diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..5990d9c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/.github/workflows/Deploy-PR.yml b/.github/workflows/Deploy-PR.yml deleted file mode 100644 index acbc34a..0000000 --- a/.github/workflows/Deploy-PR.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Build and Deploy -on: - pull_request: -jobs: - build-and-deploy: - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - persist-credentials: false - # NOTE: Python is necessary for the pre-rendering (minification) step - - name: Install python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - # NOTE: Here you can install dependencies such as matplotlib if you use - # packages such as PyPlot. - # - run: pip install matplotlib - - name: Install Julia - uses: julia-actions/setup-julia@v1 - with: - version: '1' # Latest stable Julia release. - # NOTE - # The steps below ensure that NodeJS and Franklin are loaded then it - # installs highlight.js which is needed for the prerendering step - # (code highlighting + katex prerendering). - # Then the environment is activated and instantiated to install all - # Julia packages which may be required to successfully build your site. - # The last line should be `optimize()` though you may want to give it - # specific arguments, see the documentation or ?optimize in the REPL. - - run: julia -e ' - using Pkg; Pkg.activate("."); Pkg.instantiate(); - using NodeJS; run(`$(npm_cmd()) install highlight.js`); - using Franklin; - optimize(clear=true)' - - name: Build and Deploy - uses: JamesIves/github-pages-deploy-action@releases/v3 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: gh-pages-pr - FOLDER: __site diff --git a/.github/workflows/Deploy.yml b/.github/workflows/Deploy.yml index ea9294f..0d1d04a 100644 --- a/.github/workflows/Deploy.yml +++ b/.github/workflows/Deploy.yml @@ -3,49 +3,19 @@ on: push: branches: - main -concurrency: - # Skip intermediate builds: always. - # Cancel intermediate builds: only if it is a pull request build. - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + pull_request: jobs: - build-and-deploy: + docs: runs-on: ubuntu-latest - permissions: - contents: write + permissions: write-all steps: - - name: Checkout - uses: actions/checkout@v2 - with: - persist-credentials: false - # NOTE: Python is necessary for the pre-rendering (minification) step - - name: Install python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - # NOTE: Here you can install dependencies such as matplotlib if you use - # packages such as PyPlot. - # - run: pip install matplotlib - - name: Install Julia - uses: julia-actions/setup-julia@v1 - with: - version: '1' # Latest stable Julia release. - # NOTE - # The steps below ensure that NodeJS and Franklin are loaded then it - # installs highlight.js which is needed for the prerendering step - # (code highlighting + katex prerendering). - # Then the environment is activated and instantiated to install all - # Julia packages which may be required to successfully build your site. - # The last line should be `optimize()` though you may want to give it - # specific arguments, see the documentation or ?optimize in the REPL. - - run: julia -e ' - using Pkg; Pkg.activate("."); Pkg.instantiate(); - using NodeJS; run(`$(npm_cmd()) install highlight.js`); - using Franklin; - optimize(clear=true)' - - name: Build and Deploy - uses: JamesIves/github-pages-deploy-action@releases/v3 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: gh-pages - FOLDER: __site + - uses: actions/checkout@v4 + - name: Set git user config + run: | + git config --global user.name "myusername" + git config --global user.email "myusername@modernjuliaworkflows.github" + - name: πŸš€ Build and Deploy + uses: tlienart/xranklin-build-action@main + with: + DEPLOY: ${{ github.event_name == 'push' }} + DEPLOY_BRANCH: 'gh-pages' \ No newline at end of file diff --git a/.gitignore b/.gitignore index ed60785..4aa4264 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,9 @@ +__cache/ __site/ .DS_Store node_modules/ package-lock.json .vscode -Manifest.toml - -MyPackage/ -MyProject/ \ No newline at end of file +writing/Manifest.toml +sharing/Manifest.toml +optimizing/Manifest.toml \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 781f084..c4ad4a0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,7 @@ For each tool, we aim to provide: 1. a brief introduction with the relevant links 2. a small actionable demo presented as a code block -Every time a new resource is introduced, it should be put in bold face and accompanied with a link to the relevant GitHub repository or website, like so: [**Revise.jl**](https://github.com/timholy/Revise.jl). +Every time a new resource is introduced, it should be accompanied with a link to the relevant GitHub repository or website, like so: [Revise.jl](https://github.com/timholy/Revise.jl). Links to the package documentation are not necessary, unless they are meant to highlight a specific part. Package names are written as normal text with the .jl extension, while functions or objects are written between backticks. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2d14e93 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Guillaume Dalle, Adrian Hill, and Jacobus Smit + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Manifest.toml b/Manifest.toml new file mode 100644 index 0000000..7d26aaa --- /dev/null +++ b/Manifest.toml @@ -0,0 +1,282 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.10.6" +manifest_format = "2.0" +project_hash = "fd73ef33d480b0816d7728a667dba4ef62c44c52" + +[[deps.ANSIColoredPrinters]] +git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" +uuid = "a4c015fc-c6ff-483c-b24f-f7ea428134e9" +version = "0.0.1" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.BitFlags]] +git-tree-sha1 = "0691e34b3bb8be9307330f88d1a3c3f25466c24d" +uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" +version = "0.1.9" + +[[deps.CRC32c]] +uuid = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" + +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "bce6804e5e6044c6daab27bb533d1295e4a2e759" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.6" + +[[deps.ConcurrentUtilities]] +deps = ["Serialization", "Sockets"] +git-tree-sha1 = "ea32b83ca4fefa1768dc84e504cc0a94fb1ab8d1" +uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" +version = "2.4.2" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.ExceptionUnwrapping]] +deps = ["Test"] +git-tree-sha1 = "d36f682e590a83d63d1c7dbd287573764682d12a" +uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" +version = "0.1.11" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + +[[deps.FranklinParser]] +deps = ["PrecompileTools", "REPL"] +git-tree-sha1 = "7daf95d2334d4c0f73353e110c9396e9d5258afb" +uuid = "796511e7-1510-466f-ad0c-1823c64bcafa" +version = "0.7.1" + +[[deps.HTTP]] +deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] +git-tree-sha1 = "1336e07ba2eb75614c99496501a8f4b233e9fafe" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "1.10.10" + +[[deps.IOCapture]] +deps = ["Logging", "Random"] +git-tree-sha1 = "b6d6bfdd7ce25b0f9b2f6b3dd56b2673a66c8770" +uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" +version = "0.2.5" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "be3dc50a92e5a386872a493a10050136d4703f9b" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.6.1" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.4.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.6.4+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.0+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.LiveServer]] +deps = ["HTTP", "LoggingExtras", "MIMEs", "Sockets", "Test"] +git-tree-sha1 = "564a436267fb1fc768f815dad64c4386c46623f8" +uuid = "16fef848-5104-11e9-1b77-fb7a48bbb589" +version = "1.4.0" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.LoggingExtras]] +deps = ["Dates", "Logging"] +git-tree-sha1 = "f02b56007b064fbfddb4c9cd60161b6dd0f40df3" +uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" +version = "1.1.0" + +[[deps.MIMEs]] +git-tree-sha1 = "65f28ad4b594aebe22157d6fac869786a255b7eb" +uuid = "6c6e2e6c-3030-632d-7369-2d6c69616d65" +version = "0.1.4" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] +git-tree-sha1 = "c067a280ddc25f196b5e7df3877c6b226d390aaf" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.1.9" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+1" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2023.1.10" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.NodeJS]] +deps = ["Pkg"] +git-tree-sha1 = "bf1f49fd62754064bc42490a8ddc2aa3694a8e7a" +uuid = "2bd173c7-0d6d-553b-b6af-13a54713934c" +version = "2.0.0" + +[[deps.OpenSSL]] +deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"] +git-tree-sha1 = "38cb508d080d21dc1128f7fb04f20387ed4c0af4" +uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" +version = "1.4.3" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "7493f61f55a6cce7325f197443aa80d32554ba10" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.0.15+1" + +[[deps.OrderedCollections]] +git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.3" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.10.0" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.1" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.3" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.SimpleBufferStream]] +git-tree-sha1 = "f305871d2f381d21527c770d4788c06c097c9bc1" +uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" +version = "1.2.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.TranscodingStreams]] +git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.11.3" + +[[deps.URIs]] +git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.5.1" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[deps.Xranklin]] +deps = ["ANSIColoredPrinters", "CRC32c", "Dates", "FranklinParser", "IOCapture", "LiveServer", "Logging", "Markdown", "OrderedCollections", "Pkg", "REPL", "Reexport", "Serialization", "TOML", "URIs"] +git-tree-sha1 = "00c83bd65338e9cf1ca1b8a183bd27f5df60767e" +repo-rev = "main" +repo-url = "https://github.com/tlienart/Xranklin.jl/" +uuid = "558449b0-171e-4e1f-900f-d076a5ddf486" +version = "0.1.0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+1" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.52.0+1" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+2" diff --git a/Project.toml b/Project.toml index 136b28f..4c61963 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,4 @@ [deps] -Franklin = "713c75ef-9fc9-4b05-94a9-213340da978e" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" NodeJS = "2bd173c7-0d6d-553b-b6af-13a54713934c" - -[compat] -Franklin = "0.10.86" +Xranklin = "558449b0-171e-4e1f-900f-d076a5ddf486" diff --git a/README.md b/README.md index 1b7b209..9286376 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,13 @@ # Modern Julia Workflows -Blog posts on best practices for Julia development. +[![Website][website-badge]][website-url] [![Contributor's Guide][contributing-badge]][contributing-url] -* To read: [blog preview](https://modernjuliaworkflows.github.io/) -* To discuss: [discourse post](https://discourse.julialang.org/t/seeking-feedback-blog-post-about-modern-julia-workflows/100324) -* To contribute: [guidelines](CONTRIBUTING.md) \ No newline at end of file +[Blog posts][website-url] on best practices for Julia development. + +To discuss contents, refer to this [discourse post](https://discourse.julialang.org/t/seeking-feedback-blog-post-about-modern-julia-workflows/100324) or our [GitHub discussions](https://github.com/modernjuliaworkflows/modernjuliaworkflows.github.io/discussions) page. + + +[website-url]: https://modernjuliaworkflows.org +[website-badge]: https://img.shields.io/badge/Website-stable-blue.svg +[contributing-url]: https://github.com/modernjuliaworkflows/modernjuliaworkflows.github.io/blob/main/CONTRIBUTING.md +[contributing-badge]: https://img.shields.io/badge/Contributor's%20Guide-blueviolet \ No newline at end of file diff --git a/_css/custom.css b/_css/custom.css index 1bbf59d..951a210 100644 --- a/_css/custom.css +++ b/_css/custom.css @@ -2,14 +2,17 @@ REPL HELP BLOCKS ================================================================== */ -.julia-help { +.repl-help { background-color: lightgray; - padding: 10px; + padding-left: 10px; + padding-right: 10px; + padding-top: 1px; + padding-bottom: 1px; border-radius: 10px; - margin-bottom: 0.8em; + margin-bottom: 1em; } - .julia-help h1,h2,h3 { + .repl-help h1,h2,h3 { font-size: 1em; font-weight: 500; } @@ -22,4 +25,102 @@ .im-badge img { height: 1em; object-fit: scale-down; -} \ No newline at end of file +} + +/* Custom CSS required to display the docs|stable badge on the landing page */ +img[alt=badge] { + height: 1em; + width: auto; + display: inline; + margin: 0; + padding: 0 0.5ex; +} + +/* ================================================================== + Admonitions +================================================================== */ + +.tldr { + background-color: rgba(34, 139, 34, 0.291); + border-radius: 10px; + padding: 0.5em; + margin-bottom: 1em; +} + +.tldr p { + margin-bottom: 0em; +} + +.advanced { + background: #cb3d337a; + border-radius: 10px; + padding: 0.5em; + margin-bottom: 1em; +} + +.vscode { + background-color: #9558b248; + border-radius: 10px; + padding: 0.5em; + margin-bottom: 1em; +} + + +/* ================================================================== + HLJS CODE BLOCKS +================================================================== */ + +.hljs { + position: relative; + font-size: var(--small); + line-height: 1.35em; + border-radius: 10px; +} + +.hljs-meta { font-weight: bold;} + +/* Use colors from theme specified in style.html */ +.hljs-meta.prompt_ {color: #7ee787;} +.hljs-meta.help_ {color: #f2cc60;} +.hljs-meta.shell_ {color: #ff7b72;} +.hljs-meta.infiltrator_ {color: #ffa657;} +.hljs-meta.debugger_ {color: #ffa657;} +.hljs-meta.debuggereval_ {color: #ffa657;} + +/* Add language name to code blocks */ +.hljs::before { + content: ""; + position: absolute; + top: 0; + left: 0; + padding: 0.2rem 0.5rem 0.1rem; + background-color: #27303c; /* brighter than #0d1117 */ + border-bottom-right-radius:10px; + font-size: 0.7rem; + font-weight: 700; + text-transform: uppercase; +} + +/* Define language names shown in code blocks */ +.language-bash::before {content: "Shell";} +.language-toml::before {content: "TOML";} +.language-ini::before {content: "TOML";} +.language-julia::before {content: "Julia";} +/* Special Franklin tags for Julia REPL modes */ +.language-julia-repl::before {content: "Julia REPL";} +.julia-repl::before {content: "Julia REPL";} +.julia-repl-pkg::before {content: "Julia REPL: Pkg mode";} +.julia-repl-help::before {content: "Julia REPL: Help mode";} +.julia-repl-shell::before {content: "Julia REPL: Shell mode";} +/* Julia code after REPL prompts doesn't need a language tag */ +.language-julia-repl .language-julia::before {content: "";} +.julia-repl .language-julia::before {content: "";} +.julia-repl-pkg .language-julia::before {content: "";} +.julia-repl-help .language-julia::before {content: "";} +.julia-repl-shell .language-julia::before {content: "";} +/* Julia code in doc strings printed via REPL Help mode is sometimes incorrectly +recognized as Ruby code. Since we don't have any Ruby code in MJW, this is a quick fix. +TODO: remove when not required in Xranklin. +*/ +.language-ruby::before {content: "Julia";} + diff --git a/_css/franklin.css b/_css/franklin.css index 5967e6a..81a1ea9 100644 --- a/_css/franklin.css +++ b/_css/franklin.css @@ -302,6 +302,10 @@ code { /* font-size: 90%; } */ +/* NOTE: +The highlight.js (hljs) styling of Modern Julia Workflows is in _css/custom.css. +The following commented-out code is the original Franklin CSS: + .hljs { font-size: var(--small); line-height: 1.35em; @@ -312,9 +316,7 @@ code { .hljs-meta.shell_ {color: crimson;} .hljs-meta.prompt_ {color: rgb(25, 179, 51);} -.hljs-meta.infiltrator_ {color: rgb(240, 150, 30);} -.hljs-meta.debugger_ {color: rgb(240, 150, 30);} -.hljs-meta.debuggereval_ {color: rgb(240, 150, 30);} +*/ .code-output { background: var(--output-background); diff --git a/_css/poole_hyde.css b/_css/poole_hyde.css index ab5d4d0..b3ace02 100644 --- a/_css/poole_hyde.css +++ b/_css/poole_hyde.css @@ -256,7 +256,6 @@ html { .sidebar { text-align: center; - padding: 2rem 1rem; color: rgba(255, 255, 255, 0.5); background-color: #202020; } @@ -269,6 +268,7 @@ html { bottom: 0; width: 15rem; text-align: left; + overflow-y: scroll; } } @@ -281,8 +281,7 @@ html { .sidebar-about h1 { color: #fff; margin-top: 0; - /* font-family: "Abril Fatface", sans-serif; */ /* MODIFIED */ - font-size: 3.25rem; + font-size: 40px; } /* Sidebar nav */ @@ -304,22 +303,56 @@ a.sidebar-nav-item:focus { font-weight: bold; } +.copyright-notice a { + text-decoration: none; + color: rgba(255, 255, 255, 0.5) !important; +} + +.github-link { + display: flex; + justify-content: center; + align-items: center; +} + +.github-link a { + color: rgba(255, 255, 255, 0.5) !important; +} + +/* Hide submenu by default */ +.menu-list-child-list { + display: none; + padding-left: 1.5rem; +} + +/* Show submenu when parent item has 'active' class */ +.sidebar-nav-item.active+.menu-list-child-list { + display: block; +} + +.menu-list-link { + color: rgba(255, 255, 255, 0.5) !important; + font-size: var(--small); +} + /* Sticky sidebar * * Add the `sidebar-sticky` class to the sidebar's container to affix it the * contents to the top of the sidebar in tablets and up. */ +.sidebar-sticky { + padding-top: 1rem; + padding-left: 1.5rem; + padding-right: 1.5rem; + padding-bottom: 0.5rem; +} + @media (min-width: 768px) { .sidebar-sticky { - position: absolute; - top: 1rem; - right: 1rem; - left: 1rem; + position: relative; } } - /* Container * * Align the contents of the site above the proper threshold with some margin-fu @@ -339,11 +372,14 @@ a.sidebar-nav-item:focus { } } -/* Hide the logo on screens with width less than 768px */ +/* Hide the logo and submenu on screens with width less than 768px */ @media (max-width: 767px) { .logo-container { display: none; } + .sidebar-nav-item.active+.menu-list-child-list { + display: none; + } } /* Center the logo within the logo container */ diff --git a/_layout/page_foot.html b/_layout/page_foot.html index d14843c..e51fba8 100644 --- a/_layout/page_foot.html +++ b/_layout/page_foot.html @@ -1,4 +1,4 @@
- CC BY-SA 4.0 {{ fill author }}. {{isnotpage /tag/*}}Last modified: {{ fill fd_mtime }}.{{end}}
+ CC BY-SA 4.0 {{ fill author }}. {{isnotpage /tag/*}}Last modified: {{last_modification_date}}.{{end}}
Website built with Franklin.jl and the Julia programming language.
diff --git a/_layout/sidebar.html b/_layout/sidebar.html index 77653a8..fd39261 100644 --- a/_layout/sidebar.html +++ b/_layout/sidebar.html @@ -2,19 +2,87 @@ -
+
\ No newline at end of file diff --git a/_libs/highlight/styles/github-dark.min.css b/_libs/highlight/styles/github-dark.min.css index 03b6da8..e5f30aa 100644 --- a/_libs/highlight/styles/github-dark.min.css +++ b/_libs/highlight/styles/github-dark.min.css @@ -1,4 +1,4 @@ -pre code.hljs{display:block;overflow-x:auto;padding:1em}code.hljs{padding:3px 5px}/*! +pre code.hljs{display:block;overflow-x:auto;padding:1em; padding-top:2.25em}code.hljs{padding:3px 5px}/*! Theme: GitHub Dark Description: Dark theme as seen on github.com Author: github.com diff --git a/config.md b/config.md index 0645efe..2d9f669 100644 --- a/config.md +++ b/config.md @@ -10,15 +10,35 @@ maxtoclevel = 2 # these files might be copied and, if markdown, processed by Franklin which # you might not want. Indicate directories by ending the name with a `/`. # Base files such as LICENSE.md and README.md are ignored by default. -ignore = ["node_modules/"] +ignore = [ + "node_modules/", + "CONTRIBUTING.md", +] # RSS (the website_{title, descr, url} must be defined to get RSS) generate_rss = true website_title = "Modern Julia Workflows" website_descr = "Blog posts on best practices for Julia development" -website_url = "https://modernjuliaworkflows.github.io/" +website_url = "https://modernjuliaworkflows.org/" +++ \ No newline at end of file +--> +\newcommand{\tldr}[1]{ + @@tldr + **TLDR**: !#1 + @@ +} + +\newcommand{\advanced}[1]{ + @@advanced + **Advanced**: !#1 + @@ +} + +\newcommand{\vscode}[1]{ + @@vscode + **VSCode**: !#1 + @@ +} \ No newline at end of file diff --git a/pages/further.md b/further/index.md similarity index 94% rename from pages/further.md rename to further/index.md index 39924a0..5a105f5 100644 --- a/pages/further.md +++ b/further/index.md @@ -1,4 +1,6 @@ -@def title = "Going further" ++++ +title = "Going further" ++++ # Going further @@ -20,6 +22,7 @@ * [Introduction to Julia Tutorial Workshop](https://github.com/storopoli/Julia-Workshop) by [Jose Storopoli](https://github.com/storopoli) * [Introduction to Julia Course](https://github.com/csc-training/julia-introduction) by [CSC Training](https://github.com/csc-training) * [Julia for Economists](https://github.com/cpfiffer/julia-bootcamp-2022) by [Cameron Pfiffer](https://github.com/cpfiffer) +* [Julia for High-Performance Computing](https://github.com/carstenbauer/JuliaUCL24) by [Carsten Bauer](https://github.com/carstenbauer) * [Julia programming for ML](https://adrhill.github.io/julia-ml-course/) by [Adrian Hill](https://github.com/adrhill/adrhill) * [JuliaNotes.jl](https://m3g.github.io/JuliaNotes.jl/stable/) by the [MartΓ­nez Molecular Modeling Group](https://github.com/m3g) * [Learn X in Y Minutes Where X=Julia](https://learnxinyminutes.com/docs/julia/) by [Leah Hanson et al.](https://github.com/adambard/learnxinyminutes-docs/blame/master/julia.html.markdown) diff --git a/index.md b/index.md index 1838731..ffd51e5 100644 --- a/index.md +++ b/index.md @@ -1,9 +1,12 @@ -@def title = "Modern Julia Workflows" ++++ +title = "Modern Julia Workflows" ++++ # Modern Julia Workflows A series of blog posts on best practices for Julia development. Consider this a draft: once the posts are ready, we will submit them to the [Julia language blog](https://julialang.org/blog/) to make them easily discoverable. +If you find our project useful, please star the [GitHub repo](https://github.com/modernjuliaworkflows/modernjuliaworkflows.github.io)! \toc @@ -13,22 +16,21 @@ Our purpose is to gather the hidden tips and tricks of Julia development, and ma We do not cover syntax, and assume that the reader is familiar enough with the [basics of Julia](https://julialang.org/learning/). Instead, we focus on all the tools that can make the coding experience more pleasant and efficient. -## Structure +## Contents There are three blog posts of increasing technical difficulty, plus one page of additional resources: -1. [Writing your code](/pages/writing/): from zero to a basic script -2. [Sharing your code](/pages/sharing/): from a basic script to a reliable package -3. [Optimizing your code](/pages/optimizing/): from a basic script to a light-speed algorithm -4. [Going further](/pages/further/): a collection of other websites to help you dive deeper into the rabbit hole +1. [Writing your code](/writing/): from zero to a basic script +2. [Sharing your code](/sharing/): from a basic script to a reliable package +3. [Optimizing your code](/optimizing/): from a basic script to a light-speed algorithm +4. [Going further](/further/): other websites to help you dive deeper into the rabbit hole -All pages are fairly long and not meant to be read in one sitting, so take your time. +These are all fairly long and not meant to be read in one sitting, so take your time. Keep in mind that while each of these resources _can_ be useful to you, not every one of them _will_ be. But at least you will know where to look in case you have a specific question. ## Before you start Many of the links you will see point to [GitHub](https://github.com/) repositories for Julia packages. -When you click them, they will take you to a page called `README.md`, which gives a brief description of what the package does. -Usually, this description is not enough to actually use the package. -You can often find a more thorough documentation by looking for a blue badge called [`docs|stable`](https://img.shields.io/badge/docs-stable-blue.svg) at the top of the page. +When you click them, they will take you to a page called `README.md` that contains a brief description of the package, sometimes insufficient for actual use. +You can usually find more thorough documentation by clicking on a blue badge called `docs|stable` looking like this ![badge](https://img.shields.io/badge/docs-stable-blue.svg) at the top of the `README`. diff --git a/optimizing/Project.toml b/optimizing/Project.toml new file mode 100644 index 0000000..d6bcc9c --- /dev/null +++ b/optimizing/Project.toml @@ -0,0 +1,8 @@ +[deps] +Accessors = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697" +AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a" +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +Cthulhu = "f68482b8-f384-11e8-15f7-abe071a5a75f" +DispatchDoctor = "8d63f2c5-f18a-4cf2-ba9d-b3f60fc568c8" +JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b" +StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" diff --git a/optimizing/index.md b/optimizing/index.md new file mode 100644 index 0000000..1cd6a2e --- /dev/null +++ b/optimizing/index.md @@ -0,0 +1,496 @@ ++++ +title = "Optimizing your code" ++++ + +\activate{} + +# Optimizing your code + +\toc + +## Principles + +\tldr{ +The two fundamental principles for writing fast Julia code: + +1. Ensure that **the compiler can infer the type** of every variable. +2. Avoid **unnecessary (heap) allocations**. +} + +The compiler's job is to optimize and translate Julia code it into runnable [machine code](https://en.wikipedia.org/wiki/Machine_code). +If a variable's type cannot be deduced before the code is run, then the compiler won't generate efficient code to handle that variable. +We call this phenomenon "type instability". +Enabling type inference means making sure that every variable's type in every function can be deduced from the types of the function inputs alone. + +A "heap allocation" (or simply "allocation") occurs when we create a new variable without knowing how much space it will require (like a `Vector` with flexible length). +Julia has a mark-and-sweep [garbage collector](https://docs.julialang.org/en/v1/devdocs/gc/) (GC), which runs periodically during code execution to free up space on the heap. +Execution of code is stopped while the garbage collector runs, so minimising its usage is important. + +The vast majority of performance tips come down to these two fundamental ideas. +Typically, the most common beginner pitfall is the use of [untyped global variables](https://docs.julialang.org/en/v1/manual/performance-tips/#Avoid-untyped-global-variables) without passing them as arguments. +Why is it bad? +Because the type of a global variable can change outside of the body of a function, so it causes type instabilities wherever it is used. +Those type instabilities in turn lead to more heap allocations. + +With this in mind, after you're done with the current page, you should read the [official performance tips](https://docs.julialang.org/en/v1/manual/performance-tips/): they contain some useful advice which is not repeated here for space reasons. + +## Measurements + +\tldr{Use Chairmarks.jl's `@be` with a setup phase to get the most accurate idea of your code's performance.} + +The simplest way to measure how fast a piece of code runs is to use the `@time` macro, which returns the result of the code and prints the measured runtime and allocations. +Because code needs to be compiled before it can be run, you should first run a function without timing it so it can be compiled, and then time it: + +```>time-example +sum_abs(vec) = sum(abs(x) for x in vec); +v = rand(100); + +using BenchmarkTools +@time sum_abs(v); # Inaccurate, note the >99% compilation time +@time sum_abs(v); # Accurate +``` + +Using `@time` is quick but it has flaws, because your function is only measured once. +That measurement might have been influenced by other things going on in your computer at the same time. +In general, running the same block of code multiple times is a safer measurement method, because it diminishes the probability of only observing an outlier. +The Chairmarks.jl package provides convenient syntax to do just that. + +### Chairmarks + +[Chairmarks.jl](https://github.com/LilithHafner/Chairmarks.jl) is the latest benchmarking toolkit, designed to make fast and accurate timing measurements. +Chairmarks offers `@b` (for "benchmark") which can be used in the same way as `@time` but will run the code multiple times and provide a minimum execution time. +Alternatively, Chairmarks also provides `@be` to run the same benchmark and output all of its statistics. + +```>chairmarks-example +using Chairmarks +@b sum_abs(v) +@be sum_abs(v) +``` + +Chairmarks supports a pipeline syntax with optional `init`, `setup`, `teardown`, and `keywords` arguments for more extensive control over the benchmarking process. +The `sum_abs` function could also be benchmarked using pipeline syntax as below. + +```>pipeline-example-simple +@be v sum_abs +``` + +For a more complicated example, you could write the following to benchmark a matrix multiplication function for one second, excluding the time spent to *setup* the arrays. + +```>pipeline-example-complex +my_matmul(A, b) = A * b; +@be (A=rand(1000,1000), b=rand(1000)) my_matmul(_.A, _.b) seconds=1 +``` + +See the [Chairmarks documentation](https://chairmarks.lilithhafner.com/) for more details on benchmarking options. +For better visualization, [PrettyChairmarks.jl](https://github.com/astrozot/PrettyChairmarks.jl) shows performance histograms alongside the numerical results. + +\advanced{ +No matter the benchmarking tool used, certain computations may be [optimized away by the compiler]((https://juliaci.github.io/BenchmarkTools.jl/stable/manual/#Understanding-compiler-optimizations)) before the benchmark takes place. +If you observe suspiciously fast performance, especially below the nanosecond scale, this is very likely to have happened. +} + +### Benchmark suites + +While we previously discussed the importance of documenting breaking changes in packages using [semantic versioning](/sharing/index.md#versions-and-registration), regressions in performance can also be vital to track. +Several packages exist for this purpose: + +- [PkgBenchmark.jl](https://github.com/JuliaCI/PkgBenchmark.jl) and its unmaintained but functional CI wrapper [BenchmarkCI.jl](https://github.com/tkf/BenchmarkCI.jl) +- [AirSpeedVelocity.jl](https://github.com/MilesCranmer/AirspeedVelocity.jl) +- [PkgJogger.jl](https://github.com/awadell1/PkgJogger.jl) + +### Other tools + +Chairmarks.jl works fine for relatively short and simple blocks of code (microbenchmarking). +To find bottlenecks in a larger program, you should rather use a [profiler](#profiling) or the package [TimerOutputs.jl](https://github.com/KristofferC/TimerOutputs.jl). +It allows you to label different sections of your code, then time them and display a table of grouped by label. + +[BenchmarkTools.jl](https://github.com/JuliaCI/BenchmarkTools.jl) is the older standard for benchmarking in Julia. It is still widely used today. +However, its default parameters run benchmarks for longer than Chairmarks, and it requires interpolating variables into the benchmarked expressions with `$`. + +Finally, if you know a loop is slow and you'll need to wait for it to be done, you can use [ProgressMeter.jl](https://github.com/timholy/ProgressMeter.jl) or [ProgressLogging.jl](https://github.com/JuliaLogging/ProgressLogging.jl) to track its progress. + +## Profiling + +\tldr{ + Profiling can identify performance bottlenecks at function level, and graphical tools such as ProfileView.jl are the best way to use it. +} + +### Sampling + +Whereas a benchmark measures the overall performance of some code, a profiler breaks it down function by function to identify bottlenecks. +Sampling-based profilers periodically ask the program which line it is currently executing, and aggregate results by line or by function. +Julia offers two kinds: [one for runtime](https://docs.julialang.org/en/v1/stdlib/Profile/#lib-profiling) (in the module `Profile`) and [one for memory](https://docs.julialang.org/en/v1/stdlib/Profile/#Memory-profiling) (in the submodule `Profile.Allocs`). + +These built-in profilers print textual outputs, but the result of profiling is best visualized as a flame graph. +In a flame graph, each horizontal layer corresponds to a specific level in the call stack, and the width of a tile shows how much time was spent in the corresponding function. +Here's an example: + +![flamegraph](https://github.com/pfitzseb/ProfileCanvas.jl/raw/main/assets/flamegraph.png) + +### Visualization tools + +The packages [ProfileView.jl](https://github.com/timholy/ProfileView.jl) and [PProf.jl](https://github.com/JuliaPerf/PProf.jl) both allow users to record and interact with flame graphs. +ProfileView.jl is simpler to use, but PProf is more featureful and is based on [pprof](https://github.com/google/pprof), an external tool maintained by Google which applies to more than just Julia code. +Here we only demonstrate the former: + +```julia profileview-example +using ProfileView +@profview do_work(some_input) +``` + +\vscode{ + Calling `@profview do_work(some_input)` in the integrated Julia REPL will open an interactive flame graph, similar to ProfileView.jl but without requiring a separate package. +} + +To integrate profile visualisations into environments like Jupyter and Pluto, use [ProfileSVG.jl](https://github.com/kimikage/ProfileSVG.jl) or [ProfileCanvas.jl](https://github.com/pfitzseb/ProfileCanvas.jl), whose outputs can be embedded into a notebook. + +For sharing profiles with others (e.g., on Slack or Discourse), [StatProfilerHTML.jl](https://github.com/tkluck/StatProfilerHTML.jl) is particularly useful as it generates self-contained HTML files that can be zipped and shared, allowing others to inspect the profiling results interactively without needing Julia installed. + +No matter which tool you use, if your code is too fast to collect samples, you may need to run it multiple times in a loop. + +\advanced{ + To visualize memory allocation profiles, use PProf.jl or VSCode's `@profview_allocs`. + A known issue with the allocation profiler is that it is not able to determine the type of every object allocated, instead `Profile.Allocs.UnknownType` is shown instead. + Inspecting the call graph can help identify which types are responsible for the allocations. +} + +### External profilers + +Apart from the built-in `Profile` standard library, there are a few external profilers that you can use including [Intel VTune](https://www.intel.com/content/www/us/en/developer/tools/oneapi/vtune-profiler.html) (in combination with [IntelITT.jl](https://github.com/JuliaPerf/IntelITT.jl)), [NVIDIA Nsight Systems](https://developer.nvidia.com/nsight-systems) (in combination with [NVTX.jl](https://github.com/JuliaGPU/NVTX.jl)), and [Tracy](https://docs.julialang.org/en/v1/devdocs/external_profilers/#Tracy-Profiler). + +## Type stability + +\tldr{Use JET.jl to automatically detect type instabilities in your code, and `@code_warntype` or Cthulhu.jl to do so manually. DispatchDoctor.jl can help prevent them altogether.} + +For a section of code to be considered type stable, the type inferred by the compiler must be "concrete", which means that the size of memory that needs to be allocated to store its value is known at compile time. +Types declared abstract with `abstract type` are not concrete and neither are [parametric types](https://docs.julialang.org/en/v1/manual/types/#Parametric-Types) whose parameters are not specified: + +```>isconcretetype-example +isconcretetype(Any) +isconcretetype(AbstractVector) +isconcretetype(Vector) # Shorthand for `Vector{T} where T` +isconcretetype(Vector{Real}) +isconcretetype(eltype(Vector{Real})) +isconcretetype(Vector{Int64}) +isconcretetype(eltype(Vector{Int64})) +``` + +\advanced{ +`Vector{Real}` is concrete despite `Real` being abstract for [subtle typing reasons](https://docs.julialang.org/en/v1/manual/types/#man-parametric-composite-types) but it will still be slow in practice because the type of its elements is abstract. +} + + +While type-stable function calls compile down to fast `GOTO` statements, type-unstable function calls generate code that must read the list of all methods for a given operation and find the one that matches. +This phenomenon called "dynamic dispatch" prevents further optimizations. + +Type-stability is a fragile thing: if a variable's type cannot be inferred, then the types of variables that depend on it may not be inferrable either. +As a first approximation, most code should be type-stable unless it has a good reason not to be. + +### Detecting instabilities + +The simplest way to detect an instability is with the builtin macro [`@code_warntype`](https://docs.julialang.org/en/v1/manual/performance-tips/#man-code-warntype): +The output of `@code_warntype` is difficult to parse, but the key takeaway is the return type of the function's `Body`: if it is an abstract type, like `Any`, something is wrong. +In a normal Julia REPL, such cases would show up colored in red as a warning. + +```!interactiveutils +using InteractiveUtils # hide +``` + +```> +function put_in_vec_and_sum(x) + v = [] + push!(v, x) + return sum(v) +end; + +@code_warntype put_in_vec_and_sum(1) +``` + +Unfortunately, `@code_warntype` is limited to one function body: calls to other functions are not expanded, which makes deeper type instabilities easy to miss. +That is where [JET.jl](https://github.com/aviatesk/JET.jl) can help: it provides [optimization analysis](https://aviatesk.github.io/JET.jl/stable/optanalysis/) aimed primarily at finding type instabilities. +While [test integrations](https://aviatesk.github.io/JET.jl/stable/optanalysis/#optanalysis-test-integration) are also provided, the interactive entry point of JET is the `@report_opt` macro. + +```>JET_opt +using JET +@report_opt put_in_vec_and_sum(1) +``` + +\vscode{The Julia extension features a [static linter](https://www.julia-vscode.org/docs/stable/userguide/linter/), and runtime diagnostics with JET can be automated to run periodically on your codebase and show any problems detected.} + +[Cthulhu.jl](https://github.com/JuliaDebug/Cthulhu.jl) exposes the `@descend` macro which can be used to interactively "step through" lines of the corresponding typed code, and "descend" into a particular line if needed. +This is akin to repeatedly calling `@code_warntype` deeper and deeper into your functions, slowly succumbing to the madness... +We cannot demonstrate it on a static website, but the [video example](https://www.youtube.com/watch?v=pvduxLowpPY) is a good starting point. + +### Fixing instabilities + +The Julia manual has a collection of tips to [improve type inference](https://docs.julialang.org/en/v1.12-dev/manual/performance-tips/#Type-inference). + +A more direct approach is to error whenever a type instability occurs: the macro `@stable` from [DispatchDoctor.jl](https://github.com/MilesCranmer/DispatchDoctor.jl) allows exactly that. + +## Memory management + +\tldr{You can reduce allocations with careful array management.} + +After ensuring type stability, one should try to reduce the number of heap allocations a program makes. +Again, the Julia manual has a series of tricks related to [arrays and allocations](https://docs.julialang.org/en/v1.12-dev/manual/performance-tips/#Memory-management-and-arrays) which you should take a look at. +In particular, try to modify existing arrays instead of allocating new objects (caution with array slices) and try to access arrays in the right order (column major order). + +And again, you can also choose to error whenever an allocation occurs, with the help of [AllocCheck.jl](https://github.com/JuliaLang/AllocCheck.jl). +By annotating a function with `@check_allocs`, if the function is run and the compiler detects that it might allocate, it will throw an error. +Alternatively, to ensure that non-allocating functions never regress in future versions of your code, you can write a test set to check allocations by providing the function and a concrete type-signature. + +```julia AllocCheck +@testset "non-allocating" begin + @test isempty(AllocCheck.check_allocs(my_func, (Float64, Float64))) +end +``` + +## Compilation + +\tldr{If you can anticipate which functions or packages you will need, loading time can be greatly reduced with PrecompileTools.jl or PackageCompiler.jl.} + +A number of tools allow you to reduce Julia's latency, also referred to as TTFX (time to first X, where X was historically plotting a graph). + +### Precompilation + +[PrecompileTools.jl](https://github.com/JuliaLang/PrecompileTools.jl) reduces the amount of time taken to run functions loaded from a package or local module that you wrote. +It allows module authors to give a "list" of methods to precompile when a module is loaded for the first time. +These methods then have the same latency as if they had already been run by the end user. + +Here's an example of precompilation, adapted from the package's [documentation](https://julialang.github.io/PrecompileTools.jl/stable/#Tutorial:-forcing-precompilation-with-workloads): + +```julia +module MyPackage + +using PrecompileTools: @compile_workload + +struct MyType + x::Int +end + +myfunction(a::Vector) = a[1].x + +@compile_workload begin + a = [MyType(1)] + myfunction(a) +end + +end +``` + +Note that every method that is called will be compiled, no matter how far down the call stack or which module it comes from. +To see if the intended calls were compiled correctly or diagnose other problems related to precompilation, use [SnoopCompile.jl](https://github.com/timholy/SnoopCompile.jl). +This is especially important for writers of registered Julia packages, as it allows you to diagnose recompilation that happens due to invalidation. + +### Package compilation + +To reduce the time that packages take to load, you can use [PackageCompiler.jl](https://github.com/JuliaLang/PackageCompiler.jl) to generate a custom version of Julia, called a sysimage, with its own standard library. +As packages in the standard library are already compiled, any `using` or `import` statement involving them is almost instant. + +Once PackageCompiler.jl is added to your global environment, activate a local environment for which you want to generate a sysimage, ensure all of the packages you want to compile are in its `Project.toml`, and run `create_sysimage` as in the example below. +The filetype of `sysimage_path` differs by operating system: Linux has `.so`, MacOS has `.dylib`, and Windows has `.dll`. + +```julia packagecompiler-example +packages_to_compile = ["Makie", "DifferentialEquations"] +create_sysimage(packages_to_compile; sysimage_path="MySysimage.so") +``` + +Once a sysimage is generated, it can be used with the command line flag: `julia --sysimage=path/to/sysimage`. + +\vscode{ + The generation and loading of sysimages can be [streamlined with VSCode](https://www.julia-vscode.org/docs/stable/userguide/compilesysimage/). + By default, the command sequence `Task: Run Build Task` followed by `Julia: Build custom sysimage for current environment` will compile a sysimage containing all packages in the current environment, but additional details can be specified in a `/.vscode/JuliaSysimage.toml` file. + To automatically detect and use a custom sysimage, set `useCustomSysimage` to `true` in the application settings. +} + +### Static compilation + +[PackageCompiler.jl](https://github.com/JuliaLang/PackageCompiler.jl) also facilitates the creation of [apps](https://julialang.github.io/PackageCompiler.jl/stable/apps.html) and [libraries](https://julialang.github.io/PackageCompiler.jl/stable/libs.html) that can be shared to and run on machines that don't have Julia installed. + +At a basic level, all that's required to turn a Julia module `MyModule` into an app is a function `julia_main()::Cint` that returns `0` upon successful completion. +Then, with PackageCompiler.jl loaded, run `create_app("MyModule", "MyAppCompiled")`. +Command line arguments to the resulting app are assigned to the global variable `ARGS::Array{ASCIIString}`, the handling of which can be made easier by [ArgParse.jl](https://github.com/carlobaldassi/ArgParse.jl). + +In Julia, a library is just a sysimage with some extras that enable external programs to interact with it. +Any functions in a module marked with `Base.@ccallable`, and whose type signature involves C-conforming types e.g. `Cint`, `Cstring`, and `Cvoid`, can be compiled into an externally callable library with `create_library`, similarly to `create_app`. +Unfortunately, the process of compiling and sharing a standalone executable or callable library must take [relocability](https://julialang.github.io/PackageCompiler.jl/stable/apps.html#relocatability) into account, which is beyond the scope of this blog. + +\advanced{ + +An alternative way to compile a shareable app or library that doesn't need to compile a sysimage, and therefore results in smaller binaries, is to use [StaticCompiler.jl](https://github.com/tshort/StaticCompiler.jl) and its sister package [StaticTools.jl](https://github.com/brenhinkeller/StaticTools.jl). +The biggest tradeoff of not compiling a sysimage, is that Julia's garbage collector is no longer included, so all heap allocations must be managed manually, and all code compiled _must_ be type-stable. +To get around this limitation, you can use static equivalents of dynamic types, such as a `StaticArray` ([StaticArrays.jl](https://github.com/JuliaArrays/StaticArrays.jl)) instead of an `Array` or a `StaticString` (StaticTools.jl), use `malloc` and `free` from StaticTools.jl directly, or use arena allocators with [Bumper.jl](https://github.com/MasonProtter/Bumper.jl). +The README of StaticCompiler.jl contains a more [detailed guide](https://github.com/tshort/StaticCompiler.jl?tab=readme-ov-file#guide-for-package-authors) on how to prepare code to be compiled. + +} + +## Parallelism + +\tldr{Use `Threads` or OhMyThreads.jl on a single machine, `Distributed` or MPI.jl on a computing cluster. GPU-compatible code is easy to write and run.} + +Code can be made to run faster through parallel execution with [multithreading](https://docs.julialang.org/en/v1/manual/multi-threading/) (shared-memory parallelism) or [multiprocessing / distributed computing](https://docs.julialang.org/en/v1/manual/distributed-computing/). +Many common operations such as maps and reductions can be trivially parallelised through either method by using their respective Julia packages (e.g `pmap` from Distributed.jl and `tmap` from OhMyThreads.jl). +Multithreading is available on almost all modern hardware, whereas distributed computing is most useful to users of high-performance computing clusters. + +### Multithreading + +To enable multithreading with the built-in `Threads` library, use one of the following equivalent command line flags, and give either an integer or `auto`: + +```bash threads-flag +julia --threads 4 +julia -t auto +``` + +Once Julia is running, you can check if this was successful by calling `Threads.nthreads()`. + +\vscode{ + The default number of threads can be edited by adding `"julia.NumThreads": 4,` to your settings. This will be applied to the integrated terminal. +} + +\advanced{ + Linear algebra code calls the low-level libraries [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) and [LAPACK](https://en.wikipedia.org/wiki/LAPACK). + These libraries manage their own pool of threads, so single-threaded Julia processes can still make use of multiple threads, and multi-threaded Julia processes that call these libraries may run into performance issues due to the limited number of threads available in a single core. + In this case, once `LinearAlgebra` is loaded, BLAS can be set to use only one thread by calling `BLAS.set_num_threads(1)`. + For more information see the docs on [multithreading and linear algebra](https://docs.julialang.org/en/v1/manual/performance-tips/#man-multithreading-linear-algebra). +} + +Regardless of the number of threads, you can parallelise a for loop with the macro `Threads.@threads`. +The macros `@spawn` and `@async` function similarly, but require more manual management of tasks and their results. For this reason `@threads` is recommended for those who do not wish to use third-party packages. + +When designing multithreaded code, you should generally try to write to shared memory as rarely as possible. Where it cannot be avoided, you need to be careful to avoid "race conditions", i.e. situations when competing threads try to write different things to the same memory location. +It is usually a good idea to separate memory accesses with loop indices, as in the example below: + +```julia @threads-forloop +results = zeros(Int, 4) +Threads.@threads for i in 1:4 + results[i] = i^2 +end +``` +Almost always, it is [**not** a good idea to use `threadid()`](https://julialang.org/blog/2023/07/PSA-dont-use-threadid/). + +Even if you manage to avoid any race conditions in your multithreaded code, it is very easy to run into subtle performance issues (like [false sharing](https://en.wikipedia.org/wiki/False_sharing)). For these reasons, you might want to consider using a high-level package like [OhMyThreads.jl](https://github.com/JuliaFolds2/OhMyThreads.jl), which provides a user-friendly alternative to `Threads` and makes managing threads and their memory use much easier. +The helpful [translation guide](https://juliafolds2.github.io/OhMyThreads.jl/stable/translation/) will get you started in a jiffy. + +If the latency of spinning up new threads becomes a bottleneck, check out [Polyester.jl](https://github.com/JuliaSIMD/Polyester.jl) for very lightweight threads that are quicker to start. + +If you're on Linux, you should consider using [ThreadPinning.jl](https://github.com/carstenbauer/ThreadPinning.jl) to pin your Julia threads to CPU cores to obtain stable and optimal performance. The package can also be used to visualize where the Julia threads are running on your system (see `threadinfo()`). + +\advanced{ +Some widely used parallel programming packages like [LoopVectorization.jl](https://github.com/JuliaSIMD/LoopVectorization.jl) (which also powers [Octavian.jl](https://github.com/JuliaLinearAlgebra/Octavian.jl)) or [ThreadsX.jl](https://github.com/tkf/ThreadsX.jl) are no longer maintained. +} + +### Distributed computing + +Julia's multiprocessing and distributed computing relies on the standard library `Distributed`. +The main difference compared to multi-threading is that data isn't shared between worker processes. +Once Julia is started, processes can be added with `addprocs`, and they can be queried with `nworkers`. + +The macro `Distributed.@distributed` is a _syntactic_ equivalent for `Threads.@threads`. +Hence, we can use `@distributed` to parallelise a for loop as before, but we have to additionally deal with sharing and recombining the `results` array. +We can delegate this responsibility to the standard library `SharedArrays`. +However, in order for all workers to know about a function or module, we have to load it `@everywhere`: + +``` @distributed-forloop +using Distributed + +# Add additional workers then load code on the workers +addprocs(3) +@everywhere using SharedArrays +@everywhere f(x) = 3x^2 + +results = SharedArray{Int}(4) +@sync @distributed for i in 1:4 + results[i] = f(i) +end +``` + +Note that `@distributed` does not force the main process to wait for other workers, so we must use `@sync` to block execution until all computations are done. + +One feature `@distributed` has over `@threads` is the possibility to specify a reduction function (an associative binary operator) which combines the results of each worker. +In this case `@sync` is implied, as the reduction cannot happen unless all of the workers have finished. + +```!Distributed +using Distributed # hide +``` + +```julia @distributed-sum +@distributed (+) for i in 1:4 + i^2 +end +``` + +Alternately, the convenience function `pmap` can be used to easily parallelise a `map`, both in a distributed and multi-threaded way. + +```julia +results = pmap(f, 1:100; distributed=true, batch_size=25, on_error=ex->0) +``` + +For more functionalities related to higher-order functions, [Transducers.jl](https://github.com/JuliaFolds2/Transducers.jl) and [Folds.jl](https://github.com/JuliaFolds2/Folds.jl) are the way to go. + +\advanced{ + +[MPI.jl](https://github.com/JuliaParallel/MPI.jl) implements the [Message Passing Interface standard](https://en.wikipedia.org/wiki/Message_Passing_Interface), which is heavily used in high-performance computing beyond Julia. +The C library that MPI.jl wraps is _highly_ optimized, so Julia code that needs to be scaled up to a large number of cores, such as an HPC cluster, will typically run faster with MPI than with plain `Distributed`. + +[Elemental.jl](https://github.com/JuliaParallel/Elemental.jl) is a package for distributed dense and sparse linear algebra which wraps the [Elemental](https://github.com/LLNL/Elemental) library written in C++, itself using MPI under the hood. +} + +### GPU programming + +GPUs are specialised in executing instructions in parallel over a large number of threads. +While they were originally designed for accelerating graphics rendering, more recently they have been used to train and evaluate machine learning models. + +Julia's GPU ecosystem is managed by the [JuliaGPU](https://juliagpu.org/) organisation, which provides individual packages for directly working with each GPU vendor's instruction set. +The most popular one is [CUDA.jl](https://github.com/JuliaGPU/CUDA.jl), which also simplifies installation of CUDA drivers for NVIDIA GPUs. +Through [KernelAbstractions.jl](https://github.com/JuliaGPU/KernelAbstractions.jl), you can easily write code that is agnostic to the type of GPU where it will run. + +### SIMD instructions + +In the Single Instruction, Multiple Data paradigm, several processing units perform the same instruction at the same time, differing only in their inputs. +The range of operations that can be parallelised (or "vectorized") like this is more limited than in the previous sections, and slightly harder to control. +Julia can automatically vectorize repeated numerical operations (such as those found in loops) provided a few conditions are met: + +1. Reordering operations must not change the result of the computation. +2. There must be no control flow or branches in the core computation. +3. All array accesses must follow some linear pattern. + +While this may seem straightforward, there are a number of important caveats which prevent code from being vectorized. +[Performance annotations](https://docs.julialang.org/en/v1/manual/performance-tips/#man-performance-annotations) like `@simd` or `@inbounds` help enable vectorization in some cases, as does replacing control flow with `ifelse`. + +If this isn't enough, [SIMD.jl](https://github.com/eschnett/SIMD.jl) allows users to force the use of SIMD instructions and bypass the check for whether this is possible. +One particular use-case for this is for vectorising non-contiguous memory reads and writes through `SIMD.vgather` and `SIMD.vscatter` respectively. + +\advanced{ +You can detect whether the optimizations have occurred by inspecting the output of `@code_llvm` or `@code_native` and looking for vectorised registers, types, instructions. +Note that the exact things you're looking for will vary between code and CPU instruction set, an example of what to look for can be seen in this [blog post](https://kristofferc.github.io/post/intrinsics/) by Kristoffer Carlsson. +} + +## Efficient types + +\tldr{Be aware that [StaticArrays.jl](https://github.com/JuliaArrays/StaticArrays.jl) exist and learn how they work.} + +Using an efficient data structure is a tried and true way of improving the performance. +While users can write their own efficient implementations through officially documented [interfaces](https://docs.julialang.org/en/v1/manual/interfaces/), a number of packages containing common use cases are more tightly integrated into the Julia ecosystem. + +### Static arrays + +Using [StaticArrays.jl](https://github.com/JuliaArrays/StaticArrays.jl), you can construct arrays which contain size information in their type. +Through multiple dispatch, statically sized arrays give rise to specialised, efficient methods for certain algorithms like linear algebra. +In addition, the `SArray`, `SMatrix` and `SVector` types are immutable, so the array does not need to be garbage collected as it can be stack-allocated. +Creating a new `SArray`s comes at almost no extra cost, compared to directly editing the data of a mutable object. +With `MArray`, `MMatrix`, and `MVector`, data remains mutable as in normal arrays. + +To handle mutable and immutable data structures with the same syntax, you can use [Accessors.jl](https://github.com/JuliaObjects/Accessors.jl): + +```julia accessors-example +using StaticArrays, Accessors + +sx = SA[1, 2, 3] # SA constructs an SArray +@set sx[1] = 3 # Returns a copy, does not update the variable +@reset sx[1] = 4 # Replaces the original +``` + +### Classic data structures + +All but the most obscure data structures can be found in the packages from the [JuliaCollections](https://github.com/JuliaCollections) organization, especially [DataStructures.jl](https://github.com/JuliaCollections/DataStructures.jl) which has all the standards from the computer science courses (stacks, queues, heaps, trees and the like). +Iteration and memoization utilities are also provided by packages like [IterTools.jl](https://github.com/JuliaCollections/IterTools.jl) and [Memoize.jl](https://github.com/JuliaCollections/Memoize.jl). diff --git a/pages/optimizing.md b/pages/optimizing.md deleted file mode 100644 index 595123b..0000000 --- a/pages/optimizing.md +++ /dev/null @@ -1,51 +0,0 @@ -@def title = "Optimizing your code" - -# Optimizing your code - -\toc - -## Principles - -* [performance tips](https://docs.julialang.org/en/v1/manual/performance-tips/) - -## Measurements - -* [BenchmarkTools.jl](https://github.com/JuliaCI/BenchmarkTools.jl) -* [TimerOutputs.jl](https://github.com/KristofferC/TimerOutputs.jl) -* [PkgBenchmark.jl](https://github.com/JuliaCI/PkgBenchmark.jl) - -## Profiling - -* [built-in](https://docs.julialang.org/en/v1/manual/profile/) -* [ProfileView.jl](https://github.com/timholy/ProfileView.jl) / [ProfileSVG.jl](https://github.com/kimikage/ProfileSVG.jl) -* [profiling in VSCode](https://www.julia-vscode.org/docs/stable/userguide/profiler/) - -## Type stability - -* [Cthulhu.jl](https://github.com/JuliaDebug/Cthulhu.jl) -* [JET.jl](https://github.com/aviatesk/JET.jl) -* [linting in VSCode](https://www.julia-vscode.org/docs/stable/userguide/linter/) - -## Precompilation - -* [PrecompileTools.jl](https://github.com/JuliaLang/PrecompileTools.jl) -* [PackageCompiler.jl](https://github.com/JuliaLang/PackageCompiler.jl) -* [StaticCompiler.jl](https://github.com/tshort/StaticCompiler.jl) -* [SnoopCompile.jl](https://github.com/timholy/SnoopCompile.jl) -* [compiling in VSCode](https://www.julia-vscode.org/docs/stable/userguide/compilesysimage/) - -## Parallelism - -* [distributed vs. multithreading](https://docs.julialang.org/en/v1/manual/parallel-computing/) -* [ThreadsX.jl](https://github.com/tkf/ThreadsX.jl) -* [FLoops.jl](https://github.com/JuliaFolds/FLoops.jl) - -## SIMD / GPU - -* [LoopVectorization.jl](https://github.com/JuliaSIMD/LoopVectorization.jl) -* [Tullio.jl](https://github.com/mcabbott/Tullio.jl) -* [KernelAbstractions.jl](https://github.com/JuliaGPU/KernelAbstractions.jl) - -## Miscellaneous - -* [StaticArrays.jl](https://github.com/JuliaArrays/StaticArrays.jl) diff --git a/pages/sharing.md b/pages/sharing.md deleted file mode 100644 index 00400ac..0000000 --- a/pages/sharing.md +++ /dev/null @@ -1,81 +0,0 @@ -@def title = "Sharing your code" - -# Sharing your code - -\toc - -## Setup - -* Git(Hub) -* GitHub actions: [TagBot](https://github.com/JuliaRegistries/TagBot), [CompatHelper.jl](https://github.com/JuliaRegistries/CompatHelper.jl) -* [PkgTemplates.jl](https://github.com/JuliaCI/PkgTemplates.jl) - -## Formatting - -* style guides ([BlueStyle](https://github.com/invenia/BlueStyle), [SciMLStyle](https://github.com/SciML/SciMLStyle)) -* [JuliaFormatter.jl](https://github.com/domluna/JuliaFormatter.jl) -* [formatting in VSCode](https://www.julia-vscode.org/docs/stable/userguide/formatter/) - -## Testing - -* [unit testing](https://docs.julialang.org/en/v1/stdlib/Test/) -* [TestEnv.jl](https://github.com/JuliaTesting/TestEnv.jl) -* [TestItemRunner.jl](https://github.com/julia-vscode/TestItemRunner.jl) -* [ReTest.jl](https://github.com/JuliaTesting/ReTest.jl) -* [ReferenceTests.jl](https://github.com/JuliaTesting/ReferenceTests.jl) -* [Aqua.jl](https://github.com/JuliaTesting/Aqua.jl) - -## Interfaces - -* [Interfaces.jl](https://github.com/rafaqz/Interfaces.jl) -* [RequiredInterfaces.jl](https://github.com/Seelengrab/RequiredInterfaces.jl) -* [PropCheck.jl](https://github.com/Seelengrab/PropCheck.jl) - -## Documentation - -* [docstrings](https://docs.julialang.org/en/v1/manual/documentation/) -* [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl) -* [LiveServer.jl](https://github.com/tlienart/LiveServer.jl) -* [Replay.jl](https://github.com/rafaqz/Interfaces.jl) - -## Literate programming - -* [Literate.jl](https://github.com/fredrikekre/Literate.jl) -* [Weave.jl](https://github.com/JunoLab/Weave.jl) -* [Books.jl](https://github.com/JuliaBooks/Books.jl) -* [Quarto](https://quarto.org/) - -## Compatibility - -* [semantic versioning](https://semver.org/) -* [PackageCompatUI.jl](https://github.com/GunnarFarneback/PackageCompatUI.jl) -* [CompatHelper.jl](https://github.com/JuliaRegistries/CompatHelper.jl) -* [package extensions](https://pkgdocs.julialang.org/v1/creating-packages/#Conditional-loading-of-code-in-packages-(Extensions)) -* [Requires.jl](https://github.com/JuliaPackaging/Requires.jl) -* [PackageExtensionTools.jl](https://github.com/cjdoris/PackageExtensionTools.jl) -* [SimpleUnPack.jl](https://github.com/devmotion/SimpleUnPack.jl) - -## Reproducibility - -* [StableRNGs.jl](https://github.com/JuliaRandom/StableRNGs.jl) -* [DataDeps.jl](https://github.com/oxinabox/DataDeps.jl) -* [ArtifactUtils.jl](https://github.com/JuliaPackaging/ArtifactUtils.jl) -* [DrWatson.jl](https://github.com/JuliaDynamics/DrWatson.jl) - -## Publishing - -* [general registry](https://github.com/JuliaRegistries/General) -* [Registrator.jl](https://github.com/JuliaRegistries/Registrator.jl) -* [LocalRegistry.jl](https://github.com/GunnarFarneback/LocalRegistry.jl) - -## Collaboration - -* [SciML ColPrac](https://github.com/SciML/ColPrac) -* [contribute](https://julialang.org/contribute/) -* [GitHub PRs](https://kshyatt.github.io/post/firstjuliapr/) - -## Citations - -* [Zenodo](https://zenodo.org/) -* [PkgCite.jl](https://github.com/SebastianM-C/PkgCite.jl) -* [DocumenterCitations.jl](https://github.com/ali-ramadhan/DocumenterCitations.jl) \ No newline at end of file diff --git a/pages/writing.md b/pages/writing.md deleted file mode 100644 index e1058de..0000000 --- a/pages/writing.md +++ /dev/null @@ -1,523 +0,0 @@ -@def title = "Writing your code" - -# Writing your code - -\toc - -## Installation - -> TLDR: Use `juliaup` - -Do not install Julia from the official [downloads](https://julialang.org/downloads/) page, use **[`juliaup`](https://github.com/JuliaLang/juliaup)** instead. -You can get it from the Windows store, or from the command line on Unix systems: - -```bash -curl -fsSL https://install.julialang.org | sh -``` - -It provides [various utilities](https://github.com/JuliaLang/juliaup#using-juliaup) to download, update, organize and switch between Julia versions. -As a bonus, you no longer have to manually specify the path to your executable. - -`juliaup` relies on adaptive shortcuts called "channels", which allow you to access specific Julia versions without giving their exact number. -Upon installation, the [current stable release](https://julialang.org/downloads/#current_stable_release) is downloaded and selected as the default: - -```bash -juliaup add release # done automatically -juliaup default release # done automatically -``` - -However, you can use other versions like the [long-term support version](https://julialang.org/downloads/#long_term_support_release): - -```bash -juliaup add lts -julia +lts # launch lts -``` - -You can get an overview of the channels installed on your computer: - -```bash -juliaup status -``` - -When new versions are tagged, the binding of a given channel can change, and a new executable might need to be downloaded. -If you want to catch up with the latest developments, that's easy: - -```bash -juliaup update -``` - -## REPL - -> TLDR: The REPL has 4 primary modes: Julia, package (`]`), help (`?`) and shell (`;`). - -The Read-Eval-Print Loop (or REPL) is the most basic way to interact with Julia. -Check out its [documentation](https://docs.julialang.org/en/v1/stdlib/REPL/) for details, and the [REPL mastery workshop](https://github.com/miguelraz/REPLMasteryWorkshop) for a deep dive. -You can start one by typing `julia` into a terminal, or by clicking on the Julia application in your computer. -It will allow you to play around with arbitrary Julia code: - -```> -a, b = 1, 2; -a + b -``` - -This is the standard, Julian mode of the REPL, but it also has three other modes. -Each mode is entered by typing a specific character after the `julia>` prompt, and can be exited by hitting backspace after the `julia>` prompt. - -### Help mode (`?`) - -By pressing `?` you can obtain information and metadata about Julia objects (functions, types, etc.), and unicode symbols. -The query fetches the docstring of the object, which explains how to use it. - -```? -println -``` - -If you don't know the exact name you are looking for, type a word surrounded by quotes to see in which docstrings it pops up. - -### Package mode (`]`) - -By pressing `]` you access the package manager (check out its short [documentation](https://docs.julialang.org/en/v1/stdlib/Pkg/), we will get back to it later). -It is built into Julia and allows you to: - -* `add`, `update` (or `up`) and `remove` (or `rm`) packages; -* `activate` different local, global or temporary environments; -* get the `status` (or `st`) of your current environment. - -```] -activate --temp -status -add Example -status -``` - -### Shell mode (`;`) - -By pressing `;` you enter a terminal, where you can execute any bash command you want. - -```; -echo "hello" -ls ./pages -``` - -## Editor - -> TLDR: VSCode has the best Julia support. - -Most computer programs are just plain text files with a specific extension (in our case `.jl`). -So in theory, any text editor suffices to write and modify Julia code. -In practice, an Integrated Development Environment (or IDE) makes the experience much more pleasant, thanks to code-related utilities and language-specific plugins. - -The best IDE for Julia is **[Visual Studio Code](https://code.visualstudio.com/)**, developed by Microsoft. -Indeed, the **[Julia VSCode extension](https://www.julia-vscode.org/)** is the most feature-rich and actively developed of all Julia IDE plugins. -You can download it from the VSCode Marketplace. -In what follows, we will often mention commands and keyboard shortcuts provided by this extension. -But the only shortcut you need to remember is `Ctrl + Shift + P` (or `Cmd + Shift + P` on Mac): this opens the VSCode command palette, in which you can search for any command. -Type "julia" in the command palette to see what you can do with it. - -Assuming you want to avoid the Microsoft ecosystem, [VSCodium](https://vscodium.com/) is a nearly bit-for-bit replacement for VSCode, but with an open source license and without telemetry. -If you don't want to use VSCode at all, other options include [emacs](https://www.gnu.org/software/emacs/) and [vim](https://www.vim.org/). -Check out [JuliaEditorSupport](https://github.com/JuliaEditorSupport) to see if your favorite IDE has a Julia plugin. -The available functionalities should be roughly similar to those of VSCode, at least for the basic aspects like running code. - -## Running code - -> TLDR: Open a REPL and run all your code there interactively - -You can execute a Julia script from your terminal, but in most cases that is not what you want to do. - -```bash -julia myfile.jl -``` - -Julia has a rather high startup, load and compilation latency. -If you only use scripts, you will pay this cost every time you run a slightly modified version of your code. -That is why many Julia developers fire up a [REPL](#REPL) at the beginning of the day and run all of their code there, chunk by chunk, in an interactive way. -This is made much easier by IDE integration, and here are the relevant [VSCode commands](https://www.julia-vscode.org/docs/stable/userguide/runningcode/): - -* `Julia: Start REPL` (shortcut `Alt + J` then `Alt + O`) -* `Julia: Execute Code in REPL and Move` (shortcut `Shift + Enter`). As in Jupyter, the code that gets executed is the block containing the cursor, or the selected part if there is any. -* `Julia: Execute active File in REPL` - -When keeping the same REPL open for a long time, it's common to end up with a "polluted" workspace where the definitions of certain variables or functions have been overwritten in unexpected ways. -This, along with other events like `struct` redefinitions, might force you to restart your REPL now and again. -One way to help with workspace tidiness is to take advantage of the [module system](#local_packages) to separate the reusable parts of your code from the one-off parts that are only relevant for a certain script. - -## Notebooks - -> TLDR: Jupyter or Pluto, depending on your reactivity needs - -Notebooks are a popular alternative to IDEs when it comes to short and self-contained code, typically in data science. -They are also a good fit for literate programming, where lines of code are interspersed by comments and explanations. - -The most well-known notebook ecosystem is [Jupyter](https://jupyter.org/), which supports **Ju**lia, **Pyt**hon and **R** as its three core languages. -To use it with Julia, you will need to install the **[IJulia.jl](https://github.com/JuliaLang/IJulia.jl)** backend. -Then, if you have also installed Jupyter, you can run this command to launch the server: - -```bash -jupyter notebook -``` - -If you only have IJulia.jl on your system, you can run this snippet instead: - -```julia-repl -julia> using IJulia - -julia> notebook() -``` - -A pure-Julia alternative to Jupyter is given by **[Pluto.jl](https://plutojl.org/)**. -Unlike Jupyter notebooks, Pluto notebooks are - -* Reactive: when you update a cell, the other cells depending on it are updated. -* Reproducible: they come bundled with an exhaustive list of dependencies. - -To try them out, install the package and then run - -```julia-repl -julia> using Pluto - -julia> Pluto.run() -``` - -## Environments - -> TLDR: Julia projects are made with `] activate`, and their details are stored in the `Project.toml` and `Manifest.toml`. - -Pkg.jl and the [Pkg mode](#package-mode) built in to the [REPL](#repl) let you install packages and manage environments. -A "package" is a structured way of reusing code between projects and the active "environment" is responsible for determining which versions of packages to load. - -Pkg.jl can be used from the REPL, as seen before: -```] -activate MyPackage -``` -or directly called in Julia mode as a package with the same commands: - -```> -using Pkg -Pkg.activate("MyPackage") -``` - -The `activate` command used above can be used to activate an existing project or create a new one and then activate it. - -You can also run `julia` from the command line with the [startup flag](#configuration) `--project MyProject` (which is what the [VSCode plugin](#environments_in_vscode) does). -After this, packages you install will be listed in the `Project.toml` and `Manifest.toml` files. - -The `Project.toml` contains information both about the project e.g name, uuid, authors, and its dependencies. -Its dependency-related content is visible with `]status` or just `]st` -```] -add Term OhMyREPL -st -``` - -As dependencies often have their own dependencies, potential version conflicts must be resolved for an environment to be usable. -The resolution is done automatically on package installation with `]add `, environment instantiation with `]instantiate`, and with the `]resolve` command. -The output of this resolution is stored in the `Manifest.toml`. - - - - -Sharing a project between computers is as simple as sending a folder containing your code as well as the `Project.toml` and `Manifest.toml`. -With these files, the user can run `]instantiate` and Julia will perfectly[^1] recreate the state of packages in the local environment. - -If you haven't `activate`d a local project, packages that you `add` will be installed in the "global environment" called `@v1.X`[^2] after the active version of Julia. -Packages installed globally are available no matter which local environment is active because of "environment stacking": - -When calling `using Package`, Julia determines what to load by going down the stack defined by `Base.LOAD_PATH`: - -```> -Base.LOAD_PATH -``` - -The search begins at the local environment `@`, then the global environment `@v#.#`, and finally the standard library `@stdlib` that comes pre-installed with Julia. -The two most important implications of this are firstly that development tools can be installed globally and [loaded on startup](#configuration) to be available to use, and secondly that packages in the standard library can be updated or fixed independently of the version of Julia you are using. - -[^1]: Unless the `develop` command is used, which causes the project's dependencies to be stateful. -[^2]: The `@` before the name means that the environment is ["shared"], which means you can `activate` shared environments with the `--shared` flag and it is located in `~/.julia/environments`. Notably it does _not_ imply that it is part of the environment stack. - -["shared"]: https://pkgdocs.julialang.org/v1/environments/#Shared-environments - -### Environments in VSCode - -In VSCode, if your directory contains a `Project.toml`, you will be prompted whether you want to make this the default environment. -With this option set, anytime you [open a REPL](#running-code) the environment will already be the local one. - - -## Local packages - -Local packages are a smart way of reusing code between projects. -You could load common code directly with `include("path/to/file.jl")`, but a local package allows you to benefit from package niceties: - -1. You don't have to specify the path, you can just write `using MyPackage`, -2. You can [version](sharing/#Compatibility) the package and update it without breaking code that relies on old versions of the package, -3. You can add it as a dependency to a project that you're working on, -4. (Bonus!) You get used to developing reusable, modular code. - - - - -## Configuration - -Julia accepts [startup flags](https://docs.julialang.org/en/v1/manual/command-line-interface/#command-line-interface) to handle settings such as the number of threads available. -In addition, most Julia developers also have a [startup file](https://docs.julialang.org/en/v1/manual/command-line-interface/#Startup-file) which is run automatically every time the language is started. -It is located at `.julia/config/startup.jl`. - -The basic component that everyone puts in the startup file is Revise.jl: - -```julia -try - using Revise -catch e - @warn "Error initializing Revise" -end -``` - -In addition, users commonly load packages that affect the REPL experience, as well as benchmarking or profiling utilities. -We will come back to all of these later on, but in the meantime **[StartupCustomizer.jl](https://github.com/abraemer/StartupCustomizer.jl)** can help you set them up. -More generally, the startup file allows you to define your own favorite helper functions and have them immediately available in every Julia session. - -## Esthetics - -* [Term.jl](https://github.com/FedeClaudi/Term.jl) -* [OhMyREPL.jl](https://github.com/KristofferC/OhMyREPL.jl) -* [AbbreviatedStackTraces.jl](https://github.com/BioTurboNick/AbbreviatedStackTraces.jl) -* [ProgressMeter.jl](https://github.com/timholy/ProgressMeter.jl) -* [ProgressLogging.jl](https://github.com/JuliaLogging/ProgressLogging.jl) -* [Suppressor.jl](https://github.com/JuliaIO/Suppressor.jl) - -## Debugging -> TLDR: -> Use the VSCode debugger or Infiltrator.jl. -> Use logging instead of printing. - -* [Logging][julia-docs-logging] -* [Debugging in VSCode][vscode-debugger] -* [Debugger.jl][debugger-repo] -* [Infiltrator.jl][infiltrator-repo] - - - -### Logging - -Assume you want to debug the following function, which is supposed to compute the -sum of [proper divisors](https://mathworld.wolfram.com/ProperDivisor.html) of $n$: -```julia -function sum_of_divisors(n) - divisors = filter(x -> n % x == 0, 1:n) - return sum(divisors) -end -``` -```> -sum_of_divisors(6) # should return 1 + 2 + 3 -``` - -Using `@show` or `println`, you can print local variables inside of a function: - -```julia:debugshow -function sum_of_divisors(n) - divisors = filter(x -> n % x == 0, 1:n) - @show divisors - return sum(divisors) -end -``` -```> -sum_of_divisors(6) -``` - -> The problem with `sum_of_divisors` is the range `1:n`, -> which includes `n` in the list of computed divisors. -> We can fix the function by changing the range to `1:n-1`. - -While printing might suffice to debug simple problems, we can do better. -Julia offers the logging macros `@debug`, `@info`, `@warn` and `@error` that have several advantages over printing. They: -- show the line number they were called from -- label arguments, similar to `@show` -- can be disabled and filtered according to their source module and severity level -- work well in multithreaded code -- can be written to a file - -By default, `@debug` messages are suppressed. -You can enable them through the `JULIA_DEBUG` environment variable -by specifying the source module name, e.g. `Main`. - -```julia:debuglogging -function sum_of_divisors(n) - divisors = filter(x -> n % x == 0, 1:n) - @debug "sum_of_divisors" n divisors - return sum(divisors) -end -``` - - -```julia-repl -julia> ENV["JULIA_DEBUG"] = Main # enable @debug logs -Main - -julia> sum_of_divisors(6) -β”Œ Debug: sum_of_divisors -β”‚ n = 6 -β”‚ divisors = -β”‚ 4-element Vector{Int64}: -β”‚ 1 -β”‚ 2 -β”‚ 3 -β”‚ 6 -β”” @ Main REPL[1]:3 -12 -``` - -For scripts, you can prefix your command-line call to `julia` with environment variables, -e.g. `JULIA_DEBUG=Main julia myscript.jl`. -Refer the [Julia documentation on logging][julia-docs-logging] for more information. - -### VSCode Debugger - -Using the [Julia VSCode extension][julia-vscode-repo], -click left of a line number in a VSCode editor pane to add a *breakpoint*, -which is visualized by a red circle. -In the debugging pane of the Julia VSCode extension, -click *Run and Debug* to start the debugger. -The program will automatically halt when it hits a breakpoint. - -Using the toolbar at the top of the editor, you can -*continue*, *step over*, *step into* and *step out* of your code. -The debugger will open a pane showing information about the code -such as local variables inside of the current function, -their current values and the call stack. - -For more information including explanatory screenshots, -refer to the [Julia VSCode documentation][vscode-debugger]. - -### Infiltrator.jl - -[Infiltrator.jl's][infiltrator-repo] `@infiltrate` macro allows you to directly set breakpoints in your code. -Calling a function which hits a breakpoint will activate the Infiltrator REPL-mode -and change the prompt to `infil>`. - -Typing `?` in this mode will summarize available commands. -For example, typing `@locals` in Infiltrator-mode will print local variables: - -```julia -using Infiltrator - -function sum_of_divisors(n) - divisors = filter(x -> n % x == 0, 1:n) - @infiltrate - return sum(divisors) -end -``` -```julia-repl -julia> sum_of_divisors(6) -Infiltrating (on thread 1) sum_of_divisors(n::Int64) - at REPL[4]:3 - -infil> @locals -- n::Int64 = 6 -- divisors::Vector{Int64} = [1, 2, 3, 6] -``` - -What makes Infiltrator powerful is the `@exfiltrate` macro, -which allows you to move local variables into a global storage called the `safehouse`. - -```julia-repl -infil> @exfiltrate divisors -Exfiltrating 1 local variable into the safehouse. - -infil> @continue - -12 - -julia> safehouse.divisors -4-element Vector{Int64}: - 1 - 2 - 3 - 6 -``` - -### Debugger.jl - -Using [Debugger.jl][debugger-repo]'s `@enter` macro, we can enter a function call and step through it. -The prompt changes to `1|debug>`, allowing you to use [Debugger.jl's commands][debugger-commands] -to step into and out of function calls, show local variables and set breakpoints. - -Typing `` ` `` will change the prompt to `1|julia>`, indicating evaluation mode. Any expression typed in this mode will be evaluated in the local context. -This is useful to show local variables, as demonstrated in the following example: - -```julia-repl -julia> @enter sum_of_divisors(6) -In sum_of_divisors(n) at REPL[3]:1 - 1 function sum_of_divisors(n) ->2 divisors = filter(x -> n % x == 0, 1:n) - 3 return sum(divisors) - 4 end - -About to run: (typeof)(6) -1|debug> n # n: step to next line -In sum_of_divisors(n) at REPL[3]:1 - 1 function sum_of_divisors(n) - 2 divisors = filter(x -> n % x == 0, 1:n) ->3 return sum(divisors) - 4 end - -About to run: (sum)([1, 2, 3, 6]) -1|julia> divisors # type `, then variable name -4-element Vector{Int64}: - 1 - 2 - 3 - 6 -``` - -## Other languages - -* [C and Fortran](https://docs.julialang.org/en/v1/manual/calling-c-and-fortran-code/) -* [CondaPkg.jl](https://github.com/cjdoris/CondaPkg.jl) + [PythonCall.jl](https://github.com/cjdoris/PythonCall.jl) -* [JuliaInterop](https://github.com/JuliaInterop) ([RCall.jl](https://github.com/JuliaInterop/RCall.jl), [Cxx.jl](https://github.com/JuliaInterop/Cxx.jl)) - -## Getting help - -* [StartHere.jl](https://github.com/JuliaCommunity/StartHere.jl) -* [cheatsheet](https://cheatsheet.juliadocs.org/) -* [help](https://julialang.org/about/help/) -* [community](https://julialang.org/community/) - -[julia-vscode-repo]: https://github.com/julia-vscode/julia-vscode -[vscode-debugger]: https://www.julia-vscode.org/docs/stable/userguide/debugging/ -[julia-docs-logging]: https://docs.julialang.org/en/v1/stdlib/Logging/ -[infiltrator-repo]: https://github.com/JuliaDebug/Infiltrator.jl -[debugger-repo]: https://github.com/JuliaDebug/Debugger.jl -[debugger-commands]: https://github.com/JuliaDebug/Debugger.jl#debugger-commands -[codetracking-repo]: https://github.com/timholy/CodeTracking.jl -[interactiveerrors-repo]: https://github.com/MichaelHatherly/InteractiveErrors.jl -[interactivesearch-repo]: https://github.com/tkf/InteractiveCodeSearch.jl diff --git a/sharing/Project.toml b/sharing/Project.toml new file mode 100644 index 0000000..4ac8b82 --- /dev/null +++ b/sharing/Project.toml @@ -0,0 +1,5 @@ +[deps] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b" +JuliaFormatter = "98e50ef6-434e-11e9-1051-2b60c6c9e899" +PkgTemplates = "14b8a8f1-9102-5b29-a752-f990bacb7fe1" diff --git a/sharing/index.md b/sharing/index.md new file mode 100644 index 0000000..79f575b --- /dev/null +++ b/sharing/index.md @@ -0,0 +1,364 @@ ++++ +title = "Sharing your code" +ignore_cache = true ++++ + + + +```! +# hideall +if isdir(sitepath("MyAwesomePackage")) + rm(sitepath("MyAwesomePackage"); recursive=true) +end +``` + +\activate{} + +# Sharing your code + +In this post, you will learn about tools to initialize, structure and distribute Julia packages. + +\toc + +## Setup + +A vast majority of Julia packages are hosted on [GitHub](https://github.com/) (although less common, other options like [GitLab](https://gitlab.com/) are also possible). +GitHub is a platform for collaborative software development, based on the version control system [Git](https://git-scm.com/). +If you are unfamiliar with these technologies, check out the [GitHub documentation](https://docs.github.com/en/get-started/quickstart). + +The first step is therefore [creating](https://github.com/new) an empty GitHub repository. +You should try to follow [package naming rules](https://pkgdocs.julialang.org/v1/creating-packages/#Package-naming-rules) and add a ".jl" extension at the end, like so: "MyAwesomePackage.jl". +Do not insert any files like `README.md`, `.gitignore` or `LICENSE.md`, this will be done for you in the next step. + +Indeed, we can leverage [PkgTemplates.jl](https://github.com/JuliaCI/PkgTemplates.jl) to automate package creation (like `]generate` from Pkg.jl but on steroids). +The following code gives you a basic file structure to start with: + +```>pkgtemplates1 +using PkgTemplates +t = Template(user="myuser", interactive=false); +``` + +```!pkgtemplates2 +#hideall +t = Template(dir=Utils.path(:site), user="myuser", interactive=false); +``` + +```>pkgtemplates3 +t("MyAwesomePackage") +``` + +Then, you simply need to push this new folder to the remote repository , and you're ready to go. + +The steps described above, including creation of a GitHub repo and pushing your project to it, can also be comfortably done with the help of [PackageMaker.jl](https://github.com/Eben60/PackageMaker.jl), which is a graphical wrapper around [PkgTemplates.jl](https://github.com/JuliaCI/PkgTemplates.jl) with a couple features of its own. + +The rest of this post will explain to you what each part of this folder does, and how to bend them to your will. + +To work on the package further, we switch to it's environment or "develop" it into the current one, and then import it: + +```julia-repl +julia> using Pkg # remember, you can equivalently do all that from the pkg REPL after pressing ] + +julia> Pkg.activate(path="MyAwesomePackage") +``` + +or + +```julia-repl +julia> using Pkg + +julia> Pkg.develop(path="MyAwesomePackage") +``` + +```!using-awesome1 +#hideall +using Pkg +Pkg.develop(path=sitepath("MyAwesomePackage")) # ignore sitepath +``` + +```>using-awesome2 +using MyAwesomePackage +``` + + +## GitHub Actions + +The most useful aspect of PkgTemplates.jl is that it automatically generates workflows for [GitHub Actions](https://docs.github.com/en/actions/quickstart). +These are stored as YAML files in `.github/workflows`, with a slightly convoluted syntax that you don't need to fully understand. +For instance, the file `CI.yml` contains instructions that execute the tests of your package (see below) for each pull request, tag or push to the `main` branch. +This is done on a GitHub server and should theoretically cost you money, but your GitHub repository is public, you get an unlimited workflow budget for free. + +A variety of workflows and functionalities are available through optional [plugins](https://juliaci.github.io/PkgTemplates.jl/stable/user/#Plugins-1). +The interactive setting `Template(..., interactive=true)` allows you to select the ones you want for a given package. +Otherwise, you will get the [default selection](https://juliaci.github.io/PkgTemplates.jl/stable/user/#Default-Plugins), which you are encouraged to look at. + +## Testing + +The purpose of the `test` subfolder in your package is [unit testing](https://docs.julialang.org/en/v1/stdlib/Test/): automatically checking that your code behaves the way you want it to. +For instance, if you write your own square root function, you may want to test that it gives the correct results for positive numbers, and errors for negative numbers. + +```>sqrt +using Test + +@test sqrt(4) β‰ˆ 2 + +@testset "Invalid inputs" begin + @test_throws DomainError sqrt(-1) + @test_throws MethodError sqrt("abc") +end; +``` + +Such tests belong in `test/runtests.jl`, and they are executed with the `]test` command (in the REPL's Pkg mode). +Unit testing may seem rather naive, or even superfluous, but as your code grows more complex, it becomes easier to break something without noticing. +Testing each part separately will increase the reliability of the software you write. + +\advanced{ + +To test the arguments provided to the functions within your code (for instance their sign or value), avoid `@assert` (which can be deactivated) and use [ArgCheck.jl](https://github.com/jw3126/ArgCheck.jl) instead. + +} + +At some point, your package may require [test-specific dependencies](https://pkgdocs.julialang.org/v1/creating-packages/#Adding-tests-to-the-package). +This often happens when you need to test compatibility with another package, on which you do not depend for the source code itself. +Or it may simply be due to testing-specific packages like the ones we will encounter below. +For interactive testing work, use [TestEnv.jl](https://github.com/JuliaTesting/TestEnv.jl) to activate the full test environment (faster than running `]test` repeatedly). + +\vscode{ + +The Julia extension also has its own testing framework, which relies on sprinkling "test items" throughout the code. +See [TestItemRunner.jl](https://github.com/julia-vscode/TestItemRunner.jl) for indications on how to use them optimally. + +} + +\advanced{ + +If you want to have more control over your tests, you can try + +* [ReferenceTests.jl](https://github.com/JuliaTesting/ReferenceTests.jl) to compare function outputs with reference files. +* [ReTest.jl](https://github.com/JuliaTesting/ReTest.jl) to define tests next to the source code and control their execution. +* [TestSetExtensions.jl](https://github.com/ssfrr/TestSetExtensions.jl) to make test set outputs more readable. +* [TestReadme.jl](https://github.com/thchr/TestReadme.jl) to test whatever code samples are in your README. +* [ReTestItems.jl](https://github.com/JuliaTesting/ReTestItems.jl) for an alternative take on VSCode's test item framework. + +} + +Code coverage refers to the fraction of lines in your source code that are covered by tests. +It is a good indicator of the exhaustiveness of your test suite, albeit not sufficient. +[Codecov](https://about.codecov.io/) is a website that provides easy visualization of this coverage, and many Julia packages use it. +It is available as a PkgTemplates.jl plugin, but you have to perform an [additional configuration step](https://docs.codecov.com/docs/adding-the-codecov-token) on the repo for Codecov to communicate with it. + +## Style + +To make your code easy to read, it is essential to follow a consistent set of guidelines. +The official [style guide](https://docs.julialang.org/en/v1/manual/style-guide/) is very short, so most people use third party style guides like [BlueStyle](https://github.com/JuliaDiff/BlueStyle) or [SciMLStyle](https://github.com/SciML/SciMLStyle). + +[JuliaFormatter.jl](https://github.com/domluna/JuliaFormatter.jl) is an automated formatter for Julia files which can help you enforce the style guide of your choice. +Just add a file `.JuliaFormatter.toml` at the root of your repository, containing a single line like + +```toml +style = "blue" +``` + +Then, the package directory will be formatted in the BlueStyle whenever you call + +```>format +using JuliaFormatter +JuliaFormatter.format(MyAwesomePackage) +``` + +\vscode{ + +The [default formatter](https://www.julia-vscode.org/docs/stable/userguide/formatter/) falls back on JuliaFormatter.jl. + +} + +\advanced{ + +You can format code automatically in GitHub pull requests with the [`julia-format` action](https://github.com/julia-actions/julia-format), or add the formatting check directly to your test suite. + +} + +## Code quality + +Of course, there is more to code quality than just formatting. +[Aqua.jl](https://github.com/JuliaTesting/Aqua.jl) provides a set of routines that examine other aspects of your package, from unused dependencies to ambiguous methods. +It is usually a good idea to include the following in your tests: + +```>aqua +using Aqua, MyAwesomePackage +Aqua.test_all(MyAwesomePackage) +``` + +Meanwhile, [JET.jl](https://github.com/aviatesk/JET.jl) is a complementary tool, similar to a static linter. +Here we focus on its [error analysis](https://aviatesk.github.io/JET.jl/stable/jetanalysis/), which can detect errors or typos without even running the code by leveraging type inference. +You can either use it in report mode (with a nice [VSCode display](https://www.julia-vscode.org/docs/stable/userguide/linter/#Runtime-diagnostics)) or in test mode as follows: + +```>jet +using JET, MyAwesomePackage +JET.report_package(MyAwesomePackage) +JET.test_package(MyAwesomePackage) +``` + +Note that both Aqua.jl and JET.jl might pick up false positives: refer to their respective documentations for ways to make them less sensitive. + +Finally, [ExplicitImports.jl](https://github.com/ericphanson/ExplicitImports.jl) can help you get rid of generic imports to specify where each of the names in your package comes from. +This is a good practice and makes your code more robust to name conflicts between dependencies. + +## Documentation + +Even if your code does everything it is supposed to, it will be useless to others (and pretty soon to yourself) without proper documentation. +Adding [docstrings](https://docs.julialang.org/en/v1/manual/documentation/) everywhere needs to become a second nature. +This way, readers and users of your code can query them through the REPL help mode. +[DocStringExtensions.jl](https://github.com/JuliaDocs/DocStringExtensions.jl) provides a few shortcuts that can speed up docstring creation by taking care of the obvious parts. + +```!docstring +""" + myfunc(a, b; kwargs...) + +One-line sentence describing the purpose of the function, +just below the (indented) signature. + +More details if needed. +""" +function myfunc end; +``` + + +However, package documentation is not limited to docstrings. +It can also contain high-level overviews, technical explanations, examples, tutorials, etc. +[Documenter.jl](https://github.com/JuliaDocs/Documenter.jl) allows you to design a website for all of this, based on Markdown files contained in the `docs` subfolder of your package. +Unsurprisingly, its own [documentation](https://documenter.juliadocs.org/stable/) is excellent and will teach you a lot. +To build the documentation locally, just run + +```julia-repl +julia> using Pkg + +julia> Pkg.activate("docs") + +julia> include("docs/make.jl") +``` + +Then, use [LiveServer.jl](https://github.com/tlienart/LiveServer.jl) from your package folder to visualize and automatically update the website as the code changes (similar to Revise.jl): + +```julia-repl +julia> using LiveServer + +julia> servedocs() +``` + +To host the documentation online easily, just select the [`Documenter` plugin](https://juliaci.github.io/PkgTemplates.jl/stable/user/#PkgTemplates.Documenter) from PkgTemplates.jl before creation. +Not only will this fill the `docs` subfolder with the right contents: it will also initialize a [GitHub Actions workflow](https://documenter.juliadocs.org/stable/man/hosting/#gh-pages-Branch) to build and deploy your website on [GitHub pages](https://pages.github.com/). +The only thing left to do is to [select the `gh-pages` branch as source](https://documenter.juliadocs.org/stable/man/hosting/#gh-pages-Branch). + +\advanced{ + +You may find the following Documenter plugins useful: + +1. [DocumenterCitations.jl](https://github.com/JuliaDocs/DocumenterCitations.jl) allows you to insert citations inside the documentation website from a BibTex file. +2. [DocumenterInterLinks.jl](https://github.com/JuliaDocs/DocumenterInterLinks.jl) allow you to cross-reference external documentations (Documenter and Sphinx). + +Assuming you are looking for an alternative to Documenter.jl, you can try out [Pollen.jl](https://github.com/lorenzoh/Pollen.jl). +In another category, [Replay.jl](https://github.com/AtelierArith/Replay.jl) allows you to replay instructions entered into your terminal as an ASCII video, which is nice for tutorials. + +} + +## Literate programming + +Scientific software is often hard to grasp, and the code alone may not be very enlightening. +Whether it is for package documentation or to write papers and books, you might want to interleave code with texts, formulas, images and so on. +In addition to the [Pluto.jl](https://github.com/fonsp/Pluto.jl) and [Jupyter](https://jupyter.org/) notebooks, take a look at [Literate.jl](https://github.com/fredrikekre/Literate.jl) to enrich your code with comments and translate it to various formats. +[Books.jl](https://github.com/JuliaBooks/Books.jl) is relevant to draft long documents. + +[Quarto](https://quarto.org/) is an open-source scientific and technical publishing system that supports Python, R and Julia. +Quarto can render markdown files (`.md`), Quarto markdown files (`.qmd`), and Jupyter Notebooks (`.ipynb`) into documents (Word, PDF, presentations), web pages, blog posts, books, [and more](https://quarto.org/docs/output-formats/all-formats.html). +Additionally, Quarto makes it easy to share or [publish](https://quarto.org/docs/publishing/) rendered content to Github Pages, Netlify, Confluence, Hugging Face Spaces, among others. +[Quarto Pub](https://quartopub.com/) is a free publishing service for content created with Quarto. + +## Versions and registration + +The Julia community has adopted [semantic versioning](https://semver.org/), which means every package must have a version, and the version numbering follows strict rules. +The main consequence is that you need to specify [compatibility bounds](https://pkgdocs.julialang.org/v1/compatibility/) for your dependencies: this happens in the `[compat]` section of your `Project.toml`. +To initialize these bounds, use the `]compat` command in the Pkg mode of the REPL, or the package [PackageCompatUI.jl](https://github.com/GunnarFarneback/PackageCompatUI.jl). + +As your package lives on, new versions of your dependencies will be released. +The [CompatHelper.jl](https://github.com/JuliaRegistries/CompatHelper.jl) GitHub Action will help you monitor Julia dependencies and update your `Project.toml` accordingly. +In addition, [Dependabot](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuring-dependabot-version-updates#enabling-dependabot-version-updates) can monitor the dependencies... of your GitHub actions themselves. +But don't worry: both are default plugins in the PkgTemplates.jl setup. + +\advanced{ + +It may also happen that you incorrectly promise compatibility with an old version of a package. +To prevent that, the [julia-downgrade-compat](https://github.com/julia-actions/julia-downgrade-compat) GitHub action tests your package with the oldest possible version of every dependency, and verifies that everything still works. + +} + +If your package is useful to others in the community, it may be a good idea to register it, that is, make it part of the pool of packages that can be installed with + +```julia-repl +pkg> add MyAwesomePackage # made possible by registration +``` + +Note that unregistered packages can also be installed by anyone from the GitHub URL, but this a less reproducible solution: + +```julia-repl +pkg> add https://github.com/myuser/MyAwesomePackage # not ideal +``` + +To register your package, check out the [general registry](https://github.com/JuliaRegistries/General) guidelines. +The [Registrator.jl](https://github.com/JuliaRegistries/Registrator.jl) bot can help you automate the process. +Another handy bot, provided by default with PkgTemplates.jl, is [TagBot](https://github.com/JuliaRegistries/TagBot): it automatically tags new versions of your package following each registry release. +If you have performed the [necessary SSH configuration](https://documenter.juliadocs.org/stable/man/hosting/#travis-ssh), TagBot will also trigger documentation website builds following each release. + +\advanced{ + +If your package is only interesting to you and a small group of collaborators, or if you don't want to make it public, you can still register it by setting up a local registry: see [LocalRegistry.jl](https://github.com/GunnarFarneback/LocalRegistry.jl). + +} + +## Reproducibility + +Obtaining consistent and reproducible results is an essential part of experimental science. +[DrWatson.jl](https://github.com/JuliaDynamics/DrWatson.jl) is a general toolbox for running and re-running experiments in an orderly fashion. +We now explore a few specific issues that often arise. + +A first hurdle is [random number generation](https://docs.julialang.org/en/v1/stdlib/Random/), which is not guaranteed to remain stable across Julia versions. +To ensure that the random streams remain exactly the same, you need to use [StableRNGs.jl](https://github.com/JuliaRandom/StableRNGs.jl). +Another aspect is dataset download and management. +The packages [DataDeps.jl](https://github.com/oxinabox/DataDeps.jl), [DataToolkit.jl](https://github.com/tecosaur/DataToolkit.jl) and [ArtifactUtils.jl](https://github.com/JuliaPackaging/ArtifactUtils.jl) can help you bundle non-code elements with your package. +A third thing to consider is proper citation and versioning. +Giving your package a DOI with [Zenodo](https://zenodo.org/) ensures that everyone can properly cite it in scientific publications. +Similarly, your papers should cite the packages you use as dependencies: [PkgCite.jl](https://github.com/SebastianM-C/PkgCite.jl) will help with that. + +## Interoperability + +To ensure compatibility with earlier Julia versions, [Compat.jl](https://github.com/JuliaLang/Compat.jl) is your best ally. + +Making packages play nice with one another is a key goal of the Julia ecosystem. +Since Julia 1.9, this can be done with [package extensions](https://pkgdocs.julialang.org/v1/creating-packages/#Conditional-loading-of-code-in-packages-(Extensions)), which override specific behaviors based on the presence of a given package in the environment. +[PackageExtensionTools.jl](https://github.com/cjdoris/PackageExtensionTools.jl) eases the pain of setting up extensions. + +Furthermore, the Julia ecosystem as a whole plays nice with other programming languages too. +[C and Fortran](https://docs.julialang.org/en/v1/manual/calling-c-and-fortran-code/) are natively supported. +Python can be easily interfaced with the combination of [CondaPkg.jl](https://github.com/cjdoris/CondaPkg.jl) and [PythonCall.jl](https://github.com/cjdoris/PythonCall.jl). +Other language compatibility packages can be found in the [JuliaInterop](https://github.com/JuliaInterop) organization, like [RCall.jl](https://github.com/JuliaInterop/RCall.jl). + +Part of interoperability is also flexibility and customization: the [Preferences.jl](https://github.com/JuliaPackaging/Preferences.jl) package gives a nice way to specify various options in TOML files. + +\advanced{ + +Some package developers may need to define what kind of behavior they expect from a certain type, or what a certain method should do. +When writing it in the documentation is not enough, a formal testable specification becomes necessary. +This problem of "interfaces" does not yet have a definitive solution in Julia, but several options have been proposed: [Interfaces.jl](https://github.com/rafaqz/Interfaces.jl), [RequiredInterfaces.jl](https://github.com/Seelengrab/RequiredInterfaces.jl) and [PropCheck.jl](https://github.com/Seelengrab/PropCheck.jl) are all worth checking out. + +} + +## Collaboration + +Once your package grows big enough, you might need to bring in some help. +Working together on a software project has its own set of challenges, which are partially addressed by a good set of ground rules liks [SciML ColPrac](https://github.com/SciML/ColPrac). +Of course, collaboration goes both ways: if you find a Julia package you really like, you are more than welcome to [contribute](https://julialang.org/contribute/) as well, for example by opening issues or submitting pull requests. + + + +```!cleanup +Pkg.rm("MyAwesomePackage") # hide +``` diff --git a/writing/Project.toml b/writing/Project.toml new file mode 100644 index 0000000..e69de29 diff --git a/writing/index.md b/writing/index.md new file mode 100644 index 0000000..44aa12a --- /dev/null +++ b/writing/index.md @@ -0,0 +1,748 @@ ++++ +title = "Writing your code" +ignore_cache = true ++++ + + + +```! +# hideall +if isdir(sitepath("MyPackage")) + rm(sitepath("MyPackage"); recursive=true) +end +``` + +\activate{} + +# Writing your code + +In this post, you will learn about tools to create, run and debug Julia code. + +\toc + +## Getting help + +\tldr{You're not alone!} + +Before you write any line of code, it's good to know where to find help. +The official [help page](https://julialang.org/about/help/) is a good place to start. +In particular, the Julia [community](https://julialang.org/community/) is always happy to guide beginners. + +As a rule of thumb, the [Discourse forum](https://discourse.julialang.org/) is where you should ask your questions to make the answers discoverable for future users. +If you just want to chat with someone, you have a choice between the open source [Zulip](https://julialang.zulipchat.com/register/) and the closed source [Slack](https://julialang.org/slack/). +Some of the vocabulary used by community members may appear unfamiliar, but don't worry: [StartHere.jl](https://github.com/JuliaCommunity/StartHere.jl) gives you a good overview. + +## Installation + +\tldr{Use `juliaup`} + +The most natural starting point to install Julia onto your system is the [Julia downloads page](https://julialang.org/downloads/), which will tell you to use [`juliaup`](https://github.com/JuliaLang/juliaup). + +1. Windows users can download Julia and `juliaup` together from the [Windows Store](https://www.microsoft.com/store/apps/9NJNWW8PVKMN). +2. OSX or Linux users can execute the following terminal command: + +```bash +curl -fsSL https://install.julialang.org | sh +``` + +In both cases, this will make the `juliaup` and `julia` commands accessible from the terminal (or Windows Powershell). +On Windows this will also create an application launcher. +All users can start Julia by running + +```bash +julia +``` + +Meanwhile, `juliaup` provides [various utilities](https://github.com/JuliaLang/juliaup#using-juliaup) to download, update, organize and switch between different Julia versions. +As a bonus, you no longer have to manually specify the path to your executable. +This all works thanks to adaptive shortcuts called "channels", which allow you to access specific Julia versions without giving their exact number. + +For instance, the `release` channel will always point to the [current stable version](https://julialang.org/downloads/#current_stable_release), and the `lts` channel will always point to the [long-term support version](https://julialang.org/downloads/#long_term_support_release). +Upon installation of `juliaup`, the current stable version of Julia is downloaded and selected as the default. + +\advanced{ + +To use other channels, add them to `juliaup` and put a `+` in front of the channel name when you start Julia: + +```bash +juliaup add lts +julia +lts +``` + +You can get an overview of the channels installed on your computer with + +```bash +juliaup status +``` + +When new versions are tagged, the version associated with a given channel can change, which means a new executable needs to be downloaded. +If you want to catch up with the latest developments, just do + +```bash +juliaup update +``` + +} + +## REPL + +\tldr{The Julia REPL has 4 modes: Julia, package (`]`), help (`?`) and shell (`;`).} + +The Read-Eval-Print Loop (or REPL) is the most basic way to interact with Julia, check out its [documentation](https://docs.julialang.org/en/v1/stdlib/REPL/) for details. +You can start a REPL by typing `julia` into a terminal, or by clicking on the Julia application in your computer. +It will allow you to play around with arbitrary Julia code: + +```>repl-example +a, b = 1, 2; +a + b +``` + +This is the standard (Julia) mode of the REPL, but there are three other modes you need to know. +Each mode is entered by typing a specific character after the `julia>` prompt. +Once you're in a non-Julia mode, you stay there for every command you run. +To exit it, hit backspace after the prompt and you'll get the `julia>` prompt back. + +### Help mode (`?`) + +By pressing `?` you can obtain information and metadata about Julia objects (functions, types, etc.) or unicode symbols. +The query fetches the docstring of the object, which explains how to use it. + +```?help-example +println +``` + +If you don't know the exact name you are looking for, type a word surrounded by quotes to see in which docstrings it pops up. + +### Package mode (`]`) + +By pressing `]` you access [Pkg.jl](https://github.com/JuliaLang/Pkg.jl), Julia's integrated package manager, whose [documentation](https://pkgdocs.julialang.org/v1/getting-started/) is an absolute must-read. +Pkg.jl allows you to: + +* `]activate` different local, shared or temporary environments; +* `]instantiate` them by downloading the necessary packages; +* `]add`, `]update` (or `]up`) and `]remove` (or `]rm`) packages; +* get the `]status` (or `]st`) of your current environment. + +As an illustration, we download the package Example.jl inside our current environment: + +```]pkg-example +add Example +``` + +```]pkg-example +status +``` + +Note that the same keywords are also available in Julia mode: + +```>pkg-example-2 +using Pkg +Pkg.rm("Example") +``` + +The package mode itself also has a help mode, accessed with `?`, in case you're lost among all these new keywords. + +### Shell mode (`;`) + +By pressing `;` you enter a terminal, where you can execute any command you want. +Here's an example for Unix systems: + +```;shell-example +ls ./writing +``` + +## Editor + +\tldr{VSCode is the IDE with the best Julia support.} + +Most computer programs are just plain text files with a specific extension (in our case `.jl`). +So in theory, any text editor suffices to write and modify Julia code. +In practice, an Integrated Development Environment (or IDE) makes the experience much more pleasant, thanks to code-related utilities and language-specific plugins. + +The best IDE for Julia is [Visual Studio Code](https://code.visualstudio.com/), or VSCode, developed by Microsoft. +Indeed, the [Julia VSCode extension](https://www.julia-vscode.org/) is the most feature-rich of all Julia IDE plugins. +You can download it from the VSCode Marketplace and read its [documentation](https://www.julia-vscode.org/docs/stable/). + +\vscode{ + +In what follows, we will sometimes mention commands and [keyboard shortcuts](https://www.julia-vscode.org/docs/stable/userguide/keybindings/) provided by this extension. +But the only shortcut you need to remember is `Ctrl + Shift + P` (or `Cmd + Shift + P` on Mac): this opens the VSCode command palette, in which you can search for any command. +Type "julia" in the command palette to see what you can do. + +} + +\advanced{ + +Assuming you want to avoid the Microsoft ecosystem, [VSCodium](https://vscodium.com/) is a nearly bit-for-bit replacement for VSCode, but with an open source license and without telemetry. +If you don't want to use VSCode at all, other options include [Emacs](https://www.gnu.org/software/emacs/) and [Vim](https://www.vim.org/). +Check out [JuliaEditorSupport](https://github.com/JuliaEditorSupport) to see if your favorite IDE has a Julia plugin. +The available functionalities should be roughly similar to those of VSCode, at least for the basic aspects like running code. + +You may also want to download the [JuliaMono](https://juliamono.netlify.app/) font for esthetically pleasant unicode handling. +} + +## Running code + +\tldr{Open a REPL and run all your code there interactively.} + +You can execute a Julia script from your terminal, but in most cases that is not what you want to do. + +```bash +julia myfile.jl # avoid this +``` + +Julia has a rather high startup and compilation latency. +If you only use scripts, you will pay this cost every time you run a slightly modified version of your code. +That is why many Julia developers fire up a REPL at the beginning of the day and run all of their code there, chunk by chunk, in an interactive way. +Full files can be run interactively from the REPL with the `include` function. + +```julia-repl +julia> include("myfile.jl") +``` + +Alternatively, `includet` from the [Revise.jl](https://timholy.github.io/Revise.jl/stable/user_reference/#Revise.includet) package can be used to "include and track" a file. +This will automatically update changes to function definitions in the file in the running REPL session. + +\vscode{ + +[Running code](https://www.julia-vscode.org/docs/stable/userguide/runningcode/) is made much easier by the following commands: + +* `Julia: Restart REPL` (shortcut `Alt + J` then `Alt + R`) - this will open or restart the integrated Julia REPL. It is different from opening a plain VSCode terminal and launching Julia manually from there. +* `Julia: Execute Code in REPL and Move` (shortcut `Shift + Enter`) - this will execute the selected code in the integrated Julia REPL, like a notebook. + +} + +When keeping the same REPL open for a long time, it's common to end up with a "polluted" workspace where the definitions of certain variables or functions have been overwritten in unexpected ways. +This, along with other events like `struct` redefinitions, might force you to restart your REPL now and again, and that's okay. + +## Notebooks + +\tldr{Try either Jupyter or Pluto, depending on your reactivity needs.} + +Notebooks are a popular alternative to IDEs when it comes to short and self-contained code, typically in data science. +They are also a good fit for literate programming, where lines of code are interspersed with comments and explanations. + +The most well-known notebook ecosystem is [Jupyter](https://jupyter.org/), which supports Julia, Python and R as its three core languages. +To use it with Julia, you will need to install the [IJulia.jl](https://github.com/JuliaLang/IJulia.jl) backend. +Then, if you have also [installed Jupyter](https://jupyter.org/install) with `pip install jupyterlab`, you can run this command to launch the server: + +```bash +jupyter lab +``` + +If you only have IJulia.jl on your system, you can run this snippet instead: + +```julia-repl +julia> using IJulia + +julia> IJulia.notebook() +``` + +\vscode{ + +Jupyter notebooks can be opened, modified and run directly from the editor. +Thanks to the Julia extension, you don't even need to install IJulia.jl or Jupyter first. + +} + +A pure-Julia alternative to Jupyter is given by [Pluto.jl](https://plutojl.org/). +Unlike Jupyter notebooks, Pluto notebooks are + +* Reactive: when you update a cell, the other cells depending on it are updated. +* Reproducible: they come bundled with an exhaustive list of dependencies that are installed automatically. + +To try them out, install the package and then run + +```julia-repl +julia> using Pluto + +julia> Pluto.run() +``` + +## Markdown + +\tldr{Markdown is also a good fit for literate programming, and Quarto is an alternative to notebooks.} + +[Markdown](https://www.markdownguide.org/) is a markup language used to add formatting elements to plaintext text files. + +### Plain Text Markdown +Plain text markdown files, which have the `.md` extension, are not used for interactive programming, meaning one cannot run code written in the file. +As a result, plain text markdown files are usually rendered into a final product by other software. + +This is an example of a plain text markdown file: + +````markdown +# Title + +## Section Header + +This is example text. + +```julia +println("hello world") +``` +```` + +### Quarto + +[Quarto](https://quarto.org/) "is an open-source scientific and technical publishing system." +Quarto makes a plain text markdown file (`.md`) alternative called Quarto markdown file (`.qmd`). + +Quarto markdown files like plain text markdown files also integrate with editors, such as VSCode. + +\vscode{ + +Install the Quarto [extension](https://marketplace.visualstudio.com/items?itemName=quarto.quarto) for a streamlined experience. + +} + +Unlike plain text markdown files, Quarto markdown files have executable code chunks. +These code chunks provide a functionality similar to notebooks, thus Quarto markdown files are an alternative to notebooks. +Additionally, Quarto markdown files give users additional control over output and styling via the YAML header at the top of the `.qmd` file. + +As of Quarto version 1.5, users can choose from two Julia engines to execute code - a native Julia engine and IJulia.jl. +The primary difference between the native Julia engine and IJulia.jl is that the native Julia engine does not depend on Python and can utilize local environments. +For this reason it's recommended to start with the native Julia engine. +Learn more about the native Julia engine in Quarto's [documentation](https://quarto.org/docs/blog/posts/2024-07-11-1.5-release/#native-julia-engine). + +Below is an example of a Quarto markdown file. + +````quarto +--- +title: "My document" +format: + # renders a HTML document + html: + # table of contents + toc: true +execute: + # makes code chunks invisible in the output + # code output is still visible though + echo: false + # hides warnings in the output + warning: false +# native julia engine +engine: julia +--- + +# Title + +## Section Header + +Below is an executable code chunk. + +If this file were opened in an editor such as VSCode one could execute the `println("hello world")` Julia code and view the output, like in a notebook. + +```{julia} +println("hello world") +``` +```` + +## Environments + +\tldr{Activate a local environment for each project with `]activate path`. Its details are stored in `Project.toml` and `Manifest.toml`.} + +As we have seen, Pkg.jl is the Julia equivalent of `pip` or `conda` for Python. +It lets you [install packages](https://pkgdocs.julialang.org/v1/managing-packages/) and [manage environments](https://pkgdocs.julialang.org/v1/environments/) (collections of packages with specific versions). + +You can activate an environment from the Pkg REPL by specifying its path `]activate somepath`. +Typically, you would do `]activate .` to activate the environment in the current directory. +Another option is to directly start Julia inside an environment, with the command line option `julia --project=somepath`. + +Once in an environment, the packages you `]add` will be listed in two files `somepath/Project.toml` and `somepath/Manifest.toml`: + +* `Project.toml` contains general project information (name of the package, unique id, authors) and direct dependencies with version bounds. +* `Manifest.toml` contains the exact versions of all direct and indirect dependencies + +If you haven't entered any local project, packages will be installed in the default environment, called `@v1.X` after the active version of Julia (note the `@` before the name). +Packages installed that way are available no matter which local environment is active, because of "environment [stacking](https://docs.julialang.org/en/v1/manual/code-loading/#Environment-stacks)". +It is recommended to keep the default environment very light to avoid dependencies conflicts. It should contain only essential development tools. + +\vscode{ + +You can configure the [environment](https://www.julia-vscode.org/docs/stable/userguide/env/) in which a VSCode Julia REPL opens. +Just click the `Julia env: ...` button at the bottom. +Note however that the Julia version itself will always be the default one from `juliaup`. + +} + +\advanced{ + +You can visualize the dependency graph of an environment with [PkgDependency.jl](https://github.com/peng1999/PkgDependency.jl). + +} + +## Local packages + +\tldr{A package makes your code modular and reproducible.} + +Once your code base grows beyond a few scripts, you will want to [create a package](https://pkgdocs.julialang.org/v1/creating-packages/) of your own. +The first advantage is that you don't need to specify the path of every file: `using MyPackage: myfunc` is enough to get access to the names you define. +Furthermore, you can specify versions for your package and its dependencies, making your code easier and safer to reuse. + +To create a new package locally, one easy way is to use `]generate`. We will discuss more sophisticated workflows, including a graphical tool, in the next blog post. + +```>generate-package +Pkg.generate(sitepath("MyPackage")); # ignore sitepath +``` + +This command initializes a simple folder with a `Project.toml` and a `src` subfolder. +As we have seen, the `Project.toml` specifies the dependencies. +Meanwhile, the `src` subfolder contains a file `MyPackage.jl`, where a [module](https://docs.julialang.org/en/v1/manual/modules/) called `MyPackage` is defined. +It is the heart of your package, and will typically look like this when you're done: + +```julia +module MyPackage + +# imported dependencies +using OtherPackage1 +using OtherPackage2 + +# files defining functions, types, etc. +include("file1.jl") +include("subfolder/file2.jl") + +# names you want to make public +export myfunc +export MyType + +end +``` + +## Development workflow + +\tldr{Use Revise.jl to track code changes while you play with your package in its own environment.} + +Once you have created a package, your daily routine will look like this: + +1. Open a REPL in which you import `MyPackage` +2. Run some functions interactively, either by writing them directly in the REPL or from a Julia file that you use as a notebook +3. Modify some files in `MyPackage` +4. Go back to step 2 + +For that to work well, you need code modifications to be taken into account automatically. +That is why [Revise.jl](https://github.com/timholy/Revise.jl) exists. +In fact, it is used by so many Julia developers that some wish it were part of the core language: you can read its [documentation](https://timholy.github.io/Revise.jl/stable/) for more details. +If you start every REPL session by importing Revise.jl, then all the other packages you import after that will have their code tracked. +Whenever you edit a source file and hit save, the REPL will update its state accordingly. + +\vscode{ + +The Julia extension imports Revise.jl by default when it starts a REPL, provided it is installed in the default environment. + +} + +The only remaining question is: in which environment should you work? +In general, you can work within the environment defined by your package, and add all the dependencies you need there. +To summarize, this is how you get started: + +```julia +using Revise, Pkg +Pkg.activate("./MyPackage") +using MyPackage +MyPackage.myfunc() +``` + +\advanced{ + +There are situations where the previous method does not work: + +* if you are developing several packages at once and want to use them together +* if your interactive work requires heavy dependencies that your package itself does not need (for instance plotting). + +Then, you will need to use another environment as a playground, and `]develop` (or `]dev`) your package(s) into it. +Note the new Pkg.jl keyword: `]add PackageName` is used to download a fixed version of a registered package, while `]develop path` links to the current state of the code in a local folder. +To summarize, this is how you get started: + +```julia +using Revise, Pkg +Pkg.activate("./MyPlayground") +Pkg.develop(path="./MyPackage") +using MyPackage +MyPackage.myfunc() +``` + +For the common case of dependencies needed for interactive work only, [shared](https://pkgdocs.julialang.org/v1/environments/#Shared-environments) or [stacked](https://docs.julialang.org/en/v1/manual/code-loading/#Environment-stacks) environments are another practical solution. +[ShareAdd.jl](https://github.com/Eben60/ShareAdd.jl) can help you in using and managing these (see its documentation). + +} + +## Configuration + +\tldr{Use the startup file to import packages as soon as Julia starts.} + +Julia accepts [startup flags](https://docs.julialang.org/en/v1/manual/command-line-interface/#command-line-interface) to handle settings such as the number of threads available or the environment in which it launches. +In addition, most Julia developers also have a [startup file](https://docs.julialang.org/en/v1/manual/command-line-interface/#Startup-file) which is run automatically every time the language is started. +It is located at `.julia/config/startup.jl`. + +The basic component that everyone puts in the startup file is Revise.jl: + +```julia +try + using Revise +catch e + @warn "Error initializing Revise" +end +``` + +In addition, users commonly import packages that affect the REPL experience, as well as esthetic, benchmarking or profiling utilities. +A typical example is [OhMyREPL.jl](https://github.com/KristofferC/OhMyREPL.jl) which is widely used for syntax highlighting in the REPL. +More generally, the startup file allows you to define your own favorite helper functions and have them immediately available in every Julia session. +[StartupCustomizer.jl](https://github.com/abraemer/StartupCustomizer.jl) can help you set up your startup file. + +\advanced{ + +Here are a few more startup packages that can make your life easier once you know the language better: + +* [AbbreviatedStackTraces.jl](https://github.com/BioTurboNick/AbbreviatedStackTraces.jl) allows you to shorten error stacktraces, which can sometimes get pretty long (beware of its [interactions with VSCode](https://github.com/BioTurboNick/AbbreviatedStackTraces.jl/issues/38)) +* [Term.jl](https://github.com/FedeClaudi/Term.jl) offers a completely new way to display things like types and errors (see the [advanced configuration](https://fedeclaudi.github.io/Term.jl/stable/adv/adv/) to enable it by default). + +} + +## Interactivity + +\tldr{Explore source code from within the REPL.} + +The Julia REPL comes bundled with [InteractiveUtils.jl](https://docs.julialang.org/en/v1/stdlib/InteractiveUtils/), a bunch of very useful functions for interacting with source code. + +```!interactiveutils +using InteractiveUtils # hide +``` + +Here are a few examples: + +```>interactiveutils_examples +supertypes(Int64) +subtypes(Integer) +length(methodswith(Integer)) +@which exp(1) +apropos("matrix exponential") +``` + +When you ask for help on a Julia forum, you might want to include your local Julia information: + +```> +versioninfo() +``` + +\advanced{ + +The following packages can give you even more interactive power: + +* [About.jl](https://github.com/tecosaur/About.jl) to look for detailed information about a function, type, value, or something else entirely. +* [InteractiveCodeSearch.jl](https://github.com/tkf/InteractiveCodeSearch.jl) to look for a precise implementation of a function. +* [InteractiveErrors.jl](https://github.com/MichaelHatherly/InteractiveErrors.jl) to navigate through stacktraces. +* [CodeTracking.jl](https://github.com/timholy/CodeTracking.jl) to extend InteractiveUtils.jl + +} + +## Logging + +\tldr{Logging macros are more versatile than printing.} + +When you encounter a problem in your code or want to track progress, a common reflex is to add `print` statements everywhere. + +```!printing_func +function printing_func(n) + for i in 1:n + println(i^2) + end +end +``` + +```>printing_repl +printing_func(3) +``` + +A slight improvement is given by the `@show` macro, which displays the variable name: + +```!showing_func +function showing_func(n) + for i in 1:n + @show i^2 + end +end +``` + +```>showing_repl +showing_func(3) +``` + +But you can go even further with the macros `@debug`, `@info`, `@warn` and `@error`. +They have several advantages over printing: + +* They display variable names and a custom message +* They show the line number they were called from +* They can be disabled and filtered according to source module and severity level +* They work well in multithreaded code +* They can write their output to a file + +```!warning_func +function warning_func(n) + for i in 1:n + @warn "This is bad" i^2 + end +end +``` + +```>warning_repl +warning_func(3) +``` + +Refer to the logging [documentation](https://docs.julialang.org/en/v1/stdlib/Logging/) for more information. + +\advanced{ + +In particular, note that `@debug` messages are suppressed by default. +You can enable them through the `JULIA_DEBUG` environment variable if you specify the source module name, typically `Main` or your package module. + +} + +Beyond the built-in logging utilities, [ProgressLogging.jl](https://github.com/JuliaLogging/ProgressLogging.jl) has a macro `@progress`, which interfaces nicely with VSCode and Pluto to display progress bars. +And [Suppressor.jl](https://github.com/JuliaIO/Suppressor.jl) can sometimes be handy when you need to suppress warnings or other bothersome messages (use at your own risk). + +## Debugging + +\tldr{Infiltrator.jl and Debugger.jl allow you to peek inside a function while its execution is paused.} + +The problem with printing or logging is that you cannot interact with local variables or save them for further analysis. +The following two packages solve this issue, and they probably belong in your default environment `@v1.X`, like Revise.jl. + +### Setting + +Assume you want to debug a function checking whether the $n$-th [Fermat number](https://en.wikipedia.org/wiki/Fermat_number) $F_n = 2^{2^n} + 1$ is prime: + +```!fermat +function fermat_prime(n) + k = 2^n + F = 2^k + 1 + for d in 2:isqrt(F) # integer square root + if F % d == 0 + return false + end + end + return true +end +``` + +```>fermat-test +fermat_prime(4) +fermat_prime(6) +``` + +Unfortunately, $F_4 = 65537$ is the largest known Fermat prime, which means $F_6$ is incorrectly classified. +Let's investigate why this happens! + +### Infiltrator.jl + +[Infiltrator.jl](https://github.com/JuliaDebug/Infiltrator.jl) is a lightweight inspection package, which will not slow down your code at all. +Its `@infiltrate` macro allows you to directly set breakpoints in your code. +Calling a function which hits a breakpoint will activate the Infiltrator REPL-mode +and change the prompt to `infil>`. +Typing `?` in this mode will summarize available commands. +For example, typing `@locals` in Infiltrator-mode will print local variables: + +```julia +using Infiltrator + +function fermat_prime_infil(n) + k = 2^n + F = 2^k + 1 + @infiltrate + for d in 2:isqrt(F) + if F % d == 0 + return false + end + end + return true +end +``` + +What makes Infiltrator.jl even more powerful is the `@exfiltrate` macro, which allows you to move local variables into a global storage called the `safehouse`. + +```julia-repl +julia> fermat_prime_infil(6) +Infiltrating fermat_prime_infil(n::Int64) + at REPL[2]:4 + +infil> @exfiltrate k F +Exfiltrating 2 local variables into the safehouse. + +infil> @continue + +true + +julia> safehouse.k +64 + +julia> safehouse.F +1 +``` + +The diagnosis is a classic one: [integer overflow](https://docs.julialang.org/en/v1/manual/faq/#faq-integer-arithmetic). +Indeed, $2^{64}$ is larger than the maximum integer value in Julia: + +```>typemax +typemax(Int) +2^63-1 +``` + +And the solution is to call our function on "big" integers with an arbitrary number of bits: + +```>fermat-big +fermat_prime(big(6)) +``` + +### Debugger.jl + +[Debugger.jl](https://github.com/JuliaDebug/Debugger.jl) allows us to interrupt code execution anywhere we want, even in functions we did not write. +Using its `@enter` macro, we can enter a function call and walk through the call stack, at the cost of reduced performance. + +The REPL prompt changes to `1|debug>`, allowing you to use [custom navigation commands](https://github.com/JuliaDebug/Debugger.jl#debugger-commands) to step into and out of function calls, show local variables and set breakpoints. +Typing a backtick `` ` `` will change the prompt to `1|julia>`, indicating evaluation mode. +Any expression typed in this mode will be evaluated in the local context. +This is useful to show local variables, as demonstrated in the following example: + +```julia-repl +julia> using Debugger + +julia> @enter fermat_prime(6) +In fermat_prime(n) at REPL[7]:1 + 1 function fermat_prime(n) +>2 k = 2^n + 3 F = 2^k + 1 + 4 for d in 2:isqrt(F) # integer square root + 5 if F % d == 0 + 6 return false + +About to run: (^)(2, 6) +1|debug> n +In fermat_prime(n) at REPL[7]:1 + 1 function fermat_prime(n) + 2 k = 2^n +>3 F = 2^k + 1 + 4 for d in 2:isqrt(F) # integer square root + 5 if F % d == 0 + 6 return false + 7 end + +About to run: (^)(2, 64) +1|julia> k +64 +``` + +\vscode{ + +VSCode offers a nice [graphical interface for debugging](https://www.julia-vscode.org/docs/stable/userguide/debugging/). +Click left of a line number in an editor pane to add a _breakpoint_, which is represented by a red circle. +In the debugging pane of the Julia extension, click `Run and Debug` to start the debugger. +The program will automatically halt when it hits a breakpoint. +Using the toolbar at the top of the editor, you can then _continue_, _step over_, _step into_ and _step out_ of your code. +The debugger will open a pane showing information about the code such as local variables inside of the current function, their current values and the full call stack. + +The debugger can be [sped up](https://www.julia-vscode.org/docs/dev/userguide/debugging/#Settings-to-speed-up-the-debugger) by selectively compiling modules that you will not need to step into via the `+` symbol at the bottom of the debugging pane. +It is often easiest to start by adding `ALL_MODULES_EXCEPT_MAIN` to the compiled list, and then selectively remove the modules you need to have interpreted +by typing their name into the same `+` menu but with a `-` sign in front e.g. `-MyModule`. +} + +