diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 33f29d79418..f8a257e28f3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,8 +5,18 @@ version: 2 updates: - - package-ecosystem: "cargo" # See documentation for possible values - directory: "/" # Location of package manifests + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "cargo" + directory: "/pyo3-benches/" + schedule: + interval: "weekly" + + - package-ecosystem: "cargo" + directory: "/pyo3-ffi-check/" schedule: interval: "weekly" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index b344525cabe..11375e966b3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,6 +8,6 @@ Please consider adding the following to your pull request: - docs to all new functions and / or detail in the guide - tests for all new or changed functions -PyO3's CI pipeline will check your pull request. To run its tests +PyO3's CI pipeline will check your pull request, thus make sure you have checked the `Contributing.md` guidelines. To run most of its tests locally, you can run ```nox```. See ```nox --list-sessions``` for a list of supported actions. diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 572f77efd76..62c64d9d113 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -15,10 +15,12 @@ concurrency: jobs: benchmarks: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + with: + python-version: '3.12' - uses: dtolnay/rust-toolchain@stable with: components: rust-src @@ -30,14 +32,15 @@ jobs: pyo3-benches continue-on-error: true - - name: Install cargo-codspeed - run: cargo install cargo-codspeed + - uses: taiki-e/install-action@v2 + with: + tool: cargo-codspeed - name: Install nox run: pip install nox - name: Run the benchmarks - uses: CodSpeedHQ/action@v2 + uses: CodSpeedHQ/action@v3 with: run: nox -s codspeed token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d2f74401dd6..5fb2124836e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,13 +16,13 @@ on: rust-target: required: true type: string - extra-features: + MSRV: required: true type: string jobs: build: - continue-on-error: ${{ endsWith(inputs.python-version, '-dev') || contains(fromJSON('["3.7", "pypy3.7"]'), inputs.python-version) || inputs.rust == 'beta' || inputs.rust == 'nightly' }} + continue-on-error: ${{ endsWith(inputs.python-version, '-dev') || contains(fromJSON('["3.7", "3.8"]'), inputs.python-version) || contains(fromJSON('["beta", "nightly"]'), inputs.rust) }} runs-on: ${{ inputs.os }} if: ${{ !(startsWith(inputs.python-version, 'graalpy') && startsWith(inputs.os, 'windows')) }} steps: @@ -38,6 +38,10 @@ jobs: - name: Install nox run: python -m pip install --upgrade pip && pip install nox + - if: inputs.python-version == 'graalpy24.1' + name: Install GraalPy virtualenv (only GraalPy 24.1) + run: python -m pip install 'git+https://github.com/oracle/graalpython#egg=graalpy_virtualenv_seeder&subdirectory=graalpy_virtualenv_seeder' + - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master with: @@ -54,14 +58,18 @@ jobs: name: Prepare LD_LIBRARY_PATH (Ubuntu only) run: echo LD_LIBRARY_PATH=${pythonLocation}/lib >> $GITHUB_ENV - - if: inputs.rust == '1.56.0' - name: Prepare minimal package versions (MSRV only) - run: nox -s set-minimal-package-versions + - if: inputs.rust == inputs.MSRV + name: Prepare MSRV package versions + run: nox -s set-msrv-package-versions - if: inputs.rust != 'stable' name: Ignore changed error messages when using trybuild run: echo "TRYBUILD=overwrite" >> "$GITHUB_ENV" + - if: inputs.rust == 'nightly' + name: Prepare to test on nightly rust + run: echo "MAYBE_NIGHTLY=nightly" >> "$GITHUB_ENV" + - name: Build docs run: nox -s docs @@ -88,26 +96,31 @@ jobs: - name: Build (all additive features) if: ${{ !startsWith(inputs.python-version, 'graalpy') }} - run: cargo build --lib --tests --no-default-features --features "full ${{ inputs.extra-features }}" + run: cargo build --lib --tests --no-default-features --features "multiple-pymethods full $MAYBE_NIGHTLY" - if: ${{ startsWith(inputs.python-version, 'pypy') }} - name: Build PyPy (abi3-py37) - run: cargo build --lib --tests --no-default-features --features "abi3-py37 full ${{ inputs.extra-features }}" + name: Build PyPy (abi3-py39) + run: cargo build --lib --tests --no-default-features --features "multiple-pymethods abi3-py39 full $MAYBE_NIGHTLY" # Run tests (except on PyPy, because no embedding API). - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} name: Test - run: cargo test --no-default-features --features "full ${{ inputs.extra-features }}" + run: cargo test --no-default-features --features "full $MAYBE_NIGHTLY" + + # Repeat, with multiple-pymethods feature enabled (it's not entirely additive) + - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} + name: Test + run: cargo test --no-default-features --features "multiple-pymethods full $MAYBE_NIGHTLY" # Run tests again, but in abi3 mode - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} name: Test (abi3) - run: cargo test --no-default-features --features "abi3 full ${{ inputs.extra-features }}" + run: cargo test --no-default-features --features "multiple-pymethods abi3 full $MAYBE_NIGHTLY" # Run tests again, for abi3-py37 (the minimal Python version) - if: ${{ (!startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy')) && (inputs.python-version != '3.7') }} name: Test (abi3-py37) - run: cargo test --no-default-features --features "abi3-py37 full ${{ inputs.extra-features }}" + run: cargo test --no-default-features --features "multiple-pymethods abi3-py37 full $MAYBE_NIGHTLY" - name: Test proc-macro code run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml @@ -122,8 +135,7 @@ jobs: CARGO_TARGET_DIR: ${{ github.workspace }}/target - uses: dorny/paths-filter@v3 - # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here - if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !startsWith(inputs.python-version, 'graalpy') }} + if: ${{ inputs.rust == 'stable' && !startsWith(inputs.python-version, 'graalpy') }} id: ffi-changes with: base: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} @@ -136,9 +148,8 @@ jobs: - '.github/workflows/build.yml' - name: Run pyo3-ffi-check - # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here, nor - # is pypy 3.9 on windows - if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !startsWith(inputs.python-version, 'graalpy') && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }} + # pypy 3.9 on windows is not PEP 3123 compliant, nor is graalpy + if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && !startsWith(inputs.python-version, 'graalpy') && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }} run: nox -s ffi-check env: diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 0a782b4010f..b4c3b9892f3 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -11,5 +11,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + with: + python-version: '3.12' - run: python -m pip install --upgrade pip && pip install nox - run: nox -s check-changelog diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12db5867f90..18367f353a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + with: + python-version: "3.12" - run: python -m pip install --upgrade pip && pip install nox - uses: dtolnay/rust-toolchain@stable with: @@ -31,6 +33,19 @@ jobs: - name: Check rust formatting (rustfmt) run: nox -s rustfmt + resolve: + runs-on: ubuntu-latest + outputs: + MSRV: ${{ steps.resolve-msrv.outputs.MSRV }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: resolve MSRV + id: resolve-msrv + run: echo MSRV=`python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["package"]["rust-version"])'` >> $GITHUB_OUTPUT + semver-checks: if: github.ref != 'refs/heads/main' needs: [fmt] @@ -38,28 +53,31 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + with: + python-version: "3.12" - uses: obi1kenobi/cargo-semver-checks-action@v2 check-msrv: - needs: [fmt] + needs: [fmt, resolve] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@master with: - toolchain: 1.56.0 - targets: x86_64-unknown-linux-gnu + toolchain: ${{ needs.resolve.outputs.MSRV }} components: rust-src - uses: actions/setup-python@v5 with: - architecture: "x64" + python-version: "3.12" - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.event_name != 'merge_group' }} - run: python -m pip install --upgrade pip && pip install nox - - name: Prepare minimal package versions - run: nox -s set-minimal-package-versions - - run: nox -s check-all + # This is a smoke test to confirm that CI will run on MSRV (including dev dependencies) + - name: Check with MSRV package versions + run: | + nox -s set-msrv-package-versions + nox -s check-all env: CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu @@ -72,43 +90,49 @@ jobs: fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }} matrix: rust: [stable] - platform: [ - { - os: "macos-14", # first available arm macos runner - python-architecture: "arm64", - rust-target: "aarch64-apple-darwin", - }, - { - os: "ubuntu-latest", - python-architecture: "x64", - rust-target: "x86_64-unknown-linux-gnu", - }, - { - os: "ubuntu-latest", - python-architecture: "x64", - rust-target: "powerpc64le-unknown-linux-gnu", - }, - { - os: "ubuntu-latest", - python-architecture: "x64", - rust-target: "s390x-unknown-linux-gnu", - }, - { - os: "ubuntu-latest", - python-architecture: "x64", - rust-target: "wasm32-wasi", - }, - { - os: "windows-latest", - python-architecture: "x64", - rust-target: "x86_64-pc-windows-msvc", - }, - { - os: "windows-latest", - python-architecture: "x86", - rust-target: "i686-pc-windows-msvc", - }, - ] + platform: + [ + { + os: "macos-latest", + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", + }, + { + os: "ubuntu-latest", + python-architecture: "x64", + rust-target: "x86_64-unknown-linux-gnu", + }, + { + os: "ubuntu-22.04-arm", + python-architecture: "arm64", + rust-target: "aarch64-unknown-linux-gnu", + }, + { + os: "ubuntu-latest", + python-architecture: "x64", + rust-target: "powerpc64le-unknown-linux-gnu", + }, + { + os: "ubuntu-latest", + python-architecture: "x64", + rust-target: "s390x-unknown-linux-gnu", + }, + { + os: "ubuntu-latest", + python-architecture: "x64", + rust-target: "wasm32-wasip1", + }, + { + os: "windows-latest", + python-architecture: "x64", + rust-target: "x86_64-pc-windows-msvc", + }, + { + os: "windows-latest", + python-architecture: "x86", + rust-target: "i686-pc-windows-msvc", + }, + ] include: # Run beta clippy as a way to detect any incoming lints which may affect downstream users - rust: beta @@ -129,6 +153,7 @@ jobs: components: clippy,rust-src - uses: actions/setup-python@v5 with: + python-version: "3.12" architecture: ${{ matrix.platform.python-architecture }} - uses: Swatinem/rust-cache@v2 with: @@ -141,7 +166,7 @@ jobs: build-pr: if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-build-full') && github.event_name == 'pull_request' }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} - needs: [fmt] + needs: [fmt, resolve] uses: ./.github/workflows/build.yml with: os: ${{ matrix.platform.os }} @@ -149,24 +174,22 @@ jobs: python-architecture: ${{ matrix.platform.python-architecture }} rust: ${{ matrix.rust }} rust-target: ${{ matrix.platform.rust-target }} - extra-features: ${{ matrix.platform.extra-features }} + MSRV: ${{ needs.resolve.outputs.MSRV }} secrets: inherit strategy: # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }} matrix: - extra-features: ["multiple-pymethods"] rust: [stable] python-version: ["3.12"] - platform: - [ + platform: [ { - os: "macos-14", # first available arm macos runner + os: "macos-latest", # first available arm macos runner python-architecture: "arm64", rust-target: "aarch64-apple-darwin", }, { - os: "macos-13", # last available x86_64 macos runner + os: "macos-13", # last available x86_64 macos runner python-architecture: "x64", rust-target: "x86_64-apple-darwin", }, @@ -175,6 +198,11 @@ jobs: python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu", }, + { + os: "ubuntu-22.04-arm", + python-architecture: "arm64", + rust-target: "aarch64-unknown-linux-gnu", + }, { os: "windows-latest", python-architecture: "x64", @@ -197,11 +225,10 @@ jobs: python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu", } - extra-features: "nightly multiple-pymethods" build-full: if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} - needs: [fmt] + needs: [fmt, resolve] uses: ./.github/workflows/build.yml with: os: ${{ matrix.platform.os }} @@ -209,38 +236,34 @@ jobs: python-architecture: ${{ matrix.platform.python-architecture }} rust: ${{ matrix.rust }} rust-target: ${{ matrix.platform.rust-target }} - extra-features: ${{ matrix.platform.extra-features }} + MSRV: ${{ needs.resolve.outputs.MSRV }} secrets: inherit strategy: # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }} matrix: - extra-features: ["multiple-pymethods"] # Because MSRV doesn't support this rust: [stable] - python-version: [ - "3.7", - "3.8", - "3.9", - "3.10", - "3.11", - "3.12", - "pypy3.7", - "pypy3.8", - "pypy3.9", - "pypy3.10", - "graalpy24.0", - ] + python-version: + [ + "3.7", + "3.8", + "3.9", + "3.10", + "3.11", + "3.12", + "3.13", + "pypy3.9", + "pypy3.10", + "pypy3.11", + "graalpy24.0", + "graalpy24.1", + ] platform: [ - # for the full matrix, use x86_64 macos runners because not all Python versions - # PyO3 supports are available for arm on GitHub Actions. (Availability starts - # around Python 3.10, can switch the full matrix to arm once earlier versions - # are dropped.) - # NB: if this switches to arm, switch the arm job below in the `include` to x86_64 { - os: "macos-13", - python-architecture: "x64", - rust-target: "x86_64-apple-darwin", + os: "macos-latest", + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", }, { os: "ubuntu-latest", @@ -255,7 +278,7 @@ jobs: ] include: # Test minimal supported Rust version - - rust: 1.56.0 + - rust: ${{ needs.resolve.outputs.MSRV }} python-version: "3.12" platform: { @@ -263,7 +286,6 @@ jobs: python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu", } - extra-features: "" # Test the `nightly` feature - rust: nightly @@ -274,7 +296,6 @@ jobs: python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu", } - extra-features: "nightly multiple-pymethods" # Run rust beta to help catch toolchain regressions - rust: beta @@ -285,9 +306,8 @@ jobs: python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu", } - extra-features: "multiple-pymethods" - # Test 32-bit Windows only with the latest Python version + # Test 32-bit Windows and x64 macOS only with the latest Python version - rust: stable python-version: "3.12" platform: @@ -296,19 +316,89 @@ jobs: python-architecture: "x86", rust-target: "i686-pc-windows-msvc", } - extra-features: "multiple-pymethods" - - # test arm macos runner with the latest Python version - # NB: if the full matrix switchess to arm, switch to x86_64 here - rust: stable python-version: "3.12" platform: { - os: "macos-14", + os: "macos-13", + python-architecture: "x64", + rust-target: "x86_64-apple-darwin", + } + # ubuntu-latest (24.04) no longer supports 3.7, so run on 22.04 + - rust: stable + python-version: "3.7" + platform: + { + os: "ubuntu-22.04", + python-architecture: "x64", + rust-target: "x86_64-unknown-linux-gnu", + } + + # arm64 macOS Python not available on GitHub Actions until 3.10 + # so backfill 3.7-3.9 with x64 macOS runners + - rust: stable + python-version: "3.7" + platform: + { + os: "macos-13", + python-architecture: "x64", + rust-target: "x86_64-apple-darwin", + } + - rust: stable + python-version: "3.8" + platform: + { + os: "macos-13", + python-architecture: "x64", + rust-target: "x86_64-apple-darwin", + } + - rust: stable + python-version: "3.9" + platform: + { + os: "macos-13", + python-architecture: "x64", + rust-target: "x86_64-apple-darwin", + } + # arm64 Linux runner is in public preview, so test 3.13 on it + - rust: stable + python-version: "3.13" + platform: + { + os: "ubuntu-22.04-arm", + python-architecture: "arm64", + rust-target: "aarch64-unknown-linux-gnu", + } + + exclude: + # ubuntu-latest (24.04) no longer supports 3.7 + - python-version: "3.7" + platform: { os: "ubuntu-latest" } + # arm64 macOS Python not available on GitHub Actions until 3.10 + - rust: stable + python-version: "3.7" + platform: + { + os: "macos-latest", + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", + } + - rust: stable + python-version: "3.8" + platform: + { + os: "macos-latest", + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", + } + - rust: stable + python-version: "3.9" + platform: + { + os: "macos-latest", python-architecture: "arm64", rust-target: "aarch64-apple-darwin", } - extra-features: "multiple-pymethods" valgrind: if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} @@ -317,6 +407,10 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + with: + # FIXME valgrind detects an issue with Python 3.12.5, needs investigation + # whether it's a PyO3 issue or upstream CPython. + python-version: "3.12.4" - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.event_name != 'merge_group' }} @@ -336,6 +430,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + with: + python-version: "3.12" - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.event_name != 'merge_group' }} @@ -356,49 +452,42 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + with: + python-version: "3.12" - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.event_name != 'merge_group' }} - uses: dtolnay/rust-toolchain@nightly with: components: rust-src - - run: cargo rustdoc --lib --no-default-features --features full -Zunstable-options --config "build.rustdocflags=[\"--cfg\", \"docsrs\"]" + - run: cargo rustdoc --lib --no-default-features --features full,jiff-02 -Zunstable-options --config "build.rustdocflags=[\"--cfg\", \"docsrs\"]" coverage: + if: ${{ github.event_name != 'merge_group' }} needs: [fmt] name: coverage ${{ matrix.os }} strategy: matrix: - os: ["windows-latest", "macos-14", "ubuntu-latest"] # first available arm macos runner + os: ["windows-latest", "macos-latest", "ubuntu-latest"] runs-on: ${{ matrix.os }} steps: - - if: ${{ github.event_name == 'pull_request' && matrix.os != 'ubuntu-latest' }} - id: should-skip - shell: bash - run: echo 'skip=true' >> $GITHUB_OUTPUT - uses: actions/checkout@v4 - if: steps.should-skip.outputs.skip != 'true' - uses: actions/setup-python@v5 - if: steps.should-skip.outputs.skip != 'true' + with: + python-version: "3.12" - uses: Swatinem/rust-cache@v2 - if: steps.should-skip.outputs.skip != 'true' with: save-if: ${{ github.event_name != 'merge_group' }} - uses: dtolnay/rust-toolchain@stable - if: steps.should-skip.outputs.skip != 'true' with: components: llvm-tools-preview,rust-src - name: Install cargo-llvm-cov - if: steps.should-skip.outputs.skip != 'true' uses: taiki-e/install-action@cargo-llvm-cov - run: python -m pip install --upgrade pip && pip install nox - if: steps.should-skip.outputs.skip != 'true' - run: nox -s coverage - if: steps.should-skip.outputs.skip != 'true' - - uses: codecov/codecov-action@v4 - if: steps.should-skip.outputs.skip != 'true' + - uses: codecov/codecov-action@v5 with: - file: coverage.json + files: coverage.json name: ${{ matrix.os }} token: ${{ secrets.CODECOV_TOKEN }} @@ -421,14 +510,14 @@ jobs: components: rust-src - uses: actions/setup-node@v4 with: - node-version: 14 + node-version: 18 - run: python -m pip install --upgrade pip && pip install nox - - uses: actions/cache@v4 + - uses: actions/cache/restore@v4 id: cache with: path: | .nox/emscripten - key: ${{ hashFiles('emscripten/*') }} - ${{ hashFiles('noxfile.py') }} - ${{ steps.setup-python.outputs.python-path }} + key: emscripten-${{ hashFiles('emscripten/*') }}-${{ hashFiles('noxfile.py') }}-${{ steps.setup-python.outputs.python-path }} - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.event_name != 'merge_group' }} @@ -437,6 +526,12 @@ jobs: run: nox -s build-emscripten - name: Test run: nox -s test-emscripten + - uses: actions/cache/save@v4 + if: ${{ github.event_name != 'merge_group' }} + with: + path: | + .nox/emscripten + key: emscripten-${{ hashFiles('emscripten/*') }}-${{ hashFiles('noxfile.py') }}-${{ steps.setup-python.outputs.python-path }} test-debug: if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} @@ -452,8 +547,8 @@ jobs: components: rust-src - name: Install python3 standalone debug build with nox run: | - PBS_RELEASE="20231002" - PBS_PYTHON_VERSION="3.12.0" + PBS_RELEASE="20241016" + PBS_PYTHON_VERSION="3.13.0" PBS_ARCHIVE="cpython-${PBS_PYTHON_VERSION}+${PBS_RELEASE}-x86_64-unknown-linux-gnu-debug-full.tar.zst" wget "https://github.com/indygreg/python-build-standalone/releases/download/${PBS_RELEASE}/${PBS_ARCHIVE}" tar -I zstd -xf "${PBS_ARCHIVE}" @@ -469,10 +564,10 @@ jobs: PYO3_CONFIG_FILE=$(mktemp) cat > $PYO3_CONFIG_FILE << EOF implementation=CPython - version=3.12 + version=3.13 shared=true abi3=false - lib_name=python3.12d + lib_name=python3.13d lib_dir=${{ github.workspace }}/python/install/lib executable=${{ github.workspace }}/python/install/bin/python3 pointer_width=64 @@ -482,12 +577,52 @@ jobs: echo PYO3_CONFIG_FILE=$PYO3_CONFIG_FILE >> $GITHUB_ENV - run: python3 -m nox -s test + test-free-threaded: + needs: [fmt] + name: Free threaded tests - ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + steps: + - uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.event_name != 'merge_group' }} + - uses: dtolnay/rust-toolchain@stable + with: + components: rust-src + - uses: actions/setup-python@v5.5.0 + with: + python-version: "3.13t" + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - run: python3 -m sysconfig + - run: python3 -m pip install --upgrade pip && pip install nox + - name: Prepare coverage environment + run: | + cargo llvm-cov clean --workspace --profraw-only + nox -s set-coverage-env + - run: nox -s ffi-check + - run: nox + - name: Generate coverage report + run: nox -s generate-coverage-report + - name: Upload coverage report + uses: codecov/codecov-action@v5 + with: + files: coverage.json + name: ${{ matrix.os }}-test-free-threaded + token: ${{ secrets.CODECOV_TOKEN }} + test-version-limits: needs: [fmt] if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.event_name != 'merge_group' }} @@ -496,21 +631,33 @@ jobs: - run: python3 -m nox -s test-version-limits check-feature-powerset: - needs: [fmt] + needs: [fmt, resolve] if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} runs-on: ubuntu-latest + name: check-feature-powerset ${{ matrix.rust }} + strategy: + # run on stable and MSRV to check that all combinations of features are expected to build fine on our supported + # range of compilers + matrix: + rust: ["stable"] + include: + - rust: ${{ needs.resolve.outputs.MSRV }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + with: + python-version: "3.12" - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.event_name != 'merge_group' }} - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@master with: - components: rust-src - - uses: taiki-e/install-action@cargo-hack + toolchain: stable + - uses: taiki-e/install-action@v2 + with: + tool: cargo-hack,cargo-minimal-versions - run: python3 -m pip install --upgrade pip && pip install nox - - run: python3 -m nox -s check-feature-powerset + - run: python3 -m nox -s check-feature-powerset -- ${{ matrix.rust != 'stable' && 'minimal-versions' || '' }} test-cross-compilation: needs: [fmt] @@ -535,30 +682,42 @@ jobs: # ubuntu x86_64 -> windows x86_64 - os: "ubuntu-latest" target: "x86_64-pc-windows-gnu" - flags: "-i python3.12 --features abi3 --features generate-import-lib" - manylinux: off + flags: "-i python3.12 --features generate-import-lib" # macos x86_64 -> aarch64 - - os: "macos-13" # last x86_64 macos runners + - os: "macos-13" # last x86_64 macos runners target: "aarch64-apple-darwin" # macos aarch64 -> x86_64 - - os: "macos-14" # aarch64 macos runners + - os: "macos-latest" target: "x86_64-apple-darwin" + # windows x86_64 -> aarch64 + - os: "windows-latest" + target: "aarch64-pc-windows-msvc" + flags: "-i python3.12 --features generate-import-lib" steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" - uses: Swatinem/rust-cache@v2 with: - workspaces: - examples/maturin-starter + workspaces: examples/maturin-starter save-if: ${{ github.event_name != 'merge_group' }} key: ${{ matrix.target }} - name: Setup cross-compiler if: ${{ matrix.target == 'x86_64-pc-windows-gnu' }} run: sudo apt-get install -y mingw-w64 llvm - - uses: PyO3/maturin-action@v1 + - name: Compile version-specific library + uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} manylinux: ${{ matrix.manylinux }} args: --release -m examples/maturin-starter/Cargo.toml ${{ matrix.flags }} + - name: Compile abi3 library + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux }} + args: --release -m examples/maturin-starter/Cargo.toml --features abi3 ${{ matrix.flags }} test-cross-compilation-windows: needs: [fmt] @@ -567,10 +726,11 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + with: + python-version: "3.12" - uses: Swatinem/rust-cache@v2 with: - workspaces: - examples/maturin-starter + workspaces: examples/maturin-starter save-if: ${{ github.event_name != 'merge_group' }} - uses: actions/cache/restore@v4 with: @@ -587,11 +747,11 @@ jobs: pip install cargo-xwin # abi3 cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu - cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc + cargo xwin build --cross-compiler clang --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc # non-abi3 export PYO3_CROSS_PYTHON_VERSION=3.12 cargo build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-gnu - cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-msvc + cargo xwin build --cross-compiler clang --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-msvc - if: ${{ github.ref == 'refs/heads/main' }} uses: actions/cache/save@v4 with: @@ -610,6 +770,7 @@ jobs: - coverage - emscripten - test-debug + - test-free-threaded - test-version-limits - check-feature-powerset - test-cross-compilation diff --git a/.github/workflows/coverage-pr-base.yml b/.github/workflows/coverage-pr-base.yml new file mode 100644 index 00000000000..33118697636 --- /dev/null +++ b/.github/workflows/coverage-pr-base.yml @@ -0,0 +1,42 @@ +# This runs as a separate job because it needs to run on the `pull_request_target` event +# in order to access the CODECOV_TOKEN secret. +# +# This is safe because this doesn't run arbitrary code from PRs. + +name: Set Codecov PR base +on: + # See safety note / doc at the top of this file. + pull_request_target: + +jobs: + coverage-pr-base: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Set PR base on codecov + run: | + # fetch the merge commit between the PR base and head + git fetch -u --progress --depth=1 origin "+$BASE_REF:$BASE_REF" "+$MERGE_REF:$MERGE_REF" + while [ -z "$(git merge-base "$BASE_REF" "$MERGE_REF")" ]; do + git fetch -u -q --deepen="10" origin "$BASE_REF" "$MERGE_REF"; + done + + MERGE_BASE=$(git merge-base "$BASE_REF" "$MERGE_REF") + echo "Merge base: $MERGE_BASE" + + # inform codecov about the merge base + pip install codecov-cli + codecovcli pr-base-picking \ + --base-sha $MERGE_BASE \ + --pr ${{ github.event.number }} \ + --slug PyO3/pyo3 \ + --token ${{ secrets.CODECOV_TOKEN }} \ + --service github + env: + # Don't put these in bash, because we don't want the expansion to + # risk code execution + BASE_REF: "refs/heads/${{ github.event.pull_request.base.ref }}" + MERGE_REF: "refs/pull/${{ github.event.pull_request.number }}/merge" diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index a9a7669054a..1dc5927b49e 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -22,13 +22,15 @@ jobs: tag_name: ${{ steps.prepare_tag.outputs.tag_name }} steps: - uses: actions/checkout@v4 - + - uses: actions/setup-python@v5 + with: + python-version: '3.12' - uses: dtolnay/rust-toolchain@nightly - name: Setup mdBook uses: taiki-e/install-action@v2 with: - tool: mdbook,lychee + tool: mdbook,mdbook-tabs,lychee - name: Prepare tag id: prepare_tag @@ -40,100 +42,15 @@ jobs: - name: Build the guide run: | python -m pip install --upgrade pip && pip install nox - nox -s check-guide + nox -s ${{ github.event_name == 'release' && 'build-guide' || 'check-guide' }} env: PYO3_VERSION_TAG: ${{ steps.prepare_tag.outputs.tag_name }} - name: Deploy docs and the guide if: ${{ github.event_name == 'release' }} - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./target/guide/ destination_dir: ${{ steps.prepare_tag.outputs.tag_name }} full_commit_message: "Upload documentation for ${{ steps.prepare_tag.outputs.tag_name }}" - - cargo-benchmark: - if: ${{ github.ref_name == 'main' }} - name: Cargo benchmark - needs: guide-build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - - uses: dtolnay/rust-toolchain@stable - - - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: cargo-${{ runner.os }}-bench-${{ hashFiles('**/Cargo.toml') }} - continue-on-error: true - - - name: Run benchmarks - run: | - python -m pip install --upgrade pip && pip install nox - for bench in pyo3-benches/benches/*.rs; do - bench_name=$(basename "$bench" .rs) - nox -s bench -- --bench "$bench_name" -- --output-format bencher | tee -a output.txt - done - - # Download previous benchmark result from cache (if exists) - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-benchmark - - # Run `github-action-benchmark` action - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: pyo3-bench - # What benchmark tool the output.txt came from - tool: "cargo" - # Where the output from the benchmark tool is stored - output-file-path: output.txt - # GitHub API token to make a commit comment - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: ${{ github.event_name != 'pull_request' }} - - pytest-benchmark: - if: ${{ github.ref_name == 'main' }} - name: Pytest benchmark - needs: cargo-benchmark - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - - uses: dtolnay/rust-toolchain@stable - - - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: cargo-${{ runner.os }}-pytest-bench-${{ hashFiles('**/Cargo.toml') }} - continue-on-error: true - - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-pytest-benchmark - - - name: Run benchmarks - run: | - python -m pip install --upgrade pip && pip install nox - nox -f pytests/noxfile.py -s bench -- --benchmark-json $(pwd)/output.json - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: pytest-bench - tool: "pytest" - output-file-path: output.json - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: ${{ github.event_name != 'pull_request' }} diff --git a/.netlify/build.sh b/.netlify/build.sh index e1d86788ca1..97b7bd72414 100755 --- a/.netlify/build.sh +++ b/.netlify/build.sh @@ -2,6 +2,7 @@ set -uex +rustup update nightly rustup default nightly PYO3_VERSION=$(cargo search pyo3 --limit 1 | head -1 | tr -s ' ' | cut -d ' ' -f 3 | tr -d '"') @@ -88,6 +89,14 @@ if [ "${INSTALLED_MDBOOK_LINKCHECK_VERSION}" != "mdbook v${MDBOOK_LINKCHECK_VERS cargo install mdbook-linkcheck@${MDBOOK_LINKCHECK_VERSION} --force fi +# Install latest mdbook-tabs. Netlify will cache the cargo bin dir, so this will +# only build mdbook-tabs if needed. +MDBOOK_TABS_VERSION=$(cargo search mdbook-tabs --limit 1 | head -1 | tr -s ' ' | cut -d ' ' -f 3 | tr -d '"') +INSTALLED_MDBOOK_TABS_VERSION=$(mdbook-tabs --version || echo "none") +if [ "${INSTALLED_MDBOOK_TABS_VERSION}" != "mdbook-tabs v${MDBOOK_TABS_VERSION}" ]; then + cargo install mdbook-tabs@${MDBOOK_TABS_VERSION} --force +fi + pip install nox nox -s build-guide mv target/guide/ netlify_build/main/ diff --git a/.python-version b/.python-version deleted file mode 100644 index e4fba218358..00000000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.12 diff --git a/Architecture.md b/Architecture.md index a4218a7f71f..be9d2b53d82 100644 --- a/Architecture.md +++ b/Architecture.md @@ -37,12 +37,9 @@ automated tooling because: - it gives us best control about how to adapt C conventions to Rust, and - there are many Python interpreter versions we support in a single set of files. -We aim to provide straight-forward Rust wrappers resembling the file structure of -[`cpython/Include`](https://github.com/python/cpython/tree/v3.9.2/Include). +We aim to provide straight-forward Rust wrappers resembling the file structure of [`cpython/Include`](https://github.com/python/cpython/tree/3.13/Include). -However, we still lack some APIs and are continuously updating the module to match -the file contents upstream in CPython. -The tracking issue is [#1289](https://github.com/PyO3/pyo3/issues/1289), and contribution is welcome. +We are continuously updating the module to match the latest CPython version which PyO3 supports (i.e. as of time of writing Python 3.13). The tracking issue is [#1289](https://github.com/PyO3/pyo3/issues/1289), and contribution is welcome. In the [`pyo3-ffi`] crate, there is lots of conditional compilation such as `#[cfg(Py_LIMITED_API)]`, `#[cfg(Py_3_7)]`, and `#[cfg(PyPy)]`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f4ce218021..57196517016 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,406 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.24.1] - 2025-03-31 + +### Added + +- Add `abi3-py313` feature. [#4969](https://github.com/PyO3/pyo3/pull/4969) +- Add `PyAnyMethods::getattr_opt`. [#4978](https://github.com/PyO3/pyo3/pull/4978) +- Add `PyInt::new` constructor for all supported number types (i32, u32, i64, u64, isize, usize). [#4984](https://github.com/PyO3/pyo3/pull/4984) +- Add `pyo3::sync::with_critical_section2`. [#4992](https://github.com/PyO3/pyo3/pull/4992) +- Implement `PyCallArgs` for `Borrowed<'_, 'py, PyTuple>`, `&Bound<'py, PyTuple>`, and `&Py`. [#5013](https://github.com/PyO3/pyo3/pull/5013) + +### Fixed + +- Fix `is_type_of` for native types not using same specialized check as `is_type_of_bound`. [#4981](https://github.com/PyO3/pyo3/pull/4981) +- Fix `Probe` class naming issue with `#[pymethods]`. [#4988](https://github.com/PyO3/pyo3/pull/4988) +- Fix compile failure with required `#[pyfunction]` arguments taking `Option<&str>` and `Option<&T>` (for `#[pyclass]` types). [#5002](https://github.com/PyO3/pyo3/pull/5002) +- Fix `PyString::from_object` causing of bounds reads whith `encoding` and `errors` parameters which are not nul-terminated. [#5008](https://github.com/PyO3/pyo3/pull/5008) +- Fix compile error when additional options follow after `crate` for `#[pyfunction]`. [#5015](https://github.com/PyO3/pyo3/pull/5015) + +## [0.24.0] - 2025-03-09 + +### Packaging + +- Add supported CPython/PyPy versions to cargo package metadata. [#4756](https://github.com/PyO3/pyo3/pull/4756) +- Bump `target-lexicon` dependency to 0.13. [#4822](https://github.com/PyO3/pyo3/pull/4822) +- Add optional `jiff` dependency to add conversions for `jiff` datetime types. [#4823](https://github.com/PyO3/pyo3/pull/4823) +- Add optional `uuid` dependency to add conversions for `uuid::Uuid`. [#4864](https://github.com/PyO3/pyo3/pull/4864) +- Bump minimum supported `inventory` version to 0.3.5. [#4954](https://github.com/PyO3/pyo3/pull/4954) + +### Added + +- Add `PyIterator::send` method to allow sending values into a python generator. [#4746](https://github.com/PyO3/pyo3/pull/4746) +- Add `PyCallArgs` trait for passing arguments into the Python calling protocol. This enabled using a faster calling convention for certain types, improving performance. [#4768](https://github.com/PyO3/pyo3/pull/4768) +- Add `#[pyo3(default = ...']` option for `#[derive(FromPyObject)]` to set a default value for extracted fields of named structs. [#4829](https://github.com/PyO3/pyo3/pull/4829) +- Add `#[pyo3(into_py_with = ...)]` option for `#[derive(IntoPyObject, IntoPyObjectRef)]`. [#4850](https://github.com/PyO3/pyo3/pull/4850) +- Add FFI definitions `PyThreadState_GetFrame` and `PyFrame_GetBack`. [#4866](https://github.com/PyO3/pyo3/pull/4866) +- Optimize `last` for `BoundListIterator`, `BoundTupleIterator` and `BorrowedTupleIterator`. [#4878](https://github.com/PyO3/pyo3/pull/4878) +- Optimize `Iterator::count()` for `PyDict`, `PyList`, `PyTuple` & `PySet`. [#4878](https://github.com/PyO3/pyo3/pull/4878) +- Optimize `nth`, `nth_back`, `advance_by` and `advance_back_by` for `BoundTupleIterator` [#4897](https://github.com/PyO3/pyo3/pull/4897) +- Add support for `types.GenericAlias` as `pyo3::types::PyGenericAlias`. [#4917](https://github.com/PyO3/pyo3/pull/4917) +- Add `MutextExt` trait to help avoid deadlocks with the GIL while locking a `std::sync::Mutex`. [#4934](https://github.com/PyO3/pyo3/pull/4934) +- Add `#[pyo3(rename_all = "...")]` option for `#[derive(FromPyObject)]`. [#4941](https://github.com/PyO3/pyo3/pull/4941) + +### Changed + +- Optimize `nth`, `nth_back`, `advance_by` and `advance_back_by` for `BoundListIterator`. [#4810](https://github.com/PyO3/pyo3/pull/4810) +- Use `DerefToPyAny` in blanket implementations of `From>` and `From>` for `PyObject`. [#4593](https://github.com/PyO3/pyo3/pull/4593) +- Map `io::ErrorKind::IsADirectory`/`NotADirectory` to the corresponding Python exception on Rust 1.83+. [#4747](https://github.com/PyO3/pyo3/pull/4747) +- `PyAnyMethods::call` and friends now require `PyCallArgs` for their positional arguments. [#4768](https://github.com/PyO3/pyo3/pull/4768) +- Expose FFI definitions for `PyObject_Vectorcall(Method)` on the stable abi on 3.12+. [#4853](https://github.com/PyO3/pyo3/pull/4853) +- `#[pyo3(from_py_with = ...)]` now take a path rather than a string literal [#4860](https://github.com/PyO3/pyo3/pull/4860) +- Format Python traceback in impl Debug for PyErr. [#4900](https://github.com/PyO3/pyo3/pull/4900) +- Convert `PathBuf` & `Path` into Python `pathlib.Path` instead of `PyString`. [#4925](https://github.com/PyO3/pyo3/pull/4925) +- Relax parsing of exotic Python versions. [#4949](https://github.com/PyO3/pyo3/pull/4949) +- PyO3 threads now hang instead of `pthread_exit` trying to acquire the GIL when the interpreter is shutting down. This mimics the [Python 3.14](https://github.com/python/cpython/issues/87135) behavior and avoids undefined behavior and crashes. [#4874](https://github.com/PyO3/pyo3/pull/4874) + +### Removed + +- Remove implementations of `Deref` for `PyAny` and other "native" types. [#4593](https://github.com/PyO3/pyo3/pull/4593) +- Remove implicit default of trailing optional arguments (see #2935) [#4729](https://github.com/PyO3/pyo3/pull/4729) +- Remove the deprecated implicit eq fallback for simple enums. [#4730](https://github.com/PyO3/pyo3/pull/4730) + +### Fixed + +- Correct FFI definition of `PyIter_Send` to return a `PySendResult`. [#4746](https://github.com/PyO3/pyo3/pull/4746) +- Fix a thread safety issue in the runtime borrow checker used by mutable pyclass instances on the free-threaded build. [#4948](https://github.com/PyO3/pyo3/pull/4948) + + +## [0.23.5] - 2025-02-22 + +### Packaging + +- Add support for PyPy3.11 [#4760](https://github.com/PyO3/pyo3/pull/4760) + +### Fixed + +- Fix thread-unsafe implementation of freelist pyclasses on the free-threaded build. [#4902](https://github.com/PyO3/pyo3/pull/4902) +- Re-enable a workaround for situations where CPython incorrectly does not add `__builtins__` to `__globals__` in code executed by `Python::py_run` (was removed in PyO3 0.23.0). [#4921](https://github.com/PyO3/pyo3/pull/4921) + +## [0.23.4] - 2025-01-10 + +### Added + +- Add `PyList::locked_for_each`, which uses a critical section to lock the list on the free-threaded build. [#4789](https://github.com/PyO3/pyo3/pull/4789) +- Add `pyo3_build_config::add_python_framework_link_args` build script API to set rpath when using macOS system Python. [#4833](https://github.com/PyO3/pyo3/pull/4833) + +### Changed + +- Use `datetime.fold` to distinguish ambiguous datetimes when converting to and from `chrono::DateTime` (rather than erroring). [#4791](https://github.com/PyO3/pyo3/pull/4791) +- Optimize PyList iteration on the free-threaded build. [#4789](https://github.com/PyO3/pyo3/pull/4789) + +### Fixed + +- Fix unnecessary internal `py.allow_threads` GIL-switch when attempting to access contents of a `PyErr` which originated from Python (could lead to unintended deadlocks). [#4766](https://github.com/PyO3/pyo3/pull/4766) +- Fix thread-unsafe access of dict internals in `BoundDictIterator` on the free-threaded build. [#4788](https://github.com/PyO3/pyo3/pull/4788) +* Fix unnecessary critical sections in `BoundDictIterator` on the free-threaded build. [#4788](https://github.com/PyO3/pyo3/pull/4788) +- Fix time-of-check to time-of-use issues with list iteration on the free-threaded build. [#4789](https://github.com/PyO3/pyo3/pull/4789) +- Fix `chrono::DateTime` to-Python conversion when `Tz` is `chrono_tz::Tz`. [#4790](https://github.com/PyO3/pyo3/pull/4790) +- Fix `#[pyclass]` not being able to be named `Probe`. [#4794](https://github.com/PyO3/pyo3/pull/4794) +- Fix not treating cross-compilation from x64 to aarch64 on Windows as a cross-compile. [#4800](https://github.com/PyO3/pyo3/pull/4800) +- Fix missing struct fields on GraalPy when subclassing builtin classes. [#4802](https://github.com/PyO3/pyo3/pull/4802) +- Fix generating import lib for PyPy when `abi3` feature is enabled. [#4806](https://github.com/PyO3/pyo3/pull/4806) +- Fix generating import lib for python3.13t when `abi3` feature is enabled. [#4808](https://github.com/PyO3/pyo3/pull/4808) +- Fix compile failure for raw identifiers like `r#box` in `derive(FromPyObject)`. [#4814](https://github.com/PyO3/pyo3/pull/4814) +- Fix compile failure for `#[pyclass]` enum variants with more than 12 fields. [#4832](https://github.com/PyO3/pyo3/pull/4832) + + +## [0.23.3] - 2024-12-03 + +### Packaging + +- Bump optional `python3-dll-a` dependency to 0.2.11. [#4749](https://github.com/PyO3/pyo3/pull/4749) + +### Fixed + +- Fix unresolved symbol link failures on Windows when compiling for Python 3.13t with `abi3` features enabled. [#4733](https://github.com/PyO3/pyo3/pull/4733) +- Fix unresolved symbol link failures on Windows when compiling for Python 3.13t using the `generate-import-lib` feature. [#4749](https://github.com/PyO3/pyo3/pull/4749) +- Fix compile-time regression in PyO3 0.23.0 where changing `PYO3_CONFIG_FILE` would not reconfigure PyO3 for the new interpreter. [#4758](https://github.com/PyO3/pyo3/pull/4758) + +## [0.23.2] - 2024-11-25 + +### Added + +- Add `IntoPyObjectExt` trait. [#4708](https://github.com/PyO3/pyo3/pull/4708) + +### Fixed + +- Fix compile failures when building for free-threaded Python when the `abi3` or `abi3-pyxx` features are enabled. [#4719](https://github.com/PyO3/pyo3/pull/4719) +- Fix `ambiguous_associated_items` lint error in `#[pyclass]` and `#[derive(IntoPyObject)]` macros. [#4725](https://github.com/PyO3/pyo3/pull/4725) + + +## [0.23.1] - 2024-11-16 + +Re-release of 0.23.0 with fixes to docs.rs build. + +## [0.23.0] - 2024-11-15 + +### Packaging + +- Drop support for PyPy 3.7 and 3.8. [#4582](https://github.com/PyO3/pyo3/pull/4582) +- Extend range of supported versions of `hashbrown` optional dependency to include version 0.15. [#4604](https://github.com/PyO3/pyo3/pull/4604) +- Bump minimum version of `eyre` optional dependency to 0.6.8. [#4617](https://github.com/PyO3/pyo3/pull/4617) +- Bump minimum version of `hashbrown` optional dependency to 0.14.5. [#4617](https://github.com/PyO3/pyo3/pull/4617) +- Bump minimum version of `indexmap` optional dependency to 2.5.0. [#4617](https://github.com/PyO3/pyo3/pull/4617) +- Bump minimum version of `num-complex` optional dependency to 0.4.6. [#4617](https://github.com/PyO3/pyo3/pull/4617) +- Bump minimum version of `chrono-tz` optional dependency to 0.10. [#4617](https://github.com/PyO3/pyo3/pull/4617) +- Support free-threaded Python 3.13t. [#4588](https://github.com/PyO3/pyo3/pull/4588) + +### Added + +- Add `IntoPyObject` (fallible) conversion trait to convert from Rust to Python values. [#4060](https://github.com/PyO3/pyo3/pull/4060) +- Add `#[pyclass(str="")]` option to generate `__str__` based on a `Display` implementation or format string. [#4233](https://github.com/PyO3/pyo3/pull/4233) +- Implement `PartialEq` for `Bound<'py, PyInt>` with `u8`, `u16`, `u32`, `u64`, `u128`, `usize`, `i8`, `i16`, `i32`, `i64`, `i128` and `isize`. [#4317](https://github.com/PyO3/pyo3/pull/4317) +- Implement `PartialEq` and `PartialEq` for `Bound<'py, PyFloat>`. [#4348](https://github.com/PyO3/pyo3/pull/4348) +- Add `as_super` and `into_super` methods for `Bound`. [#4351](https://github.com/PyO3/pyo3/pull/4351) +- Add FFI definitions `PyCFunctionFast` and `PyCFunctionFastWithKeywords` [#4415](https://github.com/PyO3/pyo3/pull/4415) +- Add FFI definitions for `PyMutex` on Python 3.13 and newer. [#4421](https://github.com/PyO3/pyo3/pull/4421) +- Add `PyDict::locked_for_each` to iterate efficiently on freethreaded Python. [#4439](https://github.com/PyO3/pyo3/pull/4439) +- Add FFI definitions `PyObject_GetOptionalAttr`, `PyObject_GetOptionalAttrString`, `PyObject_HasAttrWithError`, `PyObject_HasAttrStringWithError`, `Py_CONSTANT_*` constants, `Py_GetConstant`, `Py_GetConstantBorrowed`, and `PyType_GetModuleByDef` on Python 3.13 and newer. [#4447](https://github.com/PyO3/pyo3/pull/4447) +- Add FFI definitions for the Python critical section API available on Python 3.13 and newer. [#4477](https://github.com/PyO3/pyo3/pull/4477) +- Add derive macro for `IntoPyObject`. [#4495](https://github.com/PyO3/pyo3/pull/4495) +- Add `Borrowed::as_ptr`. [#4520](https://github.com/PyO3/pyo3/pull/4520) +- Add FFI definition for `PyImport_AddModuleRef`. [#4529](https://github.com/PyO3/pyo3/pull/4529) +- Add `PyAnyMethods::try_iter`. [#4553](https://github.com/PyO3/pyo3/pull/4553) +- Add `pyo3::sync::with_critical_section`, a wrapper around the Python Critical Section API added in Python 3.13. [#4587](https://github.com/PyO3/pyo3/pull/4587) +- Add `#[pymodule(gil_used = false)]` option to declare that a module supports the free-threaded build. [#4588](https://github.com/PyO3/pyo3/pull/4588) +- Add `PyModule::gil_used` method to declare that a module supports the free-threaded build. [#4588](https://github.com/PyO3/pyo3/pull/4588) +- Add FFI definition `PyDateTime_CAPSULE_NAME`. [#4634](https://github.com/PyO3/pyo3/pull/4634) +- Add `PyMappingProxy` type to represent the `mappingproxy` Python class. [#4644](https://github.com/PyO3/pyo3/pull/4644) +- Add FFI definitions `PyList_Extend` and `PyList_Clear`. [#4667](https://github.com/PyO3/pyo3/pull/4667) +- Add derive macro for `IntoPyObjectRef`. [#4674](https://github.com/PyO3/pyo3/pull/4674) +- Add `pyo3::sync::OnceExt` and `pyo3::sync::OnceLockExt` traits. [#4676](https://github.com/PyO3/pyo3/pull/4676) + +### Changed + +- Prefer `IntoPyObject` over `IntoPy>>` for `#[pyfunction]` and `#[pymethods]` return types. [#4060](https://github.com/PyO3/pyo3/pull/4060) +- Report multiple errors from `#[pyclass]` and `#[pyo3(..)]` attributes. [#4243](https://github.com/PyO3/pyo3/pull/4243) +- Nested declarative `#[pymodule]` are automatically treated as submodules (no `PyInit_` entrypoint is created). [#4308](https://github.com/PyO3/pyo3/pull/4308) +- Deprecate `PyAnyMethods::is_ellipsis` (`Py::is_ellipsis` was deprecated in PyO3 0.20). [#4322](https://github.com/PyO3/pyo3/pull/4322) +- Deprecate `PyLong` in favor of `PyInt`. [#4347](https://github.com/PyO3/pyo3/pull/4347) +- Rename `IntoPyDict::into_py_dict_bound` to `IntoPyDict::into_py_dict`. [#4388](https://github.com/PyO3/pyo3/pull/4388) +- `PyModule::from_code` now expects `&CStr` as arguments instead of `&str`. [#4404](https://github.com/PyO3/pyo3/pull/4404) +- Use "fastcall" Python calling convention for `#[pyfunction]`s when compiling on abi3 for Python 3.10 and up. [#4415](https://github.com/PyO3/pyo3/pull/4415) +- Remove `Copy` and `Clone` from `PyObject` struct FFI definition. [#4434](https://github.com/PyO3/pyo3/pull/4434) +- `Python::eval` and `Python::run` now take a `&CStr` instead of `&str`. [#4435](https://github.com/PyO3/pyo3/pull/4435) +- Deprecate `IPowModulo`, `PyClassAttributeDef`, `PyGetterDef`, `PyMethodDef`, `PyMethodDefType`, and `PySetterDef` from PyO3's public API. [#4441](https://github.com/PyO3/pyo3/pull/4441) +- `IntoPyObject` impls for `Vec`, `&[u8]`, `[u8; N]`, `Cow<[u8]>` and `SmallVec<[u8; N]>` now convert into Python `bytes` rather than a `list` of integers. [#4442](https://github.com/PyO3/pyo3/pull/4442) +- Emit a compile-time error when attempting to subclass a class that doesn't allow subclassing. [#4453](https://github.com/PyO3/pyo3/pull/4453) +- `IntoPyDict::into_py_dict` is now fallible due to `IntoPyObject` migration. [#4493](https://github.com/PyO3/pyo3/pull/4493) +- The `abi3` feature will now override config files provided via `PYO3_BUILD_CONFIG`. [#4497](https://github.com/PyO3/pyo3/pull/4497) +- Disable the `GILProtected` struct on free-threaded Python. [#4504](https://github.com/PyO3/pyo3/pull/4504) +- Updated FFI definitions for functions and struct fields that have been deprecated or removed from CPython. [#4534](https://github.com/PyO3/pyo3/pull/4534) +- Disable `PyListMethods::get_item_unchecked` on free-threaded Python. [#4539](https://github.com/PyO3/pyo3/pull/4539) +- Add `GILOnceCell::import`. [#4542](https://github.com/PyO3/pyo3/pull/4542) +- Deprecate `PyAnyMethods::iter` in favour of `PyAnyMethods::try_iter`. [#4553](https://github.com/PyO3/pyo3/pull/4553) +- The `#[pyclass]` macro now requires a types to be `Sync`. (Except for `#[pyclass(unsendable)]` types). [#4566](https://github.com/PyO3/pyo3/pull/4566) +- `PyList::new` and `PyTuple::new` are now fallible due to `IntoPyObject` migration. [#4580](https://github.com/PyO3/pyo3/pull/4580) +- `PyErr::matches` is now fallible due to `IntoPyObject` migration. [#4595](https://github.com/PyO3/pyo3/pull/4595) +- Deprecate `ToPyObject` in favour of `IntoPyObject` [#4595](https://github.com/PyO3/pyo3/pull/4595) +- Deprecate `PyWeakrefMethods::get_option`. [#4597](https://github.com/PyO3/pyo3/pull/4597) +- Seal `PyWeakrefMethods` trait. [#4598](https://github.com/PyO3/pyo3/pull/4598) +- Remove `PyNativeTypeInitializer` and `PyObjectInit` from the PyO3 public API. [#4611](https://github.com/PyO3/pyo3/pull/4611) +- Deprecate `IntoPy` in favor of `IntoPyObject` [#4618](https://github.com/PyO3/pyo3/pull/4618) +- Eagerly normalize exceptions in `PyErr::take()` and `PyErr::fetch()` on Python 3.11 and older. [#4655](https://github.com/PyO3/pyo3/pull/4655) +- Move `IntoPy::type_output` to `IntoPyObject::type_output`. [#4657](https://github.com/PyO3/pyo3/pull/4657) +- Change return type of `PyMapping::keys`, `PyMapping::values` and `PyMapping::items` to `Bound<'py, PyList>` instead of `Bound<'py, PySequence>`. [#4661](https://github.com/PyO3/pyo3/pull/4661) +- Complex enums now allow field types that either implement `IntoPyObject` by reference or by value together with `Clone`. This makes `Py` available as field type. [#4694](https://github.com/PyO3/pyo3/pull/4694) + + +### Removed + +- Remove all functionality deprecated in PyO3 0.20. [#4322](https://github.com/PyO3/pyo3/pull/4322) +- Remove all functionality deprecated in PyO3 0.21. [#4323](https://github.com/PyO3/pyo3/pull/4323) +- Deprecate `PyUnicode` in favour of `PyString`. [#4370](https://github.com/PyO3/pyo3/pull/4370) +- Remove deprecated `gil-refs` feature. [#4378](https://github.com/PyO3/pyo3/pull/4378) +- Remove private FFI definitions `_Py_IMMORTAL_REFCNT`, `_Py_IsImmortal`, `_Py_TPFLAGS_STATIC_BUILTIN`, `_Py_Dealloc`, `_Py_IncRef`, `_Py_DecRef`. [#4447](https://github.com/PyO3/pyo3/pull/4447) +- Remove private FFI definitions `_Py_c_sum`, `_Py_c_diff`, `_Py_c_neg`, `_Py_c_prod`, `_Py_c_quot`, `_Py_c_pow`, `_Py_c_abs`. [#4521](https://github.com/PyO3/pyo3/pull/4521) +- Remove `_borrowed` methods of `PyWeakRef` and `PyWeakRefProxy`. [#4528](https://github.com/PyO3/pyo3/pull/4528) +- Removed private FFI definition `_PyErr_ChainExceptions`. [#4534](https://github.com/PyO3/pyo3/pull/4534) + +### Fixed + +- Fix invalid library search path `lib_dir` when cross-compiling. [#4389](https://github.com/PyO3/pyo3/pull/4389) +- Fix FFI definition `Py_Is` for PyPy on 3.10 to call the function defined by PyPy. [#4447](https://github.com/PyO3/pyo3/pull/4447) +- Fix compile failure when using `#[cfg]` attributes for simple enum variants. [#4509](https://github.com/PyO3/pyo3/pull/4509) +- Fix compiler warning for `non_snake_case` method names inside `#[pymethods]` generated code. [#4567](https://github.com/PyO3/pyo3/pull/4567) +- Fix compile error with `#[derive(FromPyObject)]` generic struct with trait bounds. [#4645](https://github.com/PyO3/pyo3/pull/4645) +- Fix compile error for `#[classmethod]` and `#[staticmethod]` on magic methods. [#4654](https://github.com/PyO3/pyo3/pull/4654) +- Fix compile warning for `unsafe_op_in_unsafe_fn` in generated macro code. [#4674](https://github.com/PyO3/pyo3/pull/4674) +- Fix incorrect deprecation warning for `#[pyclass] enum`s with custom `__eq__` implementation. [#4692](https://github.com/PyO3/pyo3/pull/4692) +- Fix `non_upper_case_globals` lint firing for generated `__match_args__` on complex enums. [#4705](https://github.com/PyO3/pyo3/pull/4705) + +## [0.22.5] - 2024-10-15 + +### Fixed + +- Fix regression in 0.22.4 of naming collision in `__clear__` slot and `clear` method generated code. [#4619](https://github.com/PyO3/pyo3/pull/4619) + + +## [0.22.4] - 2024-10-12 + +### Added + +- Add FFI definition `PyWeakref_GetRef` and `compat::PyWeakref_GetRef`. [#4528](https://github.com/PyO3/pyo3/pull/4528) + +### Changed + +- Deprecate `_borrowed` methods on `PyWeakRef` and `PyWeakrefProxy` (just use the owning forms). [#4590](https://github.com/PyO3/pyo3/pull/4590) + +### Fixed + +- Revert removal of private FFI function `_PyLong_NumBits` on Python 3.13 and later. [#4450](https://github.com/PyO3/pyo3/pull/4450) +- Fix `__traverse__` functions for base classes not being called by subclasses created with `#[pyclass(extends = ...)]`. [#4563](https://github.com/PyO3/pyo3/pull/4563) +- Fix regression in 0.22.3 failing compiles under `#![forbid(unsafe_code)]`. [#4574](https://github.com/PyO3/pyo3/pull/4574) +- Fix `create_exception` macro triggering lint and compile errors due to interaction with `gil-refs` feature. [#4589](https://github.com/PyO3/pyo3/pull/4589) +- Workaround possible use-after-free in `_borrowed` methods on `PyWeakRef` and `PyWeakrefProxy` by leaking their contents. [#4590](https://github.com/PyO3/pyo3/pull/4590) +- Fix crash calling `PyType_GetSlot` on static types before Python 3.10. [#4599](https://github.com/PyO3/pyo3/pull/4599) + + +## [0.22.3] - 2024-09-15 + +### Added + +- Add `pyo3::ffi::compat` namespace with compatibility shims for C API functions added in recent versions of Python. +- Add FFI definition `PyDict_GetItemRef` on Python 3.13 and newer, and `compat::PyDict_GetItemRef` for all versions. [#4355](https://github.com/PyO3/pyo3/pull/4355) +- Add FFI definition `PyList_GetItemRef` on Python 3.13 and newer, and `pyo3_ffi::compat::PyList_GetItemRef` for all versions. [#4410](https://github.com/PyO3/pyo3/pull/4410) +- Add FFI definitions `compat::Py_NewRef` and `compat::Py_XNewRef`. [#4445](https://github.com/PyO3/pyo3/pull/4445) +- Add FFI definitions `compat::PyObject_CallNoArgs` and `compat::PyObject_CallMethodNoArgs`. [#4461](https://github.com/PyO3/pyo3/pull/4461) +- Add `GilOnceCell>::clone_ref`. [#4511](https://github.com/PyO3/pyo3/pull/4511) + +### Changed + +- Improve error messages for `#[pyfunction]` defined inside `#[pymethods]`. [#4349](https://github.com/PyO3/pyo3/pull/4349) +- Improve performance of calls to Python by using the vectorcall calling convention where possible. [#4456](https://github.com/PyO3/pyo3/pull/4456) +- Mention the type name in the exception message when trying to instantiate a class with no constructor defined. [#4481](https://github.com/PyO3/pyo3/pull/4481) + +### Removed + +- Remove private FFI definition `_Py_PackageContext`. [#4420](https://github.com/PyO3/pyo3/pull/4420) + +### Fixed + +- Fix compile failure in declarative `#[pymodule]` under presence of `#![no_implicit_prelude]`. [#4328](https://github.com/PyO3/pyo3/pull/4328) +- Fix use of borrowed reference in `PyDict::get_item` (unsafe in free-threaded Python). [#4355](https://github.com/PyO3/pyo3/pull/4355) +- Fix `#[pyclass(eq)]` macro hygiene issues for structs and enums. [#4359](https://github.com/PyO3/pyo3/pull/4359) +- Fix hygiene/span issues of `#[pyfunction]` and `#[pymethods]` generated code which affected expansion in `macro_rules` context. [#4382](https://github.com/PyO3/pyo3/pull/4382) +- Fix `unsafe_code` lint error in `#[pyclass]` generated code. [#4396](https://github.com/PyO3/pyo3/pull/4396) +- Fix async functions returning a tuple only returning the first element to Python. [#4407](https://github.com/PyO3/pyo3/pull/4407) +- Fix use of borrowed reference in `PyList::get_item` (unsafe in free-threaded Python). [#4410](https://github.com/PyO3/pyo3/pull/4410) +- Correct FFI definition `PyArg_ParseTupleAndKeywords` to take `*const *const c_char` instead of `*mut *mut c_char` on Python 3.13 and up. [#4420](https://github.com/PyO3/pyo3/pull/4420) +- Fix a soundness bug with `PyClassInitializer`: panic if adding subclass to existing instance via `PyClassInitializer::from(Py).add_subclass(SubClass)`. [#4454](https://github.com/PyO3/pyo3/pull/4454) +- Fix illegal reference counting op inside implementation of `__traverse__` handlers. [#4479](https://github.com/PyO3/pyo3/pull/4479) + +## [0.22.2] - 2024-07-17 + +### Packaging + +- Require opt-in to freethreaded Python using the `UNSAFE_PYO3_BUILD_FREE_THREADED=1` environment variable (it is not yet supported by PyO3). [#4327](https://github.com/PyO3/pyo3/pull/4327) + +### Changed + +- Use FFI function calls for reference counting on all abi3 versions. [#4324](https://github.com/PyO3/pyo3/pull/4324) +- `#[pymodule(...)]` now directly accepts all relevant `#[pyo3(...)]` options. [#4330](https://github.com/PyO3/pyo3/pull/4330) + +### Fixed + +- Fix compile failure in declarative `#[pymodule]` under presence of `#![no_implicit_prelude]`. [#4328](https://github.com/PyO3/pyo3/pull/4328) +- Fix compile failure due to c-string literals on Rust < 1.79. [#4353](https://github.com/PyO3/pyo3/pull/4353) + +## [0.22.1] - 2024-07-06 + +### Added + +- Add `#[pyo3(submodule)]` option for declarative `#[pymodule]`s. [#4301](https://github.com/PyO3/pyo3/pull/4301) +- Implement `PartialEq` for `Bound<'py, PyBool>`. [#4305](https://github.com/PyO3/pyo3/pull/4305) + +### Fixed + +- Return `NotImplemented` instead of raising `TypeError` from generated equality method when comparing different types. [#4287](https://github.com/PyO3/pyo3/pull/4287) +- Handle full-path `#[pyo3::prelude::pymodule]` and similar for `#[pyclass]` and `#[pyfunction]` in declarative modules. [#4288](https://github.com/PyO3/pyo3/pull/4288) +- Fix 128-bit int regression on big-endian platforms with Python <3.13. [#4291](https://github.com/PyO3/pyo3/pull/4291) +- Stop generating code that will never be covered with declarative modules. [#4297](https://github.com/PyO3/pyo3/pull/4297) +- Fix invalid deprecation warning for trailing optional on `#[setter]` function. [#4304](https://github.com/PyO3/pyo3/pull/4304) + +## [0.22.0] - 2024-06-24 + +### Packaging + +- Update `heck` dependency to 0.5. [#3966](https://github.com/PyO3/pyo3/pull/3966) +- Extend range of supported versions of `chrono-tz` optional dependency to include version 0.10. [#4061](https://github.com/PyO3/pyo3/pull/4061) +- Update MSRV to 1.63. [#4129](https://github.com/PyO3/pyo3/pull/4129) +- Add optional `num-rational` feature to add conversions with Python's `fractions.Fraction`. [#4148](https://github.com/PyO3/pyo3/pull/4148) +- Support Python 3.13. [#4184](https://github.com/PyO3/pyo3/pull/4184) + +### Added + +- Add `PyWeakref`, `PyWeakrefReference` and `PyWeakrefProxy`. [#3835](https://github.com/PyO3/pyo3/pull/3835) +- Support `#[pyclass]` on enums that have tuple variants. [#4072](https://github.com/PyO3/pyo3/pull/4072) +- Add support for scientific notation in `Decimal` conversion. [#4079](https://github.com/PyO3/pyo3/pull/4079) +- Add `pyo3_disable_reference_pool` conditional compilation flag to avoid the overhead of the global reference pool at the cost of known limitations as explained in the performance section of the guide. [#4095](https://github.com/PyO3/pyo3/pull/4095) +- Add `#[pyo3(constructor = (...))]` to customize the generated constructors for complex enum variants. [#4158](https://github.com/PyO3/pyo3/pull/4158) +- Add `PyType::module`, which always matches Python `__module__`. [#4196](https://github.com/PyO3/pyo3/pull/4196) +- Add `PyType::fully_qualified_name` which matches the "fully qualified name" defined in [PEP 737](https://peps.python.org/pep-0737). [#4196](https://github.com/PyO3/pyo3/pull/4196) +- Add `PyTypeMethods::mro` and `PyTypeMethods::bases`. [#4197](https://github.com/PyO3/pyo3/pull/4197) +- Add `#[pyclass(ord)]` to implement ordering based on `PartialOrd`. [#4202](https://github.com/PyO3/pyo3/pull/4202) +- Implement `ToPyObject` and `IntoPy` for `PyBackedStr` and `PyBackedBytes`. [#4205](https://github.com/PyO3/pyo3/pull/4205) +- Add `#[pyclass(hash)]` option to implement `__hash__` in terms of the `Hash` implementation [#4206](https://github.com/PyO3/pyo3/pull/4206) +- Add `#[pyclass(eq)]` option to generate `__eq__` based on `PartialEq`, and `#[pyclass(eq_int)]` for simple enums to implement equality based on their discriminants. [#4210](https://github.com/PyO3/pyo3/pull/4210) +- Implement `From>` for `PyClassInitializer`. [#4214](https://github.com/PyO3/pyo3/pull/4214) +- Add `as_super` methods to `PyRef` and `PyRefMut` for accesing the base class by reference. [#4219](https://github.com/PyO3/pyo3/pull/4219) +- Implement `PartialEq` for `Bound<'py, PyString>`. [#4245](https://github.com/PyO3/pyo3/pull/4245) +- Implement `PyModuleMethods::filename` on PyPy. [#4249](https://github.com/PyO3/pyo3/pull/4249) +- Implement `PartialEq<[u8]>` for `Bound<'py, PyBytes>`. [#4250](https://github.com/PyO3/pyo3/pull/4250) +- Add `pyo3_ffi::c_str` macro to create `&'static CStr` on Rust versions which don't have 1.77's `c""` literals. [#4255](https://github.com/PyO3/pyo3/pull/4255) +- Support `bool` conversion with `numpy` 2.0's `numpy.bool` type [#4258](https://github.com/PyO3/pyo3/pull/4258) +- Add `PyAnyMethods::{bitnot, matmul, floor_div, rem, divmod}`. [#4264](https://github.com/PyO3/pyo3/pull/4264) + +### Changed + +- Change the type of `PySliceIndices::slicelength` and the `length` parameter of `PySlice::indices()`. [#3761](https://github.com/PyO3/pyo3/pull/3761) +- Deprecate implicit default for trailing optional arguments [#4078](https://github.com/PyO3/pyo3/pull/4078) +- `Clone`ing pointers into the Python heap has been moved behind the `py-clone` feature, as it must panic without the GIL being held as a soundness fix. [#4095](https://github.com/PyO3/pyo3/pull/4095) +- Add `#[track_caller]` to all `Py`, `Bound<'py, T>` and `Borrowed<'a, 'py, T>` methods which can panic. [#4098](https://github.com/PyO3/pyo3/pull/4098) +- Change `PyAnyMethods::dir` to be fallible and return `PyResult>` (and similar for `PyAny::dir`). [#4100](https://github.com/PyO3/pyo3/pull/4100) +- The global reference pool (to track pending reference count decrements) is now initialized lazily to avoid the overhead of taking a mutex upon function entry when the functionality is not actually used. [#4178](https://github.com/PyO3/pyo3/pull/4178) +- Emit error messages when using `weakref` or `dict` when compiling for `abi3` for Python older than 3.9. [#4194](https://github.com/PyO3/pyo3/pull/4194) +- Change `PyType::name` to always match Python `__name__`. [#4196](https://github.com/PyO3/pyo3/pull/4196) +- Remove CPython internal ffi call for complex number including: add, sub, mul, div, neg, abs, pow. Added PyAnyMethods::{abs, pos, neg} [#4201](https://github.com/PyO3/pyo3/pull/4201) +- Deprecate implicit integer comparision for simple enums in favor of `#[pyclass(eq_int)]`. [#4210](https://github.com/PyO3/pyo3/pull/4210) +- Set the `module=` attribute of declarative modules' child `#[pymodule]`s and `#[pyclass]`es. [#4213](https://github.com/PyO3/pyo3/pull/4213) +- Set the `module` option for complex enum variants from the value set on the complex enum `module`. [#4228](https://github.com/PyO3/pyo3/pull/4228) +- Respect the Python "limited API" when building for the `abi3` feature on PyPy or GraalPy. [#4237](https://github.com/PyO3/pyo3/pull/4237) +- Optimize code generated by `#[pyo3(get)]` on `#[pyclass]` fields. [#4254](https://github.com/PyO3/pyo3/pull/4254) +- `PyCFunction::new`, `PyCFunction::new_with_keywords` and `PyCFunction::new_closure` now take `&'static CStr` name and doc arguments (previously was `&'static str`). [#4255](https://github.com/PyO3/pyo3/pull/4255) +- The `experimental-declarative-modules` feature is now stabilized and available by default. [#4257](https://github.com/PyO3/pyo3/pull/4257) + +### Fixed + +- Fix panic when `PYO3_CROSS_LIB_DIR` is set to a missing path. [#4043](https://github.com/PyO3/pyo3/pull/4043) +- Fix a compile error when exporting an exception created with `create_exception!` living in a different Rust module using the `declarative-module` feature. [#4086](https://github.com/PyO3/pyo3/pull/4086) +- Fix FFI definitions of `PY_VECTORCALL_ARGUMENTS_OFFSET` and `PyVectorcall_NARGS` to fix a false-positive assertion. [#4104](https://github.com/PyO3/pyo3/pull/4104) +- Disable `PyUnicode_DATA` on PyPy: not exposed by PyPy. [#4116](https://github.com/PyO3/pyo3/pull/4116) +- Correctly handle `#[pyo3(from_py_with = ...)]` attribute on dunder (`__magic__`) method arguments instead of silently ignoring it. [#4117](https://github.com/PyO3/pyo3/pull/4117) +- Fix a compile error when declaring a standalone function or class method with a Python name that is a Rust keyword. [#4226](https://github.com/PyO3/pyo3/pull/4226) +- Fix declarative modules discarding doc comments on the `mod` node. [#4236](https://github.com/PyO3/pyo3/pull/4236) +- Fix `__dict__` attribute missing for `#[pyclass(dict)]` instances when building for `abi3` on Python 3.9. [#4251](https://github.com/PyO3/pyo3/pull/4251) + +## [0.21.2] - 2024-04-16 + +### Changed + +- Deprecate the `PySet::empty()` gil-ref constructor. [#4082](https://github.com/PyO3/pyo3/pull/4082) + +### Fixed + +- Fix compile error for `async fn` in `#[pymethods]` with a `&self` receiver and more than one additional argument. [#4035](https://github.com/PyO3/pyo3/pull/4035) +- Improve error message for wrong receiver type in `__traverse__`. [#4045](https://github.com/PyO3/pyo3/pull/4045) +- Fix compile error when exporting a `#[pyclass]` living in a different Rust module using the `experimental-declarative-modules` feature. [#4054](https://github.com/PyO3/pyo3/pull/4054) +- Fix `missing_docs` lint triggering on documented `#[pymodule]` functions. [#4067](https://github.com/PyO3/pyo3/pull/4067) +- Fix undefined symbol errors for extension modules on AIX (by linking `libpython`). [#4073](https://github.com/PyO3/pyo3/pull/4073) + ## [0.21.1] - 2024-04-01 ### Added @@ -1731,7 +2131,22 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.21.1...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.24.1...HEAD +[0.24.1]: https://github.com/pyo3/pyo3/compare/v0.24.0...v0.24.1 +[0.24.0]: https://github.com/pyo3/pyo3/compare/v0.23.5...v0.24.0 +[0.23.5]: https://github.com/pyo3/pyo3/compare/v0.23.4...v0.23.5 +[0.23.4]: https://github.com/pyo3/pyo3/compare/v0.23.3...v0.23.4 +[0.23.3]: https://github.com/pyo3/pyo3/compare/v0.23.2...v0.23.3 +[0.23.2]: https://github.com/pyo3/pyo3/compare/v0.23.1...v0.23.2 +[0.23.1]: https://github.com/pyo3/pyo3/compare/v0.23.0...v0.23.1 +[0.23.0]: https://github.com/pyo3/pyo3/compare/v0.22.5...v0.23.0 +[0.22.5]: https://github.com/pyo3/pyo3/compare/v0.22.4...v0.22.5 +[0.22.4]: https://github.com/pyo3/pyo3/compare/v0.22.3...v0.22.4 +[0.22.3]: https://github.com/pyo3/pyo3/compare/v0.22.2...v0.22.3 +[0.22.2]: https://github.com/pyo3/pyo3/compare/v0.22.1...v0.22.2 +[0.22.1]: https://github.com/pyo3/pyo3/compare/v0.22.0...v0.22.1 +[0.22.0]: https://github.com/pyo3/pyo3/compare/v0.21.2...v0.22.0 +[0.21.2]: https://github.com/pyo3/pyo3/compare/v0.21.1...v0.21.2 [0.21.1]: https://github.com/pyo3/pyo3/compare/v0.21.0...v0.21.1 [0.21.0]: https://github.com/pyo3/pyo3/compare/v0.20.3...v0.21.0 [0.21.0-beta.0]: https://github.com/pyo3/pyo3/compare/v0.20.3...v0.21.0-beta.0 diff --git a/Cargo.toml b/Cargo.toml index fdd1fa9d29a..375b0dea024 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.21.1" +version = "0.24.1" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -12,44 +12,49 @@ categories = ["api-bindings", "development-tools::ffi"] license = "MIT OR Apache-2.0" exclude = ["/.gitignore", ".cargo/config", "/codecov.yml", "/Makefile", "/pyproject.toml", "/noxfile.py", "/.github", "/tests/test_compile_error.rs", "/tests/ui"] edition = "2021" -rust-version = "1.56" +rust-version = "1.63" [dependencies] cfg-if = "1.0" libc = "0.2.62" -parking_lot = ">= 0.11, < 0.13" memoffset = "0.9" -portable-atomic = "1.0" +once_cell = "1.13" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.21.1" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.24.1" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.21.1", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.24.1", optional = true } indoc = { version = "2.0.1", optional = true } unindent = { version = "0.2.1", optional = true } # support crate for multiple-pymethods feature -inventory = { version = "0.3.0", optional = true } +inventory = { version = "0.3.5", optional = true } # crate integrations that can be added using the eponymous features -anyhow = { version = "1.0", optional = true } +anyhow = { version = "1.0.1", optional = true } chrono = { version = "0.4.25", default-features = false, optional = true } -chrono-tz = { version = ">= 0.6, < 0.9", default-features = false, optional = true } +chrono-tz = { version = ">= 0.10, < 0.11", default-features = false, optional = true } either = { version = "1.9", optional = true } -eyre = { version = ">= 0.4, < 0.7", optional = true } -hashbrown = { version = ">= 0.9, < 0.15", optional = true } -indexmap = { version = ">= 1.6, < 3", optional = true } -num-bigint = { version = "0.4", optional = true } -num-complex = { version = ">= 0.2, < 0.5", optional = true } -rust_decimal = { version = "1.0.0", default-features = false, optional = true } +eyre = { version = ">= 0.6.8, < 0.7", optional = true } +hashbrown = { version = ">= 0.14.5, < 0.16", optional = true } +indexmap = { version = ">= 2.5.0, < 3", optional = true } +jiff-02 = { package = "jiff", version = "0.2", optional = true } +num-bigint = { version = "0.4.2", optional = true } +num-complex = { version = ">= 0.4.6, < 0.5", optional = true } +num-rational = { version = "0.4.1", optional = true } +rust_decimal = { version = "1.15", default-features = false, optional = true } serde = { version = "1.0", optional = true } smallvec = { version = "1.0", optional = true } +uuid = { version = "1.11.0", optional = true } + +[target.'cfg(not(target_has_atomic = "64"))'.dependencies] +portable-atomic = "1.0" [dev-dependencies] assert_approx_eq = "1.1.0" chrono = "0.4.25" -chrono-tz = ">= 0.6, < 0.9" +chrono-tz = ">= 0.10, < 0.11" # Required for "and $N others" normalization trybuild = ">=1.0.70" proptest = { version = "1.0", default-features = false, features = ["std"] } @@ -58,9 +63,12 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.61" rayon = "1.6.1" futures = "0.3.28" +tempfile = "3.12.0" +static_assertions = "1.1.0" +uuid = { version = "1.10.0", features = ["v4"] } [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.1", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "=0.24.1", features = ["resolve-config"] } [features] default = ["macros"] @@ -72,9 +80,6 @@ experimental-async = ["macros", "pyo3-macros/experimental-async"] # and IntoPy traits experimental-inspect = [] -# Enables annotating Rust inline modules with #[pymodule] to build Python modules declaratively -experimental-declarative-modules = ["pyo3-macros/experimental-declarative-modules", "macros"] - # Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc. macros = ["pyo3-macros", "indoc", "unindent"] @@ -95,7 +100,8 @@ abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38", "pyo3-ffi/abi3-py38"] abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39", "pyo3-ffi/abi3-py39"] abi3-py310 = ["abi3-py311", "pyo3-build-config/abi3-py310", "pyo3-ffi/abi3-py310"] abi3-py311 = ["abi3-py312", "pyo3-build-config/abi3-py311", "pyo3-ffi/abi3-py311"] -abi3-py312 = ["abi3", "pyo3-build-config/abi3-py312", "pyo3-ffi/abi3-py312"] +abi3-py312 = ["abi3-py313", "pyo3-build-config/abi3-py312", "pyo3-ffi/abi3-py312"] +abi3-py313 = ["abi3", "pyo3-build-config/abi3-py313", "pyo3-ffi/abi3-py313"] # Automatically generates `python3.dll` import libraries for Windows targets. generate-import-lib = ["pyo3-ffi/generate-import-lib"] @@ -103,8 +109,8 @@ generate-import-lib = ["pyo3-ffi/generate-import-lib"] # Changes `Python::with_gil` to automatically initialize the Python interpreter if needed. auto-initialize = [] -# Allows use of the deprecated "GIL Refs" APIs. -gil-refs = [] +# Enables `Clone`ing references to Python objects `Py` which panics if the GIL is not held. +py-clone = [] # Optimizes PyObject to Vec conversion and so on. nightly = [] @@ -113,22 +119,24 @@ nightly = [] # This is mostly intended for testing purposes - activating *all* of these isn't particularly useful. full = [ "macros", - # "multiple-pymethods", # TODO re-add this when MSRV is greater than 1.62 + # "multiple-pymethods", # Not supported by wasm "anyhow", "chrono", "chrono-tz", "either", "experimental-async", - "experimental-declarative-modules", "experimental-inspect", "eyre", "hashbrown", "indexmap", "num-bigint", "num-complex", + "num-rational", + "py-clone", "rust_decimal", "serde", "smallvec", + "uuid", ] [workspace] @@ -143,7 +151,7 @@ members = [ [package.metadata.docs.rs] no-default-features = true -features = ["full", "gil-refs"] +features = ["full"] rustdoc-args = ["--cfg", "docsrs"] [workspace.lints.clippy] @@ -164,9 +172,10 @@ used_underscore_binding = "warn" [workspace.lints.rust] elided_lifetimes_in_paths = "warn" invalid_doc_attributes = "warn" -rust_2018_idioms = "warn" +rust_2018_idioms = { level = "warn", priority = -1 } rust_2021_prelude_collisions = "warn" unused_lifetimes = "warn" +unsafe_op_in_unsafe_fn = "warn" [workspace.lints.rustdoc] broken_intra_doc_links = "warn" diff --git a/Contributing.md b/Contributing.md index 29d1bb758a7..054ef42fb88 100644 --- a/Contributing.md +++ b/Contributing.md @@ -23,10 +23,6 @@ To work and develop PyO3, you need Python & Rust installed on your system. * [virtualenv](https://virtualenv.pypa.io/en/latest/) can also be used with or without Pyenv to use specific installed Python versions. * [`nox`][nox] is used to automate many of our CI tasks. -### Caveats - -* When using pyenv on macOS, installing a Python version using `--enable-shared` is required to make it work. i.e `env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.7.12` - ### Help users identify bugs The [PyO3 Discord server](https://discord.gg/33kcChzH7f) is very active with users who are new to PyO3, and often completely new to Rust. Helping them debug is a great way to get experience with the PyO3 codebase. @@ -57,14 +53,14 @@ nox -s docs -- open #### Doctests We use lots of code blocks in our docs. Run `cargo test --doc` when making changes to check that -the doctests still work, or `cargo test` to run all the tests including doctests. See +the doctests still work, or `cargo test` to run all the Rust tests including doctests. See https://doc.rust-lang.org/rustdoc/documentation-tests.html for a guide on doctests. #### Building the guide You can preview the user guide by building it locally with `mdbook`. -First, install [`mdbook`][mdbook] and [`nox`][nox]. Then, run +First, install [`mdbook`][mdbook], the [`mdbook-tabs`][mdbook-tabs] plugin and [`nox`][nox]. Then, run ```shell nox -s build-guide -- --open @@ -90,22 +86,37 @@ Everybody is welcome to submit comments on open PRs. Please help ensure new PyO3 Here are a few things to note when you are writing PRs. -### Continuous Integration - -The PyO3 repo uses GitHub Actions. PRs are blocked from merging if CI is not successful. +### Testing and Continuous Integration -Formatting, linting and tests are checked for all Rust and Python code. In addition, all warnings in Rust code are disallowed (using `RUSTFLAGS="-D warnings"`). +The PyO3 repo uses GitHub Actions. PRs are blocked from merging if CI is not successful. Formatting, linting and tests are checked for all Rust and Python code. In addition, all warnings in Rust code are disallowed (using `RUSTFLAGS="-D warnings"`). Tests run with all supported Python versions with the latest stable Rust compiler, as well as for Python 3.9 with the minimum supported Rust version. If you are adding a new feature, you should add it to the `full` feature in our *Cargo.toml** so that it is tested in CI. -You can run these tests yourself with -`nox`. Use `nox -l` to list the full set of subcommands you can run. +You can run these checks yourself with `nox`. Use `nox -l` to list the full set of subcommands you can run. + +#### Linting Python code +`nox -s ruff` + +#### Linting Rust code +`nox -s rustfmt` + +#### Semver checks +`cargo semver-checks check-release` + +#### Clippy +`nox -s clippy-all` + +#### Tests +`nox -s test` or `cargo test` for Rust tests only, `nox -f pytests/noxfile.py -s test` for Python tests only + +#### Check all conditional compilation +`nox -s check-feature-powerset` #### UI Tests -PyO3 uses [`trybuild`][trybuild] to develop UI tests to capture error messages from the Rust compiler for some of the macro functionality. +PyO3 uses [`trybuild`](https://github.com/dtolnay/trybuild) to develop UI tests to capture error messages from the Rust compiler for some of the macro functionality. Because there are several feature combinations for these UI tests, when updating them all (e.g. for a new Rust compiler version) it may be helpful to use the `update-ui-tests` nox session: @@ -190,7 +201,7 @@ Second, there is a Python-based benchmark contained in the `pytests` subdirector You can view what code is and isn't covered by PyO3's tests. We aim to have 100% coverage - please check coverage and add tests if you notice a lack of coverage! -- First, ensure the llmv-cov cargo plugin is installed. You may need to run the plugin through cargo once before using it with `nox`. +- First, ensure the llvm-cov cargo plugin is installed. You may need to run the plugin through cargo once before using it with `nox`. ```shell cargo install cargo-llvm-cov cargo llvm-cov @@ -224,5 +235,6 @@ In the meanwhile, some of our maintainers have personal GitHub sponsorship pages - [messense](https://github.com/sponsors/messense) [mdbook]: https://rust-lang.github.io/mdBook/cli/index.html +[mdbook-tabs]: https://mdbook-plugins.rustforweb.org/tabs.html [lychee]: https://github.com/lycheeverse/lychee [nox]: https://github.com/theacodes/nox diff --git a/LICENSE-APACHE b/LICENSE-APACHE index fca31990733..72207b851d3 100644 --- a/LICENSE-APACHE +++ b/LICENSE-APACHE @@ -1,189 +1,178 @@ - Copyright (c) 2017-present PyO3 Project and Contributors. https://github.com/PyO3 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. +Copyright (c) 2017-present PyO3 Project and Contributors. https://github.com/PyO3 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md index 8e7e2f75bca..7ad5932b35d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # PyO3 [![actions status](https://img.shields.io/github/actions/workflow/status/PyO3/pyo3/ci.yml?branch=main&logo=github&style=)](https://github.com/PyO3/pyo3/actions) -[![benchmark](https://img.shields.io/badge/benchmark-✓-Green?logo=github)](https://pyo3.rs/dev/bench/) +[![benchmark](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/PyO3/pyo3) [![codecov](https://img.shields.io/codecov/c/gh/PyO3/pyo3?logo=codecov)](https://codecov.io/gh/PyO3/pyo3) [![crates.io](https://img.shields.io/crates/v/pyo3?logo=rust)](https://crates.io/crates/pyo3) -[![minimum rustc 1.56](https://img.shields.io/badge/rustc-1.56+-blue?logo=rust)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html) +[![minimum rustc 1.63](https://img.shields.io/badge/rustc-1.63+-blue?logo=rust)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html) [![discord server](https://img.shields.io/discord/1209263839632424990?logo=discord)](https://discord.gg/33kcChzH7f) [![contributing notes](https://img.shields.io/badge/contribute-on%20github-Green?logo=github)](https://github.com/PyO3/pyo3/blob/main/Contributing.md) @@ -16,9 +16,12 @@ ## Usage -PyO3 supports the following software versions: - - Python 3.7 and up (CPython, PyPy, and GraalPy) - - Rust 1.56 and up +Requires Rust 1.63 or greater. + +PyO3 supports the following Python distributions: + - CPython 3.7 or greater + - PyPy 7.3 (Python 3.9+) + - GraalPy 24.0 or greater (Python 3.10+) You can use PyO3 to write a native Python module in Rust, or to embed Python in a Rust binary. The following sections explain each of these in turn. @@ -68,7 +71,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.21.1", features = ["extension-module"] } +pyo3 = { version = "0.24.1", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -137,7 +140,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.21.1" +version = "0.24.1" features = ["auto-initialize"] ``` @@ -146,15 +149,16 @@ Example program displaying the value of `sys.version` and the current user name: ```rust use pyo3::prelude::*; use pyo3::types::IntoPyDict; +use pyo3::ffi::c_str; fn main() -> PyResult<()> { Python::with_gil(|py| { - let sys = py.import_bound("sys")?; + let sys = py.import("sys")?; let version: String = sys.getattr("version")?.extract()?; - let locals = [("os", py.import_bound("os")?)].into_py_dict_bound(py); - let code = "os.getenv('USER') or os.getenv('USERNAME') or 'Unknown'"; - let user: String = py.eval_bound(code, None, Some(&locals))?.extract()?; + let locals = [("os", py.import("os")?)].into_py_dict(py)?; + let code = c_str!("os.getenv('USER') or os.getenv('USERNAME') or 'Unknown'"); + let user: String = py.eval(code, None, Some(&locals))?.extract()?; println!("Hello {}, I'm Python {}", user, version); Ok(()) @@ -174,16 +178,17 @@ about this topic. - [dict-derive](https://github.com/gperinazzo/dict-derive) _Derive FromPyObject to automatically transform Python dicts into Rust structs_ - [pyo3-log](https://github.com/vorner/pyo3-log) _Bridge from Rust to Python logging_ - [pythonize](https://github.com/davidhewitt/pythonize) _Serde serializer for converting Rust objects to JSON-compatible Python objects_ -- [pyo3-asyncio](https://github.com/awestlake87/pyo3-asyncio) _Utilities for working with Python's Asyncio library and async functions_ +- [pyo3-async-runtimes](https://github.com/PyO3/pyo3-async-runtimes) _Utilities for interoperability with Python's Asyncio library and Rust's async runtimes._ - [rustimport](https://github.com/mityax/rustimport) _Directly import Rust files or crates from Python, without manual compilation step. Provides pyo3 integration by default and generates pyo3 binding code automatically._ +- [pyo3-arrow](https://crates.io/crates/pyo3-arrow) _Lightweight [Apache Arrow](https://arrow.apache.org/) integration for pyo3._ +- [pyo3-bytes](https://crates.io/crates/pyo3-bytes) _Integration between [`bytes`](https://crates.io/crates/bytes) and pyo3._ ## Examples -- [autopy](https://github.com/autopilot-rs/autopy) _A simple, cross-platform GUI automation library for Python and Rust._ - - Contains an example of building wheels on TravisCI and appveyor using [cibuildwheel](https://github.com/pypa/cibuildwheel) -- [ballista-python](https://github.com/apache/arrow-ballista-python) _A Python library that binds to Apache Arrow distributed query engine Ballista._ - [bed-reader](https://github.com/fastlmm/bed-reader) _Read and write the PLINK BED format, simply and efficiently._ - Shows Rayon/ndarray::parallel (including capturing errors, controlling thread num), Python types to Rust generics, Github Actions +- [cellular_raza](https://cellular-raza.com) _A cellular agent-based simulation framework for building complex models from a clean slate._ +- [connector-x](https://github.com/sfu-db/connector-x/tree/main/connectorx-python) _Fastest library to load data from DB to DataFrames in Rust and Python._ - [cryptography](https://github.com/pyca/cryptography/tree/main/src/rust) _Python cryptography library with some functionality in Rust._ - [css-inline](https://github.com/Stranger6667/css-inline/tree/master/bindings/python) _CSS inlining for Python implemented in Rust._ - [datafusion-python](https://github.com/apache/arrow-datafusion-python) _A Python library that binds to Apache Arrow in-memory query engine DataFusion._ @@ -192,35 +197,37 @@ about this topic. - [fastuuid](https://github.com/thedrow/fastuuid/) _Python bindings to Rust's UUID library._ - [feos](https://github.com/feos-org/feos) _Lightning fast thermodynamic modeling in Rust with fully developed Python interface._ - [forust](https://github.com/jinlow/forust) _A lightweight gradient boosted decision tree library written in Rust._ -- [greptimedb](https://github.com/GreptimeTeam/greptimedb/tree/main/src/script) _Support [Python scripting](https://docs.greptime.com/user-guide/python-scripts/overview) in the database_ +- [granian](https://github.com/emmett-framework/granian) _A Rust HTTP server for Python applications._ - [haem](https://github.com/BooleanCat/haem) _A Python library for working on Bioinformatics problems._ +- [html2text-rs](https://github.com/deedy5/html2text_rs) _Python library for converting HTML to markup or plain text._ - [html-py-ever](https://github.com/PyO3/setuptools-rust/tree/main/examples/html-py-ever) _Using [html5ever](https://github.com/servo/html5ever) through [kuchiki](https://github.com/kuchiki-rs/kuchiki) to speed up html parsing and css-selecting._ -- [hyperjson](https://github.com/mre/hyperjson) _A hyper-fast Python module for reading/writing JSON data using Rust's serde-json._ -- [inline-python](https://github.com/fusion-engineering/inline-python) _Inline Python code directly in your Rust code._ +- [inline-python](https://github.com/m-ou-se/inline-python) _Inline Python code directly in your Rust code._ - [johnnycanencrypt](https://github.com/kushaldas/johnnycanencrypt) OpenPGP library with Yubikey support. -- [jsonschema-rs](https://github.com/Stranger6667/jsonschema-rs/tree/master/bindings/python) _Fast JSON Schema validation library._ +- [jsonschema](https://github.com/Stranger6667/jsonschema/tree/master/crates/jsonschema-py) _A high-performance JSON Schema validator for Python._ - [mocpy](https://github.com/cds-astro/mocpy) _Astronomical Python library offering data structures for describing any arbitrary coverage regions on the unit sphere._ - [opendal](https://github.com/apache/opendal/tree/main/bindings/python) _A data access layer that allows users to easily and efficiently retrieve data from various storage services in a unified way._ - [orjson](https://github.com/ijl/orjson) _Fast Python JSON library._ - [ormsgpack](https://github.com/aviramha/ormsgpack) _Fast Python msgpack library._ -- [point-process](https://github.com/ManifoldFR/point-process-rust/tree/master/pylib) _High level API for pointprocesses as a Python library._ -- [polaroid](https://github.com/daggy1234/polaroid) _Hyper Fast and safe image manipulation library for Python written in Rust._ - [polars](https://github.com/pola-rs/polars) _Fast multi-threaded DataFrame library in Rust | Python | Node.js._ - [pydantic-core](https://github.com/pydantic/pydantic-core) _Core validation logic for pydantic written in Rust._ -- [pyheck](https://github.com/kevinheavey/pyheck) _Fast case conversion library, built by wrapping [heck](https://github.com/withoutboats/heck)._ - - Quite easy to follow as there's not much code. -- [pyre](https://github.com/Project-Dream-Weaver/pyre-http) _Fast Python HTTP server written in Rust._ -- [ril-py](https://github.com/Cryptex-github/ril-py) _A performant and high-level image processing library for Python written in Rust._ +- [primp](https://github.com/deedy5/primp) _The fastest python HTTP client that can impersonate web browsers by mimicking their headers and TLS/JA3/JA4/HTTP2 fingerprints._ +- [rateslib](https://github.com/attack68/rateslib) _A fixed income library for Python using Rust extensions._ - [river](https://github.com/online-ml/river) _Online machine learning in python, the computationally heavy statistics algorithms are implemented in Rust._ +- [robyn](https://github.com/sparckles/Robyn) A Super Fast Async Python Web Framework with a Rust runtime. - [rust-python-coverage](https://github.com/cjermain/rust-python-coverage) _Example PyO3 project with automated test coverage for Rust and Python._ +- [rnet](https://github.com/0x676e67/rnet) Asynchronous Python HTTP Client with Black Magic +- [sail](https://github.com/lakehq/sail) _Unifying stream, batch, and AI workloads with Apache Spark compatibility._ - [tiktoken](https://github.com/openai/tiktoken) _A fast BPE tokeniser for use with OpenAI's models._ - [tokenizers](https://github.com/huggingface/tokenizers/tree/main/bindings/python) _Python bindings to the Hugging Face tokenizers (NLP) written in Rust._ - [tzfpy](http://github.com/ringsaturn/tzfpy) _A fast package to convert longitude/latitude to timezone name._ - [utiles](https://github.com/jessekrubin/utiles) _Fast Python web-map tile utilities_ -- [wasmer-python](https://github.com/wasmerio/wasmer-python) _Python library to run WebAssembly binaries._ ## Articles and other media +- [(Video) PyO3: From Python to Rust and Back Again](https://www.youtube.com/watch?v=UmL_CA-v3O8) - Jul 3, 2024 +- [Parsing Python ASTs 20x Faster with Rust](https://www.gauge.sh/blog/parsing-python-asts-20x-faster-with-rust) - Jun 17, 2024 +- [(Video) How Python Harnesses Rust through PyO3](https://www.youtube.com/watch?v=UkZ_m3Wj2hA) - May 18, 2024 +- [(Video) Combining Rust and Python: The Best of Both Worlds?](https://www.youtube.com/watch?v=lyG6AKzu4ew) - Mar 1, 2024 - [(Video) Extending Python with Rust using PyO3](https://www.youtube.com/watch?v=T45ZEmSR1-s) - Dec 16, 2023 - [A Week of PyO3 + rust-numpy (How to Speed Up Your Data Pipeline X Times)](https://terencezl.github.io/blog/2023/06/06/a-week-of-pyo3-rust-numpy/) - Jun 6, 2023 - [(Podcast) PyO3 with David Hewitt](https://rustacean-station.org/episode/david-hewitt/) - May 19, 2023 diff --git a/branding/favicon/pyo3_16x16.png b/branding/favicon/pyo3_16x16.png new file mode 100644 index 00000000000..0d2d77eb151 Binary files /dev/null and b/branding/favicon/pyo3_16x16.png differ diff --git a/branding/favicon/pyo3_32x32.png b/branding/favicon/pyo3_32x32.png new file mode 100644 index 00000000000..ff1f97ae269 Binary files /dev/null and b/branding/favicon/pyo3_32x32.png differ diff --git a/branding/pyo3logo.png b/branding/pyo3logo.png new file mode 100644 index 00000000000..06cad61734e Binary files /dev/null and b/branding/pyo3logo.png differ diff --git a/branding/pyo3logo.svg b/branding/pyo3logo.svg new file mode 100644 index 00000000000..0315c63e56a --- /dev/null +++ b/branding/pyo3logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/branding/pyotr.png b/branding/pyotr.png new file mode 100644 index 00000000000..75ab9bb34b8 Binary files /dev/null and b/branding/pyotr.png differ diff --git a/branding/pyotr.svg b/branding/pyotr.svg new file mode 100644 index 00000000000..52d478e67c4 --- /dev/null +++ b/branding/pyotr.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/build.rs b/build.rs index 7f0ae6e31c8..68a658bf285 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,9 @@ use std::env; use pyo3_build_config::pyo3_build_script_impl::{cargo_env_var, errors::Result}; -use pyo3_build_config::{bail, print_feature_cfgs, InterpreterConfig}; +use pyo3_build_config::{ + add_python_framework_link_args, bail, print_feature_cfgs, InterpreterConfig, +}; fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> { if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() && !interpreter_config.shared { @@ -39,13 +41,17 @@ fn configure_pyo3() -> Result<()> { println!("{}", cfg) } - // Emit cfgs like `thread_local_const_init` + // Emit cfgs like `invalid_from_utf8_lint` print_feature_cfgs(); + // Make `cargo test` etc work on macOS with Xcode bundled Python + add_python_framework_link_args(); + Ok(()) } fn main() { + pyo3_build_config::print_expected_cfgs(); if let Err(e) = configure_pyo3() { eprintln!("error: {}", e.report()); std::process::exit(1) diff --git a/emscripten/Makefile b/emscripten/Makefile index af224854c26..54094382c1b 100644 --- a/emscripten/Makefile +++ b/emscripten/Makefile @@ -4,7 +4,7 @@ CURDIR=$(abspath .) BUILDROOT ?= $(CURDIR)/builddir PYMAJORMINORMICRO ?= 3.11.0 -EMSCRIPTEN_VERSION=3.1.13 +EMSCRIPTEN_VERSION=3.1.68 export EMSDKDIR = $(BUILDROOT)/emsdk diff --git a/examples/Cargo.toml b/examples/Cargo.toml index e54b3b5cde2..f6c77eb609c 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -3,6 +3,7 @@ name = "pyo3-examples" version = "0.0.0" publish = false edition = "2021" +rust-version = "1.63" [dev-dependencies] pyo3 = { path = "..", features = ["auto-initialize", "extension-module"] } @@ -10,5 +11,5 @@ pyo3 = { path = "..", features = ["auto-initialize", "extension-module"] } [[example]] name = "decorator" path = "decorator/src/lib.rs" -crate_type = ["cdylib"] +crate-type = ["cdylib"] doc-scrape-examples = true diff --git a/examples/README.md b/examples/README.md index baaa57b650d..3c7cc301399 100644 --- a/examples/README.md +++ b/examples/README.md @@ -9,9 +9,11 @@ Below is a brief description of each of these: | `decorator` | A project showcasing the example from the [Emulating callable objects](https://pyo3.rs/latest/class/call.html) chapter of the guide. | | `maturin-starter` | A template project which is configured to use [`maturin`](https://github.com/PyO3/maturin) for development. | | `setuptools-rust-starter` | A template project which is configured to use [`setuptools_rust`](https://github.com/PyO3/setuptools-rust/) for development. | -| `word-count` | A quick performance comparison between word counter implementations written in each of Rust and Python. | | `plugin` | Illustrates how to use Python as a scripting language within a Rust application | -| `sequential` | Illustrates how to use pyo3-ffi to write subinterpreter-safe modules | + +Note that there are also other examples in the `pyo3-ffi/examples` +directory that illustrate how to create rust extensions using raw FFI calls into +the CPython C API instead of using PyO3's abstractions. ## Creating new projects from these examples diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index 1dc689ef6d4..bb3c6cdbd7d 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.1"); +variable::set("PYO3_VERSION", "0.24.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/decorator/Cargo.toml b/examples/decorator/Cargo.toml index 3456302a9fd..785895121a3 100644 --- a/examples/decorator/Cargo.toml +++ b/examples/decorator/Cargo.toml @@ -2,6 +2,7 @@ name = "decorator" version = "0.1.0" edition = "2021" +rust-version = "1.63" [lib] name = "decorator" diff --git a/examples/decorator/src/lib.rs b/examples/decorator/src/lib.rs index cfb09c112d5..4c5471c9945 100644 --- a/examples/decorator/src/lib.rs +++ b/examples/decorator/src/lib.rs @@ -1,6 +1,6 @@ use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; -use std::cell::Cell; +use std::sync::atomic::{AtomicU64, Ordering}; /// A function decorator that keeps track how often it is called. /// @@ -9,8 +9,8 @@ use std::cell::Cell; pub struct PyCounter { // Keeps track of how many calls have gone through. // - // See the discussion at the end for why `Cell` is used. - count: Cell, + // See the discussion at the end for why `AtomicU64` is used. + count: AtomicU64, // This is the actual function being wrapped. wraps: Py, @@ -26,14 +26,14 @@ impl PyCounter { #[new] fn __new__(wraps: Py) -> Self { PyCounter { - count: Cell::new(0), + count: AtomicU64::new(0), wraps, } } #[getter] fn count(&self) -> u64 { - self.count.get() + self.count.load(Ordering::Relaxed) } #[pyo3(signature = (*args, **kwargs))] @@ -43,15 +43,13 @@ impl PyCounter { args: &Bound<'_, PyTuple>, kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - let old_count = self.count.get(); - let new_count = old_count + 1; - self.count.set(new_count); + let new_count = self.count.fetch_add(1, Ordering::Relaxed); let name = self.wraps.getattr(py, "__name__")?; println!("{} has been called {} time(s).", name, new_count); // After doing something, we finally forward the call to the wrapped function - let ret = self.wraps.call_bound(py, args, kwargs)?; + let ret = self.wraps.call(py, args, kwargs)?; // We could do something with the return value of // the function before returning it diff --git a/examples/getitem/Cargo.toml b/examples/getitem/Cargo.toml index 17020b9bd05..99430483171 100644 --- a/examples/getitem/Cargo.toml +++ b/examples/getitem/Cargo.toml @@ -2,6 +2,7 @@ name = "getitem" version = "0.1.0" edition = "2021" +rust-version = "1.63" [lib] name = "getitem" diff --git a/examples/getitem/src/lib.rs b/examples/getitem/src/lib.rs index c3c662ab92f..ba850a06b8d 100644 --- a/examples/getitem/src/lib.rs +++ b/examples/getitem/src/lib.rs @@ -2,7 +2,6 @@ use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; use pyo3::types::PySlice; -use std::os::raw::c_long; #[derive(FromPyObject)] enum IntOrSlice<'py> { @@ -29,7 +28,7 @@ impl ExampleContainer { } else if let Ok(slice) = key.downcast::() { // METHOD 1 - the use PySliceIndices to help with bounds checking and for cases when only start or end are provided // in this case the start/stop/step all filled in to give valid values based on the max_length given - let index = slice.indices(self.max_length as c_long).unwrap(); + let index = slice.indices(self.max_length as isize).unwrap(); let _delta = index.stop - index.start; // METHOD 2 - Do the getattr manually really only needed if you have some special cases for stop/_step not being present @@ -62,7 +61,7 @@ impl ExampleContainer { fn __setitem__(&self, idx: IntOrSlice, value: u32) -> PyResult<()> { match idx { IntOrSlice::Slice(slice) => { - let index = slice.indices(self.max_length as c_long).unwrap(); + let index = slice.indices(self.max_length as isize).unwrap(); println!( "Got a slice! {}-{}, step: {}, value: {}", index.start, index.stop, index.step, value @@ -76,8 +75,7 @@ impl ExampleContainer { } } -#[pymodule] -#[pyo3(name = "getitem")] +#[pymodule(name = "getitem")] fn example(m: &Bound<'_, PyModule>) -> PyResult<()> { // ? -https://github.com/PyO3/maturin/issues/475 m.add_class::()?; diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index 1dc689ef6d4..bb3c6cdbd7d 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.1"); +variable::set("PYO3_VERSION", "0.24.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/Cargo.toml b/examples/maturin-starter/Cargo.toml index 257908a4bb0..ee1ab9aff06 100644 --- a/examples/maturin-starter/Cargo.toml +++ b/examples/maturin-starter/Cargo.toml @@ -2,6 +2,7 @@ name = "maturin-starter" version = "0.1.0" edition = "2021" +rust-version = "1.63" [lib] name = "maturin_starter" diff --git a/examples/maturin-starter/src/lib.rs b/examples/maturin-starter/src/lib.rs index faa147b2a10..4c2a30d3a5d 100644 --- a/examples/maturin-starter/src/lib.rs +++ b/examples/maturin-starter/src/lib.rs @@ -27,7 +27,7 @@ fn maturin_starter(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { // Inserting to sys.modules allows importing submodules nicely from Python // e.g. from maturin_starter.submodule import SubmoduleClass - let sys = PyModule::import_bound(py, "sys")?; + let sys = PyModule::import(py, "sys")?; let sys_modules: Bound<'_, PyDict> = sys.getattr("modules")?.downcast_into()?; sys_modules.set_item("maturin_starter.submodule", m.getattr("submodule")?)?; diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index 78adb883f43..8f9d871b601 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.1"); +variable::set("PYO3_VERSION", "0.24.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/plugin/Cargo.toml b/examples/plugin/Cargo.toml index 08127b5003f..062dab1ff21 100644 --- a/examples/plugin/Cargo.toml +++ b/examples/plugin/Cargo.toml @@ -2,7 +2,7 @@ name = "plugin_example" version = "0.1.0" edition = "2021" - +rust-version = "1.63" [dependencies] pyo3={path="../../", features=["macros"]} diff --git a/examples/plugin/src/main.rs b/examples/plugin/src/main.rs index 5a54a1837cb..59442549e6d 100644 --- a/examples/plugin/src/main.rs +++ b/examples/plugin/src/main.rs @@ -13,13 +13,13 @@ fn main() -> Result<(), Box> { //do useful work Python::with_gil(|py| { //add the current directory to import path of Python (do not use this in production!) - let syspath: &PyList = py.import("sys")?.getattr("path")?.extract()?; + let syspath: Bound = py.import("sys")?.getattr("path")?.extract()?; syspath.insert(0, &path)?; println!("Import path is: {:?}", syspath); // Now we can load our python_plugin/gadget_init_plugin.py file. // It can in turn import other stuff as it deems appropriate - let plugin = PyModule::import_bound(py, "gadget_init_plugin")?; + let plugin = PyModule::import(py, "gadget_init_plugin")?; // and call start function there, which will return a python reference to Gadget. // Gadget here is a "pyclass" object reference let gadget = plugin.getattr("start")?.call0()?; diff --git a/examples/sequential/Cargo.toml b/examples/sequential/Cargo.toml deleted file mode 100644 index 4500c69b597..00000000000 --- a/examples/sequential/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "sequential" -version = "0.1.0" -edition = "2021" - -[lib] -name = "sequential" -crate-type = ["cdylib", "lib"] - -[dependencies] -pyo3-ffi = { path = "../../pyo3-ffi", features = ["extension-module"] } - -[workspace] diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index 212f62f76fe..1553f9cf931 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.1"); +variable::set("PYO3_VERSION", "0.24.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/Cargo.toml b/examples/setuptools-rust-starter/Cargo.toml index 5777cbbcd78..ffabd8df849 100644 --- a/examples/setuptools-rust-starter/Cargo.toml +++ b/examples/setuptools-rust-starter/Cargo.toml @@ -2,6 +2,7 @@ name = "setuptools-rust-starter" version = "0.1.0" edition = "2021" +rust-version = "1.63" [lib] name = "setuptools_rust_starter" diff --git a/examples/setuptools-rust-starter/src/lib.rs b/examples/setuptools-rust-starter/src/lib.rs index d31284be7a3..a26623bc044 100644 --- a/examples/setuptools-rust-starter/src/lib.rs +++ b/examples/setuptools-rust-starter/src/lib.rs @@ -27,7 +27,7 @@ fn _setuptools_rust_starter(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult // Inserting to sys.modules allows importing submodules nicely from Python // e.g. from setuptools_rust_starter.submodule import SubmoduleClass - let sys = PyModule::import_bound(py, "sys")?; + let sys = PyModule::import(py, "sys")?; let sys_modules: Bound<'_, PyDict> = sys.getattr("modules")?.downcast_into()?; sys_modules.set_item("setuptools_rust_starter.submodule", m.getattr("submodule")?)?; diff --git a/examples/string-sum/Cargo.toml b/examples/string-sum/Cargo.toml deleted file mode 100644 index 4a48b221c60..00000000000 --- a/examples/string-sum/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "string_sum" -version = "0.1.0" -edition = "2021" - -[lib] -name = "string_sum" -crate-type = ["cdylib"] - -[dependencies] -pyo3-ffi = { path = "../../pyo3-ffi", features = ["extension-module"] } - -[workspace] diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index 1dc689ef6d4..bb3c6cdbd7d 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.1"); +variable::set("PYO3_VERSION", "0.24.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/word-count/Cargo.toml b/examples/word-count/Cargo.toml index cfb3444d5fe..8d79c8a4ff9 100644 --- a/examples/word-count/Cargo.toml +++ b/examples/word-count/Cargo.toml @@ -2,6 +2,7 @@ name = "word-count" version = "0.1.0" edition = "2021" +rust-version = "1.63" [lib] name = "word_count" diff --git a/guide/book.toml b/guide/book.toml index bccc3506098..be682a64eab 100644 --- a/guide/book.toml +++ b/guide/book.toml @@ -6,7 +6,11 @@ author = "PyO3 Project and Contributors" [preprocessor.pyo3_version] command = "python3 guide/pyo3_version.py" +[preprocessor.tabs] + [output.html] git-repository-url = "https://github.com/PyO3/pyo3/tree/main/guide" edit-url-template = "https://github.com/PyO3/pyo3/edit/main/guide/{path}" playground.runnable = false +additional-css = ["theme/tabs.css"] +additional-js = ["theme/tabs.js"] diff --git a/guide/pyclass-parameters.md b/guide/pyclass-parameters.md index 6951a5b5e15..7ebca2ec821 100644 --- a/guide/pyclass-parameters.md +++ b/guide/pyclass-parameters.md @@ -2,21 +2,26 @@ | Parameter | Description | | :- | :- | +| `constructor` | This is currently only allowed on [variants of complex enums][params-constructor]. It allows customization of the generated class constructor for each variant. It uses the same syntax and supports the same options as the `signature` attribute of functions and methods. | | `crate = "some::path"` | Path to import the `pyo3` crate, if it's not accessible at `::pyo3`. | | `dict` | Gives instances of this class an empty `__dict__` to store custom attributes. | +| `eq` | Implements `__eq__` using the `PartialEq` implementation of the underlying Rust datatype. | +| `eq_int` | Implements `__eq__` using `__int__` for simple enums. | | `extends = BaseType` | Use a custom baseclass. Defaults to [`PyAny`][params-1] | | `freelist = N` | Implements a [free list][params-2] of size N. This can improve performance for types that are often created and deleted in quick succession. Profile your code to see whether `freelist` is right for you. | | `frozen` | Declares that your pyclass is immutable. It removes the borrow checker overhead when retrieving a shared reference to the Rust struct, but disables the ability to get a mutable reference. | | `get_all` | Generates getters for all fields of the pyclass. | +| `hash` | Implements `__hash__` using the `Hash` implementation of the underlying Rust datatype. | | `mapping` | Inform PyO3 that this class is a [`Mapping`][params-mapping], and so leave its implementation of sequence C-API slots empty. | | `module = "module_name"` | Python code will see the class as being defined in this module. Defaults to `builtins`. | | `name = "python_name"` | Sets the name that Python sees this class as. Defaults to the name of the Rust struct. | +| `ord` | Implements `__lt__`, `__gt__`, `__le__`, & `__ge__` using the `PartialOrd` implementation of the underlying Rust datatype. *Requires `eq`* | | `rename_all = "renaming_rule"` | Applies renaming rules to every getters and setters of a struct, or every variants of an enum. Possible values are: "camelCase", "kebab-case", "lowercase", "PascalCase", "SCREAMING-KEBAB-CASE", "SCREAMING_SNAKE_CASE", "snake_case", "UPPERCASE". | | `sequence` | Inform PyO3 that this class is a [`Sequence`][params-sequence], and so leave its C-API mapping length slot empty. | | `set_all` | Generates setters for all fields of the pyclass. | +| `str` | Implements `__str__` using the `Display` implementation of the underlying Rust datatype or by passing an optional format string `str=""`. *Note: The optional format string is only allowed for structs. `name` and `rename_all` are incompatible with the optional format string. Additional details can be found in the discussion on this [PR](https://github.com/PyO3/pyo3/pull/4233).* | | `subclass` | Allows other Python classes and `#[pyclass]` to inherit from this class. Enums cannot be subclassed. | -| `text_signature = "(arg1, arg2, ...)"` | Sets the text signature for the Python class' `__new__` method. | -| `unsendable` | Required if your struct is not [`Send`][params-3]. Rather than using `unsendable`, consider implementing your struct in a threadsafe way by e.g. substituting [`Rc`][params-4] with [`Arc`][params-5]. By using `unsendable`, your class will panic when accessed by another thread. Also note the Python's GC is multi-threaded and while unsendable classes will not be traversed on foreign threads to avoid UB, this can lead to memory leaks. | +| `unsendable` | Required if your struct is not [`Send`][params-3]. Rather than using `unsendable`, consider implementing your struct in a thread-safe way by e.g. substituting [`Rc`][params-4] with [`Arc`][params-5]. By using `unsendable`, your class will panic when accessed by another thread. Also note the Python's GC is multi-threaded and while unsendable classes will not be traversed on foreign threads to avoid UB, this can lead to memory leaks. | | `weakref` | Allows this class to be [weakly referenceable][params-6]. | All of these parameters can either be passed directly on the `#[pyclass(...)]` annotation, or as one or @@ -39,5 +44,6 @@ struct MyClass {} [params-4]: https://doc.rust-lang.org/std/rc/struct.Rc.html [params-5]: https://doc.rust-lang.org/std/sync/struct.Arc.html [params-6]: https://docs.python.org/3/library/weakref.html +[params-constructor]: https://pyo3.rs/latest/class.html#complex-enums [params-mapping]: https://pyo3.rs/latest/class/protocols.html#mapping--sequence-types [params-sequence]: https://pyo3.rs/latest/class/protocols.html#mapping--sequence-types diff --git a/guide/src/SUMMARY.md b/guide/src/SUMMARY.md index 4c22c26f587..cf987b72625 100644 --- a/guide/src/SUMMARY.md +++ b/guide/src/SUMMARY.md @@ -15,6 +15,7 @@ - [Basic object customization](class/object.md) - [Emulating numeric types](class/numeric.md) - [Emulating callable objects](class/call.md) + - [Thread safety](class/thread-safety.md) - [Calling Python from Rust](python-from-rust.md) - [Python object types](types.md) - [Python exceptions](exception.md) @@ -25,15 +26,16 @@ - [Conversion traits](conversions/traits.md) - [Using `async` and `await`](async-await.md) - [Parallelism](parallelism.md) +- [Supporting Free-Threaded Python](free-threading.md) - [Debugging](debugging.md) - [Features reference](features.md) -- [Memory management](memory.md) - [Performance](performance.md) - [Advanced topics](advanced.md) - [Building and distribution](building-and-distribution.md) - [Supporting multiple Python versions](building-and-distribution/multiple-python-versions.md) - [Useful crates](ecosystem.md) - [Logging](ecosystem/logging.md) + - [Tracing](ecosystem/tracing.md) - [Using `async` and `await`](ecosystem/async-await.md) - [FAQ and troubleshooting](faq.md) diff --git a/guide/src/async-await.md b/guide/src/async-await.md index 06fa1580ad7..27574181804 100644 --- a/guide/src/async-await.md +++ b/guide/src/async-await.md @@ -12,6 +12,7 @@ use futures::channel::oneshot; use pyo3::prelude::*; #[pyfunction] +#[pyo3(signature=(seconds, result=None))] async fn sleep(seconds: f64, result: Option) -> Option { let (tx, rx) = oneshot::channel(); thread::spawn(move || { diff --git a/guide/src/building-and-distribution.md b/guide/src/building-and-distribution.md index 780f135e211..d3474fedaf7 100644 --- a/guide/src/building-and-distribution.md +++ b/guide/src/building-and-distribution.md @@ -92,8 +92,10 @@ If you're packaging your library for redistribution, you should indicated the Py To use PyO3 with bazel one needs to manually configure PyO3, PyO3-ffi and PyO3-macros. In particular, one needs to make sure that it is compiled with the right python flags for the version you intend to use. For example see: -1. https://github.com/OliverFM/pytorch_with_gazelle -- for a minimal example of a repo that can use PyO3, PyTorch and Gazelle to generate python Build files. -2. https://github.com/TheButlah/rules_pyo3 -- which has more extensive support, but is outdated. + +1. [github.com/abrisco/rules_pyo3](https://github.com/abrisco/rules_pyo3) -- General rules for building extension modules. +2. [github.com/OliverFM/pytorch_with_gazelle](https://github.com/OliverFM/pytorch_with_gazelle) -- for a minimal example of a repo that can use PyO3, PyTorch and Gazelle to generate python Build files. +3. [github.com/TheButlah/rules_pyo3](https://github.com/TheButlah/rules_pyo3) -- is somewhat dated. #### Platform tags @@ -103,9 +105,9 @@ Rather than using just the `.so` or `.pyd` extension suggested above (depending # CPython 3.10 on macOS .cpython-310-darwin.so -# PyPy 7.3 (Python 3.8) on Linux +# PyPy 7.3 (Python 3.9) on Linux $ python -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX"))' -.pypy38-pp73-x86_64-linux-gnu.so +.pypy39-pp73-x86_64-linux-gnu.so ``` So, for example, a valid module library name on CPython 3.10 for macOS is `your_module.cpython-310-darwin.so`, and its equivalent when compiled for PyPy 7.3 on Linux would be `your_module.pypy38-pp73-x86_64-linux-gnu.so`. @@ -142,7 +144,17 @@ rustflags = [ ] ``` -Using the MacOS system python3 (`/usr/bin/python3`, as opposed to python installed via homebrew, pyenv, nix, etc.) may result in runtime errors such as `Library not loaded: @rpath/Python3.framework/Versions/3.8/Python3`. These can be resolved with another addition to `.cargo/config.toml`: +Using the MacOS system python3 (`/usr/bin/python3`, as opposed to python installed via homebrew, pyenv, nix, etc.) may result in runtime errors such as `Library not loaded: @rpath/Python3.framework/Versions/3.8/Python3`. + +The easiest way to set the correct linker arguments is to add a `build.rs` with the following content: + +```rust,ignore +fn main() { + pyo3_build_config::add_python_framework_link_args(); +} +``` + +Alternatively it can be resolved with another addition to `.cargo/config.toml`: ```toml [build] @@ -151,19 +163,9 @@ rustflags = [ ] ``` -Alternatively, on rust >= 1.56, one can include in `build.rs`: - -```rust -fn main() { - println!( - "cargo:rustc-link-arg=-Wl,-rpath,/Library/Developer/CommandLineTools/Library/Frameworks" - ); -} -``` - For more discussion on and workarounds for MacOS linking problems [see this issue](https://github.com/PyO3/pyo3/issues/1800#issuecomment-906786649). -Finally, don't forget that on MacOS the `extension-module` feature will cause `cargo test` to fail without the `--no-default-features` flag (see [the FAQ](https://pyo3.rs/main/faq.html#i-cant-run-cargo-test-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror)). +Finally, don't forget that on MacOS the `extension-module` feature will cause `cargo test` to fail without the `--no-default-features` flag (see [the FAQ](https://pyo3.rs/main/faq.html#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror)). ### The `extension-module` feature @@ -232,7 +234,7 @@ not work when compiling for `abi3`. These are: If you want to embed the Python interpreter inside a Rust program, there are two modes in which this can be done: dynamically and statically. We'll cover each of these modes in the following sections. Each of them affect how you must distribute your program. Instead of learning how to do this yourself, you might want to consider using a project like [PyOxidizer] to ship your application and all of its dependencies in a single file. -PyO3 automatically switches between the two linking modes depending on whether the Python distribution you have configured PyO3 to use ([see above](#configuring-the-python-version)) contains a shared library or a static library. The static library is most often seen in Python distributions compiled from source without the `--enable-shared` configuration option. For example, this is the default for `pyenv` on macOS. +PyO3 automatically switches between the two linking modes depending on whether the Python distribution you have configured PyO3 to use ([see above](#configuring-the-python-version)) contains a shared library or a static library. The static library is most often seen in Python distributions compiled from source without the `--enable-shared` configuration option. ### Dynamically embedding the Python interpreter diff --git a/guide/src/building-and-distribution/multiple-python-versions.md b/guide/src/building-and-distribution/multiple-python-versions.md index b328d236c51..a7879941d0c 100644 --- a/guide/src/building-and-distribution/multiple-python-versions.md +++ b/guide/src/building-and-distribution/multiple-python-versions.md @@ -1,6 +1,6 @@ # Supporting multiple Python versions -PyO3 supports all actively-supported Python 3 and PyPy versions. As much as possible, this is done internally to PyO3 so that your crate's code does not need to adapt to the differences between each version. However, as Python features grow and change between versions, PyO3 cannot a completely identical API for every Python version. This may require you to add conditional compilation to your crate or runtime checks for the Python version. +PyO3 supports all actively-supported Python 3 and PyPy versions. As much as possible, this is done internally to PyO3 so that your crate's code does not need to adapt to the differences between each version. However, as Python features grow and change between versions, PyO3 cannot offer a completely identical API for every Python version. This may require you to add conditional compilation to your crate or runtime checks for the Python version. This section of the guide first introduces the `pyo3-build-config` crate, which you can use as a `build-dependency` to add additional `#[cfg]` flags which allow you to support multiple Python versions at compile-time. diff --git a/guide/src/class.md b/guide/src/class.md index f353cc4787e..90991328fe6 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -37,14 +37,16 @@ struct Number(i32); // PyO3 supports unit-only enums (which contain only unit variants) // These simple enums behave similarly to Python's enumerations (enum.Enum) -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] enum MyEnum { Variant, OtherVariant = 30, // PyO3 supports custom discriminants. } // PyO3 supports custom discriminants in unit-only enums -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] enum HttpResponse { Ok = 200, NotFound = 404, @@ -52,15 +54,18 @@ enum HttpResponse { // ... } -// PyO3 also supports enums with non-unit variants +// PyO3 also supports enums with Struct and Tuple variants // These complex enums have sligtly different behavior from the simple enums above // They are meant to work with instance checks and match statement patterns +// The variants can be mixed and matched +// Struct variants have named fields while tuple enums generate generic names for fields in order _0, _1, _2, ... +// Apart from this both types are functionally identical #[pyclass] enum Shape { Circle { radius: f64 }, Rectangle { width: f64, height: f64 }, - RegularPolygon { side_count: u32, radius: f64 }, - Nothing {}, + RegularPolygon(u32, f64), + Nothing(), } ``` @@ -68,7 +73,7 @@ The above example generates implementations for [`PyTypeInfo`] and [`PyClass`] f ### Restrictions -To integrate Rust types with Python, PyO3 needs to place some restrictions on the types which can be annotated with `#[pyclass]`. In particular, they must have no lifetime parameters, no generic parameters, and must implement `Send`. The reason for each of these is explained below. +To integrate Rust types with Python, PyO3 needs to place some restrictions on the types which can be annotated with `#[pyclass]`. In particular, they must have no lifetime parameters, no generic parameters, and must be thread-safe. The reason for each of these is explained below. #### No lifetime parameters @@ -114,9 +119,13 @@ create_interface!(IntClass, i64); create_interface!(FloatClass, String); ``` -#### Must be Send +#### Must be thread-safe -Because Python objects are freely shared between threads by the Python interpreter, there is no guarantee which thread will eventually drop the object. Therefore all types annotated with `#[pyclass]` must implement `Send` (unless annotated with [`#[pyclass(unsendable)]`](#customizing-the-class)). +Python objects are freely shared between threads by the Python interpreter. This means that: +- Python objects may be created and destroyed by different Python threads; therefore `#[pyclass]` objects must be `Send`. +- Python objects may be accessed by multiple Python threads simultaneously; therefore `#[pyclass]` objects must be `Sync`. + +For now, don't worry about these requirements; simple classes will already be thread-safe. There is a [detailed discussion on thread-safety](./class/thread-safety.md) later in the guide. ## Constructor @@ -191,7 +200,7 @@ fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { ## Bound and interior mutability -Often is useful to turn a `#[pyclass]` type `T` into a Python object and access it from Rust code. The [`Py`] and [`Bound<'py, T>`] smart pointers are the ways to represent a Python object in PyO3's API. More detail can be found about them [in the Python objects](./types.md#pyo3s-smart-pointers) section of the guide. +It is often useful to turn a `#[pyclass]` type `T` into a Python object and access it from Rust code. The [`Py`] and [`Bound<'py, T>`] smart pointers are the ways to represent a Python object in PyO3's API. More detail can be found about them [in the Python objects](./types.md#pyo3s-smart-pointers) section of the guide. Most Python objects do not offer exclusive (`&mut`) access (see the [section on Python's memory model](./python-from-rust.md#pythons-memory-model)). However, Rust structs wrapped as Python objects (called `pyclass` types) often *do* need `&mut` access. Due to the GIL, PyO3 *can* guarantee exclusive access to them. @@ -249,7 +258,7 @@ fn return_myclass() -> Py { let obj = return_myclass(); -Python::with_gil(|py| { +Python::with_gil(move |py| { let bound = obj.bind(py); // Py::bind returns &Bound<'py, MyClass> let obj_ref = bound.borrow(); // Get PyRef assert_eq!(obj_ref.num, 1); @@ -280,6 +289,8 @@ let py_counter: Py = Python::with_gil(|py| { }); py_counter.get().value.fetch_add(1, Ordering::Relaxed); + +Python::with_gil(move |_py| drop(py_counter)); ``` Frozen classes are likely to become the default thereby guiding the PyO3 ecosystem towards a more deliberate application of interior mutability. Eventually, this should enable further optimizations of PyO3's internals and avoid downstream code paying the cost of interior mutability when it is not actually required. @@ -320,8 +331,12 @@ explicitly. To get a parent class from a child, use [`PyRef`] instead of `&self` for methods, or [`PyRefMut`] instead of `&mut self`. -Then you can access a parent class by `self_.as_ref()` as `&Self::BaseClass`, -or by `self_.into_super()` as `PyRef`. +Then you can access a parent class by `self_.as_super()` as `&PyRef`, +or by `self_.into_super()` as `PyRef` (and similar for the `PyRefMut` +case). For convenience, `self_.as_ref()` can also be used to get `&Self::BaseClass` +directly; however, this approach does not let you access base classes higher in the +inheritance hierarchy, for which you would need to chain multiple `as_super` or +`into_super` calls. ```rust # use pyo3::prelude::*; @@ -338,7 +353,7 @@ impl BaseClass { BaseClass { val1: 10 } } - pub fn method(&self) -> PyResult { + pub fn method1(&self) -> PyResult { Ok(self.val1) } } @@ -356,8 +371,8 @@ impl SubClass { } fn method2(self_: PyRef<'_, Self>) -> PyResult { - let super_ = self_.as_ref(); // Get &BaseClass - super_.method().map(|x| x * self_.val2) + let super_ = self_.as_super(); // Get &PyRef + super_.method1().map(|x| x * self_.val2) } } @@ -374,29 +389,52 @@ impl SubSubClass { } fn method3(self_: PyRef<'_, Self>) -> PyResult { + let base = self_.as_super().as_super(); // Get &PyRef<'_, BaseClass> + base.method1().map(|x| x * self_.val3) + } + + fn method4(self_: PyRef<'_, Self>) -> PyResult { let v = self_.val3; let super_ = self_.into_super(); // Get PyRef<'_, SubClass> SubClass::method2(super_).map(|x| x * v) } + fn get_values(self_: PyRef<'_, Self>) -> (usize, usize, usize) { + let val1 = self_.as_super().as_super().val1; + let val2 = self_.as_super().val2; + (val1, val2, self_.val3) + } + + fn double_values(mut self_: PyRefMut<'_, Self>) { + self_.as_super().as_super().val1 *= 2; + self_.as_super().val2 *= 2; + self_.val3 *= 2; + } + #[staticmethod] fn factory_method(py: Python<'_>, val: usize) -> PyResult { let base = PyClassInitializer::from(BaseClass::new()); let sub = base.add_subclass(SubClass { val2: val }); if val % 2 == 0 { - Ok(Py::new(py, sub)?.to_object(py)) + Ok(Py::new(py, sub)?.into_any()) } else { let sub_sub = sub.add_subclass(SubSubClass { val3: val }); - Ok(Py::new(py, sub_sub)?.to_object(py)) + Ok(Py::new(py, sub_sub)?.into_any()) } } } # Python::with_gil(|py| { # let subsub = pyo3::Py::new(py, SubSubClass::new()).unwrap(); -# pyo3::py_run!(py, subsub, "assert subsub.method3() == 3000"); +# pyo3::py_run!(py, subsub, "assert subsub.method1() == 10"); +# pyo3::py_run!(py, subsub, "assert subsub.method2() == 150"); +# pyo3::py_run!(py, subsub, "assert subsub.method3() == 200"); +# pyo3::py_run!(py, subsub, "assert subsub.method4() == 3000"); +# pyo3::py_run!(py, subsub, "assert subsub.get_values() == (10, 15, 20)"); +# pyo3::py_run!(py, subsub, "assert subsub.double_values() == None"); +# pyo3::py_run!(py, subsub, "assert subsub.get_values() == (20, 30, 40)"); # let subsub = SubSubClass::factory_method(py, 2).unwrap(); # let subsubsub = SubSubClass::factory_method(py, 3).unwrap(); -# let cls = py.get_type_bound::(); +# let cls = py.get_type::(); # pyo3::py_run!(py, subsub cls, "assert not isinstance(subsub, cls)"); # pyo3::py_run!(py, subsubsub cls, "assert isinstance(subsubsub, cls)"); # }); @@ -492,7 +530,7 @@ impl MyDict { // some custom methods that use `private` here... } # Python::with_gil(|py| { -# let cls = py.get_type_bound::(); +# let cls = py.get_type::(); # pyo3::py_run!(py, cls, "cls(a=1, b=2)") # }); # } @@ -515,6 +553,7 @@ For simple cases where a member variable is just read and written with no side e ```rust # use pyo3::prelude::*; +# #[allow(dead_code)] #[pyclass] struct MyClass { #[pyo3(get, set)] @@ -762,7 +801,7 @@ impl MyClass { } Python::with_gil(|py| { - let my_class = py.get_type_bound::(); + let my_class = py.get_type::(); pyo3::py_run!(py, my_class, "assert my_class.my_attribute == 'hello'") }); ``` @@ -933,8 +972,8 @@ impl MyClass { # # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let inspect = PyModule::import_bound(py, "inspect")?.getattr("signature")?; -# let module = PyModule::new_bound(py, "my_module")?; +# let inspect = PyModule::import(py, "inspect")?.getattr("signature")?; +# let module = PyModule::new(py, "my_module")?; # module.add_class::()?; # let class = module.getattr("MyClass")?; # @@ -998,6 +1037,44 @@ impl MyClass { Note that `text_signature` on `#[new]` is not compatible with compilation in `abi3` mode until Python 3.10 or greater. +### Method receivers and lifetime elision + +PyO3 supports writing instance methods using the normal method receivers for shared `&self` and unique `&mut self` references. This interacts with [lifetime elision][lifetime-elision] insofar as the lifetime of a such a receiver is assigned to all elided output lifetime parameters. + +This is a good default for general Rust code where return values are more likely to borrow from the receiver than from the other arguments, if they contain any lifetimes at all. However, when returning bound references `Bound<'py, T>` in PyO3-based code, the GIL lifetime `'py` should usually be derived from a GIL token `py: Python<'py>` passed as an argument instead of the receiver. + +Specifically, signatures like + +```rust,ignore +fn frobnicate(&self, py: Python) -> Bound; +``` + +will not work as they are inferred as + +```rust,ignore +fn frobnicate<'a, 'py>(&'a self, py: Python<'py>) -> Bound<'a, Foo>; +``` + +instead of the intended + +```rust,ignore +fn frobnicate<'a, 'py>(&'a self, py: Python<'py>) -> Bound<'py, Foo>; +``` + +and should usually be written as + +```rust,ignore +fn frobnicate<'py>(&self, py: Python<'py>) -> Bound<'py, Foo>; +``` + +The same problem does not exist for `#[pyfunction]`s as the special case for receiver lifetimes does not apply and indeed a signature like + +```rust,ignore +fn frobnicate(bar: &Bar, py: Python) -> Bound; +``` + +will yield compiler error [E0106 "missing lifetime specifier"][compiler-error-e0106]. + ## `#[pyclass]` enums Enum support in PyO3 comes in two flavors, depending on what kind of variants the enum has: simple and complex. @@ -1010,7 +1087,8 @@ PyO3 adds a class attribute for each variant, so you can access them in Python w ```rust # use pyo3::prelude::*; -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] enum MyEnum { Variant, OtherVariant, @@ -1019,7 +1097,7 @@ enum MyEnum { Python::with_gil(|py| { let x = Py::new(py, MyEnum::Variant).unwrap(); let y = Py::new(py, MyEnum::OtherVariant).unwrap(); - let cls = py.get_type_bound::(); + let cls = py.get_type::(); pyo3::py_run!(py, x y cls, r#" assert x == cls.Variant assert y == cls.OtherVariant @@ -1032,20 +1110,19 @@ You can also convert your simple enums into `int`: ```rust # use pyo3::prelude::*; -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] enum MyEnum { Variant, OtherVariant = 10, } Python::with_gil(|py| { - let cls = py.get_type_bound::(); + let cls = py.get_type::(); let x = MyEnum::Variant as i32; // The exact value is assigned by the compiler. pyo3::py_run!(py, cls x, r#" assert int(cls.Variant) == x assert int(cls.OtherVariant) == 10 - assert cls.OtherVariant == 10 # You can also compare against int. - assert 10 == cls.OtherVariant "#) }) ``` @@ -1054,14 +1131,15 @@ PyO3 also provides `__repr__` for enums: ```rust # use pyo3::prelude::*; -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] enum MyEnum{ Variant, OtherVariant, } Python::with_gil(|py| { - let cls = py.get_type_bound::(); + let cls = py.get_type::(); let x = Py::new(py, MyEnum::Variant).unwrap(); pyo3::py_run!(py, cls x, r#" assert repr(x) == 'MyEnum.Variant' @@ -1074,7 +1152,8 @@ All methods defined by PyO3 can be overridden. For example here's how you overri ```rust # use pyo3::prelude::*; -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] enum MyEnum { Answer = 42, } @@ -1087,7 +1166,7 @@ impl MyEnum { } Python::with_gil(|py| { - let cls = py.get_type_bound::(); + let cls = py.get_type::(); pyo3::py_run!(py, cls, "assert repr(cls.Answer) == '42'") }) ``` @@ -1096,7 +1175,8 @@ Enums and their variants can also be renamed using `#[pyo3(name)]`. ```rust # use pyo3::prelude::*; -#[pyclass(name = "RenamedEnum")] +#[pyclass(eq, eq_int, name = "RenamedEnum")] +#[derive(PartialEq)] enum MyEnum { #[pyo3(name = "UPPERCASE")] Variant, @@ -1104,7 +1184,7 @@ enum MyEnum { Python::with_gil(|py| { let x = Py::new(py, MyEnum::Variant).unwrap(); - let cls = py.get_type_bound::(); + let cls = py.get_type::(); pyo3::py_run!(py, x cls, r#" assert repr(x) == 'RenamedEnum.UPPERCASE' assert x == cls.UPPERCASE @@ -1112,6 +1192,32 @@ Python::with_gil(|py| { }) ``` +Ordering of enum variants is optionally added using `#[pyo3(ord)]`. +*Note: Implementation of the `PartialOrd` trait is required when passing the `ord` argument. If not implemented, a compile time error is raised.* + +```rust +# use pyo3::prelude::*; +#[pyclass(eq, ord)] +#[derive(PartialEq, PartialOrd)] +enum MyEnum{ + A, + B, + C, +} + +Python::with_gil(|py| { + let cls = py.get_type::(); + let a = Py::new(py, MyEnum::A).unwrap(); + let b = Py::new(py, MyEnum::B).unwrap(); + let c = Py::new(py, MyEnum::C).unwrap(); + pyo3::py_run!(py, cls a b c, r#" + assert (a < b) == True + assert (c <= b) == False + assert (c > a) == True + "#) +}) +``` + You may not use enums as a base class or let enums inherit from other classes. ```rust,compile_fail @@ -1140,7 +1246,7 @@ enum BadSubclass { An enum is complex if it has any non-unit (struct or tuple) variants. -Currently PyO3 supports only struct variants in a complex enum. Support for unit and tuple variants is planned. +PyO3 supports only struct and tuple variants in a complex enum. Unit variants aren't supported at present (the recommendation is to use an empty tuple enum instead). PyO3 adds a class attribute for each variant, which may be used to construct values and in match patterns. PyO3 also provides getter methods for all fields of each variant. @@ -1150,15 +1256,15 @@ PyO3 adds a class attribute for each variant, which may be used to construct val enum Shape { Circle { radius: f64 }, Rectangle { width: f64, height: f64 }, - RegularPolygon { side_count: u32, radius: f64 }, + RegularPolygon(u32, f64), Nothing { }, } # #[cfg(Py_3_10)] Python::with_gil(|py| { - let circle = Shape::Circle { radius: 10.0 }.into_py(py); - let square = Shape::RegularPolygon { side_count: 4, radius: 10.0 }.into_py(py); - let cls = py.get_type_bound::(); + let circle = Shape::Circle { radius: 10.0 }.into_pyobject(py)?; + let square = Shape::RegularPolygon(4, 10.0).into_pyobject(py)?; + let cls = py.get_type::(); pyo3::py_run!(py, circle square cls, r#" assert isinstance(circle, cls) assert isinstance(circle, cls.Circle) @@ -1166,8 +1272,8 @@ Python::with_gil(|py| { assert isinstance(square, cls) assert isinstance(square, cls.RegularPolygon) - assert square.side_count == 4 - assert square.radius == 10.0 + assert square[0] == 4 # Gets _0 field + assert square[1] == 10.0 # Gets _1 field def count_vertices(cls, shape): match shape: @@ -1175,18 +1281,20 @@ Python::with_gil(|py| { return 0 case cls.Rectangle(): return 4 - case cls.RegularPolygon(side_count=n): + case cls.RegularPolygon(n): return n case cls.Nothing(): return 0 assert count_vertices(cls, circle) == 0 assert count_vertices(cls, square) == 4 - "#) + "#); +# Ok::<_, PyErr>(()) }) +# .unwrap(); ``` -WARNING: `Py::new` and `.into_py` are currently inconsistent. Note how the constructed value is _not_ an instance of the specific variant. For this reason, constructing values is only recommended using `.into_py`. +WARNING: `Py::new` and `.into_pyobject` are currently inconsistent. Note how the constructed value is _not_ an instance of the specific variant. For this reason, constructing values is only recommended using `.into_pyobject`. ```rust # use pyo3::prelude::*; @@ -1197,7 +1305,7 @@ enum MyEnum { Python::with_gil(|py| { let x = Py::new(py, MyEnum::Variant { i: 42 }).unwrap(); - let cls = py.get_type_bound::(); + let cls = py.get_type::(); pyo3::py_run!(py, x cls, r#" assert isinstance(x, cls) assert not isinstance(x, cls.Variant) @@ -1205,6 +1313,46 @@ Python::with_gil(|py| { }) ``` +The constructor of each generated class can be customized using the `#[pyo3(constructor = (...))]` attribute. This uses the same syntax as the [`#[pyo3(signature = (...))]`](function/signature.md) +attribute on function and methods and supports the same options. To apply this attribute simply place it on top of a variant in a `#[pyclass]` complex enum as shown below: + +```rust +# use pyo3::prelude::*; +#[pyclass] +enum Shape { + #[pyo3(constructor = (radius=1.0))] + Circle { radius: f64 }, + #[pyo3(constructor = (*, width, height))] + Rectangle { width: f64, height: f64 }, + #[pyo3(constructor = (side_count, radius=1.0))] + RegularPolygon { side_count: u32, radius: f64 }, + Nothing { }, +} + +# #[cfg(Py_3_10)] +Python::with_gil(|py| { + let cls = py.get_type::(); + pyo3::py_run!(py, cls, r#" + circle = cls.Circle() + assert isinstance(circle, cls) + assert isinstance(circle, cls.Circle) + assert circle.radius == 1.0 + + square = cls.Rectangle(width = 1, height = 1) + assert isinstance(square, cls) + assert isinstance(square, cls.Rectangle) + assert square.width == 1 + assert square.height == 1 + + hexagon = cls.RegularPolygon(6) + assert isinstance(hexagon, cls) + assert isinstance(hexagon, cls.RegularPolygon) + assert hexagon.side_count == 6 + assert hexagon.radius == 1 + "#) +}) +``` + ## Implementation details The `#[pyclass]` macros rely on a lot of conditional code generation: each `#[pyclass]` can optionally have a `#[pymethods]` block. @@ -1219,6 +1367,7 @@ The `#[pyclass]` macro expands to roughly the code seen below. The `PyClassImplC # #[cfg(not(feature = "multiple-pymethods"))] { # use pyo3::prelude::*; // Note: the implementation differs slightly with the `multiple-pymethods` feature enabled. +# #[allow(dead_code)] struct MyClass { # #[allow(dead_code)] num: i32, @@ -1226,10 +1375,6 @@ struct MyClass { impl pyo3::types::DerefToPyAny for MyClass {} -# #[allow(deprecated)] -unsafe impl pyo3::type_object::HasPyGilRef for MyClass { - type AsRefTarget = pyo3::PyCell; -} unsafe impl pyo3::type_object::PyTypeInfo for MyClass { const NAME: &'static str = "MyClass"; const MODULE: ::std::option::Option<&'static str> = ::std::option::Option::None; @@ -1245,7 +1390,7 @@ impl pyo3::PyClass for MyClass { type Frozen = pyo3::pyclass::boolean_struct::False; } -impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a MyClass +impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py, false> for &'a MyClass { type Holder = ::std::option::Option>; @@ -1255,7 +1400,7 @@ impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a } } -impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a mut MyClass +impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py, false> for &'a mut MyClass { type Holder = ::std::option::Option>; @@ -1265,6 +1410,7 @@ impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a } } +#[allow(deprecated)] impl pyo3::IntoPy for MyClass { fn into_py(self, py: pyo3::Python<'_>) -> pyo3::PyObject { pyo3::IntoPy::into_py(pyo3::Py::new(py, self).unwrap(), py) @@ -1301,13 +1447,13 @@ impl pyo3::impl_::pyclass::PyClassImpl for MyClass { static DOC: pyo3::sync::GILOnceCell<::std::borrow::Cow<'static, ::std::ffi::CStr>> = pyo3::sync::GILOnceCell::new(); DOC.get_or_try_init(py, || { let collector = PyClassImplCollector::::new(); - build_pyclass_doc(::NAME, "\0", collector.new_text_signature()) + build_pyclass_doc(::NAME, pyo3::ffi::c_str!(""), collector.new_text_signature()) }).map(::std::ops::Deref::deref) } } # Python::with_gil(|py| { -# let cls = py.get_type_bound::(); +# let cls = py.get_type::(); # pyo3::py_run!(py, cls, "assert cls.__name__ == 'MyClass'") # }); # } @@ -1329,3 +1475,6 @@ impl pyo3::impl_::pyclass::PyClassImpl for MyClass { [classattr]: https://docs.python.org/3/tutorial/classes.html#class-and-instance-variables [`multiple-pymethods`]: features.md#multiple-pymethods + +[lifetime-elision]: https://doc.rust-lang.org/reference/lifetime-elision.html +[compiler-error-e0106]: https://doc.rust-lang.org/error_codes/E0106.html diff --git a/guide/src/class/call.md b/guide/src/class/call.md index 0890df9561a..5242a4f41a8 100644 --- a/guide/src/class/call.md +++ b/guide/src/class/call.md @@ -66,7 +66,7 @@ def Counter(wraps): return call ``` -### What is the `Cell` for? +### What is the `AtomicU64` for? A [previous implementation] used a normal `u64`, which meant it required a `&mut self` receiver to update the count: @@ -108,14 +108,15 @@ say_hello() # RuntimeError: Already borrowed ``` -The implementation in this chapter fixes that by never borrowing exclusively; all the methods take `&self` as receivers, of which multiple may exist simultaneously. This requires a shared counter and the easiest way to do that is to use [`Cell`], so that's what is used here. +The implementation in this chapter fixes that by never borrowing exclusively; all the methods take `&self` as receivers, of which multiple may exist simultaneously. This requires a shared counter and the most straightforward way to implement thread-safe interior mutability (e.g. the type does not need to accept `&mut self` to modify the "interior" state) for a `u64` is to use [`AtomicU64`], so that's what is used here. This shows the dangers of running arbitrary Python code - note that "running arbitrary Python code" can be far more subtle than the example above: - Python's asynchronous executor may park the current thread in the middle of Python code, even in Python code that *you* control, and let other Python code run. - Dropping arbitrary Python objects may invoke destructors defined in Python (`__del__` methods). - Calling Python's C-api (most PyO3 apis call C-api functions internally) may raise exceptions, which may allow Python code in signal handlers to run. +- On the free-threaded build, users might use Python's `threading` module to work with your types simultaneously from multiple OS threads. This is especially important if you are writing unsafe code; Python code must never be able to cause undefined behavior. You must ensure that your Rust code is in a consistent state before doing any of the above things. [previous implementation]: https://github.com/PyO3/pyo3/discussions/2598 "Thread Safe Decorator · Discussion #2598 · PyO3/pyo3" -[`Cell`]: https://doc.rust-lang.org/std/cell/struct.Cell.html "Cell in std::cell - Rust" +[`AtomicU64`]: https://doc.rust-lang.org/std/sync/atomic/struct.AtomicU64.html "AtomicU64 in std::sync::atomic - Rust" diff --git a/guide/src/class/numeric.md b/guide/src/class/numeric.md index 361d2fb6d36..4f73a44adab 100644 --- a/guide/src/class/numeric.md +++ b/guide/src/class/numeric.md @@ -27,7 +27,7 @@ OverflowError: Python int too large to convert to C long ``` Instead of relying on the default [`FromPyObject`] extraction to parse arguments, we can specify our -own extraction function, using the `#[pyo3(from_py_with = "...")]` attribute. Unfortunately PyO3 +own extraction function, using the `#[pyo3(from_py_with = ...)]` attribute. Unfortunately PyO3 doesn't provide a way to wrap Python integers out of the box, but we can do a Python call to mask it and cast it to an `i32`. @@ -62,7 +62,7 @@ struct Number(i32); #[pymethods] impl Number { #[new] - fn new(#[pyo3(from_py_with = "wrap")] value: i32) -> Self { + fn new(#[pyo3(from_py_with = wrap)] value: i32) -> Self { Self(value) } } @@ -171,7 +171,7 @@ impl Number { } fn __complex__<'py>(&self, py: Python<'py>) -> Bound<'py, PyComplex> { - PyComplex::from_doubles_bound(py, self.0 as f64, 0.0) + PyComplex::from_doubles(py, self.0 as f64, 0.0) } } ``` @@ -210,7 +210,7 @@ use std::hash::{Hash, Hasher}; use pyo3::exceptions::{PyValueError, PyZeroDivisionError}; use pyo3::prelude::*; use pyo3::class::basic::CompareOp; -use pyo3::types::PyComplex; +use pyo3::types::{PyComplex, PyString}; fn wrap(obj: &Bound<'_, PyAny>) -> PyResult { let val = obj.call_method1("__and__", (0xFFFFFFFF_u32,))?; @@ -225,13 +225,13 @@ struct Number(i32); #[pymethods] impl Number { #[new] - fn new(#[pyo3(from_py_with = "wrap")] value: i32) -> Self { + fn new(#[pyo3(from_py_with = wrap)] value: i32) -> Self { Self(value) } fn __repr__(slf: &Bound<'_, Self>) -> PyResult { // Get the class name dynamically in case `Number` is subclassed - let class_name: String = slf.get_type().qualname()?; + let class_name: Bound<'_, PyString> = slf.get_type().qualname()?; Ok(format!("{}({})", class_name, slf.borrow().0)) } @@ -321,7 +321,7 @@ impl Number { } fn __complex__<'py>(&self, py: Python<'py>) -> Bound<'py, PyComplex> { - PyComplex::from_doubles_bound(py, self.0 as f64, 0.0) + PyComplex::from_doubles(py, self.0 as f64, 0.0) } } @@ -330,7 +330,7 @@ fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } -# const SCRIPT: &'static str = r#" +# const SCRIPT: &'static std::ffi::CStr = pyo3::ffi::c_str!(r#" # def hash_djb2(s: str): # n = Number(0) # five = Number(5) @@ -379,17 +379,17 @@ fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { # pass # assert Number(1337).__str__() == '1337' # assert Number(1337).__repr__() == 'Number(1337)' -"#; +"#); # # use pyo3::PyTypeInfo; # # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let globals = PyModule::import_bound(py, "__main__")?.dict(); -# globals.set_item("Number", Number::type_object_bound(py))?; +# let globals = PyModule::import(py, "__main__")?.dict(); +# globals.set_item("Number", Number::type_object(py))?; # -# py.run_bound(SCRIPT, Some(&globals), None)?; +# py.run(SCRIPT, Some(&globals), None)?; # Ok(()) # }) # } diff --git a/guide/src/class/object.md b/guide/src/class/object.md index db6cc7d3234..07f445aac60 100644 --- a/guide/src/class/object.md +++ b/guide/src/class/object.md @@ -70,6 +70,48 @@ impl Number { } ``` +To automatically generate the `__str__` implementation using a `Display` trait implementation, pass the `str` argument to `pyclass`. + +```rust +# use std::fmt::{Display, Formatter}; +# use pyo3::prelude::*; +# +# #[allow(dead_code)] +# #[pyclass(str)] +# struct Coordinate { + x: i32, + y: i32, + z: i32, +} + +impl Display for Coordinate { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "({}, {}, {})", self.x, self.y, self.z) + } +} +``` + +For convenience, a shorthand format string can be passed to `str` as `str=""` for **structs only**. It expands and is passed into the `format!` macro in the following ways: + +* `"{x}"` -> `"{}", self.x` +* `"{0}"` -> `"{}", self.0` +* `"{x:?}"` -> `"{:?}", self.x` + +*Note: Depending upon the format string you use, this may require implementation of the `Display` or `Debug` traits for the given Rust types.* +*Note: the pyclass args `name` and `rename_all` are incompatible with the shorthand format string and will raise a compile time error.* + +```rust +# use pyo3::prelude::*; +# +# #[allow(dead_code)] +# #[pyclass(str="({x}, {y}, {z})")] +# struct Coordinate { + x: i32, + y: i32, + z: i32, +} +``` + #### Accessing the class name In the `__repr__`, we used a hard-coded class name. This is sometimes not ideal, @@ -80,7 +122,9 @@ the subclass name. This is typically done in Python code by accessing ```rust # use pyo3::prelude::*; +# use pyo3::types::PyString; # +# #[allow(dead_code)] # #[pyclass] # struct Number(i32); # @@ -88,7 +132,7 @@ the subclass name. This is typically done in Python code by accessing impl Number { fn __repr__(slf: &Bound<'_, Self>) -> PyResult { // This is the equivalent of `self.__class__.__name__` in Python. - let class_name: String = slf.get_type().qualname()?; + let class_name: Bound<'_, PyString> = slf.get_type().qualname()?; // To access fields of the Rust struct, we need to borrow the `PyCell`. Ok(format!("{}({})", class_name, slf.borrow().0)) } @@ -109,6 +153,7 @@ use std::hash::{Hash, Hasher}; # use pyo3::prelude::*; # +# #[allow(dead_code)] # #[pyclass] # struct Number(i32); # @@ -121,6 +166,20 @@ impl Number { } } ``` +To implement `__hash__` using the Rust [`Hash`] trait implementation, the `hash` option can be used. +This option is only available for `frozen` classes to prevent accidental hash changes from mutating the object. If you need +an `__hash__` implementation for a mutable class, use the manual method from above. This option also requires `eq`: According to the +[Python docs](https://docs.python.org/3/reference/datamodel.html#object.__hash__) "If a class does not define an `__eq__()` +method it should not define a `__hash__()` operation either" +```rust +# use pyo3::prelude::*; +# +# #[allow(dead_code)] +#[pyclass(frozen, eq, hash)] +#[derive(PartialEq, Hash)] +struct Number(i32); +``` + > **Note**: When implementing `__hash__` and comparisons, it is important that the following property holds: > @@ -159,6 +218,7 @@ use pyo3::class::basic::CompareOp; # use pyo3::prelude::*; # +# #[allow(dead_code)] # #[pyclass] # struct Number(i32); # @@ -185,6 +245,7 @@ use pyo3::class::basic::CompareOp; # use pyo3::prelude::*; # +# #[allow(dead_code)] # #[pyclass] # struct Number(i32); # @@ -226,6 +287,28 @@ impl Number { # } ``` +To implement `__eq__` using the Rust [`PartialEq`] trait implementation, the `eq` option can be used. + +```rust +# use pyo3::prelude::*; +# +# #[allow(dead_code)] +#[pyclass(eq)] +#[derive(PartialEq)] +struct Number(i32); +``` + +To implement `__lt__`, `__le__`, `__gt__`, & `__ge__` using the Rust `PartialOrd` trait implementation, the `ord` option can be used. *Note: Requires `eq`.* + +```rust +# use pyo3::prelude::*; +# +# #[allow(dead_code)] +#[pyclass(eq, ord)] +#[derive(PartialEq, PartialOrd)] +struct Number(i32); +``` + ### Truthyness We'll consider `Number` to be `True` if it is nonzero: @@ -233,6 +316,7 @@ We'll consider `Number` to be `True` if it is nonzero: ```rust # use pyo3::prelude::*; # +# #[allow(dead_code)] # #[pyclass] # struct Number(i32); # @@ -252,6 +336,7 @@ use std::hash::{Hash, Hasher}; use pyo3::prelude::*; use pyo3::class::basic::CompareOp; +use pyo3::types::PyString; #[pyclass] struct Number(i32); @@ -264,7 +349,7 @@ impl Number { } fn __repr__(slf: &Bound<'_, Self>) -> PyResult { - let class_name: String = slf.get_type().qualname()?; + let class_name: Bound<'_, PyString> = slf.get_type().qualname()?; Ok(format!("{}({})", class_name, slf.borrow().0)) } @@ -305,3 +390,4 @@ fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { [`Hasher`]: https://doc.rust-lang.org/std/hash/trait.Hasher.html [`DefaultHasher`]: https://doc.rust-lang.org/std/collections/hash_map/struct.DefaultHasher.html [SipHash]: https://en.wikipedia.org/wiki/SipHash +[`PartialEq`]: https://doc.rust-lang.org/stable/std/cmp/trait.PartialEq.html diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md index 3b12fd531c3..8a361a1442e 100644 --- a/guide/src/class/protocols.md +++ b/guide/src/class/protocols.md @@ -1,20 +1,27 @@ -# Magic methods and slots +# Class customizations -Python's object model defines several protocols for different object behavior, such as the sequence, mapping, and number protocols. You may be familiar with implementing these protocols in Python classes by "magic" methods, such as `__str__` or `__repr__`. Because of the double-underscores surrounding their name, these are also known as "dunder" methods. +Python's object model defines several protocols for different object behavior, such as the sequence, mapping, and number protocols. Python classes support these protocols by implementing "magic" methods, such as `__str__` or `__repr__`. Because of the double-underscores surrounding their name, these are also known as "dunder" methods. -In the Python C-API which PyO3 is implemented upon, many of these magic methods have to be placed into special "slots" on the class type object, as covered in the previous section. +PyO3 makes it possible for every magic method to be implemented in `#[pymethods]` just as they would be done in a regular Python class, with a few notable differences: +- `__new__` and `__init__` are replaced by the [`#[new]` attribute](../class.md#constructor). +- `__del__` is not yet supported, but may be in the future. +- `__buffer__` and `__release_buffer__` are currently not supported and instead PyO3 supports [`__getbuffer__` and `__releasebuffer__`](#buffer-objects) methods (these predate [PEP 688](https://peps.python.org/pep-0688/#python-level-buffer-protocol)), again this may change in the future. +- PyO3 adds [`__traverse__` and `__clear__`](#garbage-collector-integration) methods for controlling garbage collection. +- The Python C-API which PyO3 is implemented upon requires many magic methods to have a specific function signature in C and be placed into special "slots" on the class type object. This limits the allowed argument and return types for these methods. They are listed in detail in the section below. -If a function name in `#[pymethods]` is a recognised magic method, it will be automatically placed into the correct slot in the Python type object. The function name is taken from the usual rules for naming `#[pymethods]`: the `#[pyo3(name = "...")]` attribute is used if present, otherwise the Rust function name is used. +If a magic method is not on the list above (for example `__init_subclass__`), then it should just work in PyO3. If this is not the case, please file a bug report. -The magic methods handled by PyO3 are very similar to the standard Python ones on [this page](https://docs.python.org/3/reference/datamodel.html#special-method-names) - in particular they are the subset which have slots as [defined here](https://docs.python.org/3/c-api/typeobj.html). Some of the slots do not have a magic method in Python, which leads to a few additional magic methods defined only in PyO3: - - Magic methods for garbage collection - - Magic methods for the buffer protocol +## Magic Methods handled by PyO3 + +If a function name in `#[pymethods]` is a magic method which is known to need special handling, it will be automatically placed into the correct slot in the Python type object. The function name is taken from the usual rules for naming `#[pymethods]`: the `#[pyo3(name = "...")]` attribute is used if present, otherwise the Rust function name is used. + +The magic methods handled by PyO3 are very similar to the standard Python ones on [this page](https://docs.python.org/3/reference/datamodel.html#special-method-names) - in particular they are the subset which have slots as [defined here](https://docs.python.org/3/c-api/typeobj.html). When PyO3 handles a magic method, a couple of changes apply compared to other `#[pymethods]`: - The Rust function signature is restricted to match the magic method. - The `#[pyo3(signature = (...)]` and `#[pyo3(text_signature = "...")]` attributes are not allowed. -The following sections list of all magic methods PyO3 currently handles. The +The following sections list all magic methods for which PyO3 implements the necessary special handling. The given signatures should be interpreted as follows: - All methods take a receiver as first argument, shown as ``. It can be `&self`, `&mut self` or a `Bound` reference like `self_: PyRef<'_, Self>` and @@ -31,7 +38,6 @@ given signatures should be interpreted as follows: checked by the Python interpreter. For example, `__str__` needs to return a string object. This is indicated by `object (Python type)`. - ### Basic object customization - `__str__() -> object (str)` @@ -91,19 +97,21 @@ given signatures should be interpreted as follows: ```rust use pyo3::class::basic::CompareOp; + use pyo3::types::PyNotImplemented; # use pyo3::prelude::*; + # use pyo3::BoundObject; # # #[pyclass] # struct Number(i32); # #[pymethods] impl Number { - fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> PyObject { + fn __richcmp__<'py>(&self, other: &Self, op: CompareOp, py: Python<'py>) -> PyResult> { match op { - CompareOp::Eq => (self.0 == other.0).into_py(py), - CompareOp::Ne => (self.0 != other.0).into_py(py), - _ => py.NotImplemented(), + CompareOp::Eq => Ok((self.0 == other.0).into_pyobject(py)?.into_any()), + CompareOp::Ne => Ok((self.0 != other.0).into_pyobject(py)?.into_any()), + _ => Ok(PyNotImplemented::get(py).into_any()), } } } @@ -150,9 +158,11 @@ Example: ```rust use pyo3::prelude::*; +use std::sync::Mutex; + #[pyclass] struct MyIterator { - iter: Box + Send>, + iter: Mutex + Send>>, } #[pymethods] @@ -160,8 +170,8 @@ impl MyIterator { fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } - fn __next__(mut slf: PyRefMut<'_, Self>) -> Option { - slf.iter.next() + fn __next__(slf: PyRefMut<'_, Self>) -> Option { + slf.iter.lock().unwrap().next() } } ``` @@ -219,15 +229,15 @@ documentation](https://docs.python.org/library/stdtypes.html#iterator-types). #### Returning a value from iteration This guide has so far shown how to use `Option` to implement yielding values -during iteration. In Python a generator can also return a value. To express -this in Rust, PyO3 provides the [`IterNextOutput`] enum to both `Yield` values -and `Return` a final value - see its docs for further details and an example. +during iteration. In Python a generator can also return a value. This is done by +raising a `StopIteration` exception. To express this in Rust, return `PyResult::Err` +with a `PyStopIteration` as the error. ### Awaitable objects - `__await__() -> object` - `__aiter__() -> object` - - `__anext__() -> Option or IterANextOutput` + - `__anext__() -> Option` ### Mapping & Sequence types @@ -452,6 +462,5 @@ i.e. `Python::with_gil` will panic. > Note: these methods are part of the C API, PyPy does not necessarily honor them. If you are building for PyPy you should measure memory consumption to make sure you do not have runaway memory growth. See [this issue on the PyPy bug tracker](https://github.com/pypy/pypy/issues/3848). -[`IterNextOutput`]: {{#PYO3_DOCS_URL}}/pyo3/pyclass/enum.IterNextOutput.html [`PySequence`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PySequence.html [`CompareOp::matches`]: {{#PYO3_DOCS_URL}}/pyo3/pyclass/enum.CompareOp.html#method.matches diff --git a/guide/src/class/thread-safety.md b/guide/src/class/thread-safety.md new file mode 100644 index 00000000000..55c2a3caca8 --- /dev/null +++ b/guide/src/class/thread-safety.md @@ -0,0 +1,110 @@ +# `#[pyclass]` thread safety + +Python objects are freely shared between threads by the Python interpreter. This means that: +- there is no control which thread might eventually drop the `#[pyclass]` object, meaning `Send` is required. +- multiple threads can potentially be reading the `#[pyclass]` data simultaneously, meaning `Sync` is required. + +This section of the guide discusses various data structures which can be used to make types satisfy these requirements. + +In special cases where it is known that your Python application is never going to use threads (this is rare!), these thread-safety requirements can be opted-out with [`#[pyclass(unsendable)]`](../class.md#customizing-the-class), at the cost of making concurrent access to the Rust data be runtime errors. This is only for very specific use cases; it is almost always better to make proper thread-safe types. + +## Making `#[pyclass]` types thread-safe + +The general challenge with thread-safety is to make sure that two threads cannot produce a data race, i.e. unsynchronized writes to the same data at the same time. A data race produces an unpredictable result and is forbidden by Rust. + +By default, `#[pyclass]` employs an ["interior mutability" pattern](../class.md#bound-and-interior-mutability) to allow for either multiple `&T` references or a single exclusive `&mut T` reference to access the data. This allows for simple `#[pyclass]` types to be thread-safe automatically, at the cost of runtime checking for concurrent access. Errors will be raised if the usage overlaps. + +For example, the below simple class is thread-safe: + +```rust +# use pyo3::prelude::*; + +#[pyclass] +struct MyClass { + x: i32, + y: i32, +} + +#[pymethods] +impl MyClass { + fn get_x(&self) -> i32 { + self.x + } + + fn set_y(&mut self, value: i32) { + self.y = value; + } +} +``` + +In the above example, if calls to `get_x` and `set_y` overlap (from two different threads) then at least one of those threads will experience a runtime error indicating that the data was "already borrowed". + +To avoid these errors, you can take control of the interior mutability yourself in one of the following ways. + +### Using atomic data structures + +To remove the possibility of having overlapping `&self` and `&mut self` references produce runtime errors, consider using `#[pyclass(frozen)]` and use [atomic data structures](https://doc.rust-lang.org/std/sync/atomic/) to control modifications directly. + +For example, a thread-safe version of the above `MyClass` using atomic integers would be as follows: + +```rust +# use pyo3::prelude::*; +use std::sync::atomic::{AtomicI32, Ordering}; + +#[pyclass(frozen)] +struct MyClass { + x: AtomicI32, + y: AtomicI32, +} + +#[pymethods] +impl MyClass { + fn get_x(&self) -> i32 { + self.x.load(Ordering::Relaxed) + } + + fn set_y(&self, value: i32) { + self.y.store(value, Ordering::Relaxed) + } +} +``` + +### Using locks + +An alternative to atomic data structures is to use [locks](https://doc.rust-lang.org/std/sync/struct.Mutex.html) to make threads wait for access to shared data. + +For example, a thread-safe version of the above `MyClass` using locks would be as follows: + +```rust +# use pyo3::prelude::*; +use std::sync::Mutex; + +struct MyClassInner { + x: i32, + y: i32, +} + +#[pyclass(frozen)] +struct MyClass { + inner: Mutex +} + +#[pymethods] +impl MyClass { + fn get_x(&self) -> i32 { + self.inner.lock().expect("lock not poisoned").x + } + + fn set_y(&self, value: i32) { + self.inner.lock().expect("lock not poisoned").y = value; + } +} +``` + +If you need to lock around state stored in the Python interpreter or otherwise call into the Python C API while a lock is held, you might find the `MutexExt` trait useful. It provides a `lock_py_attached` method for `std::sync::Mutex` that avoids deadlocks with the GIL or other global synchronization events in the interpreter. + +### Wrapping unsynchronized data + +In some cases, the data structures stored within a `#[pyclass]` may themselves not be thread-safe. Rust will therefore not implement `Send` and `Sync` on the `#[pyclass]` type. + +To achieve thread-safety, a manual `Send` and `Sync` implementation is required which is `unsafe` and should only be done following careful review of the soundness of the implementation. Doing this for PyO3 types is no different than for any other Rust code, [the Rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html) has a great discussion on this. \ No newline at end of file diff --git a/guide/src/conversions.md b/guide/src/conversions.md index 991c2061042..ee8dfddebb0 100644 --- a/guide/src/conversions.md +++ b/guide/src/conversions.md @@ -1,3 +1,5 @@ # Type conversions In this portion of the guide we'll talk about the mapping of Python types to Rust types offered by PyO3, as well as the traits available to perform conversions between them. + +See also the conversion [tables](conversions/tables.md) and [traits](conversions/traits.md). diff --git a/guide/src/conversions/tables.md b/guide/src/conversions/tables.md index eb33b17acf7..531e48b8f73 100644 --- a/guide/src/conversions/tables.md +++ b/guide/src/conversions/tables.md @@ -13,12 +13,13 @@ The table below contains the Python type and the corresponding function argument | Python | Rust | Rust (Python-native) | | ------------- |:-------------------------------:|:--------------------:| | `object` | - | `PyAny` | -| `str` | `String`, `Cow`, `&str`, `char`, `OsString`, `PathBuf`, `Path` | `PyString`, `PyUnicode` | +| `str` | `String`, `Cow`, `&str`, `char`, `OsString`, `PathBuf`, `Path` | `PyString` | | `bytes` | `Vec`, `&[u8]`, `Cow<[u8]>` | `PyBytes` | | `bool` | `bool` | `PyBool` | -| `int` | `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize`, `num_bigint::BigInt`[^1], `num_bigint::BigUint`[^1] | `PyLong` | +| `int` | `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize`, `num_bigint::BigInt`[^1], `num_bigint::BigUint`[^1] | `PyInt` | | `float` | `f32`, `f64` | `PyFloat` | | `complex` | `num_complex::Complex`[^2] | `PyComplex` | +| `fractions.Fraction`| `num_rational::Ratio`[^8] | - | | `list[T]` | `Vec` | `PyList` | | `dict[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `PyDict` | | `tuple[T, U]` | `(T, U)`, `Vec` | `PyTuple` | @@ -37,8 +38,8 @@ The table below contains the Python type and the corresponding function argument | `decimal.Decimal` | `rust_decimal::Decimal`[^7] | - | | `ipaddress.IPv4Address` | `std::net::IpAddr`, `std::net::IpV4Addr` | - | | `ipaddress.IPv6Address` | `std::net::IpAddr`, `std::net::IpV6Addr` | - | -| `os.PathLike ` | `PathBuf`, `Path` | `PyString`, `PyUnicode` | -| `pathlib.Path` | `PathBuf`, `Path` | `PyString`, `PyUnicode` | +| `os.PathLike ` | `PathBuf`, `Path` | `PyString` | +| `pathlib.Path` | `PathBuf`, `Path` | `PyString` | | `typing.Optional[T]` | `Option` | - | | `typing.Sequence[T]` | `Vec` | `PySequence` | | `typing.Mapping[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `&PyMapping` | @@ -113,3 +114,5 @@ Finally, the following Rust types are also able to convert to Python as return v [^6]: Requires the `chrono-tz` optional feature. [^7]: Requires the `rust_decimal` optional feature. + +[^8]: Requires the `num-rational` optional feature. diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md old mode 100644 new mode 100755 index 65a5d150e79..848dc041ef7 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -13,7 +13,7 @@ fails, so usually you will use something like # use pyo3::types::PyList; # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let list = PyList::new_bound(py, b"foo"); +# let list = PyList::new(py, b"foo")?; let v: Vec = list.extract()?; # assert_eq!(&v, &[102, 111, 111]); # Ok(()) @@ -46,6 +46,7 @@ the Python object, i.e. `obj.getattr("my_string")`, and call `extract()` on the ```rust use pyo3::prelude::*; +use pyo3_ffi::c_str; #[derive(FromPyObject)] struct RustyStruct { @@ -54,13 +55,13 @@ struct RustyStruct { # # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let module = PyModule::from_code_bound( +# let module = PyModule::from_code( # py, -# "class Foo: +# c_str!("class Foo: # def __init__(self): -# self.my_string = 'test'", -# "", -# "", +# self.my_string = 'test'"), +# c_str!(""), +# c_str!(""), # )?; # # let class = module.getattr("Foo")?; @@ -86,7 +87,7 @@ struct RustyStruct { # use pyo3::types::PyDict; # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let dict = PyDict::new_bound(py); +# let dict = PyDict::new(py); # dict.set_item("my_string", "test")?; # # let rustystruct: RustyStruct = dict.extract()?; @@ -100,6 +101,7 @@ The argument passed to `getattr` and `get_item` can also be configured: ```rust use pyo3::prelude::*; +use pyo3_ffi::c_str; #[derive(FromPyObject)] struct RustyStruct { @@ -111,14 +113,14 @@ struct RustyStruct { # # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let module = PyModule::from_code_bound( +# let module = PyModule::from_code( # py, -# "class Foo(dict): +# c_str!("class Foo(dict): # def __init__(self): # self.name = 'test' -# self['key'] = 'test2'", -# "", -# "", +# self['key'] = 'test2'"), +# c_str!(""), +# c_str!(""), # )?; # # let class = module.getattr("Foo")?; @@ -155,7 +157,7 @@ struct RustyStruct { # # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let py_dict = py.eval_bound("{'foo': 'foo', 'bar': 'bar', 'foobar': 'foobar'}", None, None)?; +# let py_dict = py.eval(pyo3::ffi::c_str!("{'foo': 'foo', 'bar': 'bar', 'foobar': 'foobar'}"), None, None)?; # let rustystruct: RustyStruct = py_dict.extract()?; # assert_eq!(rustystruct.foo, "foo"); # assert_eq!(rustystruct.bar, "bar"); @@ -181,7 +183,7 @@ struct RustyTuple(String, String); # use pyo3::types::PyTuple; # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let tuple = PyTuple::new_bound(py, vec!["test", "test2"]); +# let tuple = PyTuple::new(py, vec!["test", "test2"])?; # # let rustytuple: RustyTuple = tuple.extract()?; # assert_eq!(rustytuple.0, "test"); @@ -204,7 +206,7 @@ struct RustyTuple((String,)); # use pyo3::types::PyTuple; # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let tuple = PyTuple::new_bound(py, vec!["test"]); +# let tuple = PyTuple::new(py, vec!["test"])?; # # let rustytuple: RustyTuple = tuple.extract()?; # assert_eq!((rustytuple.0).0, "test"); @@ -236,7 +238,7 @@ struct RustyTransparentStruct { # use pyo3::types::PyString; # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let s = PyString::new_bound(py, "test"); +# let s = PyString::new(py, "test"); # # let tup: RustyTransparentTupleStruct = s.extract()?; # assert_eq!(tup.0, "test"); @@ -262,10 +264,11 @@ attribute can be applied to single-field-variants. ```rust use pyo3::prelude::*; +use pyo3_ffi::c_str; #[derive(FromPyObject)] # #[derive(Debug)] -enum RustyEnum<'a> { +enum RustyEnum<'py> { Int(usize), // input is a positive int String(String), // input is a string IntTuple(usize, usize), // input is a 2-tuple with positive ints @@ -284,15 +287,15 @@ enum RustyEnum<'a> { b: usize, }, #[pyo3(transparent)] - CatchAll(&'a PyAny), // This extraction never fails + CatchAll(Bound<'py, PyAny>), // This extraction never fails } # # use pyo3::types::{PyBytes, PyString}; # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { # { -# let thing = 42_u8.to_object(py); -# let rust_thing: RustyEnum<'_> = thing.extract(py)?; +# let thing = 42_u8.into_pyobject(py)?; +# let rust_thing: RustyEnum<'_> = thing.extract()?; # # assert_eq!( # 42, @@ -303,7 +306,7 @@ enum RustyEnum<'a> { # ); # } # { -# let thing = PyString::new_bound(py, "text"); +# let thing = PyString::new(py, "text"); # let rust_thing: RustyEnum<'_> = thing.extract()?; # # assert_eq!( @@ -315,8 +318,8 @@ enum RustyEnum<'a> { # ); # } # { -# let thing = (32_u8, 73_u8).to_object(py); -# let rust_thing: RustyEnum<'_> = thing.extract(py)?; +# let thing = (32_u8, 73_u8).into_pyobject(py)?; +# let rust_thing: RustyEnum<'_> = thing.extract()?; # # assert_eq!( # (32, 73), @@ -327,8 +330,8 @@ enum RustyEnum<'a> { # ); # } # { -# let thing = ("foo", 73_u8).to_object(py); -# let rust_thing: RustyEnum<'_> = thing.extract(py)?; +# let thing = ("foo", 73_u8).into_pyobject(py)?; +# let rust_thing: RustyEnum<'_> = thing.extract()?; # # assert_eq!( # (String::from("foo"), 73), @@ -339,15 +342,15 @@ enum RustyEnum<'a> { # ); # } # { -# let module = PyModule::from_code_bound( +# let module = PyModule::from_code( # py, -# "class Foo(dict): +# c_str!("class Foo(dict): # def __init__(self): # self.x = 0 # self.y = 1 -# self.z = 2", -# "", -# "", +# self.z = 2"), +# c_str!(""), +# c_str!(""), # )?; # # let class = module.getattr("Foo")?; @@ -364,14 +367,14 @@ enum RustyEnum<'a> { # } # # { -# let module = PyModule::from_code_bound( +# let module = PyModule::from_code( # py, -# "class Foo(dict): +# c_str!("class Foo(dict): # def __init__(self): # self.x = 3 -# self.y = 4", -# "", -# "", +# self.y = 4"), +# c_str!(""), +# c_str!(""), # )?; # # let class = module.getattr("Foo")?; @@ -388,13 +391,13 @@ enum RustyEnum<'a> { # } # # { -# let thing = PyBytes::new_bound(py, b"text"); +# let thing = PyBytes::new(py, b"text"); # let rust_thing: RustyEnum<'_> = thing.extract()?; # # assert_eq!( # b"text", # match rust_thing { -# RustyEnum::CatchAll(i) => i.downcast::()?.as_bytes(), +# RustyEnum::CatchAll(ref i) => i.downcast::()?.as_bytes(), # other => unreachable!("Error extracting: {:?}", other), # } # ); @@ -424,8 +427,8 @@ enum RustyEnum { # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { # { -# let thing = 42_u8.to_object(py); -# let rust_thing: RustyEnum = thing.extract(py)?; +# let thing = 42_u8.into_pyobject(py)?; +# let rust_thing: RustyEnum = thing.extract()?; # # assert_eq!( # 42, @@ -437,8 +440,8 @@ enum RustyEnum { # } # # { -# let thing = "foo".to_object(py); -# let rust_thing: RustyEnum = thing.extract(py)?; +# let thing = "foo".into_pyobject(py)?; +# let rust_thing: RustyEnum = thing.extract()?; # # assert_eq!( # "foo", @@ -450,8 +453,8 @@ enum RustyEnum { # } # # { -# let thing = b"foo".to_object(py); -# let error = thing.extract::(py).unwrap_err(); +# let thing = b"foo".into_pyobject(py)?; +# let error = thing.extract::().unwrap_err(); # assert!(error.is_instance_of::(py)); # } # @@ -473,6 +476,10 @@ If the input is neither a string nor an integer, the error message will be: - changes the name of the failed variant in the generated error message in case of failure. - e.g. `pyo3("int")` reports the variant's type as `int`. - only supported for enum variants +- `pyo3(rename_all = "...")` + - renames all attributes/item keys according to the specified renaming rule + - Possible values are: "camelCase", "kebab-case", "lowercase", "PascalCase", "SCREAMING-KEBAB-CASE", "SCREAMING_SNAKE_CASE", "snake_case", "UPPERCASE". + - fields with an explicit renaming via `attribute(...)`/`item(...)` are not affected #### `#[derive(FromPyObject)]` Field Attributes - `pyo3(attribute)`, `pyo3(attribute("name"))` @@ -481,13 +488,266 @@ If the input is neither a string nor an integer, the error message will be: - `pyo3(item)`, `pyo3(item("key"))` - retrieve the field from a mapping, possibly with the custom key specified as an argument. - can be any literal that implements `ToBorrowedObject` -- `pyo3(from_py_with = "...")` +- `pyo3(from_py_with = ...)` - apply a custom function to convert the field from Python the desired Rust type. - - the argument must be the name of the function as a string. + - the argument must be the path to the function. - the function signature must be `fn(&Bound) -> PyResult` where `T` is the Rust type of the argument. +- `pyo3(default)`, `pyo3(default = ...)` + - if the argument is set, uses the given default value. + - in this case, the argument must be a Rust expression returning a value of the desired Rust type. + - if the argument is not set, [`Default::default`](https://doc.rust-lang.org/std/default/trait.Default.html#tymethod.default) is used. + - note that the default value is only used if the field is not set. + If the field is set and the conversion function from Python to Rust fails, an exception is raised and the default value is not used. + - this attribute is only supported on named fields. + +For example, the code below applies the given conversion function on the `"value"` dict item to compute its length or fall back to the type default value (0): + +```rust +use pyo3::prelude::*; + +#[derive(FromPyObject)] +struct RustyStruct { + #[pyo3(item("value"), default, from_py_with = Bound::<'_, PyAny>::len)] + len: usize, + #[pyo3(item)] + other: usize, +} +# +# use pyo3::types::PyDict; +# fn main() -> PyResult<()> { +# Python::with_gil(|py| -> PyResult<()> { +# // Filled case +# let dict = PyDict::new(py); +# dict.set_item("value", (1,)).unwrap(); +# dict.set_item("other", 1).unwrap(); +# let result = dict.extract::()?; +# assert_eq!(result.len, 1); +# assert_eq!(result.other, 1); +# +# // Empty case +# let dict = PyDict::new(py); +# dict.set_item("other", 1).unwrap(); +# let result = dict.extract::()?; +# assert_eq!(result.len, 0); +# assert_eq!(result.other, 1); +# Ok(()) +# }) +# } +``` + +### `IntoPyObject` +The [`IntoPyObject`] trait defines the to-python conversion for a Rust type. All types in PyO3 implement this trait, +as does a `#[pyclass]` which doesn't use `extends`. + +This trait defines a single method, `into_pyobject()`, which returns a [`Result`] with `Ok` and `Err` types depending on the input value. For convenience, there is a companion [`IntoPyObjectExt`] trait which adds methods such as `into_py_any()` which converts the `Ok` and `Err` types to commonly used types (in the case of `into_py_any()`, `Py` and `PyErr` respectively). + +Occasionally you may choose to implement this for custom types which are mapped to Python types +_without_ having a unique python type. + +#### derive macro + +`IntoPyObject` can be implemented using our derive macro. Both `struct`s and `enum`s are supported. + +`struct`s will turn into a `PyDict` using the field names as keys, tuple `struct`s will turn convert +into `PyTuple` with the fields in declaration order. +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use std::collections::HashMap; +# use std::hash::Hash; + +// structs convert into `PyDict` with field names as keys +#[derive(IntoPyObject)] +struct Struct { + count: usize, + obj: Py, +} + +// tuple structs convert into `PyTuple` +// lifetimes and generics are supported, the impl will be bounded by +// `K: IntoPyObject, V: IntoPyObject` +#[derive(IntoPyObject)] +struct Tuple<'a, K: Hash + Eq, V>(&'a str, HashMap); +``` + +For structs with a single field (newtype pattern) the `#[pyo3(transparent)]` option can be used to +forward the implementation to the inner type. + + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; + +// newtype tuple structs are implicitly `transparent` +#[derive(IntoPyObject)] +struct TransparentTuple(PyObject); + +#[derive(IntoPyObject)] +#[pyo3(transparent)] +struct TransparentStruct<'py> { + inner: Bound<'py, PyAny>, // `'py` lifetime will be used as the Python lifetime +} +``` + +For `enum`s each variant is converted according to the rules for `struct`s above. + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use std::collections::HashMap; +# use std::hash::Hash; + +#[derive(IntoPyObject)] +enum Enum<'a, 'py, K: Hash + Eq, V> { // enums are supported and convert using the same + TransparentTuple(PyObject), // rules on the variants as the structs above + #[pyo3(transparent)] + TransparentStruct { inner: Bound<'py, PyAny> }, + Tuple(&'a str, HashMap), + Struct { count: usize, obj: Py } +} +``` + +Additionally `IntoPyObject` can be derived for a reference to a struct or enum using the +`IntoPyObjectRef` derive macro. All the same rules from above apply as well. + +##### `#[derive(IntoPyObject)]`/`#[derive(IntoPyObjectRef)]` Field Attributes +- `pyo3(into_py_with = ...)` + - apply a custom function to convert the field from Rust into Python. + - the argument must be the function indentifier + - the function signature must be `fn(Cow<'_, T>, Python<'py>) -> PyResult>` where `T` is the Rust type of the argument. + - `#[derive(IntoPyObject)]` will invoke the function with `Cow::Owned` + - `#[derive(IntoPyObjectRef)]` will invoke the function with `Cow::Borrowed` + + ```rust + # use pyo3::prelude::*; + # use pyo3::IntoPyObjectExt; + # use std::borrow::Cow; + #[derive(Clone)] + struct NotIntoPy(usize); + + #[derive(IntoPyObject, IntoPyObjectRef)] + struct MyStruct { + #[pyo3(into_py_with = convert)] + not_into_py: NotIntoPy, + } + + /// Convert `NotIntoPy` into Python + fn convert<'py>(not_into_py: Cow<'_, NotIntoPy>, py: Python<'py>) -> PyResult> { + not_into_py.0.into_bound_py_any(py) + } + ``` + +#### manual implementation + +If the derive macro is not suitable for your use case, `IntoPyObject` can be implemented manually as +demonstrated below. + +```rust +# use pyo3::prelude::*; +# #[allow(dead_code)] +struct MyPyObjectWrapper(PyObject); + +impl<'py> IntoPyObject<'py> for MyPyObjectWrapper { + type Target = PyAny; // the Python type + type Output = Bound<'py, Self::Target>; // in most cases this will be `Bound` + type Error = std::convert::Infallible; // the conversion error type, has to be convertable to `PyErr` + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(self.0.into_bound(py)) + } +} + +// equivalent to former `ToPyObject` implementations +impl<'a, 'py> IntoPyObject<'py> for &'a MyPyObjectWrapper { + type Target = PyAny; + type Output = Borrowed<'a, 'py, Self::Target>; // `Borrowed` can be used to optimized reference counting + type Error = std::convert::Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(self.0.bind_borrowed(py)) + } +} +``` + +#### `BoundObject` for conversions that may be `Bound` or `Borrowed` + +`IntoPyObject::into_py_object` returns either `Bound` or `Borrowed` depending on the implementation for a concrete type. For example, the `IntoPyObject` implementation for `u32` produces a `Bound<'py, PyInt>` and the `bool` implementation produces a `Borrowed<'py, 'py, PyBool>`: + +```rust +use pyo3::prelude::*; +use pyo3::IntoPyObject; +use pyo3::types::{PyBool, PyInt}; + +let ints: Vec = vec![1, 2, 3, 4]; +let bools = vec![true, false, false, true]; + +Python::with_gil(|py| { + let ints_as_pyint: Vec> = ints + .iter() + .map(|x| Ok(x.into_pyobject(py)?)) + .collect::>() + .unwrap(); + + let bools_as_pybool: Vec> = bools + .iter() + .map(|x| Ok(x.into_pyobject(py)?)) + .collect::>() + .unwrap(); +}); +``` + +In this example if we wanted to combine `ints_as_pyints` and `bools_as_pybool` into a single `Vec>` to return from the `with_gil` closure, we would have to manually convert the concrete types for the smart pointers and the python types. + +Instead, we can write a function that generically converts vectors of either integers or bools into a vector of `Py` using the [`BoundObject`] trait: + +```rust +# use pyo3::prelude::*; +# use pyo3::BoundObject; +# use pyo3::IntoPyObject; + +# let bools = vec![true, false, false, true]; +# let ints = vec![1, 2, 3, 4]; + +fn convert_to_vec_of_pyobj<'py, T>(py: Python<'py>, the_vec: Vec) -> PyResult>> +where + T: IntoPyObject<'py> + Copy +{ + the_vec.iter() + .map(|x| { + Ok( + // Note: the below is equivalent to `x.into_py_any()` + // from the `IntoPyObjectExt` trait + x.into_pyobject(py) + .map_err(Into::into)? + .into_any() + .unbind() + ) + }) + .collect() +} + +let vec_of_pyobjs: Vec> = Python::with_gil(|py| { + let mut bools_as_pyany = convert_to_vec_of_pyobj(py, bools).unwrap(); + let mut ints_as_pyany = convert_to_vec_of_pyobj(py, ints).unwrap(); + let mut result: Vec> = vec![]; + result.append(&mut bools_as_pyany); + result.append(&mut ints_as_pyany); + result +}); +``` + +In the example above we used `BoundObject::into_any` and `BoundObject::unbind` to manipulate the python types and smart pointers into the result type we wanted to produce from the function. ### `IntoPy` +
+ +⚠️ Warning: API update in progress 🛠️ + +PyO3 0.23 has introduced `IntoPyObject` as the new trait for to-python conversions. While `#[pymethods]` and `#[pyfunction]` contain a compatibility layer to allow `IntoPy` as a return type, all Python API have been migrated to use `IntoPyObject`. To migrate implement `IntoPyObject` for your type. +
+ + This trait defines the to-python conversion for a Rust type. It is usually implemented as `IntoPy`, which is the trait needed for returning a value from `#[pyfunction]` and `#[pymethods]`. @@ -502,6 +762,7 @@ use pyo3::prelude::*; # #[allow(dead_code)] struct MyPyObjectWrapper(PyObject); +#[allow(deprecated)] impl IntoPy for MyPyObjectWrapper { fn into_py(self, py: Python<'_>) -> PyObject { self.0 @@ -511,6 +772,14 @@ impl IntoPy for MyPyObjectWrapper { ### The `ToPyObject` trait +
+ +⚠️ Warning: API update in progress 🛠️ + +PyO3 0.23 has introduced `IntoPyObject` as the new trait for to-python conversions. To migrate +implement `IntoPyObject` on a reference of your type (`impl<'py> IntoPyObject<'py> for &Type { ... }`). +
+ [`ToPyObject`] is a conversion trait that allows various objects to be converted into [`PyObject`]. `IntoPy` serves the same purpose, except that it consumes `self`. @@ -518,7 +787,12 @@ same purpose, except that it consumes `self`. [`IntoPy`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.IntoPy.html [`FromPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.FromPyObject.html [`ToPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.ToPyObject.html +[`IntoPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.IntoPyObject.html +[`IntoPyObjectExt`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.IntoPyObjectExt.html [`PyObject`]: {{#PYO3_DOCS_URL}}/pyo3/type.PyObject.html [`PyRef`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRef.html [`PyRefMut`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRefMut.html +[`BoundObject`]: {{#PYO3_DOCS_URL}}/pyo3/instance/trait.BoundObject.html + +[`Result`]: https://doc.rust-lang.org/stable/std/result/enum.Result.html diff --git a/guide/src/debugging.md b/guide/src/debugging.md index 00c22631c3b..2cbf867d438 100644 --- a/guide/src/debugging.md +++ b/guide/src/debugging.md @@ -34,14 +34,362 @@ Run Valgrind with `valgrind --suppressions=valgrind-python.supp ./my-command --w The best start to investigate a crash such as an segmentation fault is a backtrace. You can set `RUST_BACKTRACE=1` as an environment variable to get the stack trace on a `panic!`. Alternatively you can use a debugger such as `gdb` to explore the issue. Rust provides a wrapper, `rust-gdb`, which has pretty-printers for inspecting Rust variables. Since PyO3 uses `cdylib` for Python shared objects, it does not receive the pretty-print debug hooks in `rust-gdb` ([rust-lang/rust#96365](https://github.com/rust-lang/rust/issues/96365)). The mentioned issue contains a workaround for enabling pretty-printers in this case. - * Link against a debug build of python as described in the previous chapter - * Run `rust-gdb ` - * Set a breakpoint (`b`) on `rust_panic` if you are investigating a `panic!` - * Enter `r` to run - * After the crash occurred, enter `bt` or `bt full` to print the stacktrace +* Link against a debug build of python as described in the previous chapter +* Run `rust-gdb ` +* Set a breakpoint (`b`) on `rust_panic` if you are investigating a `panic!` +* Enter `r` to run +* After the crash occurred, enter `bt` or `bt full` to print the stacktrace Often it is helpful to run a small piece of Python code to exercise a section of Rust. ```console rust-gdb --args python -c "import my_package; my_package.sum_to_string(1, 2)" ``` + +## Setting breakpoints in your Rust code + +One of the preferred ways by developers to debug their code is by setting breakpoints. This can be achieved in PyO3 by using a debugger like `rust-gdb` or `rust-lldb` with your Python interpreter. + +For more information about how to use both `lldb` and `gdb` you can read the [gdb to lldb command map](https://lldb.llvm.org/use/map.html) from the lldb documentation. + +### Common setup + +1. Compile your extension with debug symbols: + + ```bash + # Debug is the default for maturin, but you can explicitly ensure debug symbols with: + RUSTFLAGS="-g" maturin develop + + # For setuptools-rust users: + pip install -e . + ``` + + > **Note**: When using debuggers, make sure that `python` resolves to an actual Python binary or symlink and not a shim script. Some tools like pyenv use shim scripts which can interfere with debugging. + +### Debugger specific setup + +Depeding on your OS and your preferences you can use two different debuggers, `rust-gdb` or `rust-lldb`. + +{{#tabs }} +{{#tab name="Using rust-gdb" }} + +1. Launch rust-gdb with the Python interpreter: + + ```bash + rust-gdb --args python + ``` + +2. Once in gdb, set a breakpoint in your Rust code: + + ```bash + (gdb) break your_module.rs:42 + ``` + +3. Run your Python script that imports and uses your Rust extension: + + ```bash + # Option 1: Run an inline Python command + (gdb) run -c "import your_module; your_module.your_function()" + + # Option 2: Run a Python script + (gdb) run your_script.py + + # Option 3: Run pytest tests + (gdb) run -m pytest tests/test_something.py::TestName + ``` + +{{#endtab }} +{{#tab name="Using rust-lldb (for macOS users)" }} + +1. Start rust-lldb with Python: + + ```bash + rust-lldb -- python + ``` + +2. Set breakpoints in your Rust code: + + ```bash + (lldb) breakpoint set --file your_module.rs --line 42 + ``` + +3. Run your Python script: + + ```bash + # Option 1: Run an inline Python command + (lldb) run -c "import your_module; your_module.your_function()" + + # Option 2: Run a Python script + (lldb) run your_script.py + + # Option 3: Run pytest tests + (lldb) run -m pytest tests/test_something.py::TestName + ``` + +{{#endtab }} +{{#endtabs }} + +### Using VS Code + +VS Code with the Rust and Python extensions provides an integrated debugging experience: + +1. First, install the necessary VS Code extensions: + * [Rust Analyzer](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer) + * [CodeLLDB](https://marketplace.visualstudio.com/items?itemName=vadimcn.vscode-lldb) + * [Python](https://marketplace.visualstudio.com/items?itemName=ms-python.python) + +2. Create a `.vscode/launch.json` file with a configuration that uses the LLDB Debug Launcher: + + ```json + { + "version": "0.2.0", + "configurations": [ + { + "name": "Debug PyO3", + "type": "lldb", + "request": "attach", + "program": "${workspaceFolder}/.venv/bin/python", + "pid": "${command:pickProcess}", + "sourceLanguages": [ + "rust" + ] + }, + { + "name": "Launch Python with PyO3", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/.venv/bin/python", + "args": ["${file}"], + "cwd": "${workspaceFolder}", + "sourceLanguages": ["rust"] + }, + { + "name": "Debug PyO3 with Args", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/.venv/bin/python", + "args": ["path/to/your/script.py", "arg1", "arg2"], + "cwd": "${workspaceFolder}", + "sourceLanguages": ["rust"] + }, + { + "name": "Debug PyO3 Tests", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/.venv/bin/python", + "args": ["-m", "pytest", "tests/your_test.py::test_function", "-v"], + "cwd": "${workspaceFolder}", + "sourceLanguages": ["rust"] + } + ] + } + ``` + + This configuration supports multiple debugging scenarios: + * Attaching to a running Python process + * Launching the currently open Python file + * Running a specific script with command-line arguments + * Running pytest tests + +3. Set breakpoints in your Rust code by clicking in the gutter next to line numbers. + +4. Start debugging: + * For attaching to a running Python process: First start the process, then select the "Debug PyO3" configuration and click Start Debugging (F5). You'll be prompted to select the Python process to attach to. + * For launching a Python script: Open your Python script, select the "Launch Python with PyO3" configuration and click Start Debugging (F5). + * For running with arguments: Select "Debug PyO3 with Args" (remember to edit the configuration with your actual script path and arguments). + * For running tests: Select "Debug PyO3 Tests" (edit the test path as needed). + +5. When debugging PyO3 code: + * You can inspect Rust variables and data structures + * Use the debug console to evaluate expressions + * Step through Rust code line by line using the step controls + * Set conditional breakpoints for more complex debugging scenarios + +### Advanced Debugging Configurations + +For advanced debugging scenarios, you might want to add environment variables or enable specific Rust debug flags: + +```json +{ + "name": "Debug PyO3 with Environment", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/.venv/bin/python", + "args": ["${file}"], + "env": { + "RUST_BACKTRACE": "1", + "PYTHONPATH": "${workspaceFolder}" + }, + "sourceLanguages": ["rust"] +} +``` + +### Debugging from Jupyter Notebooks + +For Jupyter Notebooks run from VS Code, you can use the following helper functions to automate the launch configuration: + +```python +from pathlib import Path +import os +import json +import sys + + +def update_launch_json(vscode_config_file_path=None): + """Update VSCode launch.json with the correct Jupyter kernel PID. + + Args: + vscode_config_file_path (str, optional): Path to the .vscode/launch.json file. + If not provided, will use the current working directory. + """ + pid = get_jupyter_kernel_pid() + if not pid: + print("Could not determine Jupyter kernel PID.") + return + + # Determine launch.json path + if vscode_config_file_path: + launch_json_path = vscode_config_file_path + else: + launch_json_path = os.path.join(Path(os.getcwd()), ".vscode", "launch.json") + + # Get Python interpreter path + python_path = sys.executable + + # Default debugger config + debug_config = { + "version": "0.2.0", + "configurations": [ + { + "name": "Debug PyO3 (Jupyter)", + "type": "lldb", + "request": "attach", + "program": python_path, + "pid": pid, + "sourceLanguages": ["rust"], + }, + { + "name": "Launch Python with PyO3", + "type": "lldb", + "request": "launch", + "program": python_path, + "args": ["${file}"], + "cwd": "${workspaceFolder}", + "sourceLanguages": ["rust"] + } + ], + } + + # Create .vscode directory if it doesn't exist + try: + os.makedirs(os.path.dirname(launch_json_path), exist_ok=True) + + # If launch.json already exists, try to update it instead of overwriting + if os.path.exists(launch_json_path): + try: + with open(launch_json_path, "r") as f: + existing_config = json.load(f) + + # Check if our configuration already exists + config_exists = False + for config in existing_config.get("configurations", []): + if config.get("name") == "Debug PyO3 (Jupyter)": + config["pid"] = pid + config["program"] = python_path + config_exists = True + + if not config_exists: + existing_config.setdefault("configurations", []).append(debug_config["configurations"][0]) + + debug_config = existing_config + except Exception: + # If reading fails, we'll just overwrite with our new configuration + pass + + with open(launch_json_path, "w") as f: + json.dump(debug_config, f, indent=4) + print(f"Updated launch.json with PID: {pid} at {launch_json_path}") + except Exception as e: + print(f"Error updating launch.json: {e}") + + +def get_jupyter_kernel_pid(): + """Find the process ID (PID) of the running Jupyter kernel. + + Returns: + int: The process ID of the Jupyter kernel, or None if not found. + """ + # Check if we're running in a Jupyter environment + if 'ipykernel' in sys.modules: + pid = os.getpid() + print(f"Jupyter kernel PID: {pid}") + return pid + else: + print("Not running in a Jupyter environment.") + return None +``` + +To use these functions: + +1. Run the cell containing these functions in your Jupyter notebook +2. Run `update_launch_json()` in a cell +3. In VS Code, select the "Debug PyO3 (Jupyter)" configuration and start debugging + + +## Thread Safety and Compiler Sanitizers + +PyO3 attempts to match the Rust language-level guarantees for thread safety, but +that does not preclude other code outside of the control of PyO3 or buggy code +managed by a PyO3 extension from creating a thread safety issue. Analyzing +whether or not a piece of Rust code that uses the CPython C API is thread safe +can be quite complicated, since many Python operations can lead to arbitrary +Python code execution. Automated ways to discover thread safety issues can often +be more fruitful than code analysis. + +[ThreadSanitizer](https://clang.llvm.org/docs/ThreadSanitizer.html) is a thread +safety checking runtime that can be used to detect data races triggered by +thread safety bugs or incorrect use of thread-unsafe data structures. While it +can only detect data races triggered by code at runtime, if it does detect +something the reports often point to exactly where the problem is happening. + +To use `ThreadSanitizer` with a library that depends on PyO3, you will need to +install a nightly Rust toolchain, along with the `rust-src` component, since you +will need to compile the Rust standard library: + +```bash +rustup install nightly +rustup override set nighty +rustup component add rust-src +``` + +You will also need a version of CPython compiled using LLVM/Clang with the same +major version of LLVM as is currently used to compile nightly Rust. As of March +2025, Rust nightly uses LLVM 20. + +The [cpython_sanity docker images](https://github.com/nascheme/cpython_sanity) +contain a development environment with a pre-compiled version of CPython 3.13 or +3.14 as well as optionally NumPy and SciPy, all compiled using LLVM 20 and +ThreadSanitizer. + +After activating a nightly Rust toolchain, you can build your project using +`ThreadSanitizer` with the following command: + +```bash +RUSTFLAGS="-Zsanitizer=thread" maturin develop -Zbuild-std --target x86_64-unknown-linux-gnu +``` + +If you are not running on an x86_64 Linux machine, you should replace +`x86_64-unknown-linux-gnu` with the [target +triple](https://doc.rust-lang.org/rustc/platform-support.html#tier-1-with-host-tools) +that is appropriate for your system. You can also replace `maturin develop` with +`cargo test` to run `cargo` tests. Note that `cargo` runs tests in a thread +pool, so `cargo` tests can be a good way to find thread safety issues. + +You can also replace `-Zsanitizer=thread` with `-Zsanitizer=address` or any of +the other sanitizers that are [supported by +Rust](https://doc.rust-lang.org/beta/unstable-book/compiler-flags/sanitizer.html). Note +that you'll need to build CPython from source with the appropriate [configure +script +flags](https://docs.python.org/3/using/configure.html#cmdoption-with-address-sanitizer) +to use the same sanitizer environment as you want to use for your Rust +code. \ No newline at end of file diff --git a/guide/src/ecosystem/async-await.md b/guide/src/ecosystem/async-await.md index 0319fa05063..9da906edeb9 100644 --- a/guide/src/ecosystem/async-await.md +++ b/guide/src/ecosystem/async-await.md @@ -3,556 +3,13 @@ *`async`/`await` support is currently being integrated in PyO3. See the [dedicated documentation](../async-await.md)* If you are working with a Python library that makes use of async functions or wish to provide -Python bindings for an async Rust library, [`pyo3-asyncio`](https://github.com/awestlake87/pyo3-asyncio) +Python bindings for an async Rust library, [`pyo3-async-runtimes`](https://github.com/PyO3/pyo3-async-runtimes) likely has the tools you need. It provides conversions between async functions in both Python and Rust and was designed with first-class support for popular Rust runtimes such as [`tokio`](https://tokio.rs/) and [`async-std`](https://async.rs/). In addition, all async Python -code runs on the default `asyncio` event loop, so `pyo3-asyncio` should work just fine with existing +code runs on the default `asyncio` event loop, so `pyo3-async-runtimes` should work just fine with existing Python libraries. -In the following sections, we'll give a general overview of `pyo3-asyncio` explaining how to call -async Python functions with PyO3, how to call async Rust functions from Python, and how to configure -your codebase to manage the runtimes of both. - -## Quickstart - -Here are some examples to get you started right away! A more detailed breakdown -of the concepts in these examples can be found in the following sections. - -### Rust Applications -Here we initialize the runtime, import Python's `asyncio` library and run the given future to completion using Python's default `EventLoop` and `async-std`. Inside the future, we convert `asyncio` sleep into a Rust future and await it. - - -```toml -# Cargo.toml dependencies -[dependencies] -pyo3 = { version = "0.14" } -pyo3-asyncio = { version = "0.14", features = ["attributes", "async-std-runtime"] } -async-std = "1.9" -``` - -```rust -//! main.rs - -use pyo3::prelude::*; - -#[pyo3_asyncio::async_std::main] -async fn main() -> PyResult<()> { - let fut = Python::with_gil(|py| { - let asyncio = py.import("asyncio")?; - // convert asyncio.sleep into a Rust Future - pyo3_asyncio::async_std::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) - })?; - - fut.await?; - - Ok(()) -} -``` - -The same application can be written to use `tokio` instead using the `#[pyo3_asyncio::tokio::main]` -attribute. - -```toml -# Cargo.toml dependencies -[dependencies] -pyo3 = { version = "0.14" } -pyo3-asyncio = { version = "0.14", features = ["attributes", "tokio-runtime"] } -tokio = "1.4" -``` - -```rust -//! main.rs - -use pyo3::prelude::*; - -#[pyo3_asyncio::tokio::main] -async fn main() -> PyResult<()> { - let fut = Python::with_gil(|py| { - let asyncio = py.import("asyncio")?; - // convert asyncio.sleep into a Rust Future - pyo3_asyncio::tokio::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) - })?; - - fut.await?; - - Ok(()) -} -``` - -More details on the usage of this library can be found in the [API docs](https://awestlake87.github.io/pyo3-asyncio/master/doc) and the primer below. - -### PyO3 Native Rust Modules - -PyO3 Asyncio can also be used to write native modules with async functions. - -Add the `[lib]` section to `Cargo.toml` to make your library a `cdylib` that Python can import. -```toml -[lib] -name = "my_async_module" -crate-type = ["cdylib"] -``` - -Make your project depend on `pyo3` with the `extension-module` feature enabled and select your -`pyo3-asyncio` runtime: - -For `async-std`: -```toml -[dependencies] -pyo3 = { version = "0.14", features = ["extension-module"] } -pyo3-asyncio = { version = "0.14", features = ["async-std-runtime"] } -async-std = "1.9" -``` - -For `tokio`: -```toml -[dependencies] -pyo3 = { version = "0.14", features = ["extension-module"] } -pyo3-asyncio = { version = "0.14", features = ["tokio-runtime"] } -tokio = "1.4" -``` - -Export an async function that makes use of `async-std`: - -```rust -//! lib.rs - -use pyo3::{prelude::*, wrap_pyfunction}; - -#[pyfunction] -fn rust_sleep(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>>> { - pyo3_asyncio::async_std::future_into_py(py, async { - async_std::task::sleep(std::time::Duration::from_secs(1)).await; - Ok(Python::with_gil(|py| py.None())) - }) -} - -#[pymodule] -fn my_async_module(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; - - Ok(()) -} -``` - -If you want to use `tokio` instead, here's what your module should look like: - -```rust -//! lib.rs - -use pyo3::{prelude::*, wrap_pyfunction}; - -#[pyfunction] -fn rust_sleep(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>>> { - pyo3_asyncio::tokio::future_into_py(py, async { - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - Ok(Python::with_gil(|py| py.None())) - }) -} - -#[pymodule] -fn my_async_module(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; - Ok(()) -} -``` - -You can build your module with maturin (see the [Using Rust in Python](https://pyo3.rs/main/#using-rust-from-python) section in the PyO3 guide for setup instructions). After that you should be able to run the Python REPL to try it out. - -```bash -maturin develop && python3 -🔗 Found pyo3 bindings -🐍 Found CPython 3.8 at python3 - Finished dev [unoptimized + debuginfo] target(s) in 0.04s -Python 3.8.5 (default, Jan 27 2021, 15:41:15) -[GCC 9.3.0] on linux -Type "help", "copyright", "credits" or "license" for more information. ->>> import asyncio ->>> ->>> from my_async_module import rust_sleep ->>> ->>> async def main(): ->>> await rust_sleep() ->>> ->>> # should sleep for 1s ->>> asyncio.run(main()) ->>> -``` - -## Awaiting an Async Python Function in Rust - -Let's take a look at a dead simple async Python function: - -```python -# Sleep for 1 second -async def py_sleep(): - await asyncio.sleep(1) -``` - -**Async functions in Python are simply functions that return a `coroutine` object**. For our purposes, -we really don't need to know much about these `coroutine` objects. The key factor here is that calling -an `async` function is _just like calling a regular function_, the only difference is that we have -to do something special with the object that it returns. - -Normally in Python, that something special is the `await` keyword, but in order to await this -coroutine in Rust, we first need to convert it into Rust's version of a `coroutine`: a `Future`. -That's where `pyo3-asyncio` comes in. -[`pyo3_asyncio::async_std::into_future`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.into_future.html) -performs this conversion for us. - -The following example uses `into_future` to call the `py_sleep` function shown above and then await the -coroutine object returned from the call: - -```rust -use pyo3::prelude::*; - -#[pyo3_asyncio::tokio::main] -async fn main() -> PyResult<()> { - let future = Python::with_gil(|py| -> PyResult<_> { - // import the module containing the py_sleep function - let example = py.import("example")?; - - // calling the py_sleep method like a normal function - // returns a coroutine - let coroutine = example.call_method0("py_sleep")?; - - // convert the coroutine into a Rust future using the - // tokio runtime - pyo3_asyncio::tokio::into_future(coroutine) - })?; - - // await the future - future.await?; - - Ok(()) -} -``` - -Alternatively, the below example shows how to write a `#[pyfunction]` which uses `into_future` to receive and await -a coroutine argument: - -```rust -#[pyfunction] -fn await_coro(coro: &Bound<'_, PyAny>>) -> PyResult<()> { - // convert the coroutine into a Rust future using the - // async_std runtime - let f = pyo3_asyncio::async_std::into_future(coro)?; - - pyo3_asyncio::async_std::run_until_complete(coro.py(), async move { - // await the future - f.await?; - Ok(()) - }) -} -``` - -This could be called from Python as: - -```python -import asyncio - -async def py_sleep(): - asyncio.sleep(1) - -await_coro(py_sleep()) -``` - -If for you wanted to pass a callable function to the `#[pyfunction]` instead, (i.e. the last line becomes `await_coro(py_sleep))`, then the above example needs to be tweaked to first call the callable to get the coroutine: - -```rust -#[pyfunction] -fn await_coro(callable: &Bound<'_, PyAny>>) -> PyResult<()> { - // get the coroutine by calling the callable - let coro = callable.call0()?; - - // convert the coroutine into a Rust future using the - // async_std runtime - let f = pyo3_asyncio::async_std::into_future(coro)?; - - pyo3_asyncio::async_std::run_until_complete(coro.py(), async move { - // await the future - f.await?; - Ok(()) - }) -} -``` - -This can be particularly helpful where you need to repeatedly create and await a coroutine. Trying to await the same coroutine multiple times will raise an error: - -```python -RuntimeError: cannot reuse already awaited coroutine -``` - -> If you're interested in learning more about `coroutines` and `awaitables` in general, check out the -> [Python 3 `asyncio` docs](https://docs.python.org/3/library/asyncio-task.html) for more information. - -## Awaiting a Rust Future in Python - -Here we have the same async function as before written in Rust using the -[`async-std`](https://async.rs/) runtime: - -```rust -/// Sleep for 1 second -async fn rust_sleep() { - async_std::task::sleep(std::time::Duration::from_secs(1)).await; -} -``` - -Similar to Python, Rust's async functions also return a special object called a -`Future`: - -```rust -let future = rust_sleep(); -``` - -We can convert this `Future` object into Python to make it `awaitable`. This tells Python that you -can use the `await` keyword with it. In order to do this, we'll call -[`pyo3_asyncio::async_std::future_into_py`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.future_into_py.html): - -```rust -use pyo3::prelude::*; - -async fn rust_sleep() { - async_std::task::sleep(std::time::Duration::from_secs(1)).await; -} - -#[pyfunction] -fn call_rust_sleep(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>>> { - pyo3_asyncio::async_std::future_into_py(py, async move { - rust_sleep().await; - Ok(Python::with_gil(|py| py.None())) - }) -} -``` - -In Python, we can call this pyo3 function just like any other async function: - -```python -from example import call_rust_sleep - -async def rust_sleep(): - await call_rust_sleep() -``` - -## Managing Event Loops - -Python's event loop requires some special treatment, especially regarding the main thread. Some of -Python's `asyncio` features, like proper signal handling, require control over the main thread, which -doesn't always play well with Rust. - -Luckily, Rust's event loops are pretty flexible and don't _need_ control over the main thread, so in -`pyo3-asyncio`, we decided the best way to handle Rust/Python interop was to just surrender the main -thread to Python and run Rust's event loops in the background. Unfortunately, since most event loop -implementations _prefer_ control over the main thread, this can still make some things awkward. - -### PyO3 Asyncio Initialization - -Because Python needs to control the main thread, we can't use the convenient proc macros from Rust -runtimes to handle the `main` function or `#[test]` functions. Instead, the initialization for PyO3 has to be done from the `main` function and the main -thread must block on [`pyo3_asyncio::async_std::run_until_complete`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.run_until_complete.html). - -Because we have to block on one of those functions, we can't use [`#[async_std::main]`](https://docs.rs/async-std/latest/async_std/attr.main.html) or [`#[tokio::main]`](https://docs.rs/tokio/1.1.0/tokio/attr.main.html) -since it's not a good idea to make long blocking calls during an async function. - -> Internally, these `#[main]` proc macros are expanded to something like this: -> ```rust -> fn main() { -> // your async main fn -> async fn _main_impl() { /* ... */ } -> Runtime::new().block_on(_main_impl()); -> } -> ``` -> Making a long blocking call inside the `Future` that's being driven by `block_on` prevents that -> thread from doing anything else and can spell trouble for some runtimes (also this will actually -> deadlock a single-threaded runtime!). Many runtimes have some sort of `spawn_blocking` mechanism -> that can avoid this problem, but again that's not something we can use here since we need it to -> block on the _main_ thread. - -For this reason, `pyo3-asyncio` provides its own set of proc macros to provide you with this -initialization. These macros are intended to mirror the initialization of `async-std` and `tokio` -while also satisfying the Python runtime's needs. - -Here's a full example of PyO3 initialization with the `async-std` runtime: -```rust -use pyo3::prelude::*; - -#[pyo3_asyncio::async_std::main] -async fn main() -> PyResult<()> { - // PyO3 is initialized - Ready to go - - let fut = Python::with_gil(|py| -> PyResult<_> { - let asyncio = py.import("asyncio")?; - - // convert asyncio.sleep into a Rust Future - pyo3_asyncio::async_std::into_future( - asyncio.call_method1("sleep", (1.into_py(py),))? - ) - })?; - - fut.await?; - - Ok(()) -} -``` - -### A Note About `asyncio.run` - -In Python 3.7+, the recommended way to run a top-level coroutine with `asyncio` -is with `asyncio.run`. In `v0.13` we recommended against using this function due to initialization issues, but in `v0.14` it's perfectly valid to use this function... with a caveat. - -Since our Rust <--> Python conversions require a reference to the Python event loop, this poses a problem. Imagine we have a PyO3 Asyncio module that defines -a `rust_sleep` function like in previous examples. You might rightfully assume that you can call pass this directly into `asyncio.run` like this: - -```python -import asyncio - -from my_async_module import rust_sleep - -asyncio.run(rust_sleep()) -``` - -You might be surprised to find out that this throws an error: -```bash -Traceback (most recent call last): - File "example.py", line 5, in - asyncio.run(rust_sleep()) -RuntimeError: no running event loop -``` - -What's happening here is that we are calling `rust_sleep` _before_ the future is -actually running on the event loop created by `asyncio.run`. This is counter-intuitive, but expected behaviour, and unfortunately there doesn't seem to be a good way of solving this problem within PyO3 Asyncio itself. - -However, we can make this example work with a simple workaround: - -```python -import asyncio - -from my_async_module import rust_sleep - -# Calling main will just construct the coroutine that later calls rust_sleep. -# - This ensures that rust_sleep will be called when the event loop is running, -# not before. -async def main(): - await rust_sleep() - -# Run the main() coroutine at the top-level instead -asyncio.run(main()) -``` - -### Non-standard Python Event Loops - -Python allows you to use alternatives to the default `asyncio` event loop. One -popular alternative is `uvloop`. In `v0.13` using non-standard event loops was -a bit of an ordeal, but in `v0.14` it's trivial. - -#### Using `uvloop` in a PyO3 Asyncio Native Extensions - -```toml -# Cargo.toml - -[lib] -name = "my_async_module" -crate-type = ["cdylib"] - -[dependencies] -pyo3 = { version = "0.14", features = ["extension-module"] } -pyo3-asyncio = { version = "0.14", features = ["tokio-runtime"] } -async-std = "1.9" -tokio = "1.4" -``` - -```rust -//! lib.rs - -use pyo3::{prelude::*, wrap_pyfunction}; - -#[pyfunction] -fn rust_sleep(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>>> { - pyo3_asyncio::tokio::future_into_py(py, async { - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - Ok(Python::with_gil(|py| py.None())) - }) -} - -#[pymodule] -fn my_async_module(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; - - Ok(()) -} -``` - -```bash -$ maturin develop && python3 -🔗 Found pyo3 bindings -🐍 Found CPython 3.8 at python3 - Finished dev [unoptimized + debuginfo] target(s) in 0.04s -Python 3.8.8 (default, Apr 13 2021, 19:58:26) -[GCC 7.3.0] :: Anaconda, Inc. on linux -Type "help", "copyright", "credits" or "license" for more information. ->>> import asyncio ->>> import uvloop ->>> ->>> import my_async_module ->>> ->>> uvloop.install() ->>> ->>> async def main(): -... await my_async_module.rust_sleep() -... ->>> asyncio.run(main()) ->>> -``` - -#### Using `uvloop` in Rust Applications - -Using `uvloop` in Rust applications is a bit trickier, but it's still possible -with relatively few modifications. - -Unfortunately, we can't make use of the `#[pyo3_asyncio::::main]` attribute with non-standard event loops. This is because the `#[pyo3_asyncio::::main]` proc macro has to interact with the Python -event loop before we can install the `uvloop` policy. - -```toml -[dependencies] -async-std = "1.9" -pyo3 = "0.14" -pyo3-asyncio = { version = "0.14", features = ["async-std-runtime"] } -``` - -```rust -//! main.rs - -use pyo3::{prelude::*, types::PyType}; - -fn main() -> PyResult<()> { - pyo3::prepare_freethreaded_python(); - - Python::with_gil(|py| { - let uvloop = py.import("uvloop")?; - uvloop.call_method0("install")?; - - // store a reference for the assertion - let uvloop = PyObject::from(uvloop); - - pyo3_asyncio::async_std::run(py, async move { - // verify that we are on a uvloop.Loop - Python::with_gil(|py| -> PyResult<()> { - assert!(pyo3_asyncio::async_std::get_current_loop(py)?.is_instance( - uvloop - .as_ref(py) - .getattr("Loop")? - )?); - Ok(()) - })?; - - async_std::task::sleep(std::time::Duration::from_secs(1)).await; - - Ok(()) - }) - }) -} -``` - ## Additional Information -- Managing event loop references can be tricky with pyo3-asyncio. See [Event Loop References](https://docs.rs/pyo3-asyncio/#event-loop-references) in the API docs to get a better intuition for how event loop references are managed in this library. -- Testing pyo3-asyncio libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/testing) +- Managing event loop references can be tricky with `pyo3-async-runtimes`. See [Event Loop References](https://docs.rs/pyo3-async-runtimes/#event-loop-references-and-contextvars) in the API docs to get a better intuition for how event loop references are managed in this library. +- Testing `pyo3-async-runtimes` libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://docs.rs/pyo3-async-runtimes/latest/pyo3_async_runtimes/testing) diff --git a/guide/src/ecosystem/tracing.md b/guide/src/ecosystem/tracing.md new file mode 100644 index 00000000000..341d0759e96 --- /dev/null +++ b/guide/src/ecosystem/tracing.md @@ -0,0 +1,107 @@ +# Tracing + +Python projects that write extension modules for performance reasons may want to +tap into [Rust's `tracing` ecosystem] to gain insight into the performance of +their extension module. + +This section of the guide describes a few crates that provide ways to do that. +They build on [`tracing_subscriber`][tracing-subscriber] and require code +changes in both Python and Rust to integrate. Note that each extension module +must configure its own `tracing` integration; one extension module will not see +`tracing` data from a different module. + +## `pyo3-tracing-subscriber` ([documentation][pyo3-tracing-subscriber-docs]) + +[`pyo3-tracing-subscriber`][pyo3-tracing-subscriber] provides a way for Python +projects to configure `tracing_subscriber`. It exposes a few +`tracing_subscriber` layers: +- `tracing_subscriber::fmt` for writing human-readable output to file or stdout +- `opentelemetry-stdout` for writing OTLP output to file or stdout +- `opentelemetry-otlp` for writing OTLP output to an OTLP endpoint + +The extension module must call [`pyo3_tracing_subscriber::add_submodule`][add-submodule] +to export the Python classes needed to configure and initialize `tracing`. + +On the Python side, use the `Tracing` context manager to initialize tracing and +run Rust code inside the context manager's block. `Tracing` takes a +`GlobalTracingConfig` instance describing the layers to be used. + +See [the README on crates.io][pyo3-tracing-subscriber] +for example code. + +## `pyo3-python-tracing-subscriber` ([documentation][pyo3-python-tracing-subscriber-docs]) + +The similarly-named [`pyo3-python-tracing-subscriber`][pyo3-python-tracing-subscriber] +implements a shim in Rust that forwards `tracing` data to a `Layer` +implementation defined in and passed in from Python. + +There are many ways an extension module could integrate `pyo3-python-tracing-subscriber` +but a simple one may look something like this: +```rust +#[tracing::instrument] +#[pyfunction] +fn fibonacci(index: usize, use_memoized: bool) -> PyResult { + // ... +} + +#[pyfunction] +pub fn initialize_tracing(py_impl: Bound<'_, PyAny>) { + tracing_subscriber::registry() + .with(pyo3_python_tracing_subscriber::PythonCallbackLayerBridge::new(py_impl)) + .init(); +} +``` +The extension module must provide some way for Python to pass in one or more +Python objects that implement [the `Layer` interface]. Then it should construct +[`pyo3_python_tracing_subscriber::PythonCallbackLayerBridge`][PythonCallbackLayerBridge] +instances with each of those Python objects and initialize `tracing_subscriber` +as shown above. + +The Python objects implement a modified version of the `Layer` interface: +- `on_new_span()` may return some state that will stored inside the Rust span +- other callbacks will be given that state as an additional positional argument + +A dummy `Layer` implementation may look like this: +```python +import rust_extension + +class MyPythonLayer: + def __init__(self): + pass + + # `on_new_span` can return some state + def on_new_span(self, span_attrs: str, span_id: str) -> int: + print(f"[on_new_span]: {span_attrs} | {span_id}") + return random.randint(1, 1000) + + # The state from `on_new_span` is passed back into other trait methods + def on_event(self, event: str, state: int): + print(f"[on_event]: {event} | {state}") + + def on_close(self, span_id: str, state: int): + print(f"[on_close]: {span_id} | {state}") + + def on_record(self, span_id: str, values: str, state: int): + print(f"[on_record]: {span_id} | {values} | {state}") + +def main(): + rust_extension.initialize_tracing(MyPythonLayer()) + + print("10th fibonacci number: ", rust_extension.fibonacci(10, True)) +``` + +`pyo3-python-tracing-subscriber` has [working examples] +showing both the Rust side and the Python side of an integration. + +[pyo3-tracing-subscriber]: https://crates.io/crates/pyo3-tracing-subscriber +[pyo3-tracing-subscriber-docs]: https://docs.rs/pyo3-tracing-subscriber +[add-submodule]: https://docs.rs/pyo3-tracing-subscriber/*/pyo3_tracing_subscriber/fn.add_submodule.html + +[pyo3-python-tracing-subscriber]: https://crates.io/crates/pyo3-python-tracing-subscriber +[pyo3-python-tracing-subscriber-docs]: https://docs.rs/pyo3-python-tracing-subscriber +[PythonCallbackLayerBridge]: https://docs.rs/pyo3-python-tracing-subscriber/*/pyo3_python_tracing_subscriber/struct.PythonCallbackLayerBridge.html +[working examples]: https://github.com/getsentry/pyo3-python-tracing-subscriber/tree/main/demo + +[Rust's `tracing` ecosystem]: https://crates.io/crates/tracing +[tracing-subscriber]: https://docs.rs/tracing-subscriber/*/tracing_subscriber/ +[the `Layer` interface]: https://docs.rs/tracing-subscriber/*/tracing_subscriber/layer/trait.Layer.html diff --git a/guide/src/exception.md b/guide/src/exception.md index 3e2f5034897..8de04ba986a 100644 --- a/guide/src/exception.md +++ b/guide/src/exception.md @@ -23,15 +23,18 @@ use pyo3::exceptions::PyException; create_exception!(mymodule, CustomError, PyException); +# fn main() -> PyResult<()> { Python::with_gil(|py| { - let ctx = [("CustomError", py.get_type_bound::())].into_py_dict_bound(py); + let ctx = [("CustomError", py.get_type::())].into_py_dict(py)?; pyo3::py_run!( py, *ctx, "assert str(CustomError) == \"\"" ); pyo3::py_run!(py, *ctx, "assert CustomError('oops').args == ('oops',)"); -}); +# Ok(()) +}) +# } ``` When using PyO3 to create an extension module, you can add the new exception to @@ -46,7 +49,7 @@ pyo3::create_exception!(mymodule, CustomError, PyException); #[pymodule] fn mymodule(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { // ... other elements added to module ... - m.add("CustomError", py.get_type_bound::())?; + m.add("CustomError", py.get_type::())?; Ok(()) } @@ -78,12 +81,15 @@ In PyO3 every object has the [`PyAny::is_instance`] and [`PyAny::is_instance_of` use pyo3::prelude::*; use pyo3::types::{PyBool, PyList}; +# fn main() -> PyResult<()> { Python::with_gil(|py| { - assert!(PyBool::new_bound(py, true).is_instance_of::()); - let list = PyList::new_bound(py, &[1, 2, 3, 4]); + assert!(PyBool::new(py, true).is_instance_of::()); + let list = PyList::new(py, &[1, 2, 3, 4])?; assert!(!list.is_instance_of::()); assert!(list.is_instance_of::()); -}); +# Ok(()) +}) +# } ``` To check the type of an exception, you can similarly do: @@ -128,5 +134,52 @@ defines exceptions for several standard library modules. [`PyErr`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html [`PyResult`]: {{#PYO3_DOCS_URL}}/pyo3/type.PyResult.html [`PyErr::from_value`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html#method.from_value -[`PyAny::is_instance`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.is_instance -[`PyAny::is_instance_of`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.is_instance_of +[`PyAny::is_instance`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.is_instance +[`PyAny::is_instance_of`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.is_instance_of + +## Creating more complex exceptions + +If you need to create an exception with more complex behavior, you can also manually create a subclass of `PyException`: + +```rust +#![allow(dead_code)] +# #[cfg(any(not(feature = "abi3")))] { +use pyo3::prelude::*; +use pyo3::types::IntoPyDict; +use pyo3::exceptions::PyException; + +#[pyclass(extends=PyException)] +struct CustomError { + #[pyo3(get)] + url: String, + + #[pyo3(get)] + message: String, +} + +#[pymethods] +impl CustomError { + #[new] + fn new(url: String, message: String) -> Self { + Self { url, message } + } +} + +# fn main() -> PyResult<()> { +Python::with_gil(|py| { + let ctx = [("CustomError", py.get_type::())].into_py_dict(py)?; + pyo3::py_run!( + py, + *ctx, + "assert str(CustomError) == \"\", repr(CustomError)" + ); + pyo3::py_run!(py, *ctx, "assert CustomError('https://example.com', 'something went bad').args == ('https://example.com', 'something went bad')"); + pyo3::py_run!(py, *ctx, "assert CustomError('https://example.com', 'something went bad').url == 'https://example.com'"); +# Ok(()) +}) +# } +# } + +``` + +Note that this is not possible when the ``abi3`` feature is enabled, as that prevents subclassing ``PyException``. diff --git a/guide/src/faq.md b/guide/src/faq.md index 19f9b5d50ab..83089cf395e 100644 --- a/guide/src/faq.md +++ b/guide/src/faq.md @@ -2,20 +2,22 @@ Sorry that you're having trouble using PyO3. If you can't find the answer to your problem in the list below, you can also reach out for help on [GitHub Discussions](https://github.com/PyO3/pyo3/discussions) and on [Discord](https://discord.gg/33kcChzH7f). -## I'm experiencing deadlocks using PyO3 with lazy_static or once_cell! +## I'm experiencing deadlocks using PyO3 with `std::sync::OnceLock`, `std::sync::LazyLock`, `lazy_static`, and `once_cell`! -`lazy_static` and `once_cell::sync` both use locks to ensure that initialization is performed only by a single thread. Because the Python GIL is an additional lock this can lead to deadlocks in the following way: +`OnceLock`, `LazyLock`, and their thirdparty predecessors use blocking to ensure only one thread ever initializes them. Because the Python GIL is an additional lock this can lead to deadlocks in the following way: -1. A thread (thread A) which has acquired the Python GIL starts initialization of a `lazy_static` value. +1. A thread (thread A) which has acquired the Python GIL starts initialization of a `OnceLock` value. 2. The initialization code calls some Python API which temporarily releases the GIL e.g. `Python::import`. -3. Another thread (thread B) acquires the Python GIL and attempts to access the same `lazy_static` value. -4. Thread B is blocked, because it waits for `lazy_static`'s initialization to lock to release. +3. Another thread (thread B) acquires the Python GIL and attempts to access the same `OnceLock` value. +4. Thread B is blocked, because it waits for `OnceLock`'s initialization to lock to release. 5. Thread A is blocked, because it waits to re-acquire the GIL which thread B still holds. 6. Deadlock. -PyO3 provides a struct [`GILOnceCell`] which works equivalently to `OnceCell` but relies solely on the Python GIL for thread safety. This means it can be used in place of `lazy_static` or `once_cell` where you are experiencing the deadlock described above. See the documentation for [`GILOnceCell`] for an example how to use it. +PyO3 provides a struct [`GILOnceCell`] which implements a single-initialization API based on these types that relies on the GIL for locking. If the GIL is released or there is no GIL, then this type allows the initialization function to race but ensures that the data is only ever initialized once. If you need to ensure that the initialization function is called once and only once, you can make use of the [`OnceExt`] and [`OnceLockExt`] extension traits that enable using the standard library types for this purpose but provide new methods for these types that avoid the risk of deadlocking with the Python GIL. This means they can be used in place of other choices when you are experiencing the deadlock described above. See the documentation for [`GILOnceCell`] and [`OnceExt`] for further details and an example how to use them. [`GILOnceCell`]: {{#PYO3_DOCS_URL}}/pyo3/sync/struct.GILOnceCell.html +[`OnceExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceExt.html +[`OnceLockExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceLockExt.html ## I can't run `cargo test`; or I can't build in a Cargo workspace: I'm having linker issues like "Symbol not found" or "Undefined reference to _PyExc_SystemError"! @@ -127,12 +129,10 @@ If you don't want that cloning to happen, a workaround is to allocate the field ```rust # use pyo3::prelude::*; #[pyclass] -#[derive(Clone)] struct Inner {/* fields omitted */} #[pyclass] struct Outer { - #[pyo3(get)] inner: Py, } @@ -144,6 +144,11 @@ impl Outer { inner: Py::new(py, Inner {})?, }) } + + #[getter] + fn inner(&self, py: Python<'_>) -> Py { + self.inner.clone_ref(py) + } } ``` This time `a` and `b` *are* the same object: @@ -178,6 +183,7 @@ done with the `crate` attribute: # use pyo3::prelude::*; # pub extern crate pyo3; # mod reexported { pub use ::pyo3; } +# #[allow(dead_code)] #[pyclass] #[pyo3(crate = "reexported::pyo3")] struct MyClass; diff --git a/guide/src/features.md b/guide/src/features.md index 0816770a781..b48c138b287 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -57,12 +57,6 @@ This feature adds support for `async fn` in `#[pyfunction]` and `#[pymethods]`. The feature has some unfinished refinements and performance improvements. To help finish this off, see [issue #1632](https://github.com/PyO3/pyo3/issues/1632) and its associated draft PRs. -### `experimental-declarative-modules` - -This feature allows to declare Python modules using `#[pymodule] mod my_module { ... }` syntax. - -The feature has some unfinished refinements and edge cases. To help finish this off, see [issue #3900](https://github.com/PyO3/pyo3/issues/3900). - ### `experimental-inspect` This feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` and `FromPyObject::type_input` APIs to produce Python type "annotations" for Rust types. @@ -75,6 +69,14 @@ This feature is a backwards-compatibility feature to allow continued use of the This feature and the APIs it enables is expected to be removed in a future PyO3 version. +### `py-clone` + +This feature was introduced to ease migration. It was found that delayed reference counts cannot be made sound and hence `Clon`ing an instance of `Py` must panic without the GIL being held. To avoid migrations introducing new panics without warning, the `Clone` implementation itself is now gated behind this feature. + +### `pyo3_disable_reference_pool` + +This is a performance-oriented conditional compilation flag, e.g. [set via `$RUSTFLAGS`][set-configuration-options], which disabled the global reference pool and the assocaited overhead for the crossing the Python-Rust boundary. However, if enabled, `Drop`ping an instance of `Py` without the GIL being held will abort the process. + ### `macros` This feature enables a dependency on the `pyo3-macros` crate, which provides the procedural macros portion of PyO3's API: @@ -93,9 +95,9 @@ These macros require a number of dependencies which may not be needed by users w ### `multiple-pymethods` -This feature enables a dependency on `inventory`, which enables each `#[pyclass]` to have more than one `#[pymethods]` block. This feature also requires a minimum Rust version of 1.62 due to limitations in the `inventory` crate. +This feature enables each `#[pyclass]` to have more than one `#[pymethods]` block. -Most users should only need a single `#[pymethods]` per `#[pyclass]`. In addition, not all platforms (e.g. Wasm) are supported by `inventory`. For this reason this feature is not enabled by default, meaning fewer dependencies and faster compilation for the majority of users. +Most users should only need a single `#[pymethods]` per `#[pyclass]`. In addition, not all platforms (e.g. Wasm) are supported by `inventory`, which is used in the implementation of the feature. For this reason this feature is not enabled by default, meaning fewer dependencies and faster compilation for the majority of users. See [the `#[pyclass]` implementation details](class.md#implementation-details) for more information. @@ -149,6 +151,18 @@ Adds a dependency on [hashbrown](https://docs.rs/hashbrown) and enables conversi Adds a dependency on [indexmap](https://docs.rs/indexmap) and enables conversions into its [`IndexMap`](https://docs.rs/indexmap/latest/indexmap/map/struct.IndexMap.html) type. +### `jiff-02` + +Adds a dependency on [jiff@0.2](https://docs.rs/jiff/0.2) and requires MSRV 1.70. Enables a conversion from [jiff](https://docs.rs/jiff)'s types to python: +- [SignedDuration](https://docs.rs/jiff/0.2/jiff/struct.SignedDuration.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html) +- [TimeZone](https://docs.rs/jiff/0.2/jiff/tz/struct.TimeZone.html) -> [`PyTzInfo`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTzInfo.html) +- [Offset](https://docs.rs/jiff/0.2/jiff/tz/struct.Offset.html) -> [`PyTzInfo`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTzInfo.html) +- [Date](https://docs.rs/jiff/0.2/jiff/civil/struct.Date.html) -> [`PyDate`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDate.html) +- [Time](https://docs.rs/jiff/0.2/jiff/civil/struct.Time.html) -> [`PyTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTime.html) +- [DateTime](https://docs.rs/jiff/0.2/jiff/civil/struct.DateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +- [Zoned](https://docs.rs/jiff/0.2/jiff/struct.Zoned.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +- [Timestamp](https://docs.rs/jiff/0.2/jiff/struct.Timestamp.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) + ### `num-bigint` Adds a dependency on [num-bigint](https://docs.rs/num-bigint) and enables conversions into its [`BigInt`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigInt.html) and [`BigUint`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigUint.html) types. @@ -157,6 +171,10 @@ Adds a dependency on [num-bigint](https://docs.rs/num-bigint) and enables conver Adds a dependency on [num-complex](https://docs.rs/num-complex) and enables conversions into its [`Complex`](https://docs.rs/num-complex/latest/num_complex/struct.Complex.html) type. +### `num-rational` + +Adds a dependency on [num-rational](https://docs.rs/num-rational) and enables conversions into its [`Ratio`](https://docs.rs/num-rational/latest/num_rational/struct.Ratio.html) type. + ### `rust_decimal` Adds a dependency on [rust_decimal](https://docs.rs/rust_decimal) and enables conversions into its [`Decimal`](https://docs.rs/rust_decimal/latest/rust_decimal/struct.Decimal.html) type. @@ -191,3 +209,9 @@ struct User { ### `smallvec` Adds a dependency on [smallvec](https://docs.rs/smallvec) and enables conversions into its [`SmallVec`](https://docs.rs/smallvec/latest/smallvec/struct.SmallVec.html) type. + +[set-configuration-options]: https://doc.rust-lang.org/reference/conditional-compilation.html#set-configuration-options + +### `uuid` + +Adds a dependency on [uuid](https://docs.rs/uuid) and enables conversions into its [`Uuid`](https://docs.rs/uuid/latest/uuid/struct.Uuid.html) type. \ No newline at end of file diff --git a/guide/src/free-threading.md b/guide/src/free-threading.md new file mode 100644 index 00000000000..ffb95d240a1 --- /dev/null +++ b/guide/src/free-threading.md @@ -0,0 +1,412 @@ +# Supporting Free-Threaded CPython + +CPython 3.13 introduces an experimental "free-threaded" build of CPython that +does not rely on the [global interpreter +lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) +(often referred to as the GIL) for thread safety. As of version 0.23, PyO3 also +has preliminary support for building Rust extensions for the free-threaded +Python build and support for calling into free-threaded Python from Rust. + +If you want more background on free-threaded Python in general, see the [what's +new](https://docs.python.org/3.13/whatsnew/3.13.html#whatsnew313-free-threaded-cpython) +entry in the CPython docs, the [HOWTO +guide](https://docs.python.org/3.13/howto/free-threading-extensions.html#freethreading-extensions-howto) +for porting C extensions, and [PEP 703](https://peps.python.org/pep-0703/), +which provides the technical background for the free-threading implementation in +CPython. + +In the GIL-enabled build, the global interpreter lock serializes access to the +Python runtime. The GIL is therefore a fundamental limitation to parallel +scaling of multithreaded Python workflows, due to [Amdahl's +law](https://en.wikipedia.org/wiki/Amdahl%27s_law), because any time spent +executing a parallel processing task on only one execution context fundamentally +cannot be sped up using parallelism. + +The free-threaded build removes this limit on multithreaded Python scaling. This +means it's much more straightforward to achieve parallelism using the Python +[`threading`] module. If you +have ever needed to use +[`multiprocessing`](https://docs.python.org/3/library/multiprocessing.html) to +achieve a parallel speedup for some Python code, free-threading will likely +allow the use of Python threads instead for the same workflow. + +PyO3's support for free-threaded Python will enable authoring native Python +extensions that are thread-safe by construction, with much stronger safety +guarantees than C extensions. Our goal is to enable ["fearless +concurrency"](https://doc.rust-lang.org/book/ch16-00-concurrency.html) in the +native Python runtime by building on the Rust [`Send` and +`Sync`](https://doc.rust-lang.org/nomicon/send-and-sync.html) traits. + +This document provides advice for porting Rust code using PyO3 to run under +free-threaded Python. + +## Supporting free-threaded Python with PyO3 + +Many simple uses of PyO3, like exposing bindings for a "pure" Rust function +with no side-effects or defining an immutable Python class, will likely work +"out of the box" on the free-threaded build. All that will be necessary is to +annotate Python modules declared by rust code in your project to declare that +they support free-threaded Python, for example by declaring the module with +`#[pymodule(gil_used = false)]`. + +More complicated `#[pyclass]` types may need to deal with thread-safety directly; there is [a dedicated section of the guide](./class/thread-safety.md) to discuss this. + +At a low-level, annotating a module sets the `Py_MOD_GIL` slot on modules +defined by an extension to `Py_MOD_GIL_NOT_USED`, which allows the interpreter +to see at runtime that the author of the extension thinks the extension is +thread-safe. You should only do this if you know that your extension is +thread-safe. Because of Rust's guarantees, this is already true for many +extensions, however see below for more discussion about how to evaluate the +thread safety of existing Rust extensions and how to think about the PyO3 API +using a Python runtime with no GIL. + +If you do not explicitly mark that modules are thread-safe, the Python +interpreter will re-enable the GIL at runtime while importing your module and +print a `RuntimeWarning` with a message containing the name of the module +causing it to re-enable the GIL. You can force the GIL to remain disabled by +setting the `PYTHON_GIL=0` as an environment variable or passing `-Xgil=0` when +starting Python (`0` means the GIL is turned off). + +If you are sure that all data structures exposed in a `PyModule` are +thread-safe, then pass `gil_used = false` as a parameter to the +`pymodule` procedural macro declaring the module or call +`PyModule::gil_used` on a `PyModule` instance. For example: + +```rust +use pyo3::prelude::*; + +/// This module supports free-threaded Python +#[pymodule(gil_used = false)] +fn my_extension(m: &Bound<'_, PyModule>) -> PyResult<()> { + // add members to the module that you know are thread-safe + Ok(()) +} +``` + +Or for a module that is set up without using the `pymodule` macro: + +```rust +use pyo3::prelude::*; + +# #[allow(dead_code)] +fn register_child_module(parent_module: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(parent_module.py(), "child_module")?; + child_module.gil_used(false)?; + parent_module.add_submodule(&child_module) +} + +``` + +For now you must explicitly opt in to free-threading support by annotating +modules defined in your extension. In a future version of `PyO3`, we plan to +make `gil_used = false` the default. + +See the +[`string-sum`](https://github.com/PyO3/pyo3/tree/main/pyo3-ffi/examples/string-sum) +example for how to declare free-threaded support using raw FFI calls for modules +using single-phase initialization and the +[`sequential`](https://github.com/PyO3/pyo3/tree/main/pyo3-ffi/examples/sequential) +example for modules using multi-phase initialization. + +If you would like to use conditional compilation to trigger different code paths +under the free-threaded build, you can use the `Py_GIL_DISABLED` attribute once +you have configured your crate to generate the necessary build configuration +data. See [the guide +section](./building-and-distribution/multiple-python-versions.md) for more +details about supporting multiple different Python versions, including the +free-threaded build. + + +## Special considerations for the free-threaded build + +The free-threaded interpreter does not have a GIL, and this can make interacting +with the PyO3 API confusing, since the API was originally designed around strong +assumptions about the GIL providing locking. Additionally, since the GIL +provided locking for operations on Python objects, many existing extensions that +provide mutable data structures relied on the GIL to make interior mutability +thread-safe. + +Working with PyO3 under the free-threaded interpreter therefore requires some +additional care and mental overhead compared with a GIL-enabled interpreter. We +discuss how to handle this below. + +### Many symbols exposed by PyO3 have `GIL` in the name + +We are aware that there are some naming issues in the PyO3 API that make it +awkward to think about a runtime environment where there is no GIL. We plan to +change the names of these types to de-emphasize the role of the GIL in future +versions of PyO3, but for now you should remember that the use of the term `GIL` +in functions and types like [`Python::with_gil`] and [`GILOnceCell`] is +historical. + +Instead, you should think about whether or not a Rust thread is attached to a +Python interpreter runtime. Calling into the CPython C API is only legal when an +OS thread is explicitly attached to the interpreter runtime. In the GIL-enabled +build, this happens when the GIL is acquired. In the free-threaded build there +is no GIL, but the same C macros that release or acquire the GIL in the +GIL-enabled build instead ask the interpreter to attach the thread to the Python +runtime, and there can be many threads simultaneously attached. See [PEP +703](https://peps.python.org/pep-0703/#thread-states) for more background about +how threads can be attached and detached from the interpreter runtime, in a +manner analagous to releasing and acquiring the GIL in the GIL-enabled build. + +In the GIL-enabled build, PyO3 uses the [`Python<'py>`] type and the `'py` +lifetime to signify that the global interpreter lock is held. In the +freethreaded build, holding a `'py` lifetime means only that the thread is +currently attached to the Python interpreter -- other threads can be +simultaneously interacting with the interpreter. + +You still need to obtain a `'py` lifetime is to interact with Python +objects or call into the CPython C API. If you are not yet attached to the +Python runtime, you can register a thread using the [`Python::with_gil`] +function. Threads created via the Python [`threading`] module do not not need to +do this, and pyo3 will handle setting up the [`Python<'py>`] token when CPython +calls into your extension. + +### Global synchronization events can cause hangs and deadlocks + +The free-threaded build triggers global synchronization events in the following +situations: + +* During garbage collection in order to get a globally consistent view of + reference counts and references between objects +* In Python 3.13, when the first background thread is started in + order to mark certain objects as immortal +* When either `sys.settrace` or `sys.setprofile` are called in order to + instrument running code objects and threads +* Before `os.fork()` is called. + +This is a non-exhaustive list and there may be other situations in future Python +versions that can trigger global synchronization events. + +This means that you should detach from the interpreter runtime using +[`Python::allow_threads`] in exactly the same situations as you should detach +from the runtime in the GIL-enabled build: when doing long-running tasks that do +not require the CPython runtime or when doing any task that needs to re-attach +to the runtime (see the [guide +section](parallelism.md#sharing-python-objects-between-rust-threads) that +covers this). In the former case, you would observe a hang on threads that are +waiting on the long-running task to complete, and in the latter case you would +see a deadlock while a thread tries to attach after the runtime triggers a +global synchronization event, but the spawning thread prevents the +synchronization event from completing. + +### Exceptions and panics for multithreaded access of mutable `pyclass` instances + +Data attached to `pyclass` instances is protected from concurrent access by a +`RefCell`-like pattern of runtime borrow checking. Like a `RefCell`, PyO3 will +raise exceptions (or in some cases panic) to enforce exclusive access for +mutable borrows. It was always possible to generate panics like this in PyO3 in +code that releases the GIL with [`Python::allow_threads`] or calling a python +method accepting `&self` from a `&mut self` (see [the docs on interior +mutability](./class.md#bound-and-interior-mutability),) but now in free-threaded +Python there are more opportunities to trigger these panics from Python because +there is no GIL to lock concurrent access to mutably borrowed data from Python. + +The most straightforward way to trigger this problem is to use the Python +[`threading`] module to simultaneously call a rust function that mutably borrows a +[`pyclass`]({{#PYO3_DOCS_URL}}/pyo3/attr.pyclass.html) in multiple threads. For +example, consider the following implementation: + +```rust +# use pyo3::prelude::*; +# fn main() { +#[pyclass] +#[derive(Default)] +struct ThreadIter { + count: usize, +} + +#[pymethods] +impl ThreadIter { + #[new] + pub fn new() -> Self { + Default::default() + } + + fn __next__(&mut self, py: Python<'_>) -> usize { + self.count += 1; + self.count + } +} +# } +``` + +And then if we do something like this in Python: + +```python +import concurrent.futures +from my_module import ThreadIter + +i = ThreadIter() + +def increment(): + next(i) + +with concurrent.futures.ThreadPoolExecutor(max_workers=16) as tpe: + futures = [tpe.submit(increment) for _ in range(100)] + [f.result() for f in futures] +``` + +We will see an exception: + +```text +Traceback (most recent call last) + File "example.py", line 5, in + next(i) +RuntimeError: Already borrowed +``` + +We plan to allow user-selectable semantics for mutable pyclass definitions in +PyO3 0.24, allowing some form of opt-in locking to emulate the GIL if that is +needed. For now you should explicitly add locking, possibly using conditional +compilation or using the critical section API, to avoid creating deadlocks with +the GIL. + +### Cannot build extensions using the limited API + +The free-threaded build uses a completely new ABI and there is not yet an +equivalent to the limited API for the free-threaded ABI. That means if your +crate depends on PyO3 using the `abi3` feature or an an `abi3-pyxx` feature, +PyO3 will print a warning and ignore that setting when building extensions using +the free-threaded interpreter. + +This means that if your package makes use of the ABI forward compatibility +provided by the limited API to upload only one wheel for each release of your +package, you will need to update your release procedure to also upload a +version-specific free-threaded wheel. + +See [the guide section](./building-and-distribution/multiple-python-versions.md) +for more details about supporting multiple different Python versions, including +the free-threaded build. + +### Thread-safe single initialization + +Until version 0.23, PyO3 provided only [`GILOnceCell`] to enable deadlock-free +single initialization of data in contexts that might execute arbitrary Python +code. While we have updated [`GILOnceCell`] to avoid thread safety issues +triggered only under the free-threaded build, the design of [`GILOnceCell`] is +inherently thread-unsafe, in a manner that can be problematic even in the +GIL-enabled build. + +If, for example, the function executed by [`GILOnceCell`] releases the GIL or +calls code that releases the GIL, then it is possible for multiple threads to +race to initialize the cell. While the cell will only ever be intialized +once, it can be problematic in some contexts that [`GILOnceCell`] does not block +like the standard library [`OnceLock`]. + +In cases where the initialization function must run exactly once, you can bring +the [`OnceExt`] or [`OnceLockExt`] traits into scope. The [`OnceExt`] trait adds +[`OnceExt::call_once_py_attached`] and [`OnceExt::call_once_force_py_attached`] +functions to the api of `std::sync::Once`, enabling use of [`Once`] in contexts +where the GIL is held. Similarly, [`OnceLockExt`] adds +[`OnceLockExt::get_or_init_py_attached`]. These functions are analogous to +[`Once::call_once`], [`Once::call_once_force`], and [`OnceLock::get_or_init`] except +they accept a [`Python<'py>`] token in addition to an `FnOnce`. All of these +functions release the GIL and re-acquire it before executing the function, +avoiding deadlocks with the GIL that are possible without using the PyO3 +extension traits. Here is an example of how to use [`OnceExt`] to +enable single-initialization of a runtime cache holding a `Py`. + +```rust +# fn main() { +# use pyo3::prelude::*; +use std::sync::Once; +use pyo3::sync::OnceExt; +use pyo3::types::PyDict; + +struct RuntimeCache { + once: Once, + cache: Option> +} + +let mut cache = RuntimeCache { + once: Once::new(), + cache: None +}; + +Python::with_gil(|py| { + // guaranteed to be called once and only once + cache.once.call_once_py_attached(py, || { + cache.cache = Some(PyDict::new(py).unbind()); + }); +}); +# } +``` + +### `GILProtected` is not exposed + +[`GILProtected`] is a PyO3 type that allows mutable access to static data by +leveraging the GIL to lock concurrent access from other threads. In +free-threaded Python there is no GIL, so you will need to replace this type with +some other form of locking. In many cases, a type from +[`std::sync::atomic`](https://doc.rust-lang.org/std/sync/atomic/) or a +[`std::sync::Mutex`](https://doc.rust-lang.org/std/sync/struct.Mutex.html) will +be sufficient. + +Before: + +```rust +# fn main() { +# #[cfg(not(Py_GIL_DISABLED))] { +# use pyo3::prelude::*; +use pyo3::sync::GILProtected; +use pyo3::types::{PyDict, PyNone}; +use std::cell::RefCell; + +static OBJECTS: GILProtected>>> = + GILProtected::new(RefCell::new(Vec::new())); + +Python::with_gil(|py| { + // stand-in for something that executes arbitrary Python code + let d = PyDict::new(py); + d.set_item(PyNone::get(py), PyNone::get(py)).unwrap(); + OBJECTS.get(py).borrow_mut().push(d.unbind()); +}); +# }} +``` + +After: + +```rust +# use pyo3::prelude::*; +# fn main() { +use pyo3::types::{PyDict, PyNone}; +use std::sync::Mutex; + +static OBJECTS: Mutex>> = Mutex::new(Vec::new()); + +Python::with_gil(|py| { + // stand-in for something that executes arbitrary Python code + let d = PyDict::new(py); + d.set_item(PyNone::get(py), PyNone::get(py)).unwrap(); + // as with any `Mutex` usage, lock the mutex for as little time as possible + // in this case, we do it just while pushing into the `Vec` + OBJECTS.lock().unwrap().push(d.unbind()); +}); +# } +``` + +If you are executing arbitrary Python code while holding the lock, then you +should import the [`MutexExt`] trait and use the `lock_py_attached` method +instead of `lock`. This ensures that global synchronization events started by +the Python runtime can proceed, avoiding possible deadlocks with the +interpreter. + +[`GILOnceCell`]: {{#PYO3_DOCS_URL}}/pyo3/sync/struct.GILOnceCell.html +[`GILProtected`]: https://docs.rs/pyo3/0.22/pyo3/sync/struct.GILProtected.html +[`MutexExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.MutexExt.html +[`Once`]: https://doc.rust-lang.org/stable/std/sync/struct.Once.html +[`Once::call_once`]: https://doc.rust-lang.org/stable/std/sync/struct.Once.html#tymethod.call_once +[`Once::call_once_force`]: https://doc.rust-lang.org/stable/std/sync/struct.Once.html#tymethod.call_once_force +[`OnceExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceExt.html +[`OnceExt::call_once_py_attached`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceExt.html#tymethod.call_once_py_attached +[`OnceExt::call_once_force_py_attached`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceExt.html#tymethod.call_once_force_py_attached +[`OnceLockExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceLockExt.html +[`OnceLockExt::get_or_init_py_attached`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceLockExt.html#tymethod.get_or_init_py_attached +[`OnceLock`]: https://doc.rust-lang.org/stable/std/sync/struct.OnceLock.html +[`OnceLock::get_or_init`]: https://doc.rust-lang.org/stable/std/sync/struct.OnceLock.html#tymethod.get_or_init +[`Python::allow_threads`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.allow_threads +[`Python::with_gil`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.with_gil +[`Python<'py>`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html +[`threading`]: https://docs.python.org/3/library/threading.html diff --git a/guide/src/function.md b/guide/src/function.md index 86ac4c89b46..323bc9c8f87 100644 --- a/guide/src/function.md +++ b/guide/src/function.md @@ -14,8 +14,7 @@ fn double(x: usize) -> usize { #[pymodule] fn my_extension(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(double, m)?)?; - Ok(()) + m.add_function(wrap_pyfunction!(double, m)?) } ``` @@ -56,8 +55,7 @@ The `#[pyo3]` attribute can be used to modify properties of the generated Python #[pymodule] fn module_with_functions(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(no_args_py, m)?)?; - Ok(()) + m.add_function(wrap_pyfunction!(no_args_py, m)?) } # Python::with_gil(|py| { @@ -103,7 +101,7 @@ The `#[pyo3]` attribute can be used to modify properties of the generated Python The `#[pyo3]` attribute can be used on individual arguments to modify properties of them in the generated function. It can take any combination of the following options: - - `#[pyo3(from_py_with = "...")]` + - `#[pyo3(from_py_with = ...)]` Set this on an option to specify a custom function to convert the function argument from Python to the desired Rust type, instead of using the default `FromPyObject` extraction. The function signature must be `fn(&Bound<'_, PyAny>) -> PyResult` where `T` is the Rust type of the argument. @@ -113,17 +111,16 @@ The `#[pyo3]` attribute can be used on individual arguments to modify properties use pyo3::prelude::*; fn get_length(obj: &Bound<'_, PyAny>) -> PyResult { - let length = obj.len()?; - Ok(length) + obj.len() } #[pyfunction] - fn object_length(#[pyo3(from_py_with = "get_length")] argument: usize) -> usize { + fn object_length(#[pyo3(from_py_with = get_length)] argument: usize) -> usize { argument } # Python::with_gil(|py| { - # let f = pyo3::wrap_pyfunction_bound!(object_length)(py).unwrap(); + # let f = pyo3::wrap_pyfunction!(object_length)(py).unwrap(); # assert_eq!(f.call1((vec![1, 2, 3],)).unwrap().extract::().unwrap(), 3); # }); ``` @@ -204,8 +201,7 @@ fn my_extension(m: &Bound<'_, PyModule>) -> PyResult<()> { x * 2 } - m.add_function(wrap_pyfunction!(double, m)?)?; - Ok(()) + m.add_function(wrap_pyfunction!(double, m)?) } ``` diff --git a/guide/src/function/error-handling.md b/guide/src/function/error-handling.md index f55fee90e54..0d7fae7976e 100644 --- a/guide/src/function/error-handling.md +++ b/guide/src/function/error-handling.md @@ -44,7 +44,7 @@ fn check_positive(x: i32) -> PyResult<()> { # # fn main(){ # Python::with_gil(|py|{ -# let fun = pyo3::wrap_pyfunction_bound!(check_positive, py).unwrap(); +# let fun = pyo3::wrap_pyfunction!(check_positive, py).unwrap(); # fun.call1((-1,)).unwrap_err(); # fun.call1((1,)).unwrap(); # }); @@ -72,7 +72,7 @@ fn parse_int(x: &str) -> Result { # fn main() { # Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction_bound!(parse_int, py).unwrap(); +# let fun = pyo3::wrap_pyfunction!(parse_int, py).unwrap(); # let value: usize = fun.call1(("5",)).unwrap().extract().unwrap(); # assert_eq!(value, 5); # }); @@ -132,7 +132,7 @@ fn connect(s: String) -> Result<(), CustomIOError> { fn main() { Python::with_gil(|py| { - let fun = pyo3::wrap_pyfunction_bound!(connect, py).unwrap(); + let fun = pyo3::wrap_pyfunction!(connect, py).unwrap(); let err = fun.call1(("0.0.0.0",)).unwrap_err(); assert!(err.is_instance_of::(py)); }); @@ -224,7 +224,7 @@ fn wrapped_get_x() -> Result { # fn main() { # Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction_bound!(wrapped_get_x, py).unwrap(); +# let fun = pyo3::wrap_pyfunction!(wrapped_get_x, py).unwrap(); # let value: usize = fun.call0().unwrap().extract().unwrap(); # assert_eq!(value, 5); # }); diff --git a/guide/src/function/signature.md b/guide/src/function/signature.md index b276fc457fb..431cad87bfd 100644 --- a/guide/src/function/signature.md +++ b/guide/src/function/signature.md @@ -2,7 +2,7 @@ The `#[pyfunction]` attribute also accepts parameters to control how the generated Python function accepts arguments. Just like in Python, arguments can be positional-only, keyword-only, or accept either. `*args` lists and `**kwargs` dicts can also be accepted. These parameters also work for `#[pymethods]` which will be introduced in the [Python Classes](../class.md) section of the guide. -Like Python, by default PyO3 accepts all arguments as either positional or keyword arguments. Most arguments are required by default, except for trailing `Option<_>` arguments, which are [implicitly given a default of `None`](#trailing-optional-arguments). This behaviour can be configured by the `#[pyo3(signature = (...))]` option which allows writing a signature in Python syntax. +Like Python, by default PyO3 accepts all arguments as either positional or keyword arguments. All arguments are required by default. This behaviour can be configured by the `#[pyo3(signature = (...))]` option which allows writing a signature in Python syntax. This section of the guide goes into detail about use of the `#[pyo3(signature = (...))]` option and its related option `#[pyo3(text_signature = "...")]` @@ -22,8 +22,7 @@ fn num_kwds(kwds: Option<&Bound<'_, PyDict>>) -> usize { #[pymodule] fn module_with_functions(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(num_kwds, m)?).unwrap(); - Ok(()) + m.add_function(wrap_pyfunction!(num_kwds, m)?) } ``` @@ -119,69 +118,6 @@ num=-1 > } > ``` -## Trailing optional arguments - -As a convenience, functions without a `#[pyo3(signature = (...))]` option will treat trailing `Option` arguments as having a default of `None`. In the example below, PyO3 will create `increment` with a signature of `increment(x, amount=None)`. - -```rust -use pyo3::prelude::*; - -/// Returns a copy of `x` increased by `amount`. -/// -/// If `amount` is unspecified or `None`, equivalent to `x + 1`. -#[pyfunction] -fn increment(x: u64, amount: Option) -> u64 { - x + amount.unwrap_or(1) -} -# -# fn main() -> PyResult<()> { -# Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction_bound!(increment, py)?; -# -# let inspect = PyModule::import_bound(py, "inspect")?.getattr("signature")?; -# let sig: String = inspect -# .call1((fun,))? -# .call_method0("__str__")? -# .extract()?; -# -# #[cfg(Py_3_8)] // on 3.7 the signature doesn't render b, upstream bug? -# assert_eq!(sig, "(x, amount=None)"); -# -# Ok(()) -# }) -# } -``` - -To make trailing `Option` arguments required, but still accept `None`, add a `#[pyo3(signature = (...))]` annotation. For the example above, this would be `#[pyo3(signature = (x, amount))]`: - -```rust -# use pyo3::prelude::*; -#[pyfunction] -#[pyo3(signature = (x, amount))] -fn increment(x: u64, amount: Option) -> u64 { - x + amount.unwrap_or(1) -} -# -# fn main() -> PyResult<()> { -# Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction_bound!(increment, py)?; -# -# let inspect = PyModule::import_bound(py, "inspect")?.getattr("signature")?; -# let sig: String = inspect -# .call1((fun,))? -# .call_method0("__str__")? -# .extract()?; -# -# #[cfg(Py_3_8)] // on 3.7 the signature doesn't render b, upstream bug? -# assert_eq!(sig, "(x, amount)"); -# -# Ok(()) -# }) -# } -``` - -To help avoid confusion, PyO3 requires `#[pyo3(signature = (...))]` when an `Option` argument is surrounded by arguments which aren't `Option`. - ## Making the function signature available to Python The function signature is exposed to Python via the `__text_signature__` attribute. PyO3 automatically generates this for every `#[pyfunction]` and all `#[pymethods]` directly from the Rust function, taking into account any override done with the `#[pyo3(signature = (...))]` option. @@ -204,12 +140,12 @@ fn add(a: u64, b: u64) -> u64 { # # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction_bound!(add, py)?; +# let fun = pyo3::wrap_pyfunction!(add, py)?; # # let doc: String = fun.getattr("__doc__")?.extract()?; # assert_eq!(doc, "This function adds two unsigned 64-bit integers."); # -# let inspect = PyModule::import_bound(py, "inspect")?.getattr("signature")?; +# let inspect = PyModule::import(py, "inspect")?.getattr("signature")?; # let sig: String = inspect # .call1((fun,))? # .call_method0("__str__")? @@ -252,12 +188,12 @@ fn add(a: u64, b: u64) -> u64 { # # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction_bound!(add, py)?; +# let fun = pyo3::wrap_pyfunction!(add, py)?; # # let doc: String = fun.getattr("__doc__")?.extract()?; # assert_eq!(doc, "This function adds two unsigned 64-bit integers."); # -# let inspect = PyModule::import_bound(py, "inspect")?.getattr("signature")?; +# let inspect = PyModule::import(py, "inspect")?.getattr("signature")?; # let sig: String = inspect # .call1((fun,))? # .call_method0("__str__")? @@ -294,7 +230,7 @@ fn add(a: u64, b: u64) -> u64 { # # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction_bound!(add, py)?; +# let fun = pyo3::wrap_pyfunction!(add, py)?; # # let doc: String = fun.getattr("__doc__")?.extract()?; # assert_eq!(doc, "This function adds two unsigned 64-bit integers."); diff --git a/guide/src/getting-started.md b/guide/src/getting-started.md index 59cf5ba6ded..e2cc040bbd7 100644 --- a/guide/src/getting-started.md +++ b/guide/src/getting-started.md @@ -6,7 +6,7 @@ To get started using PyO3 you will need three things: a Rust toolchain, a Python ## Rust -First, make sure you have Rust installed on your system. If you haven't already done so, try following the instructions [here](https://www.rust-lang.org/tools/install). PyO3 runs on both the `stable` and `nightly` versions so you can choose whichever one fits you best. The minimum required Rust version is 1.56. +First, make sure you have Rust installed on your system. If you haven't already done so, try following the instructions [here](https://www.rust-lang.org/tools/install). PyO3 runs on both the `stable` and `nightly` versions so you can choose whichever one fits you best. The minimum required Rust version is 1.63. If you can run `rustc --version` and the version is new enough you're good to go! @@ -18,19 +18,14 @@ To use PyO3, you need at least Python 3.7. While you can simply use the default While you can use any virtualenv manager you like, we recommend the use of `pyenv` in particular if you want to develop or test for multiple different Python versions, so that is what the examples in this book will use. The installation instructions for `pyenv` can be found [here](https://github.com/pyenv/pyenv#getting-pyenv). (Note: To get the `pyenv activate` and `pyenv virtualenv` commands, you will also need to install the [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv) plugin. The [pyenv installer](https://github.com/pyenv/pyenv-installer#installation--update--uninstallation) will install both together.) -If you intend to run Python from Rust (for example in unit tests) you should set the following environment variable when installing a new Python version using `pyenv`: -```bash -PYTHON_CONFIGURE_OPTS="--enable-shared" -``` +It can be useful to keep the sources used when installing using `pyenv` so that future debugging can see the original source files. This can be done by passing the `--keep` flag as part of the `pyenv install` command. For example: ```bash -env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.12 +pyenv install 3.12 --keep ``` -You can read more about `pyenv`'s configuration options [here](https://github.com/pyenv/pyenv/blob/master/plugins/python-build/README.md#building-with---enable-shared). - ### Building There are a number of build and Python package management systems such as [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](./building-and-distribution.md#manual-builds). We recommend the use of `maturin`, which you can install [here](https://maturin.rs/installation.html). It is developed to work with PyO3 and provides the most "batteries included" experience, especially if you are aiming to publish to PyPI. `maturin` is just a Python package, so you can add it in the same way you already install Python packages. @@ -164,8 +159,7 @@ fn sum_as_string(a: usize, b: usize) -> PyResult { /// import the module. #[pymodule] fn pyo3_example(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; - Ok(()) + m.add_function(wrap_pyfunction!(sum_as_string, m)?) } ``` @@ -181,3 +175,7 @@ $ python ``` For more instructions on how to use Python code from Rust, see the [Python from Rust](python-from-rust.md) page. + +## Maturin Import Hook + +In development, any changes in the code would require running `maturin develop` before testing. To streamline the development process, you may want to install [Maturin Import Hook](https://github.com/PyO3/maturin-import-hook) which will run `maturin develop` automatically when the library with code changes is being imported. diff --git a/guide/src/index.md b/guide/src/index.md index fe8b76b69c9..4e85d25d52b 100644 --- a/guide/src/index.md +++ b/guide/src/index.md @@ -10,15 +10,6 @@ The rough order of material in this user guide is as follows: Please choose from the chapters on the left to jump to individual topics, or continue below to start with PyO3's README. -
- -⚠️ Warning: API update in progress 🛠️ - -PyO3 0.21 has introduced a significant new API, termed the "Bound" API after the new smart pointer `Bound`. - -While most of this guide has been updated to the new API, it is possible some stray references to the older "GIL Refs" API such as `&PyAny` remain. -
-
{{#include ../../README.md}} diff --git a/guide/src/memory.md b/guide/src/memory.md deleted file mode 100644 index cd4af4f8f13..00000000000 --- a/guide/src/memory.md +++ /dev/null @@ -1,285 +0,0 @@ -# Memory management - -
- -⚠️ Warning: API update in progress 🛠️ - -PyO3 0.21 has introduced a significant new API, termed the "Bound" API after the new smart pointer `Bound`. - -This section on memory management is heavily weighted towards the now-deprecated "GIL Refs" API, which suffered from the drawbacks detailed here as well as CPU overheads. - -See [the smart pointer types](./types.md#pyo3s-smart-pointers) for description on the new, simplified, memory model of the Bound API, which is built as a thin wrapper on Python reference counting. -
- -Rust and Python have very different notions of memory management. Rust has -a strict memory model with concepts of ownership, borrowing, and lifetimes, -where memory is freed at predictable points in program execution. Python has -a looser memory model in which variables are reference-counted with shared, -mutable state by default. A global interpreter lock (GIL) is needed to prevent -race conditions, and a garbage collector is needed to break reference cycles. -Memory in Python is freed eventually by the garbage collector, but not usually -in a predictable way. - -PyO3 bridges the Rust and Python memory models with two different strategies for -accessing memory allocated on Python's heap from inside Rust. These are -GIL Refs such as `&'py PyAny`, and GIL-independent `Py` smart pointers. - -## GIL-bound memory - -PyO3's GIL Refs such as `&'py PyAny` make PyO3 more ergonomic to -use by ensuring that their lifetime can never be longer than the duration the -Python GIL is held. This means that most of PyO3's API can assume the GIL is -held. (If PyO3 could not assume this, every PyO3 API would need to take a -`Python` GIL token to prove that the GIL is held.) This allows us to write -very simple and easy-to-understand programs like this: - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -Python::with_gil(|py| -> PyResult<()> { - #[allow(deprecated)] // py.eval() is part of the GIL Refs API - let hello = py - .eval("\"Hello World!\"", None, None)? - .downcast::()?; - println!("Python says: {}", hello); - Ok(()) -})?; -# Ok(()) -# } -``` - -Internally, calling `Python::with_gil()` creates a `GILPool` which owns the -memory pointed to by the reference. In the example above, the lifetime of the -reference `hello` is bound to the `GILPool`. When the `with_gil()` closure ends -the `GILPool` is also dropped and the Python reference counts of the variables -it owns are decreased, releasing them to the Python garbage collector. Most -of the time we don't have to think about this, but consider the following: - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -Python::with_gil(|py| -> PyResult<()> { - for _ in 0..10 { - #[allow(deprecated)] // py.eval() is part of the GIL Refs API - let hello = py - .eval("\"Hello World!\"", None, None)? - .downcast::()?; - println!("Python says: {}", hello); - } - // There are 10 copies of `hello` on Python's heap here. - Ok(()) -})?; -# Ok(()) -# } -``` - -We might assume that the `hello` variable's memory is freed at the end of each -loop iteration, but in fact we create 10 copies of `hello` on Python's heap. -This may seem surprising at first, but it is completely consistent with Rust's -memory model. The `hello` variable is dropped at the end of each loop, but it -is only a reference to the memory owned by the `GILPool`, and its lifetime is -bound to the `GILPool`, not the for loop. The `GILPool` isn't dropped until -the end of the `with_gil()` closure, at which point the 10 copies of `hello` -are finally released to the Python garbage collector. - -
- -⚠️ Warning: `GILPool` is no longer the preferred way to manage memory with PyO3 🛠️ - -PyO3 0.21 has introduced a new API known as the Bound API, which doesn't have the same surprising results. Instead, each `Bound` smart pointer releases the Python reference immediately on drop. See [the smart pointer types](./types.md#pyo3s-smart-pointers) for more details. -
- - -In general we don't want unbounded memory growth during loops! One workaround -is to acquire and release the GIL with each iteration of the loop. - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -for _ in 0..10 { - Python::with_gil(|py| -> PyResult<()> { - #[allow(deprecated)] // py.eval() is part of the GIL Refs API - let hello = py - .eval("\"Hello World!\"", None, None)? - .downcast::()?; - println!("Python says: {}", hello); - Ok(()) - })?; // only one copy of `hello` at a time -} -# Ok(()) -# } -``` - -It might not be practical or performant to acquire and release the GIL so many -times. Another workaround is to work with the `GILPool` object directly, but -this is unsafe. - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -Python::with_gil(|py| -> PyResult<()> { - for _ in 0..10 { - #[allow(deprecated)] // `new_pool` is not needed in code not using the GIL Refs API - let pool = unsafe { py.new_pool() }; - let py = pool.python(); - #[allow(deprecated)] // py.eval() is part of the GIL Refs API - let hello = py - .eval("\"Hello World!\"", None, None)? - .downcast::()?; - println!("Python says: {}", hello); - } - Ok(()) -})?; -# Ok(()) -# } -``` - -The unsafe method `Python::new_pool` allows you to create a nested `GILPool` -from which you can retrieve a new `py: Python` GIL token. Variables created -with this new GIL token are bound to the nested `GILPool` and will be released -when the nested `GILPool` is dropped. Here, the nested `GILPool` is dropped -at the end of each loop iteration, before the `with_gil()` closure ends. - -When doing this, you must be very careful to ensure that once the `GILPool` is -dropped you do not retain access to any owned references created after the -`GILPool` was created. Read the -[documentation for `Python::new_pool()`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.new_pool) -for more information on safety. - -This memory management can also be applicable when writing extension modules. -`#[pyfunction]` and `#[pymethods]` will create a `GILPool` which lasts the entire -function call, releasing objects when the function returns. Most functions only create -a few objects, meaning this doesn't have a significant impact. Occasionally functions -with long complex loops may need to use `Python::new_pool` as shown above. - -
- -⚠️ Warning: `GILPool` is no longer the preferred way to manage memory with PyO3 🛠️ - -PyO3 0.21 has introduced a new API known as the Bound API, which doesn't have the same surprising results. Instead, each `Bound` smart pointer releases the Python reference immediately on drop. See [the smart pointer types](./types.md#pyo3s-smart-pointers) for more details. -
- -## GIL-independent memory - -Sometimes we need a reference to memory on Python's heap that can outlive the -GIL. Python's `Py` is analogous to `Arc`, but for variables whose -memory is allocated on Python's heap. Cloning a `Py` increases its -internal reference count just like cloning `Arc`. The smart pointer can -outlive the "GIL is held" period in which it was created. It isn't magic, -though. We need to reacquire the GIL to access the memory pointed to by the -`Py`. - -What happens to the memory when the last `Py` is dropped and its -reference count reaches zero? It depends whether or not we are holding the GIL. - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -Python::with_gil(|py| -> PyResult<()> { - #[allow(deprecated)] // py.eval() is part of the GIL Refs API - let hello: Py = py.eval("\"Hello World!\"", None, None)?.extract()?; - #[allow(deprecated)] // as_ref is part of the GIL Refs API - { - println!("Python says: {}", hello.as_ref(py)); - } - Ok(()) -})?; -# Ok(()) -# } -``` - -At the end of the `Python::with_gil()` closure `hello` is dropped, and then the -GIL is dropped. Since `hello` is dropped while the GIL is still held by the -current thread, its memory is released to the Python garbage collector -immediately. - -This example wasn't very interesting. We could have just used a GIL-bound -`&PyString` reference. What happens when the last `Py` is dropped while -we are *not* holding the GIL? - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -let hello: Py = Python::with_gil(|py| { - #[allow(deprecated)] // py.eval() is part of the GIL Refs API - py.eval("\"Hello World!\"", None, None)?.extract() -})?; -// Do some stuff... -// Now sometime later in the program we want to access `hello`. -Python::with_gil(|py| { - #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. - let hello = hello.as_ref(py); - println!("Python says: {}", hello); -}); -// Now we're done with `hello`. -drop(hello); // Memory *not* released here. -// Sometime later we need the GIL again for something... -Python::with_gil(|py| - // Memory for `hello` is released here. -# () -); -# Ok(()) -# } -``` - -When `hello` is dropped *nothing* happens to the pointed-to memory on Python's -heap because nothing _can_ happen if we're not holding the GIL. Fortunately, -the memory isn't leaked. PyO3 keeps track of the memory internally and will -release it the next time we acquire the GIL. - -We can avoid the delay in releasing memory if we are careful to drop the -`Py` while the GIL is held. - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -#[allow(deprecated)] // py.eval() is part of the GIL Refs API -let hello: Py = - Python::with_gil(|py| py.eval("\"Hello World!\"", None, None)?.extract())?; -// Do some stuff... -// Now sometime later in the program: -Python::with_gil(|py| { - #[allow(deprecated)] // as_ref is part of the GIL Refs API - { - println!("Python says: {}", hello.as_ref(py)); - } - drop(hello); // Memory released here. -}); -# Ok(()) -# } -``` - -We could also have used `Py::into_ref()`, which consumes `self`, instead of -`Py::as_ref()`. But note that in addition to being slower than `as_ref()`, -`into_ref()` binds the memory to the lifetime of the `GILPool`, which means -that rather than being released immediately, the memory will not be released -until the GIL is dropped. - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -#[allow(deprecated)] // py.eval() is part of the GIL Refs API -let hello: Py = - Python::with_gil(|py| py.eval("\"Hello World!\"", None, None)?.extract())?; -// Do some stuff... -// Now sometime later in the program: -Python::with_gil(|py| { - #[allow(deprecated)] // into_ref is part of the GIL Refs API - { - println!("Python says: {}", hello.into_ref(py)); - } - // Memory not released yet. - // Do more stuff... - // Memory released here at end of `with_gil()` closure. -}); -# Ok(()) -# } -``` diff --git a/guide/src/migration.md b/guide/src/migration.md index d855f69d396..35126dfcaef 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -3,10 +3,415 @@ This guide can help you upgrade code through breaking changes from one PyO3 version to the next. For a detailed list of all changes, see the [CHANGELOG](changelog.md). -## from 0.20.* to 0.21 +## from 0.22.* to 0.23
Click to expand +PyO3 0.23 is a significant rework of PyO3's internals for two major improvements: + - Support of Python 3.13's new freethreaded build (aka "3.13t") + - Rework of to-Python conversions with a new `IntoPyObject` trait. + +These changes are both substantial and reasonable efforts have been made to allow as much code as possible to continue to work as-is despite the changes. The impacts are likely to be seen in three places when upgrading: + - PyO3's data structures [are now thread-safe](#free-threaded-python-support) instead of reliant on the GIL for synchronization. In particular, `#[pyclass]` types are [now required to be `Sync`](./class/thread-safety.md). + - The [`IntoPyObject` trait](#new-intopyobject-trait-unifies-to-python-conversions) may need to be implemented for types in your codebase. In most cases this can simply be done with [`#[derive(IntoPyObject)]`](#intopyobject-and-intopyobjectref-derive-macros). There will be many deprecation warnings from the replacement of `IntoPy` and `ToPyObject` traits. + - There will be many deprecation warnings from the [final removal of the `gil-refs` feature](#gil-refs-feature-removed), which opened up API space for a cleanup and simplification to PyO3's "Bound" API. + +The sections below discuss the rationale and details of each change in more depth. +
+ +### Free-threaded Python Support +
+Click to expand + +PyO3 0.23 introduces initial support for the new free-threaded build of +CPython 3.13, aka "3.13t". + +Because this build allows multiple Python threads to operate simultaneously on underlying Rust data, the `#[pyclass]` macro now requires that types it operates on implement `Sync`. + +Aside from the change to `#[pyclass]`, most features of PyO3 work unchanged, as the changes have been to the internal data structures to make them thread-safe. An example of this is the `GILOnceCell` type, which used the GIL to synchronize single-initialization. It now uses internal locks to guarantee that only one write ever succeeds, however it allows for multiple racing runs of the initialization closure. It may be preferable to instead use `std::sync::OnceLock` in combination with the `pyo3::sync::OnceLockExt` trait which adds `OnceLock::get_or_init_py_attached` for single-initialization where the initialization closure is guaranteed only ever to run once and without deadlocking with the GIL. + +Future PyO3 versions will likely add more traits and data structures to make working with free-threaded Python easier. + +Some features are unaccessible on the free-threaded build: + - The `GILProtected` type, which relied on the GIL to expose synchronized access to inner contents + - `PyList::get_item_unchecked`, which cannot soundly be used due to races between time-of-check and time-of-use + +If you make use of these features then you will need to account for the +unavailability of the API in the free-threaded build. One way to handle it is via conditional compilation -- extensions can use `pyo3-build-config` to get access to a `#[cfg(Py_GIL_DISABLED)]` guard. + +See [the guide section on free-threaded Python](free-threading.md) for more details about supporting free-threaded Python in your PyO3 extensions. +
+ +### New `IntoPyObject` trait unifies to-Python conversions +
+Click to expand + +PyO3 0.23 introduces a new `IntoPyObject` trait to convert Rust types into Python objects which replaces both `IntoPy` and `ToPyObject`. +Notable features of this new trait include: +- conversions can now return an error +- it is designed to work efficiently for both `T` owned types and `&T` references +- compared to `IntoPy` the generic `T` moved into an associated type, so + - there is now only one way to convert a given type + - the output type is stronger typed and may return any Python type instead of just `PyAny` +- byte collections are specialized to convert into `PyBytes` now, see [below](#to-python-conversions-changed-for-byte-collections-vecu8-u8-n-and-smallvecu8-n) +- `()` (unit) is now only specialized in return position of `#[pyfunction]` and `#[pymethods]` to return `None`, in normal usage it converts into an empty `PyTuple` + +All PyO3 provided types as well as `#[pyclass]`es already implement `IntoPyObject`. Other types will +need to adapt an implementation of `IntoPyObject` to stay compatible with the Python APIs. In many cases +the new [`#[derive(IntoPyObject)]`](#intopyobject-and-intopyobjectref-derive-macros) macro can be used instead of +[manual implementations](#intopyobject-manual-implementation). + +Since `IntoPyObject::into_pyobject` may return either a `Bound` or `Borrowed`, you may find the [`BoundObject`](conversions/traits.md#boundobject-for-conversions-that-may-be-bound-or-borrowed) trait to be useful to write code that generically handles either type of smart pointer. + +Together with the introduction of `IntoPyObject` the old conversion traits `ToPyObject` and `IntoPy` +are deprecated and will be removed in a future PyO3 version. + +#### `IntoPyObject` and `IntoPyObjectRef` derive macros + +To implement the new trait you may use the new `IntoPyObject` and `IntoPyObjectRef` derive macros as below. + +```rust +# use pyo3::prelude::*; +#[derive(IntoPyObject, IntoPyObjectRef)] +struct Struct { + count: usize, + obj: Py, +} +``` + +The `IntoPyObjectRef` derive macro derives implementations for references (e.g. for `&Struct` in the example above), which is a replacement for the `ToPyObject` trait. + +#### `IntoPyObject` manual implementation + +Before: +```rust,ignore +# use pyo3::prelude::*; +# #[allow(dead_code)] +struct MyPyObjectWrapper(PyObject); + +impl IntoPy for MyPyObjectWrapper { + fn into_py(self, py: Python<'_>) -> PyObject { + self.0 + } +} + +impl ToPyObject for MyPyObjectWrapper { + fn to_object(&self, py: Python<'_>) -> PyObject { + self.0.clone_ref(py) + } +} +``` + +After: +```rust +# use pyo3::prelude::*; +# #[allow(dead_code)] +# struct MyPyObjectWrapper(PyObject); + +impl<'py> IntoPyObject<'py> for MyPyObjectWrapper { + type Target = PyAny; // the Python type + type Output = Bound<'py, Self::Target>; // in most cases this will be `Bound` + type Error = std::convert::Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(self.0.into_bound(py)) + } +} + +// `ToPyObject` implementations should be converted to implementations on reference types +impl<'a, 'py> IntoPyObject<'py> for &'a MyPyObjectWrapper { + type Target = PyAny; + type Output = Borrowed<'a, 'py, Self::Target>; // `Borrowed` can be used to optimized reference counting + type Error = std::convert::Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(self.0.bind_borrowed(py)) + } +} +``` +
+ +### To-Python conversions changed for byte collections (`Vec`, `[u8; N]` and `SmallVec<[u8; N]>`). +
+Click to expand + +With the introduction of the `IntoPyObject` trait, PyO3's macros now prefer `IntoPyObject` implementations over `IntoPy` when producing Python values. This applies to `#[pyfunction]` and `#[pymethods]` return values and also fields accessed via `#[pyo3(get)]`. + +This change has an effect on functions and methods returning _byte_ collections like +- `Vec` +- `[u8; N]` +- `SmallVec<[u8; N]>` + +In their new `IntoPyObject` implementation these will now turn into `PyBytes` rather than a +`PyList`. All other `T`s are unaffected and still convert into a `PyList`. + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +#[pyfunction] +fn foo() -> Vec { // would previously turn into a `PyList`, now `PyBytes` + vec![0, 1, 2, 3] +} + +#[pyfunction] +fn bar() -> Vec { // unaffected, returns `PyList` + vec![0, 1, 2, 3] +} +``` + +If this conversion is _not_ desired, consider building a list manually using `PyList::new`. + +The following types were previously _only_ implemented for `u8` and now allow other `T`s turn into `PyList`: +- `&[T]` +- `Cow<[T]>` + +This is purely additional and should just extend the possible return types. + +
+ +### `gil-refs` feature removed +
+Click to expand + +PyO3 0.23 completes the removal of the "GIL Refs" API in favour of the new "Bound" API introduced in PyO3 0.21. + +With the removal of the old API, many "Bound" API functions which had been introduced with `_bound` suffixes no longer need the suffixes as these names have been freed up. For example, `PyTuple::new_bound` is now just `PyTuple::new` (the existing name remains but is deprecated). + +Before: + +```rust +# #![allow(deprecated)] +# use pyo3::prelude::*; +# use pyo3::types::PyTuple; +# fn main() { +# Python::with_gil(|py| { +// For example, for PyTuple. Many such APIs have been changed. +let tup = PyTuple::new_bound(py, [1, 2, 3]); +# }) +# } +``` + +After: + +```rust +# use pyo3::prelude::*; +# use pyo3::types::PyTuple; +# fn main() { +# Python::with_gil(|py| { +// For example, for PyTuple. Many such APIs have been changed. +let tup = PyTuple::new(py, [1, 2, 3]); +# }) +# } +``` + +#### `IntoPyDict` trait adjusted for removal of `gil-refs` + +As part of this API simplification, the `IntoPyDict` trait has had a small breaking change: `IntoPyDict::into_py_dict_bound` method has been renamed to `IntoPyDict::into_py_dict`. It is also now fallible as part of the `IntoPyObject` trait addition. + +If you implemented `IntoPyDict` for your type, you should implement `into_py_dict` instead of `into_py_dict_bound`. The old name is still available for calling but deprecated. + +Before: + +```rust,ignore +# use pyo3::prelude::*; +# use pyo3::types::{PyDict, IntoPyDict}; +# use std::collections::HashMap; + +struct MyMap(HashMap); + +impl IntoPyDict for MyMap +where + K: ToPyObject, + V: ToPyObject, +{ + fn into_py_dict_bound(self, py: Python<'_>) -> Bound<'_, PyDict> { + let dict = PyDict::new_bound(py); + for (key, value) in self.0 { + dict.set_item(key, value) + .expect("Failed to set_item on dict"); + } + dict + } +} +``` + +After: + +```rust +# use pyo3::prelude::*; +# use pyo3::types::{PyDict, IntoPyDict}; +# use std::collections::HashMap; + +# #[allow(dead_code)] +struct MyMap(HashMap); + +impl<'py, K, V> IntoPyDict<'py> for MyMap +where + K: IntoPyObject<'py>, + V: IntoPyObject<'py>, +{ + fn into_py_dict(self, py: Python<'py>) -> PyResult> { + let dict = PyDict::new(py); + for (key, value) in self.0 { + dict.set_item(key, value)?; + } + Ok(dict) + } +} +``` +
+ +## from 0.21.* to 0.22 + +### Deprecation of `gil-refs` feature continues +
+Click to expand + +Following the introduction of the "Bound" API in PyO3 0.21 and the planned removal of the "GIL Refs" API, all functionality related to GIL Refs is now gated behind the `gil-refs` feature and emits a deprecation warning on use. + +See the 0.21 migration entry for help upgrading. +
+ +### Deprecation of implicit default for trailing optional arguments +
+Click to expand + +With `pyo3` 0.22 the implicit `None` default for trailing `Option` type argument is deprecated. To migrate, place a `#[pyo3(signature = (...))]` attribute on affected functions or methods and specify the desired behavior. +The migration warning specifies the corresponding signature to keep the current behavior. With 0.23 the signature will be required for any function containing `Option` type parameters to prevent accidental +and unnoticed changes in behavior. With 0.24 this restriction will be lifted again and `Option` type arguments will be treated as any other argument _without_ special handling. + +Before: + +```rust +# #![allow(deprecated, dead_code)] +# use pyo3::prelude::*; +#[pyfunction] +fn increment(x: u64, amount: Option) -> u64 { + x + amount.unwrap_or(1) +} +``` + +After: + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +#[pyfunction] +#[pyo3(signature = (x, amount=None))] +fn increment(x: u64, amount: Option) -> u64 { + x + amount.unwrap_or(1) +} +``` +
+ +### `Py::clone` is now gated behind the `py-clone` feature +
+Click to expand +If you rely on `impl Clone for Py` to fulfil trait requirements imposed by existing Rust code written without PyO3-based code in mind, the newly introduced feature `py-clone` must be enabled. + +However, take care to note that the behaviour is different from previous versions. If `Clone` was called without the GIL being held, we tried to delay the application of these reference count increments until PyO3-based code would re-acquire it. This turned out to be impossible to implement in a sound manner and hence was removed. Now, if `Clone` is called without the GIL being held, we panic instead for which calling code might not be prepared. + +It is advised to migrate off the `py-clone` feature. The simplest way to remove dependency on `impl Clone for Py` is to wrap `Py` as `Arc>` and use cloning of the arc. + +Related to this, we also added a `pyo3_disable_reference_pool` conditional compilation flag which removes the infrastructure necessary to apply delayed reference count decrements implied by `impl Drop for Py`. They do not appear to be a soundness hazard as they should lead to memory leaks in the worst case. However, the global synchronization adds significant overhead to cross the Python-Rust boundary. Enabling this feature will remove these costs and make the `Drop` implementation abort the process if called without the GIL being held instead. +
+ +### Require explicit opt-in for comparison for simple enums +
+Click to expand + +With `pyo3` 0.22 the new `#[pyo3(eq)]` options allows automatic implementation of Python equality using Rust's `PartialEq`. Previously simple enums automatically implemented equality in terms of their discriminants. To make PyO3 more consistent, this automatic equality implementation is deprecated in favour of having opt-ins for all `#[pyclass]` types. Similarly, simple enums supported comparison with integers, which is not covered by Rust's `PartialEq` derive, so has been split out into the `#[pyo3(eq_int)]` attribute. + +To migrate, place a `#[pyo3(eq, eq_int)]` attribute on simple enum classes. + +Before: + +```rust +# #![allow(deprecated, dead_code)] +# use pyo3::prelude::*; +#[pyclass] +enum SimpleEnum { + VariantA, + VariantB = 42, +} +``` + +After: + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] +enum SimpleEnum { + VariantA, + VariantB = 42, +} +``` +
+ +### `PyType::name` reworked to better match Python `__name__` +
+Click to expand + +This function previously would try to read directly from Python type objects' C API field (`tp_name`), in which case it +would return a `Cow::Borrowed`. However the contents of `tp_name` don't have well-defined semantics. + +Instead `PyType::name()` now returns the equivalent of Python `__name__` and returns `PyResult>`. + +The closest equivalent to PyO3 0.21's version of `PyType::name()` has been introduced as a new function `PyType::fully_qualified_name()`, +which is equivalent to `__module__` and `__qualname__` joined as `module.qualname`. + +Before: + +```rust,ignore +# #![allow(deprecated, dead_code)] +# use pyo3::prelude::*; +# use pyo3::types::{PyBool}; +# fn main() -> PyResult<()> { +Python::with_gil(|py| { + let bool_type = py.get_type_bound::(); + let name = bool_type.name()?.into_owned(); + println!("Hello, {}", name); + + let mut name_upper = bool_type.name()?; + name_upper.to_mut().make_ascii_uppercase(); + println!("Hello, {}", name_upper); + + Ok(()) +}) +# } +``` + +After: + +```rust,ignore +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use pyo3::types::{PyBool}; +# fn main() -> PyResult<()> { +Python::with_gil(|py| { + let bool_type = py.get_type_bound::(); + let name = bool_type.name()?; + println!("Hello, {}", name); + + // (if the full dotted path was desired, switch from `name()` to `fully_qualified_name()`) + let mut name_upper = bool_type.fully_qualified_name()?.to_string(); + name_upper.make_ascii_uppercase(); + println!("Hello, {}", name_upper); + + Ok(()) +}) +# } +``` +
+ + + +## from 0.20.* to 0.21 +
+Click to expand + PyO3 0.21 introduces a new `Bound<'py, T>` smart pointer which replaces the existing "GIL Refs" API to interact with Python objects. For example, in PyO3 0.20 the reference `&'py PyAny` would be used to interact with Python objects. In PyO3 0.21 the updated type is `Bound<'py, PyAny>`. Making this change moves Rust ownership semantics out of PyO3's internals and into user code. This change fixes [a known soundness edge case of interaction with gevent](https://github.com/PyO3/pyo3/issues/3668) as well as improves CPU and [memory performance](https://github.com/PyO3/pyo3/issues/1056). For a full history of discussion see https://github.com/PyO3/pyo3/issues/3382. The "GIL Ref" `&'py PyAny` and similar types such as `&'py PyDict` continue to be available as a deprecated API. Due to the advantages of the new API it is advised that all users make the effort to upgrade as soon as possible. @@ -22,7 +427,7 @@ The following sections are laid out in this order.
### Enable the `gil-refs` feature -
+
Click to expand To make the transition for the PyO3 ecosystem away from the GIL Refs API as smooth as possible, in PyO3 0.21 no APIs consuming or producing GIL Refs have been altered. Instead, variants using `Bound` smart pointers have been introduced, for example `PyTuple::new_bound` which returns `Bound` is the replacement form of `PyTuple::new`. The GIL Ref APIs have been deprecated, but to make migration easier it is possible to disable these deprecation warnings by enabling the `gil-refs` feature. @@ -49,18 +454,18 @@ pyo3 = { version = "0.21", features = ["gil-refs"] }
### `PyTypeInfo` and `PyTryFrom` have been adjusted -
+
Click to expand The `PyTryFrom` trait has aged poorly, its `try_from` method now conflicts with `TryFrom::try_from` in the 2021 edition prelude. A lot of its functionality was also duplicated with `PyTypeInfo`. -To tighten up the PyO3 traits as part of the deprecation of the GIL Refs API the `PyTypeInfo` trait has had a simpler companion `PyTypeCheck`. The methods [`PyAny::downcast`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.downcast) and [`PyAny::downcast_exact`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.downcast_exact) no longer use `PyTryFrom` as a bound, instead using `PyTypeCheck` and `PyTypeInfo` respectively. +To tighten up the PyO3 traits as part of the deprecation of the GIL Refs API the `PyTypeInfo` trait has had a simpler companion `PyTypeCheck`. The methods `PyAny::downcast` and `PyAny::downcast_exact` no longer use `PyTryFrom` as a bound, instead using `PyTypeCheck` and `PyTypeInfo` respectively. To migrate, switch all type casts to use `obj.downcast()` instead of `try_from(obj)` (and similar for `downcast_exact`). Before: -```rust +```rust,ignore # #![allow(deprecated)] # use pyo3::prelude::*; # use pyo3::types::{PyInt, PyList}; @@ -75,7 +480,7 @@ Python::with_gil(|py| { After: -```rust +```rust,ignore # use pyo3::prelude::*; # use pyo3::types::{PyInt, PyList}; # fn main() -> PyResult<()> { @@ -92,15 +497,14 @@ Python::with_gil(|py| {
### `Iter(A)NextOutput` are deprecated -
+
Click to expand The `__next__` and `__anext__` magic methods can now return any type convertible into Python objects directly just like all other `#[pymethods]`. The `IterNextOutput` used by `__next__` and `IterANextOutput` used by `__anext__` are subsequently deprecated. Most importantly, this change allows returning an awaitable from `__anext__` without non-sensically wrapping it into `Yield` or `Some`. Only the return types `Option` and `Result, E>` are still handled in a special manner where `Some(val)` yields `val` and `None` stops iteration. Starting with an implementation of a Python iterator using `IterNextOutput`, e.g. -```rust -#![allow(deprecated)] +```rust,ignore use pyo3::prelude::*; use pyo3::iter::IterNextOutput; @@ -214,21 +618,21 @@ impl PyClassAsyncIter {
### `PyType::name` has been renamed to `PyType::qualname` -
+
Click to expand `PyType::name` has been renamed to `PyType::qualname` to indicate that it does indeed return the [qualified name](https://docs.python.org/3/glossary.html#term-qualified-name), matching the `__qualname__` attribute. The newly added `PyType::name` yields the full name including the module name now which corresponds to `__module__.__name__` on the level of attributes.
### `PyCell` has been deprecated -
+
Click to expand Interactions with Python objects implemented in Rust no longer need to go though `PyCell`. Instead iteractions with Python object now consistently go through `Bound` or `Py` independently of whether `T` is native Python object or a `#[pyclass]` implemented in Rust. Use `Bound::new` or `Py::new` respectively to create and `Bound::borrow(_mut)` / `Py::borrow(_mut)` to borrow the Rust object.
### Migrating from the GIL Refs API to `Bound` -
+
Click to expand To minimise breakage of code using the GIL Refs API, the `Bound` smart pointer has been introduced by adding complements to all functions which accept or return GIL Refs. This allows code to migrate by replacing the deprecated APIs with the new ones. @@ -326,7 +730,7 @@ Despite a large amount of deprecations warnings produced by PyO3 to aid with the
### Deactivating the `gil-refs` feature -
+
Click to expand As a final step of migration, deactivating the `gil-refs` feature will set up code for best performance and is intended to set up a forward-compatible API for PyO3 0.22. @@ -339,12 +743,12 @@ To make PyO3's core functionality continue to work while the GIL Refs API is in PyO3 0.21 has introduced the [`PyBackedStr`]({{#PYO3_DOCS_URL}}/pyo3/pybacked/struct.PyBackedStr.html) and [`PyBackedBytes`]({{#PYO3_DOCS_URL}}/pyo3/pybacked/struct.PyBackedBytes.html) types to help with this case. The easiest way to avoid lifetime challenges from extracting `&str` is to use these. For more complex types like `Vec<&str>`, is now impossible to extract directly from a Python object and `Vec` is the recommended upgrade path. -A key thing to note here is because extracting to these types now ties them to the input lifetime, some extremely common patterns may need to be split into multiple Rust lines. For example, the following snippet of calling `.extract::<&str>()` directly on the result of `.getattr()` needs to be adjusted when deactivating the `gil-refs-migration` feature. +A key thing to note here is because extracting to these types now ties them to the input lifetime, some extremely common patterns may need to be split into multiple Rust lines. For example, the following snippet of calling `.extract::<&str>()` directly on the result of `.getattr()` needs to be adjusted when deactivating the `gil-refs` feature. Before: -```rust -# #[cfg(feature = "gil-refs-migration")] { +```rust,ignore +# #[cfg(feature = "gil-refs")] { # use pyo3::prelude::*; # use pyo3::types::{PyList, PyType}; # fn example<'py>(py: Python<'py>) -> PyResult<()> { @@ -360,7 +764,7 @@ assert_eq!(name, "list"); After: -```rust +```rust,ignore # #[cfg(any(not(Py_LIMITED_API), Py_3_10))] { # use pyo3::prelude::*; # use pyo3::types::{PyList, PyType}; @@ -381,7 +785,7 @@ To avoid needing to worry about lifetimes at all, it is also possible to use the The following example uses the same snippet as those just above, but this time the final extracted type is `PyBackedStr`: -```rust +```rust,ignore # use pyo3::prelude::*; # use pyo3::types::{PyList, PyType}; # fn example<'py>(py: Python<'py>) -> PyResult<()> { @@ -393,6 +797,7 @@ assert_eq!(&*name, "list"); # } # Python::with_gil(example).unwrap(); ``` +
## from 0.19.* to 0.20 @@ -467,7 +872,7 @@ Python::with_gil(|py| -> PyResult<()> {
Click to expand -[Trailing `Option` arguments](./function/signature.md#trailing-optional-arguments) have an automatic default of `None`. To avoid unwanted changes when modifying function signatures, in PyO3 0.18 it was deprecated to have a required argument after an `Option` argument without using `#[pyo3(signature = (...))]` to specify the intended defaults. In PyO3 0.20, this becomes a hard error. +Trailing `Option` arguments have an automatic default of `None`. To avoid unwanted changes when modifying function signatures, in PyO3 0.18 it was deprecated to have a required argument after an `Option` argument without using `#[pyo3(signature = (...))]` to specify the intended defaults. In PyO3 0.20, this becomes a hard error. Before: @@ -641,7 +1046,7 @@ drop(second); The replacement is [`Python::with_gil`](https://docs.rs/pyo3/0.18.3/pyo3/marker/struct.Python.html#method.with_gil) which is more cumbersome but enforces the proper nesting by design, e.g. -```rust +```rust,ignore # #![allow(dead_code)] # use pyo3::prelude::*; @@ -666,7 +1071,7 @@ let second = Python::with_gil(|py| Object::new(py)); drop(first); drop(second); -// Or it ensure releasing the inner lock before the outer one. +// Or it ensures releasing the inner lock before the outer one. Python::with_gil(|py| { let first = Object::new(py); let second = Python::with_gil(|py| Object::new(py)); @@ -675,7 +1080,7 @@ Python::with_gil(|py| { }); ``` -Furthermore, `Python::acquire_gil` provides ownership of a `GILGuard` which can be freely stored and passed around. This is usually not helpful as it may keep the lock held for a long time thereby blocking progress in other parts of the program. Due to the generative lifetime attached to the GIL token supplied by `Python::with_gil`, the problem is avoided as the GIL token can only be passed down the call chain. Often, this issue can also be avoided entirely as any GIL-bound reference `&'py PyAny` implies access to a GIL token `Python<'py>` via the [`PyAny::py`](https://docs.rs/pyo3/latest/pyo3/types/struct.PyAny.html#method.py) method. +Furthermore, `Python::acquire_gil` provides ownership of a `GILGuard` which can be freely stored and passed around. This is usually not helpful as it may keep the lock held for a long time thereby blocking progress in other parts of the program. Due to the generative lifetime attached to the GIL token supplied by `Python::with_gil`, the problem is avoided as the GIL token can only be passed down the call chain. Often, this issue can also be avoided entirely as any GIL-bound reference `&'py PyAny` implies access to a GIL token `Python<'py>` via the [`PyAny::py`](https://docs.rs/pyo3/0.22.5/pyo3/types/struct.PyAny.html#method.py) method.
## from 0.17.* to 0.18 @@ -690,7 +1095,7 @@ Starting with PyO3 0.18, this is deprecated and a future PyO3 version will requi Before, x in the below example would be required to be passed from Python code: -```rust,compile_fail +```rust,compile_fail,ignore # #![allow(dead_code)] # use pyo3::prelude::*; @@ -739,9 +1144,9 @@ fn function_with_defaults(a: i32, b: i32, c: i32) {} # fn main() { # Python::with_gil(|py| { -# let simple = wrap_pyfunction_bound!(simple_function, py).unwrap(); +# let simple = wrap_pyfunction!(simple_function, py).unwrap(); # assert_eq!(simple.getattr("__text_signature__").unwrap().to_string(), "(a, b, c)"); -# let defaulted = wrap_pyfunction_bound!(function_with_defaults, py).unwrap(); +# let defaulted = wrap_pyfunction!(function_with_defaults, py).unwrap(); # assert_eq!(defaulted.getattr("__text_signature__").unwrap().to_string(), "(a, b=1, c=2)"); # }) # } @@ -833,6 +1238,7 @@ Python::with_gil(|py| { After, some type annotations may be necessary: ```rust +# #![allow(deprecated)] # use pyo3::prelude::*; # # fn main() { @@ -1089,7 +1495,7 @@ An additional advantage of using Rust's indexing conventions for these types is that these types can now also support Rust's indexing operators as part of a consistent API: -```rust +```rust,ignore #![allow(deprecated)] use pyo3::{Python, types::PyList}; @@ -1311,6 +1717,7 @@ After # #[allow(dead_code)] struct MyPyObjectWrapper(PyObject); +# #[allow(deprecated)] impl IntoPy for MyPyObjectWrapper { fn into_py(self, _py: Python<'_>) -> PyObject { self.0 @@ -1330,6 +1737,7 @@ let obj = PyObject::from_py(1.234, py); After: ```rust +# #![allow(deprecated)] # use pyo3::prelude::*; # Python::with_gil(|py| { let obj: PyObject = 1.234.into_py(py); @@ -1413,7 +1821,7 @@ There can be two fixes: ``` After: - ```rust + ```rust,ignore # #![allow(dead_code)] use pyo3::prelude::*; use std::sync::{Arc, Mutex}; @@ -1565,7 +1973,7 @@ For more, see [the constructor section](class.md#constructor) of this guide.
Click to expand -PyO3 0.9 introduces [`PyCell`], which is a [`RefCell`]-like object wrapper +PyO3 0.9 introduces `PyCell`, which is a [`RefCell`]-like object wrapper for ensuring Rust's rules regarding aliasing of references are upheld. For more detail, see the [Rust Book's section on Rust's rules of references](https://doc.rust-lang.org/book/ch04-02-references-and-borrowing.html#the-rules-of-references) @@ -1614,7 +2022,7 @@ However, for `#[pyproto]` and some functions, you need to manually fix the code. In 0.8 object creation was done with `PyRef::new` and `PyRefMut::new`. In 0.9 these have both been removed. To upgrade code, please use -[`PyCell::new`]({{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyCell.html#method.new) instead. +`PyCell::new` instead. If you need [`PyRef`] or [`PyRefMut`], just call `.borrow()` or `.borrow_mut()` on the newly-created `PyCell`. @@ -1744,7 +2152,6 @@ impl PySequenceProtocol for ByteSequence { [`FromPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.FromPyObject.html [`PyAny`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html -[`PyCell`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyCell.html [`PyBorrowMutError`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyBorrowMutError.html [`PyRef`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRef.html [`PyRefMut`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRef.html diff --git a/guide/src/module.md b/guide/src/module.md index c9c7f78aaf5..1e274c7c953 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -13,8 +13,7 @@ fn double(x: usize) -> usize { /// This module is implemented in Rust. #[pymodule] fn my_extension(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(double, m)?)?; - Ok(()) + m.add_function(wrap_pyfunction!(double, m)?) } ``` @@ -32,11 +31,9 @@ fn double(x: usize) -> usize { x * 2 } -#[pymodule] -#[pyo3(name = "custom_name")] +#[pymodule(name = "custom_name")] fn my_extension(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(double, m)?)?; - Ok(()) + m.add_function(wrap_pyfunction!(double, m)?) } ``` @@ -78,10 +75,9 @@ fn parent_module(m: &Bound<'_, PyModule>) -> PyResult<()> { } fn register_child_module(parent_module: &Bound<'_, PyModule>) -> PyResult<()> { - let child_module = PyModule::new_bound(parent_module.py(), "child_module")?; + let child_module = PyModule::new(parent_module.py(), "child_module")?; child_module.add_function(wrap_pyfunction!(func, &child_module)?)?; - parent_module.add_submodule(&child_module)?; - Ok(()) + parent_module.add_submodule(&child_module) } #[pyfunction] @@ -92,10 +88,11 @@ fn func() -> String { # Python::with_gil(|py| { # use pyo3::wrap_pymodule; # use pyo3::types::IntoPyDict; +# use pyo3::ffi::c_str; # let parent_module = wrap_pymodule!(parent_module)(py); -# let ctx = [("parent_module", parent_module)].into_py_dict_bound(py); +# let ctx = [("parent_module", parent_module)].into_py_dict(py).unwrap(); # -# py.run_bound("assert parent_module.child_module.func() == 'func'", None, Some(&ctx)).unwrap(); +# py.run(c_str!("assert parent_module.child_module.func() == 'func'"), None, Some(&ctx)).unwrap(); # }) ``` @@ -106,14 +103,12 @@ submodules by using `from parent_module import child_module`. For more informati It is not necessary to add `#[pymodule]` on nested modules, which is only required on the top-level module. -## Declarative modules (experimental) +## Declarative modules Another syntax based on Rust inline modules is also available to declare modules. -The `experimental-declarative-modules` feature must be enabled to use it. For example: ```rust -# #[cfg(feature = "experimental-declarative-modules")] # mod declarative_module_test { use pyo3::prelude::*; @@ -145,15 +140,42 @@ mod my_extension { #[pymodule_init] fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { // Arbitrary code to run at the module initialization - m.add("double2", m.getattr("double")?)?; - Ok(()) + m.add("double2", m.getattr("double")?) + } +} +# } +``` + +The `#[pymodule]` macro automatically sets the `module` attribute of the `#[pyclass]` macros declared inside of it with its name. +For nested modules, the name of the parent module is automatically added. +In the following example, the `Unit` class will have for `module` `my_extension.submodule` because it is properly nested +but the `Ext` class will have for `module` the default `builtins` because it not nested. + +```rust +# mod declarative_module_module_attr_test { +use pyo3::prelude::*; + +#[pyclass] +struct Ext; + +#[pymodule] +mod my_extension { + use super::*; + + #[pymodule_export] + use super::Ext; + + #[pymodule] + mod submodule { + use super::*; + // This is a submodule + + #[pyclass] // This will be part of the module + struct Unit; } } # } ``` +It is possible to customize the `module` value for a `#[pymodule]` with the `#[pyo3(module = "MY_MODULE")]` option. -Some changes are planned to this feature before stabilization, like automatically -filling submodules into `sys.modules` to allow easier imports (see [issue #759](https://github.com/PyO3/pyo3/issues/759)) -and filling the `module` argument of inlined `#[pyclass]` automatically with the proper module name. -Macro names might also change. -See [issue #3900](https://github.com/PyO3/pyo3/issues/3900) to track this feature progress. +You can provide the `submodule` argument to `pymodule()` for modules that are not top-level modules -- it is automatically set for modules nested inside of a `#[pymodule]`. diff --git a/guide/src/parallelism.md b/guide/src/parallelism.md index 792e0ed8de4..64ff1c8c9c0 100644 --- a/guide/src/parallelism.md +++ b/guide/src/parallelism.md @@ -1,6 +1,6 @@ # Parallelism -CPython has the infamous [Global Interpreter Lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock), which prevents several threads from executing Python bytecode in parallel. This makes threading in Python a bad fit for [CPU-bound](https://stackoverflow.com/questions/868568/) tasks and often forces developers to accept the overhead of multiprocessing. +CPython has the infamous [Global Interpreter Lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) (GIL), which prevents several threads from executing Python bytecode in parallel. This makes threading in Python a bad fit for [CPU-bound](https://en.wikipedia.org/wiki/CPU-bound) tasks and often forces developers to accept the overhead of multiprocessing. There is an experimental "free-threaded" version of CPython 3.13 that does not have a GIL, see the PyO3 docs on [free-threaded Python](./free-threading.md) for more information about that. In PyO3 parallelism can be easily achieved in Rust-only code. Let's take a look at our [word-count](https://github.com/PyO3/pyo3/blob/main/examples/word-count/src/lib.rs) example, where we have a `search` function that utilizes the [rayon](https://github.com/rayon-rs/rayon) crate to count words in parallel. ```rust,no_run @@ -117,4 +117,61 @@ test_word_count_python_sequential 27.3985 (15.82) 45.452 You can see that the Python threaded version is not much slower than the Rust sequential version, which means compared to an execution on a single CPU core the speed has doubled. +## Sharing Python objects between Rust threads + +In the example above we made a Python interface to a low-level rust function, +and then leveraged the python `threading` module to run the low-level function +in parallel. It is also possible to spawn threads in Rust that acquire the GIL +and operate on Python objects. However, care must be taken to avoid writing code +that deadlocks with the GIL in these cases. + +* Note: This example is meant to illustrate how to drop and re-acquire the GIL + to avoid creating deadlocks. Unless the spawned threads subsequently + release the GIL or you are using the free-threaded build of CPython, you + will not see any speedups due to multi-threaded parallelism using `rayon` + to parallelize code that acquires and holds the GIL for the entire + execution of the spawned thread. + +In the example below, we share a `Vec` of User ID objects defined using the +`pyclass` macro and spawn threads to process the collection of data into a `Vec` +of booleans based on a predicate using a rayon parallel iterator: + +```rust,no_run +use pyo3::prelude::*; + +// These traits let us use int_par_iter and map +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + +#[pyclass] +struct UserID { + id: i64, +} + +let allowed_ids: Vec = Python::with_gil(|outer_py| { + let instances: Vec> = (0..10).map(|x| Py::new(outer_py, UserID { id: x }).unwrap()).collect(); + outer_py.allow_threads(|| { + instances.par_iter().map(|instance| { + Python::with_gil(|inner_py| { + instance.borrow(inner_py).id > 5 + }) + }).collect() + }) +}); +assert!(allowed_ids.into_iter().filter(|b| *b).count() == 4); +``` + +It's important to note that there is an `outer_py` GIL lifetime token as well as +an `inner_py` token. Sharing GIL lifetime tokens between threads is not allowed +and threads must individually acquire the GIL to access data wrapped by a python +object. + +It's also important to see that this example uses [`Python::allow_threads`] to +wrap the code that spawns OS threads via `rayon`. If this example didn't use +`allow_threads`, a rayon worker thread would block on acquiring the GIL while a +thread that owns the GIL spins forever waiting for the result of the rayon +thread. Calling `allow_threads` allows the GIL to be released in the thread +collecting the results from the worker threads. You should always call +`allow_threads` in situations that spawn worker threads, but especially so in +cases where worker threads need to acquire the GIL, to prevent deadlocks. + [`Python::allow_threads`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.allow_threads diff --git a/guide/src/performance.md b/guide/src/performance.md index c47a91deee5..5a57585c4a0 100644 --- a/guide/src/performance.md +++ b/guide/src/performance.md @@ -96,3 +96,60 @@ impl PartialEq for FooBound<'_> { } } ``` + +## Calling Python callables (`__call__`) +CPython support multiple calling protocols: [`tp_call`] and [`vectorcall`]. [`vectorcall`] is a more efficient protocol unlocking faster calls. +PyO3 will try to dispatch Python `call`s using the [`vectorcall`] calling convention to archive maximum performance if possible and falling back to [`tp_call`] otherwise. +This is implemented using the (internal) `PyCallArgs` trait. It defines how Rust types can be used as Python `call` arguments. This trait is currently implemented for +- Rust tuples, where each member implements `IntoPyObject`, +- `Bound<'_, PyTuple>` +- `Py` +Rust tuples may make use of [`vectorcall`] where as `Bound<'_, PyTuple>` and `Py` can only use [`tp_call`]. For maximum performance prefer using Rust tuples as arguments. + + +[`tp_call`]: https://docs.python.org/3/c-api/call.html#the-tp-call-protocol +[`vectorcall`]: https://docs.python.org/3/c-api/call.html#the-vectorcall-protocol + +## Disable the global reference pool + +PyO3 uses global mutable state to keep track of deferred reference count updates implied by `impl Drop for Py` being called without the GIL being held. The necessary synchronization to obtain and apply these reference count updates when PyO3-based code next acquires the GIL is somewhat expensive and can become a significant part of the cost of crossing the Python-Rust boundary. + +This functionality can be avoided by setting the `pyo3_disable_reference_pool` conditional compilation flag. This removes the global reference pool and the associated costs completely. However, it does _not_ remove the `Drop` implementation for `Py` which is necessary to interoperate with existing Rust code written without PyO3-based code in mind. To stay compatible with the wider Rust ecosystem in these cases, we keep the implementation but abort when `Drop` is called without the GIL being held. If `pyo3_leak_on_drop_without_reference_pool` is additionally enabled, objects dropped without the GIL being held will be leaked instead which is always sound but might have determinal effects like resource exhaustion in the long term. + +This limitation is important to keep in mind when this setting is used, especially when embedding Python code into a Rust application as it is quite easy to accidentally drop a `Py` (or types containing it like `PyErr`, `PyBackedStr` or `PyBackedBytes`) returned from `Python::with_gil` without making sure to re-acquire the GIL beforehand. For example, the following code + +```rust,ignore +# use pyo3::prelude::*; +# use pyo3::types::PyList; +let numbers: Py = Python::with_gil(|py| PyList::empty(py).unbind()); + +Python::with_gil(|py| { + numbers.bind(py).append(23).unwrap(); +}); + +Python::with_gil(|py| { + numbers.bind(py).append(42).unwrap(); +}); +``` + +will abort if the list not explicitly disposed via + +```rust +# use pyo3::prelude::*; +# use pyo3::types::PyList; +let numbers: Py = Python::with_gil(|py| PyList::empty(py).unbind()); + +Python::with_gil(|py| { + numbers.bind(py).append(23).unwrap(); +}); + +Python::with_gil(|py| { + numbers.bind(py).append(42).unwrap(); +}); + +Python::with_gil(move |py| { + drop(numbers); +}); +``` + +[conditional-compilation]: https://doc.rust-lang.org/reference/conditional-compilation.html diff --git a/guide/src/python-from-rust.md b/guide/src/python-from-rust.md index ee618f3fa47..ebb7fa1f4da 100644 --- a/guide/src/python-from-rust.md +++ b/guide/src/python-from-rust.md @@ -44,3 +44,7 @@ Because of the lack of exclusive `&mut` references, PyO3's APIs for Python objec [smart-pointers]: https://doc.rust-lang.org/book/ch15-00-smart-pointers.html [obtaining-py]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#obtaining-a-python-token [`pyo3::sync`]: {{#PYO3_DOCS_URL}}/pyo3/sync/index.html +[eval]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval +[import]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.import_bound +[clone_ref]: {{#PYO3_DOCS_URL}}/pyo3/prelude/struct.Py.html#method.clone_ref +[Bound]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html diff --git a/guide/src/python-from-rust/calling-existing-code.md b/guide/src/python-from-rust/calling-existing-code.md index 53051d4ce51..4eba33f11c3 100644 --- a/guide/src/python-from-rust/calling-existing-code.md +++ b/guide/src/python-from-rust/calling-existing-code.md @@ -4,8 +4,7 @@ If you already have some existing Python code that you need to execute from Rust ## Want to access Python APIs? Then use `PyModule::import`. -[`Pymodule::import`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.import) can -be used to get handle to a Python module from Rust. You can use this to import and use any Python +[`PyModule::import`] can be used to get handle to a Python module from Rust. You can use this to import and use any Python module available in your environment. ```rust @@ -13,7 +12,7 @@ use pyo3::prelude::*; fn main() -> PyResult<()> { Python::with_gil(|py| { - let builtins = PyModule::import_bound(py, "builtins")?; + let builtins = PyModule::import(py, "builtins")?; let total: i32 = builtins .getattr("sum")? .call1((vec![1, 2, 3],))? @@ -24,19 +23,22 @@ fn main() -> PyResult<()> { } ``` +[`PyModule::import`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.import + ## Want to run just an expression? Then use `eval`. [`Python::eval`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval) is -a method to execute a [Python expression](https://docs.python.org/3.7/reference/expressions.html) +a method to execute a [Python expression](https://docs.python.org/3/reference/expressions.html) and return the evaluated value as a `Bound<'py, PyAny>` object. ```rust use pyo3::prelude::*; +use pyo3::ffi::c_str; # fn main() -> Result<(), ()> { Python::with_gil(|py| { let result = py - .eval_bound("[i * 10 for i in range(5)]", None, None) + .eval(c_str!("[i * 10 for i in range(5)]"), None, None) .map_err(|e| { e.print_and_set_sys_last_vars(py); })?; @@ -50,7 +52,7 @@ Python::with_gil(|py| { ## Want to run statements? Then use `run`. [`Python::run`] is a method to execute one or more -[Python statements](https://docs.python.org/3.7/reference/simple_stmts.html). +[Python statements](https://docs.python.org/3/reference/simple_stmts.html). This method returns nothing (like any Python statement), but you can get access to manipulated objects via the `locals` dict. @@ -58,6 +60,8 @@ You can also use the [`py_run!`] macro, which is a shorthand for [`Python::run`] Since [`py_run!`] panics on exceptions, we recommend you use this macro only for quickly testing your Python extensions. +[`Python::run`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.run + ```rust use pyo3::prelude::*; use pyo3::py_run; @@ -106,27 +110,28 @@ to this function! ```rust use pyo3::{prelude::*, types::IntoPyDict}; +use pyo3_ffi::c_str; # fn main() -> PyResult<()> { Python::with_gil(|py| { - let activators = PyModule::from_code_bound( + let activators = PyModule::from_code( py, - r#" + c_str!(r#" def relu(x): """see https://en.wikipedia.org/wiki/Rectifier_(neural_networks)""" return max(0.0, x) def leaky_relu(x, slope=0.01): return x if x >= 0 else x * slope - "#, - "activators.py", - "activators", + "#), + c_str!("activators.py"), + c_str!("activators"), )?; let relu_result: f64 = activators.getattr("relu")?.call1((-1.0,))?.extract()?; assert_eq!(relu_result, 0.0); - let kwargs = [("slope", 0.2)].into_py_dict_bound(py); + let kwargs = [("slope", 0.2)].into_py_dict(py)?; let lrelu_result: f64 = activators .getattr("leaky_relu")? .call((-1.0,), Some(&kwargs))? @@ -152,6 +157,7 @@ As an example, the below adds the module `foo` to the embedded interpreter: ```rust use pyo3::prelude::*; +use pyo3::ffi::c_str; #[pyfunction] fn add_one(x: i64) -> i64 { @@ -166,7 +172,7 @@ fn foo(foo_module: &Bound<'_, PyModule>) -> PyResult<()> { fn main() -> PyResult<()> { pyo3::append_to_inittab!(foo); - Python::with_gil(|py| Python::run_bound(py, "import foo; foo.add_one(6)", None, None)) + Python::with_gil(|py| Python::run(py, c_str!("import foo; foo.add_one(6)"), None, None)) } ``` @@ -177,6 +183,7 @@ and insert it manually into `sys.modules`: ```rust use pyo3::prelude::*; use pyo3::types::PyDict; +use pyo3::ffi::c_str; #[pyfunction] pub fn add_one(x: i64) -> i64 { @@ -186,18 +193,18 @@ pub fn add_one(x: i64) -> i64 { fn main() -> PyResult<()> { Python::with_gil(|py| { // Create new module - let foo_module = PyModule::new_bound(py, "foo")?; + let foo_module = PyModule::new(py, "foo")?; foo_module.add_function(wrap_pyfunction!(add_one, &foo_module)?)?; // Import and get sys.modules - let sys = PyModule::import_bound(py, "sys")?; + let sys = PyModule::import(py, "sys")?; let py_modules: Bound<'_, PyDict> = sys.getattr("modules")?.downcast_into()?; // Insert foo into sys.modules py_modules.set_item("foo", foo_module)?; // Now we can import + run our python code - Python::run_bound(py, "import foo; foo.add_one(6)", None, None) + Python::run(py, c_str!("import foo; foo.add_one(6)"), None, None) }) } ``` @@ -249,16 +256,17 @@ The example below shows: `src/main.rs`: ```rust,ignore use pyo3::prelude::*; +use pyo3_ffi::c_str; fn main() -> PyResult<()> { - let py_foo = include_str!(concat!( + let py_foo = c_str!(include_str!(concat!( env!("CARGO_MANIFEST_DIR"), "/python_app/utils/foo.py" - )); - let py_app = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/python_app/app.py")); + ))); + let py_app = c_str!(include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/python_app/app.py"))); let from_python = Python::with_gil(|py| -> PyResult> { - PyModule::from_code_bound(py, py_foo, "utils.foo", "utils.foo")?; - let app: Py = PyModule::from_code_bound(py, py_app, "", "")? + PyModule::from_code(py, py_foo, c_str!("utils.foo"), c_str!("utils.foo"))?; + let app: Py = PyModule::from_code(py, py_app, c_str!(""), c_str!(""))? .getattr("run")? .into(); app.call0(py) @@ -283,19 +291,21 @@ that directory is `/usr/share/python_app`). ```rust,no_run use pyo3::prelude::*; use pyo3::types::PyList; +use pyo3_ffi::c_str; use std::fs; use std::path::Path; +use std::ffi::CString; fn main() -> PyResult<()> { let path = Path::new("/usr/share/python_app"); - let py_app = fs::read_to_string(path.join("app.py"))?; + let py_app = CString::new(fs::read_to_string(path.join("app.py"))?)?; let from_python = Python::with_gil(|py| -> PyResult> { let syspath = py - .import_bound("sys")? + .import("sys")? .getattr("path")? .downcast_into::()?; - syspath.insert(0, &path)?; - let app: Py = PyModule::from_code_bound(py, &py_app, "", "")? + syspath.insert(0, path)?; + let app: Py = PyModule::from_code(py, py_app.as_c_str(), c_str!(""), c_str!(""))? .getattr("run")? .into(); app.call0(py) @@ -316,12 +326,13 @@ Use context managers by directly invoking `__enter__` and `__exit__`. ```rust use pyo3::prelude::*; +use pyo3::ffi::c_str; fn main() { Python::with_gil(|py| { - let custom_manager = PyModule::from_code_bound( + let custom_manager = PyModule::from_code( py, - r#" + c_str!(r#" class House(object): def __init__(self, address): self.address = address @@ -333,9 +344,9 @@ class House(object): else: print(f"Thank you for visiting {self.address}, come again soon!") - "#, - "house.py", - "house", + "#), + c_str!("house.py"), + c_str!("house"), ) .unwrap(); @@ -344,7 +355,7 @@ class House(object): house.call_method0("__enter__").unwrap(); - let result = py.eval_bound("undefined_variable + 1", None, None); + let result = py.eval(c_str!("undefined_variable + 1"), None, None); // If the eval threw an exception we'll pass it through to the context manager. // Otherwise, __exit__ is called with empty arguments (Python "None"). @@ -360,9 +371,9 @@ class House(object): .call_method1( "__exit__", ( - e.get_type_bound(py), - e.value_bound(py), - e.traceback_bound(py), + e.get_type(py), + e.value(py), + e.traceback(py), ), ) .unwrap(); @@ -383,7 +394,7 @@ use pyo3::prelude::*; # fn main() -> PyResult<()> { Python::with_gil(|py| -> PyResult<()> { - let signal = py.import_bound("signal")?; + let signal = py.import("signal")?; // Set SIGINT to have the default action signal .getattr("signal")? diff --git a/guide/src/python-from-rust/function-calls.md b/guide/src/python-from-rust/function-calls.md index f97de1f24ce..2e5cf3c589e 100644 --- a/guide/src/python-from-rust/function-calls.md +++ b/guide/src/python-from-rust/function-calls.md @@ -19,6 +19,7 @@ The example below calls a Python function behind a `PyObject` (aka `Py`) ```rust use pyo3::prelude::*; use pyo3::types::PyTuple; +use pyo3_ffi::c_str; fn main() -> PyResult<()> { let arg1 = "arg1"; @@ -26,17 +27,17 @@ fn main() -> PyResult<()> { let arg3 = "arg3"; Python::with_gil(|py| { - let fun: Py = PyModule::from_code_bound( + let fun: Py = PyModule::from_code( py, - "def example(*args, **kwargs): + c_str!("def example(*args, **kwargs): if args != (): print('called with args', args) if kwargs != {}: print('called with kwargs', kwargs) if args == () and kwargs == {}: - print('called with no arguments')", - "", - "", + print('called with no arguments')"), + c_str!(""), + c_str!(""), )? .getattr("example")? .into(); @@ -49,7 +50,7 @@ fn main() -> PyResult<()> { fun.call1(py, args)?; // call object with Python tuple of positional arguments - let args = PyTuple::new_bound(py, &[arg1, arg2, arg3]); + let args = PyTuple::new(py, &[arg1, arg2, arg3])?; fun.call1(py, args)?; Ok(()) }) @@ -64,6 +65,7 @@ For the `call` and `call_method` APIs, `kwargs` are `Option<&Bound<'py, PyDict>> use pyo3::prelude::*; use pyo3::types::IntoPyDict; use std::collections::HashMap; +use pyo3_ffi::c_str; fn main() -> PyResult<()> { let key1 = "key1"; @@ -72,43 +74,35 @@ fn main() -> PyResult<()> { let val2 = 2; Python::with_gil(|py| { - let fun: Py = PyModule::from_code_bound( + let fun: Py = PyModule::from_code( py, - "def example(*args, **kwargs): + c_str!("def example(*args, **kwargs): if args != (): print('called with args', args) if kwargs != {}: print('called with kwargs', kwargs) if args == () and kwargs == {}: - print('called with no arguments')", - "", - "", + print('called with no arguments')"), + c_str!(""), + c_str!(""), )? .getattr("example")? .into(); // call object with PyDict - let kwargs = [(key1, val1)].into_py_dict_bound(py); - fun.call_bound(py, (), Some(&kwargs))?; + let kwargs = [(key1, val1)].into_py_dict(py)?; + fun.call(py, (), Some(&kwargs))?; // pass arguments as Vec let kwargs = vec![(key1, val1), (key2, val2)]; - fun.call_bound(py, (), Some(&kwargs.into_py_dict_bound(py)))?; + fun.call(py, (), Some(&kwargs.into_py_dict(py)?))?; // pass arguments as HashMap let mut kwargs = HashMap::<&str, i32>::new(); kwargs.insert(key1, 1); - fun.call_bound(py, (), Some(&kwargs.into_py_dict_bound(py)))?; + fun.call(py, (), Some(&kwargs.into_py_dict(py)?))?; Ok(()) }) } -``` - -
- -During PyO3's [migration from "GIL Refs" to the `Bound` smart pointer](../migration.md#migrating-from-the-gil-refs-api-to-boundt), [`Py::call`]({{#PYO3_DOCS_URL}}/pyo3/struct.Py.html#method.call) is temporarily named `call_bound` (and `call_method` is temporarily `call_method_bound`). - -(This temporary naming is only the case for the `Py` smart pointer. The methods on the `&PyAny` GIL Ref such as `call` have not been given replacements, and the methods on the `Bound` smart pointer such as [`Bound::call`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call) already use follow the newest API conventions.) - -
+``` \ No newline at end of file diff --git a/guide/src/rust-from-python.md b/guide/src/rust-from-python.md index 470d5719098..3b525d399df 100644 --- a/guide/src/rust-from-python.md +++ b/guide/src/rust-from-python.md @@ -4,10 +4,10 @@ This chapter of the guide is dedicated to explaining how to wrap Rust code into PyO3 uses Rust's "procedural macros" to provide a powerful yet simple API to denote what Rust code should map into Python objects. -The three types of Python objects which PyO3 can produce are: +PyO3 can create three types of Python objects: - Python modules, via the `#[pymodule]` macro - Python functions, via the `#[pyfunction]` macro -- Python classes, via the `#[pyclass]` macro (plus `#[pymethods]` to define methods for those clases) +- Python classes, via the `#[pyclass]` macro (plus `#[pymethods]` to define methods for those classes) The following subchapters go through each of these in turn. diff --git a/guide/src/trait-bounds.md b/guide/src/trait-bounds.md index 0644e679190..e1b8e82f1db 100644 --- a/guide/src/trait-bounds.md +++ b/guide/src/trait-bounds.md @@ -1,6 +1,6 @@ # Using in Python a Rust function with trait bounds -PyO3 allows for easy conversion from Rust to Python for certain functions and classes (see the [conversion table](conversions/tables.md). +PyO3 allows for easy conversion from Rust to Python for certain functions and classes (see the [conversion table](conversions/tables.md)). However, it is not always straightforward to convert Rust code that requires a given trait implementation as an argument. This tutorial explains how to convert a Rust function that takes a trait as argument for use in Python with classes implementing the same methods as the trait. @@ -84,7 +84,7 @@ impl Model for UserModel { Python::with_gil(|py| { self.model .bind(py) - .call_method("set_variables", (PyList::new_bound(py, var),), None) + .call_method("set_variables", (PyList::new(py, var).unwrap(),), None) .unwrap(); }) } @@ -182,7 +182,7 @@ This wrapper will also perform the type conversions between Python and Rust. # println!("Rust calling Python to set the variables"); # Python::with_gil(|py| { # self.model.bind(py) -# .call_method("set_variables", (PyList::new_bound(py, var),), None) +# .call_method("set_variables", (PyList::new(py, var).unwrap(),), None) # .unwrap(); # }) # } @@ -360,7 +360,7 @@ impl Model for UserModel { # println!("Rust calling Python to set the variables"); # Python::with_gil(|py| { # self.model.bind(py) -# .call_method("set_variables", (PyList::new_bound(py, var),), None) +# .call_method("set_variables", (PyList::new(py, var).unwrap(),), None) # .unwrap(); # }) # } @@ -419,7 +419,7 @@ impl Model for UserModel { # println!("Rust calling Python to set the variables"); # Python::with_gil(|py| { # let py_model = self.model.bind(py) -# .call_method("set_variables", (PyList::new_bound(py, var),), None) +# .call_method("set_variables", (PyList::new(py, var).unwrap(),), None) # .unwrap(); # }) # } @@ -517,7 +517,7 @@ impl Model for UserModel { Python::with_gil(|py| { self.model .bind(py) - .call_method("set_variables", (PyList::new_bound(py, var),), None) + .call_method("set_variables", (PyList::new(py, var).unwrap(),), None) .unwrap(); }) } diff --git a/guide/src/types.md b/guide/src/types.md index 4c63d175991..06559d49d13 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -2,12 +2,10 @@ PyO3 offers two main sets of types to interact with Python objects. This section of the guide expands into detail about these types and how to choose which to use. -The first set of types is are the [smart pointers][smart-pointers] which all Python objects are wrapped in. These are `Py`, `Bound<'py, T>`, and `Borrowed<'a, 'py, T>`. The [first section below](#pyo3s-smart-pointers) expands on each of these in detail and why there are three of them. +The first set of types are the [smart pointers][smart-pointers] which all Python objects are wrapped in. These are `Py`, `Bound<'py, T>`, and `Borrowed<'a, 'py, T>`. The [first section below](#pyo3s-smart-pointers) expands on each of these in detail and why there are three of them. The second set of types are types which fill in the generic parameter `T` of the smart pointers. The most common is `PyAny`, which represents any Python object (similar to Python's `typing.Any`). There are also concrete types for many Python built-in types, such as `PyList`, `PyDict`, and `PyTuple`. User defined `#[pyclass]` types also fit this category. The [second section below](#concrete-python-types) expands on how to use these types. -Before PyO3 0.21, PyO3's main API to interact with Python objects was a deprecated API known as the "GIL Refs" API, containing reference types such as `&PyAny`, `&PyList`, and `&PyCell` for user-defined `#[pyclass]` types. The [third section below](#the-gil-refs-api) details this deprecated API. - ## PyO3's smart pointers PyO3's API offers three generic smart pointers: `Py`, `Bound<'py, T>` and `Borrowed<'a, 'py, T>`. For each of these the type parameter `T` will be filled by a [concrete Python type](#concrete-python-types). For example, a Python list object can be represented by `Py`, `Bound<'py, PyList>`, and `Borrowed<'a, 'py, PyList>`. @@ -34,7 +32,7 @@ The lack of binding to the `'py` lifetime also carries drawbacks: - Almost all methods on `Py` require a `Python<'py>` token as the first argument - Other functionality, such as [`Drop`][Drop], needs to check at runtime for attachment to the Python GIL, at a small performance cost -Because of the drawbacks `Bound<'py, T>` is preferred for many of PyO3's APIs. In particular, `Bound<'py, T>` is the better for function arguments. +Because of the drawbacks `Bound<'py, T>` is preferred for many of PyO3's APIs. In particular, `Bound<'py, T>` is better for function arguments. To convert a `Py` into a `Bound<'py, T>`, the `Py::bind` and `Py::into_bound` methods are available. `Bound<'py, T>` can be converted back into `Py` using [`Bound::unbind`]. @@ -42,7 +40,7 @@ To convert a `Py` into a `Bound<'py, T>`, the `Py::bind` and `Py::into_bound` [`Bound<'py, T>`][Bound] is the counterpart to `Py` which is also bound to the `'py` lifetime. It can be thought of as equivalent to the Rust tuple `(Python<'py>, Py)`. -By having the binding to the `'py` lifetime, `Bound<'py, T>` can offer the complete PyO3 API at maximum efficiency. This means that in almost all cases where `Py` is not necessary for lifetime reasons, `Bound<'py, T>` should be used. +By having the binding to the `'py` lifetime, `Bound<'py, T>` can offer the complete PyO3 API at maximum efficiency. This means that `Bound<'py, T>` should usually be used whenever carrying this lifetime is acceptable, and `Py` otherwise. `Bound<'py, T>` engages in Python reference counting. This means that `Bound<'py, T>` owns a Python object. Rust code which just wants to borrow a Python object should use a shared reference `&Bound<'py, T>`. Just like `std::sync::Arc`, using `.clone()` and `drop()` will cheaply increment and decrement the reference count of the object (just in this case, the reference counting is implemented by the Python interpreter itself). @@ -63,7 +61,7 @@ use pyo3::prelude::*; use pyo3::types::PyList; fn example<'py>(py: Python<'py>) -> PyResult<()> { - let x: Bound<'py, PyList> = PyList::empty_bound(py); + let x: Bound<'py, PyList> = PyList::empty(py); x.append(1)?; let y: Bound<'py, PyList> = x.clone(); // y is a new reference to the same list drop(x); // release the original reference x @@ -79,7 +77,7 @@ use pyo3::prelude::*; use pyo3::types::PyList; fn example(py: Python<'_>) -> PyResult<()> { - let x = PyList::empty_bound(py); + let x = PyList::empty(py); x.append(1)?; let y = x.clone(); drop(x); @@ -114,7 +112,7 @@ fn add<'py>( left.add(right) } # Python::with_gil(|py| { -# let s = pyo3::types::PyString::new_bound(py, "s"); +# let s = pyo3::types::PyString::new(py, "s"); # assert!(add(&s, &s).unwrap().eq("ss").unwrap()); # }) ``` @@ -128,7 +126,7 @@ fn add(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult Ok(output.unbind()) } # Python::with_gil(|py| { -# let s = pyo3::types::PyString::new_bound(py, "s"); +# let s = pyo3::types::PyString::new(py, "s"); # assert!(add(&s, &s).unwrap().bind(py).eq("ss").unwrap()); # }) ``` @@ -147,7 +145,7 @@ use pyo3::types::PyTuple; # fn example<'py>(py: Python<'py>) -> PyResult<()> { // Create a new tuple with the elements (0, 1, 2) -let t = PyTuple::new_bound(py, [0, 1, 2]); +let t = PyTuple::new(py, [0, 1, 2])?; for i in 0..=2 { let entry: Borrowed<'_, 'py, PyAny> = t.get_borrowed_item(i)?; // `PyAnyMethods::extract` is available on `Borrowed` @@ -234,7 +232,7 @@ fn get_first_item<'py>(list: &Bound<'py, PyList>) -> PyResult> list.get_item(0) } # Python::with_gil(|py| { -# let l = PyList::new_bound(py, ["hello world"]); +# let l = PyList::new(py, ["hello world"]).unwrap(); # assert!(get_first_item(&l).unwrap().eq("hello world").unwrap()); # }) ``` @@ -252,7 +250,7 @@ For example, the following snippet shows how to cast `Bound<'py, PyAny>` to `Bou # use pyo3::types::PyTuple; # fn example<'py>(py: Python<'py>) -> PyResult<()> { // create a new Python `tuple`, and use `.into_any()` to erase the type -let obj: Bound<'py, PyAny> = PyTuple::empty_bound(py).into_any(); +let obj: Bound<'py, PyAny> = PyTuple::empty(py).into_any(); // use `.downcast()` to cast to `PyTuple` without transferring ownership let _: &Bound<'py, PyTuple> = obj.downcast()?; @@ -297,7 +295,7 @@ For example, the following snippet extracts a Rust tuple of integers from a Pyth # use pyo3::types::PyTuple; # fn example<'py>(py: Python<'py>) -> PyResult<()> { // create a new Python `tuple`, and use `.into_any()` to erase the type -let obj: Bound<'py, PyAny> = PyTuple::new_bound(py, [1, 2, 3]).into_any(); +let obj: Bound<'py, PyAny> = PyTuple::new(py, [1, 2, 3])?.into_any(); // extracting the Python `tuple` to a rust `(i32, i32, i32)` tuple let (x, y, z) = obj.extract::<(i32, i32, i32)>()?; @@ -310,176 +308,6 @@ assert_eq!((x, y, z), (1, 2, 3)); To avoid copying data, [`#[pyclass]`][pyclass] types can directly reference Rust data stored within the Python objects without needing to `.extract()`. See the [corresponding documentation in the class section of the guide](./class.md#bound-and-interior-mutability) for more detail. -## The GIL Refs API - -The GIL Refs API was PyO3's primary API prior to PyO3 0.21. The main difference was that instead of the `Bound<'py, PyAny>` smart pointer, the "GIL Reference" `&'py PyAny` was used. (This was similar for other Python types.) - -As of PyO3 0.21, the GIL Refs API is deprecated. See the [migration guide](./migration.md#from-020-to-021) for details on how to upgrade. - -The following sections note some historical detail about the GIL Refs API. - -### [`PyAny`][PyAny] - -**Represented:** a Python object of unspecified type. In the GIL Refs API, this was only accessed as the GIL Ref `&'py PyAny`. - -**Used:** `&'py PyAny` was used to refer to some Python object when the GIL lifetime was available for the whole duration access was needed. For example, intermediate values and arguments to `pyfunction`s or `pymethod`s implemented in Rust where any type is allowed. - -**Conversions:** - -For a `&PyAny` object reference `any` where the underlying object is a Python-native type such as -a list: - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyList; -# Python::with_gil(|py| -> PyResult<()> { -#[allow(deprecated)] // PyList::empty is part of the deprecated "GIL Refs" API. -let obj: &PyAny = PyList::empty(py); - -// To &PyList with PyAny::downcast -let _: &PyList = obj.downcast()?; - -// To Py (aka PyObject) with .into() -let _: Py = obj.into(); - -// To Py with PyAny::extract -let _: Py = obj.extract()?; -# Ok(()) -# }).unwrap(); -``` - -For a `&PyAny` object reference `any` where the underlying object is a `#[pyclass]`: - -```rust -# use pyo3::prelude::*; -# #[pyclass] #[derive(Clone)] struct MyClass { } -# Python::with_gil(|py| -> PyResult<()> { -#[allow(deprecated)] // into_ref is part of the deprecated GIL Refs API -let obj: &PyAny = Py::new(py, MyClass {})?.into_ref(py); - -// To &PyCell with PyAny::downcast -#[allow(deprecated)] // &PyCell is part of the deprecated GIL Refs API -let _: &PyCell = obj.downcast()?; - -// To Py (aka PyObject) with .into() -let _: Py = obj.into(); - -// To Py with PyAny::extract -let _: Py = obj.extract()?; - -// To MyClass with PyAny::extract, if MyClass: Clone -let _: MyClass = obj.extract()?; - -// To PyRef<'_, MyClass> or PyRefMut<'_, MyClass> with PyAny::extract -let _: PyRef<'_, MyClass> = obj.extract()?; -let _: PyRefMut<'_, MyClass> = obj.extract()?; -# Ok(()) -# }).unwrap(); -``` - -### `PyTuple`, `PyDict`, and many more - -**Represented:** a native Python object of known type. In the GIL Refs API, they were only accessed as the GIL Refs `&'py PyTuple`, `&'py PyDict`. - -**Used:** `&'py PyTuple` and similar were used to operate with native Python types while holding the GIL. Like `PyAny`, this is the most convenient form to use for function arguments and intermediate values. - -These GIL Refs implement `Deref`, so they all expose the same methods which can be found on `PyAny`. - -To see all Python types exposed by `PyO3` consult the [`pyo3::types`][pyo3::types] module. - -**Conversions:** - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyList; -# Python::with_gil(|py| -> PyResult<()> { -#[allow(deprecated)] // PyList::empty is part of the deprecated "GIL Refs" API. -let list = PyList::empty(py); - -// Use methods from PyAny on all Python types with Deref implementation -let _ = list.repr()?; - -// To &PyAny automatically with Deref implementation -let _: &PyAny = list; - -// To &PyAny explicitly with .as_ref() -#[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. -let _: &PyAny = list.as_ref(); - -// To Py with .into() or Py::from() -let _: Py = list.into(); - -// To PyObject with .into() or .to_object(py) -let _: PyObject = list.into(); -# Ok(()) -# }).unwrap(); -``` - -### `Py` and `PyObject` - -**Represented:** a GIL-independent reference to a Python object. This can be a Python native type -(like `PyTuple`), or a `pyclass` type implemented in Rust. The most commonly-used variant, -`Py`, is also known as `PyObject`. - -**Used:** Whenever you want to carry around references to a Python object without caring about a -GIL lifetime. For example, storing Python object references in a Rust struct that outlives the -Python-Rust FFI boundary, or returning objects from functions implemented in Rust back to Python. - -Can be cloned using Python reference counts with `.clone()`. - -### `PyCell` - -**Represented:** a reference to a Rust object (instance of `PyClass`) wrapped in a Python object. The cell part is an analog to stdlib's [`RefCell`][RefCell] to allow access to `&mut` references. - -**Used:** for accessing pure-Rust API of the instance (members and functions taking `&SomeType` or `&mut SomeType`) while maintaining the aliasing rules of Rust references. - -Like PyO3's Python native types, the GIL Ref `&PyCell` implements `Deref`, so it also exposed all of the methods on `PyAny`. - -**Conversions:** - -`PyCell` was used to access `&T` and `&mut T` via `PyRef` and `PyRefMut` respectively. - -```rust -# use pyo3::prelude::*; -# #[pyclass] struct MyClass { } -# Python::with_gil(|py| -> PyResult<()> { -#[allow(deprecated)] // &PyCell is part of the deprecated GIL Refs API -let cell: &PyCell = PyCell::new(py, MyClass {})?; - -// To PyRef with .borrow() or .try_borrow() -let py_ref: PyRef<'_, MyClass> = cell.try_borrow()?; -let _: &MyClass = &*py_ref; -# drop(py_ref); - -// To PyRefMut with .borrow_mut() or .try_borrow_mut() -let mut py_ref_mut: PyRefMut<'_, MyClass> = cell.try_borrow_mut()?; -let _: &mut MyClass = &mut *py_ref_mut; -# Ok(()) -# }).unwrap(); -``` - -`PyCell` was also accessed like a Python-native type. - -```rust -# use pyo3::prelude::*; -# #[pyclass] struct MyClass { } -# Python::with_gil(|py| -> PyResult<()> { -#[allow(deprecated)] // &PyCell is part of the deprecate GIL Refs API -let cell: &PyCell = PyCell::new(py, MyClass {})?; - -// Use methods from PyAny on PyCell with Deref implementation -let _ = cell.repr()?; - -// To &PyAny automatically with Deref implementation -let _: &PyAny = cell; - -// To &PyAny explicitly with .as_ref() -#[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. -let _: &PyAny = cell.as_ref(); -# Ok(()) -# }).unwrap(); -``` - [Bound]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html [`Bound::unbind`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html#method.unbind [Py]: {{#PYO3_DOCS_URL}}/pyo3/struct.Py.html diff --git a/guide/theme/tabs.css b/guide/theme/tabs.css new file mode 100644 index 00000000000..8712b859c0b --- /dev/null +++ b/guide/theme/tabs.css @@ -0,0 +1,25 @@ +.mdbook-tabs { + display: flex; +} + +.mdbook-tab { + background-color: var(--table-alternate-bg); + padding: 0.5rem 1rem; + cursor: pointer; + border: none; + font-size: 1.6rem; + line-height: 1.45em; +} + +.mdbook-tab.active { + background-color: var(--table-header-bg); + font-weight: bold; +} + +.mdbook-tab-content { + padding: 1rem 0rem; +} + +.mdbook-tab-content table { + margin: unset; +} diff --git a/guide/theme/tabs.js b/guide/theme/tabs.js new file mode 100644 index 00000000000..8ba5e878c39 --- /dev/null +++ b/guide/theme/tabs.js @@ -0,0 +1,75 @@ +/** + * Change active tab of tabs. + * + * @param {Element} container + * @param {string} name + */ +const changeTab = (container, name) => { + for (const child of container.children) { + if (!(child instanceof HTMLElement)) { + continue; + } + + if (child.classList.contains('mdbook-tabs')) { + for (const tab of child.children) { + if (!(tab instanceof HTMLElement)) { + continue; + } + + if (tab.dataset.tabname === name) { + tab.classList.add('active'); + } else { + tab.classList.remove('active'); + } + } + } else if (child.classList.contains('mdbook-tab-content')) { + if (child.dataset.tabname === name) { + child.classList.remove('hidden'); + } else { + child.classList.add('hidden'); + } + } + } +}; + +document.addEventListener('DOMContentLoaded', () => { + const tabs = document.querySelectorAll('.mdbook-tab'); + for (const tab of tabs) { + tab.addEventListener('click', () => { + if (!(tab instanceof HTMLElement)) { + return; + } + + if (!tab.parentElement || !tab.parentElement.parentElement) { + return; + } + + const container = tab.parentElement.parentElement; + const name = tab.dataset.tabname; + const global = container.dataset.tabglobal; + + changeTab(container, name); + + if (global) { + localStorage.setItem(`mdbook-tabs-${global}`, name); + + const globalContainers = document.querySelectorAll( + `.mdbook-tabs-container[data-tabglobal="${global}"]` + ); + for (const globalContainer of globalContainers) { + changeTab(globalContainer, name); + } + } + }); + } + + const containers = document.querySelectorAll('.mdbook-tabs-container[data-tabglobal]'); + for (const container of containers) { + const global = container.dataset.tabglobal; + + const name = localStorage.getItem(`mdbook-tabs-${global}`); + if (name && document.querySelector(`.mdbook-tab[data-tabname=${name}]`)) { + changeTab(container, name); + } + } +}); diff --git a/noxfile.py b/noxfile.py index 3d82c8d3746..61e8dee71fd 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,11 +5,23 @@ import shutil import subprocess import sys +import sysconfig import tempfile from functools import lru_cache from glob import glob from pathlib import Path -from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Tuple, + Generator, +) + import nox import nox.command @@ -30,8 +42,41 @@ PYO3_GUIDE_SRC = PYO3_DIR / "guide" / "src" PYO3_GUIDE_TARGET = PYO3_TARGET / "guide" PYO3_DOCS_TARGET = PYO3_TARGET / "doc" -PY_VERSIONS = ("3.7", "3.8", "3.9", "3.10", "3.11", "3.12") -PYPY_VERSIONS = ("3.7", "3.8", "3.9", "3.10") +FREE_THREADED_BUILD = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + + +def _get_output(*args: str) -> str: + return subprocess.run(args, capture_output=True, text=True, check=True).stdout + + +def _parse_supported_interpreter_version( + python_impl: str, # Literal["cpython", "pypy"], TODO update after 3.7 dropped +) -> Tuple[str, str]: + output = _get_output("cargo", "metadata", "--format-version=1", "--no-deps") + cargo_packages = json.loads(output)["packages"] + # Check Python interpreter version support in package metadata + package = "pyo3-ffi" + metadata = next(pkg["metadata"] for pkg in cargo_packages if pkg["name"] == package) + version_info = metadata[python_impl] + assert "min-version" in version_info, f"missing min-version for {python_impl}" + assert "max-version" in version_info, f"missing max-version for {python_impl}" + return version_info["min-version"], version_info["max-version"] + + +def _supported_interpreter_versions( + python_impl: str, # Literal["cpython", "pypy"], TODO update after 3.7 dropped +) -> List[str]: + min_version, max_version = _parse_supported_interpreter_version(python_impl) + major = int(min_version.split(".")[0]) + assert major == 3, f"unsupported Python major version {major}" + min_minor = int(min_version.split(".")[1]) + max_minor = int(max_version.split(".")[1]) + versions = [f"{major}.{minor}" for minor in range(min_minor, max_minor + 1)] + return versions + + +PY_VERSIONS = _supported_interpreter_versions("cpython") +PYPY_VERSIONS = _supported_interpreter_versions("pypy") @nox.session(venv_backend="none") @@ -48,11 +93,14 @@ def test_rust(session: nox.Session): _run_cargo_test(session, package="pyo3-ffi") _run_cargo_test(session) - _run_cargo_test(session, features="abi3") + # the free-threaded build ignores abi3, so we skip abi3 + # tests to avoid unnecessarily running the tests twice + if not FREE_THREADED_BUILD: + _run_cargo_test(session, features="abi3") if "skip-full" not in session.posargs: - _run_cargo_test(session, features="full") - _run_cargo_test(session, features="full gil-refs") - _run_cargo_test(session, features="abi3 full") + _run_cargo_test(session, features="full jiff-02") + if not FREE_THREADED_BUILD: + _run_cargo_test(session, features="abi3 full jiff-02") @nox.session(name="test-py", venv_backend="none") @@ -60,6 +108,8 @@ def test_py(session: nox.Session) -> None: _run(session, "nox", "-f", "pytests/noxfile.py", external=True) for example in glob("examples/*/noxfile.py"): _run(session, "nox", "-f", example, external=True) + for example in glob("pyo3-ffi/examples/*/noxfile.py"): + _run(session, "nox", "-f", example, external=True) @nox.session(venv_backend="none") @@ -67,7 +117,19 @@ def coverage(session: nox.Session) -> None: session.env.update(_get_coverage_env()) _run_cargo(session, "llvm-cov", "clean", "--workspace") test(session) + generate_coverage_report(session) + + +@nox.session(name="set-coverage-env", venv_backend="none") +def set_coverage_env(session: nox.Session) -> None: + """For use in GitHub Actions to set coverage environment variables.""" + with open(os.environ["GITHUB_ENV"], "a") as env_file: + for k, v in _get_coverage_env().items(): + print(f"{k}={v}", file=env_file) + +@nox.session(name="generate-coverage-report", venv_backend="none") +def generate_coverage_report(session: nox.Session) -> None: cov_format = "codecov" output_file = "coverage.json" @@ -320,6 +382,7 @@ def test_emscripten(session: nox.Session): f"-C link-arg=-lpython{info.pymajorminor}", "-C link-arg=-lexpat", "-C link-arg=-lmpdec", + "-C link-arg=-lsqlite3", "-C link-arg=-lz", "-C link-arg=-lbz2", "-C link-arg=-sALLOW_MEMORY_GROWTH=1", @@ -332,7 +395,7 @@ def test_emscripten(session: nox.Session): session, "bash", "-c", - f"source {info.builddir/'emsdk/emsdk_env.sh'} && cargo test", + f"source {info.builddir / 'emsdk/emsdk_env.sh'} && cargo test", ) @@ -361,6 +424,12 @@ def docs(session: nox.Session) -> None: rustdoc_flags.append(session.env.get("RUSTDOCFLAGS", "")) session.env["RUSTDOCFLAGS"] = " ".join(rustdoc_flags) + features = "full" + + if get_rust_version()[:2] >= (1, 70): + # jiff needs MSRC 1.70+ + features += ",jiff-02" + shutil.rmtree(PYO3_DOCS_TARGET, ignore_errors=True) _run_cargo( session, @@ -368,7 +437,7 @@ def docs(session: nox.Session) -> None: "doc", "--lib", "--no-default-features", - "--features=full", + f"--features={features}", "--no-deps", "--workspace", *cargo_flags, @@ -394,8 +463,17 @@ def check_guide(session: nox.Session): docs(session) session.posargs.extend(posargs) + if toml is None: + session.error("requires Python 3.11 or `toml` to be installed") + pyo3_version = toml.loads((PYO3_DIR / "Cargo.toml").read_text())["package"][ + "version" + ] + remaps = { f"file://{PYO3_GUIDE_SRC}/([^/]*/)*?%7B%7B#PYO3_DOCS_URL}}}}": f"file://{PYO3_DOCS_TARGET}", + f"https://pyo3.rs/v{pyo3_version}": f"file://{PYO3_GUIDE_TARGET}", + "https://pyo3.rs/main/": f"file://{PYO3_GUIDE_TARGET}/", + "https://pyo3.rs/latest/": f"file://{PYO3_GUIDE_TARGET}/", "%7B%7B#PYO3_DOCS_VERSION}}": "latest", } remap_args = [] @@ -415,10 +493,10 @@ def check_guide(session: nox.Session): _run( session, "lychee", - PYO3_DOCS_TARGET, - f"--remap=https://pyo3.rs/main/ file://{PYO3_GUIDE_TARGET}/", - f"--remap=https://pyo3.rs/latest/ file://{PYO3_GUIDE_TARGET}/", + str(PYO3_DOCS_TARGET), + *remap_args, f"--exclude=file://{PYO3_DOCS_TARGET}", + "--exclude=http://www.adobe.com/", *session.posargs, ) @@ -542,49 +620,34 @@ def check_changelog(session: nox.Session): print(fragment.name) -@nox.session(name="set-minimal-package-versions", venv_backend="none") -def set_minimal_package_versions(session: nox.Session): +@nox.session(name="set-msrv-package-versions", venv_backend="none") +def set_msrv_package_versions(session: nox.Session): from collections import defaultdict - if toml is None: - session.error("requires Python 3.11 or `toml` to be installed") - projects = ( - None, - "examples/decorator", - "examples/maturin-starter", - "examples/setuptools-rust-starter", - "examples/word-count", + PYO3_DIR, + *(Path(p).parent for p in glob("examples/*/Cargo.toml")), + *(Path(p).parent for p in glob("pyo3-ffi/examples/*/Cargo.toml")), ) min_pkg_versions = { - "rust_decimal": "1.26.1", - "csv": "1.1.6", - "indexmap": "1.6.2", - "hashbrown": "0.9.1", - "log": "0.4.17", - "once_cell": "1.17.2", - "rayon": "1.6.1", - "rayon-core": "1.10.2", - "regex": "1.7.3", - "proptest": "1.0.0", - "chrono": "0.4.25", - "byteorder": "1.4.3", - "crossbeam-channel": "0.5.8", - "crossbeam-deque": "0.8.3", - "crossbeam-epoch": "0.9.15", - "crossbeam-utils": "0.8.16", + "trybuild": "1.0.89", + "allocator-api2": "0.2.10", + "indexmap": "2.5.0", # to be compatible with hashbrown 0.14 + "hashbrown": "0.14.5", # https://github.com/rust-lang/hashbrown/issues/574 } # run cargo update first to ensure that everything is at highest # possible version, so that this matches what CI will resolve to. for project in projects: - if project is None: - _run_cargo(session, "update") - else: - _run_cargo(session, "update", f"--manifest-path={project}/Cargo.toml") + _run_cargo( + session, + "+stable", + "update", + f"--manifest-path={project}/Cargo.toml", + env=os.environ | {"CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS": "fallback"}, + ) - for project in projects: - lock_file = Path(project or "") / "Cargo.lock" + lock_file = project / "Cargo.lock" def load_pkg_versions(): cargo_lock = toml.loads(lock_file.read_text()) @@ -610,19 +673,15 @@ def load_pkg_versions(): # and re-read `Cargo.lock` pkg_versions = load_pkg_versions() - # As a smoke test, cargo metadata solves all dependencies, so - # will break if any crates rely on cargo features not - # supported on MSRV - for project in projects: - if project is None: - _run_cargo(session, "metadata", silent=True) - else: - _run_cargo( - session, - "metadata", - f"--manifest-path={project}/Cargo.toml", - silent=True, - ) + # As a smoke test, cargo metadata solves all dependencies, so + # will break if any crates rely on cargo features not + # supported on MSRV + _run_cargo( + session, + "metadata", + f"--manifest-path={project}/Cargo.toml", + silent=True, + ) @nox.session(name="ffi-check") @@ -641,20 +700,20 @@ def test_version_limits(session: nox.Session): config_file.set("CPython", "3.6") _run_cargo(session, "check", env=env, expect_error=True) - assert "3.13" not in PY_VERSIONS - config_file.set("CPython", "3.13") + assert "3.14" not in PY_VERSIONS + config_file.set("CPython", "3.14") _run_cargo(session, "check", env=env, expect_error=True) - # 3.13 CPython should build with forward compatibility + # 3.14 CPython should build with forward compatibility env["PYO3_USE_ABI3_FORWARD_COMPATIBILITY"] = "1" _run_cargo(session, "check", env=env) - assert "3.6" not in PYPY_VERSIONS - config_file.set("PyPy", "3.6") + assert "3.8" not in PYPY_VERSIONS + config_file.set("PyPy", "3.8") _run_cargo(session, "check", env=env, expect_error=True) - assert "3.11" not in PYPY_VERSIONS - config_file.set("PyPy", "3.11") + assert "3.12" not in PYPY_VERSIONS + config_file.set("PyPy", "3.12") _run_cargo(session, "check", env=env, expect_error=True) @@ -665,15 +724,16 @@ def check_feature_powerset(session: nox.Session): cargo_toml = toml.loads((PYO3_DIR / "Cargo.toml").read_text()) + EXPECTED_ABI3_FEATURES = {f"abi3-py3{ver.split('.')[1]}" for ver in PY_VERSIONS} + EXCLUDED_FROM_FULL = { "nightly", - "gil-refs", "extension-module", "full", "default", "auto-initialize", "generate-import-lib", - "multiple-pymethods", # TODO add this after MSRV 1.62 + "multiple-pymethods", # Because it's not supported on wasm } features = cargo_toml["features"] @@ -682,6 +742,16 @@ def check_feature_powerset(session: nox.Session): abi3_features = {feature for feature in features if feature.startswith("abi3")} abi3_version_features = abi3_features - {"abi3"} + unexpected_abi3_features = abi3_version_features - EXPECTED_ABI3_FEATURES + if unexpected_abi3_features: + session.error( + f"unexpected `abi3` features found in Cargo.toml: {unexpected_abi3_features}" + ) + + missing_abi3_features = EXPECTED_ABI3_FEATURES - abi3_version_features + if missing_abi3_features: + session.error(f"missing `abi3` features in Cargo.toml: {missing_abi3_features}") + expected_full_feature = features.keys() - EXCLUDED_FROM_FULL - abi3_features uncovered_features = expected_full_feature - full_feature @@ -709,7 +779,7 @@ def check_feature_powerset(session: nox.Session): session.error("no experimental features exist; please simplify the noxfile") features_to_skip = [ - *(EXCLUDED_FROM_FULL - {"gil-refs"}), + *(EXCLUDED_FROM_FULL), *abi3_version_features, ] @@ -718,10 +788,14 @@ def check_feature_powerset(session: nox.Session): rust_flags = env.get("RUSTFLAGS", "") env["RUSTFLAGS"] = f"{rust_flags} -Dwarnings" + subcommand = "hack" + if "minimal-versions" in session.posargs: + subcommand = "minimal-versions" + comma_join = ",".join _run_cargo( session, - "hack", + subcommand, "--feature-powerset", '--optional-deps=""', f'--skip="{comma_join(features_to_skip)}"', @@ -738,13 +812,15 @@ def update_ui_tests(session: nox.Session): env["TRYBUILD"] = "overwrite" command = ["test", "--test", "test_compile_error"] _run_cargo(session, *command, env=env) - _run_cargo(session, *command, "--features=full", env=env) - _run_cargo(session, *command, "--features=abi3,full", env=env) + _run_cargo(session, *command, "--features=full,jiff-02", env=env) + _run_cargo(session, *command, "--features=abi3,full,jiff-02", env=env) def _build_docs_for_ffi_check(session: nox.Session) -> None: # pyo3-ffi-check needs to scrape docs of pyo3-ffi - _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps") + env = os.environ.copy() + env["PYO3_PYTHON"] = sys.executable + _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps", env=env) @lru_cache() @@ -754,7 +830,7 @@ def _get_rust_info() -> Tuple[str, ...]: return tuple(output.splitlines()) -def _get_rust_version() -> Tuple[int, int, int, List[str]]: +def get_rust_version() -> Tuple[int, int, int, List[str]]: for line in _get_rust_info(): if line.startswith(_RELEASE_LINE_START): version = line[len(_RELEASE_LINE_START) :].strip() @@ -770,31 +846,30 @@ def _get_rust_default_target() -> str: @lru_cache() -def _get_feature_sets() -> Tuple[Tuple[str, ...], ...]: +def _get_feature_sets() -> Generator[Tuple[str, ...], None, None]: """Returns feature sets to use for clippy job""" - rust_version = _get_rust_version() cargo_target = os.getenv("CARGO_BUILD_TARGET", "") - if rust_version[:2] >= (1, 62) and "wasm32-wasi" not in cargo_target: - # multiple-pymethods feature not supported before 1.62 or on WASI - return ( - ("--no-default-features",), - ( - "--no-default-features", - "--features=abi3", - ), - ("--features=full gil-refs multiple-pymethods",), - ("--features=abi3 full gil-refs multiple-pymethods",), - ) - else: - return ( - ("--no-default-features",), - ( - "--no-default-features", - "--features=abi3", - ), - ("--features=full gil-refs",), - ("--features=abi3 full gil-refs",), - ) + + yield from ( + ("--no-default-features",), + ( + "--no-default-features", + "--features=abi3", + ), + ) + + features = "full" + + if "wasm32-wasip1" not in cargo_target: + # multiple-pymethods not supported on wasm + features += ",multiple-pymethods" + + if get_rust_version()[:2] >= (1, 70): + # jiff needs MSRC 1.70+ + features += ",jiff-02" + + yield (f"--features={features}",) + yield (f"--features=abi3,{features}",) _RELEASE_LINE_START = "release: " @@ -860,6 +935,8 @@ def _run_cargo_test( ) -> None: command = ["cargo"] if "careful" in session.posargs: + # do explicit setup so failures in setup can be seen + _run_cargo(session, "careful", "setup") command.append("careful") command.extend(("test", "--no-fail-fast")) if "release" in session.posargs: @@ -889,10 +966,6 @@ def _run_cargo_set_package_version( _run(session, *command, external=True) -def _get_output(*args: str) -> str: - return subprocess.run(args, capture_output=True, text=True, check=True).stdout - - def _for_all_version_configs( session: nox.Session, job: Callable[[Dict[str, str]], None] ) -> None: @@ -916,7 +989,9 @@ class _ConfigFile: def __init__(self, config_file) -> None: self._config_file = config_file - def set(self, implementation: str, version: str) -> None: + def set( + self, implementation: str, version: str, build_flags: Iterable[str] = () + ) -> None: """Set the contents of this config file to the given implementation and version.""" self._config_file.seek(0) self._config_file.truncate(0) @@ -924,6 +999,7 @@ def set(self, implementation: str, version: str) -> None: f"""\ implementation={implementation} version={version} +build_flags={",".join(build_flags)} suppress_build_script_link_lines=true """ ) diff --git a/pyo3-benches/Cargo.toml b/pyo3-benches/Cargo.toml index e99ef09e19c..24ec9b5d76e 100644 --- a/pyo3-benches/Cargo.toml +++ b/pyo3-benches/Cargo.toml @@ -9,11 +9,15 @@ publish = false [dependencies] pyo3 = { path = "../", features = ["auto-initialize", "full"] } +[build-dependencies] +pyo3-build-config = { path = "../pyo3-build-config" } + [dev-dependencies] codspeed-criterion-compat = "2.3" criterion = "0.5.1" num-bigint = "0.4.3" rust_decimal = { version = "1.0.0", default-features = false } +hashbrown = "0.15" [[bench]] name = "bench_any" @@ -47,6 +51,10 @@ harness = false name = "bench_gil" harness = false +[[bench]] +name = "bench_intopyobject" +harness = false + [[bench]] name = "bench_list" harness = false diff --git a/pyo3-benches/benches/bench_any.rs b/pyo3-benches/benches/bench_any.rs index b77ab9567a6..4ed14493873 100644 --- a/pyo3-benches/benches/bench_any.rs +++ b/pyo3-benches/benches/bench_any.rs @@ -63,7 +63,7 @@ fn find_object_type(obj: &Bound<'_, PyAny>) -> ObjectType { fn bench_identify_object_type(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let obj = py.eval_bound("object()", None, None).unwrap(); + let obj = py.eval(c"object()", None, None).unwrap(); b.iter(|| find_object_type(&obj)); @@ -73,11 +73,11 @@ fn bench_identify_object_type(b: &mut Bencher<'_>) { fn bench_collect_generic_iterator(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let collection = py.eval_bound("list(range(1 << 20))", None, None).unwrap(); + let collection = py.eval(c"list(range(1 << 20))", None, None).unwrap(); b.iter(|| { collection - .iter() + .try_iter() .unwrap() .collect::>>() .unwrap() diff --git a/pyo3-benches/benches/bench_bigint.rs b/pyo3-benches/benches/bench_bigint.rs index 99635a70279..d2c78f0ad4e 100644 --- a/pyo3-benches/benches/bench_bigint.rs +++ b/pyo3-benches/benches/bench_bigint.rs @@ -8,7 +8,7 @@ use pyo3::types::PyDict; fn extract_bigint_extract_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); + let d = PyDict::new(py).into_any(); bench.iter(|| match black_box(&d).extract::() { Ok(v) => panic!("should err {}", v), @@ -19,7 +19,7 @@ fn extract_bigint_extract_fail(bench: &mut Bencher<'_>) { fn extract_bigint_small(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = py.eval_bound("-42", None, None).unwrap(); + let int = py.eval(c"-42", None, None).unwrap(); bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); @@ -27,7 +27,7 @@ fn extract_bigint_small(bench: &mut Bencher<'_>) { fn extract_bigint_big_negative(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = py.eval_bound("-10**300", None, None).unwrap(); + let int = py.eval(c"-10**300", None, None).unwrap(); bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); @@ -35,7 +35,7 @@ fn extract_bigint_big_negative(bench: &mut Bencher<'_>) { fn extract_bigint_big_positive(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = py.eval_bound("10**300", None, None).unwrap(); + let int = py.eval(c"10**300", None, None).unwrap(); bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); @@ -43,7 +43,7 @@ fn extract_bigint_big_positive(bench: &mut Bencher<'_>) { fn extract_bigint_huge_negative(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = py.eval_bound("-10**3000", None, None).unwrap(); + let int = py.eval(c"-10**3000", None, None).unwrap(); bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); @@ -51,7 +51,7 @@ fn extract_bigint_huge_negative(bench: &mut Bencher<'_>) { fn extract_bigint_huge_positive(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = py.eval_bound("10**3000", None, None).unwrap(); + let int = py.eval(c"10**3000", None, None).unwrap(); bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); diff --git a/pyo3-benches/benches/bench_call.rs b/pyo3-benches/benches/bench_call.rs index 8470c8768d3..b6e090a7dbd 100644 --- a/pyo3-benches/benches/bench_call.rs +++ b/pyo3-benches/benches/bench_call.rs @@ -2,11 +2,13 @@ use std::hint::black_box; use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; +use pyo3::ffi::c_str; use pyo3::prelude::*; +use pyo3::types::IntoPyDict; macro_rules! test_module { ($py:ident, $code:literal) => { - PyModule::from_code_bound($py, $code, file!(), "test_module") + PyModule::from_code($py, c_str!($code), c_str!(file!()), c_str!("test_module")) .expect("module creation failed") }; } @@ -25,6 +27,62 @@ fn bench_call_0(b: &mut Bencher<'_>) { }) } +fn bench_call_1(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let module = test_module!(py, "def foo(a, b, c): pass"); + + let foo_module = &module.getattr("foo").unwrap(); + let args = ( + 1.into_pyobject(py).unwrap(), + "s".into_pyobject(py).unwrap(), + 1.23.into_pyobject(py).unwrap(), + ); + + b.iter(|| { + for _ in 0..1000 { + black_box(foo_module).call1(args.clone()).unwrap(); + } + }); + }) +} + +fn bench_call(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let module = test_module!(py, "def foo(a, b, c, d, e): pass"); + + let foo_module = &module.getattr("foo").unwrap(); + let args = ( + 1.into_pyobject(py).unwrap(), + "s".into_pyobject(py).unwrap(), + 1.23.into_pyobject(py).unwrap(), + ); + let kwargs = [("d", 1), ("e", 42)].into_py_dict(py).unwrap(); + + b.iter(|| { + for _ in 0..1000 { + black_box(foo_module) + .call(args.clone(), Some(&kwargs)) + .unwrap(); + } + }); + }) +} + +fn bench_call_one_arg(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let module = test_module!(py, "def foo(a): pass"); + + let foo_module = &module.getattr("foo").unwrap(); + let arg = 1i32.into_pyobject(py).unwrap(); + + b.iter(|| { + for _ in 0..1000 { + black_box(foo_module).call1((arg.clone(),)).unwrap(); + } + }); + }) +} + fn bench_call_method_0(b: &mut Bencher<'_>) { Python::with_gil(|py| { let module = test_module!( @@ -46,9 +104,96 @@ class Foo: }) } +fn bench_call_method_1(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let module = test_module!( + py, + " +class Foo: + def foo(self, a, b, c): + pass +" + ); + + let foo_module = &module.getattr("Foo").unwrap().call0().unwrap(); + let args = ( + 1.into_pyobject(py).unwrap(), + "s".into_pyobject(py).unwrap(), + 1.23.into_pyobject(py).unwrap(), + ); + + b.iter(|| { + for _ in 0..1000 { + black_box(foo_module) + .call_method1("foo", args.clone()) + .unwrap(); + } + }); + }) +} + +fn bench_call_method(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let module = test_module!( + py, + " +class Foo: + def foo(self, a, b, c, d, e): + pass +" + ); + + let foo_module = &module.getattr("Foo").unwrap().call0().unwrap(); + let args = ( + 1.into_pyobject(py).unwrap(), + "s".into_pyobject(py).unwrap(), + 1.23.into_pyobject(py).unwrap(), + ); + let kwargs = [("d", 1), ("e", 42)].into_py_dict(py).unwrap(); + + b.iter(|| { + for _ in 0..1000 { + black_box(foo_module) + .call_method("foo", args.clone(), Some(&kwargs)) + .unwrap(); + } + }); + }) +} + +fn bench_call_method_one_arg(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let module = test_module!( + py, + " +class Foo: + def foo(self, a): + pass +" + ); + + let foo_module = &module.getattr("Foo").unwrap().call0().unwrap(); + let arg = 1i32.into_pyobject(py).unwrap(); + + b.iter(|| { + for _ in 0..1000 { + black_box(foo_module) + .call_method1("foo", (arg.clone(),)) + .unwrap(); + } + }); + }) +} + fn criterion_benchmark(c: &mut Criterion) { c.bench_function("call_0", bench_call_0); + c.bench_function("call_1", bench_call_1); + c.bench_function("call", bench_call); + c.bench_function("call_one_arg", bench_call_one_arg); c.bench_function("call_method_0", bench_call_method_0); + c.bench_function("call_method_1", bench_call_method_1); + c.bench_function("call_method", bench_call_method); + c.bench_function("call_method_one_arg", bench_call_method_one_arg); } criterion_group!(benches, criterion_benchmark); diff --git a/pyo3-benches/benches/bench_decimal.rs b/pyo3-benches/benches/bench_decimal.rs index 53b79abbd38..65fb47e23f7 100644 --- a/pyo3-benches/benches/bench_decimal.rs +++ b/pyo3-benches/benches/bench_decimal.rs @@ -8,9 +8,9 @@ use pyo3::types::PyDict; fn decimal_via_extract(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let locals = PyDict::new_bound(py); - py.run_bound( - r#" + let locals = PyDict::new(py); + py.run( + cr#" import decimal py_dec = decimal.Decimal("0.0") "#, diff --git a/pyo3-benches/benches/bench_dict.rs b/pyo3-benches/benches/bench_dict.rs index 8c3dfe023c8..6a92cf21c5f 100644 --- a/pyo3-benches/benches/bench_dict.rs +++ b/pyo3-benches/benches/bench_dict.rs @@ -9,7 +9,10 @@ use pyo3::{prelude::*, types::PyMapping}; fn iter_dict(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); + let dict = (0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap(); let mut sum = 0; b.iter(|| { for (k, _v) in &dict { @@ -23,14 +26,22 @@ fn iter_dict(b: &mut Bencher<'_>) { fn dict_new(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - b.iter_with_large_drop(|| (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py)); + b.iter_with_large_drop(|| { + (0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap() + }); }); } fn dict_get_item(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); + let dict = (0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -48,7 +59,10 @@ fn dict_get_item(b: &mut Bencher<'_>) { fn extract_hashmap(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); + let dict = (0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap(); b.iter(|| HashMap::::extract_bound(&dict)); }); } @@ -56,16 +70,21 @@ fn extract_hashmap(b: &mut Bencher<'_>) { fn extract_btreemap(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); + let dict = (0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap(); b.iter(|| BTreeMap::::extract_bound(&dict)); }); } -#[cfg(feature = "hashbrown")] fn extract_hashbrown_map(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); + let dict = (0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap(); b.iter(|| hashbrown::HashMap::::extract_bound(&dict)); }); } @@ -73,7 +92,10 @@ fn extract_hashbrown_map(b: &mut Bencher<'_>) { fn mapping_from_dict(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let dict = &(0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); + let dict = &(0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap(); b.iter(|| black_box(dict).downcast::().unwrap()); }); } @@ -85,8 +107,6 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("extract_hashmap", extract_hashmap); c.bench_function("extract_btreemap", extract_btreemap); c.bench_function("mapping_from_dict", mapping_from_dict); - - #[cfg(feature = "hashbrown")] c.bench_function("extract_hashbrown_map", extract_hashbrown_map); } diff --git a/pyo3-benches/benches/bench_extract.rs b/pyo3-benches/benches/bench_extract.rs index 9bb7ef60ab4..2062ccba7b5 100644 --- a/pyo3-benches/benches/bench_extract.rs +++ b/pyo3-benches/benches/bench_extract.rs @@ -9,7 +9,7 @@ use pyo3::{ fn extract_str_extract_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let s = PyString::new_bound(py, "Hello, World!").into_any(); + let s = PyString::new(py, "Hello, World!").into_any(); bench.iter(|| black_box(&s).extract::<&str>().unwrap()); }); @@ -17,7 +17,7 @@ fn extract_str_extract_success(bench: &mut Bencher<'_>) { fn extract_str_extract_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); + let d = PyDict::new(py).into_any(); bench.iter(|| match black_box(&d).extract::<&str>() { Ok(v) => panic!("should err {}", v), @@ -26,10 +26,10 @@ fn extract_str_extract_fail(bench: &mut Bencher<'_>) { }); } -#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] +#[cfg(Py_3_10)] fn extract_str_downcast_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let s = PyString::new_bound(py, "Hello, World!").into_any(); + let s = PyString::new(py, "Hello, World!").into_any(); bench.iter(|| { let py_str = black_box(&s).downcast::().unwrap(); @@ -40,7 +40,7 @@ fn extract_str_downcast_success(bench: &mut Bencher<'_>) { fn extract_str_downcast_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); + let d = PyDict::new(py).into_any(); bench.iter(|| match black_box(&d).downcast::() { Ok(v) => panic!("should err {}", v), @@ -51,7 +51,7 @@ fn extract_str_downcast_fail(bench: &mut Bencher<'_>) { fn extract_int_extract_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = 123.to_object(py).into_bound(py); + let int = 123i32.into_pyobject(py).unwrap(); bench.iter(|| black_box(&int).extract::().unwrap()); }); @@ -59,7 +59,7 @@ fn extract_int_extract_success(bench: &mut Bencher<'_>) { fn extract_int_extract_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); + let d = PyDict::new(py).into_any(); bench.iter(|| match black_box(&d).extract::() { Ok(v) => panic!("should err {}", v), @@ -70,7 +70,7 @@ fn extract_int_extract_fail(bench: &mut Bencher<'_>) { fn extract_int_downcast_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = 123.to_object(py).into_bound(py); + let int = 123i32.into_pyobject(py).unwrap(); bench.iter(|| { let py_int = black_box(&int).downcast::().unwrap(); @@ -81,7 +81,7 @@ fn extract_int_downcast_success(bench: &mut Bencher<'_>) { fn extract_int_downcast_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); + let d = PyDict::new(py).into_any(); bench.iter(|| match black_box(&d).downcast::() { Ok(v) => panic!("should err {}", v), @@ -92,7 +92,7 @@ fn extract_int_downcast_fail(bench: &mut Bencher<'_>) { fn extract_float_extract_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let float = 23.42.to_object(py).into_bound(py); + let float = 23.42f64.into_pyobject(py).unwrap(); bench.iter(|| black_box(&float).extract::().unwrap()); }); @@ -100,7 +100,7 @@ fn extract_float_extract_success(bench: &mut Bencher<'_>) { fn extract_float_extract_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); + let d = PyDict::new(py).into_any(); bench.iter(|| match black_box(&d).extract::() { Ok(v) => panic!("should err {}", v), @@ -111,7 +111,7 @@ fn extract_float_extract_fail(bench: &mut Bencher<'_>) { fn extract_float_downcast_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let float = 23.42.to_object(py).into_bound(py); + let float = 23.42f64.into_pyobject(py).unwrap(); bench.iter(|| { let py_float = black_box(&float).downcast::().unwrap(); @@ -122,7 +122,7 @@ fn extract_float_downcast_success(bench: &mut Bencher<'_>) { fn extract_float_downcast_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); + let d = PyDict::new(py).into_any(); bench.iter(|| match black_box(&d).downcast::() { Ok(v) => panic!("should err {}", v), diff --git a/pyo3-benches/benches/bench_frompyobject.rs b/pyo3-benches/benches/bench_frompyobject.rs index f53f116a154..6411a391f9a 100644 --- a/pyo3-benches/benches/bench_frompyobject.rs +++ b/pyo3-benches/benches/bench_frompyobject.rs @@ -17,7 +17,7 @@ enum ManyTypes { fn enum_from_pyobject(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any = PyString::new_bound(py, "hello world").into_any(); + let any = PyString::new(py, "hello world").into_any(); b.iter(|| black_box(&any).extract::().unwrap()); }) @@ -25,7 +25,7 @@ fn enum_from_pyobject(b: &mut Bencher<'_>) { fn list_via_downcast(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any = PyList::empty_bound(py).into_any(); + let any = PyList::empty(py).into_any(); b.iter(|| black_box(&any).downcast::().unwrap()); }) @@ -33,7 +33,7 @@ fn list_via_downcast(b: &mut Bencher<'_>) { fn list_via_extract(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any = PyList::empty_bound(py).into_any(); + let any = PyList::empty(py).into_any(); b.iter(|| black_box(&any).extract::>().unwrap()); }) @@ -41,7 +41,7 @@ fn list_via_extract(b: &mut Bencher<'_>) { fn not_a_list_via_downcast(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any = PyString::new_bound(py, "foobar").into_any(); + let any = PyString::new(py, "foobar").into_any(); b.iter(|| black_box(&any).downcast::().unwrap_err()); }) @@ -49,7 +49,7 @@ fn not_a_list_via_downcast(b: &mut Bencher<'_>) { fn not_a_list_via_extract(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any = PyString::new_bound(py, "foobar").into_any(); + let any = PyString::new(py, "foobar").into_any(); b.iter(|| black_box(&any).extract::>().unwrap_err()); }) @@ -63,7 +63,7 @@ enum ListOrNotList<'a> { fn not_a_list_via_extract_enum(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any = PyString::new_bound(py, "foobar").into_any(); + let any = PyString::new(py, "foobar").into_any(); b.iter(|| match black_box(&any).extract::>() { Ok(ListOrNotList::List(_list)) => panic!(), diff --git a/pyo3-benches/benches/bench_gil.rs b/pyo3-benches/benches/bench_gil.rs index 59b9ff9686f..cede8836f35 100644 --- a/pyo3-benches/benches/bench_gil.rs +++ b/pyo3-benches/benches/bench_gil.rs @@ -1,4 +1,4 @@ -use codspeed_criterion_compat::{criterion_group, criterion_main, BatchSize, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; @@ -9,14 +9,8 @@ fn bench_clean_acquire_gil(b: &mut Bencher<'_>) { fn bench_dirty_acquire_gil(b: &mut Bencher<'_>) { let obj = Python::with_gil(|py| py.None()); - b.iter_batched( - || { - // Clone and drop an object so that the GILPool has work to do. - let _ = obj.clone(); - }, - |_| Python::with_gil(|_| {}), - BatchSize::NumBatches(1), - ); + // Drop the returned clone of the object so that the reference pool has work to do. + b.iter(|| Python::with_gil(|py| obj.clone_ref(py))); } fn criterion_benchmark(c: &mut Criterion) { diff --git a/pyo3-benches/benches/bench_intern.rs b/pyo3-benches/benches/bench_intern.rs index f9f9162a5ee..1b7dc07370a 100644 --- a/pyo3-benches/benches/bench_intern.rs +++ b/pyo3-benches/benches/bench_intern.rs @@ -8,7 +8,7 @@ use pyo3::intern; fn getattr_direct(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let sys = &py.import_bound("sys").unwrap(); + let sys = &py.import("sys").unwrap(); b.iter(|| black_box(sys).getattr("version").unwrap()); }); @@ -16,7 +16,7 @@ fn getattr_direct(b: &mut Bencher<'_>) { fn getattr_intern(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let sys = &py.import_bound("sys").unwrap(); + let sys = &py.import("sys").unwrap(); b.iter(|| black_box(sys).getattr(intern!(py, "version")).unwrap()); }); diff --git a/pyo3-benches/benches/bench_intopyobject.rs b/pyo3-benches/benches/bench_intopyobject.rs new file mode 100644 index 00000000000..42af893cd8a --- /dev/null +++ b/pyo3-benches/benches/bench_intopyobject.rs @@ -0,0 +1,95 @@ +use std::hint::black_box; + +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; + +use pyo3::conversion::IntoPyObject; +use pyo3::prelude::*; +use pyo3::types::PyBytes; + +fn bench_bytes_new(b: &mut Bencher<'_>, data: &[u8]) { + Python::with_gil(|py| { + b.iter_with_large_drop(|| PyBytes::new(py, black_box(data))); + }); +} + +fn bytes_new_small(b: &mut Bencher<'_>) { + bench_bytes_new(b, &[]); +} + +fn bytes_new_medium(b: &mut Bencher<'_>) { + let data = (0..u8::MAX).collect::>(); + bench_bytes_new(b, &data); +} + +fn bytes_new_large(b: &mut Bencher<'_>) { + let data = vec![10u8; 100_000]; + bench_bytes_new(b, &data); +} + +fn bench_bytes_into_pyobject(b: &mut Bencher<'_>, data: &[u8]) { + Python::with_gil(|py| { + b.iter_with_large_drop(|| black_box(data).into_pyobject(py)); + }); +} + +fn byte_slice_into_pyobject_small(b: &mut Bencher<'_>) { + bench_bytes_into_pyobject(b, &[]); +} + +fn byte_slice_into_pyobject_medium(b: &mut Bencher<'_>) { + let data = (0..u8::MAX).collect::>(); + bench_bytes_into_pyobject(b, &data); +} + +fn byte_slice_into_pyobject_large(b: &mut Bencher<'_>) { + let data = vec![10u8; 100_000]; + bench_bytes_into_pyobject(b, &data); +} + +#[allow(deprecated)] +fn byte_slice_into_py(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let data = (0..u8::MAX).collect::>(); + let bytes = data.as_slice(); + b.iter_with_large_drop(|| black_box(bytes).into_py(py)); + }); +} + +fn vec_into_pyobject(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let bytes = (0..u8::MAX).collect::>(); + b.iter_with_large_drop(|| black_box(&bytes).clone().into_pyobject(py)); + }); +} + +#[allow(deprecated)] +fn vec_into_py(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let bytes = (0..u8::MAX).collect::>(); + b.iter_with_large_drop(|| black_box(&bytes).clone().into_py(py)); + }); +} + +fn criterion_benchmark(c: &mut Criterion) { + c.bench_function("bytes_new_small", bytes_new_small); + c.bench_function("bytes_new_medium", bytes_new_medium); + c.bench_function("bytes_new_large", bytes_new_large); + c.bench_function( + "byte_slice_into_pyobject_small", + byte_slice_into_pyobject_small, + ); + c.bench_function( + "byte_slice_into_pyobject_medium", + byte_slice_into_pyobject_medium, + ); + c.bench_function( + "byte_slice_into_pyobject_large", + byte_slice_into_pyobject_large, + ); + c.bench_function("byte_slice_into_py", byte_slice_into_py); + c.bench_function("vec_into_pyobject", vec_into_pyobject); + c.bench_function("vec_into_py", vec_into_py); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/pyo3-benches/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs index dcbdb4779cb..7a19452455e 100644 --- a/pyo3-benches/benches/bench_list.rs +++ b/pyo3-benches/benches/bench_list.rs @@ -8,7 +8,7 @@ use pyo3::types::{PyList, PySequence}; fn iter_list(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let list = PyList::new_bound(py, 0..LEN); + let list = PyList::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for x in &list { @@ -22,14 +22,14 @@ fn iter_list(b: &mut Bencher<'_>) { fn list_new(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - b.iter_with_large_drop(|| PyList::new_bound(py, 0..LEN)); + b.iter_with_large_drop(|| PyList::new(py, 0..LEN)); }); } fn list_get_item(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let list = PyList::new_bound(py, 0..LEN); + let list = PyList::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -39,11 +39,37 @@ fn list_get_item(b: &mut Bencher<'_>) { }); } +fn list_nth(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + const LEN: usize = 50; + let list = PyList::new_bound(py, 0..LEN); + let mut sum = 0; + b.iter(|| { + for i in 0..LEN { + sum += list.iter().nth(i).unwrap().extract::().unwrap(); + } + }); + }); +} + +fn list_nth_back(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + const LEN: usize = 50; + let list = PyList::new_bound(py, 0..LEN); + let mut sum = 0; + b.iter(|| { + for i in 0..LEN { + sum += list.iter().nth_back(i).unwrap().extract::().unwrap(); + } + }); + }); +} + #[cfg(not(Py_LIMITED_API))] fn list_get_item_unchecked(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let list = PyList::new_bound(py, 0..LEN); + let list = PyList::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -58,7 +84,7 @@ fn list_get_item_unchecked(b: &mut Bencher<'_>) { fn sequence_from_list(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let list = &PyList::new_bound(py, 0..LEN); + let list = &PyList::new(py, 0..LEN).unwrap(); b.iter(|| black_box(list).downcast::().unwrap()); }); } @@ -66,8 +92,10 @@ fn sequence_from_list(b: &mut Bencher<'_>) { fn criterion_benchmark(c: &mut Criterion) { c.bench_function("iter_list", iter_list); c.bench_function("list_new", list_new); + c.bench_function("list_nth", list_nth); + c.bench_function("list_nth_back", list_nth_back); c.bench_function("list_get_item", list_get_item); - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, Py_GIL_DISABLED)))] c.bench_function("list_get_item_unchecked", list_get_item_unchecked); c.bench_function("sequence_from_list", sequence_from_list); } diff --git a/pyo3-benches/benches/bench_pyobject.rs b/pyo3-benches/benches/bench_pyobject.rs index af25d61ce6a..c731216c79f 100644 --- a/pyo3-benches/benches/bench_pyobject.rs +++ b/pyo3-benches/benches/bench_pyobject.rs @@ -1,4 +1,12 @@ -use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, BatchSize, Bencher, Criterion}; + +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc::channel, + Arc, Barrier, +}; +use std::thread::spawn; +use std::time::{Duration, Instant}; use pyo3::prelude::*; @@ -6,14 +14,102 @@ fn drop_many_objects(b: &mut Bencher<'_>) { Python::with_gil(|py| { b.iter(|| { for _ in 0..1000 { - std::mem::drop(py.None()); + drop(py.None()); } }); }); } +fn drop_many_objects_without_gil(b: &mut Bencher<'_>) { + b.iter_batched( + || Python::with_gil(|py| (0..1000).map(|_| py.None()).collect::>()), + |objs| { + drop(objs); + + Python::with_gil(|_py| ()); + }, + BatchSize::SmallInput, + ); +} + +fn drop_many_objects_multiple_threads(b: &mut Bencher<'_>) { + const THREADS: usize = 5; + + let barrier = Arc::new(Barrier::new(1 + THREADS)); + + let done = Arc::new(AtomicUsize::new(0)); + + let sender = (0..THREADS) + .map(|_| { + let (sender, receiver) = channel(); + + let barrier = barrier.clone(); + + let done = done.clone(); + + spawn(move || { + for objs in receiver { + barrier.wait(); + + drop(objs); + + done.fetch_add(1, Ordering::AcqRel); + } + }); + + sender + }) + .collect::>(); + + b.iter_custom(|iters| { + let mut duration = Duration::ZERO; + + let mut last_done = done.load(Ordering::Acquire); + + for _ in 0..iters { + for sender in &sender { + let objs = Python::with_gil(|py| { + (0..1000 / THREADS) + .map(|_| py.None()) + .collect::>() + }); + + sender.send(objs).unwrap(); + } + + barrier.wait(); + + let start = Instant::now(); + + loop { + Python::with_gil(|_py| ()); + + let done = done.load(Ordering::Acquire); + if done - last_done == THREADS { + last_done = done; + break; + } + } + + Python::with_gil(|_py| ()); + + duration += start.elapsed(); + } + + duration + }); +} + fn criterion_benchmark(c: &mut Criterion) { c.bench_function("drop_many_objects", drop_many_objects); + c.bench_function( + "drop_many_objects_without_gil", + drop_many_objects_without_gil, + ); + c.bench_function( + "drop_many_objects_multiple_threads", + drop_many_objects_multiple_threads, + ); } criterion_group!(benches, criterion_benchmark); diff --git a/pyo3-benches/benches/bench_set.rs b/pyo3-benches/benches/bench_set.rs index 18134a15bd5..2d468740ea0 100644 --- a/pyo3-benches/benches/bench_set.rs +++ b/pyo3-benches/benches/bench_set.rs @@ -1,7 +1,7 @@ use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; -use pyo3::prelude::*; use pyo3::types::PySet; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::{ collections::{BTreeSet, HashSet}, hint::black_box, @@ -12,15 +12,15 @@ fn set_new(b: &mut Bencher<'_>) { const LEN: usize = 100_000; // Create Python objects up-front, so that the benchmark doesn't need to include // the cost of allocating LEN Python integers - let elements: Vec = (0..LEN).map(|i| i.into_py(py)).collect(); - b.iter_with_large_drop(|| PySet::new_bound(py, &elements).unwrap()); + let elements: Vec = (0..LEN).map(|i| i.into_py_any(py).unwrap()).collect(); + b.iter_with_large_drop(|| PySet::new(py, &elements).unwrap()); }); } fn iter_set(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let set = PySet::new_bound(py, &(0..LEN).collect::>()).unwrap(); + let set = PySet::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for x in &set { @@ -34,9 +34,7 @@ fn iter_set(b: &mut Bencher<'_>) { fn extract_hashset(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let any = PySet::new_bound(py, &(0..LEN).collect::>()) - .unwrap() - .into_any(); + let any = PySet::new(py, 0..LEN).unwrap().into_any(); b.iter_with_large_drop(|| black_box(&any).extract::>()); }); } @@ -44,20 +42,15 @@ fn extract_hashset(b: &mut Bencher<'_>) { fn extract_btreeset(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let any = PySet::new_bound(py, &(0..LEN).collect::>()) - .unwrap() - .into_any(); + let any = PySet::new(py, 0..LEN).unwrap().into_any(); b.iter_with_large_drop(|| black_box(&any).extract::>()); }); } -#[cfg(feature = "hashbrown")] fn extract_hashbrown_set(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let any = PySet::new_bound(py, &(0..LEN).collect::>()) - .unwrap() - .into_any(); + let any = PySet::new(py, 0..LEN).unwrap().into_any(); b.iter_with_large_drop(|| black_box(&any).extract::>()); }); } @@ -67,8 +60,6 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("iter_set", iter_set); c.bench_function("extract_hashset", extract_hashset); c.bench_function("extract_btreeset", extract_btreeset); - - #[cfg(feature = "hashbrown")] c.bench_function("extract_hashbrown_set", extract_hashbrown_set); } diff --git a/pyo3-benches/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs index 3c0b56a0234..e235567e926 100644 --- a/pyo3-benches/benches/bench_tuple.rs +++ b/pyo3-benches/benches/bench_tuple.rs @@ -8,7 +8,7 @@ use pyo3::types::{PyList, PySequence, PyTuple}; fn iter_tuple(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let tuple = PyTuple::new_bound(py, 0..LEN); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for x in tuple.iter_borrowed() { @@ -22,14 +22,14 @@ fn iter_tuple(b: &mut Bencher<'_>) { fn tuple_new(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - b.iter_with_large_drop(|| PyTuple::new_bound(py, 0..LEN)); + b.iter_with_large_drop(|| PyTuple::new(py, 0..LEN).unwrap()); }); } fn tuple_get_item(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -43,7 +43,7 @@ fn tuple_get_item(b: &mut Bencher<'_>) { fn tuple_get_item_unchecked(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -58,7 +58,7 @@ fn tuple_get_item_unchecked(b: &mut Bencher<'_>) { fn tuple_get_borrowed_item(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -76,7 +76,7 @@ fn tuple_get_borrowed_item(b: &mut Bencher<'_>) { fn tuple_get_borrowed_item_unchecked(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -94,7 +94,7 @@ fn tuple_get_borrowed_item_unchecked(b: &mut Bencher<'_>) { fn sequence_from_tuple(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN).into_any(); + let tuple = PyTuple::new(py, 0..LEN).unwrap().into_any(); b.iter(|| black_box(&tuple).downcast::().unwrap()); }); } @@ -102,29 +102,68 @@ fn sequence_from_tuple(b: &mut Bencher<'_>) { fn tuple_new_list(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN); - b.iter_with_large_drop(|| PyList::new_bound(py, tuple.iter_borrowed())); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); + b.iter_with_large_drop(|| PyList::new(py, tuple.iter_borrowed())); }); } fn tuple_to_list(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); b.iter_with_large_drop(|| tuple.to_list()); }); } +#[allow(deprecated)] fn tuple_into_py(b: &mut Bencher<'_>) { Python::with_gil(|py| { b.iter(|| -> PyObject { (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12).into_py(py) }); }); } +fn tuple_into_pyobject(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + b.iter(|| { + (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) + .into_pyobject(py) + .unwrap() + }); + }); +} + +fn tuple_nth(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + const LEN: usize = 50; + let list = PyTuple::new(py, 0..LEN).unwrap(); + let mut sum = 0; + b.iter(|| { + for i in 0..LEN { + sum += list.iter().nth(i).unwrap().extract::().unwrap(); + } + }); + }); +} + +fn tuple_nth_back(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + const LEN: usize = 50; + let list = PyTuple::new(py, 0..LEN).unwrap(); + let mut sum = 0; + b.iter(|| { + for i in 0..LEN { + sum += list.iter().nth_back(i).unwrap().extract::().unwrap(); + } + }); + }); +} + fn criterion_benchmark(c: &mut Criterion) { c.bench_function("iter_tuple", iter_tuple); c.bench_function("tuple_new", tuple_new); c.bench_function("tuple_get_item", tuple_get_item); + c.bench_function("tuple_nth", tuple_nth); + c.bench_function("tuple_nth_back", tuple_nth_back); #[cfg(not(any(Py_LIMITED_API, PyPy)))] c.bench_function("tuple_get_item_unchecked", tuple_get_item_unchecked); c.bench_function("tuple_get_borrowed_item", tuple_get_borrowed_item); @@ -137,6 +176,7 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("tuple_new_list", tuple_new_list); c.bench_function("tuple_to_list", tuple_to_list); c.bench_function("tuple_into_py", tuple_into_py); + c.bench_function("tuple_into_pyobject", tuple_into_pyobject); } criterion_group!(benches, criterion_benchmark); diff --git a/pyo3-benches/build.rs b/pyo3-benches/build.rs new file mode 100644 index 00000000000..0475124bb4e --- /dev/null +++ b/pyo3-benches/build.rs @@ -0,0 +1,3 @@ +fn main() { + pyo3_build_config::use_pyo3_cfgs(); +} diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 1eb269c2132..5abb24e86d6 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.21.1" +version = "0.24.1" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -9,15 +9,16 @@ repository = "https://github.com/pyo3/pyo3" categories = ["api-bindings", "development-tools::ffi"] license = "MIT OR Apache-2.0" edition = "2021" +rust-version = "1.63" [dependencies] once_cell = "1" -python3-dll-a = { version = "0.2.6", optional = true } -target-lexicon = "0.12" +python3-dll-a = { version = "0.2.12", optional = true } +target-lexicon = "0.13" [build-dependencies] -python3-dll-a = { version = "0.2.6", optional = true } -target-lexicon = "0.12" +python3-dll-a = { version = "0.2.12", optional = true } +target-lexicon = "0.13" [features] default = [] @@ -36,7 +37,8 @@ abi3-py38 = ["abi3-py39"] abi3-py39 = ["abi3-py310"] abi3-py310 = ["abi3-py311"] abi3-py311 = ["abi3-py312"] -abi3-py312 = ["abi3"] +abi3-py312 = ["abi3-py313"] +abi3-py313 = ["abi3"] [package.metadata.docs.rs] features = ["resolve-config"] diff --git a/pyo3-build-config/build.rs b/pyo3-build-config/build.rs index 309a78c87da..a6e767edcf0 100644 --- a/pyo3-build-config/build.rs +++ b/pyo3-build-config/build.rs @@ -12,7 +12,7 @@ mod errors; use std::{env, path::Path}; use errors::{Context, Result}; -use impl_::{env_var, make_interpreter_config, InterpreterConfig}; +use impl_::{make_interpreter_config, InterpreterConfig}; fn configure(interpreter_config: Option, name: &str) -> Result { let target = Path::new(&env::var_os("OUT_DIR").unwrap()).join(name); @@ -29,28 +29,12 @@ fn configure(interpreter_config: Option, name: &str) -> Resul } } -/// If PYO3_CONFIG_FILE is set, copy it into the crate. -fn config_file() -> Result> { - if let Some(path) = env_var("PYO3_CONFIG_FILE") { - let path = Path::new(&path); - println!("cargo:rerun-if-changed={}", path.display()); - // Absolute path is necessary because this build script is run with a cwd different to the - // original `cargo build` instruction. - ensure!( - path.is_absolute(), - "PYO3_CONFIG_FILE must be an absolute path" - ); - - let interpreter_config = InterpreterConfig::from_path(path) - .context("failed to parse contents of PYO3_CONFIG_FILE")?; - Ok(Some(interpreter_config)) - } else { - Ok(None) - } -} - fn generate_build_configs() -> Result<()> { - let configured = configure(config_file()?, "pyo3-build-config-file.txt")?; + // If PYO3_CONFIG_FILE is set, copy it into the crate. + let configured = configure( + InterpreterConfig::from_pyo3_config_file_env().transpose()?, + "pyo3-build-config-file.txt", + )?; if configured { // Don't bother trying to find an interpreter on the host system diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index d5373db9655..2c4955dcc6f 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -6,6 +6,8 @@ #[path = "import_lib.rs"] mod import_lib; +#[cfg(test)] +use std::cell::RefCell; use std::{ collections::{HashMap, HashSet}, env, @@ -15,22 +17,21 @@ use std::{ io::{BufRead, BufReader, Read, Write}, path::{Path, PathBuf}, process::{Command, Stdio}, - str, - str::FromStr, + str::{self, FromStr}, }; pub use target_lexicon::Triple; -use target_lexicon::{Environment, OperatingSystem}; +use target_lexicon::{Architecture, Environment, OperatingSystem}; use crate::{ bail, ensure, errors::{Context, Error, Result}, - format_warn, warn, + warn, }; /// Minimum Python version PyO3 supports. -const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 }; +pub(crate) const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 }; /// GraalPy may implement the same CPython version over multiple releases. const MINIMUM_SUPPORTED_VERSION_GRAALPY: PythonVersion = PythonVersion { @@ -39,7 +40,12 @@ const MINIMUM_SUPPORTED_VERSION_GRAALPY: PythonVersion = PythonVersion { }; /// Maximum Python version that can be used as minimum required Python version with abi3. -const ABI3_MAX_MINOR: u8 = 12; +pub(crate) const ABI3_MAX_MINOR: u8 = 13; + +#[cfg(test)] +thread_local! { + static READ_ENV_VARS: RefCell> = const { RefCell::new(Vec::new()) }; +} /// Gets an environment variable owned by cargo. /// @@ -54,6 +60,12 @@ pub fn env_var(var: &str) -> Option { if cfg!(feature = "resolve-config") { println!("cargo:rerun-if-env-changed={}", var); } + #[cfg(test)] + { + READ_ENV_VARS.with(|env_vars| { + env_vars.borrow_mut().push(var.to_owned()); + }); + } env::var_os(var) } @@ -155,6 +167,8 @@ pub struct InterpreterConfig { /// /// Serialized to multiple `extra_build_script_line` values. pub extra_build_script_lines: Vec, + /// macOS Python3.framework requires special rpath handling + pub python_framework_prefix: Option, } impl InterpreterConfig { @@ -165,31 +179,28 @@ impl InterpreterConfig { let mut out = vec![]; - // pyo3-build-config was released when Python 3.6 was supported, so minimum flag to emit is - // Py_3_6 (to avoid silently breaking users who depend on this cfg). - for i in 6..=self.version.minor { + for i in MINIMUM_SUPPORTED_VERSION.minor..=self.version.minor { out.push(format!("cargo:rustc-cfg=Py_3_{}", i)); } - if self.implementation.is_pypy() { - out.push("cargo:rustc-cfg=PyPy".to_owned()); - if self.abi3 { - out.push(format_warn!( - "PyPy does not yet support abi3 so the build artifacts will be version-specific. \ - See https://foss.heptapod.net/pypy/pypy/-/issues/3397 for more information." - )); - } - } else if self.implementation.is_graalpy() { - println!("cargo:rustc-cfg=GraalPy"); - if self.abi3 { - warn!("GraalPy does not support abi3 so the build artifacts will be version-specific."); - } - } else if self.abi3 { + match self.implementation { + PythonImplementation::CPython => {} + PythonImplementation::PyPy => out.push("cargo:rustc-cfg=PyPy".to_owned()), + PythonImplementation::GraalPy => out.push("cargo:rustc-cfg=GraalPy".to_owned()), + } + + // If Py_GIL_DISABLED is set, do not build with limited API support + if self.abi3 && !self.is_free_threaded() { out.push("cargo:rustc-cfg=Py_LIMITED_API".to_owned()); } for flag in &self.build_flags.0 { - out.push(format!("cargo:rustc-cfg=py_sys_config=\"{}\"", flag)); + match flag { + BuildFlag::Py_GIL_DISABLED => { + out.push("cargo:rustc-cfg=Py_GIL_DISABLED".to_owned()) + } + flag => out.push(format!("cargo:rustc-cfg=py_sys_config=\"{}\"", flag)), + } } out @@ -236,6 +247,7 @@ WINDOWS = platform.system() == "Windows" # macOS framework packages use shared linking FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK")) +FRAMEWORK_PREFIX = get_config_var("PYTHONFRAMEWORKPREFIX") # unix-style shared library enabled SHARED = bool(get_config_var("Py_ENABLE_SHARED")) @@ -244,6 +256,7 @@ print("implementation", platform.python_implementation()) print("version_major", sys.version_info[0]) print("version_minor", sys.version_info[1]) print("shared", PYPY or GRAALPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) +print("python_framework_prefix", FRAMEWORK_PREFIX) print_if_set("ld_version", get_config_var("LDVERSION")) print_if_set("libdir", get_config_var("LIBDIR")) print_if_set("base_prefix", base_prefix) @@ -251,6 +264,7 @@ print("executable", sys.executable) print("calcsize_pointer", struct.calcsize("P")) print("mingw", get_platform().startswith("mingw")) print("ext_suffix", get_config_var("EXT_SUFFIX")) +print("gil_disabled", get_config_var("Py_GIL_DISABLED")) "#; let output = run_python_script(interpreter.as_ref(), SCRIPT)?; let map: HashMap = parse_script_output(&output); @@ -279,6 +293,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) }; let shared = map["shared"].as_str() == "True"; + let python_framework_prefix = map.get("python_framework_prefix").cloned(); let version = PythonVersion { major: map["version_major"] @@ -293,6 +308,13 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) let implementation = map["implementation"].parse()?; + let gil_disabled = match map["gil_disabled"].as_str() { + "1" => true, + "0" => false, + "None" => false, + _ => panic!("Unknown Py_GIL_DISABLED value"), + }; + let lib_name = if cfg!(windows) { default_lib_name_windows( version, @@ -303,13 +325,15 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) // on Windows from sysconfig - e.g. ext_suffix may be // `_d.cp312-win_amd64.pyd` for 3.12 debug build map["ext_suffix"].starts_with("_d."), - ) + gil_disabled, + )? } else { default_lib_name_unix( version, implementation, map.get("ld_version").map(String::as_str), - ) + gil_disabled, + )? }; let lib_dir = if cfg!(windows) { @@ -340,6 +364,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) build_flags: BuildFlags::from_interpreter(interpreter)?, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix, }) } @@ -377,12 +402,20 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) Some(s) => !s.is_empty(), _ => false, }; + let python_framework_prefix = sysconfigdata + .get_value("PYTHONFRAMEWORKPREFIX") + .map(str::to_string); let lib_dir = get_key!(sysconfigdata, "LIBDIR").ok().map(str::to_string); + let gil_disabled = match sysconfigdata.get_value("Py_GIL_DISABLED") { + Some(value) => value == "1", + None => false, + }; let lib_name = Some(default_lib_name_unix( version, implementation, sysconfigdata.get_value("LDVERSION"), - )); + gil_disabled, + )?); let pointer_width = parse_key!(sysconfigdata, "SIZEOF_VOID_P") .map(|bytes_width: u32| bytes_width * 8) .ok(); @@ -400,6 +433,36 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix, + }) + } + + /// Import an externally-provided config file. + /// + /// The `abi3` features, if set, may apply an `abi3` constraint to the Python version. + #[allow(dead_code)] // only used in build.rs + pub(super) fn from_pyo3_config_file_env() -> Option> { + env_var("PYO3_CONFIG_FILE").map(|path| { + let path = Path::new(&path); + println!("cargo:rerun-if-changed={}", path.display()); + // Absolute path is necessary because this build script is run with a cwd different to the + // original `cargo build` instruction. + ensure!( + path.is_absolute(), + "PYO3_CONFIG_FILE must be an absolute path" + ); + + let mut config = InterpreterConfig::from_path(path) + .context("failed to parse contents of PYO3_CONFIG_FILE")?; + // If the abi3 feature is enabled, the minimum Python version is constrained by the abi3 + // feature. + // + // TODO: abi3 is a property of the build mode, not the interpreter. Should this be + // removed from `InterpreterConfig`? + config.abi3 |= is_abi3(); + config.fixup_for_abi3_version(get_abi3_version())?; + + Ok(config) }) } @@ -444,9 +507,10 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) let mut lib_dir = None; let mut executable = None; let mut pointer_width = None; - let mut build_flags = None; + let mut build_flags: Option = None; let mut suppress_build_script_link_lines = None; let mut extra_build_script_lines = vec![]; + let mut python_framework_prefix = None; for (i, line) in lines.enumerate() { let line = line.context("failed to read line from config")?; @@ -475,6 +539,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) "extra_build_script_line" => { extra_build_script_lines.push(value.to_string()); } + "python_framework_prefix" => parse_value!(python_framework_prefix, value), unknown => warn!("unknown config key `{}`", unknown), } } @@ -482,10 +547,12 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) let version = version.ok_or("missing value for version")?; let implementation = implementation.unwrap_or(PythonImplementation::CPython); let abi3 = abi3.unwrap_or(false); + let build_flags = build_flags.unwrap_or_default(); + let gil_disabled = build_flags.0.contains(&BuildFlag::Py_GIL_DISABLED); // Fixup lib_name if it's not set let lib_name = lib_name.or_else(|| { if let Ok(Ok(target)) = env::var("TARGET").map(|target| target.parse::()) { - default_lib_name_for_target(version, implementation, abi3, &target) + default_lib_name_for_target(version, implementation, abi3, gil_disabled, &target) } else { None } @@ -500,9 +567,10 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) lib_dir, executable, pointer_width, - build_flags: build_flags.unwrap_or_default(), + build_flags, suppress_build_script_link_lines: suppress_build_script_link_lines.unwrap_or(false), extra_build_script_lines, + python_framework_prefix, }) } @@ -512,9 +580,25 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) // Auto generate python3.dll import libraries for Windows targets. if self.lib_dir.is_none() { let target = target_triple_from_env(); - let py_version = if self.abi3 { None } else { Some(self.version) }; - self.lib_dir = - import_lib::generate_import_lib(&target, self.implementation, py_version)?; + let py_version = if self.implementation == PythonImplementation::CPython + && self.abi3 + && !self.is_free_threaded() + { + None + } else { + Some(self.version) + }; + let abiflags = if self.is_free_threaded() { + Some("t") + } else { + None + }; + self.lib_dir = import_lib::generate_import_lib( + &target, + self.implementation, + py_version, + abiflags, + )?; } Ok(()) } @@ -579,6 +663,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) write_option_line!(executable)?; write_option_line!(pointer_width)?; write_line!(build_flags)?; + write_option_line!(python_framework_prefix)?; write_line!(suppress_build_script_link_lines)?; for line in &self.extra_build_script_lines { writeln!(writer, "extra_build_script_line={}", line) @@ -619,10 +704,18 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) ) } - /// Lowers the configured version to the abi3 version, if set. + pub fn is_free_threaded(&self) -> bool { + self.build_flags.0.contains(&BuildFlag::Py_GIL_DISABLED) + } + + /// Updates configured ABI to build for to the requested abi3 version + /// This is a no-op for platforms where abi3 is not supported fn fixup_for_abi3_version(&mut self, abi3_version: Option) -> Result<()> { - // PyPy doesn't support abi3; don't adjust the version - if self.implementation.is_pypy() || self.implementation.is_graalpy() { + // PyPy, GraalPy, and the free-threaded build don't support abi3; don't adjust the version + if self.implementation.is_pypy() + || self.implementation.is_graalpy() + || self.is_free_threaded() + { return Ok(()); } @@ -650,6 +743,14 @@ pub struct PythonVersion { } impl PythonVersion { + pub const PY313: Self = PythonVersion { + major: 3, + minor: 13, + }; + const PY310: Self = PythonVersion { + major: 3, + minor: 10, + }; const PY37: Self = PythonVersion { major: 3, minor: 7 }; } @@ -775,6 +876,8 @@ pub fn is_linking_libpython() -> bool { /// Must be called from a PyO3 crate build script. fn is_linking_libpython_for_target(target: &Triple) -> bool { target.operating_system == OperatingSystem::Windows + // See https://github.com/PyO3/pyo3/issues/4068#issuecomment-2051159852 + || target.operating_system == OperatingSystem::Aix || target.environment == Environment::Android || target.environment == Environment::Androideabi || !is_extension_module() @@ -809,6 +912,9 @@ pub struct CrossCompileConfig { /// The compile target triple (e.g. aarch64-unknown-linux-gnu) target: Triple, + + /// Python ABI flags, used to detect free-threaded Python builds. + abiflags: Option, } impl CrossCompileConfig { @@ -823,7 +929,7 @@ impl CrossCompileConfig { ) -> Result> { if env_vars.any() || Self::is_cross_compiling_from_to(host, target) { let lib_dir = env_vars.lib_dir_path()?; - let version = env_vars.parse_version()?; + let (version, abiflags) = env_vars.parse_version()?; let implementation = env_vars.parse_implementation()?; let target = target.clone(); @@ -832,6 +938,7 @@ impl CrossCompileConfig { version, implementation, target, + abiflags, })) } else { Ok(None) @@ -851,11 +958,13 @@ impl CrossCompileConfig { // Not cross-compiling to compile for 32-bit Python from windows 64-bit compatible |= target.operating_system == OperatingSystem::Windows - && host.operating_system == OperatingSystem::Windows; + && host.operating_system == OperatingSystem::Windows + && matches!(target.architecture, Architecture::X86_32(_)) + && host.architecture == Architecture::X86_64; // Not cross-compiling to compile for x86-64 Python from macOS arm64 and vice versa - compatible |= target.operating_system == OperatingSystem::Darwin - && host.operating_system == OperatingSystem::Darwin; + compatible |= matches!(target.operating_system, OperatingSystem::Darwin(_)) + && matches!(host.operating_system, OperatingSystem::Darwin(_)); !compatible } @@ -905,22 +1014,25 @@ impl CrossCompileEnvVars { } /// Parses `PYO3_CROSS_PYTHON_VERSION` environment variable value - /// into `PythonVersion`. - fn parse_version(&self) -> Result> { - let version = self - .pyo3_cross_python_version - .as_ref() - .map(|os_string| { + /// into `PythonVersion` and ABI flags. + fn parse_version(&self) -> Result<(Option, Option)> { + match self.pyo3_cross_python_version.as_ref() { + Some(os_string) => { let utf8_str = os_string .to_str() .ok_or("PYO3_CROSS_PYTHON_VERSION is not valid a UTF-8 string")?; - utf8_str + let (utf8_str, abiflags) = if let Some(version) = utf8_str.strip_suffix('t') { + (version, Some("t".to_string())) + } else { + (utf8_str, None) + }; + let version = utf8_str .parse() - .context("failed to parse PYO3_CROSS_PYTHON_VERSION") - }) - .transpose()?; - - Ok(version) + .context("failed to parse PYO3_CROSS_PYTHON_VERSION")?; + Ok((Some(version), abiflags)) + } + None => Ok((None, None)), + } } /// Parses `PYO3_CROSS_PYTHON_IMPLEMENTATION` environment variable value @@ -964,11 +1076,11 @@ impl CrossCompileEnvVars { /// /// This function relies on PyO3 cross-compiling environment variables: /// -/// * `PYO3_CROSS`: If present, forces PyO3 to configure as a cross-compilation. -/// * `PYO3_CROSS_LIB_DIR`: If present, must be set to the directory containing +/// * `PYO3_CROSS`: If present, forces PyO3 to configure as a cross-compilation. +/// * `PYO3_CROSS_LIB_DIR`: If present, must be set to the directory containing /// the target's libpython DSO and the associated `_sysconfigdata*.py` file for /// Unix-like targets, or the Python DLL import libraries for the Windows target. -/// * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python +/// * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python /// installation. This variable is only needed if PyO3 cannnot determine the version to target /// from `abi3-py3*` features, or if there are multiple versions of Python present in /// `PYO3_CROSS_LIB_DIR`. @@ -1001,6 +1113,7 @@ pub enum BuildFlag { Py_DEBUG, Py_REF_DEBUG, Py_TRACE_REFS, + Py_GIL_DISABLED, COUNT_ALLOCS, Other(String), } @@ -1021,14 +1134,16 @@ impl FromStr for BuildFlag { "Py_DEBUG" => Ok(BuildFlag::Py_DEBUG), "Py_REF_DEBUG" => Ok(BuildFlag::Py_REF_DEBUG), "Py_TRACE_REFS" => Ok(BuildFlag::Py_TRACE_REFS), + "Py_GIL_DISABLED" => Ok(BuildFlag::Py_GIL_DISABLED), "COUNT_ALLOCS" => Ok(BuildFlag::COUNT_ALLOCS), other => Ok(BuildFlag::Other(other.to_owned())), } } } -/// A list of python interpreter compile-time preprocessor defines that -/// we will pick up and pass to rustc via `--cfg=py_sys_config={varname}`; +/// A list of python interpreter compile-time preprocessor defines. +/// +/// PyO3 will pick these up and pass to rustc via `--cfg=py_sys_config={varname}`; /// this allows using them conditional cfg attributes in the .rs files, so /// /// ```rust @@ -1044,10 +1159,11 @@ impl FromStr for BuildFlag { pub struct BuildFlags(pub HashSet); impl BuildFlags { - const ALL: [BuildFlag; 4] = [ + const ALL: [BuildFlag; 5] = [ BuildFlag::Py_DEBUG, BuildFlag::Py_REF_DEBUG, BuildFlag::Py_TRACE_REFS, + BuildFlag::Py_GIL_DISABLED, BuildFlag::COUNT_ALLOCS, ]; @@ -1059,11 +1175,7 @@ impl BuildFlags { Self( BuildFlags::ALL .iter() - .filter(|flag| { - config_map - .get_value(&flag.to_string()) - .map_or(false, |value| value == "1") - }) + .filter(|flag| config_map.get_value(flag.to_string()) == Some("1")) .cloned() .collect(), ) @@ -1074,10 +1186,15 @@ impl BuildFlags { /// the interpreter and printing variables of interest from /// sysconfig.get_config_vars. fn from_interpreter(interpreter: impl AsRef) -> Result { - // sysconfig is missing all the flags on windows, so we can't actually - // query the interpreter directly for its build flags. + // sysconfig is missing all the flags on windows for Python 3.12 and + // older, so we can't actually query the interpreter directly for its + // build flags on those versions. if cfg!(windows) { - return Ok(Self::new()); + let script = String::from("import sys;print(sys.version_info < (3, 13))"); + let stdout = run_python_script(interpreter.as_ref(), &script)?; + if stdout.trim_end() == "True" { + return Ok(Self::new()); + } } let mut script = String::from("import sysconfig\n"); @@ -1211,7 +1328,7 @@ fn ends_with(entry: &DirEntry, pat: &str) -> bool { /// Returns `None` if the library directory is not available, and a runtime error /// when no or multiple sysconfigdata files are found. fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result> { - let mut sysconfig_paths = find_all_sysconfigdata(cross); + let mut sysconfig_paths = find_all_sysconfigdata(cross)?; if sysconfig_paths.is_empty() { if let Some(lib_dir) = cross.lib_dir.as_ref() { bail!("Could not find _sysconfigdata*.py in {}", lib_dir.display()); @@ -1274,11 +1391,16 @@ fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result> { /// /// Returns an empty vector when the target Python library directory /// is not set via `PYO3_CROSS_LIB_DIR`. -pub fn find_all_sysconfigdata(cross: &CrossCompileConfig) -> Vec { +pub fn find_all_sysconfigdata(cross: &CrossCompileConfig) -> Result> { let sysconfig_paths = if let Some(lib_dir) = cross.lib_dir.as_ref() { - search_lib_dir(lib_dir, cross) + search_lib_dir(lib_dir, cross).with_context(|| { + format!( + "failed to search the lib dir at 'PYO3_CROSS_LIB_DIR={}'", + lib_dir.display() + ) + })? } else { - return Vec::new(); + return Ok(Vec::new()); }; let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME"); @@ -1296,7 +1418,7 @@ pub fn find_all_sysconfigdata(cross: &CrossCompileConfig) -> Vec { sysconfig_paths.sort(); sysconfig_paths.dedup(); - sysconfig_paths + Ok(sysconfig_paths) } fn is_pypy_lib_dir(path: &str, v: &Option) -> bool { @@ -1327,9 +1449,14 @@ fn is_cpython_lib_dir(path: &str, v: &Option) -> bool { } /// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths -fn search_lib_dir(path: impl AsRef, cross: &CrossCompileConfig) -> Vec { +fn search_lib_dir(path: impl AsRef, cross: &CrossCompileConfig) -> Result> { let mut sysconfig_paths = vec![]; - for f in fs::read_dir(path).expect("Path does not exist") { + for f in fs::read_dir(path.as_ref()).with_context(|| { + format!( + "failed to list the entries in '{}'", + path.as_ref().display() + ) + })? { sysconfig_paths.extend(match &f { // Python 3.7+ sysconfigdata with platform specifics Ok(f) if starts_with(f, "_sysconfigdata_") && ends_with(f, "py") => vec![f.path()], @@ -1337,7 +1464,7 @@ fn search_lib_dir(path: impl AsRef, cross: &CrossCompileConfig) -> Vec, cross: &CrossCompileConfig) -> Vec, cross: &CrossCompileConfig) -> Vec Result> { if let Some(path) = find_sysconfigdata(cross_compile_config)? { let data = parse_sysconfigdata(path)?; - let config = InterpreterConfig::from_sysconfigdata(&data)?; + let mut config = InterpreterConfig::from_sysconfigdata(&data)?; + if let Some(cross_lib_dir) = cross_compile_config.lib_dir_string() { + config.lib_dir = Some(cross_lib_dir) + } Ok(Some(config)) } else { @@ -1428,22 +1558,34 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result Result Result InterpreterConfig { +fn default_abi3_config(host: &Triple, version: PythonVersion) -> Result { // FIXME: PyPy & GraalPy do not support the Stable ABI. let implementation = PythonImplementation::CPython; let abi3 = true; @@ -1483,12 +1626,13 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> InterpreterConf abi3, false, false, - )) + false, + )?) } else { None }; - InterpreterConfig { + Ok(InterpreterConfig { implementation, version, shared: true, @@ -1500,7 +1644,8 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> InterpreterConf build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], - } + python_framework_prefix: None, + }) } /// Detects the cross compilation target interpreter configuration from all @@ -1540,28 +1685,24 @@ fn load_cross_compile_config( Ok(config) } -// Link against python3.lib for the stable ABI on Windows. -// See https://www.python.org/dev/peps/pep-0384/#linkage -// -// This contains only the limited ABI symbols. +// These contains only the limited ABI symbols. const WINDOWS_ABI3_LIB_NAME: &str = "python3"; +const WINDOWS_ABI3_DEBUG_LIB_NAME: &str = "python3_d"; fn default_lib_name_for_target( version: PythonVersion, implementation: PythonImplementation, abi3: bool, + gil_disabled: bool, target: &Triple, ) -> Option { if target.operating_system == OperatingSystem::Windows { - Some(default_lib_name_windows( - version, - implementation, - abi3, - false, - false, - )) + Some( + default_lib_name_windows(version, implementation, abi3, false, false, gil_disabled) + .unwrap(), + ) } else if is_linking_libpython_for_target(target) { - Some(default_lib_name_unix(version, implementation, None)) + Some(default_lib_name_unix(version, implementation, None, gil_disabled).unwrap()) } else { None } @@ -1573,18 +1714,36 @@ fn default_lib_name_windows( abi3: bool, mingw: bool, debug: bool, -) -> String { - if debug { + gil_disabled: bool, +) -> Result { + if debug && version < PythonVersion::PY310 { // CPython bug: linking against python3_d.dll raises error // https://github.com/python/cpython/issues/101614 - format!("python{}{}_d", version.major, version.minor) - } else if abi3 && !(implementation.is_pypy() || implementation.is_graalpy()) { - WINDOWS_ABI3_LIB_NAME.to_owned() + Ok(format!("python{}{}_d", version.major, version.minor)) + } else if abi3 && !(gil_disabled || implementation.is_pypy() || implementation.is_graalpy()) { + if debug { + Ok(WINDOWS_ABI3_DEBUG_LIB_NAME.to_owned()) + } else { + Ok(WINDOWS_ABI3_LIB_NAME.to_owned()) + } } else if mingw { + ensure!( + !gil_disabled, + "MinGW free-threaded builds are not currently tested or supported" + ); // https://packages.msys2.org/base/mingw-w64-python - format!("python{}.{}", version.major, version.minor) + Ok(format!("python{}.{}", version.major, version.minor)) + } else if gil_disabled { + ensure!(version >= PythonVersion::PY313, "Cannot compile C extensions for the free-threaded build on Python versions earlier than 3.13, found {}.{}", version.major, version.minor); + if debug { + Ok(format!("python{}{}t_d", version.major, version.minor)) + } else { + Ok(format!("python{}{}t", version.major, version.minor)) + } + } else if debug { + Ok(format!("python{}{}_d", version.major, version.minor)) } else { - format!("python{}{}", version.major, version.minor) + Ok(format!("python{}{}", version.major, version.minor)) } } @@ -1592,31 +1751,32 @@ fn default_lib_name_unix( version: PythonVersion, implementation: PythonImplementation, ld_version: Option<&str>, -) -> String { + gil_disabled: bool, +) -> Result { match implementation { PythonImplementation::CPython => match ld_version { - Some(ld_version) => format!("python{}", ld_version), + Some(ld_version) => Ok(format!("python{}", ld_version)), None => { if version > PythonVersion::PY37 { // PEP 3149 ABI version tags are finally gone - format!("python{}.{}", version.major, version.minor) + if gil_disabled { + ensure!(version >= PythonVersion::PY313, "Cannot compile C extensions for the free-threaded build on Python versions earlier than 3.13, found {}.{}", version.major, version.minor); + Ok(format!("python{}.{}t", version.major, version.minor)) + } else { + Ok(format!("python{}.{}", version.major, version.minor)) + } } else { // Work around https://bugs.python.org/issue36707 - format!("python{}.{}m", version.major, version.minor) + Ok(format!("python{}.{}m", version.major, version.minor)) } } }, - PythonImplementation::PyPy => { - if version >= (PythonVersion { major: 3, minor: 9 }) { - match ld_version { - Some(ld_version) => format!("pypy{}-c", ld_version), - None => format!("pypy{}.{}-c", version.major, version.minor), - } - } else { - format!("pypy{}-c", version.major) - } - } - PythonImplementation::GraalPy => "python-native".to_string(), + PythonImplementation::PyPy => match ld_version { + Some(ld_version) => Ok(format!("pypy{}-c", ld_version)), + None => Ok(format!("pypy{}.{}-c", version.major, version.minor)), + }, + + PythonImplementation::GraalPy => Ok("python-native".to_string()), } } @@ -1786,12 +1946,19 @@ pub fn make_interpreter_config() -> Result { ); }; - let mut interpreter_config = default_abi3_config(&host, abi3_version.unwrap()); + let mut interpreter_config = default_abi3_config(&host, abi3_version.unwrap())?; // Auto generate python3.dll import libraries for Windows targets. #[cfg(feature = "python3-dll-a")] { - let py_version = if interpreter_config.abi3 { + let gil_disabled = interpreter_config + .build_flags + .0 + .contains(&BuildFlag::Py_GIL_DISABLED); + let py_version = if interpreter_config.implementation == PythonImplementation::CPython + && interpreter_config.abi3 + && !gil_disabled + { None } else { Some(interpreter_config.version) @@ -1800,6 +1967,7 @@ pub fn make_interpreter_config() -> Result { &host, interpreter_config.implementation, py_version, + None, )?; } @@ -1833,7 +2001,7 @@ fn unescape(escaped: &str) -> Vec { } } - bytes.push(unhex(chunk[0]) << 4 | unhex(chunk[1])); + bytes.push((unhex(chunk[0]) << 4) | unhex(chunk[1])); } bytes @@ -1859,6 +2027,7 @@ mod tests { version: MINIMUM_SUPPORTED_VERSION, suppress_build_script_link_lines: true, extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -1887,6 +2056,7 @@ mod tests { }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -1908,6 +2078,7 @@ mod tests { version: MINIMUM_SUPPORTED_VERSION, suppress_build_script_link_lines: true, extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -1934,6 +2105,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -1956,6 +2128,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2058,6 +2231,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2087,6 +2261,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); @@ -2113,6 +2288,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2123,7 +2299,7 @@ mod tests { let min_version = "3.7".parse().unwrap(); assert_eq!( - default_abi3_config(&host, min_version), + default_abi3_config(&host, min_version).unwrap(), InterpreterConfig { implementation: PythonImplementation::CPython, version: PythonVersion { major: 3, minor: 7 }, @@ -2136,6 +2312,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2146,7 +2323,7 @@ mod tests { let min_version = "3.9".parse().unwrap(); assert_eq!( - default_abi3_config(&host, min_version), + default_abi3_config(&host, min_version).unwrap(), InterpreterConfig { implementation: PythonImplementation::CPython, version: PythonVersion { major: 3, minor: 9 }, @@ -2159,6 +2336,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2193,6 +2371,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2227,6 +2406,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2261,6 +2441,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2297,6 +2478,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2306,75 +2488,184 @@ mod tests { use PythonImplementation::*; assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, CPython, false, false, false, - ), - "python37", + false, + ) + .unwrap(), + "python39", ); + assert!(super::default_lib_name_windows( + PythonVersion { major: 3, minor: 9 }, + CPython, + false, + false, + false, + true, + ) + .is_err()); assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, CPython, true, false, false, - ), + false, + ) + .unwrap(), "python3", ); assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, CPython, false, true, false, - ), - "python3.7", + false, + ) + .unwrap(), + "python3.9", ); assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, CPython, true, true, false, - ), + false, + ) + .unwrap(), "python3", ); assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, PyPy, true, false, false, - ), - "python37", + false, + ) + .unwrap(), + "python39", ); assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, CPython, false, false, true, - ), - "python37_d", + false, + ) + .unwrap(), + "python39_d", ); - // abi3 debug builds on windows use version-specific lib + // abi3 debug builds on windows use version-specific lib on 3.9 and older // to workaround https://github.com/python/cpython/issues/101614 assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, CPython, true, false, true, - ), - "python37_d", + false, + ) + .unwrap(), + "python39_d", + ); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 10 + }, + CPython, + true, + false, + true, + false, + ) + .unwrap(), + "python3_d", + ); + // Python versions older than 3.13 don't support gil_disabled + assert!(super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 12, + }, + CPython, + false, + false, + false, + true, + ) + .is_err()); + // mingw and free-threading are incompatible (until someone adds support) + assert!(super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 12, + }, + CPython, + false, + true, + false, + true, + ) + .is_err()); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + false, + false, + false, + true, + ) + .unwrap(), + "python313t", + ); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + true, // abi3 true should not affect the free-threaded lib name + false, + false, + true, + ) + .unwrap(), + "python313t", + ); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + false, + false, + true, + true, + ) + .unwrap(), + "python313t_d", ); } @@ -2383,16 +2674,34 @@ mod tests { use PythonImplementation::*; // Defaults to python3.7m for CPython 3.7 assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 7 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 7 }, + CPython, + None, + false + ) + .unwrap(), "python3.7m", ); // Defaults to pythonX.Y for CPython 3.8+ assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 8 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 8 }, + CPython, + None, + false + ) + .unwrap(), "python3.8", ); assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 9 }, + CPython, + None, + false + ) + .unwrap(), "python3.9", ); // Can use ldversion to override for CPython @@ -2400,22 +2709,56 @@ mod tests { super::default_lib_name_unix( PythonVersion { major: 3, minor: 9 }, CPython, - Some("3.7md") - ), + Some("3.7md"), + false + ) + .unwrap(), "python3.7md", ); - // PyPy 3.7 ignores ldversion + // PyPy 3.9 includes ldversion assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 7 }, PyPy, Some("3.7md")), - "pypy3-c", + super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, None, false) + .unwrap(), + "pypy3.9-c", ); - // PyPy 3.9 includes ldversion assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, Some("3.9d")), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 9 }, + PyPy, + Some("3.9d"), + false + ) + .unwrap(), "pypy3.9d-c", ); + + // free-threading adds a t suffix + assert_eq!( + super::default_lib_name_unix( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + None, + true + ) + .unwrap(), + "python3.13t", + ); + // 3.12 and older are incompatible with gil_disabled + assert!(super::default_lib_name_unix( + PythonVersion { + major: 3, + minor: 12, + }, + CPython, + None, + true, + ) + .is_err()); } #[test] @@ -2429,7 +2772,7 @@ mod tests { assert_eq!( env_vars.parse_version().unwrap(), - Some(PythonVersion { major: 3, minor: 9 }) + (Some(PythonVersion { major: 3, minor: 9 }), None), ); let env_vars = CrossCompileEnvVars { @@ -2439,7 +2782,25 @@ mod tests { pyo3_cross_python_implementation: None, }; - assert_eq!(env_vars.parse_version().unwrap(), None); + assert_eq!(env_vars.parse_version().unwrap(), (None, None)); + + let env_vars = CrossCompileEnvVars { + pyo3_cross: None, + pyo3_cross_lib_dir: None, + pyo3_cross_python_version: Some("3.13t".into()), + pyo3_cross_python_implementation: None, + }; + + assert_eq!( + env_vars.parse_version().unwrap(), + ( + Some(PythonVersion { + major: 3, + minor: 13 + }), + Some("t".into()) + ), + ); let env_vars = CrossCompileEnvVars { pyo3_cross: None, @@ -2465,6 +2826,7 @@ mod tests { version: PythonVersion { major: 3, minor: 7 }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; config @@ -2487,6 +2849,7 @@ mod tests { version: PythonVersion { major: 3, minor: 7 }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert!(config @@ -2522,6 +2885,11 @@ mod tests { version: Some(interpreter_config.version), implementation: Some(interpreter_config.implementation), target: triple!("x86_64-unknown-linux-gnu"), + abiflags: if interpreter_config.is_free_threaded() { + Some("t".into()) + } else { + None + }, }; let sysconfigdata_path = match find_sysconfigdata(&cross) { @@ -2546,6 +2914,7 @@ mod tests { version: interpreter_config.version, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2621,6 +2990,16 @@ mod tests { .is_none()); } + #[test] + fn test_is_cross_compiling_from_to() { + assert!(cross_compiling_from_to( + &triple!("x86_64-pc-windows-msvc"), + &triple!("aarch64-pc-windows-msvc") + ) + .unwrap() + .is_some()); + } + #[test] fn test_run_python_script() { // as above, this should be okay in CI where Python is presumed installed @@ -2650,7 +3029,7 @@ mod tests { fn test_build_script_outputs_base() { let interpreter_config = InterpreterConfig { implementation: PythonImplementation::CPython, - version: PythonVersion { major: 3, minor: 8 }, + version: PythonVersion { major: 3, minor: 9 }, shared: true, abi3: false, lib_name: Some("python3".into()), @@ -2660,13 +3039,14 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( interpreter_config.build_script_outputs(), [ - "cargo:rustc-cfg=Py_3_6".to_owned(), "cargo:rustc-cfg=Py_3_7".to_owned(), "cargo:rustc-cfg=Py_3_8".to_owned(), + "cargo:rustc-cfg=Py_3_9".to_owned(), ] ); @@ -2677,9 +3057,9 @@ mod tests { assert_eq!( interpreter_config.build_script_outputs(), [ - "cargo:rustc-cfg=Py_3_6".to_owned(), "cargo:rustc-cfg=Py_3_7".to_owned(), "cargo:rustc-cfg=Py_3_8".to_owned(), + "cargo:rustc-cfg=Py_3_9".to_owned(), "cargo:rustc-cfg=PyPy".to_owned(), ] ); @@ -2689,7 +3069,7 @@ mod tests { fn test_build_script_outputs_abi3() { let interpreter_config = InterpreterConfig { implementation: PythonImplementation::CPython, - version: PythonVersion { major: 3, minor: 7 }, + version: PythonVersion { major: 3, minor: 9 }, shared: true, abi3: true, lib_name: Some("python3".into()), @@ -2699,13 +3079,15 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( interpreter_config.build_script_outputs(), [ - "cargo:rustc-cfg=Py_3_6".to_owned(), "cargo:rustc-cfg=Py_3_7".to_owned(), + "cargo:rustc-cfg=Py_3_8".to_owned(), + "cargo:rustc-cfg=Py_3_9".to_owned(), "cargo:rustc-cfg=Py_LIMITED_API".to_owned(), ] ); @@ -2717,13 +3099,48 @@ mod tests { assert_eq!( interpreter_config.build_script_outputs(), [ - "cargo:rustc-cfg=Py_3_6".to_owned(), "cargo:rustc-cfg=Py_3_7".to_owned(), + "cargo:rustc-cfg=Py_3_8".to_owned(), + "cargo:rustc-cfg=Py_3_9".to_owned(), "cargo:rustc-cfg=PyPy".to_owned(), - "cargo:warning=PyPy does not yet support abi3 so the build artifacts \ - will be version-specific. See https://foss.heptapod.net/pypy/pypy/-/issues/3397 \ - for more information." - .to_owned(), + "cargo:rustc-cfg=Py_LIMITED_API".to_owned(), + ] + ); + } + + #[test] + fn test_build_script_outputs_gil_disabled() { + let mut build_flags = BuildFlags::default(); + build_flags.0.insert(BuildFlag::Py_GIL_DISABLED); + let interpreter_config = InterpreterConfig { + implementation: PythonImplementation::CPython, + version: PythonVersion { + major: 3, + minor: 13, + }, + shared: true, + abi3: false, + lib_name: Some("python3".into()), + lib_dir: None, + executable: None, + pointer_width: None, + build_flags, + suppress_build_script_link_lines: false, + extra_build_script_lines: vec![], + python_framework_prefix: None, + }; + + assert_eq!( + interpreter_config.build_script_outputs(), + [ + "cargo:rustc-cfg=Py_3_7".to_owned(), + "cargo:rustc-cfg=Py_3_8".to_owned(), + "cargo:rustc-cfg=Py_3_9".to_owned(), + "cargo:rustc-cfg=Py_3_10".to_owned(), + "cargo:rustc-cfg=Py_3_11".to_owned(), + "cargo:rustc-cfg=Py_3_12".to_owned(), + "cargo:rustc-cfg=Py_3_13".to_owned(), + "cargo:rustc-cfg=Py_GIL_DISABLED".to_owned(), ] ); } @@ -2744,15 +3161,44 @@ mod tests { build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( interpreter_config.build_script_outputs(), [ - "cargo:rustc-cfg=Py_3_6".to_owned(), "cargo:rustc-cfg=Py_3_7".to_owned(), "cargo:rustc-cfg=py_sys_config=\"Py_DEBUG\"".to_owned(), ] ); } + + #[test] + fn test_find_sysconfigdata_in_invalid_lib_dir() { + let e = find_all_sysconfigdata(&CrossCompileConfig { + lib_dir: Some(PathBuf::from("/abc/123/not/a/real/path")), + version: None, + implementation: None, + target: triple!("x86_64-unknown-linux-gnu"), + abiflags: None, + }) + .unwrap_err(); + + // actual error message is platform-dependent, so just check the context we add + assert!(e.report().to_string().starts_with( + "failed to search the lib dir at 'PYO3_CROSS_LIB_DIR=/abc/123/not/a/real/path'\n\ + caused by:\n \ + - 0: failed to list the entries in '/abc/123/not/a/real/path'\n \ + - 1: \ + " + )); + } + + #[test] + fn test_from_pyo3_config_file_env_rebuild() { + READ_ENV_VARS.with(|vars| vars.borrow_mut().clear()); + let _ = InterpreterConfig::from_pyo3_config_file_env(); + // it's possible that other env vars were also read, hence just checking for contains + READ_ENV_VARS.with(|vars| assert!(vars.borrow().contains(&"PYO3_CONFIG_FILE".to_string()))); + } } diff --git a/pyo3-build-config/src/import_lib.rs b/pyo3-build-config/src/import_lib.rs index 0925a861b5b..ee934441f77 100644 --- a/pyo3-build-config/src/import_lib.rs +++ b/pyo3-build-config/src/import_lib.rs @@ -19,6 +19,7 @@ pub(super) fn generate_import_lib( target: &Triple, py_impl: PythonImplementation, py_version: Option, + abiflags: Option<&str>, ) -> Result> { if target.operating_system != OperatingSystem::Windows { return Ok(None); @@ -50,6 +51,7 @@ pub(super) fn generate_import_lib( ImportLibraryGenerator::new(&arch, &env) .version(py_version.map(|v| (v.major, v.minor))) .implementation(implementation) + .abiflags(abiflags) .generate(&out_lib_dir) .context("failed to generate python3.dll import library")?; diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index eab7376d0ea..68d597a8f2e 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -18,7 +18,6 @@ use std::{ use std::{env, process::Command, str::FromStr}; -#[cfg(feature = "resolve-config")] use once_cell::sync::OnceCell; pub use impl_::{ @@ -40,9 +39,12 @@ use target_lexicon::OperatingSystem; /// | `#[cfg(PyPy)]` | This marks code which is run when compiling for PyPy. | /// | `#[cfg(GraalPy)]` | This marks code which is run when compiling for GraalPy. | /// -/// For examples of how to use these attributes, [see PyO3's guide](https://pyo3.rs/latest/building-and-distribution/multiple_python_versions.html). +/// For examples of how to use these attributes, +#[doc = concat!("[see PyO3's guide](https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/building-and-distribution/multiple_python_versions.html)")] +/// . #[cfg(feature = "resolve-config")] pub fn use_pyo3_cfgs() { + print_expected_cfgs(); for cargo_command in get().build_script_outputs() { println!("{}", cargo_command) } @@ -63,7 +65,7 @@ pub fn add_extension_module_link_args() { } fn _add_extension_module_link_args(triple: &Triple, mut writer: impl std::io::Write) { - if triple.operating_system == OperatingSystem::Darwin { + if matches!(triple.operating_system, OperatingSystem::Darwin(_)) { writeln!(writer, "cargo:rustc-cdylib-link-arg=-undefined").unwrap(); writeln!(writer, "cargo:rustc-cdylib-link-arg=dynamic_lookup").unwrap(); } else if triple == &Triple::from_str("wasm32-unknown-emscripten").unwrap() { @@ -72,6 +74,44 @@ fn _add_extension_module_link_args(triple: &Triple, mut writer: impl std::io::Wr } } +/// Adds linker arguments suitable for linking against the Python framework on macOS. +/// +/// This should be called from a build script. +/// +/// The following link flags are added: +/// - macOS: `-Wl,-rpath,` +/// +/// All other platforms currently are no-ops. +#[cfg(feature = "resolve-config")] +pub fn add_python_framework_link_args() { + let interpreter_config = pyo3_build_script_impl::resolve_interpreter_config().unwrap(); + _add_python_framework_link_args( + &interpreter_config, + &impl_::target_triple_from_env(), + impl_::is_linking_libpython(), + std::io::stdout(), + ) +} + +#[cfg(feature = "resolve-config")] +fn _add_python_framework_link_args( + interpreter_config: &InterpreterConfig, + triple: &Triple, + link_libpython: bool, + mut writer: impl std::io::Write, +) { + if matches!(triple.operating_system, OperatingSystem::Darwin(_)) && link_libpython { + if let Some(framework_prefix) = interpreter_config.python_framework_prefix.as_ref() { + writeln!( + writer, + "cargo:rustc-link-arg=-Wl,-rpath,{}", + framework_prefix + ) + .unwrap(); + } + } +} + /// Loads the configuration determined from the build environment. /// /// Because this will never change in a given compilation run, this is cached in a `once_cell`. @@ -86,6 +126,8 @@ pub fn get() -> &'static InterpreterConfig { .map(|path| path.exists()) .unwrap_or(false); + // CONFIG_FILE is generated in build.rs, so it's content can vary + #[allow(unknown_lints, clippy::const_is_empty)] if let Some(interpreter_config) = InterpreterConfig::from_cargo_dep_env() { interpreter_config } else if !CONFIG_FILE.is_empty() { @@ -126,33 +168,62 @@ fn resolve_cross_compile_config_path() -> Option { }) } +/// Helper to print a feature cfg with a minimum rust version required. +fn print_feature_cfg(minor_version_required: u32, cfg: &str) { + let minor_version = rustc_minor_version().unwrap_or(0); + + if minor_version >= minor_version_required { + println!("cargo:rustc-cfg={}", cfg); + } + + // rustc 1.80.0 stabilized `rustc-check-cfg` feature, don't emit before + if minor_version >= 80 { + println!("cargo:rustc-check-cfg=cfg({})", cfg); + } +} + /// Use certain features if we detect the compiler being used supports them. /// /// Features may be removed or added as MSRV gets bumped or new features become available, /// so this function is unstable. #[doc(hidden)] pub fn print_feature_cfgs() { - fn rustc_minor_version() -> Option { - let rustc = env::var_os("RUSTC")?; - let output = Command::new(rustc).arg("--version").output().ok()?; - let version = core::str::from_utf8(&output.stdout).ok()?; - let mut pieces = version.split('.'); - if pieces.next() != Some("rustc 1") { - return None; - } - pieces.next()?.parse().ok() - } - - let rustc_minor_version = rustc_minor_version().unwrap_or(0); + print_feature_cfg(70, "rustc_has_once_lock"); + print_feature_cfg(70, "cargo_toml_lints"); + print_feature_cfg(71, "rustc_has_extern_c_unwind"); + print_feature_cfg(74, "invalid_from_utf8_lint"); + print_feature_cfg(79, "c_str_lit"); + // Actually this is available on 1.78, but we should avoid + // https://github.com/rust-lang/rust/issues/124651 just in case + print_feature_cfg(79, "diagnostic_namespace"); + print_feature_cfg(83, "io_error_more"); + print_feature_cfg(85, "fn_ptr_eq"); +} - // Enable use of const initializer for thread_local! on Rust 1.59 and greater - if rustc_minor_version >= 59 { - println!("cargo:rustc-cfg=thread_local_const_init"); +/// Registers `pyo3`s config names as reachable cfg expressions +/// +/// - +/// - +#[doc(hidden)] +pub fn print_expected_cfgs() { + if rustc_minor_version().map_or(false, |version| version < 80) { + // rustc 1.80.0 stabilized `rustc-check-cfg` feature, don't emit before + return; } - // invalid_from_utf8 lint was added in Rust 1.74 - if rustc_minor_version >= 74 { - println!("cargo:rustc-cfg=invalid_from_utf8_lint"); + println!("cargo:rustc-check-cfg=cfg(Py_LIMITED_API)"); + println!("cargo:rustc-check-cfg=cfg(Py_GIL_DISABLED)"); + println!("cargo:rustc-check-cfg=cfg(PyPy)"); + println!("cargo:rustc-check-cfg=cfg(GraalPy)"); + println!("cargo:rustc-check-cfg=cfg(py_sys_config, values(\"Py_DEBUG\", \"Py_REF_DEBUG\", \"Py_TRACE_REFS\", \"COUNT_ALLOCS\"))"); + println!("cargo:rustc-check-cfg=cfg(pyo3_disable_reference_pool)"); + println!("cargo:rustc-check-cfg=cfg(pyo3_leak_on_drop_without_reference_pool)"); + + // allow `Py_3_*` cfgs from the minimum supported version up to the + // maximum minor version (+1 for development for the next) + // FIXME: support cfg(Py_3_14) as well due to PyGILState_Ensure + for i in impl_::MINIMUM_SUPPORTED_VERSION.minor..=std::cmp::max(14, impl_::ABI3_MAX_MINOR + 1) { + println!("cargo:rustc-check-cfg=cfg(Py_3_{i})"); } } @@ -182,6 +253,8 @@ pub mod pyo3_build_script_impl { /// correct value for CARGO_CFG_TARGET_OS). #[cfg(feature = "resolve-config")] pub fn resolve_interpreter_config() -> Result { + // CONFIG_FILE is generated in build.rs, so it's content can vary + #[allow(unknown_lints, clippy::const_is_empty)] if !CONFIG_FILE.is_empty() { let mut interperter_config = InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE))?; interperter_config.generate_import_libs()?; @@ -212,6 +285,20 @@ pub mod pyo3_build_script_impl { } } +fn rustc_minor_version() -> Option { + static RUSTC_MINOR_VERSION: OnceCell> = OnceCell::new(); + *RUSTC_MINOR_VERSION.get_or_init(|| { + let rustc = env::var_os("RUSTC")?; + let output = Command::new(rustc).arg("--version").output().ok()?; + let version = core::str::from_utf8(&output.stdout).ok()?; + let mut pieces = version.split('.'); + if pieces.next() != Some("rustc 1") { + return None; + } + pieces.next()?.parse().ok() + }) +} + #[cfg(test)] mod tests { use super::*; @@ -248,4 +335,49 @@ mod tests { cargo:rustc-cdylib-link-arg=-sWASM_BIGINT\n" ); } + + #[cfg(feature = "resolve-config")] + #[test] + fn python_framework_link_args() { + let mut buf = Vec::new(); + + let interpreter_config = InterpreterConfig { + implementation: PythonImplementation::CPython, + version: PythonVersion { + major: 3, + minor: 13, + }, + shared: true, + abi3: false, + lib_name: None, + lib_dir: None, + executable: None, + pointer_width: None, + build_flags: BuildFlags::default(), + suppress_build_script_link_lines: false, + extra_build_script_lines: vec![], + python_framework_prefix: Some( + "/Applications/Xcode.app/Contents/Developer/Library/Frameworks".to_string(), + ), + }; + // Does nothing on non-mac + _add_python_framework_link_args( + &interpreter_config, + &Triple::from_str("x86_64-pc-windows-msvc").unwrap(), + true, + &mut buf, + ); + assert_eq!(buf, Vec::new()); + + _add_python_framework_link_args( + &interpreter_config, + &Triple::from_str("x86_64-apple-darwin").unwrap(), + true, + &mut buf, + ); + assert_eq!( + std::str::from_utf8(&buf).unwrap(), + "cargo:rustc-link-arg=-Wl,-rpath,/Applications/Xcode.app/Contents/Developer/Library/Frameworks\n" + ); + } } diff --git a/pyo3-ffi-check/Cargo.toml b/pyo3-ffi-check/Cargo.toml index 776add0c910..874da3d2bef 100644 --- a/pyo3-ffi-check/Cargo.toml +++ b/pyo3-ffi-check/Cargo.toml @@ -13,7 +13,7 @@ path = "../pyo3-ffi" features = ["extension-module"] # A lazy way of skipping linking in most cases (as we don't use any runtime symbols) [build-dependencies] -bindgen = "0.66.1" +bindgen = "0.69.4" pyo3-build-config = { path = "../pyo3-build-config" } [workspace] diff --git a/pyo3-ffi-check/build.rs b/pyo3-ffi-check/build.rs index ca4a17b6a61..67808888e1e 100644 --- a/pyo3-ffi-check/build.rs +++ b/pyo3-ffi-check/build.rs @@ -8,13 +8,27 @@ fn main() { "import sysconfig; print(sysconfig.get_config_var('INCLUDEPY'), end='');", ) .expect("failed to get lib dir"); + let gil_disabled_on_windows = config + .run_python_script( + "import sysconfig; import platform; print(sysconfig.get_config_var('Py_GIL_DISABLED') == 1 and platform.system() == 'Windows');", + ) + .expect("failed to get Py_GIL_DISABLED").trim_end() == "True"; + + let clang_args = if gil_disabled_on_windows { + vec![ + format!("-I{python_include_dir}"), + "-DPy_GIL_DISABLED".to_string(), + ] + } else { + vec![format!("-I{python_include_dir}")] + }; println!("cargo:rerun-if-changed=wrapper.h"); let bindings = bindgen::Builder::default() .header("wrapper.h") - .clang_arg(format!("-I{python_include_dir}")) - .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .clang_args(clang_args) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) // blocklist some values which apparently have conflicting definitions on unix .blocklist_item("FP_NORMAL") .blocklist_item("FP_SUBNORMAL") diff --git a/pyo3-ffi-check/macro/Cargo.toml b/pyo3-ffi-check/macro/Cargo.toml index 46395fc8a8c..dfefcd2c1e1 100644 --- a/pyo3-ffi-check/macro/Cargo.toml +++ b/pyo3-ffi-check/macro/Cargo.toml @@ -10,6 +10,6 @@ proc-macro = true [dependencies] glob = "0.3" quote = "1" -proc-macro2 = "1" +proc-macro2 = "1.0.60" scraper = "0.17" pyo3-build-config = { path = "../../pyo3-build-config" } diff --git a/pyo3-ffi-check/macro/src/lib.rs b/pyo3-ffi-check/macro/src/lib.rs index e3d442c3703..8438393b9eb 100644 --- a/pyo3-ffi-check/macro/src/lib.rs +++ b/pyo3-ffi-check/macro/src/lib.rs @@ -9,6 +9,11 @@ const PY_3_12: PythonVersion = PythonVersion { minor: 12, }; +const PY_3_13: PythonVersion = PythonVersion { + major: 3, + minor: 13, +}; + /// Macro which expands to multiple macro calls, one per pyo3-ffi struct. #[proc_macro] pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -49,6 +54,14 @@ pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStrea .unwrap() .strip_suffix(".html") .unwrap(); + + if struct_name == "PyConfig" && pyo3_build_config::get().version == PY_3_13 { + // https://github.com/python/cpython/issues/130940 + // PyConfig has an ABI break on Python 3.13.1 -> 3.13.2, waiting for advice + // how to proceed in PyO3. + continue; + } + let struct_ident = Ident::new(struct_name, Span::call_site()); output.extend(quote!(#macro_name!(#struct_ident);)); } diff --git a/pyo3-ffi-check/src/main.rs b/pyo3-ffi-check/src/main.rs index 99713524702..0407a2ffa39 100644 --- a/pyo3-ffi-check/src/main.rs +++ b/pyo3-ffi-check/src/main.rs @@ -48,7 +48,8 @@ fn main() { macro_rules! check_field { ($struct_name:ident, $field:ident, $bindgen_field:ident) => {{ - #[allow(clippy::used_underscore_binding)] + // some struct fields are deprecated but still present in the ABI + #[allow(clippy::used_underscore_binding, deprecated)] let pyo3_ffi_offset = memoffset::offset_of!(pyo3_ffi::$struct_name, $field); #[allow(clippy::used_underscore_binding)] let bindgen_offset = memoffset::offset_of!(bindings::$struct_name, $bindgen_field); diff --git a/pyo3-ffi/ACKNOWLEDGEMENTS b/pyo3-ffi/ACKNOWLEDGEMENTS index 4502d7774e0..8b20727dece 100644 --- a/pyo3-ffi/ACKNOWLEDGEMENTS +++ b/pyo3-ffi/ACKNOWLEDGEMENTS @@ -3,4 +3,4 @@ for binary compatibility, with additional metadata to support PyPy. For original implementations please see: - https://github.com/python/cpython - - https://foss.heptapod.net/pypy/pypy + - https://github.com/pypy/pypy diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 64753976fab..de7ce8b317a 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.21.1" +version = "0.24.1" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -10,6 +10,7 @@ categories = ["api-bindings", "development-tools::ffi"] license = "MIT OR Apache-2.0" edition = "2021" links = "python" +rust-version = "1.63" [dependencies] libc = "0.2.62" @@ -32,13 +33,25 @@ abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38"] abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39"] abi3-py310 = ["abi3-py311", "pyo3-build-config/abi3-py310"] abi3-py311 = ["abi3-py312", "pyo3-build-config/abi3-py311"] -abi3-py312 = ["abi3", "pyo3-build-config/abi3-py312"] +abi3-py312 = ["abi3-py313", "pyo3-build-config/abi3-py312"] +abi3-py313 = ["abi3", "pyo3-build-config/abi3-py313"] # Automatically generates `python3.dll` import libraries for Windows targets. generate-import-lib = ["pyo3-build-config/python3-dll-a"] +[dev-dependencies] +paste = "1" + [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.1", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.24.1", features = ["resolve-config"] } [lints] workspace = true + +[package.metadata.cpython] +min-version = "3.7" +max-version = "3.13" # inclusive + +[package.metadata.pypy] +min-version = "3.9" +max-version = "3.11" # inclusive diff --git a/pyo3-ffi/README.md b/pyo3-ffi/README.md index 4e85e83c88f..192b33a92be 100644 --- a/pyo3-ffi/README.md +++ b/pyo3-ffi/README.md @@ -12,9 +12,12 @@ Manual][capi] for up-to-date documentation. # Minimum supported Rust and Python versions -PyO3 supports the following software versions: - - Python 3.7 and up (CPython and PyPy) - - Rust 1.56 and up +Requires Rust 1.63 or greater. + +`pyo3-ffi` supports the following Python distributions: + - CPython 3.7 or greater + - PyPy 7.3 (Python 3.9+) + - GraalPy 24.0 or greater (Python 3.10+) # Example: Building Python Native modules @@ -38,101 +41,151 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies.pyo3-ffi] -version = "*" +version = "0.24.1" features = ["extension-module"] + +[build-dependencies] +# This is only necessary if you need to configure your build based on +# the Python version or the compile-time configuration for the interpreter. +pyo3_build_config = "0.24.1" +``` + +If you need to use conditional compilation based on Python version or how +Python was compiled, you need to add `pyo3-build-config` as a +`build-dependency` in your `Cargo.toml` as in the example above and either +create a new `build.rs` file or modify an existing one so that +`pyo3_build_config::use_pyo3_cfgs()` gets called at build time: + +**`build.rs`** + +```rust,ignore +fn main() { + pyo3_build_config::use_pyo3_cfgs() +} ``` **`src/lib.rs`** ```rust -use std::os::raw::c_char; +use std::os::raw::{c_char, c_long}; use std::ptr; use pyo3_ffi::*; static mut MODULE_DEF: PyModuleDef = PyModuleDef { m_base: PyModuleDef_HEAD_INIT, - m_name: "string_sum\0".as_ptr().cast::(), - m_doc: "A Python module written in Rust.\0" - .as_ptr() - .cast::(), + m_name: c_str!("string_sum").as_ptr(), + m_doc: c_str!("A Python module written in Rust.").as_ptr(), m_size: 0, - m_methods: unsafe { METHODS.as_mut_ptr().cast() }, + m_methods: unsafe { METHODS as *const [PyMethodDef] as *mut PyMethodDef }, m_slots: std::ptr::null_mut(), m_traverse: None, m_clear: None, m_free: None, }; -static mut METHODS: [PyMethodDef; 2] = [ +static mut METHODS: &[PyMethodDef] = &[ PyMethodDef { - ml_name: "sum_as_string\0".as_ptr().cast::(), + ml_name: c_str!("sum_as_string").as_ptr(), ml_meth: PyMethodDefPointer { - _PyCFunctionFast: sum_as_string, + PyCFunctionFast: sum_as_string, }, ml_flags: METH_FASTCALL, - ml_doc: "returns the sum of two integers as a string\0" - .as_ptr() - .cast::(), + ml_doc: c_str!("returns the sum of two integers as a string").as_ptr(), }, // A zeroed PyMethodDef to mark the end of the array. - PyMethodDef::zeroed() + PyMethodDef::zeroed(), ]; // The module initialization function, which must be named `PyInit_`. #[allow(non_snake_case)] #[no_mangle] pub unsafe extern "C" fn PyInit_string_sum() -> *mut PyObject { - PyModule_Create(ptr::addr_of_mut!(MODULE_DEF)) + let module = PyModule_Create(ptr::addr_of_mut!(MODULE_DEF)); + if module.is_null() { + return module; + } + #[cfg(Py_GIL_DISABLED)] + { + if PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED) < 0 { + Py_DECREF(module); + return std::ptr::null_mut(); + } + } + module } -pub unsafe extern "C" fn sum_as_string( - _self: *mut PyObject, - args: *mut *mut PyObject, - nargs: Py_ssize_t, -) -> *mut PyObject { - if nargs != 2 { - PyErr_SetString( - PyExc_TypeError, - "sum_as_string() expected 2 positional arguments\0" - .as_ptr() - .cast::(), +/// A helper to parse function arguments +/// If we used PyO3's proc macros they'd handle all of this boilerplate for us :) +unsafe fn parse_arg_as_i32(obj: *mut PyObject, n_arg: usize) -> Option { + if PyLong_Check(obj) == 0 { + let msg = format!( + "sum_as_string expected an int for positional argument {}\0", + n_arg ); - return std::ptr::null_mut(); + PyErr_SetString(PyExc_TypeError, msg.as_ptr().cast::()); + return None; } - let arg1 = *args; - if PyLong_Check(arg1) == 0 { - PyErr_SetString( - PyExc_TypeError, - "sum_as_string() expected an int for positional argument 1\0" - .as_ptr() - .cast::(), - ); - return std::ptr::null_mut(); + // Let's keep the behaviour consistent on platforms where `c_long` is bigger than 32 bits. + // In particular, it is an i32 on Windows but i64 on most Linux systems + let mut overflow = 0; + let i_long: c_long = PyLong_AsLongAndOverflow(obj, &mut overflow); + + #[allow(irrefutable_let_patterns)] // some platforms have c_long equal to i32 + if overflow != 0 { + raise_overflowerror(obj); + None + } else if let Ok(i) = i_long.try_into() { + Some(i) + } else { + raise_overflowerror(obj); + None } +} - let arg1 = PyLong_AsLong(arg1); - if !PyErr_Occurred().is_null() { - return ptr::null_mut(); +unsafe fn raise_overflowerror(obj: *mut PyObject) { + let obj_repr = PyObject_Str(obj); + if !obj_repr.is_null() { + let mut size = 0; + let p = PyUnicode_AsUTF8AndSize(obj_repr, &mut size); + if !p.is_null() { + let s = std::str::from_utf8_unchecked(std::slice::from_raw_parts( + p.cast::(), + size as usize, + )); + let msg = format!("cannot fit {} in 32 bits\0", s); + + PyErr_SetString(PyExc_OverflowError, msg.as_ptr().cast::()); + } + Py_DECREF(obj_repr); } +} - let arg2 = *args.add(1); - if PyLong_Check(arg2) == 0 { +pub unsafe extern "C" fn sum_as_string( + _self: *mut PyObject, + args: *mut *mut PyObject, + nargs: Py_ssize_t, +) -> *mut PyObject { + if nargs != 2 { PyErr_SetString( PyExc_TypeError, - "sum_as_string() expected an int for positional argument 2\0" - .as_ptr() - .cast::(), + c_str!("sum_as_string expected 2 positional arguments").as_ptr(), ); return std::ptr::null_mut(); } - let arg2 = PyLong_AsLong(arg2); - if !PyErr_Occurred().is_null() { - return ptr::null_mut(); - } + let (first, second) = (*args, *args.add(1)); + + let first = match parse_arg_as_i32(first, 1) { + Some(x) => x, + None => return std::ptr::null_mut(), + }; + let second = match parse_arg_as_i32(second, 2) { + Some(x) => x, + None => return std::ptr::null_mut(), + }; - match arg1.checked_add(arg2) { + match first.checked_add(second) { Some(sum) => { let string = sum.to_string(); PyUnicode_FromStringAndSize(string.as_ptr().cast::(), string.len() as isize) @@ -140,7 +193,7 @@ pub unsafe extern "C" fn sum_as_string( None => { PyErr_SetString( PyExc_OverflowError, - "arguments too large to add\0".as_ptr().cast::(), + c_str!("arguments too large to add").as_ptr(), ); std::ptr::null_mut() } diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs index a5ab352b6a7..096614c7961 100644 --- a/pyo3-ffi/build.rs +++ b/pyo3-ffi/build.rs @@ -4,7 +4,7 @@ use pyo3_build_config::{ cargo_env_var, env_var, errors::Result, is_linking_libpython, resolve_interpreter_config, InterpreterConfig, PythonVersion, }, - PythonImplementation, + warn, PythonImplementation, }; /// Minimum Python version PyO3 supports. @@ -17,15 +17,15 @@ const SUPPORTED_VERSIONS_CPYTHON: SupportedVersions = SupportedVersions { min: PythonVersion { major: 3, minor: 7 }, max: PythonVersion { major: 3, - minor: 12, + minor: 13, }, }; const SUPPORTED_VERSIONS_PYPY: SupportedVersions = SupportedVersions { - min: PythonVersion { major: 3, minor: 7 }, + min: PythonVersion { major: 3, minor: 9 }, max: PythonVersion { major: 3, - minor: 10, + minor: 11, }, }; @@ -56,15 +56,22 @@ fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { interpreter_config.version, versions.min, ); - ensure!( - interpreter_config.version <= versions.max || env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY").map_or(false, |os_str| os_str == "1"), - "the configured Python interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\ - = help: please check if an updated version of PyO3 is available. Current version: {}\n\ - = help: set PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 to suppress this check and build anyway using the stable ABI", - interpreter_config.version, - versions.max, - std::env::var("CARGO_PKG_VERSION").unwrap(), - ); + if interpreter_config.version > versions.max { + ensure!(!interpreter_config.is_free_threaded(), + "The configured Python interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\ + = help: please check if an updated version of PyO3 is available. Current version: {}\n\ + = help: The free-threaded build of CPython does not support the limited API so this check cannot be suppressed.", + interpreter_config.version, versions.max, std::env::var("CARGO_PKG_VERSION").unwrap() + ); + ensure!(env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY").map_or(false, |os_str| os_str == "1"), + "the configured Python interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\ + = help: please check if an updated version of PyO3 is available. Current version: {}\n\ + = help: set PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 to suppress this check and build anyway using the stable ABI", + interpreter_config.version, + versions.max, + std::env::var("CARGO_PKG_VERSION").unwrap(), + ); + } } PythonImplementation::PyPy => { let versions = SUPPORTED_VERSIONS_PYPY; @@ -104,6 +111,25 @@ fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { } } + if interpreter_config.abi3 { + match interpreter_config.implementation { + PythonImplementation::CPython => { + if interpreter_config.is_free_threaded() { + warn!( + "The free-threaded build of CPython does not yet support abi3 so the build artifacts will be version-specific." + ) + } + } + PythonImplementation::PyPy => warn!( + "PyPy does not yet support abi3 so the build artifacts will be version-specific. \ + See https://github.com/pypy/pypy/issues/3397 for more information." + ), + PythonImplementation::GraalPy => warn!( + "GraalPy does not support abi3 so the build artifacts will be version-specific." + ), + } + } + Ok(()) } @@ -189,7 +215,7 @@ fn configure_pyo3() -> Result<()> { println!("{}", line); } - // Emit cfgs like `thread_local_const_init` + // Emit cfgs like `invalid_from_utf8_lint` print_feature_cfgs(); Ok(()) @@ -205,6 +231,7 @@ fn print_config_and_exit(config: &InterpreterConfig) { } fn main() { + pyo3_build_config::print_expected_cfgs(); if let Err(e) = configure_pyo3() { eprintln!("error: {}", e.report()); std::process::exit(1) diff --git a/pyo3-ffi/examples/README.md b/pyo3-ffi/examples/README.md new file mode 100644 index 00000000000..f02ae4ba6b4 --- /dev/null +++ b/pyo3-ffi/examples/README.md @@ -0,0 +1,21 @@ +# `pyo3-ffi` Examples + +These example crates are a collection of toy extension modules built with +`pyo3-ffi`. They are all tested using `nox` in PyO3's CI. + +Below is a brief description of each of these: + +| Example | Description | +| `word-count` | Illustrates how to use pyo3-ffi to write a static rust extension | +| `sequential` | Illustrates how to use pyo3-ffi to write subinterpreter-safe modules using multi-phase module initialization | + +## Creating new projects from these examples + +To copy an example, use [`cargo-generate`](https://crates.io/crates/cargo-generate). Follow the commands below, replacing `` with the example to start from: + +```bash +$ cargo install cargo-generate +$ cargo generate --git https://github.com/PyO3/pyo3 examples/ +``` + +(`cargo generate` will take a little while to clone the PyO3 repo first; be patient when waiting for the command to run.) diff --git a/examples/sequential/.template/Cargo.toml b/pyo3-ffi/examples/sequential/.template/Cargo.toml similarity index 100% rename from examples/sequential/.template/Cargo.toml rename to pyo3-ffi/examples/sequential/.template/Cargo.toml diff --git a/examples/sequential/.template/pre-script.rhai b/pyo3-ffi/examples/sequential/.template/pre-script.rhai similarity index 100% rename from examples/sequential/.template/pre-script.rhai rename to pyo3-ffi/examples/sequential/.template/pre-script.rhai diff --git a/examples/sequential/.template/pyproject.toml b/pyo3-ffi/examples/sequential/.template/pyproject.toml similarity index 100% rename from examples/sequential/.template/pyproject.toml rename to pyo3-ffi/examples/sequential/.template/pyproject.toml diff --git a/pyo3-ffi/examples/sequential/Cargo.toml b/pyo3-ffi/examples/sequential/Cargo.toml new file mode 100644 index 00000000000..e62693303d9 --- /dev/null +++ b/pyo3-ffi/examples/sequential/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "sequential" +version = "0.1.0" +edition = "2021" +rust-version = "1.63" + +[lib] +name = "sequential" +crate-type = ["cdylib", "lib"] + +[dependencies] +pyo3-ffi = { path = "../../", features = ["extension-module"] } + +[build-dependencies] +pyo3-build-config = { path = "../../../pyo3-build-config" } + +[workspace] diff --git a/examples/sequential/MANIFEST.in b/pyo3-ffi/examples/sequential/MANIFEST.in similarity index 100% rename from examples/sequential/MANIFEST.in rename to pyo3-ffi/examples/sequential/MANIFEST.in diff --git a/examples/sequential/README.md b/pyo3-ffi/examples/sequential/README.md similarity index 100% rename from examples/sequential/README.md rename to pyo3-ffi/examples/sequential/README.md diff --git a/pyo3-ffi/examples/sequential/build.rs b/pyo3-ffi/examples/sequential/build.rs new file mode 100644 index 00000000000..0475124bb4e --- /dev/null +++ b/pyo3-ffi/examples/sequential/build.rs @@ -0,0 +1,3 @@ +fn main() { + pyo3_build_config::use_pyo3_cfgs(); +} diff --git a/examples/sequential/cargo-generate.toml b/pyo3-ffi/examples/sequential/cargo-generate.toml similarity index 100% rename from examples/sequential/cargo-generate.toml rename to pyo3-ffi/examples/sequential/cargo-generate.toml diff --git a/examples/sequential/noxfile.py b/pyo3-ffi/examples/sequential/noxfile.py similarity index 100% rename from examples/sequential/noxfile.py rename to pyo3-ffi/examples/sequential/noxfile.py diff --git a/examples/sequential/pyproject.toml b/pyo3-ffi/examples/sequential/pyproject.toml similarity index 100% rename from examples/sequential/pyproject.toml rename to pyo3-ffi/examples/sequential/pyproject.toml diff --git a/examples/sequential/src/id.rs b/pyo3-ffi/examples/sequential/src/id.rs similarity index 81% rename from examples/sequential/src/id.rs rename to pyo3-ffi/examples/sequential/src/id.rs index d80e84b4eab..fa72bb091c7 100644 --- a/examples/sequential/src/id.rs +++ b/pyo3-ffi/examples/sequential/src/id.rs @@ -1,5 +1,6 @@ use core::sync::atomic::{AtomicU64, Ordering}; use core::{mem, ptr}; +use std::ffi::CString; use std::os::raw::{c_char, c_int, c_uint, c_ulonglong, c_void}; use pyo3_ffi::*; @@ -27,10 +28,10 @@ unsafe extern "C" fn id_new( kwds: *mut PyObject, ) -> *mut PyObject { if PyTuple_Size(args) != 0 || !kwds.is_null() { - PyErr_SetString( - PyExc_TypeError, - "Id() takes no arguments\0".as_ptr().cast::(), - ); + // We use pyo3-ffi's `c_str!` macro to create null-terminated literals because + // Rust's string literals are not null-terminated + // On Rust 1.77 or newer you can use `c"text"` instead. + PyErr_SetString(PyExc_TypeError, c_str!("Id() takes no arguments").as_ptr()); return ptr::null_mut(); } @@ -81,8 +82,12 @@ unsafe extern "C" fn id_richcompare( pyo3_ffi::Py_GT => slf > other, pyo3_ffi::Py_GE => slf >= other, unrecognized => { - let msg = format!("unrecognized richcompare opcode {}\0", unrecognized); - PyErr_SetString(PyExc_SystemError, msg.as_ptr().cast::()); + let msg = CString::new(&*format!( + "unrecognized richcompare opcode {}", + unrecognized + )) + .unwrap(); + PyErr_SetString(PyExc_SystemError, msg.as_ptr()); return ptr::null_mut(); } }; @@ -101,7 +106,7 @@ static mut SLOTS: &[PyType_Slot] = &[ }, PyType_Slot { slot: Py_tp_doc, - pfunc: "An id that is increased every time an instance is created\0".as_ptr() + pfunc: c_str!("An id that is increased every time an instance is created").as_ptr() as *mut c_void, }, PyType_Slot { @@ -123,7 +128,7 @@ static mut SLOTS: &[PyType_Slot] = &[ ]; pub static mut ID_SPEC: PyType_Spec = PyType_Spec { - name: "sequential.Id\0".as_ptr().cast::(), + name: c_str!("sequential.Id").as_ptr(), basicsize: mem::size_of::() as c_int, itemsize: 0, flags: (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE) as c_uint, diff --git a/examples/sequential/src/lib.rs b/pyo3-ffi/examples/sequential/src/lib.rs similarity index 100% rename from examples/sequential/src/lib.rs rename to pyo3-ffi/examples/sequential/src/lib.rs diff --git a/examples/sequential/src/module.rs b/pyo3-ffi/examples/sequential/src/module.rs similarity index 83% rename from examples/sequential/src/module.rs rename to pyo3-ffi/examples/sequential/src/module.rs index 5552baf3368..baa7c66f206 100644 --- a/examples/sequential/src/module.rs +++ b/pyo3-ffi/examples/sequential/src/module.rs @@ -1,13 +1,11 @@ use core::{mem, ptr}; use pyo3_ffi::*; -use std::os::raw::{c_char, c_int, c_void}; +use std::os::raw::{c_int, c_void}; pub static mut MODULE_DEF: PyModuleDef = PyModuleDef { m_base: PyModuleDef_HEAD_INIT, - m_name: "sequential\0".as_ptr().cast::(), - m_doc: "A library for generating sequential ids, written in Rust.\0" - .as_ptr() - .cast::(), + m_name: c_str!("sequential").as_ptr(), + m_doc: c_str!("A library for generating sequential ids, written in Rust.").as_ptr(), m_size: mem::size_of::() as Py_ssize_t, m_methods: std::ptr::null_mut(), m_slots: unsafe { SEQUENTIAL_SLOTS as *const [PyModuleDef_Slot] as *mut PyModuleDef_Slot }, @@ -25,6 +23,11 @@ static mut SEQUENTIAL_SLOTS: &[PyModuleDef_Slot] = &[ slot: Py_mod_multiple_interpreters, value: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED, }, + #[cfg(Py_GIL_DISABLED)] + PyModuleDef_Slot { + slot: Py_mod_gil, + value: Py_MOD_GIL_NOT_USED, + }, PyModuleDef_Slot { slot: 0, value: ptr::null_mut(), @@ -42,13 +45,13 @@ unsafe extern "C" fn sequential_exec(module: *mut PyObject) -> c_int { if id_type.is_null() { PyErr_SetString( PyExc_SystemError, - "cannot locate type object\0".as_ptr().cast::(), + c_str!("cannot locate type object").as_ptr(), ); return -1; } (*state).id_type = id_type.cast::(); - PyModule_AddObjectRef(module, "Id\0".as_ptr().cast::(), id_type) + PyModule_AddObjectRef(module, c_str!("Id").as_ptr(), id_type) } unsafe extern "C" fn sequential_traverse( diff --git a/examples/sequential/tests/test.rs b/pyo3-ffi/examples/sequential/tests/test.rs similarity index 88% rename from examples/sequential/tests/test.rs rename to pyo3-ffi/examples/sequential/tests/test.rs index 6076edd4974..f2a08433cea 100644 --- a/examples/sequential/tests/test.rs +++ b/pyo3-ffi/examples/sequential/tests/test.rs @@ -5,11 +5,13 @@ use std::thread; use pyo3_ffi::*; use sequential::PyInit_sequential; -static COMMAND: &'static str = " +static COMMAND: &'static str = c_str!( + " from sequential import Id s = sum(int(Id()) for _ in range(12)) -\0"; +" +); // Newtype to be able to pass it to another thread. struct State(*mut PyThreadState); @@ -19,10 +21,7 @@ unsafe impl Send for State {} #[test] fn lets_go_fast() -> Result<(), String> { unsafe { - let ret = PyImport_AppendInittab( - "sequential\0".as_ptr().cast::(), - Some(PyInit_sequential), - ); + let ret = PyImport_AppendInittab(c_str!("sequential").as_ptr(), Some(PyInit_sequential)); if ret == -1 { return Err("could not add module to inittab".into()); } @@ -122,11 +121,8 @@ unsafe fn fetch() -> String { fn run_code() -> Result { unsafe { - let code_obj = Py_CompileString( - COMMAND.as_ptr().cast::(), - "program\0".as_ptr().cast::(), - Py_file_input, - ); + let code_obj = + Py_CompileString(COMMAND.as_ptr(), c_str!("program").as_ptr(), Py_file_input); if code_obj.is_null() { return Err(fetch()); } @@ -138,7 +134,7 @@ fn run_code() -> Result { } else { Py_DECREF(res_ptr); } - let sum = PyDict_GetItemString(globals, "s\0".as_ptr().cast::()); /* borrowed reference */ + let sum = PyDict_GetItemString(globals, c_str!("s").as_ptr()); /* borrowed reference */ if sum.is_null() { Py_DECREF(globals); return Err("globals did not have `s`".into()); diff --git a/examples/sequential/tests/test_.py b/pyo3-ffi/examples/sequential/tests/test_.py similarity index 100% rename from examples/sequential/tests/test_.py rename to pyo3-ffi/examples/sequential/tests/test_.py diff --git a/examples/string-sum/.template/Cargo.toml b/pyo3-ffi/examples/string-sum/.template/Cargo.toml similarity index 100% rename from examples/string-sum/.template/Cargo.toml rename to pyo3-ffi/examples/string-sum/.template/Cargo.toml diff --git a/examples/string-sum/.template/pre-script.rhai b/pyo3-ffi/examples/string-sum/.template/pre-script.rhai similarity index 100% rename from examples/string-sum/.template/pre-script.rhai rename to pyo3-ffi/examples/string-sum/.template/pre-script.rhai diff --git a/examples/string-sum/.template/pyproject.toml b/pyo3-ffi/examples/string-sum/.template/pyproject.toml similarity index 100% rename from examples/string-sum/.template/pyproject.toml rename to pyo3-ffi/examples/string-sum/.template/pyproject.toml diff --git a/pyo3-ffi/examples/string-sum/Cargo.toml b/pyo3-ffi/examples/string-sum/Cargo.toml new file mode 100644 index 00000000000..b0f784d26f2 --- /dev/null +++ b/pyo3-ffi/examples/string-sum/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "string_sum" +version = "0.1.0" +edition = "2021" +rust-version = "1.63" + +[lib] +name = "string_sum" +crate-type = ["cdylib"] + +[dependencies] +pyo3-ffi = { path = "../../", features = ["extension-module"] } + +[build-dependencies] +pyo3-build-config = { path = "../../../pyo3-build-config" } + +[workspace] diff --git a/examples/string-sum/MANIFEST.in b/pyo3-ffi/examples/string-sum/MANIFEST.in similarity index 100% rename from examples/string-sum/MANIFEST.in rename to pyo3-ffi/examples/string-sum/MANIFEST.in diff --git a/examples/string-sum/README.md b/pyo3-ffi/examples/string-sum/README.md similarity index 100% rename from examples/string-sum/README.md rename to pyo3-ffi/examples/string-sum/README.md diff --git a/pyo3-ffi/examples/string-sum/build.rs b/pyo3-ffi/examples/string-sum/build.rs new file mode 100644 index 00000000000..0475124bb4e --- /dev/null +++ b/pyo3-ffi/examples/string-sum/build.rs @@ -0,0 +1,3 @@ +fn main() { + pyo3_build_config::use_pyo3_cfgs(); +} diff --git a/examples/string-sum/cargo-generate.toml b/pyo3-ffi/examples/string-sum/cargo-generate.toml similarity index 100% rename from examples/string-sum/cargo-generate.toml rename to pyo3-ffi/examples/string-sum/cargo-generate.toml diff --git a/examples/string-sum/noxfile.py b/pyo3-ffi/examples/string-sum/noxfile.py similarity index 100% rename from examples/string-sum/noxfile.py rename to pyo3-ffi/examples/string-sum/noxfile.py diff --git a/examples/string-sum/pyproject.toml b/pyo3-ffi/examples/string-sum/pyproject.toml similarity index 100% rename from examples/string-sum/pyproject.toml rename to pyo3-ffi/examples/string-sum/pyproject.toml diff --git a/examples/string-sum/src/lib.rs b/pyo3-ffi/examples/string-sum/src/lib.rs similarity index 79% rename from examples/string-sum/src/lib.rs rename to pyo3-ffi/examples/string-sum/src/lib.rs index 91072418038..7af80bc08d7 100644 --- a/examples/string-sum/src/lib.rs +++ b/pyo3-ffi/examples/string-sum/src/lib.rs @@ -5,10 +5,8 @@ use pyo3_ffi::*; static mut MODULE_DEF: PyModuleDef = PyModuleDef { m_base: PyModuleDef_HEAD_INIT, - m_name: "string_sum\0".as_ptr().cast::(), - m_doc: "A Python module written in Rust.\0" - .as_ptr() - .cast::(), + m_name: c_str!("string_sum").as_ptr(), + m_doc: c_str!("A Python module written in Rust.").as_ptr(), m_size: 0, m_methods: unsafe { METHODS as *const [PyMethodDef] as *mut PyMethodDef }, m_slots: std::ptr::null_mut(), @@ -19,14 +17,12 @@ static mut MODULE_DEF: PyModuleDef = PyModuleDef { static mut METHODS: &[PyMethodDef] = &[ PyMethodDef { - ml_name: "sum_as_string\0".as_ptr().cast::(), + ml_name: c_str!("sum_as_string").as_ptr(), ml_meth: PyMethodDefPointer { - _PyCFunctionFast: sum_as_string, + PyCFunctionFast: sum_as_string, }, ml_flags: METH_FASTCALL, - ml_doc: "returns the sum of two integers as a string\0" - .as_ptr() - .cast::(), + ml_doc: c_str!("returns the sum of two integers as a string").as_ptr(), }, // A zeroed PyMethodDef to mark the end of the array. PyMethodDef::zeroed(), @@ -36,7 +32,18 @@ static mut METHODS: &[PyMethodDef] = &[ #[allow(non_snake_case)] #[no_mangle] pub unsafe extern "C" fn PyInit_string_sum() -> *mut PyObject { - PyModule_Create(ptr::addr_of_mut!(MODULE_DEF)) + let module = PyModule_Create(ptr::addr_of_mut!(MODULE_DEF)); + if module.is_null() { + return module; + } + #[cfg(Py_GIL_DISABLED)] + { + if PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED) < 0 { + Py_DECREF(module); + return std::ptr::null_mut(); + } + } + module } /// A helper to parse function arguments @@ -56,6 +63,7 @@ unsafe fn parse_arg_as_i32(obj: *mut PyObject, n_arg: usize) -> Option { let mut overflow = 0; let i_long: c_long = PyLong_AsLongAndOverflow(obj, &mut overflow); + #[allow(irrefutable_let_patterns)] // some platforms have c_long equal to i32 if overflow != 0 { raise_overflowerror(obj); None @@ -93,9 +101,7 @@ pub unsafe extern "C" fn sum_as_string( if nargs != 2 { PyErr_SetString( PyExc_TypeError, - "sum_as_string expected 2 positional arguments\0" - .as_ptr() - .cast::(), + c_str!("sum_as_string expected 2 positional arguments").as_ptr(), ); return std::ptr::null_mut(); } @@ -119,7 +125,7 @@ pub unsafe extern "C" fn sum_as_string( None => { PyErr_SetString( PyExc_OverflowError, - "arguments too large to add\0".as_ptr().cast::(), + c_str!("arguments too large to add").as_ptr(), ); std::ptr::null_mut() } diff --git a/examples/string-sum/tests/test_.py b/pyo3-ffi/examples/string-sum/tests/test_.py similarity index 100% rename from examples/string-sum/tests/test_.py rename to pyo3-ffi/examples/string-sum/tests/test_.py diff --git a/pyo3-ffi/src/abstract_.rs b/pyo3-ffi/src/abstract_.rs index b5bf9cc3d35..a79ec43f271 100644 --- a/pyo3-ffi/src/abstract_.rs +++ b/pyo3-ffi/src/abstract_.rs @@ -1,29 +1,30 @@ use crate::object::*; use crate::pyport::Py_ssize_t; +#[cfg(any(Py_3_12, all(Py_3_8, not(Py_LIMITED_API))))] +use libc::size_t; use std::os::raw::{c_char, c_int}; -use std::ptr; - -extern "C" { - #[cfg(PyPy)] - #[link_name = "PyPyObject_DelAttrString"] - pub fn PyObject_DelAttrString(o: *mut PyObject, attr_name: *const c_char) -> c_int; -} #[inline] -#[cfg(not(PyPy))] +#[cfg(all( + not(Py_3_13), // CPython exposed as a function in 3.13, in object.h + not(all(PyPy, not(Py_3_11))) // PyPy exposed as a function until PyPy 3.10, macro in 3.11+ +))] pub unsafe fn PyObject_DelAttrString(o: *mut PyObject, attr_name: *const c_char) -> c_int { - PyObject_SetAttrString(o, attr_name, ptr::null_mut()) + PyObject_SetAttrString(o, attr_name, std::ptr::null_mut()) } #[inline] +#[cfg(all( + not(Py_3_13), // CPython exposed as a function in 3.13, in object.h + not(all(PyPy, not(Py_3_11))) // PyPy exposed as a function until PyPy 3.10, macro in 3.11+ +))] pub unsafe fn PyObject_DelAttr(o: *mut PyObject, attr_name: *mut PyObject) -> c_int { - PyObject_SetAttr(o, attr_name, ptr::null_mut()) + PyObject_SetAttr(o, attr_name, std::ptr::null_mut()) } extern "C" { #[cfg(all( not(PyPy), - not(GraalPy), any(Py_3_10, all(not(Py_LIMITED_API), Py_3_9)) // Added to python in 3.9 but to limited API in 3.10 ))] #[cfg_attr(PyPy, link_name = "PyPyObject_CallNoArgs")] @@ -77,6 +78,28 @@ extern "C" { method: *mut PyObject, ... ) -> *mut PyObject; +} +#[cfg(any(Py_3_12, all(Py_3_8, not(Py_LIMITED_API))))] +pub const PY_VECTORCALL_ARGUMENTS_OFFSET: size_t = + 1 << (8 * std::mem::size_of::() as size_t - 1); + +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyObject_Vectorcall")] + #[cfg(any(Py_3_12, all(Py_3_11, not(Py_LIMITED_API))))] + pub fn PyObject_Vectorcall( + callable: *mut PyObject, + args: *const *mut PyObject, + nargsf: size_t, + kwnames: *mut PyObject, + ) -> *mut PyObject; + + #[cfg(any(Py_3_12, all(Py_3_9, not(any(Py_LIMITED_API, PyPy, GraalPy)))))] + pub fn PyObject_VectorcallMethod( + name: *mut PyObject, + args: *const *mut PyObject, + nargsf: size_t, + kwnames: *mut PyObject, + ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyObject_Type")] pub fn PyObject_Type(o: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyObject_Size")] @@ -114,10 +137,7 @@ extern "C" { #[cfg(not(any(Py_3_8, PyPy)))] #[inline] pub unsafe fn PyIter_Check(o: *mut PyObject) -> c_int { - crate::PyObject_HasAttrString( - crate::Py_TYPE(o).cast(), - "__next__\0".as_ptr() as *const c_char, - ) + crate::PyObject_HasAttrString(crate::Py_TYPE(o).cast(), c_str!("__next__").as_ptr()) } extern "C" { @@ -129,7 +149,11 @@ extern "C" { pub fn PyIter_Next(arg1: *mut PyObject) -> *mut PyObject; #[cfg(all(not(PyPy), Py_3_10))] #[cfg_attr(PyPy, link_name = "PyPyIter_Send")] - pub fn PyIter_Send(iter: *mut PyObject, arg: *mut PyObject, presult: *mut *mut PyObject); + pub fn PyIter_Send( + iter: *mut PyObject, + arg: *mut PyObject, + presult: *mut *mut PyObject, + ) -> PySendResult; #[cfg_attr(PyPy, link_name = "PyPyNumber_Check")] pub fn PyNumber_Check(o: *mut PyObject) -> c_int; diff --git a/pyo3-ffi/src/boolobject.rs b/pyo3-ffi/src/boolobject.rs index 10b5969fa4f..eec9da707a1 100644 --- a/pyo3-ffi/src/boolobject.rs +++ b/pyo3-ffi/src/boolobject.rs @@ -4,12 +4,6 @@ use crate::object::*; use std::os::raw::{c_int, c_long}; use std::ptr::addr_of_mut; -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - #[cfg_attr(PyPy, link_name = "PyPyBool_Type")] - pub static mut PyBool_Type: PyTypeObject; -} - #[inline] pub unsafe fn PyBool_Check(op: *mut PyObject) -> c_int { (Py_TYPE(op) == addr_of_mut!(PyBool_Type)) as c_int diff --git a/pyo3-ffi/src/bytearrayobject.rs b/pyo3-ffi/src/bytearrayobject.rs index c09eac5b22c..24a97bcc31b 100644 --- a/pyo3-ffi/src/bytearrayobject.rs +++ b/pyo3-ffi/src/bytearrayobject.rs @@ -5,7 +5,6 @@ use std::ptr::addr_of_mut; #[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API)))] #[repr(C)] -#[derive(Copy, Clone)] pub struct PyByteArrayObject { pub ob_base: PyVarObject, pub ob_alloc: Py_ssize_t, diff --git a/pyo3-ffi/src/ceval.rs b/pyo3-ffi/src/ceval.rs index 7aae25f8c3e..d1839a108cb 100644 --- a/pyo3-ffi/src/ceval.rs +++ b/pyo3-ffi/src/ceval.rs @@ -24,6 +24,7 @@ extern "C" { closure: *mut PyObject, ) -> *mut PyObject; + #[cfg(not(Py_3_13))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] #[cfg_attr(PyPy, link_name = "PyPyEval_CallObjectWithKeywords")] pub fn PyEval_CallObjectWithKeywords( @@ -33,6 +34,7 @@ extern "C" { ) -> *mut PyObject; } +#[cfg(not(Py_3_13))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] #[inline] pub unsafe fn PyEval_CallObject(func: *mut PyObject, arg: *mut PyObject) -> *mut PyObject { @@ -41,9 +43,11 @@ pub unsafe fn PyEval_CallObject(func: *mut PyObject, arg: *mut PyObject) -> *mut } extern "C" { + #[cfg(not(Py_3_13))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] #[cfg_attr(PyPy, link_name = "PyPyEval_CallFunction")] pub fn PyEval_CallFunction(obj: *mut PyObject, format: *const c_char, ...) -> *mut PyObject; + #[cfg(not(Py_3_13))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] #[cfg_attr(PyPy, link_name = "PyPyEval_CallMethod")] pub fn PyEval_CallMethod( @@ -95,9 +99,22 @@ extern "C" { } extern "C" { + #[cfg(not(Py_3_13))] #[cfg_attr(PyPy, link_name = "PyPyEval_ThreadsInitialized")] + #[cfg_attr( + Py_3_9, + deprecated( + note = "Deprecated in Python 3.9, this function always returns true in Python 3.7 or newer." + ) + )] pub fn PyEval_ThreadsInitialized() -> c_int; #[cfg_attr(PyPy, link_name = "PyPyEval_InitThreads")] + #[cfg_attr( + Py_3_9, + deprecated( + note = "Deprecated in Python 3.9, this function does nothing in Python 3.7 or newer." + ) + )] pub fn PyEval_InitThreads(); pub fn PyEval_AcquireLock(); pub fn PyEval_ReleaseLock(); diff --git a/pyo3-ffi/src/compat/mod.rs b/pyo3-ffi/src/compat/mod.rs new file mode 100644 index 00000000000..11f2912848e --- /dev/null +++ b/pyo3-ffi/src/compat/mod.rs @@ -0,0 +1,59 @@ +//! C API Compatibility Shims +//! +//! Some CPython C API functions added in recent versions of Python are +//! inherently safer to use than older C API constructs. This module +//! exposes functions available on all Python versions that wrap the +//! old C API on old Python versions and wrap the function directly +//! on newer Python versions. + +// Unless otherwise noted, the compatibility shims are adapted from +// the pythoncapi-compat project: https://github.com/python/pythoncapi-compat + +/// Internal helper macro which defines compatibility shims for C API functions, deferring to a +/// re-export when that's available. +macro_rules! compat_function { + ( + originally_defined_for($cfg:meta); + + $(#[$attrs:meta])* + pub unsafe fn $name:ident($($arg_names:ident: $arg_types:ty),* $(,)?) -> $ret:ty $body:block + ) => { + // Define as a standalone function under docsrs cfg so that this shows as a unique function in the docs, + // not a re-export (the re-export has the wrong visibility) + #[cfg(any(docsrs, not($cfg)))] + #[cfg_attr(docsrs, doc(cfg(all())))] + $(#[$attrs])* + pub unsafe fn $name( + $($arg_names: $arg_types,)* + ) -> $ret $body + + #[cfg(all($cfg, not(docsrs)))] + pub use $crate::$name; + + #[cfg(test)] + paste::paste! { + // Test that the compat function does not overlap with the original function. If the + // cfgs line up, then the the two glob imports will resolve to the same item via the + // re-export. If the cfgs mismatch, then the use of $name will be ambiguous in cases + // where the function is defined twice, and the test will fail to compile. + #[allow(unused_imports)] + mod [] { + use $crate::*; + use $crate::compat::*; + + #[test] + fn test_export() { + let _ = $name; + } + } + } + }; +} + +mod py_3_10; +mod py_3_13; +mod py_3_9; + +pub use self::py_3_10::*; +pub use self::py_3_13::*; +pub use self::py_3_9::*; diff --git a/pyo3-ffi/src/compat/py_3_10.rs b/pyo3-ffi/src/compat/py_3_10.rs new file mode 100644 index 00000000000..c6e8c2cb5ca --- /dev/null +++ b/pyo3-ffi/src/compat/py_3_10.rs @@ -0,0 +1,19 @@ +compat_function!( + originally_defined_for(Py_3_10); + + #[inline] + pub unsafe fn Py_NewRef(obj: *mut crate::PyObject) -> *mut crate::PyObject { + crate::Py_INCREF(obj); + obj + } +); + +compat_function!( + originally_defined_for(Py_3_10); + + #[inline] + pub unsafe fn Py_XNewRef(obj: *mut crate::PyObject) -> *mut crate::PyObject { + crate::Py_XINCREF(obj); + obj + } +); diff --git a/pyo3-ffi/src/compat/py_3_13.rs b/pyo3-ffi/src/compat/py_3_13.rs new file mode 100644 index 00000000000..59289cb76ae --- /dev/null +++ b/pyo3-ffi/src/compat/py_3_13.rs @@ -0,0 +1,106 @@ +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyDict_GetItemRef( + dp: *mut crate::PyObject, + key: *mut crate::PyObject, + result: *mut *mut crate::PyObject, + ) -> std::os::raw::c_int { + use crate::{compat::Py_NewRef, PyDict_GetItemWithError, PyErr_Occurred}; + + let item = PyDict_GetItemWithError(dp, key); + if !item.is_null() { + *result = Py_NewRef(item); + return 1; // found + } + *result = std::ptr::null_mut(); + if PyErr_Occurred().is_null() { + return 0; // not found + } + -1 + } +); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyList_GetItemRef( + arg1: *mut crate::PyObject, + arg2: crate::Py_ssize_t, + ) -> *mut crate::PyObject { + use crate::{PyList_GetItem, Py_XINCREF}; + + let item = PyList_GetItem(arg1, arg2); + Py_XINCREF(item); + item + } +); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyImport_AddModuleRef( + name: *const std::os::raw::c_char, + ) -> *mut crate::PyObject { + use crate::{compat::Py_XNewRef, PyImport_AddModule}; + + Py_XNewRef(PyImport_AddModule(name)) + } +); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyWeakref_GetRef( + reference: *mut crate::PyObject, + pobj: *mut *mut crate::PyObject, + ) -> std::os::raw::c_int { + use crate::{ + compat::Py_NewRef, PyErr_SetString, PyExc_TypeError, PyWeakref_Check, + PyWeakref_GetObject, Py_None, + }; + + if !reference.is_null() && PyWeakref_Check(reference) == 0 { + *pobj = std::ptr::null_mut(); + PyErr_SetString(PyExc_TypeError, c_str!("expected a weakref").as_ptr()); + return -1; + } + let obj = PyWeakref_GetObject(reference); + if obj.is_null() { + // SystemError if reference is NULL + *pobj = std::ptr::null_mut(); + return -1; + } + if obj == Py_None() { + *pobj = std::ptr::null_mut(); + return 0; + } + *pobj = Py_NewRef(obj); + 1 + } +); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyList_Extend( + list: *mut crate::PyObject, + iterable: *mut crate::PyObject, + ) -> std::os::raw::c_int { + crate::PyList_SetSlice(list, crate::PY_SSIZE_T_MAX, crate::PY_SSIZE_T_MAX, iterable) + } +); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyList_Clear(list: *mut crate::PyObject) -> std::os::raw::c_int { + crate::PyList_SetSlice(list, 0, crate::PY_SSIZE_T_MAX, std::ptr::null_mut()) + } +); diff --git a/pyo3-ffi/src/compat/py_3_9.rs b/pyo3-ffi/src/compat/py_3_9.rs new file mode 100644 index 00000000000..285f2b2ae7e --- /dev/null +++ b/pyo3-ffi/src/compat/py_3_9.rs @@ -0,0 +1,21 @@ +compat_function!( + originally_defined_for(all( + not(PyPy), + not(GraalPy), + any(Py_3_10, all(not(Py_LIMITED_API), Py_3_9)) // Added to python in 3.9 but to limited API in 3.10 + )); + + #[inline] + pub unsafe fn PyObject_CallNoArgs(obj: *mut crate::PyObject) -> *mut crate::PyObject { + crate::PyObject_CallObject(obj, std::ptr::null_mut()) + } +); + +compat_function!( + originally_defined_for(all(Py_3_9, not(any(Py_LIMITED_API, PyPy, GraalPy)))); + + #[inline] + pub unsafe fn PyObject_CallMethodNoArgs(obj: *mut crate::PyObject, name: *mut crate::PyObject) -> *mut crate::PyObject { + crate::PyObject_CallMethodObjArgs(obj, name, std::ptr::null_mut::()) + } +); diff --git a/pyo3-ffi/src/complexobject.rs b/pyo3-ffi/src/complexobject.rs index a03d9b00932..283bacf6e84 100644 --- a/pyo3-ffi/src/complexobject.rs +++ b/pyo3-ffi/src/complexobject.rs @@ -2,38 +2,6 @@ use crate::object::*; use std::os::raw::{c_double, c_int}; use std::ptr::addr_of_mut; -#[repr(C)] -#[derive(Copy, Clone)] -// non-limited -pub struct Py_complex { - pub real: c_double, - pub imag: c_double, -} - -#[cfg(not(Py_LIMITED_API))] -extern "C" { - pub fn _Py_c_sum(left: Py_complex, right: Py_complex) -> Py_complex; - pub fn _Py_c_diff(left: Py_complex, right: Py_complex) -> Py_complex; - pub fn _Py_c_neg(complex: Py_complex) -> Py_complex; - pub fn _Py_c_prod(left: Py_complex, right: Py_complex) -> Py_complex; - pub fn _Py_c_quot(dividend: Py_complex, divisor: Py_complex) -> Py_complex; - pub fn _Py_c_pow(num: Py_complex, exp: Py_complex) -> Py_complex; - pub fn _Py_c_abs(arg: Py_complex) -> c_double; - #[cfg_attr(PyPy, link_name = "PyPyComplex_FromCComplex")] - pub fn PyComplex_FromCComplex(v: Py_complex) -> *mut PyObject; - #[cfg_attr(PyPy, link_name = "PyPyComplex_AsCComplex")] - pub fn PyComplex_AsCComplex(op: *mut PyObject) -> Py_complex; -} - -#[repr(C)] -#[derive(Copy, Clone)] -// non-limited -pub struct PyComplexObject { - pub ob_base: PyObject, - #[cfg(not(GraalPy))] - pub cval: Py_complex, -} - #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { #[cfg_attr(PyPy, link_name = "PyPyComplex_Type")] @@ -47,17 +15,16 @@ pub unsafe fn PyComplex_Check(op: *mut PyObject) -> c_int { #[inline] pub unsafe fn PyComplex_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == addr_of_mut!(PyComplex_Type)) as c_int + Py_IS_TYPE(op, addr_of_mut!(PyComplex_Type)) } extern "C" { // skipped non-limited PyComplex_FromCComplex #[cfg_attr(PyPy, link_name = "PyPyComplex_FromDoubles")] pub fn PyComplex_FromDoubles(real: c_double, imag: c_double) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyComplex_RealAsDouble")] pub fn PyComplex_RealAsDouble(op: *mut PyObject) -> c_double; #[cfg_attr(PyPy, link_name = "PyPyComplex_ImagAsDouble")] pub fn PyComplex_ImagAsDouble(op: *mut PyObject) -> c_double; - // skipped non-limited PyComplex_AsCComplex - // skipped non-limited _PyComplex_FormatAdvancedWriter } diff --git a/pyo3-ffi/src/cpython/abstract_.rs b/pyo3-ffi/src/cpython/abstract_.rs index cf95f6711d4..6ada1a754ef 100644 --- a/pyo3-ffi/src/cpython/abstract_.rs +++ b/pyo3-ffi/src/cpython/abstract_.rs @@ -1,11 +1,11 @@ use crate::{PyObject, Py_ssize_t}; -use std::os::raw::{c_char, c_int}; +#[cfg(any(all(Py_3_8, not(any(PyPy, GraalPy))), not(Py_3_11)))] +use std::os::raw::c_char; +use std::os::raw::c_int; #[cfg(not(Py_3_11))] use crate::Py_buffer; -#[cfg(Py_3_8)] -use crate::pyport::PY_SSIZE_T_MAX; #[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] use crate::{ vectorcallfunc, PyCallable_Check, PyThreadState, PyThreadState_GET, PyTuple_Check, @@ -41,15 +41,15 @@ extern "C" { ) -> *mut PyObject; } -#[cfg(Py_3_8)] -const PY_VECTORCALL_ARGUMENTS_OFFSET: Py_ssize_t = - 1 << (8 * std::mem::size_of::() as Py_ssize_t - 1); +#[cfg(Py_3_8)] // NB exported as public in abstract.rs from 3.12 +const PY_VECTORCALL_ARGUMENTS_OFFSET: size_t = + 1 << (8 * std::mem::size_of::() as size_t - 1); #[cfg(Py_3_8)] #[inline(always)] pub unsafe fn PyVectorcall_NARGS(n: size_t) -> Py_ssize_t { - assert!(n <= (PY_SSIZE_T_MAX as size_t)); - (n as Py_ssize_t) & !PY_VECTORCALL_ARGUMENTS_OFFSET + let n = n & !PY_VECTORCALL_ARGUMENTS_OFFSET; + n.try_into().expect("cannot fail due to mask") } #[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] @@ -63,7 +63,7 @@ pub unsafe fn PyVectorcall_Function(callable: *mut PyObject) -> Option 0); let offset = (*tp).tp_vectorcall_offset; assert!(offset > 0); - let ptr = (callable as *const c_char).offset(offset) as *const Option; + let ptr = callable.cast::().offset(offset).cast(); *ptr } @@ -91,7 +91,7 @@ pub unsafe fn _PyObject_VectorcallTstate( } } -#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] +#[cfg(all(Py_3_8, not(any(PyPy, GraalPy, Py_3_11))))] // exported as a function from 3.11, see abstract.rs #[inline(always)] pub unsafe fn PyObject_Vectorcall( callable: *mut PyObject, @@ -103,16 +103,6 @@ pub unsafe fn PyObject_Vectorcall( } extern "C" { - #[cfg(all(PyPy, Py_3_8))] - #[cfg_attr(not(Py_3_9), link_name = "_PyPyObject_Vectorcall")] - #[cfg_attr(Py_3_9, link_name = "PyPyObject_Vectorcall")] - pub fn PyObject_Vectorcall( - callable: *mut PyObject, - args: *const *mut PyObject, - nargsf: size_t, - kwnames: *mut PyObject, - ) -> *mut PyObject; - #[cfg(Py_3_8)] #[cfg_attr( all(not(any(PyPy, GraalPy)), not(Py_3_9)), @@ -184,17 +174,7 @@ pub unsafe fn PyObject_CallOneArg(func: *mut PyObject, arg: *mut PyObject) -> *m let args = args_array.as_ptr().offset(1); // For PY_VECTORCALL_ARGUMENTS_OFFSET let tstate = PyThreadState_GET(); let nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; - _PyObject_VectorcallTstate(tstate, func, args, nargsf as size_t, std::ptr::null_mut()) -} - -extern "C" { - #[cfg(all(Py_3_9, not(any(PyPy, GraalPy))))] - pub fn PyObject_VectorcallMethod( - name: *mut PyObject, - args: *const *mut PyObject, - nargsf: size_t, - kwnames: *mut PyObject, - ) -> *mut PyObject; + _PyObject_VectorcallTstate(tstate, func, args, nargsf, std::ptr::null_mut()) } #[cfg(all(Py_3_9, not(any(PyPy, GraalPy))))] @@ -203,10 +183,10 @@ pub unsafe fn PyObject_CallMethodNoArgs( self_: *mut PyObject, name: *mut PyObject, ) -> *mut PyObject { - PyObject_VectorcallMethod( + crate::PyObject_VectorcallMethod( name, &self_, - 1 | PY_VECTORCALL_ARGUMENTS_OFFSET as size_t, + 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, std::ptr::null_mut(), ) } @@ -220,10 +200,10 @@ pub unsafe fn PyObject_CallMethodOneArg( ) -> *mut PyObject { let args = [self_, arg]; assert!(!arg.is_null()); - PyObject_VectorcallMethod( + crate::PyObject_VectorcallMethod( name, args.as_ptr(), - 2 | PY_VECTORCALL_ARGUMENTS_OFFSET as size_t, + 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, std::ptr::null_mut(), ) } diff --git a/pyo3-ffi/src/cpython/bytesobject.rs b/pyo3-ffi/src/cpython/bytesobject.rs index fb0b38cf1d8..306702de25e 100644 --- a/pyo3-ffi/src/cpython/bytesobject.rs +++ b/pyo3-ffi/src/cpython/bytesobject.rs @@ -6,9 +6,12 @@ use std::os::raw::c_int; #[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API)))] #[repr(C)] -#[derive(Copy, Clone)] pub struct PyBytesObject { pub ob_base: PyVarObject, + #[cfg_attr( + Py_3_11, + deprecated(note = "Deprecated in Python 3.11 and will be removed in a future version.") + )] pub ob_shash: crate::Py_hash_t, pub ob_sval: [c_char; 1], } diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index 74586eac595..230096ca378 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -20,7 +20,11 @@ pub const _PY_MONITORING_EVENTS: usize = 17; #[repr(C)] #[derive(Clone, Copy)] pub struct _Py_LocalMonitors { - pub tools: [u8; _PY_MONITORING_UNGROUPED_EVENTS], + pub tools: [u8; if cfg!(Py_3_13) { + _PY_MONITORING_LOCAL_EVENTS + } else { + _PY_MONITORING_UNGROUPED_EVENTS + }], } #[cfg(Py_3_12)] @@ -78,7 +82,6 @@ opaque_struct!(PyCodeObject); #[cfg(all(not(any(PyPy, GraalPy)), Py_3_7, not(Py_3_8)))] #[repr(C)] -#[derive(Copy, Clone)] pub struct PyCodeObject { pub ob_base: PyObject, pub co_argcount: c_int, @@ -102,9 +105,11 @@ pub struct PyCodeObject { pub co_extra: *mut c_void, } +#[cfg(Py_3_13)] +opaque_struct!(_PyExecutorArray); + #[cfg(all(not(any(PyPy, GraalPy)), Py_3_8, not(Py_3_11)))] #[repr(C)] -#[derive(Copy, Clone)] pub struct PyCodeObject { pub ob_base: PyObject, pub co_argcount: c_int, @@ -138,7 +143,6 @@ pub struct PyCodeObject { #[cfg(all(not(any(PyPy, GraalPy)), Py_3_11))] #[repr(C)] -#[derive(Copy, Clone)] pub struct PyCodeObject { pub ob_base: PyVarObject, pub co_consts: *mut PyObject, @@ -176,6 +180,8 @@ pub struct PyCodeObject { pub _co_code: *mut PyObject, #[cfg(not(Py_3_12))] pub _co_linearray: *mut c_char, + #[cfg(Py_3_13)] + pub co_executors: *mut _PyExecutorArray, #[cfg(Py_3_12)] pub _co_cached: *mut _PyCoCached, #[cfg(Py_3_12)] @@ -189,7 +195,6 @@ pub struct PyCodeObject { #[cfg(PyPy)] #[repr(C)] -#[derive(Copy, Clone)] pub struct PyCodeObject { pub ob_base: PyObject, pub co_name: *mut PyObject, diff --git a/pyo3-ffi/src/cpython/compile.rs b/pyo3-ffi/src/cpython/compile.rs index 8bce9dacb3b..79f06c92003 100644 --- a/pyo3-ffi/src/cpython/compile.rs +++ b/pyo3-ffi/src/cpython/compile.rs @@ -30,7 +30,7 @@ pub struct PyCompilerFlags { // skipped non-limited _PyCompilerFlags_INIT -#[cfg(all(Py_3_12, not(any(PyPy, GraalPy))))] +#[cfg(all(Py_3_12, not(any(Py_3_13, PyPy, GraalPy))))] #[repr(C)] #[derive(Copy, Clone)] pub struct _PyCompilerSrcLocation { @@ -42,7 +42,7 @@ pub struct _PyCompilerSrcLocation { // skipped SRC_LOCATION_FROM_AST -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(not(any(PyPy, GraalPy, Py_3_13)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyFutureFeatures { diff --git a/pyo3-ffi/src/cpython/complexobject.rs b/pyo3-ffi/src/cpython/complexobject.rs new file mode 100644 index 00000000000..4cc86db5667 --- /dev/null +++ b/pyo3-ffi/src/cpython/complexobject.rs @@ -0,0 +1,30 @@ +use crate::PyObject; +use std::os::raw::c_double; + +#[repr(C)] +#[derive(Copy, Clone)] +pub struct Py_complex { + pub real: c_double, + pub imag: c_double, +} + +// skipped private function _Py_c_sum +// skipped private function _Py_c_diff +// skipped private function _Py_c_neg +// skipped private function _Py_c_prod +// skipped private function _Py_c_quot +// skipped private function _Py_c_pow +// skipped private function _Py_c_abs + +#[repr(C)] +pub struct PyComplexObject { + pub ob_base: PyObject, + pub cval: Py_complex, +} + +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyComplex_FromCComplex")] + pub fn PyComplex_FromCComplex(v: Py_complex) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyComplex_AsCComplex")] + pub fn PyComplex_AsCComplex(op: *mut PyObject) -> Py_complex; +} diff --git a/pyo3-ffi/src/cpython/critical_section.rs b/pyo3-ffi/src/cpython/critical_section.rs new file mode 100644 index 00000000000..97b2f5e0559 --- /dev/null +++ b/pyo3-ffi/src/cpython/critical_section.rs @@ -0,0 +1,30 @@ +#[cfg(Py_GIL_DISABLED)] +use crate::PyMutex; +use crate::PyObject; + +#[repr(C)] +#[cfg(Py_GIL_DISABLED)] +pub struct PyCriticalSection { + _cs_prev: usize, + _cs_mutex: *mut PyMutex, +} + +#[repr(C)] +#[cfg(Py_GIL_DISABLED)] +pub struct PyCriticalSection2 { + _cs_base: PyCriticalSection, + _cs_mutex2: *mut PyMutex, +} + +#[cfg(not(Py_GIL_DISABLED))] +opaque_struct!(PyCriticalSection); + +#[cfg(not(Py_GIL_DISABLED))] +opaque_struct!(PyCriticalSection2); + +extern "C" { + pub fn PyCriticalSection_Begin(c: *mut PyCriticalSection, op: *mut PyObject); + pub fn PyCriticalSection_End(c: *mut PyCriticalSection); + pub fn PyCriticalSection2_Begin(c: *mut PyCriticalSection2, a: *mut PyObject, b: *mut PyObject); + pub fn PyCriticalSection2_End(c: *mut PyCriticalSection2); +} diff --git a/pyo3-ffi/src/cpython/dictobject.rs b/pyo3-ffi/src/cpython/dictobject.rs index 74b970ebac2..79dcbfdb62e 100644 --- a/pyo3-ffi/src/cpython/dictobject.rs +++ b/pyo3-ffi/src/cpython/dictobject.rs @@ -13,6 +13,10 @@ opaque_struct!(PyDictValues); pub struct PyDictObject { pub ob_base: PyObject, pub ma_used: Py_ssize_t, + #[cfg_attr( + Py_3_12, + deprecated(note = "Deprecated in Python 3.12 and will be removed in the future.") + )] pub ma_version_tag: u64, pub ma_keys: *mut PyDictKeysObject, #[cfg(not(Py_3_11))] diff --git a/pyo3-ffi/src/cpython/floatobject.rs b/pyo3-ffi/src/cpython/floatobject.rs index 8c7ee88543d..e7caa441c5d 100644 --- a/pyo3-ffi/src/cpython/floatobject.rs +++ b/pyo3-ffi/src/cpython/floatobject.rs @@ -6,7 +6,6 @@ use std::os::raw::c_double; #[repr(C)] pub struct PyFloatObject { pub ob_base: PyObject, - #[cfg(not(GraalPy))] pub ob_fval: c_double, } diff --git a/pyo3-ffi/src/cpython/frameobject.rs b/pyo3-ffi/src/cpython/frameobject.rs index a85818ace0a..e9b9c183f37 100644 --- a/pyo3-ffi/src/cpython/frameobject.rs +++ b/pyo3-ffi/src/cpython/frameobject.rs @@ -21,7 +21,6 @@ pub struct PyTryBlock { } #[repr(C)] -#[derive(Copy, Clone)] #[cfg(not(any(PyPy, GraalPy, Py_3_11)))] pub struct PyFrameObject { pub ob_base: PyVarObject, @@ -90,7 +89,8 @@ extern "C" { pub fn PyFrame_FastToLocals(f: *mut PyFrameObject); // skipped _PyFrame_DebugMallocStats - // skipped PyFrame_GetBack + #[cfg(all(Py_3_9, not(PyPy)))] + pub fn PyFrame_GetBack(f: *mut PyFrameObject) -> *mut PyFrameObject; #[cfg(not(Py_3_9))] pub fn PyFrame_ClearFreeList() -> c_int; diff --git a/pyo3-ffi/src/cpython/genobject.rs b/pyo3-ffi/src/cpython/genobject.rs index 73ebdb491ff..c9d419e3782 100644 --- a/pyo3-ffi/src/cpython/genobject.rs +++ b/pyo3-ffi/src/cpython/genobject.rs @@ -2,14 +2,13 @@ use crate::object::*; use crate::PyFrameObject; #[cfg(not(any(PyPy, GraalPy)))] use crate::_PyErr_StackItem; -#[cfg(Py_3_11)] +#[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] use std::os::raw::c_char; use std::os::raw::c_int; use std::ptr::addr_of_mut; #[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] -#[derive(Copy, Clone)] pub struct PyGenObject { pub ob_base: PyObject, #[cfg(not(Py_3_11))] diff --git a/pyo3-ffi/src/cpython/import.rs b/pyo3-ffi/src/cpython/import.rs index aafd71a8355..697d68a419c 100644 --- a/pyo3-ffi/src/cpython/import.rs +++ b/pyo3-ffi/src/cpython/import.rs @@ -57,7 +57,7 @@ pub struct _frozen { pub size: c_int, #[cfg(Py_3_11)] pub is_package: c_int, - #[cfg(Py_3_11)] + #[cfg(all(Py_3_11, not(Py_3_13)))] pub get_code: Option *mut PyObject>, } diff --git a/pyo3-ffi/src/cpython/initconfig.rs b/pyo3-ffi/src/cpython/initconfig.rs index 17fe7559e1b..321d200e141 100644 --- a/pyo3-ffi/src/cpython/initconfig.rs +++ b/pyo3-ffi/src/cpython/initconfig.rs @@ -141,6 +141,10 @@ pub struct PyConfig { pub safe_path: c_int, #[cfg(Py_3_12)] pub int_max_str_digits: c_int, + #[cfg(Py_3_13)] + pub cpu_count: c_int, + #[cfg(Py_GIL_DISABLED)] + pub enable_gil: c_int, pub pathconfig_warnings: c_int, #[cfg(Py_3_10)] pub program_name: *mut wchar_t, @@ -165,6 +169,8 @@ pub struct PyConfig { pub run_command: *mut wchar_t, pub run_module: *mut wchar_t, pub run_filename: *mut wchar_t, + #[cfg(Py_3_13)] + pub sys_path_0: *mut wchar_t, pub _install_importlib: c_int, pub _init_main: c_int, #[cfg(all(Py_3_9, not(Py_3_12)))] @@ -173,6 +179,8 @@ pub struct PyConfig { pub _is_python_build: c_int, #[cfg(all(Py_3_9, not(Py_3_10)))] pub _orig_argv: PyWideStringList, + #[cfg(all(Py_3_13, py_sys_config = "Py_DEBUG"))] + pub run_presite: *mut wchar_t, } extern "C" { diff --git a/pyo3-ffi/src/cpython/listobject.rs b/pyo3-ffi/src/cpython/listobject.rs index ea15cfc1ff5..694e6bc4290 100644 --- a/pyo3-ffi/src/cpython/listobject.rs +++ b/pyo3-ffi/src/cpython/listobject.rs @@ -2,16 +2,15 @@ use crate::object::*; #[cfg(not(PyPy))] use crate::pyport::Py_ssize_t; -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(not(PyPy))] #[repr(C)] -#[derive(Copy, Clone)] pub struct PyListObject { pub ob_base: PyVarObject, pub ob_item: *mut *mut PyObject, pub allocated: Py_ssize_t, } -#[cfg(any(PyPy, GraalPy))] +#[cfg(PyPy)] pub struct PyListObject { pub ob_base: PyObject, } diff --git a/pyo3-ffi/src/cpython/lock.rs b/pyo3-ffi/src/cpython/lock.rs new file mode 100644 index 00000000000..6c80b00d3c1 --- /dev/null +++ b/pyo3-ffi/src/cpython/lock.rs @@ -0,0 +1,14 @@ +use std::marker::PhantomPinned; +use std::sync::atomic::AtomicU8; + +#[repr(transparent)] +#[derive(Debug)] +pub struct PyMutex { + pub(crate) _bits: AtomicU8, + pub(crate) _pin: PhantomPinned, +} + +extern "C" { + pub fn PyMutex_Lock(m: *mut PyMutex); + pub fn PyMutex_Unlock(m: *mut PyMutex); +} diff --git a/pyo3-ffi/src/cpython/longobject.rs b/pyo3-ffi/src/cpython/longobject.rs new file mode 100644 index 00000000000..45acaae577d --- /dev/null +++ b/pyo3-ffi/src/cpython/longobject.rs @@ -0,0 +1,74 @@ +use crate::longobject::*; +use crate::object::*; +#[cfg(Py_3_13)] +use crate::pyport::Py_ssize_t; +use libc::size_t; +#[cfg(Py_3_13)] +use std::os::raw::c_void; +use std::os::raw::{c_int, c_uchar}; + +#[cfg(Py_3_13)] +extern "C" { + pub fn PyLong_FromUnicodeObject(u: *mut PyObject, base: c_int) -> *mut PyObject; +} + +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_DEFAULTS: c_int = -1; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_BIG_ENDIAN: c_int = 0; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_LITTLE_ENDIAN: c_int = 1; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_NATIVE_ENDIAN: c_int = 3; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_UNSIGNED_BUFFER: c_int = 4; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_REJECT_NEGATIVE: c_int = 8; + +extern "C" { + // skipped _PyLong_Sign + + #[cfg(Py_3_13)] + pub fn PyLong_AsNativeBytes( + v: *mut PyObject, + buffer: *mut c_void, + n_bytes: Py_ssize_t, + flags: c_int, + ) -> Py_ssize_t; + + #[cfg(Py_3_13)] + pub fn PyLong_FromNativeBytes( + buffer: *const c_void, + n_bytes: size_t, + flags: c_int, + ) -> *mut PyObject; + + #[cfg(Py_3_13)] + pub fn PyLong_FromUnsignedNativeBytes( + buffer: *const c_void, + n_bytes: size_t, + flags: c_int, + ) -> *mut PyObject; + + // skipped PyUnstable_Long_IsCompact + // skipped PyUnstable_Long_CompactValue + + #[cfg_attr(PyPy, link_name = "_PyPyLong_FromByteArray")] + pub fn _PyLong_FromByteArray( + bytes: *const c_uchar, + n: size_t, + little_endian: c_int, + is_signed: c_int, + ) -> *mut PyObject; + + #[cfg_attr(PyPy, link_name = "_PyPyLong_AsByteArrayO")] + pub fn _PyLong_AsByteArray( + v: *mut PyLongObject, + bytes: *mut c_uchar, + n: size_t, + little_endian: c_int, + is_signed: c_int, + ) -> c_int; + + // skipped _PyLong_GCD +} diff --git a/pyo3-ffi/src/cpython/mod.rs b/pyo3-ffi/src/cpython/mod.rs index 1ab0e3c893f..f09d51d0e4e 100644 --- a/pyo3-ffi/src/cpython/mod.rs +++ b/pyo3-ffi/src/cpython/mod.rs @@ -5,6 +5,9 @@ pub(crate) mod bytesobject; pub(crate) mod ceval; pub(crate) mod code; pub(crate) mod compile; +pub(crate) mod complexobject; +#[cfg(Py_3_13)] +pub(crate) mod critical_section; pub(crate) mod descrobject; #[cfg(not(PyPy))] pub(crate) mod dictobject; @@ -18,6 +21,9 @@ pub(crate) mod import; pub(crate) mod initconfig; // skipped interpreteridobject.h pub(crate) mod listobject; +#[cfg(Py_3_13)] +pub(crate) mod lock; +pub(crate) mod longobject; #[cfg(all(Py_3_9, not(PyPy)))] pub(crate) mod methodobject; pub(crate) mod object; @@ -42,6 +48,9 @@ pub use self::bytesobject::*; pub use self::ceval::*; pub use self::code::*; pub use self::compile::*; +pub use self::complexobject::*; +#[cfg(Py_3_13)] +pub use self::critical_section::*; pub use self::descrobject::*; #[cfg(not(PyPy))] pub use self::dictobject::*; @@ -53,13 +62,16 @@ pub use self::import::*; #[cfg(all(Py_3_8, not(PyPy)))] pub use self::initconfig::*; pub use self::listobject::*; +#[cfg(Py_3_13)] +pub use self::lock::*; +pub use self::longobject::*; #[cfg(all(Py_3_9, not(PyPy)))] pub use self::methodobject::*; pub use self::object::*; pub use self::objimpl::*; pub use self::pydebug::*; pub use self::pyerrors::*; -#[cfg(Py_3_11)] +#[cfg(all(Py_3_11, not(PyPy)))] pub use self::pyframe::*; #[cfg(all(Py_3_8, not(PyPy)))] pub use self::pylifecycle::*; diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 0f1778f6a3d..4e6932da789 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -1,20 +1,23 @@ #[cfg(Py_3_8)] use crate::vectorcallfunc; -#[cfg(Py_3_11)] -use crate::PyModuleDef; use crate::{object, PyGetSetDef, PyMemberDef, PyMethodDef, PyObject, Py_ssize_t}; use std::mem; -use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void}; +use std::os::raw::{c_char, c_int, c_uint, c_void}; + +// skipped private _Py_NewReference +// skipped private _Py_NewReferenceNoTotal +// skipped private _Py_ResurrectReference -// skipped _Py_NewReference -// skipped _Py_ForgetReference -// skipped _Py_GetRefTotal +// skipped private _Py_GetGlobalRefTotal +// skipped private _Py_GetRefTotal +// skipped private _Py_GetLegacyRefTotal +// skipped private _PyInterpreterState_GetRefTotal -// skipped _Py_Identifier +// skipped private _Py_Identifier -// skipped _Py_static_string_init -// skipped _Py_static_string -// skipped _Py_IDENTIFIER +// skipped private _Py_static_string_init +// skipped private _Py_static_string +// skipped private _Py_IDENTIFIER #[cfg(not(Py_3_11))] // moved to src/buffer.rs from Python mod bufferinfo { @@ -205,20 +208,9 @@ pub type printfunc = unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut ::libc::FILE, arg3: c_int) -> c_int; #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] pub struct PyTypeObject { - #[cfg(all(PyPy, not(Py_3_9)))] - pub ob_refcnt: Py_ssize_t, - #[cfg(all(PyPy, not(Py_3_9)))] - pub ob_pypy_link: Py_ssize_t, - #[cfg(all(PyPy, not(Py_3_9)))] - pub ob_type: *mut PyTypeObject, - #[cfg(all(PyPy, not(Py_3_9)))] - pub ob_size: Py_ssize_t, - #[cfg(not(all(PyPy, not(Py_3_9))))] pub ob_base: object::PyVarObject, - #[cfg(GraalPy)] - pub ob_size: Py_ssize_t, pub tp_name: *const c_char, pub tp_basicsize: Py_ssize_t, pub tp_itemsize: Py_ssize_t, @@ -240,7 +232,10 @@ pub struct PyTypeObject { pub tp_getattro: Option, pub tp_setattro: Option, pub tp_as_buffer: *mut PyBufferProcs, - pub tp_flags: c_ulong, + #[cfg(not(Py_GIL_DISABLED))] + pub tp_flags: std::os::raw::c_ulong, + #[cfg(Py_GIL_DISABLED)] + pub tp_flags: crate::impl_::AtomicCULong, pub tp_doc: *const c_char, pub tp_traverse: Option, pub tp_clear: Option, @@ -292,14 +287,15 @@ pub struct PyTypeObject { #[cfg(Py_3_11)] #[repr(C)] #[derive(Clone)] -pub struct _specialization_cache { - pub getitem: *mut PyObject, +struct _specialization_cache { + getitem: *mut PyObject, #[cfg(Py_3_12)] - pub getitem_version: u32, + getitem_version: u32, + #[cfg(Py_3_13)] + init: *mut PyObject, } #[repr(C)] -#[derive(Clone)] pub struct PyHeapTypeObject { pub ht_type: PyTypeObject, pub as_async: PyAsyncMethods, @@ -314,10 +310,10 @@ pub struct PyHeapTypeObject { pub ht_cached_keys: *mut c_void, #[cfg(Py_3_9)] pub ht_module: *mut object::PyObject, - #[cfg(Py_3_11)] - pub _ht_tpname: *mut c_char, - #[cfg(Py_3_11)] - pub _spec_cache: _specialization_cache, + #[cfg(all(Py_3_11, not(PyPy)))] + _ht_tpname: *mut c_char, + #[cfg(all(Py_3_11, not(PyPy)))] + _spec_cache: _specialization_cache, } impl Default for PyHeapTypeObject { @@ -328,82 +324,75 @@ impl Default for PyHeapTypeObject { } #[inline] +#[cfg(not(Py_3_11))] pub unsafe fn PyHeapType_GET_MEMBERS(etype: *mut PyHeapTypeObject) -> *mut PyMemberDef { let py_type = object::Py_TYPE(etype as *mut object::PyObject); let ptr = etype.offset((*py_type).tp_basicsize); ptr as *mut PyMemberDef } -// skipped _PyType_Name -// skipped _PyType_Lookup -// skipped _PyType_LookupId -// skipped _PyObject_LookupSpecial -// skipped _PyType_CalculateMetaclass -// skipped _PyType_GetDocFromInternalDoc -// skipped _PyType_GetTextSignatureFromInternalDoc +// skipped private _PyType_Name +// skipped private _PyType_Lookup +// skipped private _PyType_LookupRef extern "C" { - #[cfg(Py_3_11)] - #[cfg_attr(PyPy, link_name = "PyPyType_GetModuleByDef")] - pub fn PyType_GetModuleByDef(ty: *mut PyTypeObject, def: *mut PyModuleDef) -> *mut PyObject; - #[cfg(Py_3_12)] pub fn PyType_GetDict(o: *mut PyTypeObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyObject_Print")] pub fn PyObject_Print(o: *mut PyObject, fp: *mut ::libc::FILE, flags: c_int) -> c_int; - // skipped _Py_BreakPoint - // skipped _PyObject_Dump - // skipped _PyObject_IsFreed - // skipped _PyObject_IsAbstract + // skipped private _Py_BreakPoint + // skipped private _PyObject_Dump + // skipped _PyObject_GetAttrId - // skipped _PyObject_SetAttrId - // skipped _PyObject_LookupAttr - // skipped _PyObject_LookupAttrId - // skipped _PyObject_GetMethod - #[cfg(not(PyPy))] - pub fn _PyObject_GetDictPtr(obj: *mut PyObject) -> *mut *mut PyObject; - #[cfg(not(PyPy))] - pub fn _PyObject_NextNotImplemented(arg1: *mut PyObject) -> *mut PyObject; + // skipped private _PyObject_GetDictPtr pub fn PyObject_CallFinalizer(arg1: *mut PyObject); #[cfg_attr(PyPy, link_name = "PyPyObject_CallFinalizerFromDealloc")] pub fn PyObject_CallFinalizerFromDealloc(arg1: *mut PyObject) -> c_int; - // skipped _PyObject_GenericGetAttrWithDict - // skipped _PyObject_GenericSetAttrWithDict - // skipped _PyObject_FunctionStr + // skipped private _PyObject_GenericGetAttrWithDict + // skipped private _PyObject_GenericSetAttrWithDict + // skipped private _PyObject_FunctionStr } // skipped Py_SETREF // skipped Py_XSETREF -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - pub static mut _PyNone_Type: PyTypeObject; - pub static mut _PyNotImplemented_Type: PyTypeObject; -} - -// skipped _Py_SwappedOp +// skipped private _PyObject_ASSERT_FROM +// skipped private _PyObject_ASSERT_WITH_MSG +// skipped private _PyObject_ASSERT +// skipped private _PyObject_ASSERT_FAILED_MSG +// skipped private _PyObject_AssertFailed -// skipped _PyDebugAllocatorStats -// skipped _PyObject_DebugTypeStats -// skipped _PyObject_ASSERT_FROM -// skipped _PyObject_ASSERT_WITH_MSG -// skipped _PyObject_ASSERT -// skipped _PyObject_ASSERT_FAILED_MSG -// skipped _PyObject_AssertFailed -// skipped _PyObject_CheckConsistency +// skipped private _PyTrash_begin +// skipped private _PyTrash_end // skipped _PyTrash_thread_deposit_object // skipped _PyTrash_thread_destroy_chain -// skipped _PyTrash_begin -// skipped _PyTrash_end -// skipped _PyTrash_cond -// skipped PyTrash_UNWIND_LEVEL -// skipped Py_TRASHCAN_BEGIN_CONDITION -// skipped Py_TRASHCAN_END + // skipped Py_TRASHCAN_BEGIN -// skipped Py_TRASHCAN_SAFE_BEGIN -// skipped Py_TRASHCAN_SAFE_END +// skipped Py_TRASHCAN_END + +// skipped PyObject_GetItemData + +// skipped PyObject_VisitManagedDict +// skipped _PyObject_SetManagedDict +// skipped PyObject_ClearManagedDict + +// skipped TYPE_MAX_WATCHERS + +// skipped PyType_WatchCallback +// skipped PyType_AddWatcher +// skipped PyType_ClearWatcher +// skipped PyType_Watch +// skipped PyType_Unwatch + +// skipped PyUnstable_Type_AssignVersionTag + +// skipped PyRefTracerEvent + +// skipped PyRefTracer +// skipped PyRefTracer_SetTracer +// skipped PyRefTracer_GetTracer diff --git a/pyo3-ffi/src/cpython/objimpl.rs b/pyo3-ffi/src/cpython/objimpl.rs index 3e0270ddc8f..14f7121a202 100644 --- a/pyo3-ffi/src/cpython/objimpl.rs +++ b/pyo3-ffi/src/cpython/objimpl.rs @@ -1,3 +1,4 @@ +#[cfg(not(all(Py_3_11, any(PyPy, GraalPy))))] use libc::size_t; use std::os::raw::c_int; diff --git a/pyo3-ffi/src/cpython/pyerrors.rs b/pyo3-ffi/src/cpython/pyerrors.rs index 6d17ebc8124..c6e10e5f07b 100644 --- a/pyo3-ffi/src/cpython/pyerrors.rs +++ b/pyo3-ffi/src/cpython/pyerrors.rs @@ -6,19 +6,19 @@ use crate::Py_ssize_t; #[derive(Debug)] pub struct PyBaseExceptionObject { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub dict: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub args: *mut PyObject, - #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] + #[cfg(all(Py_3_11, not(PyPy)))] pub notes: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub traceback: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub context: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub cause: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub suppress_context: char, } @@ -134,28 +134,25 @@ pub struct PyOSErrorObject { #[derive(Debug)] pub struct PyStopIterationObject { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub dict: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub args: *mut PyObject, - #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] + #[cfg(all(Py_3_11, not(PyPy)))] pub notes: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub traceback: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub context: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub cause: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub suppress_context: char, pub value: *mut PyObject, } -extern "C" { - #[cfg(not(any(PyPy, GraalPy)))] - pub fn _PyErr_ChainExceptions(typ: *mut PyObject, val: *mut PyObject, tb: *mut PyObject); -} +// skipped _PyErr_ChainExceptions // skipped PyNameErrorObject // skipped PyAttributeErrorObject diff --git a/pyo3-ffi/src/cpython/pyframe.rs b/pyo3-ffi/src/cpython/pyframe.rs index d0cfa0a2c6d..5e1e16a7d08 100644 --- a/pyo3-ffi/src/cpython/pyframe.rs +++ b/pyo3-ffi/src/cpython/pyframe.rs @@ -1,2 +1,2 @@ -#[cfg(Py_3_11)] +#[cfg(all(Py_3_11, not(PyPy)))] opaque_struct!(_PyInterpreterFrame); diff --git a/pyo3-ffi/src/cpython/pystate.rs b/pyo3-ffi/src/cpython/pystate.rs index 5481265b55d..650cd6a1f7f 100644 --- a/pyo3-ffi/src/cpython/pystate.rs +++ b/pyo3-ffi/src/cpython/pystate.rs @@ -69,21 +69,21 @@ extern "C" { pub fn PyThreadState_DeleteCurrent(); } -#[cfg(all(Py_3_9, not(Py_3_11)))] +#[cfg(all(Py_3_9, not(any(Py_3_11, PyPy))))] pub type _PyFrameEvalFunction = extern "C" fn( *mut crate::PyThreadState, *mut crate::PyFrameObject, c_int, ) -> *mut crate::object::PyObject; -#[cfg(Py_3_11)] +#[cfg(all(Py_3_11, not(PyPy)))] pub type _PyFrameEvalFunction = extern "C" fn( *mut crate::PyThreadState, *mut crate::_PyInterpreterFrame, c_int, ) -> *mut crate::object::PyObject; -#[cfg(Py_3_9)] +#[cfg(all(Py_3_9, not(PyPy)))] extern "C" { /// Get the frame evaluation function. pub fn _PyInterpreterState_GetEvalFrameFunc( diff --git a/pyo3-ffi/src/cpython/pythonrun.rs b/pyo3-ffi/src/cpython/pythonrun.rs index 94863166e11..fe78f55ca07 100644 --- a/pyo3-ffi/src/cpython/pythonrun.rs +++ b/pyo3-ffi/src/cpython/pythonrun.rs @@ -135,13 +135,9 @@ extern "C" { } #[inline] -#[cfg(not(GraalPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn Py_CompileString(string: *const c_char, p: *const c_char, s: c_int) -> *mut PyObject { - #[cfg(not(PyPy))] - return Py_CompileStringExFlags(string, p, s, std::ptr::null_mut(), -1); - - #[cfg(PyPy)] - Py_CompileStringFlags(string, p, s, std::ptr::null_mut()) + Py_CompileStringExFlags(string, p, s, std::ptr::null_mut(), -1) } #[inline] diff --git a/pyo3-ffi/src/cpython/tupleobject.rs b/pyo3-ffi/src/cpython/tupleobject.rs index 4ed8520daf3..9616d4372cc 100644 --- a/pyo3-ffi/src/cpython/tupleobject.rs +++ b/pyo3-ffi/src/cpython/tupleobject.rs @@ -5,17 +5,15 @@ use crate::pyport::Py_ssize_t; #[repr(C)] pub struct PyTupleObject { pub ob_base: PyVarObject, - #[cfg(not(GraalPy))] pub ob_item: [*mut PyObject; 1], } // skipped _PyTuple_Resize // skipped _PyTuple_MaybeUntrack -/// Macro, trading safety for speed - // skipped _PyTuple_CAST +/// Macro, trading safety for speed #[inline] #[cfg(not(PyPy))] pub unsafe fn PyTuple_GET_SIZE(op: *mut PyObject) -> Py_ssize_t { diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index 9ab523a2d7f..3527a5aeadb 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -1,7 +1,6 @@ -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(any(Py_3_11, not(PyPy)))] use crate::Py_hash_t; -use crate::{PyObject, Py_UCS1, Py_UCS2, Py_UCS4, Py_UNICODE, Py_ssize_t}; -#[cfg(not(any(Py_3_12, GraalPy)))] +use crate::{PyObject, Py_UCS1, Py_UCS2, Py_UCS4, Py_ssize_t}; use libc::wchar_t; use std::os::raw::{c_char, c_int, c_uint, c_void}; @@ -251,9 +250,8 @@ impl From for u32 { #[repr(C)] pub struct PyASCIIObject { pub ob_base: PyObject, - #[cfg(not(GraalPy))] pub length: Py_ssize_t, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(any(Py_3_11, not(PyPy)))] pub hash: Py_hash_t, /// A bit field with various properties. /// @@ -266,9 +264,8 @@ pub struct PyASCIIObject { /// unsigned int ascii:1; /// unsigned int ready:1; /// unsigned int :24; - #[cfg(not(GraalPy))] pub state: u32, - #[cfg(not(any(Py_3_12, GraalPy)))] + #[cfg(not(Py_3_12))] pub wstr: *mut wchar_t, } @@ -380,11 +377,9 @@ impl PyASCIIObject { #[repr(C)] pub struct PyCompactUnicodeObject { pub _base: PyASCIIObject, - #[cfg(not(GraalPy))] pub utf8_length: Py_ssize_t, - #[cfg(not(GraalPy))] pub utf8: *mut c_char, - #[cfg(not(any(Py_3_12, GraalPy)))] + #[cfg(not(Py_3_12))] pub wstr_length: Py_ssize_t, } @@ -399,7 +394,6 @@ pub union PyUnicodeObjectData { #[repr(C)] pub struct PyUnicodeObject { pub _base: PyCompactUnicodeObject, - #[cfg(not(GraalPy))] pub data: PyUnicodeObjectData, } @@ -449,19 +443,19 @@ pub const PyUnicode_1BYTE_KIND: c_uint = 1; pub const PyUnicode_2BYTE_KIND: c_uint = 2; pub const PyUnicode_4BYTE_KIND: c_uint = 4; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn PyUnicode_1BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS1 { PyUnicode_DATA(op) as *mut Py_UCS1 } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn PyUnicode_2BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS2 { PyUnicode_DATA(op) as *mut Py_UCS2 } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 { PyUnicode_DATA(op) as *mut Py_UCS4 @@ -487,7 +481,7 @@ pub unsafe fn _PyUnicode_COMPACT_DATA(op: *mut PyObject) -> *mut c_void { } } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void { debug_assert!(!(*(op as *mut PyUnicodeObject)).data.any.is_null()); @@ -495,7 +489,7 @@ pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void { (*(op as *mut PyUnicodeObject)).data.any } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -588,7 +582,7 @@ extern "C" { #[cfg(not(Py_3_12))] #[deprecated] #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromUnicode")] - pub fn PyUnicode_FromUnicode(u: *const Py_UNICODE, size: Py_ssize_t) -> *mut PyObject; + pub fn PyUnicode_FromUnicode(u: *const wchar_t, size: Py_ssize_t) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromKindAndData")] pub fn PyUnicode_FromKindAndData( @@ -603,7 +597,7 @@ extern "C" { #[cfg(not(Py_3_12))] #[deprecated] #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUnicode")] - pub fn PyUnicode_AsUnicode(unicode: *mut PyObject) -> *mut Py_UNICODE; + pub fn PyUnicode_AsUnicode(unicode: *mut PyObject) -> *mut wchar_t; // skipped _PyUnicode_AsUnicode @@ -613,7 +607,7 @@ extern "C" { pub fn PyUnicode_AsUnicodeAndSize( unicode: *mut PyObject, size: *mut Py_ssize_t, - ) -> *mut Py_UNICODE; + ) -> *mut wchar_t; // skipped PyUnicode_GetMax } @@ -642,14 +636,14 @@ extern "C" { // skipped _PyUnicode_AsString pub fn PyUnicode_Encode( - s: *const Py_UNICODE, + s: *const wchar_t, size: Py_ssize_t, encoding: *const c_char, errors: *const c_char, ) -> *mut PyObject; pub fn PyUnicode_EncodeUTF7( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, base64SetO: c_int, base64WhiteSpace: c_int, @@ -661,13 +655,13 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeUTF8")] pub fn PyUnicode_EncodeUTF8( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, errors: *const c_char, ) -> *mut PyObject; pub fn PyUnicode_EncodeUTF32( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, errors: *const c_char, byteorder: c_int, @@ -676,7 +670,7 @@ extern "C" { // skipped _PyUnicode_EncodeUTF32 pub fn PyUnicode_EncodeUTF16( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, errors: *const c_char, byteorder: c_int, @@ -685,13 +679,11 @@ extern "C" { // skipped _PyUnicode_EncodeUTF16 // skipped _PyUnicode_DecodeUnicodeEscape - pub fn PyUnicode_EncodeUnicodeEscape( - data: *const Py_UNICODE, - length: Py_ssize_t, - ) -> *mut PyObject; + pub fn PyUnicode_EncodeUnicodeEscape(data: *const wchar_t, length: Py_ssize_t) + -> *mut PyObject; pub fn PyUnicode_EncodeRawUnicodeEscape( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, ) -> *mut PyObject; @@ -699,7 +691,7 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeLatin1")] pub fn PyUnicode_EncodeLatin1( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, errors: *const c_char, ) -> *mut PyObject; @@ -708,13 +700,13 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeASCII")] pub fn PyUnicode_EncodeASCII( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, errors: *const c_char, ) -> *mut PyObject; pub fn PyUnicode_EncodeCharmap( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, mapping: *mut PyObject, errors: *const c_char, @@ -723,7 +715,7 @@ extern "C" { // skipped _PyUnicode_EncodeCharmap pub fn PyUnicode_TranslateCharmap( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, table: *mut PyObject, errors: *const c_char, @@ -733,17 +725,14 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeDecimal")] pub fn PyUnicode_EncodeDecimal( - s: *mut Py_UNICODE, + s: *mut wchar_t, length: Py_ssize_t, output: *mut c_char, errors: *const c_char, ) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyUnicode_TransformDecimalToASCII")] - pub fn PyUnicode_TransformDecimalToASCII( - s: *mut Py_UNICODE, - length: Py_ssize_t, - ) -> *mut PyObject; + pub fn PyUnicode_TransformDecimalToASCII(s: *mut wchar_t, length: Py_ssize_t) -> *mut PyObject; // skipped _PyUnicode_TransformDecimalAndSpaceToASCII } diff --git a/pyo3-ffi/src/cpython/weakrefobject.rs b/pyo3-ffi/src/cpython/weakrefobject.rs index 3a232c7ed38..88bb501bcc5 100644 --- a/pyo3-ffi/src/cpython/weakrefobject.rs +++ b/pyo3-ffi/src/cpython/weakrefobject.rs @@ -8,6 +8,8 @@ pub struct _PyWeakReference { pub wr_next: *mut crate::PyWeakReference, #[cfg(Py_3_11)] pub vectorcall: Option, + #[cfg(all(Py_3_13, Py_GIL_DISABLED))] + pub weakrefs_lock: *mut crate::PyMutex, } // skipped _PyWeakref_GetWeakrefCount diff --git a/pyo3-ffi/src/datetime.rs b/pyo3-ffi/src/datetime.rs index a20b76aa91d..7f2d7958364 100644 --- a/pyo3-ffi/src/datetime.rs +++ b/pyo3-ffi/src/datetime.rs @@ -3,21 +3,18 @@ //! This is the unsafe thin wrapper around the [CPython C API](https://docs.python.org/3/c-api/datetime.html), //! and covers the various date and time related objects in the Python `datetime` //! standard library module. -//! -//! A note regarding PyPy (cpyext) support: -//! -//! Support for `PyDateTime_CAPI` is limited as of PyPy 7.0.0. -//! `DateTime_FromTimestamp` and `Date_FromTimestamp` are currently not supported. +#[cfg(not(PyPy))] +use crate::PyCapsule_Import; #[cfg(GraalPy)] use crate::{PyLong_AsLong, PyLong_Check, PyObject_GetAttrString, Py_DecRef}; use crate::{PyObject, PyObject_TypeCheck, PyTypeObject, Py_TYPE}; -use std::cell::UnsafeCell; -use std::os::raw::{c_char, c_int}; +use std::os::raw::c_char; +use std::os::raw::c_int; use std::ptr; +use std::sync::Once; +use std::{cell::UnsafeCell, ffi::CStr}; #[cfg(not(PyPy))] -use {crate::PyCapsule_Import, std::ffi::CString}; -#[cfg(not(any(PyPy, GraalPy)))] use {crate::Py_hash_t, std::os::raw::c_uchar}; // Type struct wrappers const _PyDateTime_DATE_DATASIZE: usize = 4; @@ -25,17 +22,14 @@ const _PyDateTime_TIME_DATASIZE: usize = 6; const _PyDateTime_DATETIME_DATASIZE: usize = 10; #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] /// Structure representing a `datetime.timedelta`. pub struct PyDateTime_Delta { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(GraalPy))] pub days: c_int, - #[cfg(not(GraalPy))] pub seconds: c_int, - #[cfg(not(GraalPy))] pub microseconds: c_int, } @@ -44,7 +38,7 @@ pub struct PyDateTime_Delta { #[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] /// Structure representing a `datetime.time` without a `tzinfo` member. pub struct _PyDateTime_BaseTime { pub ob_base: PyObject, @@ -54,42 +48,40 @@ pub struct _PyDateTime_BaseTime { } #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] /// Structure representing a `datetime.time`. pub struct PyDateTime_Time { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(GraalPy))] pub hastzinfo: c_char, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_TIME_DATASIZE], - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub fold: c_uchar, /// # Safety /// /// Care should be taken when reading this field. If the time does not have a /// tzinfo then CPython may allocate as a `_PyDateTime_BaseTime` without this field. - #[cfg(not(GraalPy))] pub tzinfo: *mut PyObject, } #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] /// Structure representing a `datetime.date` pub struct PyDateTime_Date { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hastzinfo: c_char, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_DATE_DATASIZE], } #[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] /// Structure representing a `datetime.datetime` without a `tzinfo` member. pub struct _PyDateTime_BaseDateTime { pub ob_base: PyObject, @@ -99,23 +91,21 @@ pub struct _PyDateTime_BaseDateTime { } #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] /// Structure representing a `datetime.datetime`. pub struct PyDateTime_DateTime { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(GraalPy))] pub hastzinfo: c_char, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_DATETIME_DATASIZE], - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub fold: c_uchar, /// # Safety /// /// Care should be taken when reading this field. If the time does not have a /// tzinfo then CPython may allocate as a `_PyDateTime_BaseDateTime` without this field. - #[cfg(not(GraalPy))] pub tzinfo: *mut PyObject, } @@ -128,8 +118,8 @@ pub struct PyDateTime_DateTime { /// Returns a signed integer greater than 0. pub unsafe fn PyDateTime_GET_YEAR(o: *mut PyObject) -> c_int { // This should work for Date or DateTime - let d = *(o as *mut PyDateTime_Date); - c_int::from(d.data[0]) << 8 | c_int::from(d.data[1]) + let data = (*(o as *mut PyDateTime_Date)).data; + (c_int::from(data[0]) << 8) | c_int::from(data[1]) } #[inline] @@ -137,8 +127,8 @@ pub unsafe fn PyDateTime_GET_YEAR(o: *mut PyObject) -> c_int { /// Retrieve the month component of a `PyDateTime_Date` or `PyDateTime_DateTime`. /// Returns a signed integer in the range `[1, 12]`. pub unsafe fn PyDateTime_GET_MONTH(o: *mut PyObject) -> c_int { - let d = *(o as *mut PyDateTime_Date); - c_int::from(d.data[2]) + let data = (*(o as *mut PyDateTime_Date)).data; + c_int::from(data[2]) } #[inline] @@ -146,8 +136,8 @@ pub unsafe fn PyDateTime_GET_MONTH(o: *mut PyObject) -> c_int { /// Retrieve the day component of a `PyDateTime_Date` or `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[1, 31]`. pub unsafe fn PyDateTime_GET_DAY(o: *mut PyObject) -> c_int { - let d = *(o as *mut PyDateTime_Date); - c_int::from(d.data[3]) + let data = (*(o as *mut PyDateTime_Date)).data; + c_int::from(data[3]) } // Accessor macros for times @@ -355,8 +345,8 @@ pub unsafe fn PyDateTime_DELTA_GET_MICROSECONDS(o: *mut PyObject) -> c_int { // but copying them seems suboptimal #[inline] #[cfg(GraalPy)] -pub unsafe fn _get_attr(obj: *mut PyObject, field: &str) -> c_int { - let result = PyObject_GetAttrString(obj, field.as_ptr() as *const c_char); +pub unsafe fn _get_attr(obj: *mut PyObject, field: &std::ffi::CStr) -> c_int { + let result = PyObject_GetAttrString(obj, field.as_ptr()); Py_DecRef(result); // the original macros are borrowing if PyLong_Check(result) == 1 { PyLong_AsLong(result) as c_int @@ -368,55 +358,55 @@ pub unsafe fn _get_attr(obj: *mut PyObject, field: &str) -> c_int { #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_GET_YEAR(o: *mut PyObject) -> c_int { - _get_attr(o, "year\0") + _get_attr(o, c_str!("year")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_GET_MONTH(o: *mut PyObject) -> c_int { - _get_attr(o, "month\0") + _get_attr(o, c_str!("month")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_GET_DAY(o: *mut PyObject) -> c_int { - _get_attr(o, "day\0") + _get_attr(o, c_str!("day")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_DATE_GET_HOUR(o: *mut PyObject) -> c_int { - _get_attr(o, "hour\0") + _get_attr(o, c_str!("hour")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_DATE_GET_MINUTE(o: *mut PyObject) -> c_int { - _get_attr(o, "minute\0") + _get_attr(o, c_str!("minute")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_DATE_GET_SECOND(o: *mut PyObject) -> c_int { - _get_attr(o, "second\0") + _get_attr(o, c_str!("second")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_DATE_GET_MICROSECOND(o: *mut PyObject) -> c_int { - _get_attr(o, "microsecond\0") + _get_attr(o, c_str!("microsecond")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_DATE_GET_FOLD(o: *mut PyObject) -> c_int { - _get_attr(o, "fold\0") + _get_attr(o, c_str!("fold")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_DATE_GET_TZINFO(o: *mut PyObject) -> *mut PyObject { - let res = PyObject_GetAttrString(o, "tzinfo\0".as_ptr() as *const c_char); + let res = PyObject_GetAttrString(o, c_str!("tzinfo").as_ptr().cast()); Py_DecRef(res); // the original macros are borrowing res } @@ -424,37 +414,37 @@ pub unsafe fn PyDateTime_DATE_GET_TZINFO(o: *mut PyObject) -> *mut PyObject { #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_TIME_GET_HOUR(o: *mut PyObject) -> c_int { - _get_attr(o, "hour\0") + _get_attr(o, c_str!("hour")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_TIME_GET_MINUTE(o: *mut PyObject) -> c_int { - _get_attr(o, "minute\0") + _get_attr(o, c_str!("minute")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_TIME_GET_SECOND(o: *mut PyObject) -> c_int { - _get_attr(o, "second\0") + _get_attr(o, c_str!("second")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_TIME_GET_MICROSECOND(o: *mut PyObject) -> c_int { - _get_attr(o, "microsecond\0") + _get_attr(o, c_str!("microsecond")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_TIME_GET_FOLD(o: *mut PyObject) -> c_int { - _get_attr(o, "fold\0") + _get_attr(o, c_str!("fold")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_TIME_GET_TZINFO(o: *mut PyObject) -> *mut PyObject { - let res = PyObject_GetAttrString(o, "tzinfo\0".as_ptr() as *const c_char); + let res = PyObject_GetAttrString(o, c_str!("tzinfo").as_ptr().cast()); Py_DecRef(res); // the original macros are borrowing res } @@ -462,19 +452,19 @@ pub unsafe fn PyDateTime_TIME_GET_TZINFO(o: *mut PyObject) -> *mut PyObject { #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_DELTA_GET_DAYS(o: *mut PyObject) -> c_int { - _get_attr(o, "days\0") + _get_attr(o, c_str!("days")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_DELTA_GET_SECONDS(o: *mut PyObject) -> c_int { - _get_attr(o, "seconds\0") + _get_attr(o, c_str!("seconds")) } #[inline] #[cfg(GraalPy)] pub unsafe fn PyDateTime_DELTA_GET_MICROSECONDS(o: *mut PyObject) -> c_int { - _get_attr(o, "microseconds\0") + _get_attr(o, c_str!("microseconds")) } #[cfg(PyPy)] @@ -596,6 +586,8 @@ pub struct PyDateTime_CAPI { // Python already shares this object between threads, so it's no more evil for us to do it too! unsafe impl Sync for PyDateTime_CAPI {} +pub const PyDateTime_CAPSULE_NAME: &CStr = c_str!("datetime.datetime_CAPI"); + /// Returns a pointer to a `PyDateTime_CAPI` instance /// /// # Note @@ -603,33 +595,38 @@ unsafe impl Sync for PyDateTime_CAPI {} /// `PyDateTime_IMPORT` is called #[inline] pub unsafe fn PyDateTimeAPI() -> *mut PyDateTime_CAPI { - *PyDateTimeAPI_impl.0.get() -} - -#[inline] -pub unsafe fn PyDateTime_TimeZone_UTC() -> *mut PyObject { - (*PyDateTimeAPI()).TimeZone_UTC + *PyDateTimeAPI_impl.ptr.get() } /// Populates the `PyDateTimeAPI` object pub unsafe fn PyDateTime_IMPORT() { - // PyPy expects the C-API to be initialized via PyDateTime_Import, so trying to use - // `PyCapsule_Import` will behave unexpectedly in pypy. - #[cfg(PyPy)] - let py_datetime_c_api = PyDateTime_Import(); - - #[cfg(not(PyPy))] - let py_datetime_c_api = { - // PyDateTime_CAPSULE_NAME is a macro in C - let PyDateTime_CAPSULE_NAME = CString::new("datetime.datetime_CAPI").unwrap(); - - PyCapsule_Import(PyDateTime_CAPSULE_NAME.as_ptr(), 1) as *mut PyDateTime_CAPI - }; + if !PyDateTimeAPI_impl.once.is_completed() { + // PyPy expects the C-API to be initialized via PyDateTime_Import, so trying to use + // `PyCapsule_Import` will behave unexpectedly in pypy. + #[cfg(PyPy)] + let py_datetime_c_api = PyDateTime_Import(); + + #[cfg(not(PyPy))] + let py_datetime_c_api = + PyCapsule_Import(PyDateTime_CAPSULE_NAME.as_ptr(), 1) as *mut PyDateTime_CAPI; + + if py_datetime_c_api.is_null() { + return; + } - *PyDateTimeAPI_impl.0.get() = py_datetime_c_api; + // Protect against race conditions when the datetime API is concurrently + // initialized in multiple threads. UnsafeCell.get() cannot panic so this + // won't panic either. + PyDateTimeAPI_impl.once.call_once(|| { + *PyDateTimeAPI_impl.ptr.get() = py_datetime_c_api; + }); + } } -// skipped non-limited PyDateTime_TimeZone_UTC +#[inline] +pub unsafe fn PyDateTime_TimeZone_UTC() -> *mut PyObject { + (*PyDateTimeAPI()).TimeZone_UTC +} /// Type Check macros /// @@ -742,8 +739,13 @@ extern "C" { // Rust specific implementation details -struct PyDateTimeAPISingleton(UnsafeCell<*mut PyDateTime_CAPI>); +struct PyDateTimeAPISingleton { + once: Once, + ptr: UnsafeCell<*mut PyDateTime_CAPI>, +} unsafe impl Sync for PyDateTimeAPISingleton {} -static PyDateTimeAPI_impl: PyDateTimeAPISingleton = - PyDateTimeAPISingleton(UnsafeCell::new(ptr::null_mut())); +static PyDateTimeAPI_impl: PyDateTimeAPISingleton = PyDateTimeAPISingleton { + once: Once::new(), + ptr: UnsafeCell::new(ptr::null_mut()), +}; diff --git a/pyo3-ffi/src/dictobject.rs b/pyo3-ffi/src/dictobject.rs index 99fc56b246b..710be80243f 100644 --- a/pyo3-ffi/src/dictobject.rs +++ b/pyo3-ffi/src/dictobject.rs @@ -66,6 +66,13 @@ extern "C" { ) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyDict_DelItemString")] pub fn PyDict_DelItemString(dp: *mut PyObject, key: *const c_char) -> c_int; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyDict_GetItemRef")] + pub fn PyDict_GetItemRef( + dp: *mut PyObject, + key: *mut PyObject, + result: *mut *mut PyObject, + ) -> c_int; // skipped 3.10 / ex-non-limited PyObject_GenericGetDict } diff --git a/pyo3-ffi/src/genericaliasobject.rs b/pyo3-ffi/src/genericaliasobject.rs new file mode 100644 index 00000000000..7979d7d863e --- /dev/null +++ b/pyo3-ffi/src/genericaliasobject.rs @@ -0,0 +1,12 @@ +#[cfg(Py_3_9)] +use crate::object::{PyObject, PyTypeObject}; + +#[cfg_attr(windows, link(name = "pythonXY"))] +extern "C" { + #[cfg(Py_3_9)] + #[cfg_attr(PyPy, link_name = "PyPy_GenericAlias")] + pub fn Py_GenericAlias(origin: *mut PyObject, args: *mut PyObject) -> *mut PyObject; + + #[cfg(Py_3_9)] + pub static mut Py_GenericAliasType: PyTypeObject; +} diff --git a/pyo3-ffi/src/impl_/mod.rs b/pyo3-ffi/src/impl_/mod.rs new file mode 100644 index 00000000000..3058e852e6f --- /dev/null +++ b/pyo3-ffi/src/impl_/mod.rs @@ -0,0 +1,22 @@ +#[cfg(Py_GIL_DISABLED)] +mod atomic_c_ulong { + pub struct GetAtomicCULong(); + + pub trait AtomicCULongType { + type Type; + } + impl AtomicCULongType for GetAtomicCULong<32> { + type Type = std::sync::atomic::AtomicU32; + } + impl AtomicCULongType for GetAtomicCULong<64> { + type Type = std::sync::atomic::AtomicU64; + } + + pub type TYPE = + () * 8 }> as AtomicCULongType>::Type; +} + +/// Typedef for an atomic integer to match the platform-dependent c_ulong type. +#[cfg(Py_GIL_DISABLED)] +#[doc(hidden)] +pub type AtomicCULong = atomic_c_ulong::TYPE; diff --git a/pyo3-ffi/src/import.rs b/pyo3-ffi/src/import.rs index e00843466e8..e15a37b0a72 100644 --- a/pyo3-ffi/src/import.rs +++ b/pyo3-ffi/src/import.rs @@ -30,6 +30,9 @@ extern "C" { pub fn PyImport_AddModuleObject(name: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyImport_AddModule")] pub fn PyImport_AddModule(name: *const c_char) -> *mut PyObject; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyImport_AddModuleRef")] + pub fn PyImport_AddModuleRef(name: *const c_char) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyImport_ImportModule")] pub fn PyImport_ImportModule(name: *const c_char) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyImport_ImportModuleNoBlock")] diff --git a/pyo3-ffi/src/lib.rs b/pyo3-ffi/src/lib.rs index 7a203362ed5..b14fe1e8611 100644 --- a/pyo3-ffi/src/lib.rs +++ b/pyo3-ffi/src/lib.rs @@ -1,3 +1,4 @@ +#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] //! Raw FFI declarations for Python's C API. //! //! PyO3 can be used to write native Python modules or run Python code and modules from Rust. @@ -42,16 +43,46 @@ //! PyO3 uses `rustc`'s `--cfg` flags to enable or disable code used for different Python versions. //! If you want to do this for your own crate, you can do so with the [`pyo3-build-config`] crate. //! -//! - `Py_3_7`, `Py_3_8`, `Py_3_9`, `Py_3_10`: Marks code that is only enabled when -//! compiling for a given minimum Python version. +//! - `Py_3_7`, `Py_3_8`, `Py_3_9`, `Py_3_10`, `Py_3_11`, `Py_3_12`, `Py_3_13`: Marks code that is +//! only enabled when compiling for a given minimum Python version. //! - `Py_LIMITED_API`: Marks code enabled when the `abi3` feature flag is enabled. +//! - `Py_GIL_DISABLED`: Marks code that runs only in the free-threaded build of CPython. //! - `PyPy` - Marks code enabled when compiling for PyPy. +//! - `GraalPy` - Marks code enabled when compiling for GraalPy. +//! +//! Additionally, you can query for the values `Py_DEBUG`, `Py_REF_DEBUG`, +//! `Py_TRACE_REFS`, and `COUNT_ALLOCS` from `py_sys_config` to query for the +//! corresponding C build-time defines. For example, to conditionally define +//! debug code using `Py_DEBUG`, you could do: +//! +//! ```rust,ignore +//! #[cfg(py_sys_config = "Py_DEBUG")] +//! println!("only runs if python was compiled with Py_DEBUG") +//! ``` +//! +//! To use these attributes, add [`pyo3-build-config`] as a build dependency in +//! your `Cargo.toml`: +//! +//! ```toml +//! [build-dependencies] +#![doc = concat!("pyo3-build-config =\"", env!("CARGO_PKG_VERSION"), "\"")] +//! ``` +//! +//! And then either create a new `build.rs` file in the project root or modify +//! the existing `build.rs` file to call `use_pyo3_cfgs()`: +//! +//! ```rust,ignore +//! fn main() { +//! pyo3_build_config::use_pyo3_cfgs(); +//! } +//! ``` //! //! # Minimum supported Rust and Python versions //! -//! PyO3 supports the following software versions: -//! - Python 3.7 and up (CPython and PyPy) -//! - Rust 1.56 and up +//! `pyo3-ffi` supports the following Python distributions: +//! - CPython 3.7 or greater +//! - PyPy 7.3 (Python 3.9+) +//! - GraalPy 24.0 or greater (Python 3.10+) //! //! # Example: Building Python Native modules //! @@ -77,99 +108,148 @@ //! [dependencies.pyo3-ffi] #![doc = concat!("version = \"", env!("CARGO_PKG_VERSION"), "\"")] //! features = ["extension-module"] +//! +//! [build-dependencies] +//! # This is only necessary if you need to configure your build based on +//! # the Python version or the compile-time configuration for the interpreter. +#![doc = concat!("pyo3_build_config = \"", env!("CARGO_PKG_VERSION"), "\"")] +//! ``` +//! +//! If you need to use conditional compilation based on Python version or how +//! Python was compiled, you need to add `pyo3-build-config` as a +//! `build-dependency` in your `Cargo.toml` as in the example above and either +//! create a new `build.rs` file or modify an existing one so that +//! `pyo3_build_config::use_pyo3_cfgs()` gets called at build time: +//! +//! **`build.rs`** +//! ```rust,ignore +//! fn main() { +//! pyo3_build_config::use_pyo3_cfgs() +//! } //! ``` //! //! **`src/lib.rs`** //! ```rust -//! use std::os::raw::c_char; +//! use std::os::raw::{c_char, c_long}; //! use std::ptr; //! //! use pyo3_ffi::*; //! //! static mut MODULE_DEF: PyModuleDef = PyModuleDef { //! m_base: PyModuleDef_HEAD_INIT, -//! m_name: "string_sum\0".as_ptr().cast::(), -//! m_doc: "A Python module written in Rust.\0" -//! .as_ptr() -//! .cast::(), +//! m_name: c_str!("string_sum").as_ptr(), +//! m_doc: c_str!("A Python module written in Rust.").as_ptr(), //! m_size: 0, -//! m_methods: unsafe { METHODS.as_mut_ptr().cast() }, +//! m_methods: unsafe { METHODS as *const [PyMethodDef] as *mut PyMethodDef }, //! m_slots: std::ptr::null_mut(), //! m_traverse: None, //! m_clear: None, //! m_free: None, //! }; //! -//! static mut METHODS: [PyMethodDef; 2] = [ +//! static mut METHODS: &[PyMethodDef] = &[ //! PyMethodDef { -//! ml_name: "sum_as_string\0".as_ptr().cast::(), +//! ml_name: c_str!("sum_as_string").as_ptr(), //! ml_meth: PyMethodDefPointer { -//! _PyCFunctionFast: sum_as_string, +//! PyCFunctionFast: sum_as_string, //! }, //! ml_flags: METH_FASTCALL, -//! ml_doc: "returns the sum of two integers as a string\0" -//! .as_ptr() -//! .cast::(), +//! ml_doc: c_str!("returns the sum of two integers as a string").as_ptr(), //! }, //! // A zeroed PyMethodDef to mark the end of the array. -//! PyMethodDef::zeroed() +//! PyMethodDef::zeroed(), //! ]; //! //! // The module initialization function, which must be named `PyInit_`. //! #[allow(non_snake_case)] //! #[no_mangle] //! pub unsafe extern "C" fn PyInit_string_sum() -> *mut PyObject { -//! PyModule_Create(ptr::addr_of_mut!(MODULE_DEF)) +//! let module = PyModule_Create(ptr::addr_of_mut!(MODULE_DEF)); +//! if module.is_null() { +//! return module; +//! } +//! #[cfg(Py_GIL_DISABLED)] +//! { +//! if PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED) < 0 { +//! Py_DECREF(module); +//! return std::ptr::null_mut(); +//! } +//! } +//! module //! } //! -//! pub unsafe extern "C" fn sum_as_string( -//! _self: *mut PyObject, -//! args: *mut *mut PyObject, -//! nargs: Py_ssize_t, -//! ) -> *mut PyObject { -//! if nargs != 2 { -//! PyErr_SetString( -//! PyExc_TypeError, -//! "sum_as_string() expected 2 positional arguments\0" -//! .as_ptr() -//! .cast::(), +//! /// A helper to parse function arguments +//! /// If we used PyO3's proc macros they'd handle all of this boilerplate for us :) +//! unsafe fn parse_arg_as_i32(obj: *mut PyObject, n_arg: usize) -> Option { +//! if PyLong_Check(obj) == 0 { +//! let msg = format!( +//! "sum_as_string expected an int for positional argument {}\0", +//! n_arg //! ); -//! return std::ptr::null_mut(); +//! PyErr_SetString(PyExc_TypeError, msg.as_ptr().cast::()); +//! return None; //! } //! -//! let arg1 = *args; -//! if PyLong_Check(arg1) == 0 { -//! PyErr_SetString( -//! PyExc_TypeError, -//! "sum_as_string() expected an int for positional argument 1\0" -//! .as_ptr() -//! .cast::(), -//! ); -//! return std::ptr::null_mut(); +//! // Let's keep the behaviour consistent on platforms where `c_long` is bigger than 32 bits. +//! // In particular, it is an i32 on Windows but i64 on most Linux systems +//! let mut overflow = 0; +//! let i_long: c_long = PyLong_AsLongAndOverflow(obj, &mut overflow); +//! +//! #[allow(irrefutable_let_patterns)] // some platforms have c_long equal to i32 +//! if overflow != 0 { +//! raise_overflowerror(obj); +//! None +//! } else if let Ok(i) = i_long.try_into() { +//! Some(i) +//! } else { +//! raise_overflowerror(obj); +//! None //! } +//! } //! -//! let arg1 = PyLong_AsLong(arg1); -//! if !PyErr_Occurred().is_null() { -//! return ptr::null_mut(); +//! unsafe fn raise_overflowerror(obj: *mut PyObject) { +//! let obj_repr = PyObject_Str(obj); +//! if !obj_repr.is_null() { +//! let mut size = 0; +//! let p = PyUnicode_AsUTF8AndSize(obj_repr, &mut size); +//! if !p.is_null() { +//! let s = std::str::from_utf8_unchecked(std::slice::from_raw_parts( +//! p.cast::(), +//! size as usize, +//! )); +//! let msg = format!("cannot fit {} in 32 bits\0", s); +//! +//! PyErr_SetString(PyExc_OverflowError, msg.as_ptr().cast::()); +//! } +//! Py_DECREF(obj_repr); //! } +//! } //! -//! let arg2 = *args.add(1); -//! if PyLong_Check(arg2) == 0 { +//! pub unsafe extern "C" fn sum_as_string( +//! _self: *mut PyObject, +//! args: *mut *mut PyObject, +//! nargs: Py_ssize_t, +//! ) -> *mut PyObject { +//! if nargs != 2 { //! PyErr_SetString( //! PyExc_TypeError, -//! "sum_as_string() expected an int for positional argument 2\0" -//! .as_ptr() -//! .cast::(), +//! c_str!("sum_as_string expected 2 positional arguments").as_ptr(), //! ); //! return std::ptr::null_mut(); //! } //! -//! let arg2 = PyLong_AsLong(arg2); -//! if !PyErr_Occurred().is_null() { -//! return ptr::null_mut(); -//! } +//! let (first, second) = (*args, *args.add(1)); +//! +//! let first = match parse_arg_as_i32(first, 1) { +//! Some(x) => x, +//! None => return std::ptr::null_mut(), +//! }; +//! let second = match parse_arg_as_i32(second, 2) { +//! Some(x) => x, +//! None => return std::ptr::null_mut(), +//! }; //! -//! match arg1.checked_add(arg2) { +//! match first.checked_add(second) { //! Some(sum) => { //! let string = sum.to_string(); //! PyUnicode_FromStringAndSize(string.as_ptr().cast::(), string.len() as isize) @@ -177,7 +257,7 @@ //! None => { //! PyErr_SetString( //! PyExc_OverflowError, -//! "arguments too large to add\0".as_ptr().cast::(), +//! c_str!("arguments too large to add").as_ptr(), //! ); //! std::ptr::null_mut() //! } @@ -209,6 +289,12 @@ //! [manually][manual_builds]. Both offer more flexibility than `maturin` but require further //! configuration. //! +//! This example stores the module definition statically and uses the `PyModule_Create` function +//! in the CPython C API to register the module. This is the "old" style for registering modules +//! and has the limitation that it cannot support subinterpreters. You can also create a module +//! using the new multi-phase initialization API that does support subinterpreters. See the +//! `sequential` project located in the `examples` directory at the root of the `pyo3-ffi` crate +//! for a worked example of how to this using `pyo3-ffi`. //! //! # Using Python from Rust //! @@ -231,11 +317,10 @@ //! [`maturin`]: https://github.com/PyO3/maturin "Build and publish crates with pyo3, rust-cpython and cffi bindings as well as rust binaries as python packages" //! [`pyo3-build-config`]: https://docs.rs/pyo3-build-config //! [feature flags]: https://doc.rust-lang.org/cargo/reference/features.html "Features - The Cargo Book" -//! [manual_builds]: https://pyo3.rs/latest/building-and-distribution.html#manual-builds "Manual builds - Building and Distribution - PyO3 user guide" +#![doc = concat!("[manual_builds]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/building-and-distribution.html#manual-builds \"Manual builds - Building and Distribution - PyO3 user guide\"")] //! [setuptools-rust]: https://github.com/PyO3/setuptools-rust "Setuptools plugin for Rust extensions" //! [PEP 384]: https://www.python.org/dev/peps/pep-0384 "PEP 384 -- Defining a Stable ABI" -//! [Features chapter of the guide]: https://pyo3.rs/latest/features.html#features-reference "Features Reference - PyO3 user guide" - +#![doc = concat!("[Features chapter of the guide]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/features.html#features-reference \"Features eference - PyO3 user guide\"")] #![allow( missing_docs, non_camel_case_types, @@ -245,6 +330,11 @@ clippy::missing_safety_doc )] #![warn(elided_lifetimes_in_paths, unused_lifetimes)] +// This crate is a hand-maintained translation of CPython's headers, so requiring "unsafe" +// blocks within those translations increases maintenance burden without providing any +// additional safety. The safety of the functions in this crate is determined by the +// original CPython headers +#![allow(unsafe_op_in_unsafe_fn)] // Until `extern type` is stabilized, use the recommended approach to // model opaque types: @@ -256,6 +346,54 @@ macro_rules! opaque_struct { }; } +/// This is a helper macro to create a `&'static CStr`. +/// +/// It can be used on all Rust versions supported by PyO3, unlike c"" literals which +/// were stabilised in Rust 1.77. +/// +/// Due to the nature of PyO3 making heavy use of C FFI interop with Python, it is +/// common for PyO3 to use CStr. +/// +/// Examples: +/// +/// ```rust +/// use std::ffi::CStr; +/// +/// const HELLO: &CStr = pyo3_ffi::c_str!("hello"); +/// static WORLD: &CStr = pyo3_ffi::c_str!("world"); +/// ``` +#[macro_export] +macro_rules! c_str { + ($s:expr) => { + $crate::_cstr_from_utf8_with_nul_checked(concat!($s, "\0")) + }; +} + +/// Private helper for `c_str!` macro. +#[doc(hidden)] +pub const fn _cstr_from_utf8_with_nul_checked(s: &str) -> &CStr { + // TODO: Replace this implementation with `CStr::from_bytes_with_nul` when MSRV above 1.72. + let bytes = s.as_bytes(); + let len = bytes.len(); + assert!( + !bytes.is_empty() && bytes[bytes.len() - 1] == b'\0', + "string is not nul-terminated" + ); + let mut i = 0; + let non_null_len = len - 1; + while i < non_null_len { + assert!(bytes[i] != b'\0', "string contains null bytes"); + i += 1; + } + + unsafe { CStr::from_bytes_with_nul_unchecked(bytes) } +} + +use std::ffi::CStr; + +pub mod compat; +mod impl_; + pub use self::abstract_::*; pub use self::bltinmodule::*; pub use self::boolobject::*; @@ -277,6 +415,8 @@ pub use self::enumobject::*; pub use self::fileobject::*; pub use self::fileutils::*; pub use self::floatobject::*; +#[cfg(Py_3_9)] +pub use self::genericaliasobject::*; pub use self::import::*; pub use self::intrcheck::*; pub use self::iterobject::*; @@ -346,7 +486,7 @@ mod fileobject; mod fileutils; mod floatobject; // skipped empty frameobject.h -// skipped genericaliasobject.h +mod genericaliasobject; mod import; // skipped interpreteridobject.h mod intrcheck; diff --git a/pyo3-ffi/src/listobject.rs b/pyo3-ffi/src/listobject.rs index a6f47eadf83..881a8a8707b 100644 --- a/pyo3-ffi/src/listobject.rs +++ b/pyo3-ffi/src/listobject.rs @@ -28,6 +28,9 @@ extern "C" { pub fn PyList_Size(arg1: *mut PyObject) -> Py_ssize_t; #[cfg_attr(PyPy, link_name = "PyPyList_GetItem")] pub fn PyList_GetItem(arg1: *mut PyObject, arg2: Py_ssize_t) -> *mut PyObject; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyList_GetItemRef")] + pub fn PyList_GetItemRef(arg1: *mut PyObject, arg2: Py_ssize_t) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyList_SetItem")] pub fn PyList_SetItem(arg1: *mut PyObject, arg2: Py_ssize_t, arg3: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyList_Insert")] @@ -47,6 +50,10 @@ extern "C" { arg3: Py_ssize_t, arg4: *mut PyObject, ) -> c_int; + #[cfg(Py_3_13)] + pub fn PyList_Extend(list: *mut PyObject, iterable: *mut PyObject) -> c_int; + #[cfg(Py_3_13)] + pub fn PyList_Clear(list: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyList_Sort")] pub fn PyList_Sort(arg1: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyList_Reverse")] diff --git a/pyo3-ffi/src/longobject.rs b/pyo3-ffi/src/longobject.rs index 55ea8fa1462..68b4ecba540 100644 --- a/pyo3-ffi/src/longobject.rs +++ b/pyo3-ffi/src/longobject.rs @@ -1,19 +1,11 @@ use crate::object::*; use crate::pyport::Py_ssize_t; use libc::size_t; -#[cfg(not(Py_LIMITED_API))] -use std::os::raw::c_uchar; use std::os::raw::{c_char, c_double, c_int, c_long, c_longlong, c_ulong, c_ulonglong, c_void}; use std::ptr::addr_of_mut; opaque_struct!(PyLongObject); -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - #[cfg_attr(PyPy, link_name = "PyPyLong_Type")] - pub static mut PyLong_Type: PyTypeObject; -} - #[inline] pub unsafe fn PyLong_Check(op: *mut PyObject) -> c_int { PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS) @@ -90,34 +82,11 @@ extern "C" { arg3: c_int, ) -> *mut PyObject; } -// skipped non-limited PyLong_FromUnicodeObject -// skipped non-limited _PyLong_FromBytes #[cfg(not(Py_LIMITED_API))] extern "C" { - // skipped non-limited _PyLong_Sign - #[cfg_attr(PyPy, link_name = "_PyPyLong_NumBits")] pub fn _PyLong_NumBits(obj: *mut PyObject) -> size_t; - - // skipped _PyLong_DivmodNear - - #[cfg_attr(PyPy, link_name = "_PyPyLong_FromByteArray")] - pub fn _PyLong_FromByteArray( - bytes: *const c_uchar, - n: size_t, - little_endian: c_int, - is_signed: c_int, - ) -> *mut PyObject; - - #[cfg_attr(PyPy, link_name = "_PyPyLong_AsByteArrayO")] - pub fn _PyLong_AsByteArray( - v: *mut PyLongObject, - bytes: *mut c_uchar, - n: size_t, - little_endian: c_int, - is_signed: c_int, - ) -> c_int; } // skipped non-limited _PyLong_Format @@ -130,6 +99,5 @@ extern "C" { pub fn PyOS_strtol(arg1: *const c_char, arg2: *mut *mut c_char, arg3: c_int) -> c_long; } -// skipped non-limited _PyLong_GCD // skipped non-limited _PyLong_Rshift // skipped non-limited _PyLong_Lshift diff --git a/pyo3-ffi/src/methodobject.rs b/pyo3-ffi/src/methodobject.rs index 3ed6b770e54..3dfbbb5a208 100644 --- a/pyo3-ffi/src/methodobject.rs +++ b/pyo3-ffi/src/methodobject.rs @@ -43,26 +43,34 @@ pub type PyCFunction = unsafe extern "C" fn(slf: *mut PyObject, args: *mut PyObject) -> *mut PyObject; #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] -pub type _PyCFunctionFast = unsafe extern "C" fn( +pub type PyCFunctionFast = unsafe extern "C" fn( slf: *mut PyObject, args: *mut *mut PyObject, nargs: crate::pyport::Py_ssize_t, ) -> *mut PyObject; +#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] +#[deprecated(note = "renamed to `PyCFunctionFast`")] +pub type _PyCFunctionFast = PyCFunctionFast; + pub type PyCFunctionWithKeywords = unsafe extern "C" fn( slf: *mut PyObject, args: *mut PyObject, kwds: *mut PyObject, ) -> *mut PyObject; -#[cfg(not(Py_LIMITED_API))] -pub type _PyCFunctionFastWithKeywords = unsafe extern "C" fn( +#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] +pub type PyCFunctionFastWithKeywords = unsafe extern "C" fn( slf: *mut PyObject, args: *const *mut PyObject, nargs: crate::pyport::Py_ssize_t, kwnames: *mut PyObject, ) -> *mut PyObject; +#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] +#[deprecated(note = "renamed to `PyCFunctionFastWithKeywords`")] +pub type _PyCFunctionFastWithKeywords = PyCFunctionFastWithKeywords; + #[cfg(all(Py_3_9, not(Py_LIMITED_API)))] pub type PyCMethod = unsafe extern "C" fn( slf: *mut PyObject, @@ -77,6 +85,7 @@ extern "C" { pub fn PyCFunction_GetFunction(f: *mut PyObject) -> Option; pub fn PyCFunction_GetSelf(f: *mut PyObject) -> *mut PyObject; pub fn PyCFunction_GetFlags(f: *mut PyObject) -> c_int; + #[cfg(not(Py_3_13))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] pub fn PyCFunction_Call( f: *mut PyObject, @@ -144,11 +153,21 @@ pub union PyMethodDefPointer { /// This variant corresponds with [`METH_FASTCALL`]. #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] - pub _PyCFunctionFast: _PyCFunctionFast, + #[deprecated(note = "renamed to `PyCFunctionFast`")] + pub _PyCFunctionFast: PyCFunctionFast, + + /// This variant corresponds with [`METH_FASTCALL`]. + #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + pub PyCFunctionFast: PyCFunctionFast, + + /// This variant corresponds with [`METH_FASTCALL`] | [`METH_KEYWORDS`]. + #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + #[deprecated(note = "renamed to `PyCFunctionFastWithKeywords`")] + pub _PyCFunctionFastWithKeywords: PyCFunctionFastWithKeywords, /// This variant corresponds with [`METH_FASTCALL`] | [`METH_KEYWORDS`]. - #[cfg(not(Py_LIMITED_API))] - pub _PyCFunctionFastWithKeywords: _PyCFunctionFastWithKeywords, + #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + pub PyCFunctionFastWithKeywords: PyCFunctionFastWithKeywords, /// This variant corresponds with [`METH_METHOD`] | [`METH_FASTCALL`] | [`METH_KEYWORDS`]. #[cfg(all(Py_3_9, not(Py_LIMITED_API)))] @@ -186,9 +205,8 @@ impl std::fmt::Pointer for PyMethodDefPointer { } } -// TODO: This can be a const assert on Rust 1.57 const _: () = - [()][mem::size_of::() - mem::size_of::>()]; + assert!(mem::size_of::() == mem::size_of::>()); #[cfg(not(Py_3_9))] extern "C" { diff --git a/pyo3-ffi/src/modsupport.rs b/pyo3-ffi/src/modsupport.rs index b259c70059e..4a18d30f97c 100644 --- a/pyo3-ffi/src/modsupport.rs +++ b/pyo3-ffi/src/modsupport.rs @@ -14,9 +14,14 @@ extern "C" { arg1: *mut PyObject, arg2: *mut PyObject, arg3: *const c_char, - arg4: *mut *mut c_char, + #[cfg(not(Py_3_13))] arg4: *mut *mut c_char, + #[cfg(Py_3_13)] arg4: *const *const c_char, ... ) -> c_int; + + // skipped PyArg_VaParse + // skipped PyArg_VaParseTupleAndKeywords + pub fn PyArg_ValidateKeywordArguments(arg1: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyArg_UnpackTuple")] pub fn PyArg_UnpackTuple( @@ -26,32 +31,10 @@ extern "C" { arg4: Py_ssize_t, ... ) -> c_int; + #[cfg_attr(PyPy, link_name = "PyPy_BuildValue")] pub fn Py_BuildValue(arg1: *const c_char, ...) -> *mut PyObject; - // #[cfg_attr(PyPy, link_name = "_PyPy_BuildValue_SizeT")] - //pub fn _Py_BuildValue_SizeT(arg1: *const c_char, ...) - // -> *mut PyObject; - // #[cfg_attr(PyPy, link_name = "PyPy_VaBuildValue")] - - // skipped non-limited _PyArg_UnpackStack - // skipped non-limited _PyArg_NoKeywords - // skipped non-limited _PyArg_NoKwnames - // skipped non-limited _PyArg_NoPositional - // skipped non-limited _PyArg_BadArgument - // skipped non-limited _PyArg_CheckPositional - - //pub fn Py_VaBuildValue(arg1: *const c_char, arg2: va_list) - // -> *mut PyObject; - - // skipped non-limited _Py_VaBuildStack - // skipped non-limited _PyArg_Parser - - // skipped non-limited _PyArg_ParseTupleAndKeywordsFast - // skipped non-limited _PyArg_ParseStack - // skipped non-limited _PyArg_ParseStackAndKeywords - // skipped non-limited _PyArg_VaParseTupleAndKeywordsFast - // skipped non-limited _PyArg_UnpackKeywords - // skipped non-limited _PyArg_Fini + // skipped Py_VaBuildValue #[cfg(Py_3_10)] #[cfg_attr(PyPy, link_name = "PyPyModule_AddObjectRef")] @@ -159,9 +142,3 @@ pub unsafe fn PyModule_FromDefAndSpec(def: *mut PyModuleDef, spec: *mut PyObject }, ) } - -#[cfg(not(Py_LIMITED_API))] -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - pub static mut _Py_PackageContext: *const c_char; -} diff --git a/pyo3-ffi/src/moduleobject.rs b/pyo3-ffi/src/moduleobject.rs index f4306b18639..2417664a421 100644 --- a/pyo3-ffi/src/moduleobject.rs +++ b/pyo3-ffi/src/moduleobject.rs @@ -21,6 +21,7 @@ pub unsafe fn PyModule_CheckExact(op: *mut PyObject) -> c_int { } extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyModule_NewObject")] pub fn PyModule_NewObject(name: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyModule_New")] pub fn PyModule_New(name: *const c_char) -> *mut PyObject; @@ -52,7 +53,6 @@ extern "C" { } #[repr(C)] -#[derive(Copy, Clone)] pub struct PyModuleDef_Base { pub ob_base: PyObject, pub m_init: Option *mut PyObject>, @@ -60,6 +60,7 @@ pub struct PyModuleDef_Base { pub m_copy: *mut PyObject, } +#[allow(clippy::declare_interior_mutable_const)] pub const PyModuleDef_HEAD_INIT: PyModuleDef_Base = PyModuleDef_Base { ob_base: PyObject_HEAD_INIT, m_init: None, @@ -87,6 +88,10 @@ pub const Py_mod_create: c_int = 1; pub const Py_mod_exec: c_int = 2; #[cfg(Py_3_12)] pub const Py_mod_multiple_interpreters: c_int = 3; +#[cfg(Py_3_13)] +pub const Py_mod_gil: c_int = 4; + +// skipped private _Py_mod_LAST_SLOT #[cfg(Py_3_12)] pub const Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED: *mut c_void = 0 as *mut c_void; @@ -95,10 +100,17 @@ pub const Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED: *mut c_void = 1 as *mut c_void #[cfg(Py_3_12)] pub const Py_MOD_PER_INTERPRETER_GIL_SUPPORTED: *mut c_void = 2 as *mut c_void; -// skipped non-limited _Py_mod_LAST_SLOT +#[cfg(Py_3_13)] +pub const Py_MOD_GIL_USED: *mut c_void = 0 as *mut c_void; +#[cfg(Py_3_13)] +pub const Py_MOD_GIL_NOT_USED: *mut c_void = 1 as *mut c_void; + +#[cfg(all(not(Py_LIMITED_API), Py_GIL_DISABLED))] +extern "C" { + pub fn PyUnstable_Module_SetGIL(module: *mut PyObject, gil: *mut c_void) -> c_int; +} #[repr(C)] -#[derive(Copy, Clone)] pub struct PyModuleDef { pub m_base: PyModuleDef_Base, pub m_name: *const c_char, diff --git a/pyo3-ffi/src/object.rs b/pyo3-ffi/src/object.rs index b33ee558a37..087cd32920c 100644 --- a/pyo3-ffi/src/object.rs +++ b/pyo3-ffi/src/object.rs @@ -1,7 +1,13 @@ use crate::pyport::{Py_hash_t, Py_ssize_t}; +#[cfg(Py_GIL_DISABLED)] +use crate::PyMutex; +#[cfg(Py_GIL_DISABLED)] +use std::marker::PhantomPinned; use std::mem; use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void}; use std::ptr; +#[cfg(Py_GIL_DISABLED)] +use std::sync::atomic::{AtomicIsize, AtomicU32, AtomicU8, Ordering::Relaxed}; #[cfg(Py_LIMITED_API)] opaque_struct!(PyTypeObject); @@ -9,11 +15,8 @@ opaque_struct!(PyTypeObject); #[cfg(not(Py_LIMITED_API))] pub use crate::cpython::object::PyTypeObject; -// _PyObject_HEAD_EXTRA: conditionally defined in PyObject_HEAD_INIT -// _PyObject_EXTRA_INIT: conditionally defined in PyObject_HEAD_INIT - #[cfg(Py_3_12)] -pub const _Py_IMMORTAL_REFCNT: Py_ssize_t = { +const _Py_IMMORTAL_REFCNT: Py_ssize_t = { if cfg!(target_pointer_width = "64") { c_uint::MAX as Py_ssize_t } else { @@ -22,12 +25,31 @@ pub const _Py_IMMORTAL_REFCNT: Py_ssize_t = { } }; +#[cfg(Py_GIL_DISABLED)] +const _Py_IMMORTAL_REFCNT_LOCAL: u32 = u32::MAX; + +#[allow(clippy::declare_interior_mutable_const)] pub const PyObject_HEAD_INIT: PyObject = PyObject { #[cfg(py_sys_config = "Py_TRACE_REFS")] _ob_next: std::ptr::null_mut(), #[cfg(py_sys_config = "Py_TRACE_REFS")] _ob_prev: std::ptr::null_mut(), - #[cfg(Py_3_12)] + #[cfg(Py_GIL_DISABLED)] + ob_tid: 0, + #[cfg(Py_GIL_DISABLED)] + _padding: 0, + #[cfg(Py_GIL_DISABLED)] + ob_mutex: PyMutex { + _bits: AtomicU8::new(0), + _pin: PhantomPinned, + }, + #[cfg(Py_GIL_DISABLED)] + ob_gc_bits: 0, + #[cfg(Py_GIL_DISABLED)] + ob_ref_local: AtomicU32::new(_Py_IMMORTAL_REFCNT_LOCAL), + #[cfg(Py_GIL_DISABLED)] + ob_ref_shared: AtomicIsize::new(0), + #[cfg(all(not(Py_GIL_DISABLED), Py_3_12))] ob_refcnt: PyObjectObRefcnt { ob_refcnt: 1 }, #[cfg(not(Py_3_12))] ob_refcnt: 1, @@ -39,9 +61,22 @@ pub const PyObject_HEAD_INIT: PyObject = PyObject { // skipped PyObject_VAR_HEAD // skipped Py_INVALID_SIZE +// skipped private _Py_UNOWNED_TID + +#[cfg(Py_GIL_DISABLED)] +const _Py_REF_SHARED_SHIFT: isize = 2; +// skipped private _Py_REF_SHARED_FLAG_MASK + +// skipped private _Py_REF_SHARED_INIT +// skipped private _Py_REF_MAYBE_WEAKREF +// skipped private _Py_REF_QUEUED +// skipped private _Py_REF_MERGED + +// skipped private _Py_REF_SHARED + #[repr(C)] #[derive(Copy, Clone)] -#[cfg(Py_3_12)] +#[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] /// This union is anonymous in CPython, so the name was given by PyO3 because /// Rust unions need a name. pub union PyObjectObRefcnt { @@ -50,59 +85,103 @@ pub union PyObjectObRefcnt { pub ob_refcnt_split: [crate::PY_UINT32_T; 2], } -#[cfg(Py_3_12)] +#[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] impl std::fmt::Debug for PyObjectObRefcnt { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", unsafe { self.ob_refcnt }) } } -#[cfg(not(Py_3_12))] +#[cfg(all(not(Py_3_12), not(Py_GIL_DISABLED)))] pub type PyObjectObRefcnt = Py_ssize_t; #[repr(C)] -#[derive(Copy, Clone, Debug)] +#[derive(Debug)] pub struct PyObject { #[cfg(py_sys_config = "Py_TRACE_REFS")] pub _ob_next: *mut PyObject, #[cfg(py_sys_config = "Py_TRACE_REFS")] pub _ob_prev: *mut PyObject, + #[cfg(Py_GIL_DISABLED)] + pub ob_tid: libc::uintptr_t, + #[cfg(Py_GIL_DISABLED)] + pub _padding: u16, + #[cfg(Py_GIL_DISABLED)] + pub ob_mutex: PyMutex, // per-object lock + #[cfg(Py_GIL_DISABLED)] + pub ob_gc_bits: u8, // gc-related state + #[cfg(Py_GIL_DISABLED)] + pub ob_ref_local: AtomicU32, // local reference count + #[cfg(Py_GIL_DISABLED)] + pub ob_ref_shared: AtomicIsize, // shared reference count + #[cfg(not(Py_GIL_DISABLED))] pub ob_refcnt: PyObjectObRefcnt, #[cfg(PyPy)] pub ob_pypy_link: Py_ssize_t, pub ob_type: *mut PyTypeObject, } -// skipped _PyObject_CAST +// skipped private _PyObject_CAST #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] pub struct PyVarObject { pub ob_base: PyObject, #[cfg(not(GraalPy))] pub ob_size: Py_ssize_t, + // On GraalPy the field is physically there, but not always populated. We hide it to prevent accidental misuse + #[cfg(GraalPy)] + pub _ob_size_graalpy: Py_ssize_t, } -// skipped _PyVarObject_CAST +// skipped private _PyVarObject_CAST #[inline] +#[cfg(not(all(PyPy, Py_3_10)))] +#[cfg_attr(docsrs, doc(cfg(all())))] pub unsafe fn Py_Is(x: *mut PyObject, y: *mut PyObject) -> c_int { (x == y).into() } -#[inline] -#[cfg(Py_3_12)] -pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { - (*ob).ob_refcnt.ob_refcnt +#[cfg(all(PyPy, Py_3_10))] +#[cfg_attr(docsrs, doc(cfg(all())))] +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPy_Is")] + pub fn Py_Is(x: *mut PyObject, y: *mut PyObject) -> c_int; } +// skipped private _Py_GetThreadLocal_Addr + +// skipped private _Py_ThreadId + +// skipped private _Py_IsOwnedByCurrentThread + #[inline] -#[cfg(not(Py_3_12))] pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { - #[cfg(not(GraalPy))] - return (*ob).ob_refcnt; - #[cfg(GraalPy)] - return _Py_REFCNT(ob); + #[cfg(Py_GIL_DISABLED)] + { + let local = (*ob).ob_ref_local.load(Relaxed); + if local == _Py_IMMORTAL_REFCNT_LOCAL { + return _Py_IMMORTAL_REFCNT; + } + let shared = (*ob).ob_ref_shared.load(Relaxed); + local as Py_ssize_t + Py_ssize_t::from(shared >> _Py_REF_SHARED_SHIFT) + } + + #[cfg(all(not(Py_GIL_DISABLED), Py_3_12))] + { + (*ob).ob_refcnt.ob_refcnt + } + + #[cfg(all(not(Py_GIL_DISABLED), not(Py_3_12), not(GraalPy)))] + { + (*ob).ob_refcnt + } + + #[cfg(all(not(Py_GIL_DISABLED), not(Py_3_12), GraalPy))] + { + _Py_REFCNT(ob) + } } #[inline] @@ -113,8 +192,13 @@ pub unsafe fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject { return _Py_TYPE(ob); } -// PyLong_Type defined in longobject.rs -// PyBool_Type defined in boolobject.rs +#[cfg_attr(windows, link(name = "pythonXY"))] +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyLong_Type")] + pub static mut PyLong_Type: PyTypeObject; + #[cfg_attr(PyPy, link_name = "PyPyBool_Type")] + pub static mut PyBool_Type: PyTypeObject; +} #[inline] pub unsafe fn Py_SIZE(ob: *mut PyObject) -> Py_ssize_t { @@ -128,28 +212,31 @@ pub unsafe fn Py_SIZE(ob: *mut PyObject) -> Py_ssize_t { _Py_SIZE(ob) } +#[inline(always)] +#[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] +unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { + #[cfg(target_pointer_width = "64")] + { + (((*op).ob_refcnt.ob_refcnt as crate::PY_INT32_T) < 0) as c_int + } + + #[cfg(target_pointer_width = "32")] + { + ((*op).ob_refcnt.ob_refcnt == _Py_IMMORTAL_REFCNT) as c_int + } +} + #[inline] pub unsafe fn Py_IS_TYPE(ob: *mut PyObject, tp: *mut PyTypeObject) -> c_int { (Py_TYPE(ob) == tp) as c_int } -#[inline(always)] -#[cfg(all(Py_3_12, target_pointer_width = "64"))] -pub unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { - (((*op).ob_refcnt.ob_refcnt as crate::PY_INT32_T) < 0) as c_int -} +// skipped _Py_SetRefCnt -#[inline(always)] -#[cfg(all(Py_3_12, target_pointer_width = "32"))] -pub unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { - ((*op).ob_refcnt.ob_refcnt == _Py_IMMORTAL_REFCNT) as c_int -} - -// skipped _Py_SET_REFCNT // skipped Py_SET_REFCNT -// skipped _Py_SET_TYPE + // skipped Py_SET_TYPE -// skipped _Py_SET_SIZE + // skipped Py_SET_SIZE pub type unaryfunc = unsafe extern "C" fn(*mut PyObject) -> *mut PyObject; @@ -261,6 +348,14 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyType_GetQualName")] pub fn PyType_GetQualName(arg1: *mut PyTypeObject) -> *mut PyObject; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyType_GetFullyQualifiedName")] + pub fn PyType_GetFullyQualifiedName(arg1: *mut PyTypeObject) -> *mut PyObject; + + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyType_GetModuleName")] + pub fn PyType_GetModuleName(arg1: *mut PyTypeObject) -> *mut PyObject; + #[cfg(Py_3_12)] #[cfg_attr(PyPy, link_name = "PyPyType_FromMetaclass")] pub fn PyType_FromMetaclass( @@ -284,7 +379,7 @@ extern "C" { #[inline] pub unsafe fn PyObject_TypeCheck(ob: *mut PyObject, tp: *mut PyTypeObject) -> c_int { - (Py_TYPE(ob) == tp || PyType_IsSubtype(Py_TYPE(ob), tp) != 0) as c_int + (Py_IS_TYPE(ob, tp) != 0 || PyType_IsSubtype(Py_TYPE(ob), tp) != 0) as c_int } #[cfg_attr(windows, link(name = "pythonXY"))] @@ -341,18 +436,43 @@ extern "C" { arg2: *const c_char, arg3: *mut PyObject, ) -> c_int; + #[cfg(any(Py_3_13, all(PyPy, not(Py_3_11))))] // CPython defined in 3.12 as an inline function in abstract.h + #[cfg_attr(PyPy, link_name = "PyPyObject_DelAttrString")] + pub fn PyObject_DelAttrString(arg1: *mut PyObject, arg2: *const c_char) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyObject_HasAttrString")] pub fn PyObject_HasAttrString(arg1: *mut PyObject, arg2: *const c_char) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyObject_GetAttr")] pub fn PyObject_GetAttr(arg1: *mut PyObject, arg2: *mut PyObject) -> *mut PyObject; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyObject_GetOptionalAttr")] + pub fn PyObject_GetOptionalAttr( + arg1: *mut PyObject, + arg2: *mut PyObject, + arg3: *mut *mut PyObject, + ) -> c_int; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyObject_GetOptionalAttrString")] + pub fn PyObject_GetOptionalAttrString( + arg1: *mut PyObject, + arg2: *const c_char, + arg3: *mut *mut PyObject, + ) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyObject_SetAttr")] pub fn PyObject_SetAttr(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject) -> c_int; + #[cfg(any(Py_3_13, all(PyPy, not(Py_3_11))))] // CPython defined in 3.12 as an inline function in abstract.h + #[cfg_attr(PyPy, link_name = "PyPyObject_DelAttr")] + pub fn PyObject_DelAttr(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyObject_HasAttr")] pub fn PyObject_HasAttr(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyObject_HasAttrWithError")] + pub fn PyObject_HasAttrWithError(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyObject_HasAttrStringWithError")] + pub fn PyObject_HasAttrStringWithError(arg1: *mut PyObject, arg2: *const c_char) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyObject_SelfIter")] pub fn PyObject_SelfIter(arg1: *mut PyObject) -> *mut PyObject; - #[cfg_attr(PyPy, link_name = "PyPyObject_GenericGetAttr")] pub fn PyObject_GenericGetAttr(arg1: *mut PyObject, arg2: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyObject_GenericSetAttr")] @@ -362,7 +482,9 @@ extern "C" { arg3: *mut PyObject, ) -> c_int; #[cfg(not(all(Py_LIMITED_API, not(Py_3_10))))] + #[cfg_attr(PyPy, link_name = "PyPyObject_GenericGetDict")] pub fn PyObject_GenericGetDict(arg1: *mut PyObject, arg2: *mut c_void) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyObject_GenericSetDict")] pub fn PyObject_GenericSetDict( arg1: *mut PyObject, arg2: *mut PyObject, @@ -390,8 +512,8 @@ extern "C" { // Flag bits for printing: pub const Py_PRINT_RAW: c_int = 1; // No string quotes etc. -#[cfg(all(Py_3_12, not(Py_LIMITED_API)))] -pub const _Py_TPFLAGS_STATIC_BUILTIN: c_ulong = 1 << 1; +// skipped because is a private API +// const _Py_TPFLAGS_STATIC_BUILTIN: c_ulong = 1 << 1; #[cfg(all(Py_3_12, not(Py_LIMITED_API)))] pub const Py_TPFLAGS_MANAGED_WEAKREF: c_ulong = 1 << 3; @@ -420,7 +542,7 @@ pub const Py_TPFLAGS_BASETYPE: c_ulong = 1 << 10; /// Set if the type implements the vectorcall protocol (PEP 590) #[cfg(any(Py_3_12, all(Py_3_8, not(Py_LIMITED_API))))] pub const Py_TPFLAGS_HAVE_VECTORCALL: c_ulong = 1 << 11; -// skipped non-limited _Py_TPFLAGS_HAVE_VECTORCALL +// skipped backwards-compatibility alias _Py_TPFLAGS_HAVE_VECTORCALL /// Set if the type is 'ready' -- fully initialized pub const Py_TPFLAGS_READY: c_ulong = 1 << 12; @@ -466,14 +588,14 @@ pub const Py_TPFLAGS_HAVE_VERSION_TAG: c_ulong = 1 << 18; extern "C" { #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] - pub fn _Py_NegativeRefcount(filename: *const c_char, lineno: c_int, op: *mut PyObject); + fn _Py_NegativeRefcount(filename: *const c_char, lineno: c_int, op: *mut PyObject); #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] fn _Py_INCREF_IncRefTotal(); #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] fn _Py_DECREF_DecRefTotal(); #[cfg_attr(PyPy, link_name = "_PyPy_Dealloc")] - pub fn _Py_Dealloc(arg1: *mut PyObject); + fn _Py_Dealloc(arg1: *mut PyObject); #[cfg_attr(PyPy, link_name = "PyPy_IncRef")] #[cfg_attr(GraalPy, link_name = "_Py_IncRef")] @@ -482,54 +604,51 @@ extern "C" { #[cfg_attr(GraalPy, link_name = "_Py_DecRef")] pub fn Py_DecRef(o: *mut PyObject); - #[cfg(Py_3_10)] - #[cfg_attr(PyPy, link_name = "_PyPy_IncRef")] - pub fn _Py_IncRef(o: *mut PyObject); - #[cfg(Py_3_10)] - #[cfg_attr(PyPy, link_name = "_PyPy_DecRef")] - pub fn _Py_DecRef(o: *mut PyObject); + #[cfg(all(Py_3_10, not(PyPy)))] + fn _Py_IncRef(o: *mut PyObject); + #[cfg(all(Py_3_10, not(PyPy)))] + fn _Py_DecRef(o: *mut PyObject); #[cfg(GraalPy)] - pub fn _Py_REFCNT(arg1: *const PyObject) -> Py_ssize_t; + fn _Py_REFCNT(arg1: *const PyObject) -> Py_ssize_t; #[cfg(GraalPy)] - pub fn _Py_TYPE(arg1: *const PyObject) -> *mut PyTypeObject; + fn _Py_TYPE(arg1: *const PyObject) -> *mut PyTypeObject; #[cfg(GraalPy)] - pub fn _Py_SIZE(arg1: *const PyObject) -> Py_ssize_t; + fn _Py_SIZE(arg1: *const PyObject) -> Py_ssize_t; } #[inline(always)] pub unsafe fn Py_INCREF(op: *mut PyObject) { + // On limited API, the free-threaded build, or with refcount debugging, let the interpreter do refcounting + // TODO: reimplement the logic in the header in the free-threaded build, for a little bit of performance. #[cfg(any( - GraalPy, - all(Py_LIMITED_API, Py_3_12), - all( - py_sys_config = "Py_REF_DEBUG", - Py_3_10, - not(all(Py_3_12, not(Py_LIMITED_API))) - ) + Py_GIL_DISABLED, + Py_LIMITED_API, + py_sys_config = "Py_REF_DEBUG", + GraalPy ))] { - _Py_IncRef(op); - } + // _Py_IncRef was added to the ABI in 3.10; skips null checks + #[cfg(all(Py_3_10, not(PyPy)))] + { + _Py_IncRef(op); + } - #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_3_10)))] - { - return Py_IncRef(op); + #[cfg(any(not(Py_3_10), PyPy))] + { + Py_IncRef(op); + } } - #[cfg(any( - all(Py_LIMITED_API, not(Py_3_12)), - all( - not(Py_LIMITED_API), - not(GraalPy), - any( - not(py_sys_config = "Py_REF_DEBUG"), - all(py_sys_config = "Py_REF_DEBUG", Py_3_12), - ) - ), - ))] + // version-specific builds are allowed to directly manipulate the reference count + #[cfg(not(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + py_sys_config = "Py_REF_DEBUG", + GraalPy + )))] { #[cfg(all(Py_3_12, target_pointer_width = "64"))] { @@ -556,9 +675,6 @@ pub unsafe fn Py_INCREF(op: *mut PyObject) { // Skipped _Py_INCREF_STAT_INC - if anyone wants this, please file an issue // or submit a PR supporting Py_STATS build option and pystats.h - - #[cfg(all(py_sys_config = "Py_REF_DEBUG", Py_3_12))] - _Py_INCREF_IncRefTotal(); } } @@ -568,35 +684,34 @@ pub unsafe fn Py_INCREF(op: *mut PyObject) { track_caller )] pub unsafe fn Py_DECREF(op: *mut PyObject) { + // On limited API, the free-threaded build, or with refcount debugging, let the interpreter do refcounting + // On 3.12+ we implement refcount debugging to get better assertion locations on negative refcounts + // TODO: reimplement the logic in the header in the free-threaded build, for a little bit of performance. #[cfg(any( - GraalPy, - all(Py_LIMITED_API, Py_3_12), - all( - py_sys_config = "Py_REF_DEBUG", - Py_3_10, - not(all(Py_3_12, not(Py_LIMITED_API))) - ) + Py_GIL_DISABLED, + Py_LIMITED_API, + all(py_sys_config = "Py_REF_DEBUG", not(Py_3_12)), + GraalPy ))] { - _Py_DecRef(op); - } + // _Py_DecRef was added to the ABI in 3.10; skips null checks + #[cfg(all(Py_3_10, not(PyPy)))] + { + _Py_DecRef(op); + } - #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_3_10)))] - { - return Py_DecRef(op); + #[cfg(any(not(Py_3_10), PyPy))] + { + Py_DecRef(op); + } } - #[cfg(any( - all(Py_LIMITED_API, not(Py_3_12)), - all( - not(Py_LIMITED_API), - not(GraalPy), - any( - not(py_sys_config = "Py_REF_DEBUG"), - all(py_sys_config = "Py_REF_DEBUG", Py_3_12), - ) - ), - ))] + #[cfg(not(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + all(py_sys_config = "Py_REF_DEBUG", not(Py_3_12)), + GraalPy + )))] { #[cfg(Py_3_12)] if _Py_IsImmortal(op) != 0 { @@ -606,7 +721,7 @@ pub unsafe fn Py_DECREF(op: *mut PyObject) { // Skipped _Py_DECREF_STAT_INC - if anyone needs this, please file an issue // or submit a PR supporting Py_STATS build option and pystats.h - #[cfg(all(py_sys_config = "Py_REF_DEBUG", Py_3_12))] + #[cfg(py_sys_config = "Py_REF_DEBUG")] _Py_DECREF_DecRefTotal(); #[cfg(Py_3_12)] @@ -660,43 +775,66 @@ pub unsafe fn Py_XDECREF(op: *mut PyObject) { } extern "C" { - #[cfg(all(Py_3_10, Py_LIMITED_API))] + #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))] + #[cfg_attr(docsrs, doc(cfg(Py_3_10)))] pub fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject; - #[cfg(all(Py_3_10, Py_LIMITED_API))] + #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))] + #[cfg_attr(docsrs, doc(cfg(Py_3_10)))] pub fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject; } -// Technically these macros are only available in the C header from 3.10 and up, however their -// implementation works on all supported Python versions so we define these macros on all -// versions for simplicity. +// macro _Py_NewRef not public; reimplemented directly inside Py_NewRef here +// macro _Py_XNewRef not public; reimplemented directly inside Py_XNewRef here +#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))] +#[cfg_attr(docsrs, doc(cfg(Py_3_10)))] #[inline] -pub unsafe fn _Py_NewRef(obj: *mut PyObject) -> *mut PyObject { +pub unsafe fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject { Py_INCREF(obj); obj } +#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))] +#[cfg_attr(docsrs, doc(cfg(Py_3_10)))] #[inline] -pub unsafe fn _Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { +pub unsafe fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { Py_XINCREF(obj); obj } -#[cfg(all(Py_3_10, not(Py_LIMITED_API)))] -#[inline] -pub unsafe fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject { - _Py_NewRef(obj) -} +#[cfg(Py_3_13)] +pub const Py_CONSTANT_NONE: c_uint = 0; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_FALSE: c_uint = 1; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_TRUE: c_uint = 2; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_ELLIPSIS: c_uint = 3; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_NOT_IMPLEMENTED: c_uint = 4; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_ZERO: c_uint = 5; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_ONE: c_uint = 6; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_EMPTY_STR: c_uint = 7; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_EMPTY_BYTES: c_uint = 8; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_EMPTY_TUPLE: c_uint = 9; -#[cfg(all(Py_3_10, not(Py_LIMITED_API)))] -#[inline] -pub unsafe fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { - _Py_XNewRef(obj) +extern "C" { + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPy_GetConstant")] + pub fn Py_GetConstant(constant_id: c_uint) -> *mut PyObject; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPy_GetConstantBorrowed")] + pub fn Py_GetConstantBorrowed(constant_id: c_uint) -> *mut PyObject; } #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { - #[cfg(not(GraalPy))] + #[cfg(all(not(GraalPy), not(all(Py_3_13, Py_LIMITED_API))))] #[cfg_attr(PyPy, link_name = "_PyPy_NoneStruct")] static mut _Py_NoneStruct: PyObject; @@ -706,8 +844,12 @@ extern "C" { #[inline] pub unsafe fn Py_None() -> *mut PyObject { - #[cfg(not(GraalPy))] + #[cfg(all(not(GraalPy), all(Py_3_13, Py_LIMITED_API)))] + return Py_GetConstantBorrowed(Py_CONSTANT_NONE); + + #[cfg(all(not(GraalPy), not(all(Py_3_13, Py_LIMITED_API))))] return ptr::addr_of_mut!(_Py_NoneStruct); + #[cfg(GraalPy)] return _Py_NoneStructReference; } @@ -721,7 +863,7 @@ pub unsafe fn Py_IsNone(x: *mut PyObject) -> c_int { #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { - #[cfg(not(GraalPy))] + #[cfg(all(not(GraalPy), not(all(Py_3_13, Py_LIMITED_API))))] #[cfg_attr(PyPy, link_name = "_PyPy_NotImplementedStruct")] static mut _Py_NotImplementedStruct: PyObject; @@ -731,8 +873,12 @@ extern "C" { #[inline] pub unsafe fn Py_NotImplemented() -> *mut PyObject { - #[cfg(not(GraalPy))] + #[cfg(all(not(GraalPy), all(Py_3_13, Py_LIMITED_API)))] + return Py_GetConstantBorrowed(Py_CONSTANT_NOT_IMPLEMENTED); + + #[cfg(all(not(GraalPy), not(all(Py_3_13, Py_LIMITED_API))))] return ptr::addr_of_mut!(_Py_NotImplementedStruct); + #[cfg(GraalPy)] return _Py_NotImplementedStructReference; } @@ -759,15 +905,17 @@ pub enum PySendResult { // skipped Py_RETURN_RICHCOMPARE #[inline] -#[cfg(Py_LIMITED_API)] -pub unsafe fn PyType_HasFeature(t: *mut PyTypeObject, f: c_ulong) -> c_int { - ((PyType_GetFlags(t) & f) != 0) as c_int -} +pub unsafe fn PyType_HasFeature(ty: *mut PyTypeObject, feature: c_ulong) -> c_int { + #[cfg(Py_LIMITED_API)] + let flags = PyType_GetFlags(ty); -#[inline] -#[cfg(not(Py_LIMITED_API))] -pub unsafe fn PyType_HasFeature(t: *mut PyTypeObject, f: c_ulong) -> c_int { - (((*t).tp_flags & f) != 0) as c_int + #[cfg(all(not(Py_LIMITED_API), Py_GIL_DISABLED))] + let flags = (*ty).tp_flags.load(std::sync::atomic::Ordering::Relaxed); + + #[cfg(all(not(Py_LIMITED_API), not(Py_GIL_DISABLED)))] + let flags = (*ty).tp_flags; + + ((flags & feature) != 0) as c_int } #[inline] @@ -780,7 +928,18 @@ pub unsafe fn PyType_Check(op: *mut PyObject) -> c_int { PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS) } +// skipped _PyType_CAST + #[inline] pub unsafe fn PyType_CheckExact(op: *mut PyObject) -> c_int { Py_IS_TYPE(op, ptr::addr_of_mut!(PyType_Type)) } + +extern "C" { + #[cfg(any(Py_3_13, all(Py_3_11, not(Py_LIMITED_API))))] + #[cfg_attr(PyPy, link_name = "PyPyType_GetModuleByDef")] + pub fn PyType_GetModuleByDef( + arg1: *mut crate::PyTypeObject, + arg2: *mut crate::PyModuleDef, + ) -> *mut PyObject; +} diff --git a/pyo3-ffi/src/pybuffer.rs b/pyo3-ffi/src/pybuffer.rs index 50bf4e6109c..de7067599ff 100644 --- a/pyo3-ffi/src/pybuffer.rs +++ b/pyo3-ffi/src/pybuffer.rs @@ -103,7 +103,11 @@ extern "C" { } /// Maximum number of dimensions -pub const PyBUF_MAX_NDIM: c_int = if cfg!(PyPy) { 36 } else { 64 }; +pub const PyBUF_MAX_NDIM: usize = if cfg!(all(PyPy, not(Py_3_11))) { + 36 +} else { + 64 +}; /* Flags for getting buffers */ pub const PyBUF_SIMPLE: c_int = 0; diff --git a/pyo3-ffi/src/pyerrors.rs b/pyo3-ffi/src/pyerrors.rs index 9da00ea390e..d341239a07b 100644 --- a/pyo3-ffi/src/pyerrors.rs +++ b/pyo3-ffi/src/pyerrors.rs @@ -101,7 +101,7 @@ pub unsafe fn PyUnicodeDecodeError_Create( ) -> *mut PyObject { crate::_PyObject_CallFunction_SizeT( PyExc_UnicodeDecodeError, - b"sy#nns\0".as_ptr().cast::(), + c_str!("sy#nns").as_ptr(), encoding, object, length, @@ -116,6 +116,7 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyExc_BaseException")] pub static mut PyExc_BaseException: *mut PyObject; #[cfg(Py_3_11)] + #[cfg_attr(PyPy, link_name = "PyPyExc_BaseExceptionGroup")] pub static mut PyExc_BaseExceptionGroup: *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyExc_Exception")] pub static mut PyExc_Exception: *mut PyObject; diff --git a/pyo3-ffi/src/pyhash.rs b/pyo3-ffi/src/pyhash.rs index f42f9730f1b..4f14e04a695 100644 --- a/pyo3-ffi/src/pyhash.rs +++ b/pyo3-ffi/src/pyhash.rs @@ -1,7 +1,9 @@ -#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy)))] use crate::pyport::{Py_hash_t, Py_ssize_t}; #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] -use std::os::raw::{c_char, c_void}; +use std::os::raw::c_char; +#[cfg(not(any(Py_LIMITED_API, PyPy)))] +use std::os::raw::c_void; use std::os::raw::{c_int, c_ulong}; @@ -10,7 +12,7 @@ extern "C" { // skipped non-limited _Py_HashPointer // skipped non-limited _Py_HashPointerRaw - #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy)))] pub fn _Py_HashBytes(src: *const c_void, len: Py_ssize_t) -> Py_hash_t; } diff --git a/pyo3-ffi/src/pylifecycle.rs b/pyo3-ffi/src/pylifecycle.rs index 7f73e3f0e9b..3f051c54f7c 100644 --- a/pyo3-ffi/src/pylifecycle.rs +++ b/pyo3-ffi/src/pylifecycle.rs @@ -23,18 +23,55 @@ extern "C" { pub fn Py_Main(argc: c_int, argv: *mut *mut wchar_t) -> c_int; pub fn Py_BytesMain(argc: c_int, argv: *mut *mut c_char) -> c_int; + #[cfg_attr( + Py_3_11, + deprecated(note = "Deprecated since Python 3.11. Use `PyConfig.program_name` instead.") + )] pub fn Py_SetProgramName(arg1: *const wchar_t); #[cfg_attr(PyPy, link_name = "PyPy_GetProgramName")] + #[cfg_attr( + Py_3_13, + deprecated(note = "Deprecated since Python 3.13. Use `sys.executable` instead.") + )] pub fn Py_GetProgramName() -> *mut wchar_t; + #[cfg_attr( + Py_3_11, + deprecated(note = "Deprecated since Python 3.11. Use `PyConfig.home` instead.") + )] pub fn Py_SetPythonHome(arg1: *const wchar_t); + #[cfg_attr( + Py_3_13, + deprecated( + note = "Deprecated since Python 3.13. Use `PyConfig.home` or the value of the `PYTHONHOME` environment variable instead." + ) + )] pub fn Py_GetPythonHome() -> *mut wchar_t; - + #[cfg_attr( + Py_3_13, + deprecated(note = "Deprecated since Python 3.13. Use `sys.executable` instead.") + )] pub fn Py_GetProgramFullPath() -> *mut wchar_t; - + #[cfg_attr( + Py_3_13, + deprecated(note = "Deprecated since Python 3.13. Use `sys.prefix` instead.") + )] pub fn Py_GetPrefix() -> *mut wchar_t; + #[cfg_attr( + Py_3_13, + deprecated(note = "Deprecated since Python 3.13. Use `sys.exec_prefix` instead.") + )] pub fn Py_GetExecPrefix() -> *mut wchar_t; + #[cfg_attr( + Py_3_13, + deprecated(note = "Deprecated since Python 3.13. Use `sys.path` instead.") + )] pub fn Py_GetPath() -> *mut wchar_t; + #[cfg(not(Py_3_13))] + #[cfg_attr( + Py_3_11, + deprecated(note = "Deprecated since Python 3.11. Use `sys.path` instead.") + )] pub fn Py_SetPath(arg1: *const wchar_t); // skipped _Py_CheckPython3 diff --git a/pyo3-ffi/src/pyport.rs b/pyo3-ffi/src/pyport.rs index 741b0db7bf8..a144c67fb1b 100644 --- a/pyo3-ffi/src/pyport.rs +++ b/pyo3-ffi/src/pyport.rs @@ -11,8 +11,8 @@ pub type Py_ssize_t = ::libc::ssize_t; pub type Py_hash_t = Py_ssize_t; pub type Py_uhash_t = ::libc::size_t; -pub const PY_SSIZE_T_MIN: Py_ssize_t = std::isize::MIN as Py_ssize_t; -pub const PY_SSIZE_T_MAX: Py_ssize_t = std::isize::MAX as Py_ssize_t; +pub const PY_SSIZE_T_MIN: Py_ssize_t = isize::MIN as Py_ssize_t; +pub const PY_SSIZE_T_MAX: Py_ssize_t = isize::MAX as Py_ssize_t; #[cfg(target_endian = "big")] pub const PY_BIG_ENDIAN: usize = 1; diff --git a/pyo3-ffi/src/pystate.rs b/pyo3-ffi/src/pystate.rs index d2fd39e497d..a6caf421ff6 100644 --- a/pyo3-ffi/src/pystate.rs +++ b/pyo3-ffi/src/pystate.rs @@ -1,4 +1,5 @@ -#[cfg(any(not(PyPy), Py_3_9))] +#[cfg(all(Py_3_10, not(PyPy), not(Py_LIMITED_API)))] +use crate::frameobject::PyFrameObject; use crate::moduleobject::PyModuleDef; use crate::object::PyObject; use std::os::raw::c_int; @@ -28,15 +29,12 @@ extern "C" { #[cfg(not(PyPy))] pub fn PyInterpreterState_GetID(arg1: *mut PyInterpreterState) -> i64; - #[cfg(any(not(PyPy), Py_3_9))] // only on PyPy since 3.9 #[cfg_attr(PyPy, link_name = "PyPyState_AddModule")] pub fn PyState_AddModule(arg1: *mut PyObject, arg2: *mut PyModuleDef) -> c_int; - #[cfg(any(not(PyPy), Py_3_9))] // only on PyPy since 3.9 #[cfg_attr(PyPy, link_name = "PyPyState_RemoveModule")] pub fn PyState_RemoveModule(arg1: *mut PyModuleDef) -> c_int; - #[cfg(any(not(PyPy), Py_3_9))] // only on PyPy since 3.9 // only has PyPy prefix since 3.10 #[cfg_attr(all(PyPy, Py_3_10), link_name = "PyPyState_FindModule")] pub fn PyState_FindModule(arg1: *mut PyModuleDef) -> *mut PyObject; @@ -67,9 +65,14 @@ extern "C" { } // skipped non-limited / 3.9 PyThreadState_GetInterpreter -// skipped non-limited / 3.9 PyThreadState_GetFrame // skipped non-limited / 3.9 PyThreadState_GetID +extern "C" { + // PyThreadState_GetFrame + #[cfg(all(Py_3_10, not(PyPy), not(Py_LIMITED_API)))] + pub fn PyThreadState_GetFrame(arg1: *mut PyThreadState) -> *mut PyFrameObject; +} + #[repr(C)] #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum PyGILState_STATE { @@ -77,9 +80,73 @@ pub enum PyGILState_STATE { PyGILState_UNLOCKED, } +struct HangThread; + +impl Drop for HangThread { + fn drop(&mut self) { + loop { + #[cfg(target_family = "unix")] + unsafe { + libc::pause(); + } + #[cfg(not(target_family = "unix"))] + std::thread::sleep(std::time::Duration::from_secs(9_999_999)); + } + } +} + +// The PyGILState_Ensure function will call pthread_exit during interpreter shutdown, +// which causes undefined behavior. Redirect to the "safe" version that hangs instead, +// as Python 3.14 does. +// +// See https://github.com/rust-lang/rust/issues/135929 + +// C-unwind only supported (and necessary) since 1.71. Python 3.14+ does not do +// pthread_exit from PyGILState_Ensure (https://github.com/python/cpython/issues/87135). +mod raw { + #[cfg(all(not(Py_3_14), rustc_has_extern_c_unwind))] + extern "C-unwind" { + #[cfg_attr(PyPy, link_name = "PyPyGILState_Ensure")] + pub fn PyGILState_Ensure() -> super::PyGILState_STATE; + } + + #[cfg(not(all(not(Py_3_14), rustc_has_extern_c_unwind)))] + extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyGILState_Ensure")] + pub fn PyGILState_Ensure() -> super::PyGILState_STATE; + } +} + +#[cfg(not(Py_3_14))] +pub unsafe extern "C" fn PyGILState_Ensure() -> PyGILState_STATE { + let guard = HangThread; + // If `PyGILState_Ensure` calls `pthread_exit`, which it does on Python < 3.14 + // when the interpreter is shutting down, this will cause a forced unwind. + // doing a forced unwind through a function with a Rust destructor is unspecified + // behavior. + // + // However, currently it runs the destructor, which will cause the thread to + // hang as it should. + // + // And if we don't catch the unwinding here, then one of our callers probably has a destructor, + // so it's unspecified behavior anyway, and on many configurations causes the process to abort. + // + // The alternative is for pyo3 to contain custom C or C++ code that catches the `pthread_exit`, + // but that's also annoying from a portability point of view. + // + // On Windows, `PyGILState_Ensure` calls `_endthreadex` instead, which AFAICT can't be caught + // and therefore will cause unsafety if there are pinned objects on the stack. AFAICT there's + // nothing we can do it other than waiting for Python 3.14 or not using Windows. At least, + // if there is nothing pinned on the stack, it won't cause the process to crash. + let ret: PyGILState_STATE = raw::PyGILState_Ensure(); + std::mem::forget(guard); + ret +} + +#[cfg(Py_3_14)] +pub use self::raw::PyGILState_Ensure; + extern "C" { - #[cfg_attr(PyPy, link_name = "PyPyGILState_Ensure")] - pub fn PyGILState_Ensure() -> PyGILState_STATE; #[cfg_attr(PyPy, link_name = "PyPyGILState_Release")] pub fn PyGILState_Release(arg1: PyGILState_STATE); #[cfg(not(PyPy))] diff --git a/pyo3-ffi/src/pythonrun.rs b/pyo3-ffi/src/pythonrun.rs index 10985b6068c..e7ea2d2efd0 100644 --- a/pyo3-ffi/src/pythonrun.rs +++ b/pyo3-ffi/src/pythonrun.rs @@ -1,7 +1,7 @@ use crate::object::*; #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] use libc::FILE; -#[cfg(all(not(PyPy), any(Py_LIMITED_API, not(Py_3_10), GraalPy)))] +#[cfg(any(Py_LIMITED_API, not(Py_3_10), PyPy, GraalPy))] use std::os::raw::c_char; use std::os::raw::c_int; @@ -20,6 +20,28 @@ extern "C" { pub fn PyErr_DisplayException(exc: *mut PyObject); } +#[inline] +#[cfg(PyPy)] +pub unsafe fn Py_CompileString(string: *const c_char, p: *const c_char, s: c_int) -> *mut PyObject { + // PyPy's implementation of Py_CompileString always forwards to Py_CompileStringFlags; this + // is only available in the non-limited API and has a real definition for all versions in + // the cpython/ subdirectory. + #[cfg(Py_LIMITED_API)] + extern "C" { + #[link_name = "PyPy_CompileStringFlags"] + pub fn Py_CompileStringFlags( + string: *const c_char, + p: *const c_char, + s: c_int, + f: *mut std::os::raw::c_void, // Actually *mut Py_CompilerFlags in the real definition + ) -> *mut PyObject; + } + #[cfg(not(Py_LIMITED_API))] + use crate::Py_CompileStringFlags; + + Py_CompileStringFlags(string, p, s, std::ptr::null_mut()) +} + // skipped PyOS_InputHook pub const PYOS_STACK_MARGIN: c_int = 2048; diff --git a/pyo3-ffi/src/sysmodule.rs b/pyo3-ffi/src/sysmodule.rs index 3c552254244..6f402197ece 100644 --- a/pyo3-ffi/src/sysmodule.rs +++ b/pyo3-ffi/src/sysmodule.rs @@ -8,7 +8,19 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPySys_SetObject")] pub fn PySys_SetObject(arg1: *const c_char, arg2: *mut PyObject) -> c_int; + #[cfg_attr( + Py_3_11, + deprecated( + note = "Deprecated in Python 3.11, use `PyConfig.argv` and `PyConfig.parse_argv` instead" + ) + )] pub fn PySys_SetArgv(arg1: c_int, arg2: *mut *mut wchar_t); + #[cfg_attr( + Py_3_11, + deprecated( + note = "Deprecated in Python 3.11, use `PyConfig.argv` and `PyConfig.parse_argv` instead" + ) + )] pub fn PySys_SetArgvEx(arg1: c_int, arg2: *mut *mut wchar_t, arg3: c_int); pub fn PySys_SetPath(arg1: *const wchar_t); @@ -19,6 +31,12 @@ extern "C" { pub fn PySys_FormatStdout(format: *const c_char, ...); pub fn PySys_FormatStderr(format: *const c_char, ...); + #[cfg_attr( + Py_3_13, + deprecated( + note = "Deprecated since Python 3.13. Clear sys.warnoptions and warnings.filters instead." + ) + )] pub fn PySys_ResetWarnOptions(); #[cfg_attr(Py_3_11, deprecated(note = "Python 3.11"))] pub fn PySys_AddWarnOption(arg1: *const wchar_t); diff --git a/pyo3-ffi/src/unicodeobject.rs b/pyo3-ffi/src/unicodeobject.rs index 087160a1efc..1e0425ce2a2 100644 --- a/pyo3-ffi/src/unicodeobject.rs +++ b/pyo3-ffi/src/unicodeobject.rs @@ -6,6 +6,10 @@ use std::os::raw::{c_char, c_int, c_void}; use std::ptr::addr_of_mut; #[cfg(not(Py_LIMITED_API))] +#[cfg_attr( + Py_3_13, + deprecated(note = "Deprecated since Python 3.13. Use `libc::wchar_t` instead.") +)] pub type Py_UNICODE = wchar_t; pub type Py_UCS4 = u32; @@ -328,6 +332,15 @@ extern "C" { pub fn PyUnicode_Compare(left: *mut PyObject, right: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyUnicode_CompareWithASCIIString")] pub fn PyUnicode_CompareWithASCIIString(left: *mut PyObject, right: *const c_char) -> c_int; + #[cfg(Py_3_13)] + pub fn PyUnicode_EqualToUTF8(unicode: *mut PyObject, string: *const c_char) -> c_int; + #[cfg(Py_3_13)] + pub fn PyUnicode_EqualToUTF8AndSize( + unicode: *mut PyObject, + string: *const c_char, + size: Py_ssize_t, + ) -> c_int; + pub fn PyUnicode_RichCompare( left: *mut PyObject, right: *mut PyObject, diff --git a/pyo3-ffi/src/weakrefobject.rs b/pyo3-ffi/src/weakrefobject.rs index 7e11a9012e7..305dc290fa8 100644 --- a/pyo3-ffi/src/weakrefobject.rs +++ b/pyo3-ffi/src/weakrefobject.rs @@ -58,5 +58,12 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyWeakref_NewProxy")] pub fn PyWeakref_NewProxy(ob: *mut PyObject, callback: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyWeakref_GetObject")] - pub fn PyWeakref_GetObject(_ref: *mut PyObject) -> *mut PyObject; + #[cfg_attr( + Py_3_13, + deprecated(note = "deprecated since Python 3.13. Use `PyWeakref_GetRef` instead.") + )] + pub fn PyWeakref_GetObject(reference: *mut PyObject) -> *mut PyObject; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyWeakref_GetRef")] + pub fn PyWeakref_GetRef(reference: *mut PyObject, pobj: *mut *mut PyObject) -> c_int; } diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 6ca4eeade8c..f3de7abf71f 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.21.1" +version = "0.24.1" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -9,21 +9,25 @@ repository = "https://github.com/pyo3/pyo3" categories = ["api-bindings", "development-tools::ffi"] license = "MIT OR Apache-2.0" edition = "2021" +rust-version = "1.63" # Note: we use default-features = false for proc-macro related crates # not to depend on proc-macro itself. # See https://github.com/PyO3/pyo3/pull/810 for more. [dependencies] -heck = "0.4" -proc-macro2 = { version = "1", default-features = false } -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.1", features = ["resolve-config"] } +heck = "0.5" +proc-macro2 = { version = "1.0.60", default-features = false } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.24.1", features = ["resolve-config"] } quote = { version = "1", default-features = false } [dependencies.syn] -version = "2" +version = "2.0.59" # for `LitCStr` default-features = false features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"] +[build-dependencies] +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.24.1" } + [lints] workspace = true diff --git a/pyo3-macros-backend/build.rs b/pyo3-macros-backend/build.rs new file mode 100644 index 00000000000..55aa0ba03c5 --- /dev/null +++ b/pyo3-macros-backend/build.rs @@ -0,0 +1,4 @@ +fn main() { + pyo3_build_config::print_expected_cfgs(); + pyo3_build_config::print_feature_cfgs(); +} diff --git a/pyo3-macros-backend/src/attributes.rs b/pyo3-macros-backend/src/attributes.rs old mode 100644 new mode 100755 index e91b3b8d9a2..19a12801065 --- a/pyo3-macros-backend/src/attributes.rs +++ b/pyo3-macros-backend/src/attributes.rs @@ -1,40 +1,170 @@ use proc_macro2::TokenStream; -use quote::ToTokens; +use quote::{quote, ToTokens}; +use syn::parse::Parser; use syn::{ + ext::IdentExt, parse::{Parse, ParseStream}, punctuated::Punctuated, spanned::Spanned, token::Comma, - Attribute, Expr, ExprPath, Ident, LitStr, Path, Result, Token, + Attribute, Expr, ExprPath, Ident, Index, LitBool, LitStr, Member, Path, Result, Token, }; pub mod kw { syn::custom_keyword!(annotation); syn::custom_keyword!(attribute); syn::custom_keyword!(cancel_handle); + syn::custom_keyword!(constructor); syn::custom_keyword!(dict); + syn::custom_keyword!(eq); + syn::custom_keyword!(eq_int); syn::custom_keyword!(extends); syn::custom_keyword!(freelist); syn::custom_keyword!(from_py_with); syn::custom_keyword!(frozen); syn::custom_keyword!(get); syn::custom_keyword!(get_all); + syn::custom_keyword!(hash); + syn::custom_keyword!(into_py_with); syn::custom_keyword!(item); syn::custom_keyword!(from_item_all); syn::custom_keyword!(mapping); syn::custom_keyword!(module); syn::custom_keyword!(name); + syn::custom_keyword!(ord); syn::custom_keyword!(pass_module); syn::custom_keyword!(rename_all); syn::custom_keyword!(sequence); syn::custom_keyword!(set); syn::custom_keyword!(set_all); syn::custom_keyword!(signature); + syn::custom_keyword!(str); syn::custom_keyword!(subclass); + syn::custom_keyword!(submodule); syn::custom_keyword!(text_signature); syn::custom_keyword!(transparent); syn::custom_keyword!(unsendable); syn::custom_keyword!(weakref); + syn::custom_keyword!(gil_used); +} + +fn take_int(read: &mut &str, tracker: &mut usize) -> String { + let mut int = String::new(); + for (i, ch) in read.char_indices() { + match ch { + '0'..='9' => { + *tracker += 1; + int.push(ch) + } + _ => { + *read = &read[i..]; + break; + } + } + } + int +} + +fn take_ident(read: &mut &str, tracker: &mut usize) -> Ident { + let mut ident = String::new(); + if read.starts_with("r#") { + ident.push_str("r#"); + *tracker += 2; + *read = &read[2..]; + } + for (i, ch) in read.char_indices() { + match ch { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '_' => { + *tracker += 1; + ident.push(ch) + } + _ => { + *read = &read[i..]; + break; + } + } + } + Ident::parse_any.parse_str(&ident).unwrap() +} + +// shorthand parsing logic inspiration taken from https://github.com/dtolnay/thiserror/blob/master/impl/src/fmt.rs +fn parse_shorthand_format(fmt: LitStr) -> Result<(LitStr, Vec)> { + let span = fmt.span(); + let token = fmt.token(); + let value = fmt.value(); + let mut read = value.as_str(); + let mut out = String::new(); + let mut members = Vec::new(); + let mut tracker = 1; + while let Some(brace) = read.find('{') { + tracker += brace; + out += &read[..brace + 1]; + read = &read[brace + 1..]; + if read.starts_with('{') { + out.push('{'); + read = &read[1..]; + tracker += 2; + continue; + } + let next = match read.chars().next() { + Some(next) => next, + None => break, + }; + tracker += 1; + let member = match next { + '0'..='9' => { + let start = tracker; + let index = take_int(&mut read, &mut tracker).parse::().unwrap(); + let end = tracker; + let subspan = token.subspan(start..end).unwrap_or(span); + let idx = Index { + index, + span: subspan, + }; + Member::Unnamed(idx) + } + 'a'..='z' | 'A'..='Z' | '_' => { + let start = tracker; + let mut ident = take_ident(&mut read, &mut tracker); + let end = tracker; + let subspan = token.subspan(start..end).unwrap_or(span); + ident.set_span(subspan); + Member::Named(ident) + } + '}' | ':' => { + let start = tracker; + tracker += 1; + let end = tracker; + let subspan = token.subspan(start..end).unwrap_or(span); + // we found a closing bracket or formatting ':' without finding a member, we assume the user wants the instance formatted here + bail_spanned!(subspan.span() => "No member found, you must provide a named or positionally specified member.") + } + _ => continue, + }; + members.push(member); + } + out += read; + Ok((LitStr::new(&out, span), members)) +} + +#[derive(Clone, Debug)] +pub struct StringFormatter { + pub fmt: LitStr, + pub args: Vec, +} + +impl Parse for crate::attributes::StringFormatter { + fn parse(input: ParseStream<'_>) -> Result { + let (fmt, args) = parse_shorthand_format(input.parse()?)?; + Ok(Self { fmt, args }) + } +} + +impl ToTokens for crate::attributes::StringFormatter { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.fmt.to_tokens(tokens); + tokens.extend(quote! {self.args}) + } } #[derive(Clone, Debug)] @@ -43,6 +173,12 @@ pub struct KeywordAttribute { pub value: V, } +#[derive(Clone, Debug)] +pub struct OptionalKeywordAttribute { + pub kw: K, + pub value: Option, +} + /// A helper type which parses the inner type via a literal string /// e.g. `LitStrValue` -> parses "some::path" in quotes. #[derive(Clone, Debug, PartialEq, Eq)] @@ -68,7 +204,7 @@ pub struct NameLitStr(pub Ident); impl Parse for NameLitStr { fn parse(input: ParseStream<'_>) -> Result { let string_literal: LitStr = input.parse()?; - if let Ok(ident) = string_literal.parse() { + if let Ok(ident) = string_literal.parse_with(Ident::parse_any) { Ok(NameLitStr(ident)) } else { bail_spanned!(string_literal.span() => "expected a single identifier in double quotes") @@ -171,7 +307,10 @@ pub type FreelistAttribute = KeywordAttribute>; pub type ModuleAttribute = KeywordAttribute; pub type NameAttribute = KeywordAttribute; pub type RenameAllAttribute = KeywordAttribute; +pub type StrFormatterAttribute = OptionalKeywordAttribute; pub type TextSignatureAttribute = KeywordAttribute; +pub type SubmoduleAttribute = kw::submodule; +pub type GILUsedAttribute = KeywordAttribute; impl Parse for KeywordAttribute { fn parse(input: ParseStream<'_>) -> Result { @@ -190,7 +329,61 @@ impl ToTokens for KeywordAttribute { } } -pub type FromPyWithAttribute = KeywordAttribute>; +impl Parse for OptionalKeywordAttribute { + fn parse(input: ParseStream<'_>) -> Result { + let kw: K = input.parse()?; + let value = match input.parse::() { + Ok(_) => Some(input.parse()?), + Err(_) => None, + }; + Ok(OptionalKeywordAttribute { kw, value }) + } +} + +impl ToTokens for OptionalKeywordAttribute { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.kw.to_tokens(tokens); + if self.value.is_some() { + Token![=](self.kw.span()).to_tokens(tokens); + self.value.to_tokens(tokens); + } + } +} + +#[derive(Debug, Clone)] +pub struct ExprPathWrap { + pub from_lit_str: bool, + pub expr_path: ExprPath, +} + +impl Parse for ExprPathWrap { + fn parse(input: ParseStream<'_>) -> Result { + match input.parse::() { + Ok(expr_path) => Ok(ExprPathWrap { + from_lit_str: false, + expr_path, + }), + Err(e) => match input.parse::>() { + Ok(LitStrValue(expr_path)) => Ok(ExprPathWrap { + from_lit_str: true, + expr_path, + }), + Err(_) => Err(e), + }, + } + } +} + +impl ToTokens for ExprPathWrap { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.expr_path.to_tokens(tokens) + } +} + +pub type FromPyWithAttribute = KeywordAttribute; +pub type IntoPyWithAttribute = KeywordAttribute; + +pub type DefaultAttribute = OptionalKeywordAttribute; /// For specifying the path to the pyo3 crate. pub type CrateAttribute = KeywordAttribute>; @@ -227,13 +420,41 @@ pub fn take_attributes( pub fn take_pyo3_options(attrs: &mut Vec) -> Result> { let mut out = Vec::new(); - take_attributes(attrs, |attr| { - if let Some(options) = get_pyo3_options(attr)? { - out.extend(options); + let mut all_errors = ErrorCombiner(None); + take_attributes(attrs, |attr| match get_pyo3_options(attr) { + Ok(result) => { + if let Some(options) = result { + out.extend(options); + Ok(true) + } else { + Ok(false) + } + } + Err(err) => { + all_errors.combine(err); Ok(true) - } else { - Ok(false) } })?; + all_errors.ensure_empty()?; Ok(out) } + +pub struct ErrorCombiner(pub Option); + +impl ErrorCombiner { + pub fn combine(&mut self, error: syn::Error) { + if let Some(existing) = &mut self.0 { + existing.combine(error); + } else { + self.0 = Some(error); + } + } + + pub fn ensure_empty(self) -> Result<()> { + if let Some(error) = self.0 { + Err(error) + } else { + Ok(()) + } + } +} diff --git a/pyo3-macros-backend/src/deprecations.rs b/pyo3-macros-backend/src/deprecations.rs deleted file mode 100644 index 3f1f34144f6..00000000000 --- a/pyo3-macros-backend/src/deprecations.rs +++ /dev/null @@ -1,47 +0,0 @@ -use crate::utils::Ctx; -use proc_macro2::{Span, TokenStream}; -use quote::{quote_spanned, ToTokens}; - -pub enum Deprecation { - PyMethodsNewDeprecatedForm, -} - -impl Deprecation { - fn ident(&self, span: Span) -> syn::Ident { - let string = match self { - Deprecation::PyMethodsNewDeprecatedForm => "PYMETHODS_NEW_DEPRECATED_FORM", - }; - syn::Ident::new(string, span) - } -} - -pub struct Deprecations<'ctx>(Vec<(Deprecation, Span)>, &'ctx Ctx); - -impl<'ctx> Deprecations<'ctx> { - pub fn new(ctx: &'ctx Ctx) -> Self { - Deprecations(Vec::new(), ctx) - } - - pub fn push(&mut self, deprecation: Deprecation, span: Span) { - self.0.push((deprecation, span)) - } -} - -impl<'ctx> ToTokens for Deprecations<'ctx> { - fn to_tokens(&self, tokens: &mut TokenStream) { - let Self(deprecations, Ctx { pyo3_path }) = self; - - for (deprecation, span) in deprecations { - let pyo3_path = pyo3_path.to_tokens_spanned(*span); - let ident = deprecation.ident(*span); - quote_spanned!( - *span => - #[allow(clippy::let_unit_value)] - { - let _ = #pyo3_path::impl_::deprecations::#ident; - } - ) - .to_tokens(tokens) - } - } -} diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index 7f26e5b14fc..68f95e794a8 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -1,8 +1,12 @@ -use crate::attributes::{self, get_pyo3_options, CrateAttribute, FromPyWithAttribute}; -use crate::utils::Ctx; +use crate::attributes::{ + self, get_pyo3_options, CrateAttribute, DefaultAttribute, FromPyWithAttribute, + RenameAllAttribute, RenamingRule, +}; +use crate::utils::{self, deprecated_from_py_with, Ctx}; use proc_macro2::TokenStream; -use quote::{format_ident, quote, quote_spanned}; +use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::{ + ext::IdentExt, parenthesized, parse::{Parse, ParseStream}, parse_quote, @@ -22,7 +26,7 @@ impl<'a> Enum<'a> { /// /// `data_enum` is the `syn` representation of the input enum, `ident` is the /// `Identifier` of the enum. - fn new(data_enum: &'a DataEnum, ident: &'a Ident) -> Result { + fn new(data_enum: &'a DataEnum, ident: &'a Ident, options: ContainerOptions) -> Result { ensure_spanned!( !data_enum.variants.is_empty(), ident.span() => "cannot derive FromPyObject for empty enum" @@ -31,9 +35,21 @@ impl<'a> Enum<'a> { .variants .iter() .map(|variant| { - let attrs = ContainerOptions::from_attrs(&variant.attrs)?; + let mut variant_options = ContainerOptions::from_attrs(&variant.attrs)?; + if let Some(rename_all) = &options.rename_all { + ensure_spanned!( + variant_options.rename_all.is_none(), + variant_options.rename_all.span() => "Useless variant `rename_all` - enum is already annotated with `rename_all" + ); + variant_options.rename_all = Some(rename_all.clone()); + + } let var_ident = &variant.ident; - Container::new(&variant.fields, parse_quote!(#ident::#var_ident), attrs) + Container::new( + &variant.fields, + parse_quote!(#ident::#var_ident), + variant_options, + ) }) .collect::>>()?; @@ -44,16 +60,14 @@ impl<'a> Enum<'a> { } /// Build derivation body for enums. - fn build(&self, ctx: &Ctx) -> (TokenStream, TokenStream) { - let Ctx { pyo3_path } = ctx; + fn build(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let mut var_extracts = Vec::new(); let mut variant_names = Vec::new(); let mut error_names = Vec::new(); - let mut deprecations = TokenStream::new(); for var in &self.variants { - let (struct_derive, dep) = var.build(ctx); - deprecations.extend(dep); + let struct_derive = var.build(ctx); let ext = quote!({ let maybe_ret = || -> #pyo3_path::PyResult { #struct_derive @@ -70,22 +84,19 @@ impl<'a> Enum<'a> { error_names.push(&var.err_name); } let ty_name = self.enum_ident.to_string(); - ( - quote!( - let errors = [ - #(#var_extracts),* - ]; - ::std::result::Result::Err( - #pyo3_path::impl_::frompyobject::failed_to_extract_enum( - obj.py(), - #ty_name, - &[#(#variant_names),*], - &[#(#error_names),*], - &errors - ) + quote!( + let errors = [ + #(#var_extracts),* + ]; + ::std::result::Result::Err( + #pyo3_path::impl_::frompyobject::failed_to_extract_enum( + obj.py(), + #ty_name, + &[#(#variant_names),*], + &[#(#error_names),*], + &errors ) - ), - deprecations, + ) ) } } @@ -94,6 +105,7 @@ struct NamedStructField<'a> { ident: &'a syn::Ident, getter: Option, from_py_with: Option, + default: Option, } struct TupleStructField { @@ -130,6 +142,7 @@ struct Container<'a> { path: syn::Path, ty: ContainerType<'a>, err_name: String, + rename_rule: Option, } impl<'a> Container<'a> { @@ -139,6 +152,10 @@ impl<'a> Container<'a> { fn new(fields: &'a Fields, path: syn::Path, options: ContainerOptions) -> Result { let style = match fields { Fields::Unnamed(unnamed) if !unnamed.unnamed.is_empty() => { + ensure_spanned!( + options.rename_all.is_none(), + options.rename_all.span() => "`rename_all` is useless on tuple structs and variants." + ); let mut tuple_fields = unnamed .unnamed .iter() @@ -148,6 +165,10 @@ impl<'a> Container<'a> { attrs.getter.is_none(), field.span() => "`getter` is not permitted on tuple struct elements." ); + ensure_spanned!( + attrs.default.is_none(), + field.span() => "`default` is not permitted on tuple struct elements." + ); Ok(TupleStructField { from_py_with: attrs.from_py_with, }) @@ -197,14 +218,23 @@ impl<'a> Container<'a> { ident, getter: attrs.getter, from_py_with: attrs.from_py_with, + default: attrs.default, }) }) .collect::>>()?; - if options.transparent { + if struct_fields.iter().all(|field| field.default.is_some()) { + bail_spanned!( + fields.span() => "cannot derive FromPyObject for structs and variants with only default values" + ) + } else if options.transparent { ensure_spanned!( struct_fields.len() == 1, fields.span() => "transparent structs and variants can only have 1 field" ); + ensure_spanned!( + options.rename_all.is_none(), + options.rename_all.span() => "`rename_all` is not permitted on `transparent` structs and variants" + ); let field = struct_fields.pop().unwrap(); ensure_spanned!( field.getter.is_none(), @@ -228,6 +258,7 @@ impl<'a> Container<'a> { path, ty: style, err_name, + rename_rule: options.rename_all.map(|v| v.value.rule), }; Ok(v) } @@ -244,7 +275,7 @@ impl<'a> Container<'a> { } /// Build derivation body for a struct. - fn build(&self, ctx: &Ctx) -> (TokenStream, TokenStream) { + fn build(&self, ctx: &Ctx) -> TokenStream { match &self.ty { ContainerType::StructNewtype(ident, from_py_with) => { self.build_newtype_struct(Some(ident), from_py_with, ctx) @@ -262,139 +293,113 @@ impl<'a> Container<'a> { field_ident: Option<&Ident>, from_py_with: &Option, ctx: &Ctx, - ) -> (TokenStream, TokenStream) { - let Ctx { pyo3_path } = ctx; + ) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let self_ty = &self.path; let struct_name = self.name(); if let Some(ident) = field_ident { let field_name = ident.to_string(); - match from_py_with { - None => ( - quote! { - Ok(#self_ty { - #ident: #pyo3_path::impl_::frompyobject::extract_struct_field(obj, #struct_name, #field_name)? - }) - }, - TokenStream::new(), - ), - Some(FromPyWithAttribute { - value: expr_path, .. - }) => ( - quote! { - Ok(#self_ty { - #ident: #pyo3_path::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, #field_name)? - }) - }, - quote_spanned! { expr_path.span() => - const _: () = { - fn check_from_py_with() { - let e = #pyo3_path::impl_::deprecations::GilRefs::new(); - #pyo3_path::impl_::deprecations::inspect_fn(#expr_path, &e); - e.from_py_with_arg(); - } - }; - }, - ), + if let Some(FromPyWithAttribute { + kw, + value: expr_path, + }) = from_py_with + { + let deprecation = deprecated_from_py_with(expr_path).unwrap_or_default(); + + let extractor = quote_spanned! { kw.span => + { let from_py_with: fn(_) -> _ = #expr_path; from_py_with } + }; + quote! { + #deprecation + Ok(#self_ty { + #ident: #pyo3_path::impl_::frompyobject::extract_struct_field_with(#extractor, obj, #struct_name, #field_name)? + }) + } + } else { + quote! { + Ok(#self_ty { + #ident: #pyo3_path::impl_::frompyobject::extract_struct_field(obj, #struct_name, #field_name)? + }) + } + } + } else if let Some(FromPyWithAttribute { + kw, + value: expr_path, + }) = from_py_with + { + let deprecation = deprecated_from_py_with(expr_path).unwrap_or_default(); + + let extractor = quote_spanned! { kw.span => + { let from_py_with: fn(_) -> _ = #expr_path; from_py_with } + }; + quote! { + #deprecation + #pyo3_path::impl_::frompyobject::extract_tuple_struct_field_with(#extractor, obj, #struct_name, 0).map(#self_ty) } } else { - match from_py_with { - None => ( - quote!( - #pyo3_path::impl_::frompyobject::extract_tuple_struct_field(obj, #struct_name, 0).map(#self_ty) - ), - TokenStream::new(), - ), - Some(FromPyWithAttribute { - value: expr_path, .. - }) => ( - quote! ( - #pyo3_path::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, 0).map(#self_ty) - ), - quote_spanned! { expr_path.span() => - const _: () = { - fn check_from_py_with() { - let e = #pyo3_path::impl_::deprecations::GilRefs::new(); - #pyo3_path::impl_::deprecations::inspect_fn(#expr_path, &e); - e.from_py_with_arg(); - } - }; - }, - ), + quote! { + #pyo3_path::impl_::frompyobject::extract_tuple_struct_field(obj, #struct_name, 0).map(#self_ty) } } } - fn build_tuple_struct( - &self, - struct_fields: &[TupleStructField], - ctx: &Ctx, - ) -> (TokenStream, TokenStream) { - let Ctx { pyo3_path } = ctx; + fn build_tuple_struct(&self, struct_fields: &[TupleStructField], ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let self_ty = &self.path; let struct_name = &self.name(); let field_idents: Vec<_> = (0..struct_fields.len()) .map(|i| format_ident!("arg{}", i)) .collect(); let fields = struct_fields.iter().zip(&field_idents).enumerate().map(|(index, (field, ident))| { - match &field.from_py_with { - None => quote!( + if let Some(FromPyWithAttribute { + kw, + value: expr_path, .. + }) = &field.from_py_with { + let extractor = quote_spanned! { kw.span => + { let from_py_with: fn(_) -> _ = #expr_path; from_py_with } + }; + quote! { + #pyo3_path::impl_::frompyobject::extract_tuple_struct_field_with(#extractor, &#ident, #struct_name, #index)? + } + } else { + quote!{ #pyo3_path::impl_::frompyobject::extract_tuple_struct_field(&#ident, #struct_name, #index)? - ), - Some(FromPyWithAttribute { - value: expr_path, .. - }) => quote! ( - #pyo3_path::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path as fn(_) -> _, &#ident, #struct_name, #index)? - ), - } + }} }); let deprecations = struct_fields .iter() - .filter_map(|field| { - let FromPyWithAttribute { - value: expr_path, .. - } = field.from_py_with.as_ref()?; - Some(quote_spanned! { expr_path.span() => - const _: () = { - fn check_from_py_with() { - let e = #pyo3_path::impl_::deprecations::GilRefs::new(); - #pyo3_path::impl_::deprecations::inspect_fn(#expr_path, &e); - e.from_py_with_arg(); - } - }; - }) - }) + .filter_map(|fields| fields.from_py_with.as_ref()) + .filter_map(|kw| deprecated_from_py_with(&kw.value)) .collect::(); - ( - quote!( - match #pyo3_path::types::PyAnyMethods::extract(obj) { - ::std::result::Result::Ok((#(#field_idents),*)) => ::std::result::Result::Ok(#self_ty(#(#fields),*)), - ::std::result::Result::Err(err) => ::std::result::Result::Err(err), - } - ), - deprecations, + quote!( + #deprecations + match #pyo3_path::types::PyAnyMethods::extract(obj) { + ::std::result::Result::Ok((#(#field_idents),*)) => ::std::result::Result::Ok(#self_ty(#(#fields),*)), + ::std::result::Result::Err(err) => ::std::result::Result::Err(err), + } ) } - fn build_struct( - &self, - struct_fields: &[NamedStructField<'_>], - ctx: &Ctx, - ) -> (TokenStream, TokenStream) { - let Ctx { pyo3_path } = ctx; + fn build_struct(&self, struct_fields: &[NamedStructField<'_>], ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let self_ty = &self.path; - let struct_name = &self.name(); - let mut fields: Punctuated = Punctuated::new(); + let struct_name = self.name(); + let mut fields: Punctuated = Punctuated::new(); for field in struct_fields { - let ident = &field.ident; - let field_name = ident.to_string(); + let ident = field.ident; + let field_name = ident.unraw().to_string(); let getter = match field.getter.as_ref().unwrap_or(&FieldGetter::GetAttr(None)) { FieldGetter::GetAttr(Some(name)) => { quote!(#pyo3_path::types::PyAnyMethods::getattr(obj, #pyo3_path::intern!(obj.py(), #name))) } FieldGetter::GetAttr(None) => { - quote!(#pyo3_path::types::PyAnyMethods::getattr(obj, #pyo3_path::intern!(obj.py(), #field_name))) + let name = self + .rename_rule + .map(|rule| utils::apply_renaming_rule(rule, &field_name)); + let name = name.as_deref().unwrap_or(&field_name); + quote!(#pyo3_path::types::PyAnyMethods::getattr(obj, #pyo3_path::intern!(obj.py(), #name))) } FieldGetter::GetItem(Some(syn::Lit::Str(key))) => { quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #pyo3_path::intern!(obj.py(), #key))) @@ -403,45 +408,53 @@ impl<'a> Container<'a> { quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #key)) } FieldGetter::GetItem(None) => { - quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #pyo3_path::intern!(obj.py(), #field_name))) + let name = self + .rename_rule + .map(|rule| utils::apply_renaming_rule(rule, &field_name)); + let name = name.as_deref().unwrap_or(&field_name); + quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #pyo3_path::intern!(obj.py(), #name))) } }; - let extractor = match &field.from_py_with { - None => { - quote!(#pyo3_path::impl_::frompyobject::extract_struct_field(&#getter?, #struct_name, #field_name)?) - } - Some(FromPyWithAttribute { - value: expr_path, .. - }) => { - quote! (#pyo3_path::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, &#getter?, #struct_name, #field_name)?) - } + let extractor = if let Some(FromPyWithAttribute { + kw, + value: expr_path, + }) = &field.from_py_with + { + let extractor = quote_spanned! { kw.span => + { let from_py_with: fn(_) -> _ = #expr_path; from_py_with } + }; + quote! (#pyo3_path::impl_::frompyobject::extract_struct_field_with(#extractor, &#getter?, #struct_name, #field_name)?) + } else { + quote!(#pyo3_path::impl_::frompyobject::extract_struct_field(&value, #struct_name, #field_name)?) + }; + let extracted = if let Some(default) = &field.default { + let default_expr = if let Some(default_expr) = &default.value { + default_expr.to_token_stream() + } else { + quote!(::std::default::Default::default()) + }; + quote!(if let ::std::result::Result::Ok(value) = #getter { + #extractor + } else { + #default_expr + }) + } else { + quote!({ + let value = #getter?; + #extractor + }) }; - fields.push(quote!(#ident: #extractor)); + fields.push(quote!(#ident: #extracted)); } - let deprecations = struct_fields + let d = struct_fields .iter() - .filter_map(|field| { - let FromPyWithAttribute { - value: expr_path, .. - } = field.from_py_with.as_ref()?; - Some(quote_spanned! { expr_path.span() => - const _: () = { - fn check_from_py_with() { - let e = #pyo3_path::impl_::deprecations::GilRefs::new(); - #pyo3_path::impl_::deprecations::inspect_fn(#expr_path, &e); - e.from_py_with_arg(); - } - }; - }) - }) + .filter_map(|field| field.from_py_with.as_ref()) + .filter_map(|kw| deprecated_from_py_with(&kw.value)) .collect::(); - ( - quote!(::std::result::Result::Ok(#self_ty{#fields})), - deprecations, - ) + quote!(#d ::std::result::Result::Ok(#self_ty{#fields})) } } @@ -455,6 +468,8 @@ struct ContainerOptions { annotation: Option, /// Change the path for the pyo3 crate krate: Option, + /// Converts the field idents according to the [RenamingRule] before extraction + rename_all: Option, } /// Attributes for deriving FromPyObject scoped on containers. @@ -467,6 +482,8 @@ enum ContainerPyO3Attribute { ErrorAnnotation(LitStr), /// Change the path for the pyo3 crate Crate(CrateAttribute), + /// Converts the field idents according to the [RenamingRule] before extraction + RenameAll(RenameAllAttribute), } impl Parse for ContainerPyO3Attribute { @@ -484,6 +501,8 @@ impl Parse for ContainerPyO3Attribute { input.parse().map(ContainerPyO3Attribute::ErrorAnnotation) } else if lookahead.peek(Token![crate]) { input.parse().map(ContainerPyO3Attribute::Crate) + } else if lookahead.peek(attributes::kw::rename_all) { + input.parse().map(ContainerPyO3Attribute::RenameAll) } else { Err(lookahead.error()) } @@ -526,6 +545,13 @@ impl ContainerOptions { ); options.krate = Some(path); } + ContainerPyO3Attribute::RenameAll(rename_all) => { + ensure_spanned!( + options.rename_all.is_none(), + rename_all.span() => "`rename_all` may only be provided once" + ); + options.rename_all = Some(rename_all); + } } } } @@ -539,6 +565,7 @@ impl ContainerOptions { struct FieldPyO3Attributes { getter: Option, from_py_with: Option, + default: Option, } #[derive(Clone, Debug)] @@ -550,6 +577,7 @@ enum FieldGetter { enum FieldPyO3Attribute { Getter(FieldGetter), FromPyWith(FromPyWithAttribute), + Default(DefaultAttribute), } impl Parse for FieldPyO3Attribute { @@ -593,6 +621,8 @@ impl Parse for FieldPyO3Attribute { } } else if lookahead.peek(attributes::kw::from_py_with) { input.parse().map(FieldPyO3Attribute::FromPyWith) + } else if lookahead.peek(Token![default]) { + input.parse().map(FieldPyO3Attribute::Default) } else { Err(lookahead.error()) } @@ -604,6 +634,7 @@ impl FieldPyO3Attributes { fn from_attrs(attrs: &[Attribute]) -> Result { let mut getter = None; let mut from_py_with = None; + let mut default = None; for attr in attrs { if let Some(pyo3_attrs) = get_pyo3_options(attr)? { @@ -623,6 +654,13 @@ impl FieldPyO3Attributes { ); from_py_with = Some(from_py_with_attr); } + FieldPyO3Attribute::Default(default_attr) => { + ensure_spanned!( + default.is_none(), + attr.span() => "`default` may only be provided once" + ); + default = Some(default_attr); + } } } } @@ -631,6 +669,7 @@ impl FieldPyO3Attributes { Ok(FieldPyO3Attributes { getter, from_py_with, + default, }) } } @@ -654,32 +693,35 @@ fn verify_and_get_lifetime(generics: &syn::Generics) -> Result Foo(T)` /// adds `T: FromPyObject` on the derived implementation. pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { + let options = ContainerOptions::from_attrs(&tokens.attrs)?; + let ctx = &Ctx::new(&options.krate, None); + let Ctx { pyo3_path, .. } = &ctx; + + let (_, ty_generics, _) = tokens.generics.split_for_impl(); let mut trait_generics = tokens.generics.clone(); - let generics = &tokens.generics; - let lt_param = if let Some(lt) = verify_and_get_lifetime(generics)? { + let lt_param = if let Some(lt) = verify_and_get_lifetime(&trait_generics)? { lt.clone() } else { trait_generics.params.push(parse_quote!('py)); parse_quote!('py) }; - let mut where_clause: syn::WhereClause = parse_quote!(where); - for param in generics.type_params() { + let (impl_generics, _, where_clause) = trait_generics.split_for_impl(); + + let mut where_clause = where_clause.cloned().unwrap_or_else(|| parse_quote!(where)); + for param in trait_generics.type_params() { let gen_ident = ¶m.ident; where_clause .predicates - .push(parse_quote!(#gen_ident: FromPyObject<#lt_param>)) + .push(parse_quote!(#gen_ident: #pyo3_path::FromPyObject<'py>)) } - let options = ContainerOptions::from_attrs(&tokens.attrs)?; - let ctx = &Ctx::new(&options.krate); - let Ctx { pyo3_path } = &ctx; - let (derives, from_py_with_deprecations) = match &tokens.data { + let derives = match &tokens.data { syn::Data::Enum(en) => { if options.transparent || options.annotation.is_some() { bail_spanned!(tokens.span() => "`transparent` or `annotation` is not supported \ at top level for enums"); } - let en = Enum::new(en, &tokens.ident)?; + let en = Enum::new(en, &tokens.ident, options)?; en.build(ctx) } syn::Data::Struct(st) => { @@ -698,12 +740,10 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { let ident = &tokens.ident; Ok(quote!( #[automatically_derived] - impl #trait_generics #pyo3_path::FromPyObject<#lt_param> for #ident #generics #where_clause { + impl #impl_generics #pyo3_path::FromPyObject<#lt_param> for #ident #ty_generics #where_clause { fn extract_bound(obj: &#pyo3_path::Bound<#lt_param, #pyo3_path::PyAny>) -> #pyo3_path::PyResult { #derives } } - - #from_py_with_deprecations )) } diff --git a/pyo3-macros-backend/src/intopyobject.rs b/pyo3-macros-backend/src/intopyobject.rs new file mode 100755 index 00000000000..787d1dd6259 --- /dev/null +++ b/pyo3-macros-backend/src/intopyobject.rs @@ -0,0 +1,679 @@ +use crate::attributes::{self, get_pyo3_options, CrateAttribute, IntoPyWithAttribute}; +use crate::utils::Ctx; +use proc_macro2::{Span, TokenStream}; +use quote::{format_ident, quote, quote_spanned}; +use syn::ext::IdentExt; +use syn::parse::{Parse, ParseStream}; +use syn::spanned::Spanned as _; +use syn::{ + parenthesized, parse_quote, Attribute, DataEnum, DeriveInput, Fields, Ident, Index, Result, + Token, +}; + +/// Attributes for deriving `IntoPyObject` scoped on containers. +enum ContainerPyO3Attribute { + /// Treat the Container as a Wrapper, directly convert its field into the output object. + Transparent(attributes::kw::transparent), + /// Change the path for the pyo3 crate + Crate(CrateAttribute), +} + +impl Parse for ContainerPyO3Attribute { + fn parse(input: ParseStream<'_>) -> Result { + let lookahead = input.lookahead1(); + if lookahead.peek(attributes::kw::transparent) { + let kw: attributes::kw::transparent = input.parse()?; + Ok(ContainerPyO3Attribute::Transparent(kw)) + } else if lookahead.peek(Token![crate]) { + input.parse().map(ContainerPyO3Attribute::Crate) + } else { + Err(lookahead.error()) + } + } +} + +#[derive(Default)] +struct ContainerOptions { + /// Treat the Container as a Wrapper, directly convert its field into the output object. + transparent: Option, + /// Change the path for the pyo3 crate + krate: Option, +} + +impl ContainerOptions { + fn from_attrs(attrs: &[Attribute]) -> Result { + let mut options = ContainerOptions::default(); + + for attr in attrs { + if let Some(pyo3_attrs) = get_pyo3_options(attr)? { + pyo3_attrs + .into_iter() + .try_for_each(|opt| options.set_option(opt))?; + } + } + Ok(options) + } + + fn set_option(&mut self, option: ContainerPyO3Attribute) -> syn::Result<()> { + macro_rules! set_option { + ($key:ident) => { + { + ensure_spanned!( + self.$key.is_none(), + $key.span() => concat!("`", stringify!($key), "` may only be specified once") + ); + self.$key = Some($key); + } + }; + } + + match option { + ContainerPyO3Attribute::Transparent(transparent) => set_option!(transparent), + ContainerPyO3Attribute::Crate(krate) => set_option!(krate), + } + Ok(()) + } +} + +#[derive(Debug, Clone)] +struct ItemOption { + field: Option, + span: Span, +} + +impl ItemOption { + fn span(&self) -> Span { + self.span + } +} + +enum FieldAttribute { + Item(ItemOption), + IntoPyWith(IntoPyWithAttribute), +} + +impl Parse for FieldAttribute { + fn parse(input: ParseStream<'_>) -> Result { + let lookahead = input.lookahead1(); + if lookahead.peek(attributes::kw::attribute) { + let attr: attributes::kw::attribute = input.parse()?; + bail_spanned!(attr.span => "`attribute` is not supported by `IntoPyObject`"); + } else if lookahead.peek(attributes::kw::item) { + let attr: attributes::kw::item = input.parse()?; + if input.peek(syn::token::Paren) { + let content; + let _ = parenthesized!(content in input); + let key = content.parse()?; + if !content.is_empty() { + return Err( + content.error("expected at most one argument: `item` or `item(key)`") + ); + } + Ok(FieldAttribute::Item(ItemOption { + field: Some(key), + span: attr.span, + })) + } else { + Ok(FieldAttribute::Item(ItemOption { + field: None, + span: attr.span, + })) + } + } else if lookahead.peek(attributes::kw::into_py_with) { + input.parse().map(FieldAttribute::IntoPyWith) + } else { + Err(lookahead.error()) + } + } +} + +#[derive(Clone, Debug, Default)] +struct FieldAttributes { + item: Option, + into_py_with: Option, +} + +impl FieldAttributes { + /// Extract the field attributes. + fn from_attrs(attrs: &[Attribute]) -> Result { + let mut options = FieldAttributes::default(); + + for attr in attrs { + if let Some(pyo3_attrs) = get_pyo3_options(attr)? { + pyo3_attrs + .into_iter() + .try_for_each(|opt| options.set_option(opt))?; + } + } + Ok(options) + } + + fn set_option(&mut self, option: FieldAttribute) -> syn::Result<()> { + macro_rules! set_option { + ($key:ident) => { + { + ensure_spanned!( + self.$key.is_none(), + $key.span() => concat!("`", stringify!($key), "` may only be specified once") + ); + self.$key = Some($key); + } + }; + } + + match option { + FieldAttribute::Item(item) => set_option!(item), + FieldAttribute::IntoPyWith(into_py_with) => set_option!(into_py_with), + } + Ok(()) + } +} + +enum IntoPyObjectTypes { + Transparent(syn::Type), + Opaque { + target: TokenStream, + output: TokenStream, + error: TokenStream, + }, +} + +struct IntoPyObjectImpl { + types: IntoPyObjectTypes, + body: TokenStream, +} + +struct NamedStructField<'a> { + ident: &'a syn::Ident, + field: &'a syn::Field, + item: Option, + into_py_with: Option, +} + +struct TupleStructField<'a> { + field: &'a syn::Field, + into_py_with: Option, +} + +/// Container Style +/// +/// Covers Structs, Tuplestructs and corresponding Newtypes. +enum ContainerType<'a> { + /// Struct Container, e.g. `struct Foo { a: String }` + /// + /// Variant contains the list of field identifiers and the corresponding extraction call. + Struct(Vec>), + /// Newtype struct container, e.g. `#[transparent] struct Foo { a: String }` + /// + /// The field specified by the identifier is extracted directly from the object. + StructNewtype(&'a syn::Field), + /// Tuple struct, e.g. `struct Foo(String)`. + /// + /// Variant contains a list of conversion methods for each of the fields that are directly + /// extracted from the tuple. + Tuple(Vec>), + /// Tuple newtype, e.g. `#[transparent] struct Foo(String)` + /// + /// The wrapped field is directly extracted from the object. + TupleNewtype(&'a syn::Field), +} + +/// Data container +/// +/// Either describes a struct or an enum variant. +struct Container<'a, const REF: bool> { + path: syn::Path, + receiver: Option, + ty: ContainerType<'a>, +} + +/// Construct a container based on fields, identifier and attributes. +impl<'a, const REF: bool> Container<'a, REF> { + /// + /// Fails if the variant has no fields or incompatible attributes. + fn new( + receiver: Option, + fields: &'a Fields, + path: syn::Path, + options: ContainerOptions, + ) -> Result { + let style = match fields { + Fields::Unnamed(unnamed) if !unnamed.unnamed.is_empty() => { + let mut tuple_fields = unnamed + .unnamed + .iter() + .map(|field| { + let attrs = FieldAttributes::from_attrs(&field.attrs)?; + ensure_spanned!( + attrs.item.is_none(), + attrs.item.unwrap().span() => "`item` is not permitted on tuple struct elements." + ); + Ok(TupleStructField { + field, + into_py_with: attrs.into_py_with, + }) + }) + .collect::>>()?; + if tuple_fields.len() == 1 { + // Always treat a 1-length tuple struct as "transparent", even without the + // explicit annotation. + let TupleStructField { + field, + into_py_with, + } = tuple_fields.pop().unwrap(); + ensure_spanned!( + into_py_with.is_none(), + into_py_with.span() => "`into_py_with` is not permitted on `transparent` structs" + ); + ContainerType::TupleNewtype(field) + } else if options.transparent.is_some() { + bail_spanned!( + fields.span() => "transparent structs and variants can only have 1 field" + ); + } else { + ContainerType::Tuple(tuple_fields) + } + } + Fields::Named(named) if !named.named.is_empty() => { + if options.transparent.is_some() { + ensure_spanned!( + named.named.iter().count() == 1, + fields.span() => "transparent structs and variants can only have 1 field" + ); + + let field = named.named.iter().next().unwrap(); + let attrs = FieldAttributes::from_attrs(&field.attrs)?; + ensure_spanned!( + attrs.item.is_none(), + attrs.item.unwrap().span() => "`transparent` structs may not have `item` for the inner field" + ); + ensure_spanned!( + attrs.into_py_with.is_none(), + attrs.into_py_with.span() => "`into_py_with` is not permitted on `transparent` structs or variants" + ); + ContainerType::StructNewtype(field) + } else { + let struct_fields = named + .named + .iter() + .map(|field| { + let ident = field + .ident + .as_ref() + .expect("Named fields should have identifiers"); + + let attrs = FieldAttributes::from_attrs(&field.attrs)?; + + Ok(NamedStructField { + ident, + field, + item: attrs.item, + into_py_with: attrs.into_py_with, + }) + }) + .collect::>>()?; + ContainerType::Struct(struct_fields) + } + } + _ => bail_spanned!( + fields.span() => "cannot derive `IntoPyObject` for empty structs" + ), + }; + + let v = Container { + path, + receiver, + ty: style, + }; + Ok(v) + } + + fn match_pattern(&self) -> TokenStream { + let path = &self.path; + let pattern = match &self.ty { + ContainerType::Struct(fields) => fields + .iter() + .enumerate() + .map(|(i, f)| { + let ident = f.ident; + let new_ident = format_ident!("arg{i}"); + quote! {#ident: #new_ident,} + }) + .collect::(), + ContainerType::StructNewtype(field) => { + let ident = field.ident.as_ref().unwrap(); + quote!(#ident: arg0) + } + ContainerType::Tuple(fields) => { + let i = (0..fields.len()).map(Index::from); + let idents = (0..fields.len()).map(|i| format_ident!("arg{i}")); + quote! { #(#i: #idents,)* } + } + ContainerType::TupleNewtype(_) => quote!(0: arg0), + }; + + quote! { #path{ #pattern } } + } + + /// Build derivation body for a struct. + fn build(&self, ctx: &Ctx) -> IntoPyObjectImpl { + match &self.ty { + ContainerType::StructNewtype(field) | ContainerType::TupleNewtype(field) => { + self.build_newtype_struct(field, ctx) + } + ContainerType::Tuple(fields) => self.build_tuple_struct(fields, ctx), + ContainerType::Struct(fields) => self.build_struct(fields, ctx), + } + } + + fn build_newtype_struct(&self, field: &syn::Field, ctx: &Ctx) -> IntoPyObjectImpl { + let Ctx { pyo3_path, .. } = ctx; + let ty = &field.ty; + + let unpack = self + .receiver + .as_ref() + .map(|i| { + let pattern = self.match_pattern(); + quote! { let #pattern = #i;} + }) + .unwrap_or_default(); + + IntoPyObjectImpl { + types: IntoPyObjectTypes::Transparent(ty.clone()), + body: quote_spanned! { ty.span() => + #unpack + #pyo3_path::conversion::IntoPyObject::into_pyobject(arg0, py) + }, + } + } + + fn build_struct(&self, fields: &[NamedStructField<'_>], ctx: &Ctx) -> IntoPyObjectImpl { + let Ctx { pyo3_path, .. } = ctx; + + let unpack = self + .receiver + .as_ref() + .map(|i| { + let pattern = self.match_pattern(); + quote! { let #pattern = #i;} + }) + .unwrap_or_default(); + + let setter = fields + .iter() + .enumerate() + .map(|(i, f)| { + let key = f + .item + .as_ref() + .and_then(|item| item.field.as_ref()) + .map(|item| item.value()) + .unwrap_or_else(|| f.ident.unraw().to_string()); + let value = Ident::new(&format!("arg{i}"), f.field.ty.span()); + + if let Some(expr_path) = f.into_py_with.as_ref().map(|i|&i.value) { + let cow = if REF { + quote!(::std::borrow::Cow::Borrowed(#value)) + } else { + quote!(::std::borrow::Cow::Owned(#value)) + }; + quote! { + let into_py_with: fn(::std::borrow::Cow<'_, _>, #pyo3_path::Python<'py>) -> #pyo3_path::PyResult<#pyo3_path::Bound<'py, #pyo3_path::PyAny>> = #expr_path; + #pyo3_path::types::PyDictMethods::set_item(&dict, #key, into_py_with(#cow, py)?)?; + } + } else { + quote! { + #pyo3_path::types::PyDictMethods::set_item(&dict, #key, #value)?; + } + } + }) + .collect::(); + + IntoPyObjectImpl { + types: IntoPyObjectTypes::Opaque { + target: quote!(#pyo3_path::types::PyDict), + output: quote!(#pyo3_path::Bound<'py, Self::Target>), + error: quote!(#pyo3_path::PyErr), + }, + body: quote! { + #unpack + let dict = #pyo3_path::types::PyDict::new(py); + #setter + ::std::result::Result::Ok::<_, Self::Error>(dict) + }, + } + } + + fn build_tuple_struct(&self, fields: &[TupleStructField<'_>], ctx: &Ctx) -> IntoPyObjectImpl { + let Ctx { pyo3_path, .. } = ctx; + + let unpack = self + .receiver + .as_ref() + .map(|i| { + let pattern = self.match_pattern(); + quote! { let #pattern = #i;} + }) + .unwrap_or_default(); + + let setter = fields + .iter() + .enumerate() + .map(|(i, f)| { + let ty = &f.field.ty; + let value = Ident::new(&format!("arg{i}"), f.field.ty.span()); + + if let Some(expr_path) = f.into_py_with.as_ref().map(|i|&i.value) { + let cow = if REF { + quote!(::std::borrow::Cow::Borrowed(#value)) + } else { + quote!(::std::borrow::Cow::Owned(#value)) + }; + quote_spanned! { ty.span() => + { + let into_py_with: fn(::std::borrow::Cow<'_, _>, #pyo3_path::Python<'py>) -> #pyo3_path::PyResult<#pyo3_path::Bound<'py, #pyo3_path::PyAny>> = #expr_path; + into_py_with(#cow, py)? + }, + } + } else { + quote_spanned! { ty.span() => + #pyo3_path::conversion::IntoPyObject::into_pyobject(#value, py) + .map(#pyo3_path::BoundObject::into_any) + .map(#pyo3_path::BoundObject::into_bound)?, + } + } + }) + .collect::(); + + IntoPyObjectImpl { + types: IntoPyObjectTypes::Opaque { + target: quote!(#pyo3_path::types::PyTuple), + output: quote!(#pyo3_path::Bound<'py, Self::Target>), + error: quote!(#pyo3_path::PyErr), + }, + body: quote! { + #unpack + #pyo3_path::types::PyTuple::new(py, [#setter]) + }, + } + } +} + +/// Describes derivation input of an enum. +struct Enum<'a, const REF: bool> { + variants: Vec>, +} + +impl<'a, const REF: bool> Enum<'a, REF> { + /// Construct a new enum representation. + /// + /// `data_enum` is the `syn` representation of the input enum, `ident` is the + /// `Identifier` of the enum. + fn new(data_enum: &'a DataEnum, ident: &'a Ident) -> Result { + ensure_spanned!( + !data_enum.variants.is_empty(), + ident.span() => "cannot derive `IntoPyObject` for empty enum" + ); + let variants = data_enum + .variants + .iter() + .map(|variant| { + let attrs = ContainerOptions::from_attrs(&variant.attrs)?; + let var_ident = &variant.ident; + + ensure_spanned!( + !variant.fields.is_empty(), + variant.ident.span() => "cannot derive `IntoPyObject` for empty variants" + ); + + Container::new( + None, + &variant.fields, + parse_quote!(#ident::#var_ident), + attrs, + ) + }) + .collect::>>()?; + + Ok(Enum { variants }) + } + + /// Build derivation body for enums. + fn build(&self, ctx: &Ctx) -> IntoPyObjectImpl { + let Ctx { pyo3_path, .. } = ctx; + + let variants = self + .variants + .iter() + .map(|v| { + let IntoPyObjectImpl { body, .. } = v.build(ctx); + let pattern = v.match_pattern(); + quote! { + #pattern => { + {#body} + .map(#pyo3_path::BoundObject::into_any) + .map(#pyo3_path::BoundObject::into_bound) + .map_err(::std::convert::Into::<#pyo3_path::PyErr>::into) + } + } + }) + .collect::(); + + IntoPyObjectImpl { + types: IntoPyObjectTypes::Opaque { + target: quote!(#pyo3_path::types::PyAny), + output: quote!(#pyo3_path::Bound<'py, >::Target>), + error: quote!(#pyo3_path::PyErr), + }, + body: quote! { + match self { + #variants + } + }, + } + } +} + +// if there is a `'py` lifetime, we treat it as the `Python<'py>` lifetime +fn verify_and_get_lifetime(generics: &syn::Generics) -> Option<&syn::LifetimeParam> { + let mut lifetimes = generics.lifetimes(); + lifetimes.find(|l| l.lifetime.ident == "py") +} + +pub fn build_derive_into_pyobject(tokens: &DeriveInput) -> Result { + let options = ContainerOptions::from_attrs(&tokens.attrs)?; + let ctx = &Ctx::new(&options.krate, None); + let Ctx { pyo3_path, .. } = &ctx; + + let (_, ty_generics, _) = tokens.generics.split_for_impl(); + let mut trait_generics = tokens.generics.clone(); + if REF { + trait_generics.params.push(parse_quote!('_a)); + } + let lt_param = if let Some(lt) = verify_and_get_lifetime(&trait_generics) { + lt.clone() + } else { + trait_generics.params.push(parse_quote!('py)); + parse_quote!('py) + }; + let (impl_generics, _, where_clause) = trait_generics.split_for_impl(); + + let mut where_clause = where_clause.cloned().unwrap_or_else(|| parse_quote!(where)); + for param in trait_generics.type_params() { + let gen_ident = ¶m.ident; + where_clause.predicates.push(if REF { + parse_quote!(&'_a #gen_ident: #pyo3_path::conversion::IntoPyObject<'py>) + } else { + parse_quote!(#gen_ident: #pyo3_path::conversion::IntoPyObject<'py>) + }) + } + + let IntoPyObjectImpl { types, body } = match &tokens.data { + syn::Data::Enum(en) => { + if options.transparent.is_some() { + bail_spanned!(tokens.span() => "`transparent` is not supported at top level for enums"); + } + let en = Enum::::new(en, &tokens.ident)?; + en.build(ctx) + } + syn::Data::Struct(st) => { + let ident = &tokens.ident; + let st = Container::::new( + Some(Ident::new("self", Span::call_site())), + &st.fields, + parse_quote!(#ident), + options, + )?; + st.build(ctx) + } + syn::Data::Union(_) => bail_spanned!( + tokens.span() => "#[derive(`IntoPyObject`)] is not supported for unions" + ), + }; + + let (target, output, error) = match types { + IntoPyObjectTypes::Transparent(ty) => { + if REF { + ( + quote! { <&'_a #ty as #pyo3_path::IntoPyObject<'py>>::Target }, + quote! { <&'_a #ty as #pyo3_path::IntoPyObject<'py>>::Output }, + quote! { <&'_a #ty as #pyo3_path::IntoPyObject<'py>>::Error }, + ) + } else { + ( + quote! { <#ty as #pyo3_path::IntoPyObject<'py>>::Target }, + quote! { <#ty as #pyo3_path::IntoPyObject<'py>>::Output }, + quote! { <#ty as #pyo3_path::IntoPyObject<'py>>::Error }, + ) + } + } + IntoPyObjectTypes::Opaque { + target, + output, + error, + } => (target, output, error), + }; + + let ident = &tokens.ident; + let ident = if REF { + quote! { &'_a #ident} + } else { + quote! { #ident } + }; + Ok(quote!( + #[automatically_derived] + impl #impl_generics #pyo3_path::conversion::IntoPyObject<#lt_param> for #ident #ty_generics #where_clause { + type Target = #target; + type Output = #output; + type Error = #error; + + fn into_pyobject(self, py: #pyo3_path::Python<#lt_param>) -> ::std::result::Result< + ::Output, + ::Error, + > { + #body + } + } + )) +} diff --git a/pyo3-macros-backend/src/konst.rs b/pyo3-macros-backend/src/konst.rs index 9a41a2b7178..ae88f785249 100644 --- a/pyo3-macros-backend/src/konst.rs +++ b/pyo3-macros-backend/src/konst.rs @@ -1,12 +1,9 @@ use std::borrow::Cow; +use std::ffi::CString; -use crate::utils::Ctx; -use crate::{ - attributes::{self, get_pyo3_options, take_attributes, NameAttribute}, - deprecations::Deprecations, -}; -use proc_macro2::{Ident, TokenStream}; -use quote::quote; +use crate::attributes::{self, get_pyo3_options, take_attributes, NameAttribute}; +use crate::utils::{Ctx, LitCStr}; +use proc_macro2::{Ident, Span}; use syn::{ ext::IdentExt, parse::{Parse, ParseStream}, @@ -14,12 +11,12 @@ use syn::{ Result, }; -pub struct ConstSpec<'ctx> { +pub struct ConstSpec { pub rust_ident: syn::Ident, - pub attributes: ConstAttributes<'ctx>, + pub attributes: ConstAttributes, } -impl ConstSpec<'_> { +impl ConstSpec { pub fn python_name(&self) -> Cow<'_, Ident> { if let Some(name) = &self.attributes.name { Cow::Borrowed(&name.value.0) @@ -29,16 +26,15 @@ impl ConstSpec<'_> { } /// Null-terminated Python name - pub fn null_terminated_python_name(&self) -> TokenStream { - let name = format!("{}\0", self.python_name()); - quote!({#name}) + pub fn null_terminated_python_name(&self, ctx: &Ctx) -> LitCStr { + let name = self.python_name().to_string(); + LitCStr::new(CString::new(name).unwrap(), Span::call_site(), ctx) } } -pub struct ConstAttributes<'ctx> { +pub struct ConstAttributes { pub is_class_attr: bool, pub name: Option, - pub deprecations: Deprecations<'ctx>, } pub enum PyO3ConstAttribute { @@ -56,12 +52,11 @@ impl Parse for PyO3ConstAttribute { } } -impl<'ctx> ConstAttributes<'ctx> { - pub fn from_attrs(attrs: &mut Vec, ctx: &'ctx Ctx) -> syn::Result { +impl ConstAttributes { + pub fn from_attrs(attrs: &mut Vec) -> syn::Result { let mut attributes = ConstAttributes { is_class_attr: false, name: None, - deprecations: Deprecations::new(ctx), }; take_attributes(attrs, |attr| { diff --git a/pyo3-macros-backend/src/lib.rs b/pyo3-macros-backend/src/lib.rs index a9d75a2a6fe..7893a94af98 100644 --- a/pyo3-macros-backend/src/lib.rs +++ b/pyo3-macros-backend/src/lib.rs @@ -9,8 +9,8 @@ mod utils; mod attributes; -mod deprecations; mod frompyobject; +mod intopyobject; mod konst; mod method; mod module; @@ -19,9 +19,11 @@ mod pyclass; mod pyfunction; mod pyimpl; mod pymethod; +mod pyversions; mod quotes; pub use frompyobject::build_derive_from_pyobject; +pub use intopyobject::build_derive_into_pyobject; pub use module::{pymodule_function_impl, pymodule_module_impl, PyModuleOptions}; pub use pyclass::{build_py_class, build_py_enum, PyClassArgs}; pub use pyfunction::{build_py_function, PyFunctionOptions}; diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 155c554025d..a1d7a95df35 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -1,35 +1,133 @@ +use std::borrow::Cow; +use std::ffi::CString; use std::fmt::Display; use proc_macro2::{Span, TokenStream}; use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::{ext::IdentExt, spanned::Spanned, Ident, Result}; -use crate::utils::Ctx; +use crate::pyversions::is_abi3_before; +use crate::utils::{Ctx, LitCStr}; use crate::{ - attributes::{TextSignatureAttribute, TextSignatureAttributeValue}, - deprecations::{Deprecation, Deprecations}, + attributes::{FromPyWithAttribute, TextSignatureAttribute, TextSignatureAttributeValue}, params::{impl_arg_params, Holders}, pyfunction::{ FunctionSignature, PyFunctionArgPyO3Attributes, PyFunctionOptions, SignatureAttribute, }, quotes, - utils::{self, is_abi3, PythonDoc}, + utils::{self, PythonDoc}, }; #[derive(Clone, Debug)] -pub struct FnArg<'a> { +pub struct RegularArg<'a> { + pub name: Cow<'a, syn::Ident>, + pub ty: &'a syn::Type, + pub from_py_with: Option, + pub default_value: Option, + pub option_wrapped_type: Option<&'a syn::Type>, +} + +/// Pythons *args argument +#[derive(Clone, Debug)] +pub struct VarargsArg<'a> { + pub name: Cow<'a, syn::Ident>, + pub ty: &'a syn::Type, +} + +/// Pythons **kwarg argument +#[derive(Clone, Debug)] +pub struct KwargsArg<'a> { + pub name: Cow<'a, syn::Ident>, + pub ty: &'a syn::Type, +} + +#[derive(Clone, Debug)] +pub struct CancelHandleArg<'a> { pub name: &'a syn::Ident, pub ty: &'a syn::Type, - pub optional: Option<&'a syn::Type>, - pub default: Option, - pub py: bool, - pub attrs: PyFunctionArgPyO3Attributes, - pub is_varargs: bool, - pub is_kwargs: bool, - pub is_cancel_handle: bool, +} + +#[derive(Clone, Debug)] +pub struct PyArg<'a> { + pub name: &'a syn::Ident, + pub ty: &'a syn::Type, +} + +#[derive(Clone, Debug)] +pub enum FnArg<'a> { + Regular(RegularArg<'a>), + VarArgs(VarargsArg<'a>), + KwArgs(KwargsArg<'a>), + Py(PyArg<'a>), + CancelHandle(CancelHandleArg<'a>), } impl<'a> FnArg<'a> { + pub fn name(&self) -> &syn::Ident { + match self { + FnArg::Regular(RegularArg { name, .. }) => name, + FnArg::VarArgs(VarargsArg { name, .. }) => name, + FnArg::KwArgs(KwargsArg { name, .. }) => name, + FnArg::Py(PyArg { name, .. }) => name, + FnArg::CancelHandle(CancelHandleArg { name, .. }) => name, + } + } + + pub fn ty(&self) -> &'a syn::Type { + match self { + FnArg::Regular(RegularArg { ty, .. }) => ty, + FnArg::VarArgs(VarargsArg { ty, .. }) => ty, + FnArg::KwArgs(KwargsArg { ty, .. }) => ty, + FnArg::Py(PyArg { ty, .. }) => ty, + FnArg::CancelHandle(CancelHandleArg { ty, .. }) => ty, + } + } + + #[allow(clippy::wrong_self_convention)] + pub fn from_py_with(&self) -> Option<&FromPyWithAttribute> { + if let FnArg::Regular(RegularArg { from_py_with, .. }) = self { + from_py_with.as_ref() + } else { + None + } + } + + pub fn to_varargs_mut(&mut self) -> Result<&mut Self> { + if let Self::Regular(RegularArg { + name, + ty, + option_wrapped_type: None, + .. + }) = self + { + *self = Self::VarArgs(VarargsArg { + name: name.clone(), + ty, + }); + Ok(self) + } else { + bail_spanned!(self.name().span() => "args cannot be optional") + } + } + + pub fn to_kwargs_mut(&mut self) -> Result<&mut Self> { + if let Self::Regular(RegularArg { + name, + ty, + option_wrapped_type: Some(..), + .. + }) = self + { + *self = Self::KwArgs(KwargsArg { + name: name.clone(), + ty, + }); + Ok(self) + } else { + bail_spanned!(self.name().span() => "kwargs must be Option<_>") + } + } + /// Transforms a rust fn arg parsed with syn into a method::FnArg pub fn parse(arg: &'a mut syn::FnArg) -> Result { match arg { @@ -41,32 +139,43 @@ impl<'a> FnArg<'a> { bail_spanned!(cap.ty.span() => IMPL_TRAIT_ERR); } - let arg_attrs = PyFunctionArgPyO3Attributes::from_attrs(&mut cap.attrs)?; + let PyFunctionArgPyO3Attributes { + from_py_with, + cancel_handle, + } = PyFunctionArgPyO3Attributes::from_attrs(&mut cap.attrs)?; let ident = match &*cap.pat { syn::Pat::Ident(syn::PatIdent { ident, .. }) => ident, other => return Err(handle_argument_error(other)), }; - let is_cancel_handle = arg_attrs.cancel_handle.is_some(); + if utils::is_python(&cap.ty) { + return Ok(Self::Py(PyArg { + name: ident, + ty: &cap.ty, + })); + } + + if cancel_handle.is_some() { + // `PyFunctionArgPyO3Attributes::from_attrs` validates that + // only compatible attributes are specified, either + // `cancel_handle` or `from_py_with`, dublicates and any + // combination of the two are already rejected. + return Ok(Self::CancelHandle(CancelHandleArg { + name: ident, + ty: &cap.ty, + })); + } - Ok(FnArg { - name: ident, + Ok(Self::Regular(RegularArg { + name: Cow::Borrowed(ident), ty: &cap.ty, - optional: utils::option_type_argument(&cap.ty), - default: None, - py: utils::is_python(&cap.ty), - attrs: arg_attrs, - is_varargs: false, - is_kwargs: false, - is_cancel_handle, - }) + from_py_with, + default_value: None, + option_wrapped_type: utils::option_type_argument(&cap.ty), + })) } } } - - pub fn is_regular(&self) -> bool { - !self.py && !self.is_cancel_handle && !self.is_kwargs && !self.is_varargs - } } fn handle_argument_error(pat: &syn::Pat) -> syn::Error { @@ -82,16 +191,26 @@ fn handle_argument_error(pat: &syn::Pat) -> syn::Error { syn::Error::new(span, msg) } +/// Represents what kind of a function a pyfunction or pymethod is #[derive(Clone, Debug)] pub enum FnType { + /// Represents a pymethod annotated with `#[getter]` Getter(SelfType), + /// Represents a pymethod annotated with `#[setter]` Setter(SelfType), + /// Represents a regular pymethod Fn(SelfType), + /// Represents a pymethod annotated with `#[new]`, i.e. the `__new__` dunder. FnNew, + /// Represents a pymethod annotated with both `#[new]` and `#[classmethod]` (in either order) FnNewClass(Span), + /// Represents a pymethod annotated with `#[classmethod]`, like a `@classmethod` FnClass(Span), + /// Represents a pyfunction or a pymethod annotated with `#[staticmethod]`, like a `@staticmethod` FnStatic, + /// Represents a pyfunction annotated with `#[pyo3(pass_module)] FnModule(Span), + /// Represents a pymethod or associated constant annotated with `#[classattr]` ClassAttribute, } @@ -108,14 +227,28 @@ impl FnType { } } + pub fn signature_attribute_allowed(&self) -> bool { + match self { + FnType::Fn(_) + | FnType::FnNew + | FnType::FnStatic + | FnType::FnClass(_) + | FnType::FnNewClass(_) + | FnType::FnModule(_) => true, + // Setter, Getter and ClassAttribute all have fixed signatures (either take 0 or 1 + // arguments) so cannot have a `signature = (...)` attribute. + FnType::Getter(_) | FnType::Setter(_) | FnType::ClassAttribute => false, + } + } + pub fn self_arg( &self, cls: Option<&syn::Type>, error_mode: ExtractErrorMode, holders: &mut Holders, ctx: &Ctx, - ) -> TokenStream { - let Ctx { pyo3_path } = ctx; + ) -> Option { + let Ctx { pyo3_path, .. } = ctx; match self { FnType::Getter(st) | FnType::Setter(st) | FnType::Fn(st) => { let mut receiver = st.receiver( @@ -125,35 +258,35 @@ impl FnType { ctx, ); syn::Token![,](Span::call_site()).to_tokens(&mut receiver); - receiver - } - FnType::FnNew | FnType::FnStatic | FnType::ClassAttribute => { - quote!() + Some(receiver) } FnType::FnClass(span) | FnType::FnNewClass(span) => { let py = syn::Ident::new("py", Span::call_site()); - let slf: Ident = syn::Ident::new("_slf_ref", Span::call_site()); + let slf: Ident = syn::Ident::new("_slf", Span::call_site()); let pyo3_path = pyo3_path.to_tokens_spanned(*span); - quote_spanned! { *span => + let ret = quote_spanned! { *span => #[allow(clippy::useless_conversion)] ::std::convert::Into::into( - #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &*(#slf as *const _ as *const *mut _)) + #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &*(&#slf as *const _ as *const *mut _)) .downcast_unchecked::<#pyo3_path::types::PyType>() - ), - } + ) + }; + Some(quote! { unsafe { #ret }, }) } FnType::FnModule(span) => { let py = syn::Ident::new("py", Span::call_site()); - let slf: Ident = syn::Ident::new("_slf_ref", Span::call_site()); + let slf: Ident = syn::Ident::new("_slf", Span::call_site()); let pyo3_path = pyo3_path.to_tokens_spanned(*span); - quote_spanned! { *span => + let ret = quote_spanned! { *span => #[allow(clippy::useless_conversion)] ::std::convert::Into::into( - #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &*(#slf as *const _ as *const *mut _)) + #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &*(&#slf as *const _ as *const *mut _)) .downcast_unchecked::<#pyo3_path::types::PyModule>() - ), - } + ) + }; + Some(quote! { unsafe { #ret }, }) } + FnType::FnNew | FnType::FnStatic | FnType::ClassAttribute => None, } } } @@ -172,13 +305,13 @@ pub enum ExtractErrorMode { impl ExtractErrorMode { pub fn handle_error(self, extract: TokenStream, ctx: &Ctx) -> TokenStream { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; match self { ExtractErrorMode::Raise => quote! { #extract? }, ExtractErrorMode::NotImplemented => quote! { match #extract { ::std::result::Result::Ok(value) => value, - ::std::result::Result::Err(_) => { return #pyo3_path::callback::convert(py, py.NotImplemented()); }, + ::std::result::Result::Err(_) => { return #pyo3_path::impl_::callback::convert(py, py.NotImplemented()); }, } }, } @@ -197,7 +330,9 @@ impl SelfType { // main macro callsite. let py = syn::Ident::new("py", Span::call_site()); let slf = syn::Ident::new("_slf", Span::call_site()); - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; + let bound_ref = + quote! { unsafe { #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf) } }; match self { SelfType::Receiver { span, mutable } => { let method = if *mutable { @@ -210,7 +345,7 @@ impl SelfType { error_mode.handle_error( quote_spanned! { *span => #pyo3_path::impl_::extract_argument::#method::<#cls>( - #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf).0, + #bound_ref.0, &mut #holder, ) }, @@ -221,7 +356,7 @@ impl SelfType { let pyo3_path = pyo3_path.to_tokens_spanned(*span); error_mode.handle_error( quote_spanned! { *span => - #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf).downcast::<#cls>() + #bound_ref.downcast::<#cls>() .map_err(::std::convert::Into::<#pyo3_path::PyErr>::into) .and_then( #[allow(unknown_lints, clippy::unnecessary_fallible_conversions)] // In case slf is Py (unknown_lints can be removed when MSRV is 1.75+) @@ -241,7 +376,7 @@ impl SelfType { pub enum CallingConvention { Noargs, // METH_NOARGS Varargs, // METH_VARARGS | METH_KEYWORDS - Fastcall, // METH_FASTCALL | METH_KEYWORDS (not compatible with `abi3` feature) + Fastcall, // METH_FASTCALL | METH_KEYWORDS (not compatible with `abi3` feature before 3.10) TpNew, // special convention for tp_new } @@ -253,11 +388,11 @@ impl CallingConvention { pub fn from_signature(signature: &FunctionSignature<'_>) -> Self { if signature.python_signature.has_no_args() { Self::Noargs - } else if signature.python_signature.kwargs.is_some() { - // for functions that accept **kwargs, always prefer varargs - Self::Varargs - } else if !is_abi3() { - // FIXME: available in the stable ABI since 3.10 + } else if signature.python_signature.kwargs.is_none() && !is_abi3_before(3, 10) { + // For functions that accept **kwargs, always prefer varargs for now based on + // historical performance testing. + // + // FASTCALL not compatible with `abi3` before 3.10 Self::Fastcall } else { Self::Varargs @@ -277,7 +412,6 @@ pub struct FnSpec<'a> { pub text_signature: Option, pub asyncness: Option, pub unsafety: Option, - pub deprecations: Deprecations<'a>, } pub fn parse_method_receiver(arg: &syn::FnArg) -> Result { @@ -309,7 +443,6 @@ impl<'a> FnSpec<'a> { sig: &'a mut syn::Signature, meth_attrs: &mut Vec, options: PyFunctionOptions, - ctx: &'a Ctx, ) -> Result> { let PyFunctionOptions { text_signature, @@ -319,9 +452,8 @@ impl<'a> FnSpec<'a> { } = options; let mut python_name = name.map(|name| name.value.0); - let mut deprecations = Deprecations::new(ctx); - let fn_type = Self::parse_fn_type(sig, meth_attrs, &mut python_name, &mut deprecations)?; + let fn_type = Self::parse_fn_type(sig, meth_attrs, &mut python_name)?; ensure_signatures_on_valid_method(&fn_type, signature.as_ref(), text_signature.as_ref())?; let name = &sig.ident; @@ -341,7 +473,7 @@ impl<'a> FnSpec<'a> { let signature = if let Some(signature) = signature { FunctionSignature::from_arguments_and_attribute(arguments, signature)? } else { - FunctionSignature::from_arguments(arguments)? + FunctionSignature::from_arguments(arguments) }; let convention = if matches!(fn_type, FnType::FnNew | FnType::FnNewClass(_)) { @@ -359,21 +491,21 @@ impl<'a> FnSpec<'a> { text_signature, asyncness: sig.asyncness, unsafety: sig.unsafety, - deprecations, }) } - pub fn null_terminated_python_name(&self) -> syn::LitStr { - syn::LitStr::new(&format!("{}\0", self.python_name), self.python_name.span()) + pub fn null_terminated_python_name(&self, ctx: &Ctx) -> LitCStr { + let name = self.python_name.to_string(); + let name = CString::new(name).unwrap(); + LitCStr::new(name, self.python_name.span(), ctx) } fn parse_fn_type( sig: &syn::Signature, meth_attrs: &mut Vec, python_name: &mut Option, - deprecations: &mut Deprecations<'_>, ) -> Result { - let mut method_attributes = parse_method_attributes(meth_attrs, deprecations)?; + let mut method_attributes = parse_method_attributes(meth_attrs)?; let name = &sig.ident; let parse_receiver = |msg: &'static str| { @@ -487,17 +619,22 @@ impl<'a> FnSpec<'a> { cls: Option<&syn::Type>, ctx: &Ctx, ) -> Result { - let Ctx { pyo3_path } = ctx; + let Ctx { + pyo3_path, + output_span, + } = ctx; let mut cancel_handle_iter = self .signature .arguments .iter() - .filter(|arg| arg.is_cancel_handle); + .filter(|arg| matches!(arg, FnArg::CancelHandle(..))); let cancel_handle = cancel_handle_iter.next(); - if let Some(arg) = cancel_handle { - ensure_spanned!(self.asyncness.is_some(), arg.name.span() => "`cancel_handle` attribute can only be used with `async fn`"); - if let Some(arg2) = cancel_handle_iter.next() { - bail_spanned!(arg2.name.span() => "`cancel_handle` may only be specified once"); + if let Some(FnArg::CancelHandle(CancelHandleArg { name, .. })) = cancel_handle { + ensure_spanned!(self.asyncness.is_some(), name.span() => "`cancel_handle` attribute can only be used with `async fn`"); + if let Some(FnArg::CancelHandle(CancelHandleArg { name, .. })) = + cancel_handle_iter.next() + { + bail_spanned!(name.span() => "`cancel_handle` may only be specified once"); } } @@ -522,48 +659,35 @@ impl<'a> FnSpec<'a> { Some(cls) => quote!(Some(<#cls as #pyo3_path::PyTypeInfo>::NAME)), None => quote!(None), }; - let evaluate_args = || -> (Vec, TokenStream) { - let mut arg_names = Vec::with_capacity(args.len()); - let mut evaluate_arg = quote! {}; - for arg in &args { - let arg_name = format_ident!("arg_{}", arg_names.len()); - arg_names.push(arg_name.clone()); - evaluate_arg.extend(quote! { - let #arg_name = #arg - }); - } - (arg_names, evaluate_arg) - }; + let arg_names = (0..args.len()) + .map(|i| format_ident!("arg_{}", i)) + .collect::>(); let future = match self.tp { FnType::Fn(SelfType::Receiver { mutable: false, .. }) => { - let (arg_name, evaluate_arg) = evaluate_args(); quote! {{ - #evaluate_arg; - let __guard = #pyo3_path::impl_::coroutine::RefGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; - async move { function(&__guard, #(#arg_name),*).await } + #(let #arg_names = #args;)* + let __guard = unsafe { #pyo3_path::impl_::coroutine::RefGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))? }; + async move { function(&__guard, #(#arg_names),*).await } }} } FnType::Fn(SelfType::Receiver { mutable: true, .. }) => { - let (arg_name, evaluate_arg) = evaluate_args(); quote! {{ - #evaluate_arg; - let mut __guard = #pyo3_path::impl_::coroutine::RefMutGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; - async move { function(&mut __guard, #(#arg_name),*).await } + #(let #arg_names = #args;)* + let mut __guard = unsafe { #pyo3_path::impl_::coroutine::RefMutGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))? }; + async move { function(&mut __guard, #(#arg_names),*).await } }} } _ => { - let self_arg = self_arg(); - if self_arg.is_empty() { - quote! { function(#(#args),*) } - } else { - let self_checker = holders.push_gil_refs_checker(self_arg.span()); + if let Some(self_arg) = self_arg() { quote! { function( // NB #self_arg includes a comma, so none inserted here - #pyo3_path::impl_::deprecations::inspect_type(#self_arg &#self_checker), + #self_arg #(#args),* ) } + } else { + quote! { function(#(#args),*) } } } }; @@ -573,7 +697,10 @@ impl<'a> FnSpec<'a> { #pyo3_path::intern!(py, stringify!(#python_name)), #qualname_prefix, #throw_callback, - async move { #pyo3_path::impl_::wrap::OkWrap::wrap(future.await) }, + async move { + let fut = future.await; + #pyo3_path::impl_::wrap::converter(&fut).wrap(fut) + }, ) }}; if cancel_handle.is_some() { @@ -584,22 +711,30 @@ impl<'a> FnSpec<'a> { }}; } call - } else { - let self_arg = self_arg(); - if self_arg.is_empty() { - quote! { function(#(#args),*) } - } else { - let self_checker = holders.push_gil_refs_checker(self_arg.span()); - quote! { - function( - // NB #self_arg includes a comma, so none inserted here - #pyo3_path::impl_::deprecations::inspect_type(#self_arg &#self_checker), - #(#args),* - ) - } + } else if let Some(self_arg) = self_arg() { + quote! { + function( + // NB #self_arg includes a comma, so none inserted here + #self_arg + #(#args),* + ) } + } else { + quote! { function(#(#args),*) } }; - quotes::map_result_into_ptr(quotes::ok_wrap(call, ctx), ctx) + + // We must assign the output_span to the return value of the call, + // but *not* of the call itself otherwise the spans get really weird + let ret_ident = Ident::new("ret", *output_span); + let ret_expr = quote! { let #ret_ident = #call; }; + let return_conversion = + quotes::map_result_into_ptr(quotes::ok_wrap(ret_ident.to_token_stream(), ctx), ctx); + quote! { + { + #ret_expr + #return_conversion + } + } }; let func_name = &self.name; @@ -616,40 +751,31 @@ impl<'a> FnSpec<'a> { .signature .arguments .iter() - .map(|arg| { - if arg.py { - quote!(py) - } else if arg.is_cancel_handle { - quote!(__cancel_handle) - } else { - unreachable!() - } + .map(|arg| match arg { + FnArg::Py(..) => quote!(py), + FnArg::CancelHandle(..) => quote!(__cancel_handle), + _ => unreachable!("`CallingConvention::Noargs` should not contain any arguments (reaching Python) except for `self`, which is handled below."), }) .collect(); let call = rust_call(args, &mut holders); - let check_gil_refs = holders.check_gil_refs(); let init_holders = holders.init_holders(ctx); - quote! { unsafe fn #ident<'py>( py: #pyo3_path::Python<'py>, _slf: *mut #pyo3_path::ffi::PyObject, ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { - let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 #init_holders let result = #call; - #check_gil_refs result } } } CallingConvention::Fastcall => { let mut holders = Holders::new(); - let (arg_convert, args) = impl_arg_params(self, cls, true, &mut holders, ctx)?; + let (arg_convert, args) = impl_arg_params(self, cls, true, &mut holders, ctx); let call = rust_call(args, &mut holders); let init_holders = holders.init_holders(ctx); - let check_gil_refs = holders.check_gil_refs(); quote! { unsafe fn #ident<'py>( @@ -659,22 +785,19 @@ impl<'a> FnSpec<'a> { _nargs: #pyo3_path::ffi::Py_ssize_t, _kwnames: *mut #pyo3_path::ffi::PyObject ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { - let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert #init_holders let result = #call; - #check_gil_refs result } } } CallingConvention::Varargs => { let mut holders = Holders::new(); - let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx)?; + let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx); let call = rust_call(args, &mut holders); let init_holders = holders.init_holders(ctx); - let check_gil_refs = holders.check_gil_refs(); quote! { unsafe fn #ident<'py>( @@ -683,25 +806,22 @@ impl<'a> FnSpec<'a> { _args: *mut #pyo3_path::ffi::PyObject, _kwargs: *mut #pyo3_path::ffi::PyObject ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { - let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert #init_holders let result = #call; - #check_gil_refs result } } } CallingConvention::TpNew => { let mut holders = Holders::new(); - let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx)?; + let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx); let self_arg = self .tp .self_arg(cls, ExtractErrorMode::Raise, &mut holders, ctx); - let call = quote! { #rust_name(#self_arg #(#args),*) }; + let call = quote_spanned! {*output_span=> #rust_name(#self_arg #(#args),*) }; let init_holders = holders.init_holders(ctx); - let check_gil_refs = holders.check_gil_refs(); quote! { unsafe fn #ident( py: #pyo3_path::Python<'_>, @@ -709,14 +829,12 @@ impl<'a> FnSpec<'a> { _args: *mut #pyo3_path::ffi::PyObject, _kwargs: *mut #pyo3_path::ffi::PyObject ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { - use #pyo3_path::callback::IntoPyCallbackOutput; - let _slf_ref = &_slf; + use #pyo3_path::impl_::callback::IntoPyCallbackOutput; let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert #init_holders let result = #call; let initializer: #pyo3_path::PyClassInitializer::<#cls> = result.convert(py)?; - #check_gil_refs #pyo3_path::impl_::pymethods::tp_new_impl(py, initializer, _slf) } } @@ -727,33 +845,35 @@ impl<'a> FnSpec<'a> { /// Return a `PyMethodDef` constructor for this function, matching the selected /// calling convention. pub fn get_methoddef(&self, wrapper: impl ToTokens, doc: &PythonDoc, ctx: &Ctx) -> TokenStream { - let Ctx { pyo3_path } = ctx; - let python_name = self.null_terminated_python_name(); + let Ctx { pyo3_path, .. } = ctx; + let python_name = self.null_terminated_python_name(ctx); match self.convention { CallingConvention::Noargs => quote! { #pyo3_path::impl_::pymethods::PyMethodDef::noargs( #python_name, - #pyo3_path::impl_::pymethods::PyCFunction({ + { unsafe extern "C" fn trampoline( _slf: *mut #pyo3_path::ffi::PyObject, _args: *mut #pyo3_path::ffi::PyObject, ) -> *mut #pyo3_path::ffi::PyObject { - #pyo3_path::impl_::trampoline::noargs( - _slf, - _args, - #wrapper - ) + unsafe { + #pyo3_path::impl_::trampoline::noargs( + _slf, + _args, + #wrapper + ) + } } trampoline - }), + }, #doc, ) }, CallingConvention::Fastcall => quote! { #pyo3_path::impl_::pymethods::PyMethodDef::fastcall_cfunction_with_keywords( #python_name, - #pyo3_path::impl_::pymethods::PyCFunctionFastWithKeywords({ + { unsafe extern "C" fn trampoline( _slf: *mut #pyo3_path::ffi::PyObject, _args: *const *mut #pyo3_path::ffi::PyObject, @@ -770,14 +890,14 @@ impl<'a> FnSpec<'a> { ) } trampoline - }), + }, #doc, ) }, CallingConvention::Varargs => quote! { #pyo3_path::impl_::pymethods::PyMethodDef::cfunction_with_keywords( #python_name, - #pyo3_path::impl_::pymethods::PyCFunctionWithKeywords({ + { unsafe extern "C" fn trampoline( _slf: *mut #pyo3_path::ffi::PyObject, _args: *mut #pyo3_path::ffi::PyObject, @@ -792,7 +912,7 @@ impl<'a> FnSpec<'a> { ) } trampoline - }), + }, #doc, ) }, @@ -801,11 +921,11 @@ impl<'a> FnSpec<'a> { } /// Forwards to [utils::get_doc] with the text signature of this spec. - pub fn get_doc(&self, attrs: &[syn::Attribute]) -> PythonDoc { + pub fn get_doc(&self, attrs: &[syn::Attribute], ctx: &Ctx) -> PythonDoc { let text_signature = self .text_signature_call_signature() .map(|sig| format!("{}{}", self.python_name, sig)); - utils::get_doc(attrs, text_signature) + utils::get_doc(attrs, text_signature, ctx) } /// Creates the parenthesised arguments list for `__text_signature__` snippet based on this spec's signature @@ -854,10 +974,7 @@ impl MethodTypeAttribute { /// If the attribute does not match one of the attribute names, returns `Ok(None)`. /// /// Otherwise will either return a parse error or the attribute. - fn parse_if_matching_attribute( - attr: &syn::Attribute, - deprecations: &mut Deprecations<'_>, - ) -> Result> { + fn parse_if_matching_attribute(attr: &syn::Attribute) -> Result> { fn ensure_no_arguments(meta: &syn::Meta, ident: &str) -> syn::Result<()> { match meta { syn::Meta::Path(_) => Ok(()), @@ -901,11 +1018,6 @@ impl MethodTypeAttribute { if path.is_ident("new") { ensure_no_arguments(meta, "new")?; Ok(Some(MethodTypeAttribute::New(path.span()))) - } else if path.is_ident("__new__") { - let span = path.span(); - deprecations.push(Deprecation::PyMethodsNewDeprecatedForm, span); - ensure_no_arguments(meta, "__new__")?; - Ok(Some(MethodTypeAttribute::New(span))) } else if path.is_ident("classmethod") { ensure_no_arguments(meta, "classmethod")?; Ok(Some(MethodTypeAttribute::ClassMethod(path.span()))) @@ -940,15 +1052,12 @@ impl Display for MethodTypeAttribute { } } -fn parse_method_attributes( - attrs: &mut Vec, - deprecations: &mut Deprecations<'_>, -) -> Result> { +fn parse_method_attributes(attrs: &mut Vec) -> Result> { let mut new_attrs = Vec::new(); let mut found_attrs = Vec::new(); for attr in attrs.drain(..) { - match MethodTypeAttribute::parse_if_matching_attribute(&attr, deprecations)? { + match MethodTypeAttribute::parse_if_matching_attribute(&attr)? { Some(attr) => found_attrs.push(attr), None => new_attrs.push(attr), } @@ -972,15 +1081,18 @@ fn ensure_signatures_on_valid_method( if let Some(signature) = signature { match fn_type { FnType::Getter(_) => { + debug_assert!(!fn_type.signature_attribute_allowed()); bail_spanned!(signature.kw.span() => "`signature` not allowed with `getter`") } FnType::Setter(_) => { + debug_assert!(!fn_type.signature_attribute_allowed()); bail_spanned!(signature.kw.span() => "`signature` not allowed with `setter`") } FnType::ClassAttribute => { + debug_assert!(!fn_type.signature_attribute_allowed()); bail_spanned!(signature.kw.span() => "`signature` not allowed with `classattr`") } - _ => {} + _ => debug_assert!(fn_type.signature_attribute_allowed()), } } if let Some(text_signature) = text_signature { diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 080a279a88c..d9fac3cbd7b 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -1,82 +1,117 @@ //! Code generation for the function that initializes a python module and adds classes and function. -use crate::utils::Ctx; use crate::{ - attributes::{self, take_attributes, take_pyo3_options, CrateAttribute, NameAttribute}, + attributes::{ + self, kw, take_attributes, take_pyo3_options, CrateAttribute, GILUsedAttribute, + ModuleAttribute, NameAttribute, SubmoduleAttribute, + }, get_doc, + pyclass::PyClassPyO3Option, pyfunction::{impl_wrap_pyfunction, PyFunctionOptions}, + utils::{has_attribute, has_attribute_with_namespace, Ctx, IdentOrStr, LitCStr}, }; -use proc_macro2::TokenStream; +use proc_macro2::{Span, TokenStream}; use quote::quote; +use std::ffi::CString; use syn::{ ext::IdentExt, parse::{Parse, ParseStream}, parse_quote, parse_quote_spanned, + punctuated::Punctuated, spanned::Spanned, token::Comma, - Item, Path, Result, + Item, Meta, Path, Result, }; #[derive(Default)] pub struct PyModuleOptions { krate: Option, - name: Option, + name: Option, + module: Option, + submodule: Option, + gil_used: Option, } -impl PyModuleOptions { - pub fn from_attrs(attrs: &mut Vec) -> Result { +impl Parse for PyModuleOptions { + fn parse(input: ParseStream<'_>) -> syn::Result { let mut options: PyModuleOptions = Default::default(); - for option in take_pyo3_options(attrs)? { - match option { - PyModulePyO3Option::Name(name) => options.set_name(name.value.0)?, - PyModulePyO3Option::Crate(path) => options.set_crate(path)?, - } - } + options.add_attributes( + Punctuated::::parse_terminated(input)?, + )?; Ok(options) } +} - fn set_name(&mut self, name: syn::Ident) -> Result<()> { - ensure_spanned!( - self.name.is_none(), - name.span() => "`name` may only be specified once" - ); - - self.name = Some(name); - Ok(()) +impl PyModuleOptions { + fn take_pyo3_options(&mut self, attrs: &mut Vec) -> Result<()> { + self.add_attributes(take_pyo3_options(attrs)?) } - fn set_crate(&mut self, path: CrateAttribute) -> Result<()> { - ensure_spanned!( - self.krate.is_none(), - path.span() => "`crate` may only be specified once" - ); - - self.krate = Some(path); + fn add_attributes( + &mut self, + attrs: impl IntoIterator, + ) -> Result<()> { + macro_rules! set_option { + ($key:ident $(, $extra:literal)?) => { + { + ensure_spanned!( + self.$key.is_none(), + $key.span() => concat!("`", stringify!($key), "` may only be specified once" $(, $extra)?) + ); + self.$key = Some($key); + } + }; + } + for attr in attrs { + match attr { + PyModulePyO3Option::Crate(krate) => set_option!(krate), + PyModulePyO3Option::Name(name) => set_option!(name), + PyModulePyO3Option::Module(module) => set_option!(module), + PyModulePyO3Option::Submodule(submodule) => set_option!( + submodule, + " (it is implicitly always specified for nested modules)" + ), + PyModulePyO3Option::GILUsed(gil_used) => { + set_option!(gil_used) + } + } + } Ok(()) } } -pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { +pub fn pymodule_module_impl( + module: &mut syn::ItemMod, + mut options: PyModuleOptions, +) -> Result { let syn::ItemMod { attrs, vis, unsafety: _, ident, - mod_token: _, + mod_token, content, semi: _, - } = &mut module; + } = module; let items = if let Some((_, items)) = content { items } else { - bail_spanned!(module.span() => "`#[pymodule]` can only be used on inline modules") + bail_spanned!(mod_token.span() => "`#[pymodule]` can only be used on inline modules") + }; + options.take_pyo3_options(attrs)?; + let ctx = &Ctx::new(&options.krate, None); + let Ctx { pyo3_path, .. } = ctx; + let doc = get_doc(attrs, None, ctx); + let name = options + .name + .map_or_else(|| ident.unraw(), |name| name.value.0); + let full_name = if let Some(module) = &options.module { + format!("{}.{}", module.value.value(), name) + } else { + name.to_string() }; - let options = PyModuleOptions::from_attrs(attrs)?; - let ctx = &Ctx::new(&options.krate); - let Ctx { pyo3_path } = ctx; - let doc = get_doc(attrs, None); let mut module_items = Vec::new(); let mut module_items_cfg_attrs = Vec::new(); @@ -143,7 +178,18 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { ); ensure_spanned!(pymodule_init.is_none(), item_fn.span() => "only one `#[pymodule_init]` may be specified"); pymodule_init = Some(quote! { #ident(module)?; }); - } else if has_attribute(&item_fn.attrs, "pyfunction") { + } else if has_attribute(&item_fn.attrs, "pyfunction") + || has_attribute_with_namespace( + &item_fn.attrs, + Some(pyo3_path), + &["pyfunction"], + ) + || has_attribute_with_namespace( + &item_fn.attrs, + Some(pyo3_path), + &["prelude", "pyfunction"], + ) + { module_items.push(ident.clone()); module_items_cfg_attrs.push(get_cfg_attributes(&item_fn.attrs)); } @@ -153,9 +199,27 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { !has_attribute(&item_struct.attrs, "pymodule_export"), item.span() => "`#[pymodule_export]` may only be used on `use` statements" ); - if has_attribute(&item_struct.attrs, "pyclass") { + if has_attribute(&item_struct.attrs, "pyclass") + || has_attribute_with_namespace( + &item_struct.attrs, + Some(pyo3_path), + &["pyclass"], + ) + || has_attribute_with_namespace( + &item_struct.attrs, + Some(pyo3_path), + &["prelude", "pyclass"], + ) + { module_items.push(item_struct.ident.clone()); module_items_cfg_attrs.push(get_cfg_attributes(&item_struct.attrs)); + if !has_pyo3_module_declared::( + &item_struct.attrs, + "pyclass", + |option| matches!(option, PyClassPyO3Option::Module(_)), + )? { + set_module_attribute(&mut item_struct.attrs, &full_name); + } } } Item::Enum(item_enum) => { @@ -163,9 +227,23 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { !has_attribute(&item_enum.attrs, "pymodule_export"), item.span() => "`#[pymodule_export]` may only be used on `use` statements" ); - if has_attribute(&item_enum.attrs, "pyclass") { + if has_attribute(&item_enum.attrs, "pyclass") + || has_attribute_with_namespace(&item_enum.attrs, Some(pyo3_path), &["pyclass"]) + || has_attribute_with_namespace( + &item_enum.attrs, + Some(pyo3_path), + &["prelude", "pyclass"], + ) + { module_items.push(item_enum.ident.clone()); module_items_cfg_attrs.push(get_cfg_attributes(&item_enum.attrs)); + if !has_pyo3_module_declared::( + &item_enum.attrs, + "pyclass", + |option| matches!(option, PyClassPyO3Option::Module(_)), + )? { + set_module_attribute(&mut item_enum.attrs, &full_name); + } } } Item::Mod(item_mod) => { @@ -173,9 +251,26 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { !has_attribute(&item_mod.attrs, "pymodule_export"), item.span() => "`#[pymodule_export]` may only be used on `use` statements" ); - if has_attribute(&item_mod.attrs, "pymodule") { + if has_attribute(&item_mod.attrs, "pymodule") + || has_attribute_with_namespace(&item_mod.attrs, Some(pyo3_path), &["pymodule"]) + || has_attribute_with_namespace( + &item_mod.attrs, + Some(pyo3_path), + &["prelude", "pymodule"], + ) + { module_items.push(item_mod.ident.clone()); module_items_cfg_attrs.push(get_cfg_attributes(&item_mod.attrs)); + if !has_pyo3_module_declared::( + &item_mod.attrs, + "pymodule", + |option| matches!(option, PyModulePyO3Option::Module(_)), + )? { + set_module_attribute(&mut item_mod.attrs, &full_name); + } + item_mod + .attrs + .push(parse_quote_spanned!(item_mod.mod_token.span()=> #[pyo3(submodule)])); } } Item::ForeignMod(item) => { @@ -242,27 +337,32 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { } } - let initialization = module_initialization(options, ident); + let module_def = quote! {{ + use #pyo3_path::impl_::pymodule as impl_; + const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(__pyo3_pymodule); + unsafe { + impl_::ModuleDef::new( + __PYO3_NAME, + #doc, + INITIALIZER + ) + } + }}; + let initialization = module_initialization( + &name, + ctx, + module_def, + options.submodule.is_some(), + options.gil_used.map_or(true, |op| op.value.value), + ); + Ok(quote!( - #vis mod #ident { + #(#attrs)* + #vis #mod_token #ident { #(#items)* #initialization - impl MakeDef { - const fn make_def() -> #pyo3_path::impl_::pymodule::ModuleDef { - use #pyo3_path::impl_::pymodule as impl_; - const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(__pyo3_pymodule); - unsafe { - impl_::ModuleDef::new( - __PYO3_NAME, - #doc, - INITIALIZER - ) - } - } - } - fn __pyo3_pymodule(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { use #pyo3_path::impl_::pymodule::PyAddToModule; #( @@ -270,7 +370,7 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { #module_items::_PYO3_DEF.add_to_module(module)?; )* #pymodule_init - Ok(()) + ::std::result::Result::Ok(()) } } )) @@ -278,17 +378,28 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { /// Generates the function that is called by the python interpreter to initialize the native /// module -pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result { - let options = PyModuleOptions::from_attrs(&mut function.attrs)?; - process_functions_in_module(&options, &mut function)?; - let ctx = &Ctx::new(&options.krate); - let stmts = std::mem::take(&mut function.block.stmts); - let Ctx { pyo3_path } = ctx; +pub fn pymodule_function_impl( + function: &mut syn::ItemFn, + mut options: PyModuleOptions, +) -> Result { + options.take_pyo3_options(&mut function.attrs)?; + process_functions_in_module(&options, function)?; + let ctx = &Ctx::new(&options.krate, None); + let Ctx { pyo3_path, .. } = ctx; let ident = &function.sig.ident; + let name = options + .name + .map_or_else(|| ident.unraw(), |name| name.value.0); let vis = &function.vis; - let doc = get_doc(&function.attrs, None); + let doc = get_doc(&function.attrs, None, ctx); - let initialization = module_initialization(options, ident); + let initialization = module_initialization( + &name, + ctx, + quote! { MakeDef::make_def() }, + false, + options.gil_used.map_or(true, |op| op.value.value), + ); // Module function called with optional Python<'_> marker as first arg, followed by the module. let mut module_args = Vec::new(); @@ -298,32 +409,8 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result module_args .push(quote!(::std::convert::Into::into(#pyo3_path::impl_::pymethods::BoundRef(module)))); - let extractors = function - .sig - .inputs - .iter() - .filter_map(|param| { - if let syn::FnArg::Typed(pat_type) = param { - if let syn::Pat::Ident(pat_ident) = &*pat_type.pat { - let ident: &syn::Ident = &pat_ident.ident; - return Some([ - parse_quote!{ let check_gil_refs = #pyo3_path::impl_::deprecations::GilRefs::new(); }, - parse_quote! { let #ident = #pyo3_path::impl_::deprecations::inspect_type(#ident, &check_gil_refs); }, - parse_quote_spanned! { pat_type.span() => check_gil_refs.function_arg(); }, - ]); - } - } - None - }) - .flatten(); - - function.block.stmts = extractors.chain(stmts).collect(); - function - .attrs - .push(parse_quote!(#[allow(clippy::used_underscore_binding)])); - Ok(quote! { - #function + #[doc(hidden)] #vis mod #ident { #initialization } @@ -332,6 +419,7 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result // this avoids complications around the fact that the generated module has a different scope // (and `super` doesn't always refer to the outer scope, e.g. if the `#[pymodule] is // inside a function body) + #[allow(unknown_lints, non_local_definitions)] impl #ident::MakeDef { const fn make_def() -> #pyo3_path::impl_::pymodule::ModuleDef { fn __pyo3_pymodule(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { @@ -351,38 +439,48 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result }) } -fn module_initialization(options: PyModuleOptions, ident: &syn::Ident) -> TokenStream { - let name = options.name.unwrap_or_else(|| ident.unraw()); - let ctx = &Ctx::new(&options.krate); - let Ctx { pyo3_path } = ctx; +fn module_initialization( + name: &syn::Ident, + ctx: &Ctx, + module_def: TokenStream, + is_submodule: bool, + gil_used: bool, +) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let pyinit_symbol = format!("PyInit_{}", name); + let name = name.to_string(); + let pyo3_name = LitCStr::new(CString::new(name).unwrap(), Span::call_site(), ctx); - quote! { + let mut result = quote! { #[doc(hidden)] - pub const __PYO3_NAME: &'static str = concat!(stringify!(#name), "\0"); + pub const __PYO3_NAME: &'static ::std::ffi::CStr = #pyo3_name; pub(super) struct MakeDef; #[doc(hidden)] - pub static _PYO3_DEF: #pyo3_path::impl_::pymodule::ModuleDef = MakeDef::make_def(); - - /// This autogenerated function is called by the python interpreter when importing - /// the module. + pub static _PYO3_DEF: #pyo3_path::impl_::pymodule::ModuleDef = #module_def; #[doc(hidden)] - #[export_name = #pyinit_symbol] - pub unsafe extern "C" fn __pyo3_init() -> *mut #pyo3_path::ffi::PyObject { - #pyo3_path::impl_::trampoline::module_init(|py| _PYO3_DEF.make_module(py)) - } + // so wrapped submodules can see what gil_used is + pub static __PYO3_GIL_USED: bool = #gil_used; + }; + if !is_submodule { + result.extend(quote! { + /// This autogenerated function is called by the python interpreter when importing + /// the module. + #[doc(hidden)] + #[export_name = #pyinit_symbol] + pub unsafe extern "C" fn __pyo3_init() -> *mut #pyo3_path::ffi::PyObject { + unsafe { #pyo3_path::impl_::trampoline::module_init(|py| _PYO3_DEF.make_module(py, #gil_used)) } + } + }); } + result } /// Finds and takes care of the #[pyfn(...)] in `#[pymodule]` fn process_functions_in_module(options: &PyModuleOptions, func: &mut syn::ItemFn) -> Result<()> { - let ctx = &Ctx::new(&options.krate); - let Ctx { pyo3_path } = ctx; - let mut stmts: Vec = vec![syn::parse_quote!( - #[allow(unknown_lints, unused_imports, redundant_imports)] - use #pyo3_path::{PyNativeType, types::PyModuleMethods}; - )]; + let ctx = &Ctx::new(&options.krate, None); + let Ctx { pyo3_path, .. } = ctx; + let mut stmts: Vec = Vec::new(); for mut stmt in func.block.stmts.drain(..) { if let syn::Stmt::Item(Item::Fn(func)) = &mut stmt { @@ -392,7 +490,10 @@ fn process_functions_in_module(options: &PyModuleOptions, func: &mut syn::ItemFn let name = &func.sig.ident; let statements: Vec = syn::parse_quote! { #wrapped_function - #module_name.as_borrowed().add_function(#pyo3_path::wrap_pyfunction!(#name, #module_name.as_borrowed())?)?; + { + use #pyo3_path::types::PyModuleMethods; + #module_name.add_function(#pyo3_path::wrap_pyfunction!(#name, #module_name.as_borrowed())?)?; + } }; stmts.extend(statements); } @@ -478,13 +579,44 @@ fn find_and_remove_attribute(attrs: &mut Vec, ident: &str) -> bo found } -fn has_attribute(attrs: &[syn::Attribute], ident: &str) -> bool { - attrs.iter().any(|attr| attr.path().is_ident(ident)) +impl PartialEq for IdentOrStr<'_> { + fn eq(&self, other: &syn::Ident) -> bool { + match self { + IdentOrStr::Str(s) => other == s, + IdentOrStr::Ident(i) => other == i, + } + } +} + +fn set_module_attribute(attrs: &mut Vec, module_name: &str) { + attrs.push(parse_quote!(#[pyo3(module = #module_name)])); +} + +fn has_pyo3_module_declared( + attrs: &[syn::Attribute], + root_attribute_name: &str, + is_module_option: impl Fn(&T) -> bool + Copy, +) -> Result { + for attr in attrs { + if (attr.path().is_ident("pyo3") || attr.path().is_ident(root_attribute_name)) + && matches!(attr.meta, Meta::List(_)) + { + for option in &attr.parse_args_with(Punctuated::::parse_terminated)? { + if is_module_option(option) { + return Ok(true); + } + } + } + } + Ok(false) } enum PyModulePyO3Option { + Submodule(SubmoduleAttribute), Crate(CrateAttribute), Name(NameAttribute), + Module(ModuleAttribute), + GILUsed(GILUsedAttribute), } impl Parse for PyModulePyO3Option { @@ -494,6 +626,12 @@ impl Parse for PyModulePyO3Option { input.parse().map(PyModulePyO3Option::Name) } else if lookahead.peek(syn::Token![crate]) { input.parse().map(PyModulePyO3Option::Crate) + } else if lookahead.peek(attributes::kw::module) { + input.parse().map(PyModulePyO3Option::Module) + } else if lookahead.peek(attributes::kw::submodule) { + input.parse().map(PyModulePyO3Option::Submodule) + } else if lookahead.peek(attributes::kw::gil_used) { + input.parse().map(PyModulePyO3Option::GILUsed) } else { Err(lookahead.error()) } diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index fa50d260986..ae7a6c916a8 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -1,24 +1,22 @@ -use crate::utils::Ctx; +use crate::utils::{deprecated_from_py_with, Ctx, TypeExt as _}; use crate::{ - method::{FnArg, FnSpec}, + attributes::FromPyWithAttribute, + method::{FnArg, FnSpec, RegularArg}, pyfunction::FunctionSignature, quotes::some_wrap, }; use proc_macro2::{Span, TokenStream}; -use quote::{quote, quote_spanned}; +use quote::{format_ident, quote, quote_spanned}; use syn::spanned::Spanned; -use syn::Result; pub struct Holders { holders: Vec, - gil_refs_checkers: Vec, } impl Holders { pub fn new() -> Self { Holders { holders: Vec::new(), - gil_refs_checkers: Vec::new(), } } @@ -28,71 +26,33 @@ impl Holders { holder } - pub fn push_gil_refs_checker(&mut self, span: Span) -> syn::Ident { - let gil_refs_checker = syn::Ident::new( - &format!("gil_refs_checker_{}", self.gil_refs_checkers.len()), - span, - ); - self.gil_refs_checkers.push(gil_refs_checker.clone()); - gil_refs_checker - } - pub fn init_holders(&self, ctx: &Ctx) -> TokenStream { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let holders = &self.holders; - let gil_refs_checkers = &self.gil_refs_checkers; quote! { #[allow(clippy::let_unit_value)] #(let mut #holders = #pyo3_path::impl_::extract_argument::FunctionArgumentHolder::INIT;)* - #(let #gil_refs_checkers = #pyo3_path::impl_::deprecations::GilRefs::new();)* } } - - pub fn check_gil_refs(&self) -> TokenStream { - self.gil_refs_checkers - .iter() - .map(|e| quote_spanned! { e.span() => #e.function_arg(); }) - .collect() - } } /// Return true if the argument list is simply (*args, **kwds). pub fn is_forwarded_args(signature: &FunctionSignature<'_>) -> bool { matches!( signature.arguments.as_slice(), - [ - FnArg { - is_varargs: true, - .. - }, - FnArg { - is_kwargs: true, - .. - }, - ] + [FnArg::VarArgs(..), FnArg::KwArgs(..),] ) } -pub(crate) fn check_arg_for_gil_refs( - tokens: TokenStream, - gil_refs_checker: syn::Ident, - ctx: &Ctx, -) -> TokenStream { - let Ctx { pyo3_path } = ctx; - quote! { - #pyo3_path::impl_::deprecations::inspect_type(#tokens, &#gil_refs_checker) - } -} - pub fn impl_arg_params( spec: &FnSpec<'_>, self_: Option<&syn::Type>, fastcall: bool, holders: &mut Holders, ctx: &Ctx, -) -> Result<(TokenStream, Vec)> { +) -> (TokenStream, Vec) { let args_array = syn::Ident::new("output", Span::call_site()); - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let from_py_with = spec .signature @@ -100,13 +60,12 @@ pub fn impl_arg_params( .iter() .enumerate() .filter_map(|(i, arg)| { - let from_py_with = &arg.attrs.from_py_with.as_ref()?.value; - let from_py_with_holder = - syn::Ident::new(&format!("from_py_with_{}", i), Span::call_site()); + let from_py_with = &arg.from_py_with()?.value; + let from_py_with_holder = format_ident!("from_py_with_{}", i); + let d = deprecated_from_py_with(from_py_with).unwrap_or_default(); Some(quote_spanned! { from_py_with.span() => - let e = #pyo3_path::impl_::deprecations::GilRefs::new(); - let #from_py_with_holder = #pyo3_path::impl_::deprecations::inspect_fn(#from_py_with, &e); - e.from_py_with_arg(); + #d + let #from_py_with_holder = #from_py_with; }) }) .collect::(); @@ -119,28 +78,16 @@ pub fn impl_arg_params( .arguments .iter() .enumerate() - .map(|(i, arg)| { - let from_py_with = - syn::Ident::new(&format!("from_py_with_{}", i), Span::call_site()); - let arg_value = quote!(#args_array[0].as_deref()); - - impl_arg_param(arg, from_py_with, arg_value, holders, ctx).map(|tokens| { - check_arg_for_gil_refs( - tokens, - holders.push_gil_refs_checker(arg.ty.span()), - ctx, - ) - }) - }) - .collect::>()?; - return Ok(( + .map(|(i, arg)| impl_arg_param(arg, i, &mut 0, holders, ctx)) + .collect(); + return ( quote! { - let _args = #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_args); + let _args = unsafe { #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_args) }; let _kwargs = #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr_or_opt(py, &_kwargs); #from_py_with }, arg_convert, - )); + ); }; let positional_parameter_names = &spec.signature.python_signature.positional_parameters; @@ -171,18 +118,8 @@ pub fn impl_arg_params( .arguments .iter() .enumerate() - .map(|(i, arg)| { - let from_py_with = syn::Ident::new(&format!("from_py_with_{}", i), Span::call_site()); - let arg_value = quote!(#args_array[#option_pos].as_deref()); - if arg.is_regular() { - option_pos += 1; - } - - impl_arg_param(arg, from_py_with, arg_value, holders, ctx).map(|tokens| { - check_arg_for_gil_refs(tokens, holders.push_gil_refs_checker(arg.ty.span()), ctx) - }) - }) - .collect::>()?; + .map(|(i, arg)| impl_arg_param(arg, i, &mut option_pos, holders, ctx)) + .collect(); let args_handler = if spec.signature.python_signature.varargs.is_some() { quote! { #pyo3_path::impl_::extract_argument::TupleVarargs } @@ -224,7 +161,7 @@ pub fn impl_arg_params( }; // create array of arguments, and then parse - Ok(( + ( quote! { const DESCRIPTION: #pyo3_path::impl_::extract_argument::FunctionDescription = #pyo3_path::impl_::extract_argument::FunctionDescription { cls_name: #cls_name, @@ -239,91 +176,93 @@ pub fn impl_arg_params( #from_py_with }, param_conversion, - )) + ) +} + +fn impl_arg_param( + arg: &FnArg<'_>, + pos: usize, + option_pos: &mut usize, + holders: &mut Holders, + ctx: &Ctx, +) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; + let args_array = syn::Ident::new("output", Span::call_site()); + + match arg { + FnArg::Regular(arg) => { + let from_py_with = format_ident!("from_py_with_{}", pos); + let arg_value = quote!(#args_array[#option_pos].as_deref()); + *option_pos += 1; + impl_regular_arg_param(arg, from_py_with, arg_value, holders, ctx) + } + FnArg::VarArgs(arg) => { + let holder = holders.push_holder(arg.name.span()); + let name_str = arg.name.to_string(); + quote_spanned! { arg.name.span() => + #pyo3_path::impl_::extract_argument::extract_argument::<_, false>( + &_args, + &mut #holder, + #name_str + )? + } + } + FnArg::KwArgs(arg) => { + let holder = holders.push_holder(arg.name.span()); + let name_str = arg.name.to_string(); + quote_spanned! { arg.name.span() => + #pyo3_path::impl_::extract_argument::extract_optional_argument::<_, false>( + _kwargs.as_deref(), + &mut #holder, + #name_str, + || ::std::option::Option::None + )? + } + } + FnArg::Py(..) => quote! { py }, + FnArg::CancelHandle(..) => quote! { __cancel_handle }, + } } /// Re option_pos: The option slice doesn't contain the py: Python argument, so the argument /// index and the index in option diverge when using py: Python -pub(crate) fn impl_arg_param( - arg: &FnArg<'_>, +pub(crate) fn impl_regular_arg_param( + arg: &RegularArg<'_>, from_py_with: syn::Ident, arg_value: TokenStream, // expected type: Option<&'a Bound<'py, PyAny>> holders: &mut Holders, ctx: &Ctx, -) -> Result { - let Ctx { pyo3_path } = ctx; +) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let pyo3_path = pyo3_path.to_tokens_spanned(arg.ty.span()); // Use this macro inside this function, to ensure that all code generated here is associated // with the function argument + let use_probe = quote!(use #pyo3_path::impl_::pyclass::Probe as _;); macro_rules! quote_arg_span { - ($($tokens:tt)*) => { quote_spanned!(arg.ty.span() => $($tokens)*) } - } - - if arg.py { - return Ok(quote! { py }); - } - - if arg.is_cancel_handle { - return Ok(quote! { __cancel_handle }); + ($($tokens:tt)*) => { quote_spanned!(arg.ty.span() => { #use_probe $($tokens)* }) } } - let name = arg.name; - let name_str = name.to_string(); - - if arg.is_varargs { - ensure_spanned!( - arg.optional.is_none(), - arg.name.span() => "args cannot be optional" - ); - let holder = holders.push_holder(arg.ty.span()); - return Ok(quote_arg_span! { - #pyo3_path::impl_::extract_argument::extract_argument( - &_args, - &mut #holder, - #name_str - )? - }); - } else if arg.is_kwargs { - ensure_spanned!( - arg.optional.is_some(), - arg.name.span() => "kwargs must be Option<_>" - ); - let holder = holders.push_holder(arg.name.span()); - return Ok(quote_arg_span! { - #pyo3_path::impl_::extract_argument::extract_optional_argument( - _kwargs.as_deref(), - &mut #holder, - #name_str, - || ::std::option::Option::None - )? - }); - } - - let mut default = arg.default.as_ref().map(|expr| quote!(#expr)); + let name_str = arg.name.to_string(); + let mut default = arg.default_value.as_ref().map(|expr| quote!(#expr)); // Option arguments have special treatment: the default should be specified _without_ the // Some() wrapper. Maybe this should be changed in future?! - if arg.optional.is_some() { - default = Some(default.map_or_else( - || quote!(::std::option::Option::None), - |tokens| some_wrap(tokens, ctx), - )); + if arg.option_wrapped_type.is_some() { + default = default.map(|tokens| some_wrap(tokens, ctx)); } - let tokens = if arg - .attrs - .from_py_with - .as_ref() - .map(|attr| &attr.value) - .is_some() - { + let arg_ty = arg.ty.clone().elide_lifetimes(); + if let Some(FromPyWithAttribute { kw, .. }) = arg.from_py_with { + let extractor = quote_spanned! { kw.span => + { let from_py_with: fn(_) -> _ = #from_py_with; from_py_with } + }; if let Some(default) = default { quote_arg_span! { #pyo3_path::impl_::extract_argument::from_py_with_with_default( #arg_value, #name_str, - #from_py_with as fn(_) -> _, + #extractor, #[allow(clippy::redundant_closure)] { || #default @@ -331,50 +270,61 @@ pub(crate) fn impl_arg_param( )? } } else { + let unwrap = quote! {unsafe { #pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value) }}; quote_arg_span! { #pyo3_path::impl_::extract_argument::from_py_with( - #pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value), + #unwrap, #name_str, - #from_py_with as fn(_) -> _, + #extractor, )? } } - } else if arg.optional.is_some() { - let holder = holders.push_holder(arg.name.span()); - quote_arg_span! { - #pyo3_path::impl_::extract_argument::extract_optional_argument( - #arg_value, - &mut #holder, - #name_str, - #[allow(clippy::redundant_closure)] - { - || #default - } - )? - } } else if let Some(default) = default { let holder = holders.push_holder(arg.name.span()); - quote_arg_span! { - #pyo3_path::impl_::extract_argument::extract_argument_with_default( - #arg_value, - &mut #holder, - #name_str, - #[allow(clippy::redundant_closure)] - { - || #default - } - )? + if let Some(arg_ty) = arg.option_wrapped_type { + let arg_ty = arg_ty.clone().elide_lifetimes(); + quote_arg_span! { + #pyo3_path::impl_::extract_argument::extract_optional_argument::< + _, + { #pyo3_path::impl_::pyclass::IsOption::<#arg_ty>::VALUE } + >( + #arg_value, + &mut #holder, + #name_str, + #[allow(clippy::redundant_closure)] + { + || #default + } + )? + } + } else { + quote_arg_span! { + #pyo3_path::impl_::extract_argument::extract_argument_with_default::< + _, + { #pyo3_path::impl_::pyclass::IsOption::<#arg_ty>::VALUE } + >( + #arg_value, + &mut #holder, + #name_str, + #[allow(clippy::redundant_closure)] + { + || #default + } + )? + } } } else { let holder = holders.push_holder(arg.name.span()); + let unwrap = quote! {unsafe { #pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value) }}; quote_arg_span! { - #pyo3_path::impl_::extract_argument::extract_argument( - #pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value), + #pyo3_path::impl_::extract_argument::extract_argument::< + _, + { #pyo3_path::impl_::pyclass::IsOption::<#arg_ty>::VALUE } + >( + #unwrap, &mut #holder, #name_str )? } - }; - - Ok(tokens) + } } diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index cf8d9aae801..7512d63c9fb 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1,27 +1,31 @@ use std::borrow::Cow; +use std::fmt::Debug; + +use proc_macro2::{Ident, Span, TokenStream}; +use quote::{format_ident, quote, quote_spanned, ToTokens}; +use syn::ext::IdentExt; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; +use syn::{parse_quote, parse_quote_spanned, spanned::Spanned, ImplItemFn, Result, Token}; use crate::attributes::kw::frozen; use crate::attributes::{ - self, kw, take_pyo3_options, CrateAttribute, ExtendsAttribute, FreelistAttribute, - ModuleAttribute, NameAttribute, NameLitStr, RenameAllAttribute, + self, kw, take_pyo3_options, CrateAttribute, ErrorCombiner, ExtendsAttribute, + FreelistAttribute, ModuleAttribute, NameAttribute, NameLitStr, RenameAllAttribute, + StrFormatterAttribute, }; -use crate::deprecations::Deprecations; use crate::konst::{ConstAttributes, ConstSpec}; -use crate::method::{FnArg, FnSpec}; -use crate::pyimpl::{gen_py_const, PyClassMethodsType}; +use crate::method::{FnArg, FnSpec, PyArg, RegularArg}; +use crate::pyfunction::ConstructorAttribute; +use crate::pyimpl::{gen_py_const, get_cfg_attributes, PyClassMethodsType}; use crate::pymethod::{ - impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, MethodAndSlotDef, PropertyType, - SlotDef, __INT__, __REPR__, __RICHCMP__, + impl_py_class_attribute, impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, + MethodAndSlotDef, PropertyType, SlotDef, __GETITEM__, __HASH__, __INT__, __LEN__, __REPR__, + __RICHCMP__, __STR__, }; -use crate::utils::Ctx; -use crate::utils::{self, apply_renaming_rule, PythonDoc}; +use crate::pyversions::is_abi3_before; +use crate::utils::{self, apply_renaming_rule, Ctx, LitCStr, PythonDoc}; use crate::PyFunctionOptions; -use proc_macro2::{Ident, Span, TokenStream}; -use quote::{format_ident, quote}; -use syn::ext::IdentExt; -use syn::parse::{Parse, ParseStream}; -use syn::punctuated::Punctuated; -use syn::{parse_quote, spanned::Spanned, Result, Token}; /// If the class is derived from a Rust `struct` or `enum`. #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -45,7 +49,7 @@ impl PyClassArgs { }) } - pub fn parse_stuct_args(input: ParseStream<'_>) -> syn::Result { + pub fn parse_struct_args(input: ParseStream<'_>) -> syn::Result { Self::parse(input, PyClassKind::Struct) } @@ -58,34 +62,44 @@ impl PyClassArgs { pub struct PyClassPyO3Options { pub krate: Option, pub dict: Option, + pub eq: Option, + pub eq_int: Option, pub extends: Option, pub get_all: Option, pub freelist: Option, pub frozen: Option, + pub hash: Option, pub mapping: Option, pub module: Option, pub name: Option, + pub ord: Option, pub rename_all: Option, pub sequence: Option, pub set_all: Option, + pub str: Option, pub subclass: Option, pub unsendable: Option, pub weakref: Option, } -enum PyClassPyO3Option { +pub enum PyClassPyO3Option { Crate(CrateAttribute), Dict(kw::dict), + Eq(kw::eq), + EqInt(kw::eq_int), Extends(ExtendsAttribute), Freelist(FreelistAttribute), Frozen(kw::frozen), GetAll(kw::get_all), + Hash(kw::hash), Mapping(kw::mapping), Module(ModuleAttribute), Name(NameAttribute), + Ord(kw::ord), RenameAll(RenameAllAttribute), Sequence(kw::sequence), SetAll(kw::set_all), + Str(StrFormatterAttribute), Subclass(kw::subclass), Unsendable(kw::unsendable), Weakref(kw::weakref), @@ -98,6 +112,10 @@ impl Parse for PyClassPyO3Option { input.parse().map(PyClassPyO3Option::Crate) } else if lookahead.peek(kw::dict) { input.parse().map(PyClassPyO3Option::Dict) + } else if lookahead.peek(kw::eq) { + input.parse().map(PyClassPyO3Option::Eq) + } else if lookahead.peek(kw::eq_int) { + input.parse().map(PyClassPyO3Option::EqInt) } else if lookahead.peek(kw::extends) { input.parse().map(PyClassPyO3Option::Extends) } else if lookahead.peek(attributes::kw::freelist) { @@ -106,18 +124,24 @@ impl Parse for PyClassPyO3Option { input.parse().map(PyClassPyO3Option::Frozen) } else if lookahead.peek(attributes::kw::get_all) { input.parse().map(PyClassPyO3Option::GetAll) + } else if lookahead.peek(attributes::kw::hash) { + input.parse().map(PyClassPyO3Option::Hash) } else if lookahead.peek(attributes::kw::mapping) { input.parse().map(PyClassPyO3Option::Mapping) } else if lookahead.peek(attributes::kw::module) { input.parse().map(PyClassPyO3Option::Module) } else if lookahead.peek(kw::name) { input.parse().map(PyClassPyO3Option::Name) + } else if lookahead.peek(attributes::kw::ord) { + input.parse().map(PyClassPyO3Option::Ord) } else if lookahead.peek(kw::rename_all) { input.parse().map(PyClassPyO3Option::RenameAll) } else if lookahead.peek(attributes::kw::sequence) { input.parse().map(PyClassPyO3Option::Sequence) } else if lookahead.peek(attributes::kw::set_all) { input.parse().map(PyClassPyO3Option::SetAll) + } else if lookahead.peek(attributes::kw::str) { + input.parse().map(PyClassPyO3Option::Str) } else if lookahead.peek(attributes::kw::subclass) { input.parse().map(PyClassPyO3Option::Subclass) } else if lookahead.peek(attributes::kw::unsendable) { @@ -164,20 +188,37 @@ impl PyClassPyO3Options { match option { PyClassPyO3Option::Crate(krate) => set_option!(krate), - PyClassPyO3Option::Dict(dict) => set_option!(dict), + PyClassPyO3Option::Dict(dict) => { + ensure_spanned!( + !is_abi3_before(3, 9), + dict.span() => "`dict` requires Python >= 3.9 when using the `abi3` feature" + ); + set_option!(dict); + } + PyClassPyO3Option::Eq(eq) => set_option!(eq), + PyClassPyO3Option::EqInt(eq_int) => set_option!(eq_int), PyClassPyO3Option::Extends(extends) => set_option!(extends), PyClassPyO3Option::Freelist(freelist) => set_option!(freelist), PyClassPyO3Option::Frozen(frozen) => set_option!(frozen), PyClassPyO3Option::GetAll(get_all) => set_option!(get_all), + PyClassPyO3Option::Hash(hash) => set_option!(hash), PyClassPyO3Option::Mapping(mapping) => set_option!(mapping), PyClassPyO3Option::Module(module) => set_option!(module), PyClassPyO3Option::Name(name) => set_option!(name), + PyClassPyO3Option::Ord(ord) => set_option!(ord), PyClassPyO3Option::RenameAll(rename_all) => set_option!(rename_all), PyClassPyO3Option::Sequence(sequence) => set_option!(sequence), PyClassPyO3Option::SetAll(set_all) => set_option!(set_all), + PyClassPyO3Option::Str(str) => set_option!(str), PyClassPyO3Option::Subclass(subclass) => set_option!(subclass), PyClassPyO3Option::Unsendable(unsendable) => set_option!(unsendable), - PyClassPyO3Option::Weakref(weakref) => set_option!(weakref), + PyClassPyO3Option::Weakref(weakref) => { + ensure_spanned!( + !is_abi3_before(3, 9), + weakref.span() => "`weakref` requires Python >= 3.9 when using the `abi3` feature" + ); + set_option!(weakref); + } } Ok(()) } @@ -189,42 +230,56 @@ pub fn build_py_class( methods_type: PyClassMethodsType, ) -> syn::Result { args.options.take_pyo3_options(&mut class.attrs)?; - let doc = utils::get_doc(&class.attrs, None); - let ctx = &Ctx::new(&args.options.krate); + let ctx = &Ctx::new(&args.options.krate, None); + let doc = utils::get_doc(&class.attrs, None, ctx); if let Some(lt) = class.generics.lifetimes().next() { bail_spanned!( - lt.span() => - "#[pyclass] cannot have lifetime parameters. \ - For an explanation, see https://pyo3.rs/latest/class.html#no-lifetime-parameters" + lt.span() => concat!( + "#[pyclass] cannot have lifetime parameters. For an explanation, see \ + https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#no-lifetime-parameters" + ) ); } ensure_spanned!( class.generics.params.is_empty(), - class.generics.span() => - "#[pyclass] cannot have generic parameters. \ - For an explanation, see https://pyo3.rs/latest/class.html#no-generic-parameters" + class.generics.span() => concat!( + "#[pyclass] cannot have generic parameters. For an explanation, see \ + https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#no-generic-parameters" + ) ); + let mut all_errors = ErrorCombiner(None); + let mut field_options: Vec<(&syn::Field, FieldPyO3Options)> = match &mut class.fields { syn::Fields::Named(fields) => fields .named .iter_mut() - .map(|field| { - FieldPyO3Options::take_pyo3_options(&mut field.attrs) - .map(move |options| (&*field, options)) - }) - .collect::>()?, + .filter_map( + |field| match FieldPyO3Options::take_pyo3_options(&mut field.attrs) { + Ok(options) => Some((&*field, options)), + Err(e) => { + all_errors.combine(e); + None + } + }, + ) + .collect::>(), syn::Fields::Unnamed(fields) => fields .unnamed .iter_mut() - .map(|field| { - FieldPyO3Options::take_pyo3_options(&mut field.attrs) - .map(move |options| (&*field, options)) - }) - .collect::>()?, + .filter_map( + |field| match FieldPyO3Options::take_pyo3_options(&mut field.attrs) { + Ok(options) => Some((&*field, options)), + Err(e) => { + all_errors.combine(e); + None + } + }, + ) + .collect::>(), syn::Fields::Unit => { if let Some(attr) = args.options.set_all { return Err(syn::Error::new_spanned(attr, UNIT_SET)); @@ -237,6 +292,8 @@ pub fn build_py_class( } }; + all_errors.ensure_empty()?; + if let Some(attr) = args.options.get_all { for (_, FieldPyO3Options { get, .. }) in &mut field_options { if let Some(old_get) = get.replace(Annotated::Struct(attr)) { @@ -346,8 +403,32 @@ fn impl_class( methods_type: PyClassMethodsType, ctx: &Ctx, ) -> syn::Result { - let Ctx { pyo3_path } = ctx; - let pytypeinfo_impl = impl_pytypeinfo(cls, args, None, ctx); + let Ctx { pyo3_path, .. } = ctx; + let pytypeinfo_impl = impl_pytypeinfo(cls, args, ctx); + + if let Some(str) = &args.options.str { + if str.value.is_some() { + // check if any renaming is present + let no_naming_conflict = field_options.iter().all(|x| x.1.name.is_none()) + & args.options.name.is_none() + & args.options.rename_all.is_none(); + ensure_spanned!(no_naming_conflict, str.value.span() => "The format string syntax is incompatible with any renaming via `name` or `rename_all`"); + } + } + + let (default_str, default_str_slot) = + implement_pyclass_str(&args.options, &syn::parse_quote!(#cls), ctx); + + let (default_richcmp, default_richcmp_slot) = + pyclass_richcmp(&args.options, &syn::parse_quote!(#cls), ctx)?; + + let (default_hash, default_hash_slot) = + pyclass_hash(&args.options, &syn::parse_quote!(#cls), ctx)?; + + let mut slots = Vec::new(); + slots.extend(default_richcmp_slot); + slots.extend(default_hash_slot); + slots.extend(default_str_slot); let py_class_impl = PyClassImplsBuilder::new( cls, @@ -360,7 +441,7 @@ fn impl_class( field_options, ctx, )?, - vec![], + slots, ) .doc(doc) .impl_all(ctx)?; @@ -371,6 +452,14 @@ fn impl_class( #pytypeinfo_impl #py_class_impl + + #[doc(hidden)] + #[allow(non_snake_case)] + impl #cls { + #default_richcmp + #default_hash + #default_str + } }) } @@ -403,7 +492,7 @@ pub fn build_py_enum( ) -> syn::Result { args.options.take_pyo3_options(&mut enum_.attrs)?; - let ctx = &Ctx::new(&args.options.krate); + let ctx = &Ctx::new(&args.options.krate, None); if let Some(extends) = &args.options.extends { bail_spanned!(extends.span() => "enums can't extend from other classes"); } else if let Some(subclass) = &args.options.subclass { @@ -412,7 +501,7 @@ pub fn build_py_enum( bail_spanned!(enum_.brace_token.span.join() => "#[pyclass] can't be used on enums without any variants"); } - let doc = utils::get_doc(&enum_.attrs, None); + let doc = utils::get_doc(&enum_.attrs, None, ctx); let enum_ = PyClassEnum::new(enum_)?; impl_enum(enum_, &args, doc, method_type, ctx) } @@ -445,7 +534,12 @@ impl<'a> PyClassSimpleEnum<'a> { _ => bail_spanned!(variant.span() => "Must be a unit variant."), }; let options = EnumVariantPyO3Options::take_pyo3_options(&mut variant.attrs)?; - Ok(PyClassEnumUnitVariant { ident, options }) + let cfg_attrs = get_cfg_attributes(&variant.attrs); + Ok(PyClassEnumUnitVariant { + ident, + options, + cfg_attrs, + }) } let ident = &enum_.ident; @@ -503,10 +597,10 @@ impl<'a> PyClassComplexEnum<'a> { let variant = match &variant.fields { Fields::Unit => { bail_spanned!(variant.span() => format!( - "Unit variant `{ident}` is not yet supported in a complex enum\n\ - = help: change to a struct variant with no fields: `{ident} {{ }}`\n\ - = note: the enum is complex because of non-unit variant `{witness}`", - ident=ident, witness=witness)) + "Unit variant `{ident}` is not yet supported in a complex enum\n\ + = help: change to an empty tuple variant instead: `{ident}()`\n\ + = note: the enum is complex because of non-unit variant `{witness}`", + ident=ident, witness=witness)) } Fields::Named(fields) => { let fields = fields @@ -525,12 +619,21 @@ impl<'a> PyClassComplexEnum<'a> { options, }) } - Fields::Unnamed(_) => { - bail_spanned!(variant.span() => format!( - "Tuple variant `{ident}` is not yet supported in a complex enum\n\ - = help: change to a struct variant with named fields: `{ident} {{ /* fields */ }}`\n\ - = note: the enum is complex because of non-unit variant `{witness}`", - ident=ident, witness=witness)) + Fields::Unnamed(types) => { + let fields = types + .unnamed + .iter() + .map(|field| PyClassEnumVariantUnnamedField { + ty: &field.ty, + span: field.span(), + }) + .collect(); + + PyClassEnumVariant::Tuple(PyClassEnumTupleVariant { + ident, + fields, + options, + }) } }; @@ -552,7 +655,7 @@ impl<'a> PyClassComplexEnum<'a> { enum PyClassEnumVariant<'a> { // TODO(mkovaxx): Unit(PyClassEnumUnitVariant<'a>), Struct(PyClassEnumStructVariant<'a>), - // TODO(mkovaxx): Tuple(PyClassEnumTupleVariant<'a>), + Tuple(PyClassEnumTupleVariant<'a>), } trait EnumVariant { @@ -576,16 +679,18 @@ trait EnumVariant { } } -impl<'a> EnumVariant for PyClassEnumVariant<'a> { +impl EnumVariant for PyClassEnumVariant<'_> { fn get_ident(&self) -> &syn::Ident { match self { PyClassEnumVariant::Struct(struct_variant) => struct_variant.ident, + PyClassEnumVariant::Tuple(tuple_variant) => tuple_variant.ident, } } fn get_options(&self) -> &EnumVariantPyO3Options { match self { PyClassEnumVariant::Struct(struct_variant) => &struct_variant.options, + PyClassEnumVariant::Tuple(tuple_variant) => &tuple_variant.options, } } } @@ -594,9 +699,10 @@ impl<'a> EnumVariant for PyClassEnumVariant<'a> { struct PyClassEnumUnitVariant<'a> { ident: &'a syn::Ident, options: EnumVariantPyO3Options, + cfg_attrs: Vec<&'a syn::Attribute>, } -impl<'a> EnumVariant for PyClassEnumUnitVariant<'a> { +impl EnumVariant for PyClassEnumUnitVariant<'_> { fn get_ident(&self) -> &syn::Ident { self.ident } @@ -613,19 +719,33 @@ struct PyClassEnumStructVariant<'a> { options: EnumVariantPyO3Options, } +struct PyClassEnumTupleVariant<'a> { + ident: &'a syn::Ident, + fields: Vec>, + options: EnumVariantPyO3Options, +} + struct PyClassEnumVariantNamedField<'a> { ident: &'a syn::Ident, ty: &'a syn::Type, span: Span, } +struct PyClassEnumVariantUnnamedField<'a> { + ty: &'a syn::Type, + span: Span, +} + /// `#[pyo3()]` options for pyclass enum variants +#[derive(Clone, Default)] struct EnumVariantPyO3Options { name: Option, + constructor: Option, } enum EnumVariantPyO3Option { Name(NameAttribute), + Constructor(ConstructorAttribute), } impl Parse for EnumVariantPyO3Option { @@ -633,6 +753,8 @@ impl Parse for EnumVariantPyO3Option { let lookahead = input.lookahead1(); if lookahead.peek(attributes::kw::name) { input.parse().map(EnumVariantPyO3Option::Name) + } else if lookahead.peek(attributes::kw::constructor) { + input.parse().map(EnumVariantPyO3Option::Constructor) } else { Err(lookahead.error()) } @@ -641,21 +763,87 @@ impl Parse for EnumVariantPyO3Option { impl EnumVariantPyO3Options { fn take_pyo3_options(attrs: &mut Vec) -> Result { - let mut options = EnumVariantPyO3Options { name: None }; + let mut options = EnumVariantPyO3Options::default(); - for option in take_pyo3_options(attrs)? { - match option { - EnumVariantPyO3Option::Name(name) => { + take_pyo3_options(attrs)? + .into_iter() + .try_for_each(|option| options.set_option(option))?; + + Ok(options) + } + + fn set_option(&mut self, option: EnumVariantPyO3Option) -> syn::Result<()> { + macro_rules! set_option { + ($key:ident) => { + { ensure_spanned!( - options.name.is_none(), - name.span() => "`name` may only be specified once" + self.$key.is_none(), + $key.span() => concat!("`", stringify!($key), "` may only be specified once") ); - options.name = Some(name); + self.$key = Some($key); } - } + }; } - Ok(options) + match option { + EnumVariantPyO3Option::Constructor(constructor) => set_option!(constructor), + EnumVariantPyO3Option::Name(name) => set_option!(name), + } + Ok(()) + } +} + +// todo(remove this dead code allowance once __repr__ is implemented +#[allow(dead_code)] +pub enum PyFmtName { + Str, + Repr, +} + +fn implement_py_formatting( + ty: &syn::Type, + ctx: &Ctx, + option: &StrFormatterAttribute, +) -> (ImplItemFn, MethodAndSlotDef) { + let mut fmt_impl = match &option.value { + Some(opt) => { + let fmt = &opt.fmt; + let args = &opt + .args + .iter() + .map(|member| quote! {self.#member}) + .collect::>(); + let fmt_impl: ImplItemFn = syn::parse_quote! { + fn __pyo3__generated____str__(&self) -> ::std::string::String { + ::std::format!(#fmt, #(#args, )*) + } + }; + fmt_impl + } + None => { + let fmt_impl: syn::ImplItemFn = syn::parse_quote! { + fn __pyo3__generated____str__(&self) -> ::std::string::String { + ::std::format!("{}", &self) + } + }; + fmt_impl + } + }; + let fmt_slot = generate_protocol_slot(ty, &mut fmt_impl, &__STR__, "__str__", ctx).unwrap(); + (fmt_impl, fmt_slot) +} + +fn implement_pyclass_str( + options: &PyClassPyO3Options, + ty: &syn::Type, + ctx: &Ctx, +) -> (Option, Option) { + match &options.str { + Some(option) => { + let (default_str, default_str_slot) = implement_py_formatting(ty, ctx, option); + (Some(default_str), Some(default_str_slot)) + } + _ => (None, None), } } @@ -666,6 +854,10 @@ fn impl_enum( methods_type: PyClassMethodsType, ctx: &Ctx, ) -> Result { + if let Some(str_fmt) = &args.options.str { + ensure_spanned!(str_fmt.value.is_none(), str_fmt.value.span() => "The format string syntax cannot be used with enums") + } + match enum_ { PyClassEnum::Simple(simple_enum) => { impl_simple_enum(simple_enum, args, doc, methods_type, ctx) @@ -683,26 +875,32 @@ fn impl_simple_enum( methods_type: PyClassMethodsType, ctx: &Ctx, ) -> Result { - let Ctx { pyo3_path } = ctx; let cls = simple_enum.ident; let ty: syn::Type = syn::parse_quote!(#cls); let variants = simple_enum.variants; - let pytypeinfo = impl_pytypeinfo(cls, args, None, ctx); + let pytypeinfo = impl_pytypeinfo(cls, args, ctx); + + for variant in &variants { + ensure_spanned!(variant.options.constructor.is_none(), variant.options.constructor.span() => "`constructor` can't be used on a simple enum variant"); + } + + let variant_cfg_check = generate_cfg_check(&variants, cls); let (default_repr, default_repr_slot) = { let variants_repr = variants.iter().map(|variant| { let variant_name = variant.ident; + let cfg_attrs = &variant.cfg_attrs; // Assuming all variants are unit variants because they are the only type we support. let repr = format!( "{}.{}", get_class_python_name(cls, args), variant.get_python_name(args), ); - quote! { #cls::#variant_name => #repr, } + quote! { #(#cfg_attrs)* #cls::#variant_name => #repr, } }); let mut repr_impl: syn::ImplItemFn = syn::parse_quote! { fn __pyo3__repr__(&self) -> &'static str { - match self { + match *self { #(#variants_repr)* } } @@ -712,17 +910,20 @@ fn impl_simple_enum( (repr_impl, repr_slot) }; + let (default_str, default_str_slot) = implement_pyclass_str(&args.options, &ty, ctx); + let repr_type = &simple_enum.repr_type; let (default_int, default_int_slot) = { // This implementation allows us to convert &T to #repr_type without implementing `Copy` let variants_to_int = variants.iter().map(|variant| { let variant_name = variant.ident; - quote! { #cls::#variant_name => #cls::#variant_name as #repr_type, } + let cfg_attrs = &variant.cfg_attrs; + quote! { #(#cfg_attrs)* #cls::#variant_name => #cls::#variant_name as #repr_type, } }); let mut int_impl: syn::ImplItemFn = syn::parse_quote! { fn __pyo3__int__(&self) -> #repr_type { - match self { + match *self { #(#variants_to_int)* } } @@ -731,50 +932,14 @@ fn impl_simple_enum( (int_impl, int_slot) }; - let (default_richcmp, default_richcmp_slot) = { - let mut richcmp_impl: syn::ImplItemFn = syn::parse_quote! { - fn __pyo3__richcmp__( - &self, - py: #pyo3_path::Python, - other: &#pyo3_path::Bound<'_, #pyo3_path::PyAny>, - op: #pyo3_path::basic::CompareOp - ) -> #pyo3_path::PyResult<#pyo3_path::PyObject> { - use #pyo3_path::conversion::ToPyObject; - use #pyo3_path::types::PyAnyMethods; - use ::core::result::Result::*; - match op { - #pyo3_path::basic::CompareOp::Eq => { - let self_val = self.__pyo3__int__(); - if let Ok(i) = other.extract::<#repr_type>() { - return Ok((self_val == i).to_object(py)); - } - if let Ok(other) = other.extract::<#pyo3_path::PyRef>() { - return Ok((self_val == other.__pyo3__int__()).to_object(py)); - } - - return Ok(py.NotImplemented()); - } - #pyo3_path::basic::CompareOp::Ne => { - let self_val = self.__pyo3__int__(); - if let Ok(i) = other.extract::<#repr_type>() { - return Ok((self_val != i).to_object(py)); - } - if let Ok(other) = other.extract::<#pyo3_path::PyRef>() { - return Ok((self_val != other.__pyo3__int__()).to_object(py)); - } - - return Ok(py.NotImplemented()); - } - _ => Ok(py.NotImplemented()), - } - } - }; - let richcmp_slot = - generate_default_protocol_slot(&ty, &mut richcmp_impl, &__RICHCMP__, ctx).unwrap(); - (richcmp_impl, richcmp_slot) - }; + let (default_richcmp, default_richcmp_slot) = + pyclass_richcmp_simple_enum(&args.options, &ty, repr_type, ctx)?; + let (default_hash, default_hash_slot) = pyclass_hash(&args.options, &ty, ctx)?; - let default_slots = vec![default_repr_slot, default_int_slot, default_richcmp_slot]; + let mut default_slots = vec![default_repr_slot, default_int_slot]; + default_slots.extend(default_richcmp_slot); + default_slots.extend(default_hash_slot); + default_slots.extend(default_str_slot); let pyclass_impls = PyClassImplsBuilder::new( cls, @@ -782,7 +947,9 @@ fn impl_simple_enum( methods_type, simple_enum_default_methods( cls, - variants.iter().map(|v| (v.ident, v.get_python_name(args))), + variants + .iter() + .map(|v| (v.ident, v.get_python_name(args), &v.cfg_attrs)), ctx, ), default_slots, @@ -791,6 +958,8 @@ fn impl_simple_enum( .impl_all(ctx)?; Ok(quote! { + #variant_cfg_check + #pytypeinfo #pyclass_impls @@ -801,6 +970,8 @@ fn impl_simple_enum( #default_repr #default_int #default_richcmp + #default_hash + #default_str } }) } @@ -812,7 +983,9 @@ fn impl_complex_enum( methods_type: PyClassMethodsType, ctx: &Ctx, ) -> Result { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; + let cls = complex_enum.ident; + let ty: syn::Type = syn::parse_quote!(#cls); // Need to rig the enum PyClass options let args = { @@ -824,12 +997,20 @@ fn impl_complex_enum( rigged_args }; - let ctx = &Ctx::new(&args.options.krate); + let ctx = &Ctx::new(&args.options.krate, None); let cls = complex_enum.ident; let variants = complex_enum.variants; - let pytypeinfo = impl_pytypeinfo(cls, &args, None, ctx); + let pytypeinfo = impl_pytypeinfo(cls, &args, ctx); + + let (default_richcmp, default_richcmp_slot) = pyclass_richcmp(&args.options, &ty, ctx)?; + let (default_hash, default_hash_slot) = pyclass_hash(&args.options, &ty, ctx)?; - let default_slots = vec![]; + let (default_str, default_str_slot) = implement_pyclass_str(&args.options, &ty, ctx); + + let mut default_slots = vec![]; + default_slots.extend(default_richcmp_slot); + default_slots.extend(default_hash_slot); + default_slots.extend(default_str_slot); let impl_builder = PyClassImplsBuilder::new( cls, @@ -855,7 +1036,7 @@ fn impl_complex_enum( let variant_cls = gen_complex_enum_variant_class_ident(cls, variant.get_ident()); quote! { #cls::#variant_ident { .. } => { - let pyclass_init = #pyo3_path::PyClassInitializer::from(self).add_subclass(#variant_cls); + let pyclass_init = <#pyo3_path::PyClassInitializer as ::std::convert::From>::from(self).add_subclass(#variant_cls); let variant_value = #pyo3_path::Py::new(py, pyclass_init).unwrap(); #pyo3_path::IntoPy::into_py(variant_value, py) } @@ -864,6 +1045,7 @@ fn impl_complex_enum( .collect(); quote! { + #[allow(deprecated)] impl #pyo3_path::IntoPy<#pyo3_path::PyObject> for #cls { fn into_py(self, py: #pyo3_path::Python) -> #pyo3_path::PyObject { match self { @@ -874,10 +1056,43 @@ fn impl_complex_enum( } }; + let enum_into_pyobject_impl = { + let match_arms = variants + .iter() + .map(|variant| { + let variant_ident = variant.get_ident(); + let variant_cls = gen_complex_enum_variant_class_ident(cls, variant.get_ident()); + quote! { + #cls::#variant_ident { .. } => { + let pyclass_init = <#pyo3_path::PyClassInitializer as ::std::convert::From>::from(self).add_subclass(#variant_cls); + unsafe { #pyo3_path::Bound::new(py, pyclass_init).map(|b| #pyo3_path::types::PyAnyMethods::downcast_into_unchecked(b.into_any())) } + } + } + }); + + quote! { + impl<'py> #pyo3_path::conversion::IntoPyObject<'py> for #cls { + type Target = Self; + type Output = #pyo3_path::Bound<'py, >::Target>; + type Error = #pyo3_path::PyErr; + + fn into_pyobject(self, py: #pyo3_path::Python<'py>) -> ::std::result::Result< + ::Output, + ::Error, + > { + match self { + #(#match_arms)* + } + } + } + } + }; + let pyclass_impls: TokenStream = [ impl_builder.impl_pyclass(ctx), impl_builder.impl_extractext(ctx), enum_into_py_impl, + enum_into_pyobject_impl, impl_builder.impl_pyclassimpl(ctx)?, impl_builder.impl_add_to_module(ctx), impl_builder.impl_freelist(ctx), @@ -889,7 +1104,7 @@ fn impl_complex_enum( let mut variant_cls_pytypeinfos = vec![]; let mut variant_cls_pyclass_impls = vec![]; let mut variant_cls_impls = vec![]; - for variant in &variants { + for variant in variants { let variant_cls = gen_complex_enum_variant_class_ident(cls, variant.get_ident()); let variant_cls_zst = quote! { @@ -902,23 +1117,30 @@ fn impl_complex_enum( let variant_args = PyClassArgs { class_kind: PyClassKind::Struct, // TODO(mkovaxx): propagate variant.options - options: parse_quote!(extends = #cls, frozen), + options: { + let mut rigged_options: PyClassPyO3Options = parse_quote!(extends = #cls, frozen); + // If a specific module was given to the base class, use it for all variants. + rigged_options.module.clone_from(&args.options.module); + rigged_options + }, }; - let variant_cls_pytypeinfo = impl_pytypeinfo(&variant_cls, &variant_args, None, ctx); + let variant_cls_pytypeinfo = impl_pytypeinfo(&variant_cls, &variant_args, ctx); variant_cls_pytypeinfos.push(variant_cls_pytypeinfo); - let variant_new = complex_enum_variant_new(cls, variant, ctx)?; - - let (variant_cls_impl, field_getters) = impl_complex_enum_variant_cls(cls, variant, ctx)?; + let (variant_cls_impl, field_getters, mut slots) = + impl_complex_enum_variant_cls(cls, &variant, ctx)?; variant_cls_impls.push(variant_cls_impl); + let variant_new = complex_enum_variant_new(cls, variant, ctx)?; + slots.push(variant_new); + let pyclass_impl = PyClassImplsBuilder::new( &variant_cls, &variant_args, methods_type, field_getters, - vec![variant_new], + slots, ) .impl_all(ctx)?; @@ -932,7 +1154,11 @@ fn impl_complex_enum( #[doc(hidden)] #[allow(non_snake_case)] - impl #cls {} + impl #cls { + #default_richcmp + #default_hash + #default_str + } #(#variant_cls_zsts)* @@ -948,20 +1174,50 @@ fn impl_complex_enum_variant_cls( enum_name: &syn::Ident, variant: &PyClassEnumVariant<'_>, ctx: &Ctx, -) -> Result<(TokenStream, Vec)> { +) -> Result<(TokenStream, Vec, Vec)> { match variant { PyClassEnumVariant::Struct(struct_variant) => { impl_complex_enum_struct_variant_cls(enum_name, struct_variant, ctx) } + PyClassEnumVariant::Tuple(tuple_variant) => { + impl_complex_enum_tuple_variant_cls(enum_name, tuple_variant, ctx) + } } } +fn impl_complex_enum_variant_match_args( + ctx @ Ctx { pyo3_path, .. }: &Ctx, + variant_cls_type: &syn::Type, + field_names: &mut Vec, +) -> syn::Result<(MethodAndMethodDef, syn::ImplItemFn)> { + let ident = format_ident!("__match_args__"); + let mut match_args_impl: syn::ImplItemFn = { + parse_quote! { + #[classattr] + fn #ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::Bound<'_, #pyo3_path::types::PyTuple>> { + #pyo3_path::types::PyTuple::new::<&str, _>(py, [ + #(stringify!(#field_names),)* + ]) + } + } + }; + + let spec = FnSpec::parse( + &mut match_args_impl.sig, + &mut match_args_impl.attrs, + Default::default(), + )?; + let variant_match_args = impl_py_class_attribute(variant_cls_type, &spec, ctx)?; + + Ok((variant_match_args, match_args_impl)) +} + fn impl_complex_enum_struct_variant_cls( enum_name: &syn::Ident, variant: &PyClassEnumStructVariant<'_>, ctx: &Ctx, -) -> Result<(TokenStream, Vec)> { - let Ctx { pyo3_path } = ctx; +) -> Result<(TokenStream, Vec, Vec)> { + let Ctx { pyo3_path, .. } = ctx; let variant_ident = &variant.ident; let variant_cls = gen_complex_enum_variant_class_ident(enum_name, variant.ident); let variant_cls_type = parse_quote!(#variant_cls); @@ -979,10 +1235,17 @@ fn impl_complex_enum_struct_variant_cls( complex_enum_variant_field_getter(&variant_cls_type, field_name, field.span, ctx)?; let field_getter_impl = quote! { - fn #field_name(slf: #pyo3_path::PyRef) -> #pyo3_path::PyResult<#field_type> { + fn #field_name(slf: #pyo3_path::PyRef) -> #pyo3_path::PyResult<#pyo3_path::PyObject> { + #[allow(unused_imports)] + use #pyo3_path::impl_::pyclass::Probe; + let py = slf.py(); match &*slf.into_super() { - #enum_name::#variant_ident { #field_name, .. } => Ok(#field_name.clone()), - _ => unreachable!("Wrong complex enum variant found in variant wrapper PyClass"), + #enum_name::#variant_ident { #field_name, .. } => + #pyo3_path::impl_::pyclass::ConvertField::< + { #pyo3_path::impl_::pyclass::IsIntoPyObjectRef::<#field_type>::VALUE }, + { #pyo3_path::impl_::pyclass::IsIntoPyObject::<#field_type>::VALUE }, + >::convert_field::<#field_type>(#field_name, py), + _ => ::core::unreachable!("Wrong complex enum variant found in variant wrapper PyClass"), } } }; @@ -993,26 +1256,229 @@ fn impl_complex_enum_struct_variant_cls( field_getter_impls.push(field_getter_impl); } + let (variant_match_args, match_args_const_impl) = + impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names)?; + + field_getters.push(variant_match_args); + let cls_impl = quote! { #[doc(hidden)] #[allow(non_snake_case)] impl #variant_cls { + #[allow(clippy::too_many_arguments)] fn __pymethod_constructor__(py: #pyo3_path::Python<'_>, #(#fields_with_types,)*) -> #pyo3_path::PyClassInitializer<#variant_cls> { let base_value = #enum_name::#variant_ident { #(#field_names,)* }; - #pyo3_path::PyClassInitializer::from(base_value).add_subclass(#variant_cls) + <#pyo3_path::PyClassInitializer<#enum_name> as ::std::convert::From<#enum_name>>::from(base_value).add_subclass(#variant_cls) + } + + #match_args_const_impl + + #(#field_getter_impls)* + } + }; + + Ok((cls_impl, field_getters, Vec::new())) +} + +fn impl_complex_enum_tuple_variant_field_getters( + ctx: &Ctx, + variant: &PyClassEnumTupleVariant<'_>, + enum_name: &syn::Ident, + variant_cls_type: &syn::Type, + variant_ident: &&Ident, + field_names: &mut Vec, + fields_types: &mut Vec, +) -> Result<(Vec, Vec)> { + let Ctx { pyo3_path, .. } = ctx; + + let mut field_getters = vec![]; + let mut field_getter_impls = vec![]; + + for (index, field) in variant.fields.iter().enumerate() { + let field_name = format_ident!("_{}", index); + let field_type = field.ty; + + let field_getter = + complex_enum_variant_field_getter(variant_cls_type, &field_name, field.span, ctx)?; + + // Generate the match arms needed to destructure the tuple and access the specific field + let field_access_tokens: Vec<_> = (0..variant.fields.len()) + .map(|i| { + if i == index { + quote! { val } + } else { + quote! { _ } + } + }) + .collect(); + let field_getter_impl: syn::ImplItemFn = parse_quote! { + fn #field_name(slf: #pyo3_path::PyRef) -> #pyo3_path::PyResult<#pyo3_path::PyObject> { + #[allow(unused_imports)] + use #pyo3_path::impl_::pyclass::Probe; + let py = slf.py(); + match &*slf.into_super() { + #enum_name::#variant_ident ( #(#field_access_tokens), *) => + #pyo3_path::impl_::pyclass::ConvertField::< + { #pyo3_path::impl_::pyclass::IsIntoPyObjectRef::<#field_type>::VALUE }, + { #pyo3_path::impl_::pyclass::IsIntoPyObject::<#field_type>::VALUE }, + >::convert_field::<#field_type>(val, py), + _ => ::core::unreachable!("Wrong complex enum variant found in variant wrapper PyClass"), + } + } + }; + + field_names.push(field_name); + fields_types.push(field_type.clone()); + field_getters.push(field_getter); + field_getter_impls.push(field_getter_impl); + } + + Ok((field_getters, field_getter_impls)) +} + +fn impl_complex_enum_tuple_variant_len( + ctx: &Ctx, + + variant_cls_type: &syn::Type, + num_fields: usize, +) -> Result<(MethodAndSlotDef, syn::ImplItemFn)> { + let Ctx { pyo3_path, .. } = ctx; + + let mut len_method_impl: syn::ImplItemFn = parse_quote! { + fn __len__(slf: #pyo3_path::PyRef) -> #pyo3_path::PyResult { + ::std::result::Result::Ok(#num_fields) + } + }; + + let variant_len = + generate_default_protocol_slot(variant_cls_type, &mut len_method_impl, &__LEN__, ctx)?; + + Ok((variant_len, len_method_impl)) +} + +fn impl_complex_enum_tuple_variant_getitem( + ctx: &Ctx, + variant_cls: &syn::Ident, + variant_cls_type: &syn::Type, + num_fields: usize, +) -> Result<(MethodAndSlotDef, syn::ImplItemFn)> { + let Ctx { pyo3_path, .. } = ctx; + + let match_arms: Vec<_> = (0..num_fields) + .map(|i| { + let field_access = format_ident!("_{}", i); + quote! { #i => + #pyo3_path::IntoPyObjectExt::into_py_any(#variant_cls::#field_access(slf)?, py) + } + }) + .collect(); + + let mut get_item_method_impl: syn::ImplItemFn = parse_quote! { + fn __getitem__(slf: #pyo3_path::PyRef, idx: usize) -> #pyo3_path::PyResult< #pyo3_path::PyObject> { + let py = slf.py(); + match idx { + #( #match_arms, )* + _ => ::std::result::Result::Err(#pyo3_path::exceptions::PyIndexError::new_err("tuple index out of range")), + } + } + }; + + let variant_getitem = generate_default_protocol_slot( + variant_cls_type, + &mut get_item_method_impl, + &__GETITEM__, + ctx, + )?; + + Ok((variant_getitem, get_item_method_impl)) +} + +fn impl_complex_enum_tuple_variant_cls( + enum_name: &syn::Ident, + variant: &PyClassEnumTupleVariant<'_>, + ctx: &Ctx, +) -> Result<(TokenStream, Vec, Vec)> { + let Ctx { pyo3_path, .. } = ctx; + let variant_ident = &variant.ident; + let variant_cls = gen_complex_enum_variant_class_ident(enum_name, variant.ident); + let variant_cls_type = parse_quote!(#variant_cls); + + let mut slots = vec![]; + + // represents the index of the field + let mut field_names: Vec = vec![]; + let mut field_types: Vec = vec![]; + + let (mut field_getters, field_getter_impls) = impl_complex_enum_tuple_variant_field_getters( + ctx, + variant, + enum_name, + &variant_cls_type, + variant_ident, + &mut field_names, + &mut field_types, + )?; + + let num_fields = variant.fields.len(); + + let (variant_len, len_method_impl) = + impl_complex_enum_tuple_variant_len(ctx, &variant_cls_type, num_fields)?; + + slots.push(variant_len); + + let (variant_getitem, getitem_method_impl) = + impl_complex_enum_tuple_variant_getitem(ctx, &variant_cls, &variant_cls_type, num_fields)?; + + slots.push(variant_getitem); + + let (variant_match_args, match_args_method_impl) = + impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names)?; + + field_getters.push(variant_match_args); + + let cls_impl = quote! { + #[doc(hidden)] + #[allow(non_snake_case)] + impl #variant_cls { + #[allow(clippy::too_many_arguments)] + fn __pymethod_constructor__(py: #pyo3_path::Python<'_>, #(#field_names : #field_types,)*) -> #pyo3_path::PyClassInitializer<#variant_cls> { + let base_value = #enum_name::#variant_ident ( #(#field_names,)* ); + <#pyo3_path::PyClassInitializer<#enum_name> as ::std::convert::From<#enum_name>>::from(base_value).add_subclass(#variant_cls) } + #len_method_impl + + #getitem_method_impl + + #match_args_method_impl + #(#field_getter_impls)* } }; - Ok((cls_impl, field_getters)) + Ok((cls_impl, field_getters, slots)) } fn gen_complex_enum_variant_class_ident(enum_: &syn::Ident, variant: &syn::Ident) -> syn::Ident { format_ident!("{}_{}", enum_, variant) } +fn generate_protocol_slot( + cls: &syn::Type, + method: &mut syn::ImplItemFn, + slot: &SlotDef, + name: &str, + ctx: &Ctx, +) -> syn::Result { + let spec = FnSpec::parse( + &mut method.sig, + &mut Vec::new(), + PyFunctionOptions::default(), + ) + .unwrap(); + slot.generate_type_slot(&syn::parse_quote!(#cls), &spec, name, ctx) +} + fn generate_default_protocol_slot( cls: &syn::Type, method: &mut syn::ImplItemFn, @@ -1023,7 +1489,6 @@ fn generate_default_protocol_slot( &mut method.sig, &mut Vec::new(), PyFunctionOptions::default(), - ctx, ) .unwrap(); let name = spec.name.to_string(); @@ -1037,7 +1502,13 @@ fn generate_default_protocol_slot( fn simple_enum_default_methods<'a>( cls: &'a syn::Ident, - unit_variant_names: impl IntoIterator)>, + unit_variant_names: impl IntoIterator< + Item = ( + &'a syn::Ident, + Cow<'a, syn::Ident>, + &'a Vec<&'a syn::Attribute>, + ), + >, ctx: &Ctx, ) -> Vec { let cls_type = syn::parse_quote!(#cls); @@ -1049,12 +1520,29 @@ fn simple_enum_default_methods<'a>( kw: syn::parse_quote! { name }, value: NameLitStr(py_ident.clone()), }), - deprecations: Deprecations::new(ctx), }, }; unit_variant_names .into_iter() - .map(|(var, py_name)| gen_py_const(&cls_type, &variant_to_attribute(var, &py_name), ctx)) + .map(|(var, py_name, attrs)| { + let method = gen_py_const(&cls_type, &variant_to_attribute(var, &py_name), ctx); + let associated_method_tokens = method.associated_method; + let method_def_tokens = method.method_def; + + let associated_method = quote! { + #(#attrs)* + #associated_method_tokens + }; + let method_def = quote! { + #(#attrs)* + #method_def_tokens + }; + + MethodAndMethodDef { + associated_method, + method_def, + } + }) .collect() } @@ -1072,7 +1560,6 @@ fn complex_enum_default_methods<'a>( kw: syn::parse_quote! { name }, value: NameLitStr(py_ident.clone()), }), - deprecations: Deprecations::new(ctx), }, }; variant_names @@ -1086,30 +1573,30 @@ fn complex_enum_default_methods<'a>( pub fn gen_complex_enum_variant_attr( cls: &syn::Ident, cls_type: &syn::Type, - spec: &ConstSpec<'_>, + spec: &ConstSpec, ctx: &Ctx, ) -> MethodAndMethodDef { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let member = &spec.rust_ident; let wrapper_ident = format_ident!("__pymethod_variant_cls_{}__", member); - let deprecations = &spec.attributes.deprecations; - let python_name = &spec.null_terminated_python_name(); + let python_name = spec.null_terminated_python_name(ctx); let variant_cls = format_ident!("{}_{}", cls, member); let associated_method = quote! { fn #wrapper_ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::PyObject> { - #deprecations - ::std::result::Result::Ok(py.get_type_bound::<#variant_cls>().into_any().unbind()) + ::std::result::Result::Ok(py.get_type::<#variant_cls>().into_any().unbind()) } }; let method_def = quote! { - #pyo3_path::class::PyMethodDefType::ClassAttribute({ - #pyo3_path::class::PyClassAttributeDef::new( - #python_name, - #pyo3_path::impl_::pymethods::PyClassAttributeFactory(#cls_type::#wrapper_ident) - ) - }) + #pyo3_path::impl_::pyclass::MaybeRuntimePyMethodDef::Static( + #pyo3_path::impl_::pymethods::PyMethodDefType::ClassAttribute({ + #pyo3_path::impl_::pymethods::PyClassAttributeDef::new( + #python_name, + #cls_type::#wrapper_ident + ) + }) + ) }; MethodAndMethodDef { @@ -1120,22 +1607,25 @@ pub fn gen_complex_enum_variant_attr( fn complex_enum_variant_new<'a>( cls: &'a syn::Ident, - variant: &'a PyClassEnumVariant<'a>, + variant: PyClassEnumVariant<'a>, ctx: &Ctx, ) -> Result { match variant { PyClassEnumVariant::Struct(struct_variant) => { complex_enum_struct_variant_new(cls, struct_variant, ctx) } + PyClassEnumVariant::Tuple(tuple_variant) => { + complex_enum_tuple_variant_new(cls, tuple_variant, ctx) + } } } fn complex_enum_struct_variant_new<'a>( cls: &'a syn::Ident, - variant: &'a PyClassEnumStructVariant<'a>, + variant: PyClassEnumStructVariant<'a>, ctx: &Ctx, ) -> Result { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let variant_cls = format_ident!("{}_{}", cls, variant.ident); let variant_cls_type: syn::Type = parse_quote!(#variant_cls); @@ -1143,40 +1633,88 @@ fn complex_enum_struct_variant_new<'a>( let arg_py_type: syn::Type = parse_quote!(#pyo3_path::Python<'_>); let args = { - let mut no_pyo3_attrs = vec![]; - let attrs = crate::pyfunction::PyFunctionArgPyO3Attributes::from_attrs(&mut no_pyo3_attrs)?; - let mut args = vec![ // py: Python<'_> - FnArg { + FnArg::Py(PyArg { name: &arg_py_ident, ty: &arg_py_type, - optional: None, - default: None, - py: true, - attrs: attrs.clone(), - is_varargs: false, - is_kwargs: false, - is_cancel_handle: false, - }, + }), ]; for field in &variant.fields { - args.push(FnArg { - name: field.ident, + args.push(FnArg::Regular(RegularArg { + name: Cow::Borrowed(field.ident), ty: field.ty, - optional: None, - default: None, - py: false, - attrs: attrs.clone(), - is_varargs: false, - is_kwargs: false, - is_cancel_handle: false, - }); + from_py_with: None, + default_value: None, + option_wrapped_type: None, + })); } args }; - let signature = crate::pyfunction::FunctionSignature::from_arguments(args)?; + + let signature = if let Some(constructor) = variant.options.constructor { + crate::pyfunction::FunctionSignature::from_arguments_and_attribute( + args, + constructor.into_signature(), + )? + } else { + crate::pyfunction::FunctionSignature::from_arguments(args) + }; + + let spec = FnSpec { + tp: crate::method::FnType::FnNew, + name: &format_ident!("__pymethod_constructor__"), + python_name: format_ident!("__new__"), + signature, + convention: crate::method::CallingConvention::TpNew, + text_signature: None, + asyncness: None, + unsafety: None, + }; + + crate::pymethod::impl_py_method_def_new(&variant_cls_type, &spec, ctx) +} + +fn complex_enum_tuple_variant_new<'a>( + cls: &'a syn::Ident, + variant: PyClassEnumTupleVariant<'a>, + ctx: &Ctx, +) -> Result { + let Ctx { pyo3_path, .. } = ctx; + + let variant_cls: Ident = format_ident!("{}_{}", cls, variant.ident); + let variant_cls_type: syn::Type = parse_quote!(#variant_cls); + + let arg_py_ident: syn::Ident = parse_quote!(py); + let arg_py_type: syn::Type = parse_quote!(#pyo3_path::Python<'_>); + + let args = { + let mut args = vec![FnArg::Py(PyArg { + name: &arg_py_ident, + ty: &arg_py_type, + })]; + + for (i, field) in variant.fields.iter().enumerate() { + args.push(FnArg::Regular(RegularArg { + name: std::borrow::Cow::Owned(format_ident!("_{}", i)), + ty: field.ty, + from_py_with: None, + default_value: None, + option_wrapped_type: None, + })); + } + args + }; + + let signature = if let Some(constructor) = variant.options.constructor { + crate::pyfunction::FunctionSignature::from_arguments_and_attribute( + args, + constructor.into_signature(), + )? + } else { + crate::pyfunction::FunctionSignature::from_arguments(args) + }; let spec = FnSpec { tp: crate::method::FnType::FnNew, @@ -1187,7 +1725,6 @@ fn complex_enum_struct_variant_new<'a>( text_signature: None, asyncness: None, unsafety: None, - deprecations: Deprecations::new(ctx), }; crate::pymethod::impl_py_method_def_new(&variant_cls_type, &spec, ctx) @@ -1199,7 +1736,7 @@ fn complex_enum_variant_field_getter<'a>( field_span: Span, ctx: &Ctx, ) -> Result { - let signature = crate::pyfunction::FunctionSignature::from_arguments(vec![])?; + let signature = crate::pyfunction::FunctionSignature::from_arguments(vec![]); let self_type = crate::method::SelfType::TryFromBoundRef(field_span); @@ -1212,13 +1749,12 @@ fn complex_enum_variant_field_getter<'a>( text_signature: None, asyncness: None, unsafety: None, - deprecations: Deprecations::new(ctx), }; let property_type = crate::pymethod::PropertyType::Function { self_type: &self_type, spec: &spec, - doc: crate::get_doc(&[], None), + doc: crate::get_doc(&[], None, ctx), }; let getter = crate::pymethod::impl_py_getter_def(variant_cls_type, property_type, ctx)?; @@ -1276,13 +1812,8 @@ fn descriptors_to_items( Ok(items) } -fn impl_pytypeinfo( - cls: &syn::Ident, - attr: &PyClassArgs, - deprecations: Option<&Deprecations<'_>>, - ctx: &Ctx, -) -> TokenStream { - let Ctx { pyo3_path } = ctx; +fn impl_pytypeinfo(cls: &syn::Ident, attr: &PyClassArgs, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let cls_name = get_class_python_name(cls, attr).to_string(); let module = if let Some(ModuleAttribute { value, .. }) = &attr.options.module { @@ -1292,11 +1823,6 @@ fn impl_pytypeinfo( }; quote! { - #[allow(deprecated)] - unsafe impl #pyo3_path::type_object::HasPyGilRef for #cls { - type AsRefTarget = #pyo3_path::PyCell; - } - unsafe impl #pyo3_path::type_object::PyTypeInfo for #cls { const NAME: &'static str = #cls_name; const MODULE: ::std::option::Option<&'static str> = #module; @@ -1304,8 +1830,6 @@ fn impl_pytypeinfo( #[inline] fn type_object_raw(py: #pyo3_path::Python<'_>) -> *mut #pyo3_path::ffi::PyTypeObject { use #pyo3_path::prelude::PyTypeMethods; - #deprecations - <#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::lazy_type_object() .get_or_init(py) .as_type_ptr() @@ -1314,6 +1838,191 @@ fn impl_pytypeinfo( } } +fn pyclass_richcmp_arms( + options: &PyClassPyO3Options, + ctx: &Ctx, +) -> std::result::Result { + let Ctx { pyo3_path, .. } = ctx; + + let eq_arms = options + .eq + .map(|eq| eq.span) + .or(options.eq_int.map(|eq_int| eq_int.span)) + .map(|span| { + quote_spanned! { span => + #pyo3_path::pyclass::CompareOp::Eq => { + #pyo3_path::IntoPyObjectExt::into_py_any(self_val == other, py) + }, + #pyo3_path::pyclass::CompareOp::Ne => { + #pyo3_path::IntoPyObjectExt::into_py_any(self_val != other, py) + }, + } + }) + .unwrap_or_default(); + + if let Some(ord) = options.ord { + ensure_spanned!(options.eq.is_some(), ord.span() => "The `ord` option requires the `eq` option."); + } + + let ord_arms = options + .ord + .map(|ord| { + quote_spanned! { ord.span() => + #pyo3_path::pyclass::CompareOp::Gt => { + #pyo3_path::IntoPyObjectExt::into_py_any(self_val > other, py) + }, + #pyo3_path::pyclass::CompareOp::Lt => { + #pyo3_path::IntoPyObjectExt::into_py_any(self_val < other, py) + }, + #pyo3_path::pyclass::CompareOp::Le => { + #pyo3_path::IntoPyObjectExt::into_py_any(self_val <= other, py) + }, + #pyo3_path::pyclass::CompareOp::Ge => { + #pyo3_path::IntoPyObjectExt::into_py_any(self_val >= other, py) + }, + } + }) + .unwrap_or_else(|| quote! { _ => ::std::result::Result::Ok(py.NotImplemented()) }); + + Ok(quote! { + #eq_arms + #ord_arms + }) +} + +fn pyclass_richcmp_simple_enum( + options: &PyClassPyO3Options, + cls: &syn::Type, + repr_type: &syn::Ident, + ctx: &Ctx, +) -> Result<(Option, Option)> { + let Ctx { pyo3_path, .. } = ctx; + + if let Some(eq_int) = options.eq_int { + ensure_spanned!(options.eq.is_some(), eq_int.span() => "The `eq_int` option requires the `eq` option."); + } + + if options.eq.is_none() && options.eq_int.is_none() { + return Ok((None, None)); + } + + let arms = pyclass_richcmp_arms(options, ctx)?; + + let eq = options.eq.map(|eq| { + quote_spanned! { eq.span() => + let self_val = self; + if let ::std::result::Result::Ok(other) = #pyo3_path::types::PyAnyMethods::downcast::(other) { + let other = &*other.borrow(); + return match op { + #arms + } + } + } + }); + + let eq_int = options.eq_int.map(|eq_int| { + quote_spanned! { eq_int.span() => + let self_val = self.__pyo3__int__(); + if let ::std::result::Result::Ok(other) = #pyo3_path::types::PyAnyMethods::extract::<#repr_type>(other).or_else(|_| { + #pyo3_path::types::PyAnyMethods::downcast::(other).map(|o| o.borrow().__pyo3__int__()) + }) { + return match op { + #arms + } + } + } + }); + + let mut richcmp_impl = parse_quote! { + fn __pyo3__generated____richcmp__( + &self, + py: #pyo3_path::Python, + other: &#pyo3_path::Bound<'_, #pyo3_path::PyAny>, + op: #pyo3_path::pyclass::CompareOp + ) -> #pyo3_path::PyResult<#pyo3_path::PyObject> { + #eq + + #eq_int + + ::std::result::Result::Ok(py.NotImplemented()) + } + }; + let richcmp_slot = if options.eq.is_some() { + generate_protocol_slot(cls, &mut richcmp_impl, &__RICHCMP__, "__richcmp__", ctx).unwrap() + } else { + generate_default_protocol_slot(cls, &mut richcmp_impl, &__RICHCMP__, ctx).unwrap() + }; + Ok((Some(richcmp_impl), Some(richcmp_slot))) +} + +fn pyclass_richcmp( + options: &PyClassPyO3Options, + cls: &syn::Type, + ctx: &Ctx, +) -> Result<(Option, Option)> { + let Ctx { pyo3_path, .. } = ctx; + if let Some(eq_int) = options.eq_int { + bail_spanned!(eq_int.span() => "`eq_int` can only be used on simple enums.") + } + + let arms = pyclass_richcmp_arms(options, ctx)?; + if options.eq.is_some() { + let mut richcmp_impl = parse_quote! { + fn __pyo3__generated____richcmp__( + &self, + py: #pyo3_path::Python, + other: &#pyo3_path::Bound<'_, #pyo3_path::PyAny>, + op: #pyo3_path::pyclass::CompareOp + ) -> #pyo3_path::PyResult<#pyo3_path::PyObject> { + let self_val = self; + if let ::std::result::Result::Ok(other) = #pyo3_path::types::PyAnyMethods::downcast::(other) { + let other = &*other.borrow(); + match op { + #arms + } + } else { + ::std::result::Result::Ok(py.NotImplemented()) + } + } + }; + let richcmp_slot = + generate_protocol_slot(cls, &mut richcmp_impl, &__RICHCMP__, "__richcmp__", ctx) + .unwrap(); + Ok((Some(richcmp_impl), Some(richcmp_slot))) + } else { + Ok((None, None)) + } +} + +fn pyclass_hash( + options: &PyClassPyO3Options, + cls: &syn::Type, + ctx: &Ctx, +) -> Result<(Option, Option)> { + if options.hash.is_some() { + ensure_spanned!( + options.frozen.is_some(), options.hash.span() => "The `hash` option requires the `frozen` option."; + options.eq.is_some(), options.hash.span() => "The `hash` option requires the `eq` option."; + ); + } + // FIXME: Use hash.map(...).unzip() on MSRV >= 1.66 + match options.hash { + Some(opt) => { + let mut hash_impl = parse_quote_spanned! { opt.span() => + fn __pyo3__generated____hash__(&self) -> u64 { + let mut s = ::std::collections::hash_map::DefaultHasher::new(); + ::std::hash::Hash::hash(self, &mut s); + ::std::hash::Hasher::finish(&s) + } + }; + let hash_slot = + generate_protocol_slot(cls, &mut hash_impl, &__HASH__, "__hash__", ctx).unwrap(); + Ok((Some(hash_impl), Some(hash_slot))) + } + None => Ok((None, None)), + } +} + /// Implements most traits used by `#[pyclass]`. /// /// Specifically, it implements traits that only depend on class name, @@ -1368,7 +2077,7 @@ impl<'a> PyClassImplsBuilder<'a> { } fn impl_pyclass(&self, ctx: &Ctx) -> TokenStream { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let cls = self.cls; let frozen = if self.attr.options.frozen.is_some() { @@ -1384,11 +2093,11 @@ impl<'a> PyClassImplsBuilder<'a> { } } fn impl_extractext(&self, ctx: &Ctx) -> TokenStream { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let cls = self.cls; if self.attr.options.frozen.is_some() { quote! { - impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a #cls + impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py, false> for &'a #cls { type Holder = ::std::option::Option<#pyo3_path::PyRef<'py, #cls>>; @@ -1400,7 +2109,7 @@ impl<'a> PyClassImplsBuilder<'a> { } } else { quote! { - impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a #cls + impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py, false> for &'a #cls { type Holder = ::std::option::Option<#pyo3_path::PyRef<'py, #cls>>; @@ -1410,7 +2119,7 @@ impl<'a> PyClassImplsBuilder<'a> { } } - impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a mut #cls + impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py, false> for &'a mut #cls { type Holder = ::std::option::Option<#pyo3_path::PyRefMut<'py, #cls>>; @@ -1424,26 +2133,43 @@ impl<'a> PyClassImplsBuilder<'a> { } fn impl_into_py(&self, ctx: &Ctx) -> TokenStream { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let cls = self.cls; let attr = self.attr; // If #cls is not extended type, we allow Self->PyObject conversion if attr.options.extends.is_none() { quote! { + #[allow(deprecated)] impl #pyo3_path::IntoPy<#pyo3_path::PyObject> for #cls { - fn into_py(self, py: #pyo3_path::Python) -> #pyo3_path::PyObject { + fn into_py(self, py: #pyo3_path::Python<'_>) -> #pyo3_path::PyObject { #pyo3_path::IntoPy::into_py(#pyo3_path::Py::new(py, self).unwrap(), py) } } + + impl<'py> #pyo3_path::conversion::IntoPyObject<'py> for #cls { + type Target = Self; + type Output = #pyo3_path::Bound<'py, >::Target>; + type Error = #pyo3_path::PyErr; + + fn into_pyobject(self, py: #pyo3_path::Python<'py>) -> ::std::result::Result< + ::Output, + ::Error, + > { + #pyo3_path::Bound::new(py, self) + } + } } } else { quote! {} } } fn impl_pyclassimpl(&self, ctx: &Ctx) -> Result { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let cls = self.cls; - let doc = self.doc.as_ref().map_or(quote! {"\0"}, |doc| quote! {#doc}); + let doc = self.doc.as_ref().map_or( + LitCStr::empty(ctx).to_token_stream(), + PythonDoc::to_token_stream, + ); let is_basetype = self.attr.options.subclass.is_some(); let base = match &self.attr.options.extends { Some(extends_attr) => extends_attr.value.clone(), @@ -1553,7 +2279,33 @@ impl<'a> PyClassImplsBuilder<'a> { quote! { #pyo3_path::PyAny } }; + let pyclass_base_type_impl = attr.options.subclass.map(|subclass| { + quote_spanned! { subclass.span() => + impl #pyo3_path::impl_::pyclass::PyClassBaseType for #cls { + type LayoutAsBase = #pyo3_path::impl_::pycell::PyClassObject; + type BaseNativeType = ::BaseNativeType; + type Initializer = #pyo3_path::pyclass_init::PyClassInitializer; + type PyClassMutability = ::PyClassMutability; + } + } + }); + + let assertions = if attr.options.unsendable.is_some() { + TokenStream::new() + } else { + let assert = quote_spanned! { cls.span() => #pyo3_path::impl_::pyclass::assert_pyclass_sync::<#cls>(); }; + quote! { + const _: () = { + #assert + }; + } + }; + Ok(quote! { + #assertions + + #pyclass_base_type_impl + impl #pyo3_path::impl_::pyclass::PyClassImpl for #cls { const IS_BASETYPE: bool = #is_basetype; const IS_SUBCLASS: bool = #is_subclass; @@ -1583,7 +2335,7 @@ impl<'a> PyClassImplsBuilder<'a> { static DOC: #pyo3_path::sync::GILOnceCell<::std::borrow::Cow<'static, ::std::ffi::CStr>> = #pyo3_path::sync::GILOnceCell::new(); DOC.get_or_try_init(py, || { let collector = PyClassImplCollector::::new(); - build_pyclass_doc(<#cls as #pyo3_path::PyTypeInfo>::NAME, #doc, collector.new_text_signature()) + build_pyclass_doc(::NAME, #doc, collector.new_text_signature()) }).map(::std::ops::Deref::deref) } @@ -1609,34 +2361,32 @@ impl<'a> PyClassImplsBuilder<'a> { } fn impl_add_to_module(&self, ctx: &Ctx) -> TokenStream { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let cls = self.cls; quote! { impl #cls { #[doc(hidden)] - const _PYO3_DEF: #pyo3_path::impl_::pymodule::AddClassToModule = #pyo3_path::impl_::pymodule::AddClassToModule::new(); + pub const _PYO3_DEF: #pyo3_path::impl_::pymodule::AddClassToModule = #pyo3_path::impl_::pymodule::AddClassToModule::new(); } } } fn impl_freelist(&self, ctx: &Ctx) -> TokenStream { let cls = self.cls; - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; self.attr.options.freelist.as_ref().map_or(quote!{}, |freelist| { let freelist = &freelist.value; quote! { impl #pyo3_path::impl_::pyclass::PyClassWithFreeList for #cls { #[inline] - fn get_free_list(py: #pyo3_path::Python<'_>) -> &mut #pyo3_path::impl_::freelist::FreeList<*mut #pyo3_path::ffi::PyObject> { - static mut FREELIST: *mut #pyo3_path::impl_::freelist::FreeList<*mut #pyo3_path::ffi::PyObject> = 0 as *mut _; - unsafe { - if FREELIST.is_null() { - FREELIST = ::std::boxed::Box::into_raw(::std::boxed::Box::new( - #pyo3_path::impl_::freelist::FreeList::with_capacity(#freelist))); - } - &mut *FREELIST - } + fn get_free_list(py: #pyo3_path::Python<'_>) -> &'static ::std::sync::Mutex<#pyo3_path::impl_::freelist::PyObjectFreeList> { + static FREELIST: #pyo3_path::sync::GILOnceCell<::std::sync::Mutex<#pyo3_path::impl_::freelist::PyObjectFreeList>> = #pyo3_path::sync::GILOnceCell::new(); + // If there's a race to fill the cell, the object created + // by the losing thread will be deallocated via RAII + &FREELIST.get_or_init(py, || { + ::std::sync::Mutex::new(#pyo3_path::impl_::freelist::PyObjectFreeList::with_capacity(#freelist)) + }) } } } @@ -1644,7 +2394,7 @@ impl<'a> PyClassImplsBuilder<'a> { } fn freelist_slots(&self, ctx: &Ctx) -> Vec { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let cls = self.cls; if self.attr.options.freelist.is_some() { @@ -1669,7 +2419,7 @@ impl<'a> PyClassImplsBuilder<'a> { } fn define_inventory_class(inventory_class_name: &syn::Ident, ctx: &Ctx) -> TokenStream { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; quote! { #[doc(hidden)] pub struct #inventory_class_name { @@ -1691,6 +2441,37 @@ fn define_inventory_class(inventory_class_name: &syn::Ident, ctx: &Ctx) -> Token } } +fn generate_cfg_check(variants: &[PyClassEnumUnitVariant<'_>], cls: &syn::Ident) -> TokenStream { + if variants.is_empty() { + return quote! {}; + } + + let mut conditions = Vec::new(); + + for variant in variants { + let cfg_attrs = &variant.cfg_attrs; + + if cfg_attrs.is_empty() { + // There's at least one variant of the enum without cfg attributes, + // so the check is not necessary + return quote! {}; + } + + for attr in cfg_attrs { + if let syn::Meta::List(meta) = &attr.meta { + let cfg_tokens = &meta.tokens; + conditions.push(quote! { not(#cfg_tokens) }); + } + } + } + + quote_spanned! { + cls.span() => + #[cfg(all(#(#conditions),*))] + ::core::compile_error!(concat!("#[pyclass] can't be used on enums without any variants - all variants of enum `", stringify!(#cls), "` have been configured out by cfg attributes")); + } +} + const UNIQUE_GET: &str = "`get` may only be specified once"; const UNIQUE_SET: &str = "`set` may only be specified once"; const UNIQUE_NAME: &str = "`name` may only be specified once"; diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 9f9557a3664..c87492f095c 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -4,21 +4,18 @@ use crate::{ self, get_pyo3_options, take_attributes, take_pyo3_options, CrateAttribute, FromPyWithAttribute, NameAttribute, TextSignatureAttribute, }, - deprecations::Deprecations, method::{self, CallingConvention, FnArg}, pymethod::check_generic, }; use proc_macro2::TokenStream; use quote::{format_ident, quote}; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; use syn::{ext::IdentExt, spanned::Spanned, Result}; -use syn::{ - parse::{Parse, ParseStream}, - token::Comma, -}; mod signature; -pub use self::signature::{FunctionSignature, SignatureAttribute}; +pub use self::signature::{ConstructorAttribute, FunctionSignature, SignatureAttribute}; #[derive(Clone, Debug)] pub struct PyFunctionArgPyO3Attributes { @@ -97,24 +94,8 @@ impl Parse for PyFunctionOptions { fn parse(input: ParseStream<'_>) -> Result { let mut options = PyFunctionOptions::default(); - while !input.is_empty() { - let lookahead = input.lookahead1(); - if lookahead.peek(attributes::kw::name) - || lookahead.peek(attributes::kw::pass_module) - || lookahead.peek(attributes::kw::signature) - || lookahead.peek(attributes::kw::text_signature) - { - options.add_attributes(std::iter::once(input.parse()?))?; - if !input.is_empty() { - let _: Comma = input.parse()?; - } - } else if lookahead.peek(syn::Token![crate]) { - // TODO needs duplicate check? - options.krate = Some(input.parse()?); - } else { - return Err(lookahead.error()); - } - } + let attrs = Punctuated::::parse_terminated(input)?; + options.add_attributes(attrs)?; Ok(options) } @@ -205,10 +186,13 @@ pub fn impl_wrap_pyfunction( krate, } = options; - let ctx = &Ctx::new(&krate); - let Ctx { pyo3_path } = &ctx; + let ctx = &Ctx::new(&krate, Some(&func.sig)); + let Ctx { pyo3_path, .. } = &ctx; - let python_name = name.map_or_else(|| func.sig.ident.unraw(), |name| name.value.0); + let python_name = name + .as_ref() + .map_or_else(|| &func.sig.ident, |name| &name.value.0) + .unraw(); let tp = if pass_module.is_some() { let span = match func.sig.inputs.first() { @@ -237,7 +221,7 @@ pub fn impl_wrap_pyfunction( let signature = if let Some(signature) = signature { FunctionSignature::from_arguments_and_attribute(arguments, signature)? } else { - FunctionSignature::from_arguments(arguments)? + FunctionSignature::from_arguments(arguments) }; let spec = method::FnSpec { @@ -249,7 +233,6 @@ pub fn impl_wrap_pyfunction( text_signature, asyncness: func.sig.asyncness, unsafety: func.sig.unsafety, - deprecations: Deprecations::new(ctx), }; let vis = &func.vis; @@ -257,7 +240,7 @@ pub fn impl_wrap_pyfunction( let wrapper_ident = format_ident!("__pyfunction_{}", spec.name); let wrapper = spec.get_wrapper_function(&wrapper_ident, None, ctx)?; - let methoddef = spec.get_methoddef(wrapper_ident, &spec.get_doc(&func.attrs), ctx); + let methoddef = spec.get_methoddef(wrapper_ident, &spec.get_doc(&func.attrs, ctx), ctx); let wrapped_pyfunction = quote! { @@ -273,6 +256,7 @@ pub fn impl_wrap_pyfunction( // this avoids complications around the fact that the generated module has a different scope // (and `super` doesn't always refer to the outer scope, e.g. if the `#[pyfunction] is // inside a function body) + #[allow(unknown_lints, non_local_definitions)] impl #name::MakeDef { const _PYO3_DEF: #pyo3_path::impl_::pymethods::PyMethodDef = #methoddef; } diff --git a/pyo3-macros-backend/src/pyfunction/signature.rs b/pyo3-macros-backend/src/pyfunction/signature.rs index baf01285658..deea3dfa052 100644 --- a/pyo3-macros-backend/src/pyfunction/signature.rs +++ b/pyo3-macros-backend/src/pyfunction/signature.rs @@ -10,9 +10,10 @@ use syn::{ use crate::{ attributes::{kw, KeywordAttribute}, - method::FnArg, + method::{FnArg, RegularArg}, }; +#[derive(Clone)] pub struct Signature { paren_token: syn::token::Paren, pub items: Punctuated, @@ -36,35 +37,35 @@ impl ToTokens for Signature { } } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct SignatureItemArgument { pub ident: syn::Ident, pub eq_and_default: Option<(Token![=], syn::Expr)>, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct SignatureItemPosargsSep { pub slash: Token![/], } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct SignatureItemVarargsSep { pub asterisk: Token![*], } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct SignatureItemVarargs { pub sep: SignatureItemVarargsSep, pub ident: syn::Ident, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct SignatureItemKwargs { pub asterisks: (Token![*], Token![*]), pub ident: syn::Ident, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum SignatureItem { Argument(Box), PosargsSep(SignatureItemPosargsSep), @@ -195,6 +196,16 @@ impl ToTokens for SignatureItemPosargsSep { } pub type SignatureAttribute = KeywordAttribute; +pub type ConstructorAttribute = KeywordAttribute; + +impl ConstructorAttribute { + pub fn into_signature(self) -> SignatureAttribute { + SignatureAttribute { + kw: kw::signature(self.kw.span), + value: self.value, + } + } +} #[derive(Default)] pub struct PythonSignature { @@ -351,36 +362,39 @@ impl<'a> FunctionSignature<'a> { let mut next_non_py_argument_checked = |name: &syn::Ident| { for fn_arg in args_iter.by_ref() { - if fn_arg.py { - // If the user incorrectly tried to include py: Python in the - // signature, give a useful error as a hint. - ensure_spanned!( - name != fn_arg.name, - name.span() => "arguments of type `Python` must not be part of the signature" - ); - // Otherwise try next argument. - continue; - } - if fn_arg.is_cancel_handle { - // If the user incorrectly tried to include cancel: CoroutineCancel in the - // signature, give a useful error as a hint. - ensure_spanned!( - name != fn_arg.name, - name.span() => "`cancel_handle` argument must not be part of the signature" - ); - // Otherwise try next argument. - continue; + match fn_arg { + crate::method::FnArg::Py(..) => { + // If the user incorrectly tried to include py: Python in the + // signature, give a useful error as a hint. + ensure_spanned!( + name != fn_arg.name(), + name.span() => "arguments of type `Python` must not be part of the signature" + ); + // Otherwise try next argument. + continue; + } + crate::method::FnArg::CancelHandle(..) => { + // If the user incorrectly tried to include cancel: CoroutineCancel in the + // signature, give a useful error as a hint. + ensure_spanned!( + name != fn_arg.name(), + name.span() => "`cancel_handle` argument must not be part of the signature" + ); + // Otherwise try next argument. + continue; + } + _ => { + ensure_spanned!( + name == fn_arg.name(), + name.span() => format!( + "expected argument from function definition `{}` but got argument `{}`", + fn_arg.name().unraw(), + name.unraw(), + ) + ); + return Ok(fn_arg); + } } - - ensure_spanned!( - name == fn_arg.name, - name.span() => format!( - "expected argument from function definition `{}` but got argument `{}`", - fn_arg.name.unraw(), - name.unraw(), - ) - ); - return Ok(fn_arg); } bail_spanned!( name.span() => "signature entry does not have a corresponding function argument" @@ -398,7 +412,15 @@ impl<'a> FunctionSignature<'a> { arg.span(), )?; if let Some((_, default)) = &arg.eq_and_default { - fn_arg.default = Some(default.clone()); + if let FnArg::Regular(arg) = fn_arg { + arg.default_value = Some(default.clone()); + } else { + unreachable!( + "`Python` and `CancelHandle` are already handled above and `*args`/`**kwargs` are \ + parsed and transformed below. Because the have to come last and are only allowed \ + once, this has to be a regular argument." + ); + } } } SignatureItem::VarargsSep(sep) => { @@ -406,12 +428,12 @@ impl<'a> FunctionSignature<'a> { } SignatureItem::Varargs(varargs) => { let fn_arg = next_non_py_argument_checked(&varargs.ident)?; - fn_arg.is_varargs = true; + fn_arg.to_varargs_mut()?; parse_state.add_varargs(&mut python_signature, varargs)?; } SignatureItem::Kwargs(kwargs) => { let fn_arg = next_non_py_argument_checked(&kwargs.ident)?; - fn_arg.is_kwargs = true; + fn_arg.to_kwargs_mut()?; parse_state.add_kwargs(&mut python_signature, kwargs)?; } SignatureItem::PosargsSep(sep) => { @@ -421,9 +443,11 @@ impl<'a> FunctionSignature<'a> { } // Ensure no non-py arguments remain - if let Some(arg) = args_iter.find(|arg| !arg.py && !arg.is_cancel_handle) { + if let Some(arg) = + args_iter.find(|arg| !matches!(arg, FnArg::Py(..) | FnArg::CancelHandle(..))) + { bail_spanned!( - attribute.kw.span() => format!("missing signature entry for argument `{}`", arg.name) + attribute.kw.span() => format!("missing signature entry for argument `{}`", arg.name()) ); } @@ -435,20 +459,19 @@ impl<'a> FunctionSignature<'a> { } /// Without `#[pyo3(signature)]` or `#[args]` - just take the Rust function arguments as positional. - pub fn from_arguments(arguments: Vec>) -> syn::Result { + pub fn from_arguments(arguments: Vec>) -> Self { let mut python_signature = PythonSignature::default(); for arg in &arguments { // Python<'_> arguments don't show in Python signature - if arg.py || arg.is_cancel_handle { + if matches!(arg, FnArg::Py(..) | FnArg::CancelHandle(..)) { continue; } - if arg.optional.is_none() { + if let FnArg::Regular(RegularArg { .. }) = arg { // This argument is required, all previous arguments must also have been required - ensure_spanned!( - python_signature.required_positional_parameters == python_signature.positional_parameters.len(), - arg.ty.span() => "required arguments after an `Option<_>` argument are ambiguous\n\ - = help: add a `#[pyo3(signature)]` annotation on this function to unambiguously specify the default values for all optional parameters" + assert_eq!( + python_signature.required_positional_parameters, + python_signature.positional_parameters.len(), ); python_signature.required_positional_parameters = @@ -457,20 +480,24 @@ impl<'a> FunctionSignature<'a> { python_signature .positional_parameters - .push(arg.name.unraw().to_string()); + .push(arg.name().unraw().to_string()); } - Ok(Self { + Self { arguments, python_signature, attribute: None, - }) + } } fn default_value_for_parameter(&self, parameter: &str) -> String { let mut default = "...".to_string(); - if let Some(fn_arg) = self.arguments.iter().find(|arg| arg.name == parameter) { - if let Some(arg_default) = fn_arg.default.as_ref() { + if let Some(fn_arg) = self.arguments.iter().find(|arg| arg.name() == parameter) { + if let FnArg::Regular(RegularArg { + default_value: Some(arg_default), + .. + }) = fn_arg + { match arg_default { // literal values syn::Expr::Lit(syn::ExprLit { lit, .. }) => match lit { @@ -496,7 +523,11 @@ impl<'a> FunctionSignature<'a> { // others, unsupported yet so defaults to `...` _ => {} } - } else if fn_arg.optional.is_some() { + } else if let FnArg::Regular(RegularArg { + option_wrapped_type: Some(..), + .. + }) = fn_arg + { // functions without a `#[pyo3(signature = (...))]` option // will treat trailing `Option` arguments as having a default of `None` default = "None".to_string(); diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index cf27cf37066..72f06721ec4 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -1,6 +1,6 @@ use std::collections::HashSet; -use crate::utils::Ctx; +use crate::utils::{has_attribute, has_attribute_with_namespace, Ctx, PyO3CratePath}; use crate::{ attributes::{take_pyo3_options, CrateAttribute}, konst::{ConstAttributes, ConstSpec}, @@ -10,6 +10,7 @@ use crate::{ use proc_macro2::TokenStream; use pymethod::GeneratedPyMethod; use quote::{format_ident, quote}; +use syn::ImplItemFn; use syn::{ parse::{Parse, ParseStream}, spanned::Spanned, @@ -84,13 +85,31 @@ pub fn build_py_methods( } } +fn check_pyfunction(pyo3_path: &PyO3CratePath, meth: &mut ImplItemFn) -> syn::Result<()> { + let mut error = None; + + meth.attrs.retain(|attr| { + let attrs = [attr.clone()]; + + if has_attribute(&attrs, "pyfunction") + || has_attribute_with_namespace(&attrs, Some(pyo3_path), &["pyfunction"]) + || has_attribute_with_namespace(&attrs, Some(pyo3_path), &["prelude", "pyfunction"]) { + error = Some(err_spanned!(meth.sig.span() => "functions inside #[pymethods] do not need to be annotated with #[pyfunction]")); + false + } else { + true + } + }); + + error.map_or(Ok(()), Err) +} + pub fn impl_methods( ty: &syn::Type, impls: &mut [syn::ImplItem], methods_type: PyClassMethodsType, options: PyImplOptions, ) -> syn::Result { - let ctx = &Ctx::new(&options.krate); let mut trait_impls = Vec::new(); let mut proto_impls = Vec::new(); let mut methods = Vec::new(); @@ -101,8 +120,12 @@ pub fn impl_methods( for iimpl in impls { match iimpl { syn::ImplItem::Fn(meth) => { + let ctx = &Ctx::new(&options.krate, Some(&meth.sig)); let mut fun_options = PyFunctionOptions::from_attrs(&mut meth.attrs)?; fun_options.krate = fun_options.krate.or_else(|| options.krate.clone()); + + check_pyfunction(&ctx.pyo3_path, meth)?; + match pymethod::gen_py_method(ty, &mut meth.sig, &mut meth.attrs, fun_options, ctx)? { GeneratedPyMethod::Method(MethodAndMethodDef { @@ -129,7 +152,8 @@ pub fn impl_methods( } } syn::ImplItem::Const(konst) => { - let attributes = ConstAttributes::from_attrs(&mut konst.attrs, ctx)?; + let ctx = &Ctx::new(&options.krate, None); + let attributes = ConstAttributes::from_attrs(&mut konst.attrs)?; if attributes.is_class_attr { let spec = ConstSpec { rust_ident: konst.ident.clone(), @@ -159,11 +183,10 @@ pub fn impl_methods( _ => {} } } + let ctx = &Ctx::new(&options.krate, None); add_shared_proto_slots(ty, &mut proto_impls, implemented_proto_fragments, ctx); - let ctx = &Ctx::new(&options.krate); - let items = match methods_type { PyClassMethodsType::Specialization => impl_py_methods(ty, methods, proto_impls, ctx), PyClassMethodsType::Inventory => submit_methods_inventory(ty, methods, proto_impls, ctx), @@ -182,27 +205,27 @@ pub fn impl_methods( }) } -pub fn gen_py_const(cls: &syn::Type, spec: &ConstSpec<'_>, ctx: &Ctx) -> MethodAndMethodDef { +pub fn gen_py_const(cls: &syn::Type, spec: &ConstSpec, ctx: &Ctx) -> MethodAndMethodDef { let member = &spec.rust_ident; let wrapper_ident = format_ident!("__pymethod_{}__", member); - let deprecations = &spec.attributes.deprecations; - let python_name = &spec.null_terminated_python_name(); - let Ctx { pyo3_path } = ctx; + let python_name = spec.null_terminated_python_name(ctx); + let Ctx { pyo3_path, .. } = ctx; let associated_method = quote! { fn #wrapper_ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::PyObject> { - #deprecations - ::std::result::Result::Ok(#pyo3_path::IntoPy::into_py(#cls::#member, py)) + #pyo3_path::IntoPyObjectExt::into_py_any(#cls::#member, py) } }; let method_def = quote! { - #pyo3_path::class::PyMethodDefType::ClassAttribute({ - #pyo3_path::class::PyClassAttributeDef::new( - #python_name, - #pyo3_path::impl_::pymethods::PyClassAttributeFactory(#cls::#wrapper_ident) - ) - }) + #pyo3_path::impl_::pyclass::MaybeRuntimePyMethodDef::Static( + #pyo3_path::impl_::pymethods::PyMethodDefType::ClassAttribute({ + #pyo3_path::impl_::pymethods::PyClassAttributeDef::new( + #python_name, + #cls::#wrapper_ident + ) + }) + ) }; MethodAndMethodDef { @@ -217,8 +240,9 @@ fn impl_py_methods( proto_impls: Vec, ctx: &Ctx, ) -> TokenStream { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; quote! { + #[allow(unknown_lints, non_local_definitions)] impl #pyo3_path::impl_::pyclass::PyMethods<#ty> for #pyo3_path::impl_::pyclass::PyClassImplCollector<#ty> { @@ -239,7 +263,7 @@ fn add_shared_proto_slots( mut implemented_proto_fragments: HashSet, ctx: &Ctx, ) { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; macro_rules! try_add_shared_slot { ($slot:ident, $($fragments:literal),*) => {{ let mut implemented = false; @@ -297,7 +321,7 @@ fn submit_methods_inventory( proto_impls: Vec, ctx: &Ctx, ) -> TokenStream { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; quote! { #pyo3_path::inventory::submit! { type Inventory = <#ty as #pyo3_path::impl_::pyclass::PyClassImpl>::Inventory; @@ -306,7 +330,7 @@ fn submit_methods_inventory( } } -fn get_cfg_attributes(attrs: &[syn::Attribute]) -> Vec<&syn::Attribute> { +pub(crate) fn get_cfg_attributes(attrs: &[syn::Attribute]) -> Vec<&syn::Attribute> { attrs .iter() .filter(|attr| attr.path().is_ident("cfg")) diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index ee7d3d7aaee..a1689e4e75c 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -1,10 +1,11 @@ use std::borrow::Cow; +use std::ffi::CString; -use crate::attributes::{NameAttribute, RenamingRule}; -use crate::method::{CallingConvention, ExtractErrorMode}; -use crate::params::{check_arg_for_gil_refs, impl_arg_param, Holders}; -use crate::utils::Ctx; -use crate::utils::PythonDoc; +use crate::attributes::{FromPyWithAttribute, NameAttribute, RenamingRule}; +use crate::method::{CallingConvention, ExtractErrorMode, PyArg}; +use crate::params::{impl_regular_arg_param, Holders}; +use crate::utils::{deprecated_from_py_with, PythonDoc, TypeExt as _}; +use crate::utils::{Ctx, LitCStr}; use crate::{ method::{FnArg, FnSpec, FnType, SelfType}, pyfunction::PyFunctionOptions, @@ -95,7 +96,6 @@ impl PyMethodKind { "__ior__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__IOR__)), "__getbuffer__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__GETBUFFER__)), "__releasebuffer__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__RELEASEBUFFER__)), - "__clear__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__CLEAR__)), // Protocols implemented through traits "__getattribute__" => { PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__GETATTRIBUTE__)) @@ -144,6 +144,7 @@ impl PyMethodKind { // Some tricky protocols which don't fit the pattern of the rest "__call__" => PyMethodKind::Proto(PyMethodProtoKind::Call), "__traverse__" => PyMethodKind::Proto(PyMethodProtoKind::Traverse), + "__clear__" => PyMethodKind::Proto(PyMethodProtoKind::Clear), // Not a proto _ => PyMethodKind::Fn, } @@ -154,6 +155,7 @@ enum PyMethodProtoKind { Slot(&'static SlotDef), Call, Traverse, + Clear, SlotFragment(&'static SlotFragmentDef), } @@ -162,9 +164,8 @@ impl<'a> PyMethod<'a> { sig: &'a mut syn::Signature, meth_attrs: &mut Vec, options: PyFunctionOptions, - ctx: &'a Ctx, ) -> Result { - let spec = FnSpec::parse(sig, meth_attrs, options, ctx)?; + let spec = FnSpec::parse(sig, meth_attrs, options)?; let method_name = spec.python_name.to_string(); let kind = PyMethodKind::from_name(&method_name); @@ -193,9 +194,9 @@ pub fn gen_py_method( ) -> Result { check_generic(sig)?; ensure_function_options_valid(&options)?; - let method = PyMethod::parse(sig, meth_attrs, options, ctx)?; + let method = PyMethod::parse(sig, meth_attrs, options)?; let spec = &method.spec; - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; Ok(match (method.kind, &spec.tp) { // Class attributes go before protos so that class attributes can be used to set proto @@ -216,6 +217,9 @@ pub fn gen_py_method( PyMethodProtoKind::Traverse => { GeneratedPyMethod::Proto(impl_traverse_slot(cls, spec, ctx)?) } + PyMethodProtoKind::Clear => { + GeneratedPyMethod::Proto(impl_clear_slot(cls, spec, ctx)?) + } PyMethodProtoKind::SlotFragment(slot_fragment_def) => { let proto = slot_fragment_def.generate_pyproto_fragment(cls, spec, ctx)?; GeneratedPyMethod::SlotTraitImpl(method.method_name, proto) @@ -226,21 +230,21 @@ pub fn gen_py_method( (_, FnType::Fn(_)) => GeneratedPyMethod::Method(impl_py_method_def( cls, spec, - &spec.get_doc(meth_attrs), + &spec.get_doc(meth_attrs, ctx), None, ctx, )?), (_, FnType::FnClass(_)) => GeneratedPyMethod::Method(impl_py_method_def( cls, spec, - &spec.get_doc(meth_attrs), + &spec.get_doc(meth_attrs, ctx), Some(quote!(#pyo3_path::ffi::METH_CLASS)), ctx, )?), (_, FnType::FnStatic) => GeneratedPyMethod::Method(impl_py_method_def( cls, spec, - &spec.get_doc(meth_attrs), + &spec.get_doc(meth_attrs, ctx), Some(quote!(#pyo3_path::ffi::METH_STATIC)), ctx, )?), @@ -254,7 +258,7 @@ pub fn gen_py_method( PropertyType::Function { self_type, spec, - doc: spec.get_doc(meth_attrs), + doc: spec.get_doc(meth_attrs, ctx), }, ctx, )?), @@ -263,7 +267,7 @@ pub fn gen_py_method( PropertyType::Function { self_type, spec, - doc: spec.get_doc(meth_attrs), + doc: spec.get_doc(meth_attrs, ctx), }, ctx, )?), @@ -317,7 +321,7 @@ pub fn impl_py_method_def( flags: Option, ctx: &Ctx, ) -> Result { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let wrapper_ident = format_ident!("__pymethod_{}__", spec.python_name); let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls), ctx)?; let add_flags = flags.map(|flags| quote!(.flags(#flags))); @@ -328,7 +332,9 @@ pub fn impl_py_method_def( }; let methoddef = spec.get_methoddef(quote! { #cls::#wrapper_ident }, doc, ctx); let method_def = quote! { - #pyo3_path::class::PyMethodDefType::#methoddef_type(#methoddef #add_flags) + #pyo3_path::impl_::pyclass::MaybeRuntimePyMethodDef::Static( + #pyo3_path::impl_::pymethods::PyMethodDefType::#methoddef_type(#methoddef #add_flags) + ) }; Ok(MethodAndMethodDef { associated_method, @@ -342,7 +348,7 @@ pub fn impl_py_method_def_new( spec: &FnSpec<'_>, ctx: &Ctx, ) -> Result { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let wrapper_ident = syn::Ident::new("__pymethod___new____", Span::call_site()); let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls), ctx)?; // Use just the text_signature_call_signature() because the class' Python name @@ -352,7 +358,6 @@ pub fn impl_py_method_def_new( || quote!(::std::option::Option::None), |text_signature| quote!(::std::option::Option::Some(#text_signature)), ); - let deprecations = &spec.deprecations; let slot_def = quote! { #pyo3_path::ffi::PyType_Slot { slot: #pyo3_path::ffi::Py_tp_new, @@ -361,12 +366,9 @@ pub fn impl_py_method_def_new( subtype: *mut #pyo3_path::ffi::PyTypeObject, args: *mut #pyo3_path::ffi::PyObject, kwargs: *mut #pyo3_path::ffi::PyObject, - ) -> *mut #pyo3_path::ffi::PyObject - { - #deprecations - - use #pyo3_path::impl_::pyclass::*; - impl PyClassNewTextSignature<#cls> for PyClassImplCollector<#cls> { + ) -> *mut #pyo3_path::ffi::PyObject { + #[allow(unknown_lints, non_local_definitions)] + impl #pyo3_path::impl_::pyclass::PyClassNewTextSignature<#cls> for #pyo3_path::impl_::pyclass::PyClassImplCollector<#cls> { #[inline] fn new_text_signature(self) -> ::std::option::Option<&'static str> { #text_signature_body @@ -391,7 +393,7 @@ pub fn impl_py_method_def_new( } fn impl_call_slot(cls: &syn::Type, mut spec: FnSpec<'_>, ctx: &Ctx) -> Result { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; // HACK: __call__ proto slot must always use varargs calling convention, so change the spec. // Probably indicates there's a refactoring opportunity somewhere. @@ -431,12 +433,27 @@ fn impl_traverse_slot( spec: &FnSpec<'_>, ctx: &Ctx, ) -> syn::Result { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; if let (Some(py_arg), _) = split_off_python_arg(&spec.signature.arguments) { return Err(syn::Error::new_spanned(py_arg.ty, "__traverse__ may not take `Python`. \ - Usually, an implementation of `__traverse__` should do nothing but calls to `visit.call`. \ - Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, \ - i.e. `Python::with_gil` will panic.")); + Usually, an implementation of `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` \ + should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited \ + inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic.")); + } + + // check that the receiver does not try to smuggle an (implicit) `Python` token into here + if let FnType::Fn(SelfType::TryFromBoundRef(span)) + | FnType::Fn(SelfType::Receiver { + mutable: true, + span, + }) = spec.tp + { + bail_spanned! { span => + "__traverse__ may not take a receiver other than `&self`. Usually, an implementation of \ + `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` \ + should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited \ + inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic." + } } let rust_fn_ident = spec.name; @@ -447,7 +464,7 @@ fn impl_traverse_slot( visit: #pyo3_path::ffi::visitproc, arg: *mut ::std::os::raw::c_void, ) -> ::std::os::raw::c_int { - #pyo3_path::impl_::pymethods::_call_traverse::<#cls>(slf, #cls::#rust_fn_ident, visit, arg) + #pyo3_path::impl_::pymethods::_call_traverse::<#cls>(slf, #cls::#rust_fn_ident, visit, arg, #cls::__pymethod_traverse__) } }; let slot_def = quote! { @@ -462,16 +479,62 @@ fn impl_traverse_slot( }) } -fn impl_py_class_attribute( +fn impl_clear_slot(cls: &syn::Type, spec: &FnSpec<'_>, ctx: &Ctx) -> syn::Result { + let Ctx { pyo3_path, .. } = ctx; + let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); + let self_type = match &spec.tp { + FnType::Fn(self_type) => self_type, + _ => bail_spanned!(spec.name.span() => "expected instance method for `__clear__` function"), + }; + let mut holders = Holders::new(); + let slf = self_type.receiver(cls, ExtractErrorMode::Raise, &mut holders, ctx); + + if let [arg, ..] = args { + bail_spanned!(arg.ty().span() => "`__clear__` function expected to have no arguments"); + } + + let name = &spec.name; + let holders = holders.init_holders(ctx); + let fncall = if py_arg.is_some() { + quote!(#cls::#name(#slf, py)) + } else { + quote!(#cls::#name(#slf)) + }; + + let associated_method = quote! { + pub unsafe extern "C" fn __pymethod___clear____( + _slf: *mut #pyo3_path::ffi::PyObject, + ) -> ::std::os::raw::c_int { + #pyo3_path::impl_::pymethods::_call_clear(_slf, |py, _slf| { + #holders + let result = #fncall; + let result = #pyo3_path::impl_::wrap::converter(&result).wrap(result)?; + ::std::result::Result::Ok(result) + }, #cls::__pymethod___clear____) + } + }; + let slot_def = quote! { + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::Py_tp_clear, + pfunc: #cls::__pymethod___clear____ as #pyo3_path::ffi::inquiry as _ + } + }; + Ok(MethodAndSlotDef { + associated_method, + slot_def, + }) +} + +pub(crate) fn impl_py_class_attribute( cls: &syn::Type, spec: &FnSpec<'_>, ctx: &Ctx, ) -> syn::Result { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); ensure_spanned!( args.is_empty(), - args[0].ty.span() => "#[classattr] can only have one argument (of type pyo3::Python)" + args[0].ty().span() => "#[classattr] can only have one argument (of type pyo3::Python)" ); let name = &spec.name; @@ -482,23 +545,26 @@ fn impl_py_class_attribute( }; let wrapper_ident = format_ident!("__pymethod_{}__", name); - let python_name = spec.null_terminated_python_name(); + let python_name = spec.null_terminated_python_name(ctx); let body = quotes::ok_wrap(fncall, ctx); let associated_method = quote! { fn #wrapper_ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::PyObject> { let function = #cls::#name; // Shadow the method name to avoid #3017 - #pyo3_path::impl_::wrap::map_result_into_py(py, #body) + let result = #body; + #pyo3_path::impl_::wrap::converter(&result).map_into_pyobject(py, result) } }; let method_def = quote! { - #pyo3_path::class::PyMethodDefType::ClassAttribute({ - #pyo3_path::class::PyClassAttributeDef::new( - #python_name, - #pyo3_path::impl_::pymethods::PyClassAttributeFactory(#cls::#wrapper_ident) - ) - }) + #pyo3_path::impl_::pyclass::MaybeRuntimePyMethodDef::Static( + #pyo3_path::impl_::pymethods::PyMethodDefType::ClassAttribute({ + #pyo3_path::impl_::pymethods::PyClassAttributeDef::new( + #python_name, + #cls::#wrapper_ident + ) + }) + ) }; Ok(MethodAndMethodDef { @@ -521,7 +587,7 @@ fn impl_call_setter( bail_spanned!(spec.name.span() => "setter function expected to have one argument"); } else if args.len() > 1 { bail_spanned!( - args[1].ty.span() => + args[1].ty().span() => "setter function can have at most two arguments ([pyo3::Python,] and value)" ); } @@ -542,9 +608,9 @@ pub fn impl_py_setter_def( property_type: PropertyType<'_>, ctx: &Ctx, ) -> Result { - let Ctx { pyo3_path } = ctx; - let python_name = property_type.null_terminated_python_name()?; - let doc = property_type.doc(); + let Ctx { pyo3_path, .. } = ctx; + let python_name = property_type.null_terminated_python_name(ctx)?; + let doc = property_type.doc(ctx); let mut holders = Holders::new(); let setter_impl = match property_type { PropertyType::Descriptor { @@ -590,36 +656,35 @@ pub fn impl_py_setter_def( PropertyType::Function { spec, .. } => { let (_, args) = split_off_python_arg(&spec.signature.arguments); let value_arg = &args[0]; - let (from_py_with, ident) = if let Some(from_py_with) = - &value_arg.attrs.from_py_with.as_ref().map(|f| &f.value) - { - let ident = syn::Ident::new("from_py_with", from_py_with.span()); - ( - quote_spanned! { from_py_with.span() => - let e = #pyo3_path::impl_::deprecations::GilRefs::new(); - let #ident = #pyo3_path::impl_::deprecations::inspect_fn(#from_py_with, &e); - e.from_py_with_arg(); - }, - ident, - ) + let (from_py_with, ident) = + if let Some(from_py_with) = &value_arg.from_py_with().as_ref().map(|f| &f.value) { + let ident = syn::Ident::new("from_py_with", from_py_with.span()); + let d = deprecated_from_py_with(from_py_with).unwrap_or_default(); + ( + quote_spanned! { from_py_with.span() => + #d + let #ident = #from_py_with; + }, + ident, + ) + } else { + (quote!(), syn::Ident::new("dummy", Span::call_site())) + }; + + let arg = if let FnArg::Regular(arg) = &value_arg { + arg } else { - (quote!(), syn::Ident::new("dummy", Span::call_site())) + bail_spanned!(value_arg.name().span() => "The #[setter] value argument can't be *args, **kwargs or `cancel_handle`."); }; - let extract = impl_arg_param( - &args[0], + let extract = impl_regular_arg_param( + arg, ident, quote!(::std::option::Option::Some(_value.into())), &mut holders, ctx, - ) - .map(|tokens| { - check_arg_for_gil_refs( - tokens, - holders.push_gil_refs_checker(value_arg.ty.span()), - ctx, - ) - })?; + ); + quote! { #from_py_with let _val = #extract; @@ -634,12 +699,13 @@ pub fn impl_py_setter_def( .unwrap_or_default(); let holder = holders.push_holder(span); - let gil_refs_checker = holders.push_gil_refs_checker(span); + let ty = field.ty.clone().elide_lifetimes(); quote! { - let _val = #pyo3_path::impl_::deprecations::inspect_type( - #pyo3_path::impl_::extract_argument::extract_argument(_value.into(), &mut #holder, #name)?, - &#gil_refs_checker - ); + use #pyo3_path::impl_::pyclass::Probe as _; + let _val = #pyo3_path::impl_::extract_argument::extract_argument::< + _, + { #pyo3_path::impl_::pyclass::IsOption::<#ty>::VALUE } + >(_value.into(), &mut #holder, #name)?; } } }; @@ -656,7 +722,6 @@ pub fn impl_py_setter_def( } let init_holders = holders.init_holders(ctx); - let check_gil_refs = holders.check_gil_refs(); let associated_method = quote! { #cfg_attrs unsafe fn #wrapper_ident( @@ -672,18 +737,19 @@ pub fn impl_py_setter_def( #init_holders #extract let result = #setter_impl; - #check_gil_refs - #pyo3_path::callback::convert(py, result) + #pyo3_path::impl_::callback::convert(py, result) } }; let method_def = quote! { #cfg_attrs - #pyo3_path::class::PyMethodDefType::Setter( - #pyo3_path::class::PySetterDef::new( - #python_name, - #pyo3_path::impl_::pymethods::PySetter(#cls::#wrapper_ident), - #doc + #pyo3_path::impl_::pyclass::MaybeRuntimePyMethodDef::Static( + #pyo3_path::impl_::pymethods::PyMethodDefType::Setter( + #pyo3_path::impl_::pymethods::PySetterDef::new( + #python_name, + #cls::#wrapper_ident, + #doc + ) ) ) }; @@ -705,7 +771,7 @@ fn impl_call_getter( let slf = self_type.receiver(cls, ExtractErrorMode::Raise, holders, ctx); ensure_spanned!( args.is_empty(), - args[0].ty.span() => "getter function can only have one argument (of type pyo3::Python)" + args[0].ty().span() => "getter function can only have one argument (of type pyo3::Python)" ); let name = &spec.name; @@ -724,112 +790,123 @@ pub fn impl_py_getter_def( property_type: PropertyType<'_>, ctx: &Ctx, ) -> Result { - let Ctx { pyo3_path } = ctx; - let python_name = property_type.null_terminated_python_name()?; - let doc = property_type.doc(); + let Ctx { pyo3_path, .. } = ctx; + let python_name = property_type.null_terminated_python_name(ctx)?; + let doc = property_type.doc(ctx); + + let mut cfg_attrs = TokenStream::new(); + if let PropertyType::Descriptor { field, .. } = &property_type { + for attr in field + .attrs + .iter() + .filter(|attr| attr.path().is_ident("cfg")) + { + attr.to_tokens(&mut cfg_attrs); + } + } let mut holders = Holders::new(); - let body = match property_type { + match property_type { PropertyType::Descriptor { field_index, field, .. } => { - let slf = SelfType::Receiver { - mutable: false, - span: Span::call_site(), - } - .receiver(cls, ExtractErrorMode::Raise, &mut holders, ctx); - let field_token = if let Some(ident) = &field.ident { - // named struct field + let ty = &field.ty; + let field = if let Some(ident) = &field.ident { ident.to_token_stream() } else { - // tuple struct field syn::Index::from(field_index).to_token_stream() }; - quotes::map_result_into_ptr( - quotes::ok_wrap( - quote! { - ::std::clone::Clone::clone(&(#slf.#field_token)) - }, - ctx, - ), - ctx, - ) + + // TODO: on MSRV 1.77+, we can use `::std::mem::offset_of!` here, and it should + // make it possible for the `MaybeRuntimePyMethodDef` to be a `Static` variant. + let generator = quote_spanned! { ty.span() => + #pyo3_path::impl_::pyclass::MaybeRuntimePyMethodDef::Runtime( + || GENERATOR.generate(#python_name, #doc) + ) + }; + // This is separate so that the unsafe below does not inherit the span and thus does not + // trigger the `unsafe_code` lint + let method_def = quote! { + #cfg_attrs + { + #[allow(unused_imports)] // might not be used if all probes are positve + use #pyo3_path::impl_::pyclass::Probe; + + struct Offset; + unsafe impl #pyo3_path::impl_::pyclass::OffsetCalculator<#cls, #ty> for Offset { + fn offset() -> usize { + #pyo3_path::impl_::pyclass::class_offset::<#cls>() + + #pyo3_path::impl_::pyclass::offset_of!(#cls, #field) + } + } + + const GENERATOR: #pyo3_path::impl_::pyclass::PyClassGetterGenerator::< + #cls, + #ty, + Offset, + { #pyo3_path::impl_::pyclass::IsPyT::<#ty>::VALUE }, + { #pyo3_path::impl_::pyclass::IsToPyObject::<#ty>::VALUE }, + { #pyo3_path::impl_::pyclass::IsIntoPy::<#ty>::VALUE }, + { #pyo3_path::impl_::pyclass::IsIntoPyObjectRef::<#ty>::VALUE }, + { #pyo3_path::impl_::pyclass::IsIntoPyObject::<#ty>::VALUE }, + > = unsafe { #pyo3_path::impl_::pyclass::PyClassGetterGenerator::new() }; + #generator + } + }; + + Ok(MethodAndMethodDef { + associated_method: quote! {}, + method_def, + }) } // Forward to `IntoPyCallbackOutput`, to handle `#[getter]`s returning results. PropertyType::Function { spec, self_type, .. } => { + let wrapper_ident = format_ident!("__pymethod_get_{}__", spec.name); let call = impl_call_getter(cls, spec, self_type, &mut holders, ctx)?; - quote! { - #pyo3_path::callback::convert(py, #call) - } - } - }; + let body = quote! { + #pyo3_path::impl_::callback::convert(py, #call) + }; - let wrapper_ident = match property_type { - PropertyType::Descriptor { - field: syn::Field { - ident: Some(ident), .. - }, - .. - } => { - format_ident!("__pymethod_get_{}__", ident) - } - PropertyType::Descriptor { field_index, .. } => { - format_ident!("__pymethod_get_field_{}__", field_index) - } - PropertyType::Function { spec, .. } => { - format_ident!("__pymethod_get_{}__", spec.name) - } - }; + let init_holders = holders.init_holders(ctx); + let associated_method = quote! { + #cfg_attrs + unsafe fn #wrapper_ident( + py: #pyo3_path::Python<'_>, + _slf: *mut #pyo3_path::ffi::PyObject + ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { + #init_holders + let result = #body; + result + } + }; - let mut cfg_attrs = TokenStream::new(); - if let PropertyType::Descriptor { field, .. } = &property_type { - for attr in field - .attrs - .iter() - .filter(|attr| attr.path().is_ident("cfg")) - { - attr.to_tokens(&mut cfg_attrs); - } - } + let method_def = quote! { + #cfg_attrs + #pyo3_path::impl_::pyclass::MaybeRuntimePyMethodDef::Static( + #pyo3_path::impl_::pymethods::PyMethodDefType::Getter( + #pyo3_path::impl_::pymethods::PyGetterDef::new( + #python_name, + #cls::#wrapper_ident, + #doc + ) + ) + ) + }; - let init_holders = holders.init_holders(ctx); - let check_gil_refs = holders.check_gil_refs(); - let associated_method = quote! { - #cfg_attrs - unsafe fn #wrapper_ident( - py: #pyo3_path::Python<'_>, - _slf: *mut #pyo3_path::ffi::PyObject - ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { - #init_holders - let result = #body; - #check_gil_refs - result + Ok(MethodAndMethodDef { + associated_method, + method_def, + }) } - }; - - let method_def = quote! { - #cfg_attrs - #pyo3_path::class::PyMethodDefType::Getter( - #pyo3_path::class::PyGetterDef::new( - #python_name, - #pyo3_path::impl_::pymethods::PyGetter(#cls::#wrapper_ident), - #doc - ) - ) - }; - - Ok(MethodAndMethodDef { - associated_method, - method_def, - }) + } } /// Split an argument of pyo3::Python from the front of the arg list, if present -fn split_off_python_arg<'a>(args: &'a [FnArg<'a>]) -> (Option<&FnArg<'_>>, &[FnArg<'_>]) { +fn split_off_python_arg<'a, 'b>(args: &'a [FnArg<'b>]) -> (Option<&'a PyArg<'b>>, &'a [FnArg<'b>]) { match args { - [py, args @ ..] if utils::is_python(py.ty) => (Some(py), args), + [FnArg::Py(py), args @ ..] => (Some(py), args), args => (None, args), } } @@ -849,7 +926,7 @@ pub enum PropertyType<'a> { } impl PropertyType<'_> { - fn null_terminated_python_name(&self) -> Result { + fn null_terminated_python_name(&self, ctx: &Ctx) -> Result { match self { PropertyType::Descriptor { field, @@ -864,35 +941,35 @@ impl PropertyType<'_> { if let Some(rule) = renaming_rule { name = utils::apply_renaming_rule(*rule, &name); } - name.push('\0'); name } (None, None) => { bail_spanned!(field.span() => "`get` and `set` with tuple struct fields require `name`"); } }; - Ok(syn::LitStr::new(&name, field.span())) + let name = CString::new(name).unwrap(); + Ok(LitCStr::new(name, field.span(), ctx)) } - PropertyType::Function { spec, .. } => Ok(spec.null_terminated_python_name()), + PropertyType::Function { spec, .. } => Ok(spec.null_terminated_python_name(ctx)), } } - fn doc(&self) -> Cow<'_, PythonDoc> { + fn doc(&self, ctx: &Ctx) -> Cow<'_, PythonDoc> { match self { PropertyType::Descriptor { field, .. } => { - Cow::Owned(utils::get_doc(&field.attrs, None)) + Cow::Owned(utils::get_doc(&field.attrs, None, ctx)) } PropertyType::Function { doc, .. } => Cow::Borrowed(doc), } } } -const __STR__: SlotDef = SlotDef::new("Py_tp_str", "reprfunc"); +pub const __STR__: SlotDef = SlotDef::new("Py_tp_str", "reprfunc"); pub const __REPR__: SlotDef = SlotDef::new("Py_tp_repr", "reprfunc"); -const __HASH__: SlotDef = SlotDef::new("Py_tp_hash", "hashfunc") +pub const __HASH__: SlotDef = SlotDef::new("Py_tp_hash", "hashfunc") .ret_ty(Ty::PyHashT) .return_conversion(TokenGenerator( - |Ctx { pyo3_path }: &Ctx| quote! { #pyo3_path::callback::HashCallbackOutput }, + |Ctx { pyo3_path, .. }: &Ctx| quote! { #pyo3_path::impl_::callback::HashCallbackOutput }, )); pub const __RICHCMP__: SlotDef = SlotDef::new("Py_tp_richcompare", "richcmpfunc") .extract_error_mode(ExtractErrorMode::NotImplemented) @@ -913,7 +990,7 @@ const __ANEXT__: SlotDef = SlotDef::new("Py_am_anext", "unaryfunc").return_speci ), TokenGenerator(|_| quote! { async_iter_tag }), ); -const __LEN__: SlotDef = SlotDef::new("Py_mp_length", "lenfunc").ret_ty(Ty::PySsizeT); +pub const __LEN__: SlotDef = SlotDef::new("Py_mp_length", "lenfunc").ret_ty(Ty::PySsizeT); const __CONTAINS__: SlotDef = SlotDef::new("Py_sq_contains", "objobjproc") .arguments(&[Ty::Object]) .ret_ty(Ty::Int); @@ -923,7 +1000,8 @@ const __INPLACE_CONCAT__: SlotDef = SlotDef::new("Py_sq_concat", "binaryfunc").arguments(&[Ty::Object]); const __INPLACE_REPEAT__: SlotDef = SlotDef::new("Py_sq_repeat", "ssizeargfunc").arguments(&[Ty::PySsizeT]); -const __GETITEM__: SlotDef = SlotDef::new("Py_mp_subscript", "binaryfunc").arguments(&[Ty::Object]); +pub const __GETITEM__: SlotDef = + SlotDef::new("Py_mp_subscript", "binaryfunc").arguments(&[Ty::Object]); const __POS__: SlotDef = SlotDef::new("Py_nb_positive", "unaryfunc"); const __NEG__: SlotDef = SlotDef::new("Py_nb_negative", "unaryfunc"); @@ -1014,7 +1092,11 @@ enum Ty { impl Ty { fn ffi_type(self, ctx: &Ctx) -> TokenStream { - let Ctx { pyo3_path } = ctx; + let Ctx { + pyo3_path, + output_span, + } = ctx; + let pyo3_path = pyo3_path.to_tokens_spanned(*output_span); match self { Ty::Object | Ty::MaybeNullObject => quote! { *mut #pyo3_path::ffi::PyObject }, Ty::NonNullObject => quote! { ::std::ptr::NonNull<#pyo3_path::ffi::PyObject> }, @@ -1035,21 +1117,19 @@ impl Ty { holders: &mut Holders, ctx: &Ctx, ) -> TokenStream { - let Ctx { pyo3_path } = ctx; - let name_str = arg.name.unraw().to_string(); + let Ctx { pyo3_path, .. } = ctx; match self { Ty::Object => extract_object( extract_error_mode, holders, - &name_str, + arg, quote! { #ident }, - arg.ty.span(), ctx ), Ty::MaybeNullObject => extract_object( extract_error_mode, holders, - &name_str, + arg, quote! { if #ident.is_null() { #pyo3_path::ffi::Py_None() @@ -1057,23 +1137,20 @@ impl Ty { #ident } }, - arg.ty.span(), ctx ), Ty::NonNullObject => extract_object( extract_error_mode, holders, - &name_str, + arg, quote! { #ident.as_ptr() }, - arg.ty.span(), ctx ), Ty::IPowModulo => extract_object( extract_error_mode, holders, - &name_str, + arg, quote! { #ident.as_ptr() }, - arg.ty.span(), ctx ), Ty::CompareOp => extract_error_mode.handle_error( @@ -1084,7 +1161,7 @@ impl Ty { ctx ), Ty::PySsizeT => { - let ty = arg.ty; + let ty = arg.ty(); extract_error_mode.handle_error( quote! { ::std::convert::TryInto::<#ty>::try_into(#ident).map_err(|e| #pyo3_path::exceptions::PyValueError::new_err(e.to_string())) @@ -1101,27 +1178,47 @@ impl Ty { fn extract_object( extract_error_mode: ExtractErrorMode, holders: &mut Holders, - name: &str, + arg: &FnArg<'_>, source_ptr: TokenStream, - span: Span, ctx: &Ctx, ) -> TokenStream { - let Ctx { pyo3_path } = ctx; - let holder = holders.push_holder(Span::call_site()); - let gil_refs_checker = holders.push_gil_refs_checker(span); - let extracted = extract_error_mode.handle_error( + let Ctx { pyo3_path, .. } = ctx; + let name = arg.name().unraw().to_string(); + + let extract = if let Some(FromPyWithAttribute { + kw, + value: extractor, + }) = arg.from_py_with() + { + let extractor = quote_spanned! { kw.span => + { let from_py_with: fn(_) -> _ = #extractor; from_py_with } + }; + quote! { - #pyo3_path::impl_::extract_argument::extract_argument( - #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &#source_ptr).0, + #pyo3_path::impl_::extract_argument::from_py_with( + unsafe { #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &#source_ptr).0 }, + #name, + #extractor, + ) + } + } else { + let holder = holders.push_holder(Span::call_site()); + let ty = arg.ty().clone().elide_lifetimes(); + quote! {{ + use #pyo3_path::impl_::pyclass::Probe as _; + #pyo3_path::impl_::extract_argument::extract_argument::< + _, + { #pyo3_path::impl_::pyclass::IsOption::<#ty>::VALUE } + >( + unsafe { #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &#source_ptr).0 }, &mut #holder, #name ) - }, - ctx, - ); - quote! { - #pyo3_path::impl_::deprecations::inspect_type(#extracted, &#gil_refs_checker) - } + }} + }; + + let extracted = extract_error_mode.handle_error(extract, ctx); + quote!(#extracted) } enum ReturnMode { @@ -1131,16 +1228,14 @@ enum ReturnMode { } impl ReturnMode { - fn return_call_output(&self, call: TokenStream, ctx: &Ctx, holders: &Holders) -> TokenStream { - let Ctx { pyo3_path } = ctx; - let check_gil_refs = holders.check_gil_refs(); + fn return_call_output(&self, call: TokenStream, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; match self { ReturnMode::Conversion(conversion) => { let conversion = TokenGeneratorCtx(*conversion, ctx); quote! { - let _result: #pyo3_path::PyResult<#conversion> = #pyo3_path::callback::convert(py, #call); - #check_gil_refs - #pyo3_path::callback::convert(py, _result) + let _result: #pyo3_path::PyResult<#conversion> = #pyo3_path::impl_::callback::convert(py, #call); + #pyo3_path::impl_::callback::convert(py, _result) } } ReturnMode::SpecializedConversion(traits, tag) => { @@ -1149,14 +1244,12 @@ impl ReturnMode { quote! { let _result = #call; use #pyo3_path::impl_::pymethods::{#traits}; - #check_gil_refs (&_result).#tag().convert(py, _result) } } ReturnMode::ReturnSelf => quote! { - let _result: #pyo3_path::PyResult<()> = #pyo3_path::callback::convert(py, #call); + let _result: #pyo3_path::PyResult<()> = #pyo3_path::impl_::callback::convert(py, #call); _result?; - #check_gil_refs #pyo3_path::ffi::Py_XINCREF(_raw_slf); ::std::result::Result::Ok(_raw_slf) }, @@ -1235,7 +1328,7 @@ impl SlotDef { method_name: &str, ctx: &Ctx, ) -> Result { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let SlotDef { slot, func_ty, @@ -1270,6 +1363,7 @@ impl SlotDef { let name = spec.name; let holders = holders.init_holders(ctx); let associated_method = quote! { + #[allow(non_snake_case)] unsafe fn #wrapper_ident( py: #pyo3_path::Python<'_>, _raw_slf: *mut #pyo3_path::ffi::PyObject, @@ -1315,7 +1409,7 @@ fn generate_method_body( return_mode: Option<&ReturnMode>, ctx: &Ctx, ) -> Result { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let self_arg = spec .tp .self_arg(Some(cls), extract_error_mode, holders, ctx); @@ -1323,13 +1417,11 @@ fn generate_method_body( let args = extract_proto_arguments(spec, arguments, extract_error_mode, holders, ctx)?; let call = quote! { #cls::#rust_name(#self_arg #(#args),*) }; Ok(if let Some(return_mode) = return_mode { - return_mode.return_call_output(call, ctx, holders) + return_mode.return_call_output(call, ctx) } else { - let check_gil_refs = holders.check_gil_refs(); quote! { let result = #call; - #check_gil_refs; - #pyo3_path::callback::convert(py, result) + #pyo3_path::impl_::callback::convert(py, result) } }) } @@ -1367,7 +1459,7 @@ impl SlotFragmentDef { spec: &FnSpec<'_>, ctx: &Ctx, ) -> Result { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; let SlotFragmentDef { fragment, arguments, @@ -1395,6 +1487,7 @@ impl SlotFragmentDef { let holders = holders.init_holders(ctx); Ok(quote! { impl #cls { + #[allow(non_snake_case)] unsafe fn #wrapper_ident( py: #pyo3_path::Python, _raw_slf: *mut #pyo3_path::ffi::PyObject, @@ -1507,12 +1600,12 @@ fn extract_proto_arguments( let mut non_python_args = 0; for arg in &spec.signature.arguments { - if arg.py { + if let FnArg::Py(..) = arg { args.push(quote! { py }); } else { let ident = syn::Ident::new(&format!("arg{}", non_python_args), Span::call_site()); let conversions = proto_args.get(non_python_args) - .ok_or_else(|| err_spanned!(arg.ty.span() => format!("Expected at most {} non-python arguments", proto_args.len())))? + .ok_or_else(|| err_spanned!(arg.ty().span() => format!("Expected at most {} non-python arguments", proto_args.len())))? .extract(&ident, arg, extract_error_mode, holders, ctx); non_python_args += 1; args.push(conversions); diff --git a/pyo3-macros-backend/src/pyversions.rs b/pyo3-macros-backend/src/pyversions.rs new file mode 100644 index 00000000000..4c0998667d8 --- /dev/null +++ b/pyo3-macros-backend/src/pyversions.rs @@ -0,0 +1,6 @@ +use pyo3_build_config::PythonVersion; + +pub fn is_abi3_before(major: u8, minor: u8) -> bool { + let config = pyo3_build_config::get(); + config.abi3 && config.version < PythonVersion { major, minor } +} diff --git a/pyo3-macros-backend/src/quotes.rs b/pyo3-macros-backend/src/quotes.rs index ceef23fb034..d961b4c0acd 100644 --- a/pyo3-macros-backend/src/quotes.rs +++ b/pyo3-macros-backend/src/quotes.rs @@ -1,23 +1,36 @@ use crate::utils::Ctx; use proc_macro2::TokenStream; -use quote::quote; +use quote::{quote, quote_spanned}; pub(crate) fn some_wrap(obj: TokenStream, ctx: &Ctx) -> TokenStream { - let Ctx { pyo3_path } = ctx; + let Ctx { pyo3_path, .. } = ctx; quote! { #pyo3_path::impl_::wrap::SomeWrap::wrap(#obj) } } pub(crate) fn ok_wrap(obj: TokenStream, ctx: &Ctx) -> TokenStream { - let Ctx { pyo3_path } = ctx; - quote! { - #pyo3_path::impl_::wrap::OkWrap::wrap(#obj) - .map_err(::core::convert::Into::<#pyo3_path::PyErr>::into) - } + let Ctx { + pyo3_path, + output_span, + } = ctx; + let pyo3_path = pyo3_path.to_tokens_spanned(*output_span); + quote_spanned! { *output_span => { + let obj = #obj; + #[allow(clippy::useless_conversion)] + #pyo3_path::impl_::wrap::converter(&obj).wrap(obj).map_err(::core::convert::Into::<#pyo3_path::PyErr>::into) + }} } pub(crate) fn map_result_into_ptr(result: TokenStream, ctx: &Ctx) -> TokenStream { - let Ctx { pyo3_path } = ctx; - quote! { #pyo3_path::impl_::wrap::map_result_into_ptr(py, #result) } + let Ctx { + pyo3_path, + output_span, + } = ctx; + let pyo3_path = pyo3_path.to_tokens_spanned(*output_span); + let py = syn::Ident::new("py", proc_macro2::Span::call_site()); + quote_spanned! { *output_span => { + let result = #result; + #pyo3_path::impl_::wrap::converter(&result).map_into_ptr(#py, result) + }} } diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index ca32abb42b3..bdec23388df 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -1,9 +1,10 @@ +use crate::attributes::{CrateAttribute, ExprPathWrap, RenamingRule}; use proc_macro2::{Span, TokenStream}; -use quote::ToTokens; +use quote::{quote, quote_spanned, ToTokens}; +use std::ffi::CString; +use syn::spanned::Spanned; use syn::{punctuated::Punctuated, Token}; -use crate::attributes::{CrateAttribute, RenamingRule}; - /// Macro inspired by `anyhow::anyhow!` to create a compiler error with the given span. macro_rules! err_spanned { ($span:expr => $msg:expr) => { @@ -25,7 +26,20 @@ macro_rules! ensure_spanned { if !($condition) { bail_spanned!($span => $msg); } - } + }; + ($($condition:expr, $span:expr => $msg:expr;)*) => { + if let Some(e) = [$( + (!($condition)).then(|| err_spanned!($span => $msg)), + )*] + .into_iter() + .flatten() + .reduce(|mut acc, e| { + acc.combine(e); + acc + }) { + return Err(e); + } + }; } /// Check if the given type `ty` is `pyo3::Python`. @@ -54,21 +68,71 @@ pub fn option_type_argument(ty: &syn::Type) -> Option<&syn::Type> { None } +// TODO: Replace usage of this by [`syn::LitCStr`] when on MSRV 1.77 +#[derive(Clone)] +pub struct LitCStr { + lit: CString, + span: Span, + pyo3_path: PyO3CratePath, +} + +impl LitCStr { + pub fn new(lit: CString, span: Span, ctx: &Ctx) -> Self { + Self { + lit, + span, + pyo3_path: ctx.pyo3_path.clone(), + } + } + + pub fn empty(ctx: &Ctx) -> Self { + Self { + lit: CString::new("").unwrap(), + span: Span::call_site(), + pyo3_path: ctx.pyo3_path.clone(), + } + } +} + +impl quote::ToTokens for LitCStr { + fn to_tokens(&self, tokens: &mut TokenStream) { + if cfg!(c_str_lit) { + syn::LitCStr::new(&self.lit, self.span).to_tokens(tokens); + } else { + let pyo3_path = &self.pyo3_path; + let lit = self.lit.to_str().unwrap(); + tokens.extend(quote::quote_spanned!(self.span => #pyo3_path::ffi::c_str!(#lit))); + } + } +} + /// A syntax tree which evaluates to a nul-terminated docstring for Python. /// /// Typically the tokens will just be that string, but if the original docs included macro /// expressions then the tokens will be a concat!("...", "\n", "\0") expression of the strings and -/// macro parts. -/// contents such as parse the string contents. +/// macro parts. contents such as parse the string contents. +#[derive(Clone)] +pub struct PythonDoc(PythonDocKind); + #[derive(Clone)] -pub struct PythonDoc(TokenStream); +enum PythonDocKind { + LitCStr(LitCStr), + // There is currently no way to `concat!` c-string literals, we fallback to the `c_str!` macro in + // this case. + Tokens(TokenStream), +} /// Collects all #[doc = "..."] attributes into a TokenStream evaluating to a null-terminated string. /// /// If this doc is for a callable, the provided `text_signature` can be passed to prepend /// this to the documentation suitable for Python to extract this into the `__text_signature__` /// attribute. -pub fn get_doc(attrs: &[syn::Attribute], mut text_signature: Option) -> PythonDoc { +pub fn get_doc( + attrs: &[syn::Attribute], + mut text_signature: Option, + ctx: &Ctx, +) -> PythonDoc { + let Ctx { pyo3_path, .. } = ctx; // insert special divider between `__text_signature__` and doc // (assume text_signature is itself well-formed) if let Some(text_signature) = &mut text_signature { @@ -120,20 +184,28 @@ pub fn get_doc(attrs: &[syn::Attribute], mut text_signature: Option) -> syn::token::Bracket(Span::call_site()).surround(&mut tokens, |tokens| { parts.to_tokens(tokens); syn::token::Comma(Span::call_site()).to_tokens(tokens); - syn::LitStr::new("\0", Span::call_site()).to_tokens(tokens); }); - PythonDoc(tokens) + PythonDoc(PythonDocKind::Tokens( + quote!(#pyo3_path::ffi::c_str!(#tokens)), + )) } else { // Just a string doc - return directly with nul terminator - current_part.push('\0'); - PythonDoc(current_part.to_token_stream()) + let docs = CString::new(current_part).unwrap(); + PythonDoc(PythonDocKind::LitCStr(LitCStr::new( + docs, + Span::call_site(), + ctx, + ))) } } impl quote::ToTokens for PythonDoc { fn to_tokens(&self, tokens: &mut TokenStream) { - self.0.to_tokens(tokens) + match &self.0 { + PythonDocKind::LitCStr(lit) => lit.to_tokens(tokens), + PythonDocKind::Tokens(toks) => toks.to_tokens(tokens), + } } } @@ -145,20 +217,39 @@ pub fn unwrap_ty_group(mut ty: &syn::Type) -> &syn::Type { } pub struct Ctx { + /// Where we can find the pyo3 crate pub pyo3_path: PyO3CratePath, + + /// If we are in a pymethod or pyfunction, + /// this will be the span of the return type + pub output_span: Span, } impl Ctx { - pub(crate) fn new(attr: &Option) -> Self { + pub(crate) fn new(attr: &Option, signature: Option<&syn::Signature>) -> Self { let pyo3_path = match attr { Some(attr) => PyO3CratePath::Given(attr.value.0.clone()), None => PyO3CratePath::Default, }; - Self { pyo3_path } + let output_span = if let Some(syn::Signature { + output: syn::ReturnType::Type(_, output_type), + .. + }) = &signature + { + output_type.span() + } else { + Span::call_site() + }; + + Self { + pyo3_path, + output_span, + } } } +#[derive(Clone)] pub enum PyO3CratePath { Given(syn::Path), Default, @@ -197,6 +288,122 @@ pub fn apply_renaming_rule(rule: RenamingRule, name: &str) -> String { } } -pub(crate) fn is_abi3() -> bool { - pyo3_build_config::get().abi3 +pub(crate) enum IdentOrStr<'a> { + Str(&'a str), + Ident(syn::Ident), +} + +pub(crate) fn has_attribute(attrs: &[syn::Attribute], ident: &str) -> bool { + has_attribute_with_namespace(attrs, None, &[ident]) +} + +pub(crate) fn has_attribute_with_namespace( + attrs: &[syn::Attribute], + crate_path: Option<&PyO3CratePath>, + idents: &[&str], +) -> bool { + let mut segments = vec![]; + if let Some(c) = crate_path { + match c { + PyO3CratePath::Given(paths) => { + for p in &paths.segments { + segments.push(IdentOrStr::Ident(p.ident.clone())); + } + } + PyO3CratePath::Default => segments.push(IdentOrStr::Str("pyo3")), + } + }; + for i in idents { + segments.push(IdentOrStr::Str(i)); + } + + attrs.iter().any(|attr| { + segments + .iter() + .eq(attr.path().segments.iter().map(|v| &v.ident)) + }) +} + +pub(crate) fn deprecated_from_py_with(expr_path: &ExprPathWrap) -> Option { + let path = quote!(#expr_path).to_string(); + let msg = + format!("remove the quotes from the literal\n= help: use `{path}` instead of `\"{path}\"`"); + expr_path.from_lit_str.then(|| { + quote_spanned! { expr_path.span() => + #[deprecated(since = "0.24.0", note = #msg)] + #[allow(dead_code)] + const LIT_STR_DEPRECATION: () = (); + let _: () = LIT_STR_DEPRECATION; + } + }) +} + +pub(crate) trait TypeExt { + /// Replaces all explicit lifetimes in `self` with elided (`'_`) lifetimes + /// + /// This is useful if `Self` is used in `const` context, where explicit + /// lifetimes are not allowed (yet). + fn elide_lifetimes(self) -> Self; +} + +impl TypeExt for syn::Type { + fn elide_lifetimes(mut self) -> Self { + fn elide_lifetimes(ty: &mut syn::Type) { + match ty { + syn::Type::Path(type_path) => { + if let Some(qself) = &mut type_path.qself { + elide_lifetimes(&mut qself.ty) + } + for seg in &mut type_path.path.segments { + if let syn::PathArguments::AngleBracketed(args) = &mut seg.arguments { + for generic_arg in &mut args.args { + match generic_arg { + syn::GenericArgument::Lifetime(lt) => { + *lt = syn::Lifetime::new("'_", lt.span()); + } + syn::GenericArgument::Type(ty) => elide_lifetimes(ty), + syn::GenericArgument::AssocType(assoc) => { + elide_lifetimes(&mut assoc.ty) + } + + syn::GenericArgument::Const(_) + | syn::GenericArgument::AssocConst(_) + | syn::GenericArgument::Constraint(_) + | _ => {} + } + } + } + } + } + syn::Type::Reference(type_ref) => { + if let Some(lt) = type_ref.lifetime.as_mut() { + *lt = syn::Lifetime::new("'_", lt.span()); + } + elide_lifetimes(&mut type_ref.elem); + } + syn::Type::Tuple(type_tuple) => { + for ty in &mut type_tuple.elems { + elide_lifetimes(ty); + } + } + syn::Type::Array(type_array) => elide_lifetimes(&mut type_array.elem), + syn::Type::Slice(ty) => elide_lifetimes(&mut ty.elem), + syn::Type::Group(ty) => elide_lifetimes(&mut ty.elem), + syn::Type::Paren(ty) => elide_lifetimes(&mut ty.elem), + syn::Type::Ptr(ty) => elide_lifetimes(&mut ty.elem), + + syn::Type::BareFn(_) + | syn::Type::ImplTrait(_) + | syn::Type::Infer(_) + | syn::Type::Macro(_) + | syn::Type::Never(_) + | syn::Type::TraitObject(_) + | syn::Type::Verbatim(_) + | _ => {} + } + } + + elide_lifetimes(&mut self); + self + } } diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 39a4b9198c6..c089ce53215 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.21.1" +version = "0.24.1" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -9,6 +9,7 @@ repository = "https://github.com/pyo3/pyo3" categories = ["api-bindings", "development-tools::ffi"] license = "MIT OR Apache-2.0" edition = "2021" +rust-version = "1.63" [lib] proc-macro = true @@ -16,13 +17,12 @@ proc-macro = true [features] multiple-pymethods = [] experimental-async = ["pyo3-macros-backend/experimental-async"] -experimental-declarative-modules = [] [dependencies] -proc-macro2 = { version = "1", default-features = false } +proc-macro2 = { version = "1.0.60", default-features = false } quote = "1" syn = { version = "2", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.21.1" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.24.1" } [lints] workspace = true diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index 64756a1c73b..2621bea4c6e 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -5,12 +5,12 @@ use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use pyo3_macros_backend::{ - build_derive_from_pyobject, build_py_class, build_py_enum, build_py_function, build_py_methods, - pymodule_function_impl, pymodule_module_impl, PyClassArgs, PyClassMethodsType, - PyFunctionOptions, + build_derive_from_pyobject, build_derive_into_pyobject, build_py_class, build_py_enum, + build_py_function, build_py_methods, pymodule_function_impl, pymodule_module_impl, PyClassArgs, + PyClassMethodsType, PyFunctionOptions, PyModuleOptions, }; use quote::quote; -use syn::{parse::Nothing, parse_macro_input, Item}; +use syn::{parse_macro_input, Item}; /// A proc macro used to implement Python modules. /// @@ -24,6 +24,9 @@ use syn::{parse::Nothing, parse_macro_input, Item}; /// | Annotation | Description | /// | :- | :- | /// | `#[pyo3(name = "...")]` | Defines the name of the module in Python. | +/// | `#[pyo3(submodule)]` | Skips adding a `PyInit_` FFI symbol to the compiled binary. | +/// | `#[pyo3(module = "...")]` | Defines the Python `dotted.path` to the parent module for use in introspection. | +/// | `#[pyo3(crate = "pyo3")]` | Defines the path to PyO3 to use code generated by the macro. | /// /// For more on creating Python modules see the [module section of the guide][1]. /// @@ -32,26 +35,32 @@ use syn::{parse::Nothing, parse_macro_input, Item}; /// `#[pymodule]` implementation generates a hidden module with the same name containing /// metadata about the module, which is used by `wrap_pymodule!`). /// -/// [1]: https://pyo3.rs/latest/module.html +#[doc = concat!("[1]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/module.html")] #[proc_macro_attribute] pub fn pymodule(args: TokenStream, input: TokenStream) -> TokenStream { - parse_macro_input!(args as Nothing); - match parse_macro_input!(input as Item) { - Item::Mod(module) => if cfg!(feature = "experimental-declarative-modules") { - pymodule_module_impl(module) - } else { - Err(syn::Error::new_spanned( - module, - "#[pymodule] requires the 'experimental-declarative-modules' feature to be used on Rust modules.", - )) - }, - Item::Fn(function) => pymodule_function_impl(function), + let options = parse_macro_input!(args as PyModuleOptions); + + let mut ast = parse_macro_input!(input as Item); + let expanded = match &mut ast { + Item::Mod(module) => { + match pymodule_module_impl(module, options) { + // #[pymodule] on a module will rebuild the original ast, so we don't emit it here + Ok(expanded) => return expanded.into(), + Err(e) => Err(e), + } + } + Item::Fn(function) => pymodule_function_impl(function, options), unsupported => Err(syn::Error::new_spanned( unsupported, "#[pymodule] only supports modules and functions.", )), } - .unwrap_or_compile_error() + .unwrap_or_compile_error(); + + quote!( + #ast + #expanded + ) .into() } @@ -90,17 +99,17 @@ pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream { /// multiple `#[pymethods]` blocks for a single `#[pyclass]`. /// This will add a transitive dependency on the [`inventory`][3] crate. /// -/// [1]: https://pyo3.rs/latest/class.html#instance-methods -/// [2]: https://pyo3.rs/latest/features.html#multiple-pymethods +#[doc = concat!("[1]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#instance-methods")] +#[doc = concat!("[2]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/features.html#multiple-pymethods")] /// [3]: https://docs.rs/inventory/ -/// [4]: https://pyo3.rs/latest/class.html#constructor -/// [5]: https://pyo3.rs/latest/class.html#object-properties-using-getter-and-setter -/// [6]: https://pyo3.rs/latest/class.html#static-methods -/// [7]: https://pyo3.rs/latest/class.html#class-methods -/// [8]: https://pyo3.rs/latest/class.html#callable-objects -/// [9]: https://pyo3.rs/latest/class.html#class-attributes -/// [10]: https://pyo3.rs/latest/class.html#method-arguments -/// [11]: https://pyo3.rs/latest/class.html#object-properties-using-pyo3get-set +#[doc = concat!("[4]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#constructor")] +#[doc = concat!("[5]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#object-properties-using-getter-and-setter")] +#[doc = concat!("[6]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#static-methods")] +#[doc = concat!("[7]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#class-methods")] +#[doc = concat!("[8]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#callable-objects")] +#[doc = concat!("[9]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#class-attributes")] +#[doc = concat!("[10]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#method-arguments")] +#[doc = concat!("[11]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#object-properties-using-pyo3get-set")] #[proc_macro_attribute] pub fn pymethods(attr: TokenStream, input: TokenStream) -> TokenStream { let methods_type = if cfg!(feature = "multiple-pymethods") { @@ -129,7 +138,7 @@ pub fn pymethods(attr: TokenStream, input: TokenStream) -> TokenStream { /// `#[pyfunction]` implementation generates a hidden module with the same name containing /// metadata about the function, which is used by `wrap_pyfunction!`). /// -/// [1]: https://pyo3.rs/latest/function.html +#[doc = concat!("[1]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/function.html")] #[proc_macro_attribute] pub fn pyfunction(attr: TokenStream, input: TokenStream) -> TokenStream { let mut ast = parse_macro_input!(input as syn::ItemFn); @@ -144,6 +153,27 @@ pub fn pyfunction(attr: TokenStream, input: TokenStream) -> TokenStream { .into() } +#[proc_macro_derive(IntoPyObject, attributes(pyo3))] +pub fn derive_into_py_object(item: TokenStream) -> TokenStream { + let ast = parse_macro_input!(item as syn::DeriveInput); + let expanded = build_derive_into_pyobject::(&ast).unwrap_or_compile_error(); + quote!( + #expanded + ) + .into() +} + +#[proc_macro_derive(IntoPyObjectRef, attributes(pyo3))] +pub fn derive_into_py_object_ref(item: TokenStream) -> TokenStream { + let ast = parse_macro_input!(item as syn::DeriveInput); + let expanded = + pyo3_macros_backend::build_derive_into_pyobject::(&ast).unwrap_or_compile_error(); + quote!( + #expanded + ) + .into() +} + #[proc_macro_derive(FromPyObject, attributes(pyo3))] pub fn derive_from_py_object(item: TokenStream) -> TokenStream { let ast = parse_macro_input!(item as syn::DeriveInput); @@ -159,7 +189,7 @@ fn pyclass_impl( mut ast: syn::ItemStruct, methods_type: PyClassMethodsType, ) -> TokenStream { - let args = parse_macro_input!(attrs with PyClassArgs::parse_stuct_args); + let args = parse_macro_input!(attrs with PyClassArgs::parse_struct_args); let expanded = build_py_class(&mut ast, args, methods_type).unwrap_or_compile_error(); quote!( diff --git a/pyproject.toml b/pyproject.toml index d474753ccd1..84f250e7863 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,9 @@ -[tool.ruff.extend-per-file-ignores] +[tool.ruff.lint.extend-per-file-ignores] "__init__.py" = ["F403"] [tool.towncrier] filename = "CHANGELOG.md" -version = "0.21.1" +version = "0.24.1" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}" diff --git a/pytests/Cargo.toml b/pytests/Cargo.toml index 255094a6c40..1fee3093275 100644 --- a/pytests/Cargo.toml +++ b/pytests/Cargo.toml @@ -5,6 +5,7 @@ version = "0.1.0" description = "Python-based tests for PyO3" edition = "2021" publish = false +rust-version = "1.63" [dependencies] pyo3 = { path = "../", features = ["extension-module"] } diff --git a/pytests/conftest.py b/pytests/conftest.py new file mode 100644 index 00000000000..ce729689355 --- /dev/null +++ b/pytests/conftest.py @@ -0,0 +1,22 @@ +import sysconfig +import sys +import pytest + +FREE_THREADED_BUILD = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + +gil_enabled_at_start = True +if FREE_THREADED_BUILD: + gil_enabled_at_start = sys._is_gil_enabled() + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + if FREE_THREADED_BUILD and not gil_enabled_at_start and sys._is_gil_enabled(): + tr = terminalreporter + tr.ensure_newline() + tr.section("GIL re-enabled", sep="=", red=True, bold=True) + tr.line("The GIL was re-enabled at runtime during the tests.") + tr.line("") + tr.line("Please ensure all new modules declare support for running") + tr.line("without the GIL. Any new tests that intentionally imports ") + tr.line("code that re-enables the GIL should do so in a subprocess.") + pytest.exit("GIL re-enabled during tests", returncode=1) diff --git a/pytests/noxfile.py b/pytests/noxfile.py index 7c681ab1aa8..f5a332c7a3c 100644 --- a/pytests/noxfile.py +++ b/pytests/noxfile.py @@ -9,11 +9,19 @@ def test(session: nox.Session): session.env["MATURIN_PEP517_ARGS"] = "--profile=dev" session.run_always("python", "-m", "pip", "install", "-v", ".[dev]") - try: - session.install("--only-binary=numpy", "numpy>=1.16") - except CommandFailed: - # No binary wheel for numpy available on this platform - pass + + def try_install_binary(package: str, constraint: str): + try: + session.install("--only-binary=:all:", f"{package}{constraint}") + except CommandFailed: + # No binary wheel available on this platform + pass + + try_install_binary("numpy", ">=1.16") + # https://github.com/zopefoundation/zope.interface/issues/316 + # - is a dependency of gevent + try_install_binary("zope.interface", "<7") + try_install_binary("gevent", ">=22.10.2") ignored_paths = [] if sys.version_info < (3, 10): # Match syntax is only available in Python >= 3.10 diff --git a/pytests/pyproject.toml b/pytests/pyproject.toml index aace57dd4d4..5f78a573124 100644 --- a/pytests/pyproject.toml +++ b/pytests/pyproject.toml @@ -20,7 +20,6 @@ classifiers = [ [project.optional-dependencies] dev = [ - "gevent>=22.10.2; implementation_name == 'cpython'", "hypothesis>=3.55", "pytest-asyncio>=0.21", "pytest-benchmark>=3.4", diff --git a/pytests/src/awaitable.rs b/pytests/src/awaitable.rs index 5e3b98e14ea..01a93c70a0d 100644 --- a/pytests/src/awaitable.rs +++ b/pytests/src/awaitable.rs @@ -2,7 +2,7 @@ //! awaitable protocol. //! //! Both IterAwaitable and FutureAwaitable will return a value immediately -//! when awaited, see guide examples related to pyo3-asyncio for ways +//! when awaited, see guide examples related to pyo3-async-runtimes for ways //! to suspend tasks and await results. use pyo3::exceptions::PyStopIteration; @@ -78,7 +78,7 @@ impl FutureAwaitable { } } -#[pymodule] +#[pymodule(gil_used = false)] pub fn awaitable(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; diff --git a/pytests/src/buf_and_str.rs b/pytests/src/buf_and_str.rs index 879d76af883..15230a5e153 100644 --- a/pytests/src/buf_and_str.rs +++ b/pytests/src/buf_and_str.rs @@ -36,18 +36,18 @@ impl BytesExtractor { #[staticmethod] pub fn from_buffer(buf: &Bound<'_, PyAny>) -> PyResult { - let buf = PyBuffer::::get_bound(buf)?; + let buf = PyBuffer::::get(buf)?; Ok(buf.item_count()) } } #[pyfunction] fn return_memoryview(py: Python<'_>) -> PyResult> { - let bytes = PyBytes::new_bound(py, b"hello world"); - PyMemoryView::from_bound(&bytes) + let bytes = PyBytes::new(py, b"hello world"); + PyMemoryView::from(&bytes) } -#[pymodule] +#[pymodule(gil_used = false)] pub fn buf_and_str(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_function(wrap_pyfunction!(return_memoryview, m)?)?; diff --git a/pytests/src/comparisons.rs b/pytests/src/comparisons.rs index fa35acf8e1a..4ed79e42790 100644 --- a/pytests/src/comparisons.rs +++ b/pytests/src/comparisons.rs @@ -34,6 +34,18 @@ impl EqDefaultNe { } } +#[pyclass(eq)] +#[derive(PartialEq, Eq)] +struct EqDerived(i64); + +#[pymethods] +impl EqDerived { + #[new] + fn new(value: i64) -> Self { + Self(value) + } +} + #[pyclass] struct Ordered(i64); @@ -100,10 +112,11 @@ impl OrderedDefaultNe { } } -#[pymodule] +#[pymodule(gil_used = false)] pub fn comparisons(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; Ok(()) diff --git a/pytests/src/datetime.rs b/pytests/src/datetime.rs index d0de99ae406..5162b3508a5 100644 --- a/pytests/src/datetime.rs +++ b/pytests/src/datetime.rs @@ -8,12 +8,12 @@ use pyo3::types::{ #[pyfunction] fn make_date(py: Python<'_>, year: i32, month: u8, day: u8) -> PyResult> { - PyDate::new_bound(py, year, month, day) + PyDate::new(py, year, month, day) } #[pyfunction] -fn get_date_tuple<'py>(d: &Bound<'py, PyDate>) -> Bound<'py, PyTuple> { - PyTuple::new_bound( +fn get_date_tuple<'py>(d: &Bound<'py, PyDate>) -> PyResult> { + PyTuple::new( d.py(), [d.get_year(), d.get_month() as i32, d.get_day() as i32], ) @@ -21,10 +21,11 @@ fn get_date_tuple<'py>(d: &Bound<'py, PyDate>) -> Bound<'py, PyTuple> { #[pyfunction] fn date_from_timestamp(py: Python<'_>, timestamp: i64) -> PyResult> { - PyDate::from_timestamp_bound(py, timestamp) + PyDate::from_timestamp(py, timestamp) } #[pyfunction] +#[pyo3(signature=(hour, minute, second, microsecond, tzinfo=None))] fn make_time<'py>( py: Python<'py>, hour: u8, @@ -33,7 +34,7 @@ fn make_time<'py>( microsecond: u32, tzinfo: Option<&Bound<'py, PyTzInfo>>, ) -> PyResult> { - PyTime::new_bound(py, hour, minute, second, microsecond, tzinfo) + PyTime::new(py, hour, minute, second, microsecond, tzinfo) } #[pyfunction] @@ -47,12 +48,12 @@ fn time_with_fold<'py>( tzinfo: Option<&Bound<'py, PyTzInfo>>, fold: bool, ) -> PyResult> { - PyTime::new_bound_with_fold(py, hour, minute, second, microsecond, tzinfo, fold) + PyTime::new_with_fold(py, hour, minute, second, microsecond, tzinfo, fold) } #[pyfunction] -fn get_time_tuple<'py>(dt: &Bound<'py, PyTime>) -> Bound<'py, PyTuple> { - PyTuple::new_bound( +fn get_time_tuple<'py>(dt: &Bound<'py, PyTime>) -> PyResult> { + PyTuple::new( dt.py(), [ dt.get_hour() as u32, @@ -64,8 +65,8 @@ fn get_time_tuple<'py>(dt: &Bound<'py, PyTime>) -> Bound<'py, PyTuple> { } #[pyfunction] -fn get_time_tuple_fold<'py>(dt: &Bound<'py, PyTime>) -> Bound<'py, PyTuple> { - PyTuple::new_bound( +fn get_time_tuple_fold<'py>(dt: &Bound<'py, PyTime>) -> PyResult> { + PyTuple::new( dt.py(), [ dt.get_hour() as u32, @@ -84,12 +85,12 @@ fn make_delta( seconds: i32, microseconds: i32, ) -> PyResult> { - PyDelta::new_bound(py, days, seconds, microseconds, true) + PyDelta::new(py, days, seconds, microseconds, true) } #[pyfunction] -fn get_delta_tuple<'py>(delta: &Bound<'py, PyDelta>) -> Bound<'py, PyTuple> { - PyTuple::new_bound( +fn get_delta_tuple<'py>(delta: &Bound<'py, PyDelta>) -> PyResult> { + PyTuple::new( delta.py(), [ delta.get_days(), @@ -101,6 +102,7 @@ fn get_delta_tuple<'py>(delta: &Bound<'py, PyDelta>) -> Bound<'py, PyTuple> { #[allow(clippy::too_many_arguments)] #[pyfunction] +#[pyo3(signature=(year, month, day, hour, minute, second, microsecond, tzinfo=None))] fn make_datetime<'py>( py: Python<'py>, year: i32, @@ -112,7 +114,7 @@ fn make_datetime<'py>( microsecond: u32, tzinfo: Option<&Bound<'py, PyTzInfo>>, ) -> PyResult> { - PyDateTime::new_bound( + PyDateTime::new( py, year, month, @@ -126,8 +128,8 @@ fn make_datetime<'py>( } #[pyfunction] -fn get_datetime_tuple<'py>(dt: &Bound<'py, PyDateTime>) -> Bound<'py, PyTuple> { - PyTuple::new_bound( +fn get_datetime_tuple<'py>(dt: &Bound<'py, PyDateTime>) -> PyResult> { + PyTuple::new( dt.py(), [ dt.get_year(), @@ -142,8 +144,8 @@ fn get_datetime_tuple<'py>(dt: &Bound<'py, PyDateTime>) -> Bound<'py, PyTuple> { } #[pyfunction] -fn get_datetime_tuple_fold<'py>(dt: &Bound<'py, PyDateTime>) -> Bound<'py, PyTuple> { - PyTuple::new_bound( +fn get_datetime_tuple_fold<'py>(dt: &Bound<'py, PyDateTime>) -> PyResult> { + PyTuple::new( dt.py(), [ dt.get_year(), @@ -159,22 +161,23 @@ fn get_datetime_tuple_fold<'py>(dt: &Bound<'py, PyDateTime>) -> Bound<'py, PyTup } #[pyfunction] +#[pyo3(signature=(ts, tz=None))] fn datetime_from_timestamp<'py>( py: Python<'py>, ts: f64, tz: Option<&Bound<'py, PyTzInfo>>, ) -> PyResult> { - PyDateTime::from_timestamp_bound(py, ts, tz) + PyDateTime::from_timestamp(py, ts, tz) } #[pyfunction] fn get_datetime_tzinfo<'py>(dt: &Bound<'py, PyDateTime>) -> Option> { - dt.get_tzinfo_bound() + dt.get_tzinfo() } #[pyfunction] fn get_time_tzinfo<'py>(dt: &Bound<'py, PyTime>) -> Option> { - dt.get_tzinfo_bound() + dt.get_tzinfo() } #[pyclass(extends=PyTzInfo)] @@ -188,7 +191,7 @@ impl TzClass { } fn utcoffset<'py>(&self, dt: &Bound<'py, PyDateTime>) -> PyResult> { - PyDelta::new_bound(dt.py(), 0, 3600, 0, true) + PyDelta::new(dt.py(), 0, 3600, 0, true) } fn tzname(&self, _dt: &Bound<'_, PyDateTime>) -> String { @@ -200,7 +203,7 @@ impl TzClass { } } -#[pymodule] +#[pymodule(gil_used = false)] pub fn datetime(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(make_date, m)?)?; m.add_function(wrap_pyfunction!(get_date_tuple, m)?)?; diff --git a/pytests/src/enums.rs b/pytests/src/enums.rs index 4bb269fbbd2..8652321700a 100644 --- a/pytests/src/enums.rs +++ b/pytests/src/enums.rs @@ -1,17 +1,25 @@ use pyo3::{ - pyclass, pyfunction, pymodule, types::PyModule, wrap_pyfunction_bound, Bound, PyResult, + pyclass, pyfunction, pymodule, + types::{PyModule, PyModuleMethods}, + wrap_pyfunction, Bound, PyResult, }; -#[pymodule] +#[pymodule(gil_used = false)] pub fn enums(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; - m.add_wrapped(wrap_pyfunction_bound!(do_simple_stuff))?; - m.add_wrapped(wrap_pyfunction_bound!(do_complex_stuff))?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_wrapped(wrap_pyfunction!(do_simple_stuff))?; + m.add_wrapped(wrap_pyfunction!(do_complex_stuff))?; + m.add_wrapped(wrap_pyfunction!(do_tuple_stuff))?; + m.add_wrapped(wrap_pyfunction!(do_mixed_complex_stuff))?; Ok(()) } -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] pub enum SimpleEnum { Sunday, Monday, @@ -37,11 +45,26 @@ pub fn do_simple_stuff(thing: &SimpleEnum) -> SimpleEnum { #[pyclass] pub enum ComplexEnum { - Int { i: i32 }, - Float { f: f64 }, - Str { s: String }, + Int { + i: i32, + }, + Float { + f: f64, + }, + Str { + s: String, + }, EmptyStruct {}, - MultiFieldStruct { a: i32, b: f64, c: bool }, + MultiFieldStruct { + a: i32, + b: f64, + c: bool, + }, + #[pyo3(constructor = (a = 42, b = None))] + VariantWithDefault { + a: i32, + b: Option, + }, } #[pyfunction] @@ -56,5 +79,46 @@ pub fn do_complex_stuff(thing: &ComplexEnum) -> ComplexEnum { b: *b, c: *c, }, + ComplexEnum::VariantWithDefault { a, b } => ComplexEnum::VariantWithDefault { + a: 2 * a, + b: b.as_ref().map(|s| s.to_uppercase()), + }, + } +} + +#[pyclass] +enum SimpleTupleEnum { + Int(i32), + Str(String), +} + +#[pyclass] +pub enum TupleEnum { + #[pyo3(constructor = (_0 = 1, _1 = 1.0, _2 = true))] + FullWithDefault(i32, f64, bool), + Full(i32, f64, bool), + EmptyTuple(), +} + +#[pyfunction] +pub fn do_tuple_stuff(thing: &TupleEnum) -> TupleEnum { + match thing { + TupleEnum::FullWithDefault(a, b, c) => TupleEnum::FullWithDefault(*a, *b, *c), + TupleEnum::Full(a, b, c) => TupleEnum::Full(*a, *b, *c), + TupleEnum::EmptyTuple() => TupleEnum::EmptyTuple(), + } +} + +#[pyclass] +pub enum MixedComplexEnum { + Nothing {}, + Empty(), +} + +#[pyfunction] +pub fn do_mixed_complex_stuff(thing: &MixedComplexEnum) -> MixedComplexEnum { + match thing { + MixedComplexEnum::Nothing {} => MixedComplexEnum::Empty(), + MixedComplexEnum::Empty() => MixedComplexEnum::Nothing {}, } } diff --git a/pytests/src/lib.rs b/pytests/src/lib.rs index cbd65c8012c..b6c32230dac 100644 --- a/pytests/src/lib.rs +++ b/pytests/src/lib.rs @@ -17,7 +17,7 @@ pub mod pyfunctions; pub mod sequence; pub mod subclassing; -#[pymodule] +#[pymodule(gil_used = false)] fn pyo3_pytests(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pymodule!(awaitable::awaitable))?; #[cfg(not(Py_LIMITED_API))] @@ -39,7 +39,7 @@ fn pyo3_pytests(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { // Inserting to sys.modules allows importing submodules nicely from Python // e.g. import pyo3_pytests.buf_and_str as bas - let sys = PyModule::import_bound(py, "sys")?; + let sys = PyModule::import(py, "sys")?; let sys_modules = sys.getattr("modules")?.downcast_into::()?; sys_modules.set_item("pyo3_pytests.awaitable", m.getattr("awaitable")?)?; sys_modules.set_item("pyo3_pytests.buf_and_str", m.getattr("buf_and_str")?)?; diff --git a/pytests/src/misc.rs b/pytests/src/misc.rs index 7704098bd5b..b3ef5ee283e 100644 --- a/pytests/src/misc.rs +++ b/pytests/src/misc.rs @@ -1,5 +1,7 @@ -use pyo3::{prelude::*, types::PyDict}; -use std::borrow::Cow; +use pyo3::{ + prelude::*, + types::{PyDict, PyString}, +}; #[pyfunction] fn issue_219() { @@ -7,9 +9,33 @@ fn issue_219() { Python::with_gil(|_| {}); } +#[pyclass] +struct LockHolder { + #[allow(unused)] + // Mutex needed for the MSRV + sender: std::sync::Mutex>, +} + +// This will hammer the GIL once the LockHolder is dropped. +#[pyfunction] +fn hammer_gil_in_thread() -> LockHolder { + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + receiver.recv().ok(); + // now the interpreter has shut down, so hammer the GIL. In buggy + // versions of PyO3 this will cause a crash. + loop { + Python::with_gil(|_py| ()); + } + }); + LockHolder { + sender: std::sync::Mutex::new(sender), + } +} + #[pyfunction] -fn get_type_full_name(obj: &Bound<'_, PyAny>) -> PyResult { - obj.get_type().name().map(Cow::into_owned) +fn get_type_fully_qualified_name<'py>(obj: &Bound<'py, PyAny>) -> PyResult> { + obj.get_type().fully_qualified_name() } #[pyfunction] @@ -30,10 +56,11 @@ fn get_item_and_run_callback(dict: Bound<'_, PyDict>, callback: Bound<'_, PyAny> Ok(()) } -#[pymodule] +#[pymodule(gil_used = false)] pub fn misc(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(issue_219, m)?)?; - m.add_function(wrap_pyfunction!(get_type_full_name, m)?)?; + m.add_function(wrap_pyfunction!(hammer_gil_in_thread, m)?)?; + m.add_function(wrap_pyfunction!(get_type_fully_qualified_name, m)?)?; m.add_function(wrap_pyfunction!(accepts_bool, m)?)?; m.add_function(wrap_pyfunction!(get_item_and_run_callback, m)?)?; Ok(()) diff --git a/pytests/src/objstore.rs b/pytests/src/objstore.rs index 9a005c0ec97..8e729052992 100644 --- a/pytests/src/objstore.rs +++ b/pytests/src/objstore.rs @@ -13,12 +13,12 @@ impl ObjStore { ObjStore::default() } - fn push(&mut self, py: Python<'_>, obj: &Bound<'_, PyAny>) { - self.obj.push(obj.to_object(py)); + fn push(&mut self, obj: &Bound<'_, PyAny>) { + self.obj.push(obj.clone().unbind()); } } -#[pymodule] +#[pymodule(gil_used = false)] pub fn objstore(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::() } diff --git a/pytests/src/othermod.rs b/pytests/src/othermod.rs index 29ca8121890..0de912d7d04 100644 --- a/pytests/src/othermod.rs +++ b/pytests/src/othermod.rs @@ -28,14 +28,14 @@ fn double(x: i32) -> i32 { x * 2 } -#[pymodule] +#[pymodule(gil_used = false)] pub fn othermod(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(double, m)?)?; m.add_class::()?; - m.add("USIZE_MIN", usize::min_value())?; - m.add("USIZE_MAX", usize::max_value())?; + m.add("USIZE_MIN", usize::MIN)?; + m.add("USIZE_MAX", usize::MAX)?; Ok(()) } diff --git a/pytests/src/path.rs b/pytests/src/path.rs index 0675e56d13a..b52c038ed34 100644 --- a/pytests/src/path.rs +++ b/pytests/src/path.rs @@ -11,7 +11,7 @@ fn take_pathbuf(path: PathBuf) -> PathBuf { path } -#[pymodule] +#[pymodule(gil_used = false)] pub fn path(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(make_path, m)?)?; m.add_function(wrap_pyfunction!(take_pathbuf, m)?)?; diff --git a/pytests/src/pyclasses.rs b/pytests/src/pyclasses.rs index ac817627cfe..3af08c053cc 100644 --- a/pytests/src/pyclasses.rs +++ b/pytests/src/pyclasses.rs @@ -1,3 +1,5 @@ +use std::{thread, time}; + use pyo3::exceptions::{PyStopIteration, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyType; @@ -43,6 +45,29 @@ impl PyClassIter { } } +#[pyclass] +#[derive(Default)] +struct PyClassThreadIter { + count: usize, +} + +#[pymethods] +impl PyClassThreadIter { + #[new] + pub fn new() -> Self { + Default::default() + } + + fn __next__(&mut self, py: Python<'_>) -> usize { + let current_count = self.count; + self.count += 1; + if current_count == 0 { + py.allow_threads(|| thread::sleep(time::Duration::from_millis(100))); + } + self.count + } +} + /// Demonstrates a base class which can operate on the relevant subclass in its constructor. #[pyclass(subclass)] #[derive(Clone, Debug)] @@ -63,39 +88,31 @@ impl AssertingBaseClass { } } -#[allow(deprecated)] -mod deprecated { - use super::*; - - #[pyclass(subclass)] - #[derive(Clone, Debug)] - pub struct AssertingBaseClassGilRef; - - #[pymethods] - impl AssertingBaseClassGilRef { - #[new] - #[classmethod] - fn new(cls: &PyType, expected_type: &PyType) -> PyResult { - if !cls.is(expected_type) { - return Err(PyValueError::new_err(format!( - "{:?} != {:?}", - cls, expected_type - ))); - } - Ok(Self) - } - } -} - #[pyclass] struct ClassWithoutConstructor; -#[pymodule] +#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] +#[pyclass(dict)] +struct ClassWithDict; + +#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] +#[pymethods] +impl ClassWithDict { + #[new] + fn new() -> Self { + ClassWithDict + } +} + +#[pymodule(gil_used = false)] pub fn pyclasses(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; - m.add_class::()?; m.add_class::()?; + #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + m.add_class::()?; + Ok(()) } diff --git a/pytests/src/pyfunctions.rs b/pytests/src/pyfunctions.rs index 77496198bb9..024641d3d2e 100644 --- a/pytests/src/pyfunctions.rs +++ b/pytests/src/pyfunctions.rs @@ -67,7 +67,7 @@ fn args_kwargs<'py>( (args, kwargs) } -#[pymodule] +#[pymodule(gil_used = false)] pub fn pyfunctions(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(none, m)?)?; m.add_function(wrap_pyfunction!(simple, m)?)?; diff --git a/pytests/src/sequence.rs b/pytests/src/sequence.rs index 0e48a161bd3..175f5fba8aa 100644 --- a/pytests/src/sequence.rs +++ b/pytests/src/sequence.rs @@ -12,11 +12,11 @@ fn array_to_array_i32(arr: [i32; 3]) -> [i32; 3] { } #[pyfunction] -fn vec_to_vec_pystring(vec: Vec<&PyString>) -> Vec<&PyString> { +fn vec_to_vec_pystring(vec: Vec>) -> Vec> { vec } -#[pymodule] +#[pymodule(gil_used = false)] pub fn sequence(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(vec_to_vec_i32, m)?)?; m.add_function(wrap_pyfunction!(array_to_array_i32, m)?)?; diff --git a/pytests/src/subclassing.rs b/pytests/src/subclassing.rs index 8e451cd9183..0f00e74c19d 100644 --- a/pytests/src/subclassing.rs +++ b/pytests/src/subclassing.rs @@ -17,7 +17,7 @@ impl Subclassable { } } -#[pymodule] +#[pymodule(gil_used = false)] pub fn subclassing(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) diff --git a/pytests/tests/test_comparisons.py b/pytests/tests/test_comparisons.py index 508cdeb2465..fe4d8f31f62 100644 --- a/pytests/tests/test_comparisons.py +++ b/pytests/tests/test_comparisons.py @@ -1,7 +1,14 @@ from typing import Type, Union +import sys import pytest -from pyo3_pytests.comparisons import Eq, EqDefaultNe, Ordered, OrderedDefaultNe +from pyo3_pytests.comparisons import ( + Eq, + EqDefaultNe, + EqDerived, + Ordered, + OrderedDefaultNe, +) from typing_extensions import Self @@ -9,15 +16,28 @@ class PyEq: def __init__(self, x: int) -> None: self.x = x - def __eq__(self, other: Self) -> bool: - return self.x == other.x + def __eq__(self, other: object) -> bool: + if isinstance(other, self.__class__): + return self.x == other.x + else: + return NotImplemented def __ne__(self, other: Self) -> bool: - return self.x != other.x + if isinstance(other, self.__class__): + return self.x != other.x + else: + return NotImplemented -@pytest.mark.parametrize("ty", (Eq, PyEq), ids=("rust", "python")) -def test_eq(ty: Type[Union[Eq, PyEq]]): +@pytest.mark.skipif( + sys.implementation.name == "graalpy" + and __graalpython__.get_graalvm_version().startswith("24.1"), # noqa: F821 + reason="Bug in GraalPy 24.1", +) +@pytest.mark.parametrize( + "ty", (Eq, EqDerived, PyEq), ids=("rust", "rust-derived", "python") +) +def test_eq(ty: Type[Union[Eq, EqDerived, PyEq]]): a = ty(0) b = ty(0) c = ty(1) @@ -32,6 +52,13 @@ def test_eq(ty: Type[Union[Eq, PyEq]]): assert b != c assert not (b == c) + assert not a == 0 + assert a != 0 + assert not b == 0 + assert b != 1 + assert not c == 1 + assert c != 1 + with pytest.raises(TypeError): assert a <= b diff --git a/pytests/tests/test_datetime.py b/pytests/tests/test_datetime.py index c81d13a929a..6484b926a96 100644 --- a/pytests/tests/test_datetime.py +++ b/pytests/tests/test_datetime.py @@ -56,11 +56,11 @@ def tzname(self, dt): IS_WINDOWS = sys.platform == "win32" if IS_WINDOWS: - MIN_DATETIME = pdt.datetime(1971, 1, 2, 0, 0) + MIN_DATETIME = pdt.datetime(1970, 1, 1, 0, 0, 0) if IS_32_BIT: - MAX_DATETIME = pdt.datetime(3001, 1, 19, 4, 59, 59) + MAX_DATETIME = pdt.datetime(2038, 1, 18, 23, 59, 59) else: - MAX_DATETIME = pdt.datetime(3001, 1, 19, 7, 59, 59) + MAX_DATETIME = pdt.datetime(3000, 12, 31, 23, 59, 59) else: if IS_32_BIT: # TS ±2147483648 (2**31) @@ -93,11 +93,21 @@ def test_invalid_date_fails(): @given(d=st.dates(MIN_DATETIME.date(), MAX_DATETIME.date())) def test_date_from_timestamp(d): - if PYPY and d < pdt.date(1900, 1, 1): - pytest.xfail("pdt.datetime.timestamp will raise on PyPy with dates before 1900") - - ts = pdt.datetime.timestamp(pdt.datetime.combine(d, pdt.time(0))) - assert rdt.date_from_timestamp(int(ts)) == pdt.date.fromtimestamp(ts) + try: + ts = pdt.datetime.timestamp(d) + except Exception: + # out of range for timestamp + return + + try: + expected = pdt.date.fromtimestamp(ts) + except Exception as pdt_fail: + # date from timestamp failed; expect the same from Rust binding + with pytest.raises(type(pdt_fail)) as exc_info: + rdt.date_from_timestamp(ts) + assert str(exc_info.value) == str(pdt_fail) + else: + assert rdt.date_from_timestamp(int(ts)) == expected @pytest.mark.parametrize( @@ -229,11 +239,21 @@ def test_datetime_typeerror(): @given(dt=st.datetimes(MIN_DATETIME, MAX_DATETIME)) @example(dt=pdt.datetime(1971, 1, 2, 0, 0)) def test_datetime_from_timestamp(dt): - if PYPY and dt < pdt.datetime(1900, 1, 1): - pytest.xfail("pdt.datetime.timestamp will raise on PyPy with dates before 1900") - - ts = pdt.datetime.timestamp(dt) - assert rdt.datetime_from_timestamp(ts) == pdt.datetime.fromtimestamp(ts) + try: + ts = pdt.datetime.timestamp(dt) + except Exception: + # out of range for timestamp + return + + try: + expected = pdt.datetime.fromtimestamp(ts) + except Exception as pdt_fail: + # datetime from timestamp failed; expect the same from Rust binding + with pytest.raises(type(pdt_fail)) as exc_info: + rdt.datetime_from_timestamp(ts) + assert str(exc_info.value) == str(pdt_fail) + else: + assert rdt.datetime_from_timestamp(ts) == expected def test_datetime_from_timestamp_tzinfo(): diff --git a/pytests/tests/test_enums.py b/pytests/tests/test_enums.py index 04b0cdca431..cd4f7e124c9 100644 --- a/pytests/tests/test_enums.py +++ b/pytests/tests/test_enums.py @@ -18,6 +18,12 @@ def test_complex_enum_variant_constructors(): multi_field_struct_variant = enums.ComplexEnum.MultiFieldStruct(42, 3.14, True) assert isinstance(multi_field_struct_variant, enums.ComplexEnum.MultiFieldStruct) + variant_with_default_1 = enums.ComplexEnum.VariantWithDefault() + assert isinstance(variant_with_default_1, enums.ComplexEnum.VariantWithDefault) + + variant_with_default_2 = enums.ComplexEnum.VariantWithDefault(25, "Hello") + assert isinstance(variant_with_default_2, enums.ComplexEnum.VariantWithDefault) + @pytest.mark.parametrize( "variant", @@ -27,6 +33,7 @@ def test_complex_enum_variant_constructors(): enums.ComplexEnum.Str("hello"), enums.ComplexEnum.EmptyStruct(), enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + enums.ComplexEnum.VariantWithDefault(), ], ) def test_complex_enum_variant_subclasses(variant: enums.ComplexEnum): @@ -48,6 +55,10 @@ def test_complex_enum_field_getters(): assert multi_field_struct_variant.b == 3.14 assert multi_field_struct_variant.c is True + variant_with_default = enums.ComplexEnum.VariantWithDefault() + assert variant_with_default.a == 42 + assert variant_with_default.b is None + @pytest.mark.parametrize( "variant", @@ -57,6 +68,7 @@ def test_complex_enum_field_getters(): enums.ComplexEnum.Str("hello"), enums.ComplexEnum.EmptyStruct(), enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + enums.ComplexEnum.VariantWithDefault(), ], ) def test_complex_enum_desugared_match(variant: enums.ComplexEnum): @@ -78,6 +90,11 @@ def test_complex_enum_desugared_match(variant: enums.ComplexEnum): assert x == 42 assert y == 3.14 assert z is True + elif isinstance(variant, enums.ComplexEnum.VariantWithDefault): + x = variant.a + y = variant.b + assert x == 42 + assert y is None else: assert False @@ -90,6 +107,7 @@ def test_complex_enum_desugared_match(variant: enums.ComplexEnum): enums.ComplexEnum.Str("hello"), enums.ComplexEnum.EmptyStruct(), enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + enums.ComplexEnum.VariantWithDefault(b="hello"), ], ) def test_complex_enum_pyfunction_in_out_desugared_match(variant: enums.ComplexEnum): @@ -112,5 +130,74 @@ def test_complex_enum_pyfunction_in_out_desugared_match(variant: enums.ComplexEn assert x == 42 assert y == 3.14 assert z is True + elif isinstance(variant, enums.ComplexEnum.VariantWithDefault): + x = variant.a + y = variant.b + assert x == 84 + assert y == "HELLO" else: assert False + + +def test_tuple_enum_variant_constructors(): + tuple_variant = enums.TupleEnum.Full(42, 3.14, False) + assert isinstance(tuple_variant, enums.TupleEnum.Full) + + empty_tuple_variant = enums.TupleEnum.EmptyTuple() + assert isinstance(empty_tuple_variant, enums.TupleEnum.EmptyTuple) + + +@pytest.mark.parametrize( + "variant", + [ + enums.TupleEnum.FullWithDefault(), + enums.TupleEnum.Full(42, 3.14, False), + enums.TupleEnum.EmptyTuple(), + ], +) +def test_tuple_enum_variant_subclasses(variant: enums.TupleEnum): + assert isinstance(variant, enums.TupleEnum) + + +def test_tuple_enum_defaults(): + variant = enums.TupleEnum.FullWithDefault() + assert variant._0 == 1 + assert variant._1 == 1.0 + assert variant._2 is True + + +def test_tuple_enum_field_getters(): + tuple_variant = enums.TupleEnum.Full(42, 3.14, False) + assert tuple_variant._0 == 42 + assert tuple_variant._1 == 3.14 + assert tuple_variant._2 is False + + +def test_tuple_enum_index_getter(): + tuple_variant = enums.TupleEnum.Full(42, 3.14, False) + assert len(tuple_variant) == 3 + assert tuple_variant[0] == 42 + + +@pytest.mark.parametrize( + "variant", + [enums.MixedComplexEnum.Nothing()], +) +def test_mixed_complex_enum_pyfunction_instance_nothing( + variant: enums.MixedComplexEnum, +): + assert isinstance(variant, enums.MixedComplexEnum.Nothing) + assert isinstance( + enums.do_mixed_complex_stuff(variant), enums.MixedComplexEnum.Empty + ) + + +@pytest.mark.parametrize( + "variant", + [enums.MixedComplexEnum.Empty()], +) +def test_mixed_complex_enum_pyfunction_instance_empty(variant: enums.MixedComplexEnum): + assert isinstance(variant, enums.MixedComplexEnum.Empty) + assert isinstance( + enums.do_mixed_complex_stuff(variant), enums.MixedComplexEnum.Nothing + ) diff --git a/pytests/tests/test_enums_match.py b/pytests/tests/test_enums_match.py index 4d55bbbe351..6c4b5f6aa07 100644 --- a/pytests/tests/test_enums_match.py +++ b/pytests/tests/test_enums_match.py @@ -57,3 +57,102 @@ def test_complex_enum_pyfunction_in_out(variant: enums.ComplexEnum): assert z is True case _: assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + ], +) +def test_complex_enum_partial_match(variant: enums.ComplexEnum): + match variant: + case enums.ComplexEnum.MultiFieldStruct(a): + assert a == 42 + case _: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.TupleEnum.Full(42, 3.14, True), + enums.TupleEnum.EmptyTuple(), + ], +) +def test_tuple_enum_match_statement(variant: enums.TupleEnum): + match variant: + case enums.TupleEnum.Full(_0=x, _1=y, _2=z): + assert x == 42 + assert y == 3.14 + assert z is True + case enums.TupleEnum.EmptyTuple(): + assert True + case _: + print(variant) + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.SimpleTupleEnum.Int(42), + enums.SimpleTupleEnum.Str("hello"), + ], +) +def test_simple_tuple_enum_match_statement(variant: enums.SimpleTupleEnum): + match variant: + case enums.SimpleTupleEnum.Int(x): + assert x == 42 + case enums.SimpleTupleEnum.Str(x): + assert x == "hello" + case _: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.TupleEnum.Full(42, 3.14, True), + ], +) +def test_tuple_enum_match_match_args(variant: enums.TupleEnum): + match variant: + case enums.TupleEnum.Full(x, y, z): + assert x == 42 + assert y == 3.14 + assert z is True + assert True + case _: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.TupleEnum.Full(42, 3.14, True), + ], +) +def test_tuple_enum_partial_match(variant: enums.TupleEnum): + match variant: + case enums.TupleEnum.Full(a): + assert a == 42 + case _: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.MixedComplexEnum.Nothing(), + enums.MixedComplexEnum.Empty(), + ], +) +def test_mixed_complex_enum_match_statement(variant: enums.MixedComplexEnum): + match variant: + case enums.MixedComplexEnum.Nothing(): + assert True + case enums.MixedComplexEnum.Empty(): + assert True + case _: + assert False diff --git a/pytests/tests/test_hammer_gil_in_thread.py b/pytests/tests/test_hammer_gil_in_thread.py new file mode 100644 index 00000000000..9eed640f65c --- /dev/null +++ b/pytests/tests/test_hammer_gil_in_thread.py @@ -0,0 +1,28 @@ +import sysconfig + +import pytest + +from pyo3_pytests import misc + + +def make_loop(): + # create a reference loop that will only be destroyed when the GC is called at the end + # of execution + start = [] + cur = [start] + for _ in range(1000 * 1000 * 10): + cur = [cur] + start.append(cur) + return start + + +# set a bomb that will explode when modules are cleaned up +loopy = [make_loop()] + + +@pytest.mark.skipif( + sysconfig.get_config_var("Py_DEBUG"), + reason="causes a crash on debug builds, see discussion in https://github.com/PyO3/pyo3/pull/4874", +) +def test_hammer_gil(): + loopy.append(misc.hammer_gil_in_thread()) diff --git a/pytests/tests/test_misc.py b/pytests/tests/test_misc.py index de75f1c8a80..dd7f8007e81 100644 --- a/pytests/tests/test_misc.py +++ b/pytests/tests/test_misc.py @@ -5,6 +5,11 @@ import pyo3_pytests.misc import pytest +if sys.version_info >= (3, 13): + subinterpreters = pytest.importorskip("_interpreters") +else: + subinterpreters = pytest.importorskip("_xxsubinterpreters") + def test_issue_219(): # Should not deadlock @@ -31,29 +36,39 @@ def test_multiple_imports_same_interpreter_ok(): reason="PyPy and GraalPy do not support subinterpreters", ) def test_import_in_subinterpreter_forbidden(): - import _xxsubinterpreters - + sub_interpreter = subinterpreters.create() if sys.version_info < (3, 12): expected_error = "PyO3 modules do not yet support subinterpreters, see https://github.com/PyO3/pyo3/issues/576" else: expected_error = "module pyo3_pytests.pyo3_pytests does not support loading in subinterpreters" - sub_interpreter = _xxsubinterpreters.create() - with pytest.raises( - _xxsubinterpreters.RunFailedError, - match=expected_error, - ): - _xxsubinterpreters.run_string( + if sys.version_info < (3, 13): + # Python 3.12 subinterpreters had a special error for this + with pytest.raises( + subinterpreters.RunFailedError, + match=expected_error, + ): + subinterpreters.run_string( + sub_interpreter, "import pyo3_pytests.pyo3_pytests" + ) + else: + res = subinterpreters.run_string( sub_interpreter, "import pyo3_pytests.pyo3_pytests" ) + assert res.type.__name__ == "ImportError" + assert res.msg == expected_error - _xxsubinterpreters.destroy(sub_interpreter) + subinterpreters.destroy(sub_interpreter) -def test_type_full_name_includes_module(): +def test_type_fully_qualified_name_includes_module(): numpy = pytest.importorskip("numpy") - assert pyo3_pytests.misc.get_type_full_name(numpy.bool_(True)) == "numpy.bool_" + # For numpy 1.x and 2.x + assert pyo3_pytests.misc.get_type_fully_qualified_name(numpy.bool_(True)) in [ + "numpy.bool", + "numpy.bool_", + ] def test_accepts_numpy_bool(): diff --git a/pytests/tests/test_othermod.py b/pytests/tests/test_othermod.py index ff67bba435c..f2dd9ad8fd2 100644 --- a/pytests/tests/test_othermod.py +++ b/pytests/tests/test_othermod.py @@ -3,11 +3,16 @@ from pyo3_pytests import othermod -INTEGER32_ST = st.integers(min_value=(-(2**31)), max_value=(2**31 - 1)) +INTEGER31_ST = st.integers(min_value=(-(2**30)), max_value=(2**30 - 1)) USIZE_ST = st.integers(min_value=othermod.USIZE_MIN, max_value=othermod.USIZE_MAX) -@given(x=INTEGER32_ST) +# If the full 32 bits are used here, then you can get failures that look like this: +# hypothesis.errors.FailedHealthCheck: It looks like your strategy is filtering out a lot of data. +# Health check found 50 filtered examples but only 7 good ones. +# +# Limit the range to 31 bits to avoid this problem. +@given(x=INTEGER31_ST) def test_double(x): expected = x * 2 assume(-(2**31) <= expected <= (2**31 - 1)) diff --git a/pytests/tests/test_path.py b/pytests/tests/test_path.py index 21240187356..d1d6eb83924 100644 --- a/pytests/tests/test_path.py +++ b/pytests/tests/test_path.py @@ -7,21 +7,21 @@ def test_make_path(): p = rpath.make_path() - assert p == "/root" + assert p == pathlib.Path("/root") def test_take_pathbuf(): p = "/root" - assert rpath.take_pathbuf(p) == p + assert rpath.take_pathbuf(p) == pathlib.Path(p) def test_take_pathlib(): p = pathlib.Path("/root") - assert rpath.take_pathbuf(p) == str(p) + assert rpath.take_pathbuf(p) == p def test_take_pathlike(): - assert rpath.take_pathbuf(PathLike("/root")) == "/root" + assert rpath.take_pathbuf(PathLike("/root")) == pathlib.Path("/root") def test_take_invalid_pathlike(): diff --git a/pytests/tests/test_pyclasses.py b/pytests/tests/test_pyclasses.py index 74f883e3808..9f611b634b6 100644 --- a/pytests/tests/test_pyclasses.py +++ b/pytests/tests/test_pyclasses.py @@ -1,3 +1,4 @@ +import platform from typing import Type import pytest @@ -53,22 +54,32 @@ def test_iter(): assert excinfo.value.value == "Ended" -class AssertingSubClass(pyclasses.AssertingBaseClass): - pass +@pytest.mark.skipif( + platform.machine() in ["wasm32", "wasm64"], + reason="not supporting threads in CI for WASM yet", +) +def test_parallel_iter(): + import concurrent.futures + i = pyclasses.PyClassThreadIter() -def test_new_classmethod(): - # The `AssertingBaseClass` constructor errors if it is not passed the - # relevant subclass. - _ = AssertingSubClass(expected_type=AssertingSubClass) - with pytest.raises(ValueError): - _ = AssertingSubClass(expected_type=str) + def func(): + next(i) + # the second thread attempts to borrow a reference to the instance's + # state while the first thread is still sleeping, so we trigger a + # runtime borrow-check error + with pytest.raises(RuntimeError, match="Already borrowed"): + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as tpe: + futures = [tpe.submit(func), tpe.submit(func)] + [f.result() for f in futures] + + +class AssertingSubClass(pyclasses.AssertingBaseClass): + pass -def test_new_classmethod_gil_ref(): - class AssertingSubClass(pyclasses.AssertingBaseClassGilRef): - pass +def test_new_classmethod(): # The `AssertingBaseClass` constructor errors if it is not passed the # relevant subclass. _ = AssertingSubClass(expected_type=AssertingSubClass) @@ -76,13 +87,13 @@ class AssertingSubClass(pyclasses.AssertingBaseClassGilRef): _ = AssertingSubClass(expected_type=str) -class ClassWithoutConstructorPy: +class ClassWithoutConstructor: def __new__(cls): - raise TypeError("No constructor defined") + raise TypeError("No constructor defined for ClassWithoutConstructor") @pytest.mark.parametrize( - "cls", [pyclasses.ClassWithoutConstructor, ClassWithoutConstructorPy] + "cls", [pyclasses.ClassWithoutConstructor, ClassWithoutConstructor] ) def test_no_constructor_defined_propagates_cause(cls: Type): original_error = ValueError("Original message") @@ -90,8 +101,23 @@ def test_no_constructor_defined_propagates_cause(cls: Type): try: raise original_error except Exception: - cls() # should raise TypeError("No constructor defined") + cls() # should raise TypeError("No constructor defined for ...") assert exc_info.type is TypeError - assert exc_info.value.args == ("No constructor defined",) + assert exc_info.value.args == ( + "No constructor defined for ClassWithoutConstructor", + ) assert exc_info.value.__context__ is original_error + + +def test_dict(): + try: + ClassWithDict = pyclasses.ClassWithDict + except AttributeError: + pytest.skip("not defined using abi3 < 3.9") + + d = ClassWithDict() + assert d.__dict__ == {} + + d.foo = 42 + assert d.__dict__ == {"foo": 42} diff --git a/src/buffer.rs b/src/buffer.rs index 84b08289771..2d94681a5c7 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -18,8 +18,8 @@ // DEALINGS IN THE SOFTWARE. //! `PyBuffer` implementation +use crate::Bound; use crate::{err, exceptions::PyBufferError, ffi, FromPyObject, PyAny, PyResult, Python}; -use crate::{Bound, PyNativeType}; use std::marker::PhantomData; use std::os::raw; use std::pin::Pin; @@ -182,35 +182,23 @@ pub unsafe trait Element: Copy { fn is_compatible_format(format: &CStr) -> bool; } -impl<'py, T: Element> FromPyObject<'py> for PyBuffer { +impl FromPyObject<'_> for PyBuffer { fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult> { - Self::get_bound(obj) + Self::get(obj) } } impl PyBuffer { - /// Deprecated form of [`PyBuffer::get_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyBuffer::get` will be replaced by `PyBuffer::get_bound` in a future PyO3 version" - ) - )] - pub fn get(obj: &PyAny) -> PyResult> { - Self::get_bound(&obj.as_borrowed()) - } - /// Gets the underlying buffer from the specified python object. - pub fn get_bound(obj: &Bound<'_, PyAny>) -> PyResult> { - // TODO: use nightly API Box::new_uninit() once stable + pub fn get(obj: &Bound<'_, PyAny>) -> PyResult> { + // TODO: use nightly API Box::new_uninit() once our MSRV is 1.82 let mut buf = Box::new(mem::MaybeUninit::uninit()); let buf: Box = { err::error_on_minusone(obj.py(), unsafe { ffi::PyObject_GetBuffer(obj.as_ptr(), buf.as_mut_ptr(), ffi::PyBUF_FULL_RO) })?; // Safety: buf is initialized by PyObject_GetBuffer. - // TODO: use nightly API Box::assume_init() once stable + // TODO: use nightly API Box::assume_init() once our MSRV is 1.82 unsafe { mem::transmute(buf) } }; // Create PyBuffer immediately so that if validation checks fail, the PyBuffer::drop code @@ -236,6 +224,13 @@ impl PyBuffer { } } + /// Deprecated name for [`PyBuffer::get`]. + #[deprecated(since = "0.23.0", note = "renamed to `PyBuffer::get`")] + #[inline] + pub fn get_bound(obj: &Bound<'_, PyAny>) -> PyResult> { + Self::get(obj) + } + /// Gets the pointer to the start of the buffer memory. /// /// Warning: the buffer memory might be mutated by other Python functions, @@ -263,7 +258,7 @@ impl PyBuffer { }, #[cfg(Py_3_11)] { - indices.as_ptr() as *const ffi::Py_ssize_t + indices.as_ptr().cast() }, #[cfg(not(Py_3_11))] { @@ -317,7 +312,7 @@ impl PyBuffer { /// However, dimensions of length 0 are possible and might need special attention. #[inline] pub fn shape(&self) -> &[usize] { - unsafe { slice::from_raw_parts(self.0.shape as *const usize, self.0.ndim as usize) } + unsafe { slice::from_raw_parts(self.0.shape.cast(), self.0.ndim as usize) } } /// Returns an array that holds, for each dimension, the number of bytes to skip to get to the next element in the dimension. @@ -352,7 +347,7 @@ impl PyBuffer { #[inline] pub fn format(&self) -> &CStr { if self.0.format.is_null() { - CStr::from_bytes_with_nul(b"B\0").unwrap() + ffi::c_str!("B") } else { unsafe { CStr::from_ptr(self.0.format) } } @@ -361,23 +356,13 @@ impl PyBuffer { /// Gets whether the buffer is contiguous in C-style order (last index varies fastest when visiting items in order of memory address). #[inline] pub fn is_c_contiguous(&self) -> bool { - unsafe { - ffi::PyBuffer_IsContiguous( - &*self.0 as *const ffi::Py_buffer, - b'C' as std::os::raw::c_char, - ) != 0 - } + unsafe { ffi::PyBuffer_IsContiguous(&*self.0, b'C' as std::os::raw::c_char) != 0 } } /// Gets whether the buffer is contiguous in Fortran-style order (first index varies fastest when visiting items in order of memory address). #[inline] pub fn is_fortran_contiguous(&self) -> bool { - unsafe { - ffi::PyBuffer_IsContiguous( - &*self.0 as *const ffi::Py_buffer, - b'F' as std::os::raw::c_char, - ) != 0 - } + unsafe { ffi::PyBuffer_IsContiguous(&*self.0, b'F' as std::os::raw::c_char) != 0 } } /// Gets the buffer memory as a slice. @@ -609,7 +594,7 @@ impl PyBuffer { }, #[cfg(Py_3_11)] { - source.as_ptr() as *const raw::c_void + source.as_ptr().cast() }, #[cfg(not(Py_3_11))] { @@ -707,8 +692,8 @@ mod tests { #[test] fn test_debug() { Python::with_gil(|py| { - let bytes = py.eval_bound("b'abcde'", None, None).unwrap(); - let buffer: PyBuffer = PyBuffer::get_bound(&bytes).unwrap(); + let bytes = py.eval(ffi::c_str!("b'abcde'"), None, None).unwrap(); + let buffer: PyBuffer = PyBuffer::get(&bytes).unwrap(); let expected = format!( concat!( "PyBuffer {{ buf: {:?}, obj: {:?}, ", @@ -733,125 +718,124 @@ mod tests { fn test_element_type_from_format() { use super::ElementType; use super::ElementType::*; - use std::ffi::CStr; use std::mem::size_of; use std::os::raw; - for (cstr, expected) in &[ + for (cstr, expected) in [ // @ prefix goes to native_element_type_from_type_char ( - "@b\0", + ffi::c_str!("@b"), SignedInteger { bytes: size_of::(), }, ), ( - "@c\0", + ffi::c_str!("@c"), UnsignedInteger { bytes: size_of::(), }, ), ( - "@b\0", + ffi::c_str!("@b"), SignedInteger { bytes: size_of::(), }, ), ( - "@B\0", + ffi::c_str!("@B"), UnsignedInteger { bytes: size_of::(), }, ), - ("@?\0", Bool), + (ffi::c_str!("@?"), Bool), ( - "@h\0", + ffi::c_str!("@h"), SignedInteger { bytes: size_of::(), }, ), ( - "@H\0", + ffi::c_str!("@H"), UnsignedInteger { bytes: size_of::(), }, ), ( - "@i\0", + ffi::c_str!("@i"), SignedInteger { bytes: size_of::(), }, ), ( - "@I\0", + ffi::c_str!("@I"), UnsignedInteger { bytes: size_of::(), }, ), ( - "@l\0", + ffi::c_str!("@l"), SignedInteger { bytes: size_of::(), }, ), ( - "@L\0", + ffi::c_str!("@L"), UnsignedInteger { bytes: size_of::(), }, ), ( - "@q\0", + ffi::c_str!("@q"), SignedInteger { bytes: size_of::(), }, ), ( - "@Q\0", + ffi::c_str!("@Q"), UnsignedInteger { bytes: size_of::(), }, ), ( - "@n\0", + ffi::c_str!("@n"), SignedInteger { bytes: size_of::(), }, ), ( - "@N\0", + ffi::c_str!("@N"), UnsignedInteger { bytes: size_of::(), }, ), - ("@e\0", Float { bytes: 2 }), - ("@f\0", Float { bytes: 4 }), - ("@d\0", Float { bytes: 8 }), - ("@z\0", Unknown), + (ffi::c_str!("@e"), Float { bytes: 2 }), + (ffi::c_str!("@f"), Float { bytes: 4 }), + (ffi::c_str!("@d"), Float { bytes: 8 }), + (ffi::c_str!("@z"), Unknown), // = prefix goes to standard_element_type_from_type_char - ("=b\0", SignedInteger { bytes: 1 }), - ("=c\0", UnsignedInteger { bytes: 1 }), - ("=B\0", UnsignedInteger { bytes: 1 }), - ("=?\0", Bool), - ("=h\0", SignedInteger { bytes: 2 }), - ("=H\0", UnsignedInteger { bytes: 2 }), - ("=l\0", SignedInteger { bytes: 4 }), - ("=l\0", SignedInteger { bytes: 4 }), - ("=I\0", UnsignedInteger { bytes: 4 }), - ("=L\0", UnsignedInteger { bytes: 4 }), - ("=q\0", SignedInteger { bytes: 8 }), - ("=Q\0", UnsignedInteger { bytes: 8 }), - ("=e\0", Float { bytes: 2 }), - ("=f\0", Float { bytes: 4 }), - ("=d\0", Float { bytes: 8 }), - ("=z\0", Unknown), - ("=0\0", Unknown), + (ffi::c_str!("=b"), SignedInteger { bytes: 1 }), + (ffi::c_str!("=c"), UnsignedInteger { bytes: 1 }), + (ffi::c_str!("=B"), UnsignedInteger { bytes: 1 }), + (ffi::c_str!("=?"), Bool), + (ffi::c_str!("=h"), SignedInteger { bytes: 2 }), + (ffi::c_str!("=H"), UnsignedInteger { bytes: 2 }), + (ffi::c_str!("=l"), SignedInteger { bytes: 4 }), + (ffi::c_str!("=l"), SignedInteger { bytes: 4 }), + (ffi::c_str!("=I"), UnsignedInteger { bytes: 4 }), + (ffi::c_str!("=L"), UnsignedInteger { bytes: 4 }), + (ffi::c_str!("=q"), SignedInteger { bytes: 8 }), + (ffi::c_str!("=Q"), UnsignedInteger { bytes: 8 }), + (ffi::c_str!("=e"), Float { bytes: 2 }), + (ffi::c_str!("=f"), Float { bytes: 4 }), + (ffi::c_str!("=d"), Float { bytes: 8 }), + (ffi::c_str!("=z"), Unknown), + (ffi::c_str!("=0"), Unknown), // unknown prefix -> Unknown - (":b\0", Unknown), + (ffi::c_str!(":b"), Unknown), ] { assert_eq!( - ElementType::from_format(CStr::from_bytes_with_nul(cstr.as_bytes()).unwrap()), - *expected, + ElementType::from_format(cstr), + expected, "element from format &Cstr: {:?}", cstr, ); @@ -870,8 +854,8 @@ mod tests { #[test] fn test_bytes_buffer() { Python::with_gil(|py| { - let bytes = py.eval_bound("b'abcde'", None, None).unwrap(); - let buffer = PyBuffer::get_bound(&bytes).unwrap(); + let bytes = py.eval(ffi::c_str!("b'abcde'"), None, None).unwrap(); + let buffer = PyBuffer::get(&bytes).unwrap(); assert_eq!(buffer.dimensions(), 1); assert_eq!(buffer.item_count(), 5); assert_eq!(buffer.format().to_str().unwrap(), "B"); @@ -903,11 +887,11 @@ mod tests { fn test_array_buffer() { Python::with_gil(|py| { let array = py - .import_bound("array") + .import("array") .unwrap() .call_method("array", ("f", (1.0, 1.5, 2.0, 2.5)), None) .unwrap(); - let buffer = PyBuffer::get_bound(&array).unwrap(); + let buffer = PyBuffer::get(&array).unwrap(); assert_eq!(buffer.dimensions(), 1); assert_eq!(buffer.item_count(), 4); assert_eq!(buffer.format().to_str().unwrap(), "f"); @@ -937,7 +921,7 @@ mod tests { assert_eq!(buffer.to_vec(py).unwrap(), [10.0, 11.0, 12.0, 13.0]); // F-contiguous fns - let buffer = PyBuffer::get_bound(&array).unwrap(); + let buffer = PyBuffer::get(&array).unwrap(); let slice = buffer.as_fortran_slice(py).unwrap(); assert_eq!(slice.len(), 4); assert_eq!(slice[1].get(), 11.0); diff --git a/src/call.rs b/src/call.rs new file mode 100644 index 00000000000..cf9bb16ca3d --- /dev/null +++ b/src/call.rs @@ -0,0 +1,315 @@ +//! Defines how Python calls are dispatched, see [`PyCallArgs`].for more information. + +use crate::ffi_ptr_ext::FfiPtrExt as _; +use crate::types::{PyAnyMethods as _, PyDict, PyString, PyTuple}; +use crate::{ffi, Borrowed, Bound, IntoPyObjectExt as _, Py, PyAny, PyResult}; + +pub(crate) mod private { + use super::*; + + pub trait Sealed {} + + impl Sealed for () {} + impl Sealed for Bound<'_, PyTuple> {} + impl Sealed for &'_ Bound<'_, PyTuple> {} + impl Sealed for Py {} + impl Sealed for &'_ Py {} + impl Sealed for Borrowed<'_, '_, PyTuple> {} + pub struct Token; +} + +/// This trait marks types that can be used as arguments to Python function +/// calls. +/// +/// This trait is currently implemented for Rust tuple (up to a size of 12), +/// [`Bound<'py, PyTuple>`] and [`Py`]. Custom types that are +/// convertable to `PyTuple` via `IntoPyObject` need to do so before passing it +/// to `call`. +/// +/// This trait is not intended to used by downstream crates directly. As such it +/// has no publicly available methods and cannot be implemented ouside of +/// `pyo3`. The corresponding public API is available through [`call`] +/// ([`call0`], [`call1`] and friends) on [`PyAnyMethods`]. +/// +/// # What is `PyCallArgs` used for? +/// `PyCallArgs` is used internally in `pyo3` to dispatch the Python calls in +/// the most optimal way for the current build configuration. Certain types, +/// such as Rust tuples, do allow the usage of a faster calling convention of +/// the Python interpreter (if available). More types that may take advantage +/// from this may be added in the future. +/// +/// [`call0`]: crate::types::PyAnyMethods::call0 +/// [`call1`]: crate::types::PyAnyMethods::call1 +/// [`call`]: crate::types::PyAnyMethods::call +/// [`PyAnyMethods`]: crate::types::PyAnyMethods +#[cfg_attr( + diagnostic_namespace, + diagnostic::on_unimplemented( + message = "`{Self}` cannot used as a Python `call` argument", + note = "`PyCallArgs` is implemented for Rust tuples, `Bound<'py, PyTuple>` and `Py`", + note = "if your type is convertable to `PyTuple` via `IntoPyObject`, call `.into_pyobject(py)` manually", + note = "if you meant to pass the type as a single argument, wrap it in a 1-tuple, `(,)`" + ) +)] +pub trait PyCallArgs<'py>: Sized + private::Sealed { + #[doc(hidden)] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult>; + + #[doc(hidden)] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult>; + + #[doc(hidden)] + fn call_method_positional( + self, + object: Borrowed<'_, 'py, PyAny>, + method_name: Borrowed<'_, 'py, PyString>, + _: private::Token, + ) -> PyResult> { + object + .getattr(method_name) + .and_then(|method| method.call1(self)) + } +} + +impl<'py> PyCallArgs<'py> for () { + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult> { + let args = self.into_pyobject_or_pyerr(function.py())?; + args.call(function, kwargs, token) + } + + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult> { + let args = self.into_pyobject_or_pyerr(function.py())?; + args.call_positional(function, token) + } +} + +impl<'py> PyCallArgs<'py> for Bound<'py, PyTuple> { + #[inline] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult> { + self.as_borrowed().call(function, kwargs, token) + } + + #[inline] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult> { + self.as_borrowed().call_positional(function, token) + } +} + +impl<'py> PyCallArgs<'py> for &'_ Bound<'py, PyTuple> { + #[inline] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult> { + self.as_borrowed().call(function, kwargs, token) + } + + #[inline] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult> { + self.as_borrowed().call_positional(function, token) + } +} + +impl<'py> PyCallArgs<'py> for Py { + #[inline] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult> { + self.bind_borrowed(function.py()) + .call(function, kwargs, token) + } + + #[inline] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult> { + self.bind_borrowed(function.py()) + .call_positional(function, token) + } +} + +impl<'py> PyCallArgs<'py> for &'_ Py { + #[inline] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult> { + self.bind_borrowed(function.py()) + .call(function, kwargs, token) + } + + #[inline] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult> { + self.bind_borrowed(function.py()) + .call_positional(function, token) + } +} + +impl<'py> PyCallArgs<'py> for Borrowed<'_, 'py, PyTuple> { + #[inline] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + _: private::Token, + ) -> PyResult> { + unsafe { + ffi::PyObject_Call(function.as_ptr(), self.as_ptr(), kwargs.as_ptr()) + .assume_owned_or_err(function.py()) + } + } + + #[inline] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + _: private::Token, + ) -> PyResult> { + unsafe { + ffi::PyObject_Call(function.as_ptr(), self.as_ptr(), std::ptr::null_mut()) + .assume_owned_or_err(function.py()) + } + } +} + +#[cfg(test)] +#[cfg(feature = "macros")] +mod tests { + use crate::{ + pyfunction, + types::{PyDict, PyTuple}, + Py, + }; + + #[pyfunction(signature = (*args, **kwargs), crate = "crate")] + fn args_kwargs( + args: Py, + kwargs: Option>, + ) -> (Py, Option>) { + (args, kwargs) + } + + #[test] + fn test_call() { + use crate::{ + types::{IntoPyDict, PyAnyMethods, PyDict, PyTuple}, + wrap_pyfunction, Py, Python, + }; + + Python::with_gil(|py| { + let f = wrap_pyfunction!(args_kwargs, py).unwrap(); + + let args = PyTuple::new(py, [1, 2, 3]).unwrap(); + let kwargs = &[("foo", 1), ("bar", 2)].into_py_dict(py).unwrap(); + + macro_rules! check_call { + ($args:expr, $kwargs:expr) => { + let (a, k): (Py, Py) = f + .call(args.clone(), Some(kwargs)) + .unwrap() + .extract() + .unwrap(); + assert!(a.is(&args)); + assert!(k.is(kwargs)); + }; + } + + // Bound<'py, PyTuple> + check_call!(args.clone(), kwargs); + + // &Bound<'py, PyTuple> + check_call!(&args, kwargs); + + // Py + check_call!(args.clone().unbind(), kwargs); + + // &Py + check_call!(&args.as_unbound(), kwargs); + + // Borrowed<'_, '_, PyTuple> + check_call!(args.as_borrowed(), kwargs); + }) + } + + #[test] + fn test_call_positional() { + use crate::{ + types::{PyAnyMethods, PyNone, PyTuple}, + wrap_pyfunction, Py, Python, + }; + + Python::with_gil(|py| { + let f = wrap_pyfunction!(args_kwargs, py).unwrap(); + + let args = PyTuple::new(py, [1, 2, 3]).unwrap(); + + macro_rules! check_call { + ($args:expr, $kwargs:expr) => { + let (a, k): (Py, Py) = + f.call1(args.clone()).unwrap().extract().unwrap(); + assert!(a.is(&args)); + assert!(k.is_none(py)); + }; + } + + // Bound<'py, PyTuple> + check_call!(args.clone(), kwargs); + + // &Bound<'py, PyTuple> + check_call!(&args, kwargs); + + // Py + check_call!(args.clone().unbind(), kwargs); + + // &Py + check_call!(args.as_unbound(), kwargs); + + // Borrowed<'_, '_, PyTuple> + check_call!(args.as_borrowed(), kwargs); + }) + } +} diff --git a/src/conversion.rs b/src/conversion.rs index dfa53eac83e..82ad4d84977 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -1,15 +1,14 @@ //! Defines conversions between Rust and Python types. -use crate::err::{self, PyDowncastError, PyResult}; +use crate::err::PyResult; #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::pyclass::boolean_struct::False; -use crate::type_object::PyTypeInfo; use crate::types::any::PyAnyMethods; use crate::types::PyTuple; use crate::{ - ffi, gil, Borrowed, Bound, Py, PyAny, PyClass, PyNativeType, PyObject, PyRef, PyRefMut, Python, + ffi, Borrowed, Bound, BoundObject, Py, PyAny, PyClass, PyErr, PyObject, PyRef, PyRefMut, Python, }; -use std::ptr::NonNull; +use std::convert::Infallible; /// Returns a borrowed pointer to a Python object. /// @@ -20,16 +19,17 @@ use std::ptr::NonNull; /// /// ```rust /// use pyo3::prelude::*; -/// use pyo3::types::PyString; /// use pyo3::ffi; /// /// Python::with_gil(|py| { -/// let s: Py = "foo".into_py(py); +/// let s = "foo".into_pyobject(py)?; /// let ptr = s.as_ptr(); /// /// let is_really_a_pystring = unsafe { ffi::PyUnicode_CheckExact(ptr) }; /// assert_eq!(is_really_a_pystring, 1); -/// }); +/// # Ok::<_, PyErr>(()) +/// }) +/// # .unwrap(); /// ``` /// /// # Safety @@ -42,18 +42,21 @@ use std::ptr::NonNull; /// # use pyo3::ffi; /// # /// Python::with_gil(|py| { -/// let ptr: *mut ffi::PyObject = 0xabad1dea_u32.into_py(py).as_ptr(); +/// // ERROR: calling `.as_ptr()` will throw away the temporary object and leave `ptr` dangling. +/// let ptr: *mut ffi::PyObject = 0xabad1dea_u32.into_pyobject(py)?.as_ptr(); /// /// let isnt_a_pystring = unsafe { /// // `ptr` is dangling, this is UB /// ffi::PyUnicode_CheckExact(ptr) /// }; -/// # assert_eq!(isnt_a_pystring, 0); -/// }); +/// # assert_eq!(isnt_a_pystring, 0); +/// # Ok::<_, PyErr>(()) +/// }) +/// # .unwrap(); /// ``` /// /// This happens because the pointer returned by `as_ptr` does not carry any lifetime information -/// and the Python object is dropped immediately after the `0xabad1dea_u32.into_py(py).as_ptr()` +/// and the Python object is dropped immediately after the `0xabad1dea_u32.into_pyobject(py).as_ptr()` /// expression is evaluated. To fix the problem, bind Python object to a local variable like earlier /// to keep the Python object alive until the end of its scope. /// @@ -64,6 +67,10 @@ pub unsafe trait AsPyPointer { } /// Conversion trait that allows various objects to be converted into `PyObject`. +#[deprecated( + since = "0.23.0", + note = "`ToPyObject` is going to be replaced by `IntoPyObject`. See the migration guide (https://pyo3.rs/v0.23.0/migration) for more information." +)] pub trait ToPyObject { /// Converts self into a Python object. fn to_object(&self, py: Python<'_>) -> PyObject; @@ -83,6 +90,7 @@ pub trait ToPyObject { /// ```rust /// use pyo3::prelude::*; /// +/// # #[allow(dead_code)] /// #[pyclass] /// struct Number { /// #[pyo3(get, set)] @@ -96,6 +104,7 @@ pub trait ToPyObject { /// However, it may not be desirable to expose the existence of `Number` to Python code. /// `IntoPy` allows us to define a conversion to an appropriate Python object. /// ```rust +/// #![allow(deprecated)] /// use pyo3::prelude::*; /// /// # #[allow(dead_code)] @@ -117,6 +126,7 @@ pub trait ToPyObject { /// This is useful for types like enums that can carry different types. /// /// ```rust +/// #![allow(deprecated)] /// use pyo3::prelude::*; /// /// enum Value { @@ -148,10 +158,65 @@ pub trait ToPyObject { /// # } /// ``` /// Python code will see this as any of the `int`, `string` or `None` objects. -#[doc(alias = "IntoPyCallbackOutput")] +#[cfg_attr( + diagnostic_namespace, + diagnostic::on_unimplemented( + message = "`{Self}` cannot be converted to a Python object", + note = "`IntoPy` is automatically implemented by the `#[pyclass]` macro", + note = "if you do not wish to have a corresponding Python type, implement it manually", + note = "if you do not own `{Self}` you can perform a manual conversion to one of the types in `pyo3::types::*`" + ) +)] +#[deprecated( + since = "0.23.0", + note = "`IntoPy` is going to be replaced by `IntoPyObject`. See the migration guide (https://pyo3.rs/v0.23.0/migration) for more information." +)] pub trait IntoPy: Sized { /// Performs the conversion. fn into_py(self, py: Python<'_>) -> T; +} + +/// Defines a conversion from a Rust type to a Python object, which may fail. +/// +/// This trait has `#[derive(IntoPyObject)]` to automatically implement it for simple types and +/// `#[derive(IntoPyObjectRef)]` to implement the same for references. +/// +/// It functions similarly to std's [`TryInto`] trait, but requires a [GIL token](Python) +/// as an argument. +/// +/// The [`into_pyobject`][IntoPyObject::into_pyobject] method is designed for maximum flexibility and efficiency; it +/// - allows for a concrete Python type to be returned (the [`Target`][IntoPyObject::Target] associated type) +/// - allows for the smart pointer containing the Python object to be either `Bound<'py, Self::Target>` or `Borrowed<'a, 'py, Self::Target>` +/// to avoid unnecessary reference counting overhead +/// - allows for a custom error type to be returned in the event of a conversion error to avoid +/// unnecessarily creating a Python exception +/// +/// # See also +/// +/// - The [`IntoPyObjectExt`] trait, which provides convenience methods for common usages of +/// `IntoPyObject` which erase type information and convert errors to `PyErr`. +#[cfg_attr( + diagnostic_namespace, + diagnostic::on_unimplemented( + message = "`{Self}` cannot be converted to a Python object", + note = "`IntoPyObject` is automatically implemented by the `#[pyclass]` macro", + note = "if you do not wish to have a corresponding Python type, implement it manually", + note = "if you do not own `{Self}` you can perform a manual conversion to one of the types in `pyo3::types::*`" + ) +)] +pub trait IntoPyObject<'py>: Sized { + /// The Python output type + type Target; + /// The smart pointer type to use. + /// + /// This will usually be [`Bound<'py, Target>`], but in special cases [`Borrowed<'a, 'py, Target>`] can be + /// used to minimize reference counting overhead. + type Output: BoundObject<'py, Self::Target>; + /// The type returned in the event of a conversion error. + type Error: Into; + + /// Performs the conversion. + fn into_pyobject(self, py: Python<'py>) -> Result; /// Extracts the type hint information for this type when it appears as a return value. /// @@ -164,8 +229,177 @@ pub trait IntoPy: Sized { fn type_output() -> TypeInfo { TypeInfo::Any } + + /// Converts sequence of Self into a Python object. Used to specialize `Vec`, `[u8; N]` + /// and `SmallVec<[u8; N]>` as a sequence of bytes into a `bytes` object. + #[doc(hidden)] + fn owned_sequence_into_pyobject( + iter: I, + py: Python<'py>, + _: private::Token, + ) -> Result, PyErr> + where + I: IntoIterator + AsRef<[Self]>, + I::IntoIter: ExactSizeIterator, + { + let mut iter = iter.into_iter().map(|e| e.into_bound_py_any(py)); + let list = crate::types::list::try_new_from_iter(py, &mut iter); + list.map(Bound::into_any) + } + + /// Converts sequence of Self into a Python object. Used to specialize `&[u8]` and `Cow<[u8]>` + /// as a sequence of bytes into a `bytes` object. + #[doc(hidden)] + fn borrowed_sequence_into_pyobject( + iter: I, + py: Python<'py>, + _: private::Token, + ) -> Result, PyErr> + where + Self: private::Reference, + I: IntoIterator + AsRef<[::BaseType]>, + I::IntoIter: ExactSizeIterator, + { + let mut iter = iter.into_iter().map(|e| e.into_bound_py_any(py)); + let list = crate::types::list::try_new_from_iter(py, &mut iter); + list.map(Bound::into_any) + } +} + +pub(crate) mod private { + pub struct Token; + + pub trait Reference { + type BaseType; + } + + impl Reference for &'_ T { + type BaseType = T; + } +} + +impl<'py, T> IntoPyObject<'py> for Bound<'py, T> { + type Target = T; + type Output = Bound<'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, _py: Python<'py>) -> Result { + Ok(self) + } +} + +impl<'a, 'py, T> IntoPyObject<'py> for &'a Bound<'py, T> { + type Target = T; + type Output = Borrowed<'a, 'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, _py: Python<'py>) -> Result { + Ok(self.as_borrowed()) + } +} + +impl<'a, 'py, T> IntoPyObject<'py> for Borrowed<'a, 'py, T> { + type Target = T; + type Output = Borrowed<'a, 'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, _py: Python<'py>) -> Result { + Ok(self) + } +} + +impl<'a, 'py, T> IntoPyObject<'py> for &Borrowed<'a, 'py, T> { + type Target = T; + type Output = Borrowed<'a, 'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, _py: Python<'py>) -> Result { + Ok(*self) + } +} + +impl<'py, T> IntoPyObject<'py> for Py { + type Target = T; + type Output = Bound<'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(self.into_bound(py)) + } } +impl<'a, 'py, T> IntoPyObject<'py> for &'a Py { + type Target = T; + type Output = Borrowed<'a, 'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(self.bind_borrowed(py)) + } +} + +impl<'a, 'py, T> IntoPyObject<'py> for &&'a T +where + &'a T: IntoPyObject<'py>, +{ + type Target = <&'a T as IntoPyObject<'py>>::Target; + type Output = <&'a T as IntoPyObject<'py>>::Output; + type Error = <&'a T as IntoPyObject<'py>>::Error; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) + } +} + +mod into_pyobject_ext { + pub trait Sealed {} + impl<'py, T> Sealed for T where T: super::IntoPyObject<'py> {} +} + +/// Convenience methods for common usages of [`IntoPyObject`]. Every type that implements +/// [`IntoPyObject`] also implements this trait. +/// +/// These methods: +/// - Drop type information from the output, returning a `PyAny` object. +/// - Always convert the `Error` type to `PyErr`, which may incur a performance penalty but it +/// more convenient in contexts where the `?` operator would produce a `PyErr` anyway. +pub trait IntoPyObjectExt<'py>: IntoPyObject<'py> + into_pyobject_ext::Sealed { + /// Converts `self` into an owned Python object, dropping type information. + #[inline] + fn into_bound_py_any(self, py: Python<'py>) -> PyResult> { + match self.into_pyobject(py) { + Ok(obj) => Ok(obj.into_any().into_bound()), + Err(err) => Err(err.into()), + } + } + + /// Converts `self` into an owned Python object, dropping type information and unbinding it + /// from the `'py` lifetime. + #[inline] + fn into_py_any(self, py: Python<'py>) -> PyResult> { + match self.into_pyobject(py) { + Ok(obj) => Ok(obj.into_any().unbind()), + Err(err) => Err(err.into()), + } + } + + /// Converts `self` into a Python object. + /// + /// This is equivalent to calling [`into_pyobject`][IntoPyObject::into_pyobject] followed + /// with `.map_err(Into::into)` to convert the error type to [`PyErr`]. This is helpful + /// for generic code which wants to make use of the `?` operator. + #[inline] + fn into_pyobject_or_pyerr(self, py: Python<'py>) -> PyResult { + match self.into_pyobject(py) { + Ok(obj) => Ok(obj), + Err(err) => Err(err.into()), + } + } +} + +impl<'py, T> IntoPyObjectExt<'py> for T where T: IntoPyObject<'py> {} + /// Extract a type from a Python object. /// /// @@ -180,7 +414,7 @@ pub trait IntoPy: Sized { /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { /// // Calling `.extract()` on a `Bound` smart pointer -/// let obj: Bound<'_, PyString> = PyString::new_bound(py, "blah"); +/// let obj: Bound<'_, PyString> = PyString::new(py, "blah"); /// let s: String = obj.extract()?; /// # assert_eq!(s, "blah"); /// @@ -211,29 +445,20 @@ pub trait IntoPy: Sized { /// infinite recursion, implementors must implement at least one of these methods. The recommendation /// is to implement `extract_bound` and leave `extract` as the default implementation. pub trait FromPyObject<'py>: Sized { - /// Extracts `Self` from the source GIL Ref `obj`. - /// - /// Implementors are encouraged to implement `extract_bound` and leave this method as the - /// default implementation, which will forward calls to `extract_bound`. - fn extract(ob: &'py PyAny) -> PyResult { - Self::extract_bound(&ob.as_borrowed()) - } - /// Extracts `Self` from the bound smart pointer `obj`. /// /// Implementors are encouraged to implement this method and leave `extract` defaulted, as /// this will be most compatible with PyO3's future API. - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - Self::extract(ob.clone().into_gil_ref()) - } + fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult; /// Extracts the type hint information for this type when it appears as an argument. /// /// For example, `Vec` would return `Sequence[int]`. /// The default implementation returns `Any`, which is correct for any type. /// - /// For most types, the return value for this method will be identical to that of [`IntoPy::type_output`]. - /// It may be different for some types, such as `Dict`, to allow duck-typing: functions return `Dict` but take `Mapping` as argument. + /// For most types, the return value for this method will be identical to that of + /// [`IntoPyObject::type_output`]. It may be different for some types, such as `Dict`, + /// to allow duck-typing: functions return `Dict` but take `Mapping` as argument. #[cfg(feature = "experimental-inspect")] fn type_input() -> TypeInfo { TypeInfo::Any @@ -250,13 +475,9 @@ mod from_py_object_bound_sealed { // This generic implementation is why the seal is separate from // `crate::sealed::Sealed`. impl<'py, T> Sealed for T where T: super::FromPyObject<'py> {} - #[cfg(not(feature = "gil-refs"))] impl Sealed for &'_ str {} - #[cfg(not(feature = "gil-refs"))] impl Sealed for std::borrow::Cow<'_, str> {} - #[cfg(not(feature = "gil-refs"))] impl Sealed for &'_ [u8] {} - #[cfg(not(feature = "gil-refs"))] impl Sealed for std::borrow::Cow<'_, [u8]> {} } @@ -297,8 +518,9 @@ pub trait FromPyObjectBound<'a, 'py>: Sized + from_py_object_bound_sealed::Seale /// For example, `Vec` would return `Sequence[int]`. /// The default implementation returns `Any`, which is correct for any type. /// - /// For most types, the return value for this method will be identical to that of [`IntoPy::type_output`]. - /// It may be different for some types, such as `Dict`, to allow duck-typing: functions return `Dict` but take `Mapping` as argument. + /// For most types, the return value for this method will be identical to that of + /// [`IntoPyObject::type_output`]. It may be different for some types, such as `Dict`, + /// to allow duck-typing: functions return `Dict` but take `Mapping` as argument. #[cfg(feature = "experimental-inspect")] fn type_input() -> TypeInfo { TypeInfo::Any @@ -321,6 +543,7 @@ where /// Identity conversion: allows using existing `PyObject` instances where /// `T: ToPyObject` is expected. +#[allow(deprecated)] impl ToPyObject for &'_ T { #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { @@ -328,33 +551,6 @@ impl ToPyObject for &'_ T { } } -impl IntoPy for &'_ PyAny { - #[inline] - fn into_py(self, py: Python<'_>) -> PyObject { - unsafe { PyObject::from_borrowed_ptr(py, self.as_ptr()) } - } -} - -impl IntoPy for &'_ T -where - T: AsRef, -{ - #[inline] - fn into_py(self, py: Python<'_>) -> PyObject { - unsafe { PyObject::from_borrowed_ptr(py, self.as_ref().as_ptr()) } - } -} - -#[allow(deprecated)] -impl<'py, T> FromPyObject<'py> for &'py crate::PyCell -where - T: PyClass, -{ - fn extract(obj: &'py PyAny) -> PyResult { - obj.downcast().map_err(Into::into) - } -} - impl FromPyObject<'_> for T where T: PyClass + Clone, @@ -383,275 +579,21 @@ where } } -/// Trait implemented by Python object types that allow a checked downcast. -/// If `T` implements `PyTryFrom`, we can convert `&PyAny` to `&T`. -/// -/// This trait is similar to `std::convert::TryFrom` -#[deprecated(since = "0.21.0")] -pub trait PyTryFrom<'v>: Sized + PyNativeType { - /// Cast from a concrete Python object type to PyObject. - #[deprecated( - since = "0.21.0", - note = "use `value.downcast::()` instead of `T::try_from(value)`" - )] - fn try_from>(value: V) -> Result<&'v Self, PyDowncastError<'v>>; - - /// Cast from a concrete Python object type to PyObject. With exact type check. - #[deprecated( - since = "0.21.0", - note = "use `value.downcast_exact::()` instead of `T::try_from_exact(value)`" - )] - fn try_from_exact>(value: V) -> Result<&'v Self, PyDowncastError<'v>>; - - /// Cast a PyAny to a specific type of PyObject. The caller must - /// have already verified the reference is for this type. - /// - /// # Safety - /// - /// Callers must ensure that the type is valid or risk type confusion. - #[deprecated( - since = "0.21.0", - note = "use `value.downcast_unchecked::()` instead of `T::try_from_unchecked(value)`" - )] - unsafe fn try_from_unchecked>(value: V) -> &'v Self; -} - -/// Trait implemented by Python object types that allow a checked downcast. -/// This trait is similar to `std::convert::TryInto` -#[deprecated(since = "0.21.0")] -pub trait PyTryInto: Sized { - /// Cast from PyObject to a concrete Python object type. - #[deprecated( - since = "0.21.0", - note = "use `value.downcast()` instead of `value.try_into()`" - )] - fn try_into(&self) -> Result<&T, PyDowncastError<'_>>; - - /// Cast from PyObject to a concrete Python object type. With exact type check. - #[deprecated( - since = "0.21.0", - note = "use `value.downcast()` instead of `value.try_into_exact()`" - )] - fn try_into_exact(&self) -> Result<&T, PyDowncastError<'_>>; -} - -#[allow(deprecated)] -mod implementations { - use super::*; - - // TryFrom implies TryInto - impl PyTryInto for PyAny - where - U: for<'v> PyTryFrom<'v>, - { - fn try_into(&self) -> Result<&U, PyDowncastError<'_>> { - >::try_from(self) - } - fn try_into_exact(&self) -> Result<&U, PyDowncastError<'_>> { - U::try_from_exact(self) - } - } - - impl<'v, T> PyTryFrom<'v> for T - where - T: PyTypeInfo + PyNativeType, - { - fn try_from>(value: V) -> Result<&'v Self, PyDowncastError<'v>> { - value.into().downcast() - } - - fn try_from_exact>(value: V) -> Result<&'v Self, PyDowncastError<'v>> { - value.into().downcast_exact() - } - - #[inline] - unsafe fn try_from_unchecked>(value: V) -> &'v Self { - value.into().downcast_unchecked() - } - } - - impl<'v, T> PyTryFrom<'v> for crate::PyCell - where - T: 'v + PyClass, - { - fn try_from>(value: V) -> Result<&'v Self, PyDowncastError<'v>> { - value.into().downcast() - } - fn try_from_exact>(value: V) -> Result<&'v Self, PyDowncastError<'v>> { - let value = value.into(); - unsafe { - if T::is_exact_type_of(value) { - Ok(Self::try_from_unchecked(value)) - } else { - Err(PyDowncastError::new(value, T::NAME)) - } - } - } - #[inline] - unsafe fn try_from_unchecked>(value: V) -> &'v Self { - value.into().downcast_unchecked() - } - } -} - /// Converts `()` to an empty Python tuple. +#[allow(deprecated)] impl IntoPy> for () { fn into_py(self, py: Python<'_>) -> Py { - PyTuple::empty_bound(py).unbind() + PyTuple::empty(py).unbind() } } -/// Raw level conversion between `*mut ffi::PyObject` and PyO3 types. -/// -/// # Safety -/// -/// See safety notes on individual functions. -#[deprecated(since = "0.21.0")] -pub unsafe trait FromPyPointer<'p>: Sized { - /// Convert from an arbitrary `PyObject`. - /// - /// # Safety - /// - /// Implementations must ensure the object does not get freed during `'p` - /// and ensure that `ptr` is of the correct type. - /// Note that it must be safe to decrement the reference count of `ptr`. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_owned_ptr_or_opt(py, ptr)` or `Bound::from_owned_ptr_or_opt(py, ptr)` instead" - ) - )] - unsafe fn from_owned_ptr_or_opt(py: Python<'p>, ptr: *mut ffi::PyObject) -> Option<&'p Self>; - /// Convert from an arbitrary `PyObject` or panic. - /// - /// # Safety - /// - /// Relies on [`from_owned_ptr_or_opt`](#method.from_owned_ptr_or_opt). - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" - ) - )] - unsafe fn from_owned_ptr_or_panic(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { - #[allow(deprecated)] - Self::from_owned_ptr_or_opt(py, ptr).unwrap_or_else(|| err::panic_after_error(py)) - } - /// Convert from an arbitrary `PyObject` or panic. - /// - /// # Safety - /// - /// Relies on [`from_owned_ptr_or_opt`](#method.from_owned_ptr_or_opt). - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" - ) - )] - unsafe fn from_owned_ptr(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { - #[allow(deprecated)] - Self::from_owned_ptr_or_panic(py, ptr) - } - /// Convert from an arbitrary `PyObject`. - /// - /// # Safety - /// - /// Relies on [`from_owned_ptr_or_opt`](#method.from_owned_ptr_or_opt). - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_owned_ptr_or_err(py, ptr)` or `Bound::from_owned_ptr_or_err(py, ptr)` instead" - ) - )] - unsafe fn from_owned_ptr_or_err(py: Python<'p>, ptr: *mut ffi::PyObject) -> PyResult<&'p Self> { - #[allow(deprecated)] - Self::from_owned_ptr_or_opt(py, ptr).ok_or_else(|| err::PyErr::fetch(py)) - } - /// Convert from an arbitrary borrowed `PyObject`. - /// - /// # Safety - /// - /// Implementations must ensure the object does not get freed during `'p` and avoid type confusion. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_borrowed_ptr_or_opt(py, ptr)` or `Bound::from_borrowed_ptr_or_opt(py, ptr)` instead" - ) - )] - unsafe fn from_borrowed_ptr_or_opt(py: Python<'p>, ptr: *mut ffi::PyObject) - -> Option<&'p Self>; - /// Convert from an arbitrary borrowed `PyObject`. - /// - /// # Safety - /// - /// Relies on unsafe fn [`from_borrowed_ptr_or_opt`](#method.from_borrowed_ptr_or_opt). - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_borrowed_ptr(py, ptr)` or `Bound::from_borrowed_ptr(py, ptr)` instead" - ) - )] - unsafe fn from_borrowed_ptr_or_panic(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { - #[allow(deprecated)] - Self::from_borrowed_ptr_or_opt(py, ptr).unwrap_or_else(|| err::panic_after_error(py)) - } - /// Convert from an arbitrary borrowed `PyObject`. - /// - /// # Safety - /// - /// Relies on unsafe fn [`from_borrowed_ptr_or_opt`](#method.from_borrowed_ptr_or_opt). - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_borrowed_ptr(py, ptr)` or `Bound::from_borrowed_ptr(py, ptr)` instead" - ) - )] - unsafe fn from_borrowed_ptr(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { - #[allow(deprecated)] - Self::from_borrowed_ptr_or_panic(py, ptr) - } - /// Convert from an arbitrary borrowed `PyObject`. - /// - /// # Safety - /// - /// Relies on unsafe fn [`from_borrowed_ptr_or_opt`](#method.from_borrowed_ptr_or_opt). - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_borrowed_ptr_or_err(py, ptr)` or `Bound::from_borrowed_ptr_or_err(py, ptr)` instead" - ) - )] - unsafe fn from_borrowed_ptr_or_err( - py: Python<'p>, - ptr: *mut ffi::PyObject, - ) -> PyResult<&'p Self> { - #[allow(deprecated)] - Self::from_borrowed_ptr_or_opt(py, ptr).ok_or_else(|| err::PyErr::fetch(py)) - } -} +impl<'py> IntoPyObject<'py> for () { + type Target = PyTuple; + type Output = Bound<'py, Self::Target>; + type Error = Infallible; -#[allow(deprecated)] -unsafe impl<'p, T> FromPyPointer<'p> for T -where - T: 'p + crate::PyNativeType, -{ - unsafe fn from_owned_ptr_or_opt(py: Python<'p>, ptr: *mut ffi::PyObject) -> Option<&'p Self> { - gil::register_owned(py, NonNull::new(ptr)?); - Some(&*(ptr as *mut Self)) - } - unsafe fn from_borrowed_ptr_or_opt( - _py: Python<'p>, - ptr: *mut ffi::PyObject, - ) -> Option<&'p Self> { - NonNull::new(ptr as *mut Self).map(|p| &*p.as_ptr()) + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(PyTuple::empty(py)) } } @@ -671,50 +613,3 @@ where /// }) /// ``` mod test_no_clone {} - -#[cfg(test)] -mod tests { - #[allow(deprecated)] - mod deprecated { - use super::super::PyTryFrom; - use crate::types::{IntoPyDict, PyAny, PyDict, PyList}; - use crate::{Python, ToPyObject}; - - #[test] - fn test_try_from() { - Python::with_gil(|py| { - let list: &PyAny = vec![3, 6, 5, 4, 7].to_object(py).into_ref(py); - let dict: &PyAny = vec![("reverse", true)].into_py_dict(py).as_ref(); - - assert!(>::try_from(list).is_ok()); - assert!(>::try_from(dict).is_ok()); - - assert!(>::try_from(list).is_ok()); - assert!(>::try_from(dict).is_ok()); - }); - } - - #[test] - fn test_try_from_exact() { - Python::with_gil(|py| { - let list: &PyAny = vec![3, 6, 5, 4, 7].to_object(py).into_ref(py); - let dict: &PyAny = vec![("reverse", true)].into_py_dict(py).as_ref(); - - assert!(PyList::try_from_exact(list).is_ok()); - assert!(PyDict::try_from_exact(dict).is_ok()); - - assert!(PyAny::try_from_exact(list).is_err()); - assert!(PyAny::try_from_exact(dict).is_err()); - }); - } - - #[test] - fn test_try_from_unchecked() { - Python::with_gil(|py| { - let list = PyList::new(py, [1, 2, 3]); - let val = unsafe { ::try_from_unchecked(list.as_ref()) }; - assert!(list.is(val)); - }); - } - } -} diff --git a/src/conversions/anyhow.rs b/src/conversions/anyhow.rs index 623ee7d548c..d2cb3f3eb60 100644 --- a/src/conversions/anyhow.rs +++ b/src/conversions/anyhow.rs @@ -1,9 +1,6 @@ #![cfg(feature = "anyhow")] -//! A conversion from -//! [anyhow](https://docs.rs/anyhow/ "A trait object based error system for easy idiomatic error handling in Rust applications.")’s -//! [`Error`](https://docs.rs/anyhow/latest/anyhow/struct.Error.html "Anyhows `Error` type, a wrapper around a dynamic error type") -//! type to [`PyErr`]. +//! A conversion from [anyhow]’s [`Error`][anyhow-error] type to [`PyErr`]. //! //! Use of an error handling library like [anyhow] is common in application code and when you just //! want error handling to be easy. If you are writing a library or you need more control over your @@ -47,7 +44,7 @@ //! //! fn main() { //! let error = Python::with_gil(|py| -> PyResult> { -//! let fun = wrap_pyfunction_bound!(py_open, py)?; +//! let fun = wrap_pyfunction!(py_open, py)?; //! let text = fun.call1(("foo.txt",))?.extract::>()?; //! Ok(text) //! }).unwrap_err(); @@ -74,9 +71,9 @@ //! // could call inside an application... //! // This might return a `PyErr`. //! let res = Python::with_gil(|py| { -//! let zlib = PyModule::import_bound(py, "zlib")?; +//! let zlib = PyModule::import(py, "zlib")?; //! let decompress = zlib.getattr("decompress")?; -//! let bytes = PyBytes::new_bound(py, bytes); +//! let bytes = PyBytes::new(py, bytes); //! let value = decompress.call1((bytes,))?; //! value.extract::>() //! })?; @@ -99,6 +96,8 @@ //! } //! ``` //! +//! [anyhow]: https://docs.rs/anyhow/ "A trait object based error system for easy idiomatic error handling in Rust applications." +//! [anyhow-error]: https://docs.rs/anyhow/latest/anyhow/struct.Error.html "Anyhows `Error` type, a wrapper around a dynamic error type" //! [`RuntimeError`]: https://docs.python.org/3/library/exceptions.html#RuntimeError "Built-in Exceptions — Python documentation" //! [Error handling]: https://doc.rust-lang.org/book/ch09-02-recoverable-errors-with-result.html "Recoverable Errors with Result - The Rust Programming Language" @@ -121,8 +120,8 @@ impl From for PyErr { #[cfg(test)] mod test_anyhow { use crate::exceptions::{PyRuntimeError, PyValueError}; - use crate::prelude::*; use crate::types::IntoPyDict; + use crate::{ffi, prelude::*}; use anyhow::{anyhow, bail, Context, Result}; @@ -146,9 +145,11 @@ mod test_anyhow { let pyerr = PyErr::from(err); Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict_bound(py); - let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); - assert_eq!(pyerr.value_bound(py).to_string(), expected_contents); + let locals = [("err", pyerr)].into_py_dict(py).unwrap(); + let pyerr = py + .run(ffi::c_str!("raise err"), None, Some(&locals)) + .unwrap_err(); + assert_eq!(pyerr.value(py).to_string(), expected_contents); }) } @@ -163,9 +164,11 @@ mod test_anyhow { let pyerr = PyErr::from(err); Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict_bound(py); - let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); - assert_eq!(pyerr.value_bound(py).to_string(), expected_contents); + let locals = [("err", pyerr)].into_py_dict(py).unwrap(); + let pyerr = py + .run(ffi::c_str!("raise err"), None, Some(&locals)) + .unwrap_err(); + assert_eq!(pyerr.value(py).to_string(), expected_contents); }) } diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 544d1cf2663..342ed659e22 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -19,59 +19,83 @@ //! # Example: Convert a `datetime.datetime` to chrono's `DateTime` //! //! ```rust -//! # // `chrono::Duration` has been renamed to `chrono::TimeDelta` and its constructors changed -//! # // TODO: upgrade to Chrono 0.4.35+ after upgrading our MSRV to 1.61+ -//! # #![allow(deprecated)] //! use chrono::{DateTime, Duration, TimeZone, Utc}; -//! use pyo3::{Python, ToPyObject}; +//! use pyo3::{Python, PyResult, IntoPyObject, types::PyAnyMethods}; //! -//! fn main() { +//! fn main() -> PyResult<()> { //! pyo3::prepare_freethreaded_python(); //! Python::with_gil(|py| { //! // Build some chrono values //! let chrono_datetime = Utc.with_ymd_and_hms(2022, 1, 1, 12, 0, 0).unwrap(); //! let chrono_duration = Duration::seconds(1); //! // Convert them to Python -//! let py_datetime = chrono_datetime.to_object(py); -//! let py_timedelta = chrono_duration.to_object(py); +//! let py_datetime = chrono_datetime.into_pyobject(py)?; +//! let py_timedelta = chrono_duration.into_pyobject(py)?; //! // Do an operation in Python -//! let py_sum = py_datetime.call_method1(py, "__add__", (py_timedelta,)).unwrap(); +//! let py_sum = py_datetime.call_method1("__add__", (py_timedelta,))?; //! // Convert back to Rust -//! let chrono_sum: DateTime = py_sum.extract(py).unwrap(); +//! let chrono_sum: DateTime = py_sum.extract()?; //! println!("DateTime: {}", chrono_datetime); -//! }); +//! Ok(()) +//! }) //! } //! ``` -// `chrono::Duration` has been renamed to `chrono::TimeDelta` and its constructors changed -// TODO: upgrade to Chrono 0.4.35+ after upgrading our MSRV to 1.61+ -#![allow(deprecated)] - +use crate::conversion::IntoPyObject; use crate::exceptions::{PyTypeError, PyUserWarning, PyValueError}; #[cfg(Py_LIMITED_API)] -use crate::sync::GILOnceCell; +use crate::intern; use crate::types::any::PyAnyMethods; #[cfg(not(Py_LIMITED_API))] use crate::types::datetime::timezone_from_offset; +#[cfg(Py_LIMITED_API)] +use crate::types::datetime_abi3::{check_type, timezone_utc, DatetimeTypes}; +#[cfg(Py_LIMITED_API)] +use crate::types::IntoPyDict; +use crate::types::PyNone; #[cfg(not(Py_LIMITED_API))] use crate::types::{ - timezone_utc_bound, PyDate, PyDateAccess, PyDateTime, PyDelta, PyDeltaAccess, PyTime, - PyTimeAccess, PyTzInfo, PyTzInfoAccess, + timezone_utc, PyDate, PyDateAccess, PyDateTime, PyDelta, PyDeltaAccess, PyTime, PyTimeAccess, + PyTzInfo, PyTzInfoAccess, }; -#[cfg(Py_LIMITED_API)] -use crate::{intern, DowncastError}; -use crate::{Bound, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject}; +use crate::{ffi, Bound, FromPyObject, IntoPyObjectExt, PyAny, PyErr, PyObject, PyResult, Python}; +#[allow(deprecated)] +use crate::{IntoPy, ToPyObject}; use chrono::offset::{FixedOffset, Utc}; use chrono::{ - DateTime, Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime, Offset, TimeZone, Timelike, + DateTime, Datelike, Duration, LocalResult, NaiveDate, NaiveDateTime, NaiveTime, Offset, + TimeZone, Timelike, }; +#[allow(deprecated)] impl ToPyObject for Duration { + #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { + self.into_pyobject(py).unwrap().into_any().unbind() + } +} + +#[allow(deprecated)] +impl IntoPy for Duration { + #[inline] + fn into_py(self, py: Python<'_>) -> PyObject { + self.into_pyobject(py).unwrap().into_any().unbind() + } +} + +impl<'py> IntoPyObject<'py> for Duration { + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyDelta; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { // Total number of days let days = self.num_days(); // Remainder of seconds - let secs_dur = *self - Duration::days(days); + let secs_dur = self - Duration::days(days); let secs = secs_dur.num_seconds(); // Fractional part of the microseconds let micros = (secs_dur - Duration::seconds(secs_dur.num_seconds())) @@ -87,29 +111,34 @@ impl ToPyObject for Duration { // We pass true as the `normalize` parameter since we'd need to do several checks here to // avoid that, and it shouldn't have a big performance impact. // The seconds and microseconds cast should never overflow since it's at most the number of seconds per day - PyDelta::new_bound( + PyDelta::new( py, days.try_into().unwrap_or(i32::MAX), - secs.try_into().unwrap(), - micros.try_into().unwrap(), + secs.try_into()?, + micros.try_into()?, true, ) - .expect("failed to construct delta") - .into() } + #[cfg(Py_LIMITED_API)] { - DatetimeTypes::get(py) - .timedelta - .call1(py, (days, secs, micros)) - .expect("failed to construct datetime.timedelta") + DatetimeTypes::try_get(py) + .and_then(|dt| dt.timedelta.bind(py).call1((days, secs, micros))) } } } -impl IntoPy for Duration { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) +impl<'py> IntoPyObject<'py> for &Duration { + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyDelta; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } @@ -145,28 +174,55 @@ impl FromPyObject<'_> for Duration { } } +#[allow(deprecated)] impl ToPyObject for NaiveDate { + #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { - let DateArgs { year, month, day } = self.into(); + self.into_pyobject(py).unwrap().into_any().unbind() + } +} + +#[allow(deprecated)] +impl IntoPy for NaiveDate { + #[inline] + fn into_py(self, py: Python<'_>) -> PyObject { + self.into_pyobject(py).unwrap().into_any().unbind() + } +} + +impl<'py> IntoPyObject<'py> for NaiveDate { + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyDate; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let DateArgs { year, month, day } = (&self).into(); #[cfg(not(Py_LIMITED_API))] { - PyDate::new_bound(py, year, month, day) - .expect("failed to construct date") - .into() + PyDate::new(py, year, month, day) } + #[cfg(Py_LIMITED_API)] { - DatetimeTypes::get(py) - .date - .call1(py, (year, month, day)) - .expect("failed to construct datetime.date") + DatetimeTypes::try_get(py).and_then(|dt| dt.date.bind(py).call1((year, month, day))) } } } -impl IntoPy for NaiveDate { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) +impl<'py> IntoPyObject<'py> for &NaiveDate { + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyDate; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } @@ -185,34 +241,65 @@ impl FromPyObject<'_> for NaiveDate { } } +#[allow(deprecated)] impl ToPyObject for NaiveTime { + #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { + self.into_pyobject(py).unwrap().into_any().unbind() + } +} + +#[allow(deprecated)] +impl IntoPy for NaiveTime { + #[inline] + fn into_py(self, py: Python<'_>) -> PyObject { + self.into_pyobject(py).unwrap().into_any().unbind() + } +} + +impl<'py> IntoPyObject<'py> for NaiveTime { + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyTime; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { let TimeArgs { hour, min, sec, micro, truncated_leap_second, - } = self.into(); + } = (&self).into(); + #[cfg(not(Py_LIMITED_API))] - let time = - PyTime::new_bound(py, hour, min, sec, micro, None).expect("Failed to construct time"); + let time = PyTime::new(py, hour, min, sec, micro, None)?; + #[cfg(Py_LIMITED_API)] - let time = DatetimeTypes::get(py) - .time - .bind(py) - .call1((hour, min, sec, micro)) - .expect("failed to construct datetime.time"); + let time = DatetimeTypes::try_get(py) + .and_then(|dt| dt.time.bind(py).call1((hour, min, sec, micro)))?; + if truncated_leap_second { warn_truncated_leap_second(&time); } - time.into() + + Ok(time) } } -impl IntoPy for NaiveTime { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) +impl<'py> IntoPyObject<'py> for &NaiveTime { + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyTime; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } @@ -231,15 +318,69 @@ impl FromPyObject<'_> for NaiveTime { } } +#[allow(deprecated)] impl ToPyObject for NaiveDateTime { + #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { - naive_datetime_to_py_datetime(py, self, None) + self.into_pyobject(py).unwrap().into_any().unbind() } } +#[allow(deprecated)] impl IntoPy for NaiveDateTime { + #[inline] fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) + self.into_pyobject(py).unwrap().into_any().unbind() + } +} + +impl<'py> IntoPyObject<'py> for NaiveDateTime { + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyDateTime; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let DateArgs { year, month, day } = (&self.date()).into(); + let TimeArgs { + hour, + min, + sec, + micro, + truncated_leap_second, + } = (&self.time()).into(); + + #[cfg(not(Py_LIMITED_API))] + let datetime = PyDateTime::new(py, year, month, day, hour, min, sec, micro, None)?; + + #[cfg(Py_LIMITED_API)] + let datetime = DatetimeTypes::try_get(py).and_then(|dt| { + dt.datetime + .bind(py) + .call1((year, month, day, hour, min, sec, micro)) + })?; + + if truncated_leap_second { + warn_truncated_leap_second(&datetime); + } + + Ok(datetime) + } +} + +impl<'py> IntoPyObject<'py> for &NaiveDateTime { + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyDateTime; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } @@ -254,7 +395,7 @@ impl FromPyObject<'_> for NaiveDateTime { // we return a hard error. We could silently remove tzinfo, or assume local timezone // and do a conversion, but better leave this decision to the user of the library. #[cfg(not(Py_LIMITED_API))] - let has_tzinfo = dt.get_tzinfo_bound().is_some(); + let has_tzinfo = dt.get_tzinfo().is_some(); #[cfg(Py_LIMITED_API)] let has_tzinfo = !dt.getattr(intern!(dt.py(), "tzinfo"))?.is_none(); if has_tzinfo { @@ -266,6 +407,7 @@ impl FromPyObject<'_> for NaiveDateTime { } } +#[allow(deprecated)] impl ToPyObject for DateTime { fn to_object(&self, py: Python<'_>) -> PyObject { // FIXME: convert to better timezone representation here than just convert to fixed offset @@ -276,12 +418,81 @@ impl ToPyObject for DateTime { } } +#[allow(deprecated)] impl IntoPy for DateTime { fn into_py(self, py: Python<'_>) -> PyObject { self.to_object(py) } } +impl<'py, Tz: TimeZone> IntoPyObject<'py> for DateTime +where + Tz: IntoPyObject<'py>, +{ + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyDateTime; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (&self).into_pyobject(py) + } +} + +impl<'py, Tz: TimeZone> IntoPyObject<'py> for &DateTime +where + Tz: IntoPyObject<'py>, +{ + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyDateTime; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let tz = self.timezone().into_bound_py_any(py)?; + + #[cfg(not(Py_LIMITED_API))] + let tz = tz.downcast()?; + + let DateArgs { year, month, day } = (&self.naive_local().date()).into(); + let TimeArgs { + hour, + min, + sec, + micro, + truncated_leap_second, + } = (&self.naive_local().time()).into(); + + let fold = matches!( + self.timezone().offset_from_local_datetime(&self.naive_local()), + LocalResult::Ambiguous(_, latest) if self.offset().fix() == latest.fix() + ); + + #[cfg(not(Py_LIMITED_API))] + let datetime = + PyDateTime::new_with_fold(py, year, month, day, hour, min, sec, micro, Some(tz), fold)?; + + #[cfg(Py_LIMITED_API)] + let datetime = DatetimeTypes::try_get(py).and_then(|dt| { + dt.datetime.bind(py).call( + (year, month, day, hour, min, sec, micro, tz), + Some(&[("fold", fold as u8)].into_py_dict(py)?), + ) + })?; + + if truncated_leap_second { + warn_truncated_leap_second(&datetime); + } + + Ok(datetime) + } +} + impl FromPyObject<'py>> FromPyObject<'_> for DateTime { fn extract_bound(dt: &Bound<'_, PyAny>) -> PyResult> { #[cfg(not(Py_LIMITED_API))] @@ -290,7 +501,7 @@ impl FromPyObject<'py>> FromPyObject<'_> for DateTime> = dt.getattr(intern!(dt.py(), "tzinfo"))?.extract()?; @@ -302,41 +513,80 @@ impl FromPyObject<'py>> FromPyObject<'_> for DateTime Ok(value), + LocalResult::Ambiguous(earliest, latest) => { + #[cfg(not(Py_LIMITED_API))] + let fold = dt.get_fold(); + + #[cfg(Py_LIMITED_API)] + let fold = dt.getattr(intern!(dt.py(), "fold"))?.extract::()? > 0; + + if fold { + Ok(latest) + } else { + Ok(earliest) + } + } + LocalResult::None => Err(PyValueError::new_err(format!( + "The datetime {:?} contains an incompatible timezone", dt - )) - }) + ))), + } } } +#[allow(deprecated)] impl ToPyObject for FixedOffset { + #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { - let seconds_offset = self.local_minus_utc(); + self.into_pyobject(py).unwrap().into_any().unbind() + } +} +#[allow(deprecated)] +impl IntoPy for FixedOffset { + #[inline] + fn into_py(self, py: Python<'_>) -> PyObject { + self.into_pyobject(py).unwrap().into_any().unbind() + } +} + +impl<'py> IntoPyObject<'py> for FixedOffset { + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyTzInfo; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let seconds_offset = self.local_minus_utc(); #[cfg(not(Py_LIMITED_API))] { - let td = PyDelta::new_bound(py, 0, seconds_offset, 0, true) - .expect("failed to construct timedelta"); + let td = PyDelta::new(py, 0, seconds_offset, 0, true)?; timezone_from_offset(&td) - .expect("Failed to construct PyTimezone") - .into() } + #[cfg(Py_LIMITED_API)] { - let td = Duration::seconds(seconds_offset.into()).into_py(py); - DatetimeTypes::get(py) - .timezone - .call1(py, (td,)) - .expect("failed to construct datetime.timezone") + let td = Duration::seconds(seconds_offset.into()).into_pyobject(py)?; + DatetimeTypes::try_get(py).and_then(|dt| dt.timezone.bind(py).call1((td,))) } } } -impl IntoPy for FixedOffset { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) +impl<'py> IntoPyObject<'py> for &FixedOffset { + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyTzInfo; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } @@ -347,16 +597,16 @@ impl FromPyObject<'_> for FixedOffset { /// does not supports microseconds. fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { #[cfg(not(Py_LIMITED_API))] - let ob: &PyTzInfo = ob.extract()?; + let ob = ob.downcast::()?; #[cfg(Py_LIMITED_API)] check_type(ob, &DatetimeTypes::get(ob.py()).tzinfo, "PyTzInfo")?; - // Passing `()` (so Python's None) to the `utcoffset` function will only + // Passing Python's None to the `utcoffset` function will only // work for timezones defined as fixed offsets in Python. // Any other timezone would require a datetime as the parameter, and return // None if the datetime is not provided. // Trying to convert None to a PyDelta in the next line will then fail. - let py_timedelta = ob.call_method1("utcoffset", ((),))?; + let py_timedelta = ob.call_method1("utcoffset", (PyNone::get(ob.py()),))?; if py_timedelta.is_none() { return Err(PyTypeError::new_err(format!( "{:?} is not a fixed offset timezone", @@ -371,21 +621,59 @@ impl FromPyObject<'_> for FixedOffset { } } +#[allow(deprecated)] impl ToPyObject for Utc { + #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { - timezone_utc_bound(py).into() + self.into_pyobject(py).unwrap().into_any().unbind() } } +#[allow(deprecated)] impl IntoPy for Utc { + #[inline] fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) + self.into_pyobject(py).unwrap().into_any().unbind() + } +} + +impl<'py> IntoPyObject<'py> for Utc { + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyTzInfo; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + #[cfg(Py_LIMITED_API)] + { + Ok(timezone_utc(py).into_any()) + } + #[cfg(not(Py_LIMITED_API))] + { + Ok(timezone_utc(py)) + } + } +} + +impl<'py> IntoPyObject<'py> for &Utc { + #[cfg(Py_LIMITED_API)] + type Target = PyAny; + #[cfg(not(Py_LIMITED_API))] + type Target = PyTzInfo; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } impl FromPyObject<'_> for Utc { fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { - let py_utc = timezone_utc_bound(ob.py()); + let py_utc = timezone_utc(ob.py()); if ob.eq(py_utc)? { Ok(Utc) } else { @@ -449,7 +737,7 @@ fn naive_datetime_to_py_datetime( truncated_leap_second, } = (&naive_datetime.time()).into(); #[cfg(not(Py_LIMITED_API))] - let datetime = PyDateTime::new_bound(py, year, month, day, hour, min, sec, micro, tzinfo) + let datetime = PyDateTime::new(py, year, month, day, hour, min, sec, micro, tzinfo) .expect("failed to construct datetime"); #[cfg(Py_LIMITED_API)] let datetime = DatetimeTypes::get(py) @@ -465,13 +753,13 @@ fn naive_datetime_to_py_datetime( fn warn_truncated_leap_second(obj: &Bound<'_, PyAny>) { let py = obj.py(); - if let Err(e) = PyErr::warn_bound( + if let Err(e) = PyErr::warn( py, - &py.get_type_bound::(), - "ignored leap-second, `datetime` does not support leap-seconds", + &py.get_type::(), + ffi::c_str!("ignored leap-second, `datetime` does not support leap-seconds"), 0, ) { - e.write_unraisable_bound(py, Some(&obj.as_borrowed())) + e.write_unraisable(py, Some(obj)) }; } @@ -523,56 +811,10 @@ fn py_time_to_naive_time(py_time: &Bound<'_, PyAny>) -> PyResult { .ok_or_else(|| PyValueError::new_err("invalid or out-of-range time")) } -#[cfg(Py_LIMITED_API)] -fn check_type(value: &Bound<'_, PyAny>, t: &PyObject, type_name: &'static str) -> PyResult<()> { - if !value.is_instance(t.bind(value.py()))? { - return Err(DowncastError::new(value, type_name).into()); - } - Ok(()) -} - -#[cfg(Py_LIMITED_API)] -struct DatetimeTypes { - date: PyObject, - datetime: PyObject, - time: PyObject, - timedelta: PyObject, - timezone: PyObject, - timezone_utc: PyObject, - tzinfo: PyObject, -} - -#[cfg(Py_LIMITED_API)] -impl DatetimeTypes { - fn get(py: Python<'_>) -> &Self { - static TYPES: GILOnceCell = GILOnceCell::new(); - TYPES - .get_or_try_init(py, || { - let datetime = py.import_bound("datetime")?; - let timezone = datetime.getattr("timezone")?; - Ok::<_, PyErr>(Self { - date: datetime.getattr("date")?.into(), - datetime: datetime.getattr("datetime")?.into(), - time: datetime.getattr("time")?.into(), - timedelta: datetime.getattr("timedelta")?.into(), - timezone_utc: timezone.getattr("utc")?.into(), - timezone: timezone.into(), - tzinfo: datetime.getattr("tzinfo")?.into(), - }) - }) - .expect("failed to load datetime module") - } -} - -#[cfg(Py_LIMITED_API)] -fn timezone_utc_bound(py: Python<'_>) -> Bound<'_, PyAny> { - DatetimeTypes::get(py).timezone_utc.bind(py).clone() -} - #[cfg(test)] mod tests { use super::*; - use crate::{types::PyTuple, Py}; + use crate::{types::PyTuple, BoundObject}; use std::{cmp::Ordering, panic}; #[test] @@ -581,13 +823,14 @@ mod tests { // tzdata there to make this work. #[cfg(all(Py_3_9, not(target_os = "windows")))] fn test_zoneinfo_is_not_fixed_offset() { + use crate::ffi; use crate::types::any::PyAnyMethods; use crate::types::dict::PyDictMethods; Python::with_gil(|py| { - let locals = crate::types::PyDict::new_bound(py); - py.run_bound( - "import zoneinfo; zi = zoneinfo.ZoneInfo('Europe/London')", + let locals = crate::types::PyDict::new(py); + py.run( + ffi::c_str!("import zoneinfo; zi = zoneinfo.ZoneInfo('Europe/London')"), None, Some(&locals), ) @@ -596,7 +839,7 @@ mod tests { assert!(result.is_err()); let res = result.err().unwrap(); // Also check the error message is what we expect - let msg = res.value_bound(py).repr().unwrap().to_string(); + let msg = res.value(py).repr().unwrap().to_string(); assert_eq!(msg, "TypeError(\"zoneinfo.ZoneInfo(key='Europe/London') is not a fixed offset timezone\")"); }); } @@ -611,7 +854,7 @@ mod tests { // Now test that converting a PyDateTime with tzinfo to a NaiveDateTime fails let res: PyResult = py_datetime.extract(); assert_eq!( - res.unwrap_err().value_bound(py).repr().unwrap().to_string(), + res.unwrap_err().value(py).repr().unwrap().to_string(), "TypeError('expected a datetime without tzinfo')" ); }); @@ -626,14 +869,14 @@ mod tests { // Now test that converting a PyDateTime with tzinfo to a NaiveDateTime fails let res: PyResult> = py_datetime.extract(); assert_eq!( - res.unwrap_err().value_bound(py).repr().unwrap().to_string(), + res.unwrap_err().value(py).repr().unwrap().to_string(), "TypeError('expected a datetime with non-None tzinfo')" ); // Now test that converting a PyDateTime with tzinfo to a NaiveDateTime fails let res: PyResult> = py_datetime.extract(); assert_eq!( - res.unwrap_err().value_bound(py).repr().unwrap().to_string(), + res.unwrap_err().value(py).repr().unwrap().to_string(), "TypeError('expected a datetime with non-None tzinfo')" ); }); @@ -683,15 +926,15 @@ mod tests { } #[test] - fn test_pyo3_timedelta_topyobject() { + fn test_pyo3_timedelta_into_pyobject() { // Utility function used to check different durations. // The `name` parameter is used to identify the check in case of a failure. let check = |name: &'static str, delta: Duration, py_days, py_seconds, py_ms| { Python::with_gil(|py| { - let delta = delta.to_object(py); + let delta = delta.into_pyobject(py).unwrap(); let py_delta = new_py_datetime_ob(py, "timedelta", (py_days, py_seconds, py_ms)); assert!( - delta.bind(py).eq(&py_delta).unwrap(), + delta.eq(&py_delta).unwrap(), "{}: {} != {}", name, delta, @@ -712,10 +955,14 @@ mod tests { let delta = Duration::seconds(86399999999999) + Duration::nanoseconds(999999000); // max check("delta max value", delta, 999999999, 86399, 999999); - // Also check that trying to convert an out of bound value panics. + // Also check that trying to convert an out of bound value errors. Python::with_gil(|py| { - assert!(panic::catch_unwind(|| Duration::min_value().to_object(py)).is_err()); - assert!(panic::catch_unwind(|| Duration::max_value().to_object(py)).is_err()); + // min_value and max_value were deprecated in chrono 0.4.39 + #[allow(deprecated)] + { + assert!(Duration::min_value().into_pyobject(py).is_err()); + assert!(Duration::max_value().into_pyobject(py).is_err()); + } }); } @@ -779,15 +1026,16 @@ mod tests { } #[test] - fn test_pyo3_date_topyobject() { + fn test_pyo3_date_into_pyobject() { let eq_ymd = |name: &'static str, year, month, day| { Python::with_gil(|py| { let date = NaiveDate::from_ymd_opt(year, month, day) .unwrap() - .to_object(py); + .into_pyobject(py) + .unwrap(); let py_date = new_py_datetime_ob(py, "date", (year, month, day)); assert_eq!( - date.bind(py).compare(&py_date).unwrap(), + date.compare(&py_date).unwrap(), Ordering::Equal, "{}: {} != {}", name, @@ -821,7 +1069,7 @@ mod tests { } #[test] - fn test_pyo3_datetime_topyobject_utc() { + fn test_pyo3_datetime_into_pyobject_utc() { Python::with_gil(|py| { let check_utc = |name: &'static str, year, month, day, hour, minute, second, ms, py_ms| { @@ -830,7 +1078,7 @@ mod tests { .and_hms_micro_opt(hour, minute, second, ms) .unwrap() .and_utc(); - let datetime = datetime.to_object(py); + let datetime = datetime.into_pyobject(py).unwrap(); let py_datetime = new_py_datetime_ob( py, "datetime", @@ -846,7 +1094,7 @@ mod tests { ), ); assert_eq!( - datetime.bind(py).compare(&py_datetime).unwrap(), + datetime.compare(&py_datetime).unwrap(), Ordering::Equal, "{}: {} != {}", name, @@ -857,6 +1105,7 @@ mod tests { check_utc("regular", 2014, 5, 6, 7, 8, 9, 999_999, 999_999); + #[cfg(not(Py_GIL_DISABLED))] assert_warnings!( py, check_utc("leap second", 2014, 5, 6, 7, 8, 59, 1_999_999, 999_999), @@ -869,7 +1118,7 @@ mod tests { } #[test] - fn test_pyo3_datetime_topyobject_fixed_offset() { + fn test_pyo3_datetime_into_pyobject_fixed_offset() { Python::with_gil(|py| { let check_fixed_offset = |name: &'static str, year, month, day, hour, minute, second, ms, py_ms| { @@ -880,15 +1129,15 @@ mod tests { .unwrap() .and_local_timezone(offset) .unwrap(); - let datetime = datetime.to_object(py); - let py_tz = offset.to_object(py); + let datetime = datetime.into_pyobject(py).unwrap(); + let py_tz = offset.into_pyobject(py).unwrap(); let py_datetime = new_py_datetime_ob( py, "datetime", (year, month, day, hour, minute, second, py_ms, py_tz), ); assert_eq!( - datetime.bind(py).compare(&py_datetime).unwrap(), + datetime.compare(&py_datetime).unwrap(), Ordering::Equal, "{}: {} != {}", name, @@ -899,6 +1148,7 @@ mod tests { check_fixed_offset("regular", 2014, 5, 6, 7, 8, 9, 999_999, 999_999); + #[cfg(not(Py_GIL_DISABLED))] assert_warnings!( py, check_fixed_offset("leap second", 2014, 5, 6, 7, 8, 59, 1_999_999, 999_999), @@ -910,6 +1160,35 @@ mod tests { }) } + #[test] + #[cfg(all(Py_3_9, feature = "chrono-tz", not(windows)))] + fn test_pyo3_datetime_into_pyobject_tz() { + Python::with_gil(|py| { + let datetime = NaiveDate::from_ymd_opt(2024, 12, 11) + .unwrap() + .and_hms_opt(23, 3, 13) + .unwrap() + .and_local_timezone(chrono_tz::Tz::Europe__London) + .unwrap(); + let datetime = datetime.into_pyobject(py).unwrap(); + let py_datetime = new_py_datetime_ob( + py, + "datetime", + ( + 2024, + 12, + 11, + 23, + 3, + 13, + 0, + python_zoneinfo(py, "Europe/London"), + ), + ); + assert_eq!(datetime.compare(&py_datetime).unwrap(), Ordering::Equal); + }) + } + #[test] fn test_pyo3_datetime_frompyobject_utc() { Python::with_gil(|py| { @@ -920,7 +1199,7 @@ mod tests { let minute = 8; let second = 9; let micro = 999_999; - let tz_utc = timezone_utc_bound(py); + let tz_utc = timezone_utc(py); let py_datetime = new_py_datetime_ob( py, "datetime", @@ -947,7 +1226,7 @@ mod tests { let second = 9; let micro = 999_999; let offset = FixedOffset::east_opt(3600).unwrap(); - let py_tz = offset.to_object(py); + let py_tz = offset.into_pyobject(py).unwrap(); let py_datetime = new_py_datetime_ob( py, "datetime", @@ -980,21 +1259,27 @@ mod tests { } #[test] - fn test_pyo3_offset_fixed_topyobject() { + fn test_pyo3_offset_fixed_into_pyobject() { Python::with_gil(|py| { // Chrono offset - let offset = FixedOffset::east_opt(3600).unwrap().to_object(py); + let offset = FixedOffset::east_opt(3600) + .unwrap() + .into_pyobject(py) + .unwrap(); // Python timezone from timedelta let td = new_py_datetime_ob(py, "timedelta", (0, 3600, 0)); let py_timedelta = new_py_datetime_ob(py, "timezone", (td,)); // Should be equal - assert!(offset.bind(py).eq(py_timedelta).unwrap()); + assert!(offset.eq(py_timedelta).unwrap()); // Same but with negative values - let offset = FixedOffset::east_opt(-3600).unwrap().to_object(py); + let offset = FixedOffset::east_opt(-3600) + .unwrap() + .into_pyobject(py) + .unwrap(); let td = new_py_datetime_ob(py, "timedelta", (0, -3600, 0)); let py_timedelta = new_py_datetime_ob(py, "timezone", (td,)); - assert!(offset.bind(py).eq(py_timedelta).unwrap()); + assert!(offset.eq(py_timedelta).unwrap()); }) } @@ -1009,11 +1294,11 @@ mod tests { } #[test] - fn test_pyo3_offset_utc_topyobject() { + fn test_pyo3_offset_utc_into_pyobject() { Python::with_gil(|py| { - let utc = Utc.to_object(py); + let utc = Utc.into_pyobject(py).unwrap(); let py_utc = python_utc(py); - assert!(utc.bind(py).is(&py_utc)); + assert!(utc.is(&py_utc)); }) } @@ -1036,15 +1321,16 @@ mod tests { } #[test] - fn test_pyo3_time_topyobject() { + fn test_pyo3_time_into_pyobject() { Python::with_gil(|py| { let check_time = |name: &'static str, hour, minute, second, ms, py_ms| { let time = NaiveTime::from_hms_micro_opt(hour, minute, second, ms) .unwrap() - .to_object(py); + .into_pyobject(py) + .unwrap(); let py_time = new_py_datetime_ob(py, "time", (hour, minute, second, py_ms)); assert!( - time.bind(py).eq(&py_time).unwrap(), + time.eq(&py_time).unwrap(), "{}: {} != {}", name, time, @@ -1054,6 +1340,7 @@ mod tests { check_time("regular", 3, 5, 7, 999_999, 999_999); + #[cfg(not(Py_GIL_DISABLED))] assert_warnings!( py, check_time("leap second", 3, 5, 59, 1_999_999, 999_999), @@ -1079,21 +1366,25 @@ mod tests { }) } - fn new_py_datetime_ob<'py>( - py: Python<'py>, - name: &str, - args: impl IntoPy>, - ) -> Bound<'py, PyAny> { - py.import_bound("datetime") + fn new_py_datetime_ob<'py, A>(py: Python<'py>, name: &str, args: A) -> Bound<'py, PyAny> + where + A: IntoPyObject<'py, Target = PyTuple>, + { + py.import("datetime") .unwrap() .getattr(name) .unwrap() - .call1(args) + .call1( + args.into_pyobject(py) + .map_err(Into::into) + .unwrap() + .into_bound(), + ) .unwrap() } fn python_utc(py: Python<'_>) -> Bound<'_, PyAny> { - py.import_bound("datetime") + py.import("datetime") .unwrap() .getattr("timezone") .unwrap() @@ -1101,12 +1392,23 @@ mod tests { .unwrap() } - #[cfg(not(target_arch = "wasm32"))] + #[cfg(all(Py_3_9, feature = "chrono-tz", not(windows)))] + fn python_zoneinfo<'py>(py: Python<'py>, timezone: &str) -> Bound<'py, PyAny> { + py.import("zoneinfo") + .unwrap() + .getattr("ZoneInfo") + .unwrap() + .call1((timezone,)) + .unwrap() + } + + #[cfg(not(any(target_arch = "wasm32", Py_GIL_DISABLED)))] mod proptests { use super::*; use crate::tests::common::CatchWarnings; use crate::types::IntoPyDict; use proptest::prelude::*; + use std::ffi::CString; proptest! { @@ -1115,9 +1417,9 @@ mod tests { fn test_pyo3_offset_fixed_frompyobject_created_in_python(timestamp in 0..(i32::MAX as i64), timedelta in -86399i32..=86399i32) { Python::with_gil(|py| { - let globals = [("datetime", py.import_bound("datetime").unwrap())].into_py_dict_bound(py); + let globals = [("datetime", py.import("datetime").unwrap())].into_py_dict(py).unwrap(); let code = format!("datetime.datetime.fromtimestamp({}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={})))", timestamp, timedelta); - let t = py.eval_bound(&code, Some(&globals), None).unwrap(); + let t = py.eval(&CString::new(code).unwrap(), Some(&globals), None).unwrap(); // Get ISO 8601 string from python let py_iso_str = t.call_method0("isoformat").unwrap(); @@ -1142,8 +1444,8 @@ mod tests { // python values of durations (from -999999999 to 999999999 days), Python::with_gil(|py| { let dur = Duration::days(days); - let py_delta = dur.into_py(py); - let roundtripped: Duration = py_delta.extract(py).expect("Round trip"); + let py_delta = dur.into_pyobject(py).unwrap(); + let roundtripped: Duration = py_delta.extract().expect("Round trip"); assert_eq!(dur, roundtripped); }) } @@ -1152,8 +1454,8 @@ mod tests { fn test_fixed_offset_roundtrip(secs in -86399i32..=86399i32) { Python::with_gil(|py| { let offset = FixedOffset::east_opt(secs).unwrap(); - let py_offset = offset.into_py(py); - let roundtripped: FixedOffset = py_offset.extract(py).expect("Round trip"); + let py_offset = offset.into_pyobject(py).unwrap(); + let roundtripped: FixedOffset = py_offset.extract().expect("Round trip"); assert_eq!(offset, roundtripped); }) } @@ -1170,8 +1472,8 @@ mod tests { // We use to `from_ymd_opt` constructor so that we only test valid `NaiveDate`s. // This is to skip the test if we are creating an invalid date, like February 31. if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) { - let py_date = date.to_object(py); - let roundtripped: NaiveDate = py_date.extract(py).expect("Round trip"); + let py_date = date.into_pyobject(py).unwrap(); + let roundtripped: NaiveDate = py_date.extract().expect("Round trip"); assert_eq!(date, roundtripped); } }) @@ -1191,8 +1493,8 @@ mod tests { Python::with_gil(|py| { if let Some(time) = NaiveTime::from_hms_micro_opt(hour, min, sec, micro) { // Wrap in CatchWarnings to avoid to_object firing warning for truncated leap second - let py_time = CatchWarnings::enter(py, |_| Ok(time.to_object(py))).unwrap(); - let roundtripped: NaiveTime = py_time.extract(py).expect("Round trip"); + let py_time = CatchWarnings::enter(py, |_| time.into_pyobject(py)).unwrap(); + let roundtripped: NaiveTime = py_time.extract().expect("Round trip"); // Leap seconds are not roundtripped let expected_roundtrip_time = micro.checked_sub(1_000_000).map(|micro| NaiveTime::from_hms_micro_opt(hour, min, sec, micro).unwrap()).unwrap_or(time); assert_eq!(expected_roundtrip_time, roundtripped); @@ -1215,8 +1517,8 @@ mod tests { let time_opt = NaiveTime::from_hms_micro_opt(hour, min, sec, micro); if let (Some(date), Some(time)) = (date_opt, time_opt) { let dt = NaiveDateTime::new(date, time); - let pydt = dt.to_object(py); - let roundtripped: NaiveDateTime = pydt.extract(py).expect("Round trip"); + let pydt = dt.into_pyobject(py).unwrap(); + let roundtripped: NaiveDateTime = pydt.extract().expect("Round trip"); assert_eq!(dt, roundtripped); } }) @@ -1238,8 +1540,8 @@ mod tests { if let (Some(date), Some(time)) = (date_opt, time_opt) { let dt: DateTime = NaiveDateTime::new(date, time).and_utc(); // Wrap in CatchWarnings to avoid into_py firing warning for truncated leap second - let py_dt = CatchWarnings::enter(py, |_| Ok(dt.into_py(py))).unwrap(); - let roundtripped: DateTime = py_dt.extract(py).expect("Round trip"); + let py_dt = CatchWarnings::enter(py, |_| dt.into_pyobject(py)).unwrap(); + let roundtripped: DateTime = py_dt.extract().expect("Round trip"); // Leap seconds are not roundtripped let expected_roundtrip_time = micro.checked_sub(1_000_000).map(|micro| NaiveTime::from_hms_micro_opt(hour, min, sec, micro).unwrap()).unwrap_or(time); let expected_roundtrip_dt: DateTime = NaiveDateTime::new(date, expected_roundtrip_time).and_utc(); @@ -1266,8 +1568,8 @@ mod tests { if let (Some(date), Some(time)) = (date_opt, time_opt) { let dt: DateTime = NaiveDateTime::new(date, time).and_local_timezone(offset).unwrap(); // Wrap in CatchWarnings to avoid into_py firing warning for truncated leap second - let py_dt = CatchWarnings::enter(py, |_| Ok(dt.into_py(py))).unwrap(); - let roundtripped: DateTime = py_dt.extract(py).expect("Round trip"); + let py_dt = CatchWarnings::enter(py, |_| dt.into_pyobject(py)).unwrap(); + let roundtripped: DateTime = py_dt.extract().expect("Round trip"); // Leap seconds are not roundtripped let expected_roundtrip_time = micro.checked_sub(1_000_000).map(|micro| NaiveTime::from_hms_micro_opt(hour, min, sec, micro).unwrap()).unwrap_or(time); let expected_roundtrip_dt: DateTime = NaiveDateTime::new(date, expected_roundtrip_time).and_local_timezone(offset).unwrap(); diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs index 845814c4dab..60a3bab4918 100644 --- a/src/conversions/chrono_tz.rs +++ b/src/conversions/chrono_tz.rs @@ -21,43 +21,67 @@ //! //! ```rust,no_run //! use chrono_tz::Tz; -//! use pyo3::{Python, ToPyObject}; +//! use pyo3::{Python, PyResult, IntoPyObject, types::PyAnyMethods}; //! -//! fn main() { +//! fn main() -> PyResult<()> { //! pyo3::prepare_freethreaded_python(); //! Python::with_gil(|py| { //! // Convert to Python -//! let py_tzinfo = Tz::Europe__Paris.to_object(py); +//! let py_tzinfo = Tz::Europe__Paris.into_pyobject(py)?; //! // Convert back to Rust -//! assert_eq!(py_tzinfo.extract::(py).unwrap(), Tz::Europe__Paris); -//! }); +//! assert_eq!(py_tzinfo.extract::()?, Tz::Europe__Paris); +//! Ok(()) +//! }) //! } //! ``` +use crate::conversion::IntoPyObject; use crate::exceptions::PyValueError; use crate::pybacked::PyBackedStr; use crate::sync::GILOnceCell; use crate::types::{any::PyAnyMethods, PyType}; -use crate::{ - intern, Bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, -}; +use crate::{intern, Bound, FromPyObject, Py, PyAny, PyErr, PyObject, PyResult, Python}; +#[allow(deprecated)] +use crate::{IntoPy, ToPyObject}; use chrono_tz::Tz; use std::str::FromStr; +#[allow(deprecated)] impl ToPyObject for Tz { + #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { - static ZONE_INFO: GILOnceCell> = GILOnceCell::new(); - ZONE_INFO - .get_or_try_init_type_ref(py, "zoneinfo", "ZoneInfo") - .unwrap() - .call1((self.name(),)) - .unwrap() - .unbind() + self.into_pyobject(py).unwrap().unbind() } } +#[allow(deprecated)] impl IntoPy for Tz { + #[inline] fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) + self.into_pyobject(py).unwrap().unbind() + } +} + +impl<'py> IntoPyObject<'py> for Tz { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + static ZONE_INFO: GILOnceCell> = GILOnceCell::new(); + ZONE_INFO + .import(py, "zoneinfo", "ZoneInfo") + .and_then(|obj| obj.call1((self.name(),))) + } +} + +impl<'py> IntoPyObject<'py> for &Tz { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } @@ -74,6 +98,10 @@ impl FromPyObject<'_> for Tz { #[cfg(all(test, not(windows)))] // Troubles loading timezones on Windows mod tests { use super::*; + use crate::prelude::PyAnyMethods; + use crate::Python; + use chrono::{DateTime, Utc}; + use chrono_tz::Tz; #[test] fn test_frompyobject() { @@ -91,19 +119,68 @@ mod tests { } #[test] - fn test_topyobject() { + fn test_ambiguous_datetime_to_pyobject() { + let dates = [ + DateTime::::from_str("2020-10-24 23:00:00 UTC").unwrap(), + DateTime::::from_str("2020-10-25 00:00:00 UTC").unwrap(), + DateTime::::from_str("2020-10-25 01:00:00 UTC").unwrap(), + ]; + + let dates = dates.map(|dt| dt.with_timezone(&Tz::Europe__London)); + + assert_eq!( + dates.map(|dt| dt.to_string()), + [ + "2020-10-25 00:00:00 BST", + "2020-10-25 01:00:00 BST", + "2020-10-25 01:00:00 GMT" + ] + ); + + let dates = Python::with_gil(|py| { + let pydates = dates.map(|dt| dt.into_pyobject(py).unwrap()); + assert_eq!( + pydates + .clone() + .map(|dt| dt.getattr("hour").unwrap().extract::().unwrap()), + [0, 1, 1] + ); + + assert_eq!( + pydates + .clone() + .map(|dt| dt.getattr("fold").unwrap().extract::().unwrap() > 0), + [false, false, true] + ); + + pydates.map(|dt| dt.extract::>().unwrap()) + }); + + assert_eq!( + dates.map(|dt| dt.to_string()), + [ + "2020-10-25 00:00:00 BST", + "2020-10-25 01:00:00 BST", + "2020-10-25 01:00:00 GMT" + ] + ); + } + + #[test] + #[cfg(not(Py_GIL_DISABLED))] // https://github.com/python/cpython/issues/116738#issuecomment-2404360445 + fn test_into_pyobject() { Python::with_gil(|py| { - let assert_eq = |l: PyObject, r: Bound<'_, PyAny>| { - assert!(l.bind(py).eq(r).unwrap()); + let assert_eq = |l: Bound<'_, PyAny>, r: Bound<'_, PyAny>| { + assert!(l.eq(&r).unwrap(), "{:?} != {:?}", l, r); }; assert_eq( - Tz::Europe__Paris.to_object(py), + Tz::Europe__Paris.into_pyobject(py).unwrap(), new_zoneinfo(py, "Europe/Paris"), ); - assert_eq(Tz::UTC.to_object(py), new_zoneinfo(py, "UTC")); + assert_eq(Tz::UTC.into_pyobject(py).unwrap(), new_zoneinfo(py, "UTC")); assert_eq( - Tz::Etc__GMTMinus5.to_object(py), + Tz::Etc__GMTMinus5.into_pyobject(py).unwrap(), new_zoneinfo(py, "Etc/GMT-5"), ); }); @@ -114,9 +191,6 @@ mod tests { } fn zoneinfo_class(py: Python<'_>) -> Bound<'_, PyAny> { - py.import_bound("zoneinfo") - .unwrap() - .getattr("ZoneInfo") - .unwrap() + py.import("zoneinfo").unwrap().getattr("ZoneInfo").unwrap() } } diff --git a/src/conversions/either.rs b/src/conversions/either.rs index 84ec88ea009..a514b1fde8d 100644 --- a/src/conversions/either.rs +++ b/src/conversions/either.rs @@ -26,18 +26,19 @@ //! //! ```rust //! use either::Either; -//! use pyo3::{Python, ToPyObject}; +//! use pyo3::{Python, PyResult, IntoPyObject, types::PyAnyMethods}; //! -//! fn main() { +//! fn main() -> PyResult<()> { //! pyo3::prepare_freethreaded_python(); //! Python::with_gil(|py| { //! // Create a string and an int in Python. -//! let py_str = "crab".to_object(py); -//! let py_int = 42.to_object(py); +//! let py_str = "crab".into_pyobject(py)?; +//! let py_int = 42i32.into_pyobject(py)?; //! // Now convert it to an Either. -//! let either_str: Either = py_str.extract(py).unwrap(); -//! let either_int: Either = py_int.extract(py).unwrap(); -//! }); +//! let either_str: Either = py_str.extract()?; +//! let either_int: Either = py_int.extract()?; +//! Ok(()) +//! }) //! } //! ``` //! @@ -46,12 +47,15 @@ #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ - exceptions::PyTypeError, types::any::PyAnyMethods, Bound, FromPyObject, IntoPy, PyAny, - PyObject, PyResult, Python, ToPyObject, + exceptions::PyTypeError, types::any::PyAnyMethods, Bound, FromPyObject, IntoPyObject, + IntoPyObjectExt, PyAny, PyErr, PyObject, PyResult, Python, }; +#[allow(deprecated)] +use crate::{IntoPy, ToPyObject}; use either::Either; #[cfg_attr(docsrs, doc(cfg(feature = "either")))] +#[allow(deprecated)] impl IntoPy for Either where L: IntoPy, @@ -67,6 +71,43 @@ where } #[cfg_attr(docsrs, doc(cfg(feature = "either")))] +impl<'py, L, R> IntoPyObject<'py> for Either +where + L: IntoPyObject<'py>, + R: IntoPyObject<'py>, +{ + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + match self { + Either::Left(l) => l.into_bound_py_any(py), + Either::Right(r) => r.into_bound_py_any(py), + } + } +} + +#[cfg_attr(docsrs, doc(cfg(feature = "either")))] +impl<'a, 'py, L, R> IntoPyObject<'py> for &'a Either +where + &'a L: IntoPyObject<'py>, + &'a R: IntoPyObject<'py>, +{ + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + match self { + Either::Left(l) => l.into_bound_py_any(py), + Either::Right(r) => r.into_bound_py_any(py), + } + } +} + +#[cfg_attr(docsrs, doc(cfg(feature = "either")))] +#[allow(deprecated)] impl ToPyObject for Either where L: ToPyObject, @@ -116,8 +157,9 @@ mod tests { use std::borrow::Cow; use crate::exceptions::PyTypeError; - use crate::{Python, ToPyObject}; + use crate::{IntoPyObject, Python}; + use crate::types::PyAnyMethods; use either::Either; #[test] @@ -128,30 +170,30 @@ mod tests { Python::with_gil(|py| { let l = E::Left(42); - let obj_l = l.to_object(py); - assert_eq!(obj_l.extract::(py).unwrap(), 42); - assert_eq!(obj_l.extract::(py).unwrap(), l); + let obj_l = (&l).into_pyobject(py).unwrap(); + assert_eq!(obj_l.extract::().unwrap(), 42); + assert_eq!(obj_l.extract::().unwrap(), l); let r = E::Right("foo".to_owned()); - let obj_r = r.to_object(py); - assert_eq!(obj_r.extract::>(py).unwrap(), "foo"); - assert_eq!(obj_r.extract::(py).unwrap(), r); + let obj_r = (&r).into_pyobject(py).unwrap(); + assert_eq!(obj_r.extract::>().unwrap(), "foo"); + assert_eq!(obj_r.extract::().unwrap(), r); - let obj_s = "foo".to_object(py); - let err = obj_s.extract::(py).unwrap_err(); + let obj_s = "foo".into_pyobject(py).unwrap(); + let err = obj_s.extract::().unwrap_err(); assert!(err.is_instance_of::(py)); assert_eq!( err.to_string(), "TypeError: failed to convert the value to 'Union[i32, f32]'" ); - let obj_i = 42.to_object(py); - assert_eq!(obj_i.extract::(py).unwrap(), E1::Left(42)); - assert_eq!(obj_i.extract::(py).unwrap(), E2::Left(42.0)); + let obj_i = 42i32.into_pyobject(py).unwrap(); + assert_eq!(obj_i.extract::().unwrap(), E1::Left(42)); + assert_eq!(obj_i.extract::().unwrap(), E2::Left(42.0)); - let obj_f = 42.0.to_object(py); - assert_eq!(obj_f.extract::(py).unwrap(), E1::Right(42.0)); - assert_eq!(obj_f.extract::(py).unwrap(), E2::Left(42.0)); + let obj_f = 42.0f64.into_pyobject(py).unwrap(); + assert_eq!(obj_f.extract::().unwrap(), E1::Right(42.0)); + assert_eq!(obj_f.extract::().unwrap(), E2::Left(42.0)); }); } } diff --git a/src/conversions/eyre.rs b/src/conversions/eyre.rs index d4704e411c5..42d7a12c872 100644 --- a/src/conversions/eyre.rs +++ b/src/conversions/eyre.rs @@ -46,7 +46,7 @@ //! //! fn main() { //! let error = Python::with_gil(|py| -> PyResult> { -//! let fun = wrap_pyfunction_bound!(py_open, py)?; +//! let fun = wrap_pyfunction!(py_open, py)?; //! let text = fun.call1(("foo.txt",))?.extract::>()?; //! Ok(text) //! }).unwrap_err(); @@ -73,9 +73,9 @@ //! // could call inside an application... //! // This might return a `PyErr`. //! let res = Python::with_gil(|py| { -//! let zlib = PyModule::import_bound(py, "zlib")?; +//! let zlib = PyModule::import(py, "zlib")?; //! let decompress = zlib.getattr("decompress")?; -//! let bytes = PyBytes::new_bound(py, bytes); +//! let bytes = PyBytes::new(py, bytes); //! let value = decompress.call1((bytes,))?; //! value.extract::>() //! })?; @@ -126,8 +126,8 @@ impl From for PyErr { #[cfg(test)] mod tests { use crate::exceptions::{PyRuntimeError, PyValueError}; - use crate::prelude::*; use crate::types::IntoPyDict; + use crate::{ffi, prelude::*}; use eyre::{bail, eyre, Report, Result, WrapErr}; @@ -151,9 +151,11 @@ mod tests { let pyerr = PyErr::from(err); Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict_bound(py); - let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); - assert_eq!(pyerr.value_bound(py).to_string(), expected_contents); + let locals = [("err", pyerr)].into_py_dict(py).unwrap(); + let pyerr = py + .run(ffi::c_str!("raise err"), None, Some(&locals)) + .unwrap_err(); + assert_eq!(pyerr.value(py).to_string(), expected_contents); }) } @@ -168,9 +170,11 @@ mod tests { let pyerr = PyErr::from(err); Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict_bound(py); - let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); - assert_eq!(pyerr.value_bound(py).to_string(), expected_contents); + let locals = [("err", pyerr)].into_py_dict(py).unwrap(); + let pyerr = py + .run(ffi::c_str!("raise err"), None, Some(&locals)) + .unwrap_err(); + assert_eq!(pyerr.value(py).to_string(), expected_contents); }) } diff --git a/src/conversions/hashbrown.rs b/src/conversions/hashbrown.rs index 9eea7734bfc..0efe7f5161f 100644 --- a/src/conversions/hashbrown.rs +++ b/src/conversions/hashbrown.rs @@ -17,15 +17,21 @@ //! Note that you must use compatible versions of hashbrown and PyO3. //! The required hashbrown version may vary based on the version of PyO3. use crate::{ - types::any::PyAnyMethods, - types::dict::PyDictMethods, - types::frozenset::PyFrozenSetMethods, - types::set::{new_from_iter, PySetMethods}, - types::{IntoPyDict, PyDict, PyFrozenSet, PySet}, - Bound, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject, + conversion::IntoPyObject, + types::{ + any::PyAnyMethods, + dict::PyDictMethods, + frozenset::PyFrozenSetMethods, + set::{new_from_iter, try_new_from_iter, PySetMethods}, + PyDict, PyFrozenSet, PySet, + }, + Bound, FromPyObject, PyAny, PyErr, PyObject, PyResult, Python, }; +#[allow(deprecated)] +use crate::{IntoPy, ToPyObject}; use std::{cmp, hash}; +#[allow(deprecated)] impl ToPyObject for hashbrown::HashMap where K: hash::Hash + cmp::Eq + ToPyObject, @@ -33,10 +39,15 @@ where H: hash::BuildHasher, { fn to_object(&self, py: Python<'_>) -> PyObject { - IntoPyDict::into_py_dict_bound(self, py).into() + let dict = PyDict::new(py); + for (k, v) in self { + dict.set_item(k.to_object(py), v.to_object(py)).unwrap(); + } + dict.into_any().unbind() } } +#[allow(deprecated)] impl IntoPy for hashbrown::HashMap where K: hash::Hash + cmp::Eq + IntoPy, @@ -44,10 +55,49 @@ where H: hash::BuildHasher, { fn into_py(self, py: Python<'_>) -> PyObject { - let iter = self - .into_iter() - .map(|(k, v)| (k.into_py(py), v.into_py(py))); - IntoPyDict::into_py_dict_bound(iter, py).into() + let dict = PyDict::new(py); + for (k, v) in self { + dict.set_item(k.into_py(py), v.into_py(py)).unwrap(); + } + dict.into_any().unbind() + } +} + +impl<'py, K, V, H> IntoPyObject<'py> for hashbrown::HashMap +where + K: IntoPyObject<'py> + cmp::Eq + hash::Hash, + V: IntoPyObject<'py>, + H: hash::BuildHasher, +{ + type Target = PyDict; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let dict = PyDict::new(py); + for (k, v) in self { + dict.set_item(k, v)?; + } + Ok(dict) + } +} + +impl<'a, 'py, K, V, H> IntoPyObject<'py> for &'a hashbrown::HashMap +where + &'a K: IntoPyObject<'py> + cmp::Eq + hash::Hash, + &'a V: IntoPyObject<'py>, + H: hash::BuildHasher, +{ + type Target = PyDict; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let dict = PyDict::new(py); + for (k, v) in self { + dict.set_item(k, v)?; + } + Ok(dict) } } @@ -67,6 +117,7 @@ where } } +#[allow(deprecated)] impl ToPyObject for hashbrown::HashSet where T: hash::Hash + Eq + ToPyObject, @@ -78,6 +129,7 @@ where } } +#[allow(deprecated)] impl IntoPy for hashbrown::HashSet where K: IntoPy + Eq + hash::Hash, @@ -90,6 +142,34 @@ where } } +impl<'py, K, H> IntoPyObject<'py> for hashbrown::HashSet +where + K: IntoPyObject<'py> + cmp::Eq + hash::Hash, + H: hash::BuildHasher, +{ + type Target = PySet; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + try_new_from_iter(py, self) + } +} + +impl<'a, 'py, K, H> IntoPyObject<'py> for &'a hashbrown::HashSet +where + &'a K: IntoPyObject<'py> + cmp::Eq + hash::Hash, + H: hash::BuildHasher, +{ + type Target = PySet; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + try_new_from_iter(py, self) + } +} + impl<'py, K, S> FromPyObject<'py> for hashbrown::HashSet where K: FromPyObject<'py> + cmp::Eq + hash::Hash, @@ -112,15 +192,15 @@ where #[cfg(test)] mod tests { use super::*; + use crate::types::IntoPyDict; #[test] - fn test_hashbrown_hashmap_to_python() { + fn test_hashbrown_hashmap_into_pyobject() { Python::with_gil(|py| { let mut map = hashbrown::HashMap::::new(); map.insert(1, 1); - let m = map.to_object(py); - let py_map = m.downcast_bound::(py).unwrap(); + let py_map = (&map).into_pyobject(py).unwrap(); assert!(py_map.len() == 1); assert!( @@ -135,27 +215,6 @@ mod tests { assert_eq!(map, py_map.extract().unwrap()); }); } - #[test] - fn test_hashbrown_hashmap_into_python() { - Python::with_gil(|py| { - let mut map = hashbrown::HashMap::::new(); - map.insert(1, 1); - - let m: PyObject = map.into_py(py); - let py_map = m.downcast_bound::(py).unwrap(); - - assert!(py_map.len() == 1); - assert!( - py_map - .get_item(1) - .unwrap() - .unwrap() - .extract::() - .unwrap() - == 1 - ); - }); - } #[test] fn test_hashbrown_hashmap_into_dict() { @@ -163,7 +222,7 @@ mod tests { let mut map = hashbrown::HashMap::::new(); map.insert(1, 1); - let py_map = map.into_py_dict_bound(py); + let py_map = map.into_py_dict(py).unwrap(); assert_eq!(py_map.len(), 1); assert_eq!( @@ -181,24 +240,24 @@ mod tests { #[test] fn test_extract_hashbrown_hashset() { Python::with_gil(|py| { - let set = PySet::new_bound(py, &[1, 2, 3, 4, 5]).unwrap(); + let set = PySet::new(py, [1, 2, 3, 4, 5]).unwrap(); let hash_set: hashbrown::HashSet = set.extract().unwrap(); assert_eq!(hash_set, [1, 2, 3, 4, 5].iter().copied().collect()); - let set = PyFrozenSet::new_bound(py, &[1, 2, 3, 4, 5]).unwrap(); + let set = PyFrozenSet::new(py, [1, 2, 3, 4, 5]).unwrap(); let hash_set: hashbrown::HashSet = set.extract().unwrap(); assert_eq!(hash_set, [1, 2, 3, 4, 5].iter().copied().collect()); }); } #[test] - fn test_hashbrown_hashset_into_py() { + fn test_hashbrown_hashset_into_pyobject() { Python::with_gil(|py| { let hs: hashbrown::HashSet = [1, 2, 3, 4, 5].iter().cloned().collect(); - let hso: PyObject = hs.clone().into_py(py); + let hso = hs.clone().into_pyobject(py).unwrap(); - assert_eq!(hs, hso.extract(py).unwrap()); + assert_eq!(hs, hso.extract().unwrap()); }); } } diff --git a/src/conversions/indexmap.rs b/src/conversions/indexmap.rs index fdbe057f32d..e3787e68091 100644 --- a/src/conversions/indexmap.rs +++ b/src/conversions/indexmap.rs @@ -87,10 +87,14 @@ //! # if another hash table was used, the order could be random //! ``` +use crate::conversion::IntoPyObject; use crate::types::*; -use crate::{Bound, FromPyObject, IntoPy, PyErr, PyObject, Python, ToPyObject}; +use crate::{Bound, FromPyObject, PyErr, PyObject, Python}; +#[allow(deprecated)] +use crate::{IntoPy, ToPyObject}; use std::{cmp, hash}; +#[allow(deprecated)] impl ToPyObject for indexmap::IndexMap where K: hash::Hash + cmp::Eq + ToPyObject, @@ -98,10 +102,15 @@ where H: hash::BuildHasher, { fn to_object(&self, py: Python<'_>) -> PyObject { - IntoPyDict::into_py_dict_bound(self, py).into() + let dict = PyDict::new(py); + for (k, v) in self { + dict.set_item(k.to_object(py), v.to_object(py)).unwrap(); + } + dict.into_any().unbind() } } +#[allow(deprecated)] impl IntoPy for indexmap::IndexMap where K: hash::Hash + cmp::Eq + IntoPy, @@ -109,10 +118,49 @@ where H: hash::BuildHasher, { fn into_py(self, py: Python<'_>) -> PyObject { - let iter = self - .into_iter() - .map(|(k, v)| (k.into_py(py), v.into_py(py))); - IntoPyDict::into_py_dict_bound(iter, py).into() + let dict = PyDict::new(py); + for (k, v) in self { + dict.set_item(k.into_py(py), v.into_py(py)).unwrap(); + } + dict.into_any().unbind() + } +} + +impl<'py, K, V, H> IntoPyObject<'py> for indexmap::IndexMap +where + K: IntoPyObject<'py> + cmp::Eq + hash::Hash, + V: IntoPyObject<'py>, + H: hash::BuildHasher, +{ + type Target = PyDict; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let dict = PyDict::new(py); + for (k, v) in self { + dict.set_item(k, v)?; + } + Ok(dict) + } +} + +impl<'a, 'py, K, V, H> IntoPyObject<'py> for &'a indexmap::IndexMap +where + &'a K: IntoPyObject<'py> + cmp::Eq + hash::Hash, + &'a V: IntoPyObject<'py>, + H: hash::BuildHasher, +{ + type Target = PyDict; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let dict = PyDict::new(py); + for (k, v) in self { + dict.set_item(k, v)?; + } + Ok(dict) } } @@ -136,16 +184,15 @@ where mod test_indexmap { use crate::types::*; - use crate::{IntoPy, PyObject, Python, ToPyObject}; + use crate::{IntoPyObject, Python}; #[test] - fn test_indexmap_indexmap_to_python() { + fn test_indexmap_indexmap_into_pyobject() { Python::with_gil(|py| { let mut map = indexmap::IndexMap::::new(); map.insert(1, 1); - let m = map.to_object(py); - let py_map = m.downcast_bound::(py).unwrap(); + let py_map = (&map).into_pyobject(py).unwrap(); assert!(py_map.len() == 1); assert!( @@ -164,35 +211,13 @@ mod test_indexmap { }); } - #[test] - fn test_indexmap_indexmap_into_python() { - Python::with_gil(|py| { - let mut map = indexmap::IndexMap::::new(); - map.insert(1, 1); - - let m: PyObject = map.into_py(py); - let py_map = m.downcast_bound::(py).unwrap(); - - assert!(py_map.len() == 1); - assert!( - py_map - .get_item(1) - .unwrap() - .unwrap() - .extract::() - .unwrap() - == 1 - ); - }); - } - #[test] fn test_indexmap_indexmap_into_dict() { Python::with_gil(|py| { let mut map = indexmap::IndexMap::::new(); map.insert(1, 1); - let py_map = map.into_py_dict_bound(py); + let py_map = map.into_py_dict(py).unwrap(); assert_eq!(py_map.len(), 1); assert_eq!( @@ -221,7 +246,7 @@ mod test_indexmap { } } - let py_map = map.clone().into_py_dict_bound(py); + let py_map = (&map).into_py_dict(py).unwrap(); let trip_map = py_map.extract::>().unwrap(); diff --git a/src/conversions/jiff.rs b/src/conversions/jiff.rs new file mode 100644 index 00000000000..23ffddf99eb --- /dev/null +++ b/src/conversions/jiff.rs @@ -0,0 +1,1340 @@ +#![cfg(feature = "jiff-02")] + +//! Conversions to and from [jiff](https://docs.rs/jiff/)’s `Span`, `SignedDuration`, `TimeZone`, +//! `Offset`, `Date`, `Time`, `DateTime`, `Zoned`, and `Timestamp`. +//! +//! # Setup +//! +//! To use this feature, add this to your **`Cargo.toml`**: +//! +//! ```toml +//! [dependencies] +//! jiff = "0.2" +#![doc = concat!("pyo3 = { version = \"", env!("CARGO_PKG_VERSION"), "\", features = [\"jiff-02\"] }")] +//! ``` +//! +//! Note that you must use compatible versions of jiff and PyO3. +//! The required jiff version may vary based on the version of PyO3. Jiff also requires a MSRV +//! of 1.70. +//! +//! # Example: Convert a `datetime.datetime` to jiff `Zoned` +//! +//! ```rust +//! # #![cfg_attr(windows, allow(unused_imports))] +//! # use jiff_02 as jiff; +//! use jiff::{Zoned, SignedDuration, ToSpan}; +//! use pyo3::{Python, PyResult, IntoPyObject, types::PyAnyMethods}; +//! +//! # #[cfg(windows)] +//! # fn main() -> () {} +//! # #[cfg(not(windows))] +//! fn main() -> PyResult<()> { +//! pyo3::prepare_freethreaded_python(); +//! Python::with_gil(|py| { +//! // Build some jiff values +//! let jiff_zoned = Zoned::now(); +//! let jiff_span = 1.second(); +//! // Convert them to Python +//! let py_datetime = jiff_zoned.into_pyobject(py)?; +//! let py_timedelta = SignedDuration::try_from(jiff_span)?.into_pyobject(py)?; +//! // Do an operation in Python +//! let py_sum = py_datetime.call_method1("__add__", (py_timedelta,))?; +//! // Convert back to Rust +//! let jiff_sum: Zoned = py_sum.extract()?; +//! println!("Zoned: {}", jiff_sum); +//! Ok(()) +//! }) +//! } +//! ``` +use crate::exceptions::{PyTypeError, PyValueError}; +use crate::pybacked::PyBackedStr; +use crate::sync::GILOnceCell; +#[cfg(not(Py_LIMITED_API))] +use crate::types::datetime::timezone_from_offset; +#[cfg(Py_LIMITED_API)] +use crate::types::datetime_abi3::{check_type, timezone_utc, DatetimeTypes}; +#[cfg(Py_LIMITED_API)] +use crate::types::IntoPyDict; +#[cfg(not(Py_LIMITED_API))] +use crate::types::{ + timezone_utc, PyDate, PyDateAccess, PyDateTime, PyDelta, PyDeltaAccess, PyTime, PyTimeAccess, + PyTzInfo, PyTzInfoAccess, +}; +use crate::types::{PyAnyMethods, PyNone, PyType}; +use crate::{intern, Bound, FromPyObject, IntoPyObject, Py, PyAny, PyErr, PyResult, Python}; +use jiff::civil::{Date, DateTime, Time}; +use jiff::tz::{Offset, TimeZone}; +use jiff::{SignedDuration, Span, Timestamp, Zoned}; +#[cfg(feature = "jiff-02")] +use jiff_02 as jiff; + +#[cfg(not(Py_LIMITED_API))] +fn datetime_to_pydatetime<'py>( + py: Python<'py>, + datetime: &DateTime, + fold: bool, + timezone: Option<&TimeZone>, +) -> PyResult> { + PyDateTime::new_with_fold( + py, + datetime.year().into(), + datetime.month().try_into()?, + datetime.day().try_into()?, + datetime.hour().try_into()?, + datetime.minute().try_into()?, + datetime.second().try_into()?, + (datetime.subsec_nanosecond() / 1000).try_into()?, + timezone + .map(|tz| tz.into_pyobject(py)) + .transpose()? + .as_ref(), + fold, + ) +} + +#[cfg(Py_LIMITED_API)] +fn datetime_to_pydatetime<'py>( + py: Python<'py>, + datetime: &DateTime, + fold: bool, + timezone: Option<&TimeZone>, +) -> PyResult> { + DatetimeTypes::try_get(py)?.datetime.bind(py).call( + ( + datetime.year(), + datetime.month(), + datetime.day(), + datetime.hour(), + datetime.minute(), + datetime.second(), + datetime.subsec_nanosecond() / 1000, + timezone, + ), + Some(&[("fold", fold as u8)].into_py_dict(py)?), + ) +} + +#[cfg(not(Py_LIMITED_API))] +fn pytime_to_time(time: &impl PyTimeAccess) -> PyResult