diff --git a/.github/setup-rust-windows/action.yml b/.github/setup-rust-windows/action.yml new file mode 100644 index 00000000000000..97e0288592b146 --- /dev/null +++ b/.github/setup-rust-windows/action.yml @@ -0,0 +1,38 @@ +name: Set up Rust and LLVM for Windows +description: >- + Install the pinned Rust toolchain with Windows cross-compile targets and + a version of LLVM whose libclang is loadable by bindgen. + +inputs: + arch: + description: CPU architecture (x64, Win32, arm64) + required: true + +runs: + using: composite + steps: + - uses: dtolnay/rust-toolchain@1.91.1 + with: + targets: i686-pc-windows-msvc,x86_64-pc-windows-msvc,aarch64-pc-windows-msvc + # LIBCLANG_PATH must be set explicitly so the vcxproj uses the LLVM we + # install here rather than the VS-bundled LLVM whose clang headers have + # AVX intrinsic bugs. The values are hardcoded literals (no attacker- + # controlled input), so the GITHUB_ENV writes are safe. + - name: Install LLVM for bindgen + if: inputs.arch != 'arm64' + shell: cmd + run: | # zizmor: ignore[github-env] + choco install llvm --allow-downgrade --no-progress --version 21.1.0 + if not exist "C:\Program Files\LLVM\bin\libclang.dll" exit /b 1 + echo LIBCLANG_PATH=C:\Program Files\LLVM\bin>> "%GITHUB_ENV%" + # Chocolatey's LLVM package only ships x64 binaries, which an ARM64-native + # cargo process cannot load. Install the official ARM64 build directly. + - name: Install LLVM for bindgen (ARM64) + if: inputs.arch == 'arm64' + shell: pwsh + run: | # zizmor: ignore[github-env] + $installer = Join-Path $env:RUNNER_TEMP 'LLVM-21.1.0-woa64.exe' + Invoke-WebRequest 'https://github.com/llvm/llvm-project/releases/download/llvmorg-21.1.0/LLVM-21.1.0-woa64.exe' -OutFile $installer + Start-Process -Wait -FilePath $installer -ArgumentList '/S','/D=C:\Program Files\LLVM' + if (!(Test-Path 'C:\Program Files\LLVM\bin\libclang.dll')) { exit 1 } + echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $env:GITHUB_ENV diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a0f60c30ac8a60..ab9a73627ae1df 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -383,6 +383,9 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false + - uses: dtolnay/rust-toolchain@1.91.1 + with: + targets: ${{ matrix.arch }}-linux-android - name: Build and test run: ./Android/android.py ci --fast-ci ${{ matrix.arch }}-linux-android @@ -396,7 +399,9 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - + - uses: dtolnay/rust-toolchain@1.91.1 + with: + targets: aarch64-apple-ios-sim # GitHub recommends explicitly selecting the desired Xcode version: # https://github.com/actions/runner-images/issues/12541#issuecomment-3083850140 # This became a necessity as a result of diff --git a/.github/workflows/cargo.yml b/.github/workflows/cargo.yml new file mode 100644 index 00000000000000..c9fb50043d33a5 --- /dev/null +++ b/.github/workflows/cargo.yml @@ -0,0 +1,27 @@ +name: cargo + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + timeout-minutes: 15 + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + - run: ./configure + - run: cargo test + - run: cargo fmt --check + - run: cargo clippy diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 3349eb042425dd..dc4210c59e195b 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -100,7 +100,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.11' - # PCbuild downloads LLVM automatically: - name: Windows if: runner.os == 'Windows' diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 8f412288f530bc..b0462a82ed34a1 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -48,6 +48,9 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.x' + - uses: dtolnay/rust-toolchain@1.91.1 + with: + targets: wasm32-wasip1 - name: "Runner image version" run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: "Configure build Python" diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index c95e40a38095f9..af9139d161f6b3 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -26,6 +26,9 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false + - uses: ./.github/setup-rust-windows + with: + arch: ${{ inputs.arch }} - name: Build CPython installer run: ./Tools/msi/build.bat --doc -"${ARCH}" shell: bash diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 0648b770753255..8132d205f9bf62 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -29,6 +29,9 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false + - uses: ./.github/setup-rust-windows + with: + arch: ${{ inputs.arch }} - name: Register MSVC problem matcher if: inputs.arch != 'Win32' run: echo "::add-matcher::.github/problem-matchers/msvc.json" diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index e99e317182eaa6..795e2722d65144 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -78,7 +78,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Native Windows (debug) if: runner.os == 'Windows' && matrix.architecture != 'ARM64' shell: cmd diff --git a/.gitignore b/.gitignore index 2bf4925647ddcd..228455787e5fdf 100644 --- a/.gitignore +++ b/.gitignore @@ -178,3 +178,6 @@ CLAUDE.local.md #### main branch only stuff below this line, things to backport go above. #### # main branch only: ABI files are not checked/maintained. Doc/data/python*.abi + +# Rust build artifacts +/target/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000000000..132b913b06d5e5 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,251 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "_base64" +version = "0.1.0" +dependencies = [ + "cpython-build-helper", + "cpython-sys", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cpython-build-helper" +version = "0.1.0" +dependencies = [ + "shlex", +] + +[[package]] +name = "cpython-rust-staticlib" +version = "0.1.0" +dependencies = [ + "_base64", +] + +[[package]] +name = "cpython-sys" +version = "0.1.0" +dependencies = [ + "bindgen", + "shlex", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "libc" +version = "0.2.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.110" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000000000..768d289fd77653 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] +resolver = "3" +members = [ + "Modules/_base64", "Modules/cpython-build-helper", "Modules/cpython-rust-staticlib", + "Modules/cpython-sys" +] diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index fe669bb04df02a..fd9750eae80445 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -3261,6 +3261,7 @@ def test_basic_multiple_interpreters_main_no_reset(self): # * m_copy was copied from interp2 (was from interp1) # * module's global state was updated, not reset + @unittest.skip("gh-131229: This is suddenly very flaky") @no_rerun(reason="rerun not possible; module state is never cleared (see gh-102251)") @requires_subinterpreters def test_basic_multiple_interpreters_deleted_no_reset(self): diff --git a/Makefile.pre.in b/Makefile.pre.in index dd28ff5d2a3ed1..40d67c1e220134 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -58,6 +58,14 @@ DTRACE_HEADERS= @DTRACE_HEADERS@ DTRACE_OBJS= @DTRACE_OBJS@ DSYMUTIL= @DSYMUTIL@ DSYMUTIL_PATH= @DSYMUTIL_PATH@ +CARGO_HOME=@CARGO_HOME@ +CARGO_TARGET_DIR=@CARGO_TARGET_DIR@ +CARGO_PROFILE=@CARGO_PROFILE@ +CARGO_TARGET=@CARGO_TARGET@ +CARGO_TARGET_LINKER_ENV=@CARGO_TARGET_LINKER_ENV@ +CARGO_DYLIB_SUFFIX=@CARGO_DYLIB_SUFFIX@ +LLVM_TARGET=@LLVM_TARGET@ +RUST_STATICLIB_DEP=@RUST_STATICLIB_DEP@ GNULD= @GNULD@ @@ -174,7 +182,8 @@ CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION) SHLIB_SUFFIX= @SHLIB_SUFFIX@ EXT_SUFFIX= @EXT_SUFFIX@ LDSHARED= @LDSHARED@ $(PY_LDFLAGS) -BLDSHARED= @BLDSHARED@ $(PY_CORE_LDFLAGS) +BLDSHARED_EXE= @BLDSHARED_EXE@ +BLDSHARED_ARGS= @BLDSHARED_ARGS@ $(PY_CORE_LDFLAGS) LDCXXSHARED= @LDCXXSHARED@ $(PY_LDFLAGS) DESTSHARED= $(BINLIBDEST)/lib-dynload @@ -984,7 +993,7 @@ clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py -f $(srcdir)/Lib/test/clinic.test.c # Build the interpreter -$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) +$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) $(RUST_STATICLIB_DEP) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt @@ -1017,16 +1026,16 @@ $(LIBRARY): $(LIBRARY_OBJS) libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS) # AIX Linker don't support "-h" option if test "$(MACHDEP)" != "aix"; then \ - $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(BLDSHARED_EXE) $(BLDSHARED_ARGS) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ else \ - $(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(BLDSHARED_EXE) $(BLDSHARED_ARGS) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ fi if test $(INSTSONAME) != $@; then \ $(LN) -f $(INSTSONAME) $@; \ fi libpython3.so: libpython$(LDVERSION).so - $(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^ + $(BLDSHARED_EXE) $(BLDSHARED_ARGS) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^ libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ @@ -3254,8 +3263,15 @@ profile-removal: rm -f profile-run-stamp rm -f profile-bolt-stamp +.PHONY: clean-rust +clean-rust: + @if test @CARGO_HOME@ != ''; then \ + echo Running cargo clean...; \ + $(CARGO_HOME)/bin/cargo clean; \ + fi + .PHONY: clean -clean: clean-retain-profile clean-bolt +clean: clean-retain-profile clean-bolt clean-rust @if test @DEF_MAKE_ALL_RULE@ = profile-opt -o @DEF_MAKE_ALL_RULE@ = bolt-opt; then \ rm -f profile-gen-stamp profile-clean-stamp; \ $(MAKE) profile-removal; \ @@ -3362,6 +3378,14 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h ########################################################################## # Module dependencies and platform-specific files +cpython-sys: Modules/cpython-sys/Cargo.toml Modules/cpython-sys/build.rs Modules/cpython-sys/wrapper.h Modules/cpython-sys/parser.h + CARGO_TARGET_DIR=$(abs_builddir)/target PYTHON_BUILD_DIR=$(abs_builddir) PY_CC="$(CC)" PY_CPPFLAGS="$(CPPFLAGS)" PY_CFLAGS="$(CFLAGS)" LLVM_TARGET="$(LLVM_TARGET)" IPHONEOS_DEPLOYMENT_TARGET=$(IPHONEOS_DEPLOYMENT_TARGET) $(CARGO_HOME)/bin/cargo build --lib --locked --package cpython-sys --profile $(CARGO_PROFILE) $(if $(CARGO_TARGET),--target=$(CARGO_TARGET)) --manifest-path $(srcdir)/Cargo.toml + +RUST_STATICLIB_A= target/$(if $(CARGO_TARGET),$(CARGO_TARGET)/$(CARGO_TARGET_DIR),$(CARGO_TARGET_DIR))/libcpython_rust_staticlib.a + +cpython-rust-staticlib: cpython-sys + CARGO_TARGET_DIR=$(abs_builddir)/target PYTHON_BUILD_DIR=$(abs_builddir) $(CARGO_HOME)/bin/cargo build --lib --locked --package cpython-rust-staticlib --profile $(CARGO_PROFILE) $(if $(CARGO_TARGET),--target=$(CARGO_TARGET)) --manifest-path $(srcdir)/Cargo.toml + # force rebuild when header file or module build flavor (static/shared) is changed MODULE_DEPS_STATIC=Modules/config.c MODULE_DEPS_SHARED=@MODULE_DEPS_SHARED@ diff --git a/Misc/platform_triplet.c b/Misc/platform_triplet.c index f5cd73bdea8333..f6d584189eb4ec 100644 --- a/Misc/platform_triplet.c +++ b/Misc/platform_triplet.c @@ -1,5 +1,12 @@ /* Detect platform triplet from builtin defines * cc -E Misc/platform_triplet.c | grep '^PLATFORM_TRIPLET=' | tr -d ' ' + * + * Also detects the closest Rust target triple for cargo/rustc: + * cc -E Misc/platform_triplet.c | grep '^RUST_TARGET=' | tr -d ' ' + * + * And the LLVM/clang target triple (for bindgen), only when it differs + * from RUST_TARGET: + * cc -E Misc/platform_triplet.c | grep '^LLVM_TARGET=' | tr -d ' ' */ #undef bfin #undef cris @@ -16,12 +23,16 @@ #if defined(__ANDROID__) # if defined(__x86_64__) PLATFORM_TRIPLET=x86_64-linux-android +RUST_TARGET=x86_64-linux-android # elif defined(__i386__) PLATFORM_TRIPLET=i686-linux-android +RUST_TARGET=i686-linux-android # elif defined(__aarch64__) PLATFORM_TRIPLET=aarch64-linux-android +RUST_TARGET=aarch64-linux-android # elif defined(__arm__) PLATFORM_TRIPLET=arm-linux-androideabi +RUST_TARGET=arm-linux-androideabi # else # error unknown Android platform # endif @@ -39,18 +50,25 @@ PLATFORM_TRIPLET=arm-linux-androideabi # elif defined(__GLIBC__) # define LIBC gnu # define LIBC_X32 gnux32 +# define RUST_LIBC gnu +# define RUST_LIBC_X32 gnux32 # if defined(__ARM_PCS_VFP) # define LIBC_ARM gnueabihf +# define RUST_LIBC_ARM gnueabihf # else # define LIBC_ARM gnueabi +# define RUST_LIBC_ARM gnueabi # endif # if defined(__loongarch__) # if defined(__loongarch_soft_float) # define LIBC_LA gnusf +# define RUST_LIBC_LA unknown # elif defined(__loongarch_single_float) # define LIBC_LA gnuf32 +# define RUST_LIBC_LA unknown # elif defined(__loongarch_double_float) # define LIBC_LA gnu +# define RUST_LIBC_LA gnu # else # error unknown loongarch floating-point base abi # endif @@ -59,20 +77,26 @@ PLATFORM_TRIPLET=arm-linux-androideabi # if defined(__mips_hard_float) # if defined(_ABIO32) && _MIPS_SIM == _ABIO32 # define LIBC_MIPS gnu +# define RUST_LIBC_MIPS gnu # elif defined(_ABIN32) && _MIPS_SIM == _ABIN32 # define LIBC_MIPS gnuabin32 +# define RUST_LIBC_MIPS unknown # elif defined(_ABI64) && _MIPS_SIM == _ABI64 # define LIBC_MIPS gnuabi64 +# define RUST_LIBC_MIPS gnuabi64 # else # error unknown mips sim value # endif # else # if defined(_ABIO32) && _MIPS_SIM == _ABIO32 # define LIBC_MIPS gnusf +# define RUST_LIBC_MIPS unknown # elif defined(_ABIN32) && _MIPS_SIM == _ABIN32 # define LIBC_MIPS gnuabin32sf +# define RUST_LIBC_MIPS unknown # elif defined(_ABI64) && _MIPS_SIM == _ABI64 # define LIBC_MIPS gnuabi64sf +# define RUST_LIBC_MIPS unknown # else # error unknown mips sim value # endif @@ -80,8 +104,10 @@ PLATFORM_TRIPLET=arm-linux-androideabi # endif # if defined(__SPE__) # define LIBC_PPC gnuspe +# define RUST_LIBC_PPC gnuspe # else # define LIBC_PPC gnu +# define RUST_LIBC_PPC gnu # endif # else // Heuristic to detect musl libc @@ -89,18 +115,25 @@ PLATFORM_TRIPLET=arm-linux-androideabi # ifdef __DEFINED_va_list # define LIBC musl # define LIBC_X32 muslx32 +# define RUST_LIBC musl +# define RUST_LIBC_X32 unknown # if defined(__ARM_PCS_VFP) # define LIBC_ARM musleabihf +# define RUST_LIBC_ARM musleabihf # else # define LIBC_ARM musleabi +# define RUST_LIBC_ARM musleabi # endif # if defined(__loongarch__) # if defined(__loongarch_soft_float) # define LIBC_LA muslsf +# define RUST_LIBC_LA unknown # elif defined(__loongarch_single_float) # define LIBC_LA muslf32 +# define RUST_LIBC_LA unknown # elif defined(__loongarch_double_float) # define LIBC_LA musl +# define RUST_LIBC_LA musl # else # error unknown loongarch floating-point base abi # endif @@ -109,20 +142,27 @@ PLATFORM_TRIPLET=arm-linux-androideabi # if defined(__mips_hard_float) # if defined(_ABIO32) && _MIPS_SIM == _ABIO32 # define LIBC_MIPS musl +# define RUST_LIBC_MIPS musl # elif defined(_ABIN32) && _MIPS_SIM == _ABIN32 # define LIBC_MIPS musln32 +# define RUST_LIBC_MIPS unknown # elif defined(_ABI64) && _MIPS_SIM == _ABI64 # define LIBC_MIPS musl +# define RUST_LIBC_MIPS muslabi64 +# define LLVM_LIBC_MIPS musl # else # error unknown mips sim value # endif # else # if defined(_ABIO32) && _MIPS_SIM == _ABIO32 # define LIBC_MIPS muslsf +# define RUST_LIBC_MIPS unknown # elif defined(_ABIN32) && _MIPS_SIM == _ABIN32 # define LIBC_MIPS musln32sf +# define RUST_LIBC_MIPS unknown # elif defined(_ABI64) && _MIPS_SIM == _ABI64 # define LIBC_MIPS muslsf +# define RUST_LIBC_MIPS unknown # else # error unknown mips sim value # endif @@ -130,8 +170,10 @@ PLATFORM_TRIPLET=arm-linux-androideabi # endif # if defined(_SOFT_FLOAT) || defined(__NO_FPRS__) # define LIBC_PPC muslsf +# define RUST_LIBC_PPC unknown # else # define LIBC_PPC musl +# define RUST_LIBC_PPC musl # endif # else # error unknown libc @@ -140,85 +182,126 @@ PLATFORM_TRIPLET=arm-linux-androideabi # if defined(__x86_64__) && defined(__LP64__) PLATFORM_TRIPLET=x86_64-linux-LIBC +RUST_TARGET=x86_64-unknown-linux-RUST_LIBC # elif defined(__x86_64__) && defined(__ILP32__) PLATFORM_TRIPLET=x86_64-linux-LIBC_X32 +RUST_TARGET=x86_64-unknown-linux-RUST_LIBC_X32 # elif defined(__i386__) PLATFORM_TRIPLET=i386-linux-LIBC +RUST_TARGET=i686-unknown-linux-RUST_LIBC # elif defined(__aarch64__) && defined(__AARCH64EL__) # if defined(__ILP32__) PLATFORM_TRIPLET=aarch64_ilp32-linux-LIBC +RUST_TARGET=unknown # else PLATFORM_TRIPLET=aarch64-linux-LIBC +RUST_TARGET=aarch64-unknown-linux-RUST_LIBC # endif # elif defined(__aarch64__) && defined(__AARCH64EB__) # if defined(__ILP32__) PLATFORM_TRIPLET=aarch64_be_ilp32-linux-LIBC +RUST_TARGET=unknown # else PLATFORM_TRIPLET=aarch64_be-linux-LIBC +RUST_TARGET=aarch64_be-unknown-linux-RUST_LIBC # endif # elif defined(__alpha__) PLATFORM_TRIPLET=alpha-linux-LIBC +RUST_TARGET=unknown # elif defined(__ARM_EABI__) # if defined(__ARMEL__) PLATFORM_TRIPLET=arm-linux-LIBC_ARM +RUST_TARGET=arm-unknown-linux-RUST_LIBC_ARM # else PLATFORM_TRIPLET=armeb-linux-LIBC_ARM +RUST_TARGET=armeb-unknown-linux-RUST_LIBC_ARM # endif # elif defined(__hppa__) PLATFORM_TRIPLET=hppa-linux-LIBC +RUST_TARGET=unknown # elif defined(__ia64__) PLATFORM_TRIPLET=ia64-linux-LIBC +RUST_TARGET=unknown # elif defined(__loongarch__) && defined(__loongarch_lp64) PLATFORM_TRIPLET=loongarch64-linux-LIBC_LA +RUST_TARGET=loongarch64-unknown-linux-RUST_LIBC_LA # elif defined(__m68k__) && !defined(__mcoldfire__) PLATFORM_TRIPLET=m68k-linux-LIBC +RUST_TARGET=m68k-unknown-linux-RUST_LIBC # elif defined(__mips__) # if defined(__mips_isa_rev) && (__mips_isa_rev >=6) # if defined(_MIPSEL) && defined(__mips64) PLATFORM_TRIPLET=mipsisa64r6el-linux-LIBC_MIPS +RUST_TARGET=mipsisa64r6el-unknown-linux-RUST_LIBC_MIPS # elif defined(_MIPSEL) PLATFORM_TRIPLET=mipsisa32r6el-linux-LIBC_MIPS +RUST_TARGET=mipsisa32r6el-unknown-linux-RUST_LIBC_MIPS # elif defined(__mips64) PLATFORM_TRIPLET=mipsisa64r6-linux-LIBC_MIPS +RUST_TARGET=mipsisa64r6-unknown-linux-RUST_LIBC_MIPS # else PLATFORM_TRIPLET=mipsisa32r6-linux-LIBC_MIPS +RUST_TARGET=mipsisa32r6-unknown-linux-RUST_LIBC_MIPS # endif # else # if defined(_MIPSEL) && defined(__mips64) PLATFORM_TRIPLET=mips64el-linux-LIBC_MIPS +RUST_TARGET=mips64el-unknown-linux-RUST_LIBC_MIPS +# ifdef LLVM_LIBC_MIPS +LLVM_TARGET=mips64el-unknown-linux-LLVM_LIBC_MIPS +# endif # elif defined(_MIPSEL) PLATFORM_TRIPLET=mipsel-linux-LIBC_MIPS +RUST_TARGET=mipsel-unknown-linux-RUST_LIBC_MIPS # elif defined(__mips64) PLATFORM_TRIPLET=mips64-linux-LIBC_MIPS +RUST_TARGET=mips64-unknown-linux-RUST_LIBC_MIPS +# ifdef LLVM_LIBC_MIPS +LLVM_TARGET=mips64-unknown-linux-LLVM_LIBC_MIPS +# endif # else PLATFORM_TRIPLET=mips-linux-LIBC_MIPS +RUST_TARGET=mips-unknown-linux-RUST_LIBC_MIPS # endif # endif # elif defined(__or1k__) PLATFORM_TRIPLET=or1k-linux-LIBC +RUST_TARGET=unknown # elif defined(__powerpc64__) # if defined(__LITTLE_ENDIAN__) PLATFORM_TRIPLET=powerpc64le-linux-LIBC +RUST_TARGET=powerpc64le-unknown-linux-RUST_LIBC # else PLATFORM_TRIPLET=powerpc64-linux-LIBC +RUST_TARGET=powerpc64-unknown-linux-RUST_LIBC # endif # elif defined(__powerpc__) PLATFORM_TRIPLET=powerpc-linux-LIBC_PPC +RUST_TARGET=powerpc-unknown-linux-RUST_LIBC_PPC # elif defined(__s390x__) PLATFORM_TRIPLET=s390x-linux-LIBC +RUST_TARGET=s390x-unknown-linux-RUST_LIBC # elif defined(__s390__) PLATFORM_TRIPLET=s390-linux-LIBC +RUST_TARGET=unknown # elif defined(__sh__) && defined(__LITTLE_ENDIAN__) PLATFORM_TRIPLET=sh4-linux-LIBC +RUST_TARGET=unknown # elif defined(__sparc__) && defined(__arch64__) PLATFORM_TRIPLET=sparc64-linux-LIBC +RUST_TARGET=sparc64-unknown-linux-RUST_LIBC # elif defined(__sparc__) PLATFORM_TRIPLET=sparc-linux-LIBC +RUST_TARGET=sparc-unknown-linux-RUST_LIBC # elif defined(__riscv) # if __riscv_xlen == 32 PLATFORM_TRIPLET=riscv32-linux-LIBC +RUST_TARGET=riscv32gc-unknown-linux-RUST_LIBC +LLVM_TARGET=riscv32-unknown-linux-RUST_LIBC # elif __riscv_xlen == 64 PLATFORM_TRIPLET=riscv64-linux-LIBC +RUST_TARGET=riscv64gc-unknown-linux-RUST_LIBC +LLVM_TARGET=riscv64-unknown-linux-RUST_LIBC # else # error unknown platform triplet # endif @@ -231,16 +314,20 @@ PLATFORM_TRIPLET=riscv64-linux-LIBC #elif defined(__FreeBSD_kernel__) # if defined(__LP64__) PLATFORM_TRIPLET=x86_64-kfreebsd-gnu +RUST_TARGET=unknown # elif defined(__i386__) PLATFORM_TRIPLET=i386-kfreebsd-gnu +RUST_TARGET=unknown # else # error unknown platform triplet # endif #elif defined(__gnu_hurd__) # if defined(__x86_64__) && defined(__LP64__) PLATFORM_TRIPLET=x86_64-gnu +RUST_TARGET=x86_64-unknown-hurd-gnu # elif defined(__i386__) PLATFORM_TRIPLET=i386-gnu +RUST_TARGET=i686-unknown-hurd-gnu # else # error unknown platform triplet # endif @@ -251,28 +338,48 @@ PLATFORM_TRIPLET=i386-gnu # if defined(TARGET_OS_SIMULATOR) && TARGET_OS_SIMULATOR # if __x86_64__ PLATFORM_TRIPLET=x86_64-iphonesimulator +RUST_TARGET=x86_64-apple-ios +LLVM_TARGET=x86_64-apple-ios-simulator # else PLATFORM_TRIPLET=arm64-iphonesimulator +RUST_TARGET=aarch64-apple-ios-sim +LLVM_TARGET=arm64-apple-ios-simulator # endif # else PLATFORM_TRIPLET=arm64-iphoneos +RUST_TARGET=aarch64-apple-ios +LLVM_TARGET=arm64-apple-ios # endif // Older macOS SDKs do not define TARGET_OS_OSX # elif !defined(TARGET_OS_OSX) || TARGET_OS_OSX PLATFORM_TRIPLET=darwin +# if defined(__x86_64__) +RUST_TARGET=x86_64-apple-darwin +LLVM_TARGET=x86_64-apple-macosx +# elif defined(__aarch64__) +RUST_TARGET=aarch64-apple-darwin +LLVM_TARGET=arm64-apple-macosx +# else +RUST_TARGET=unknown +# endif # else # error unknown Apple platform # endif #elif defined(__VXWORKS__) PLATFORM_TRIPLET=vxworks +RUST_TARGET=unknown #elif defined(__wasm32__) # if defined(__EMSCRIPTEN__) PLATFORM_TRIPLET=wasm32-emscripten +RUST_TARGET=wasm32-unknown-emscripten # elif defined(__wasi__) # if defined(_REENTRANT) PLATFORM_TRIPLET=wasm32-wasi-threads +RUST_TARGET=wasm32-wasip1-threads +LLVM_TARGET=wasm32-wasi # else PLATFORM_TRIPLET=wasm32-wasi +RUST_TARGET=wasm32-wasip1 # endif # else # error unknown wasm32 platform @@ -280,8 +387,10 @@ PLATFORM_TRIPLET=wasm32-wasi #elif defined(__wasm64__) # if defined(__EMSCRIPTEN__) PLATFORM_TRIPLET=wasm64-emscripten +RUST_TARGET=unknown # elif defined(__wasi__) PLATFORM_TRIPLET=wasm64-wasi +RUST_TARGET=unknown # else # error unknown wasm64 platform # endif diff --git a/Modules/Setup b/Modules/Setup index 8a54c0aaec60a0..86b43541f37ecc 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -182,6 +182,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_codecs_tw cjkcodecs/_codecs_tw.c #_multibytecodec cjkcodecs/multibytecodec.c #unicodedata unicodedata.c +#_base64 _base64/Cargo.toml _base64/src/lib.rs lib_base64.a # Modules with some UNIX dependencies diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 2c3013e3d0c144..6b56dd858ce585 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -117,6 +117,11 @@ @MODULE__MULTIBYTECODEC_TRUE@_multibytecodec cjkcodecs/multibytecodec.c @MODULE_UNICODEDATA_TRUE@unicodedata unicodedata.c +############################################################################ +# Rust modules +# +@MODULE__BASE64_TRUE@_base64 _base64/Cargo.toml _base64/src/lib.rs + ############################################################################ # Modules with some UNIX dependencies # diff --git a/Modules/_base64/Cargo.toml b/Modules/_base64/Cargo.toml new file mode 100644 index 00000000000000..f90350624729d6 --- /dev/null +++ b/Modules/_base64/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "_base64" +version = "0.1.0" +edition = "2024" + +[dependencies] +cpython-sys ={ path = "../cpython-sys" } + +[build-dependencies] +cpython-build-helper = { path = "../cpython-build-helper" } + +[lib] +name = "_base64" +# For shared builds, we generate a c dynamic library. Static builds use the rlib +crate-type = ["cdylib", "rlib"] diff --git a/Modules/_base64/build.rs b/Modules/_base64/build.rs new file mode 100644 index 00000000000000..8cca32f8f8c3df --- /dev/null +++ b/Modules/_base64/build.rs @@ -0,0 +1,5 @@ +use cpython_build_helper::print_linker_args; + +fn main() { + print_linker_args(); +} diff --git a/Modules/_base64/src/lib.rs b/Modules/_base64/src/lib.rs new file mode 100644 index 00000000000000..64e4d718789372 --- /dev/null +++ b/Modules/_base64/src/lib.rs @@ -0,0 +1,249 @@ +use std::cell::UnsafeCell; +use std::ffi::{c_char, c_int, c_void}; +use std::mem::MaybeUninit; +use std::ptr; +use std::slice; + +use cpython_sys::METH_FASTCALL; +use cpython_sys::Py_DecRef; +use cpython_sys::Py_buffer; +use cpython_sys::Py_ssize_t; +use cpython_sys::PyBuffer_Release; +use cpython_sys::PyBytes_AsString; +use cpython_sys::PyBytes_FromStringAndSize; +use cpython_sys::PyErr_NoMemory; +use cpython_sys::PyErr_SetString; +use cpython_sys::PyExc_TypeError; +use cpython_sys::PyMethodDef; +use cpython_sys::PyMethodDefFuncPointer; +use cpython_sys::PyModuleDef; +use cpython_sys::PyModuleDef_HEAD_INIT; +use cpython_sys::PyModuleDef_Init; +use cpython_sys::PyObject; +use cpython_sys::PyObject_GetBuffer; + +const PYBUF_SIMPLE: c_int = 0; +const PAD_BYTE: u8 = b'='; +const ENCODE_TABLE: [u8; 64] = *b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#[inline] +fn encoded_output_len(input_len: usize) -> Option { + input_len + .checked_add(2) + .map(|n| n / 3) + .and_then(|blocks| blocks.checked_mul(4)) +} + +#[inline] +fn encode_into(input: &[u8], output: &mut [u8]) -> usize { + let mut src_index = 0; + let mut dst_index = 0; + let len = input.len(); + + while src_index + 3 <= len { + let chunk = (u32::from(input[src_index]) << 16) + | (u32::from(input[src_index + 1]) << 8) + | u32::from(input[src_index + 2]); + output[dst_index] = ENCODE_TABLE[((chunk >> 18) & 0x3f) as usize]; + output[dst_index + 1] = ENCODE_TABLE[((chunk >> 12) & 0x3f) as usize]; + output[dst_index + 2] = ENCODE_TABLE[((chunk >> 6) & 0x3f) as usize]; + output[dst_index + 3] = ENCODE_TABLE[(chunk & 0x3f) as usize]; + src_index += 3; + dst_index += 4; + } + + match len - src_index { + 0 => {} + 1 => { + let chunk = u32::from(input[src_index]) << 16; + output[dst_index] = ENCODE_TABLE[((chunk >> 18) & 0x3f) as usize]; + output[dst_index + 1] = ENCODE_TABLE[((chunk >> 12) & 0x3f) as usize]; + output[dst_index + 2] = PAD_BYTE; + output[dst_index + 3] = PAD_BYTE; + dst_index += 4; + } + 2 => { + let chunk = + (u32::from(input[src_index]) << 16) | (u32::from(input[src_index + 1]) << 8); + output[dst_index] = ENCODE_TABLE[((chunk >> 18) & 0x3f) as usize]; + output[dst_index + 1] = ENCODE_TABLE[((chunk >> 12) & 0x3f) as usize]; + output[dst_index + 2] = ENCODE_TABLE[((chunk >> 6) & 0x3f) as usize]; + output[dst_index + 3] = PAD_BYTE; + dst_index += 4; + } + _ => unreachable!("len - src_index cannot exceed 2"), + } + + dst_index +} + +struct BorrowedBuffer { + view: Py_buffer, +} + +impl BorrowedBuffer { + fn from_object(obj: &PyObject) -> Result { + let mut view = MaybeUninit::::uninit(); + let buffer = unsafe { + if PyObject_GetBuffer(obj.as_raw(), view.as_mut_ptr(), PYBUF_SIMPLE) != 0 { + return Err(()); + } + Self { + view: view.assume_init(), + } + }; + Ok(buffer) + } + + fn len(&self) -> Py_ssize_t { + self.view.len + } + + fn as_ptr(&self) -> *const u8 { + self.view.buf.cast::() as *const u8 + } +} + +impl Drop for BorrowedBuffer { + fn drop(&mut self) { + unsafe { + PyBuffer_Release(&mut self.view); + } + } +} + +/// # Safety +/// `module` must be a valid pointer of PyObject representing the module. +/// `args` must be a valid pointer to an array of valid PyObject pointers with length `nargs`. +pub unsafe extern "C" fn standard_b64encode( + _module: *mut PyObject, + args: *mut *mut PyObject, + nargs: Py_ssize_t, +) -> *mut PyObject { + if nargs != 1 { + unsafe { + PyErr_SetString( + PyExc_TypeError, + c"standard_b64encode() takes exactly one argument".as_ptr(), + ); + } + return ptr::null_mut(); + } + + let source = unsafe { &**args }; + + // Safe cast by Safety + match standard_b64encode_impl(source) { + Ok(result) => result, + Err(_) => ptr::null_mut(), + } +} + +fn standard_b64encode_impl(source: &PyObject) -> Result<*mut PyObject, ()> { + let buffer = match BorrowedBuffer::from_object(source) { + Ok(buf) => buf, + Err(_) => return Err(()), + }; + + let view_len = buffer.len(); + if view_len < 0 { + unsafe { + PyErr_SetString( + PyExc_TypeError, + c"standard_b64encode() argument has negative length".as_ptr(), + ); + } + return Err(()); + } + + let input_len = view_len as usize; + let input = unsafe { slice::from_raw_parts(buffer.as_ptr(), input_len) }; + + let Some(output_len) = encoded_output_len(input_len) else { + unsafe { + PyErr_NoMemory(); + } + return Err(()); + }; + + if output_len > isize::MAX as usize { + unsafe { + PyErr_NoMemory(); + } + return Err(()); + } + + let result = unsafe { PyBytes_FromStringAndSize(ptr::null(), output_len as Py_ssize_t) }; + if result.is_null() { + return Err(()); + } + + let dest_ptr = unsafe { PyBytes_AsString(result) }; + if dest_ptr.is_null() { + unsafe { + Py_DecRef(result); + } + return Err(()); + } + let dest = unsafe { slice::from_raw_parts_mut(dest_ptr.cast::(), output_len) }; + + let written = encode_into(input, dest); + debug_assert_eq!(written, output_len); + Ok(result) +} + +pub extern "C" fn _base64_clear(_obj: *mut PyObject) -> c_int { + //TODO + 0 +} + +pub extern "C" fn _base64_free(_o: *mut c_void) { + //TODO +} + +pub struct ModuleDef { + ffi: UnsafeCell, +} + +impl ModuleDef { + fn init_multi_phase(&'static self) -> *mut PyObject { + unsafe { PyModuleDef_Init(self.ffi.get()) } + } +} + +unsafe impl Sync for ModuleDef {} + +pub static _BASE64_MODULE_METHODS: [PyMethodDef; 2] = { + [ + PyMethodDef { + ml_name: c"standard_b64encode".as_ptr() as *mut c_char, + ml_meth: PyMethodDefFuncPointer { + PyCFunctionFast: standard_b64encode, + }, + ml_flags: METH_FASTCALL, + ml_doc: c"Demo for the _base64 module".as_ptr() as *mut c_char, + }, + PyMethodDef::zeroed(), + ] +}; + +pub static _BASE64_MODULE: ModuleDef = { + ModuleDef { + ffi: UnsafeCell::new(PyModuleDef { + m_base: PyModuleDef_HEAD_INIT, + m_name: c"_base64".as_ptr() as *mut _, + m_doc: c"A test Rust module".as_ptr() as *mut _, + m_size: 0, + m_methods: &_BASE64_MODULE_METHODS as *const PyMethodDef as *mut _, + m_slots: std::ptr::null_mut(), + m_traverse: None, + m_clear: Some(_base64_clear), + m_free: Some(_base64_free), + }), + } +}; + +#[unsafe(no_mangle)] +pub extern "C" fn PyInit__base64() -> *mut PyObject { + _BASE64_MODULE.init_multi_phase() +} diff --git a/Modules/cpython-build-helper/Cargo.toml b/Modules/cpython-build-helper/Cargo.toml new file mode 100644 index 00000000000000..1369020965012a --- /dev/null +++ b/Modules/cpython-build-helper/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "cpython-build-helper" +version = "0.1.0" +edition = "2024" + +[dependencies] +shlex = "1.3" diff --git a/Modules/cpython-build-helper/README.md b/Modules/cpython-build-helper/README.md new file mode 100644 index 00000000000000..f42946cfefc31c --- /dev/null +++ b/Modules/cpython-build-helper/README.md @@ -0,0 +1,4 @@ +# cpython-build-helper + +This crate is used in Rust modules `build.rs` files to help do common tasks such +as passing necessary link arguments. \ No newline at end of file diff --git a/Modules/cpython-build-helper/src/lib.rs b/Modules/cpython-build-helper/src/lib.rs new file mode 100644 index 00000000000000..e715f0ecb8755d --- /dev/null +++ b/Modules/cpython-build-helper/src/lib.rs @@ -0,0 +1,124 @@ +use std::env; + +/// Print necessary link arguments for the library depending on the build +/// configuration (static or shared) +pub fn print_linker_args() { + println!("cargo:rerun-if-env-changed=RUST_SHARED_BUILD"); + println!("cargo:rerun-if-env-changed=BLDSHARED_EXE"); + println!("cargo:rerun-if-env-changed=BLDSHARED_ARGS"); + println!("cargo:rerun-if-env-changed=LIBPYTHON"); + println!("cargo:rerun-if-env-changed=PY_CC"); + println!("cargo:rerun-if-env-changed=PYTHON_BUILD_DIR"); + println!("cargo:rerun-if-env-changed=CARGO_CFG_TARGET_OS"); + + let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + let shared_build = + env::var("RUST_SHARED_BUILD").unwrap_or_default() == "1" || target_os == "ios"; + + // On Apple platforms (macOS, iOS), Cargo's cdylib produces a Mach-O + // dynamiclib (via -dynamiclib), but CPython's C extensions are built as + // bundles (via -bundle). Unlike bundles, dynamiclibs require all symbols + // to be resolved at link time. Pass -undefined dynamic_lookup so that + // Python C API symbols are resolved at load time by the interpreter. + if target_os == "macos" || target_os == "ios" { + println!("cargo:rustc-cdylib-link-arg=-undefined"); + println!("cargo:rustc-cdylib-link-arg=dynamic_lookup"); + } + + // Apple framework builds for iOS encode framework search/link flags in + // BLDSHARED_EXE. Skip the linker executable itself and filter out flags + // that conflict with Cargo's own cdylib invocation. + if shared_build && let Ok(args) = env::var("BLDSHARED_EXE") { + print_link_args(&args, true); + } + + // Pass platform-specific shared link arguments (e.g. PY_CORE_LDFLAGS) + // from the CPython build system. + if let Ok(args) = env::var("BLDSHARED_ARGS") { + print_link_args(&args, false); + } + + // On Android (and Cygwin), extension modules must link against libpython. + // LIBPYTHON is set by the CPython build system on these platforms and + // typically contains "-L. -lpython3.X". The "-L." is relative to the + // make build directory, so resolve it to an absolute path using + // PYTHON_BUILD_DIR. + if let Ok(libpython) = env::var("LIBPYTHON") { + let builddir = env::var("PYTHON_BUILD_DIR").ok(); + for arg in shlex::split(&libpython).expect("Invalid LIBPYTHON") { + if arg == "-L." + && let Some(ref dir) = builddir + { + println!("cargo:rustc-cdylib-link-arg=-L{}", dir); + continue; + } + println!("cargo:rustc-cdylib-link-arg={}", arg); + } + } + + // Static linker configuration is in cpython-rust-staticlib +} + +fn print_link_args(raw: &str, skip_first: bool) { + let mut args = shlex::split(raw).expect("Invalid linker args"); + if skip_first { + args = strip_linker_executable(args, env::var("PY_CC").ok().as_deref()); + } + + let build_dir = env::var("PYTHON_BUILD_DIR").ok(); + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + // -bundle_loader only works with -bundle, not with Cargo's + // cdylib mode (-dynamiclib). + "-bundle_loader" => { + i += 2; + } + // Cargo chooses the Mach-O output type for cdylib already. + "-bundle" | "-dynamiclib" => { + i += 1; + } + // dynamic_lookup is added explicitly for Apple targets above. + "-undefined" if i + 1 < args.len() && args[i + 1] == "dynamic_lookup" => { + i += 2; + } + "-F" if i + 1 < args.len() => { + let value = if args[i + 1] == "." { + build_dir.clone().unwrap_or_else(|| ".".to_string()) + } else { + args[i + 1].clone() + }; + println!("cargo:rustc-cdylib-link-arg=-F"); + println!("cargo:rustc-cdylib-link-arg={value}"); + i += 2; + } + _ => { + println!("cargo:rustc-cdylib-link-arg={}", args[i]); + i += 1; + } + } + } +} + +// BLDSHARED_EXE may start with a wrapper/compiler command such as +// "xcrun --sdk iphoneos clang" or "ccache clang". Strip that prefix so only +// linker flags are forwarded to rustc. +fn strip_linker_executable(args: Vec, compiler: Option<&str>) -> Vec { + if let Some(compiler) = compiler + && let Some(prefix) = compiler_command_prefix(compiler) + && args.as_slice().starts_with(prefix.as_slice()) + { + return args[prefix.len()..].to_vec(); + } + + if args.is_empty() { + return args; + } + args[1..].to_vec() +} + +fn compiler_command_prefix(raw: &str) -> Option> { + let tokens = shlex::split(raw)?; + let last_non_flag = tokens.iter().rposition(|token| !token.starts_with('-'))?; + Some(tokens.into_iter().take(last_non_flag + 1).collect()) +} diff --git a/Modules/cpython-rust-staticlib/Cargo.toml b/Modules/cpython-rust-staticlib/Cargo.toml new file mode 100644 index 00000000000000..d7f52b0f3e9602 --- /dev/null +++ b/Modules/cpython-rust-staticlib/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "cpython-rust-staticlib" +version = "0.1.0" +edition = "2024" + +[dependencies] +_base64 ={ path = "../_base64" } + +[lib] +name = "cpython_rust_staticlib" +crate-type = ["staticlib"] diff --git a/Modules/cpython-rust-staticlib/src/lib.rs b/Modules/cpython-rust-staticlib/src/lib.rs new file mode 100644 index 00000000000000..a440707833f65d --- /dev/null +++ b/Modules/cpython-rust-staticlib/src/lib.rs @@ -0,0 +1 @@ +pub use _base64::PyInit__base64; diff --git a/Modules/cpython-sys/Cargo.toml b/Modules/cpython-sys/Cargo.toml new file mode 100644 index 00000000000000..a12556c6e305a1 --- /dev/null +++ b/Modules/cpython-sys/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "cpython-sys" +version = "0.1.0" +edition = "2024" + +[dependencies] + +[build-dependencies] +bindgen = "0.72.1" +shlex = "1.3" diff --git a/Modules/cpython-sys/build.rs b/Modules/cpython-sys/build.rs new file mode 100644 index 00000000000000..03335128e9fbe5 --- /dev/null +++ b/Modules/cpython-sys/build.rs @@ -0,0 +1,410 @@ +use std::env; +use std::path::{Path, PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let srcdir = manifest_dir + .parent() + .and_then(Path::parent) + .expect("expected Modules/cpython-sys to live under the source tree"); + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + let builddir = env::var("PYTHON_BUILD_DIR").ok(); + emit_rerun_instructions(builddir.as_deref()); + let gil_disabled = gil_disabled(srcdir, builddir.as_deref()); + if gil_disabled { + println!("cargo:rustc-cfg=py_gil_disabled"); + } + println!("cargo::rustc-check-cfg=cfg(py_gil_disabled)"); + generate_c_api_bindings( + srcdir, + builddir.as_deref(), + out_path.as_path(), + gil_disabled, + ); +} + +// Bindgen depends on build-time env and, on iOS, can also inherit the +// deployment target from the generated Makefile. Declare both so Cargo reruns +// the build script when those inputs change. +fn emit_rerun_instructions(builddir: Option<&str>) { + for var in [ + "IPHONEOS_DEPLOYMENT_TARGET", + "LLVM_TARGET", + "PY_DEBUG", + "PY_GIL_DISABLED", + "PYTHON_BUILD_DIR", + "PY_CC", + "PY_CPPFLAGS", + "PY_CFLAGS", + "TARGET", + "WASI_SDK_PATH", + ] { + println!("cargo:rerun-if-env-changed={var}"); + } + + if let Some(builddir) = builddir { + let makefile = Path::new(builddir).join("Makefile"); + println!("cargo:rerun-if-changed={}", makefile.display()); + } +} + +/// Find the newest clang resource directory on the system. +/// +/// Ubuntu 24.04 ships libclang-18 whose built-in headers (stdatomic.h, +/// mmintrin.h) are broken. CI jobs install a newer clang (e.g. clang-20) +/// whose resource directory at /usr/lib/llvm-20/lib/clang/20 has working +/// headers. We pass -resource-dir to bindgen's clang so it picks up those +/// headers instead of the broken libclang-18 ones. +fn newest_clang_resource_dir() -> Option { + // On Windows, derive the resource directory from LIBCLANG_PATH so that + // bindgen uses headers matching the libclang.dll it loads, rather than + // picking up a different, incompatible, system LLVM installation + if cfg!(windows) { + return clang_resource_dir_from_libclang_path(); + } + + let base = Path::new("/usr/lib"); + let mut best: Option<(u32, PathBuf)> = None; + for entry in std::fs::read_dir(base).ok()?.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if let Some(ver_str) = name.strip_prefix("llvm-") + && let Ok(ver) = ver_str.parse::() + { + // Resource dir: /usr/lib/llvm-/lib/clang/ + let resource_dir = entry.path().join("lib").join("clang").join(ver_str); + if resource_dir.join("include").is_dir() + && best.as_ref().map_or(true, |(v, _)| ver > *v) + { + best = Some((ver, resource_dir)); + } + } + } + best.map(|(_, p)| p) +} + +/// Derive the clang resource directory from LIBCLANG_PATH. +/// +/// When LIBCLANG_PATH points to e.g. `...\bin`, the resource directory is +/// at `...\lib\clang\`. We pick the highest version found. +fn clang_resource_dir_from_libclang_path() -> Option { + let libclang_path = env::var("LIBCLANG_PATH").ok()?; + let bin_dir = Path::new(&libclang_path); + // LIBCLANG_PATH typically points to a `bin` directory; the resource + // dir lives under the sibling `lib/clang/`. + let clang_lib_dir = bin_dir.parent()?.join("lib").join("clang"); + let mut best: Option<(u32, PathBuf)> = None; + for entry in std::fs::read_dir(&clang_lib_dir).ok()?.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + // Version directories can be just a major number (e.g. "18") or + // a full dotted version (e.g. "18.1.3"). Parse the major part. + if let Ok(ver) = name.split('.').next().unwrap_or("").parse::() { + let resource_dir = entry.path(); + if resource_dir.join("include").is_dir() + && best.as_ref().map_or(true, |(v, _)| ver > *v) + { + best = Some((ver, resource_dir)); + } + } + } + best.map(|(_, p)| p) +} + +fn gil_disabled(srcdir: &Path, builddir: Option<&str>) -> bool { + if env_var_is_truthy("PY_GIL_DISABLED") { + return true; + } + + let mut candidates = Vec::new(); + if let Some(build) = builddir { + candidates.push(PathBuf::from(build)); + } + candidates.push(srcdir.to_path_buf()); + for base in candidates { + let path = base.join("pyconfig.h"); + if let Ok(contents) = std::fs::read_to_string(&path) + && contents.contains("Py_GIL_DISABLED 1") + { + return true; + } + } + false +} + +fn env_var_is_truthy(name: &str) -> bool { + env::var(name) + .map(|v| matches!(v.to_ascii_lowercase().as_str(), "1" | "true" | "yes")) + .unwrap_or(false) +} + +fn generate_c_api_bindings( + srcdir: &Path, + builddir: Option<&str>, + out_path: &Path, + gil_disabled: bool, +) { + let mut builder = bindgen::Builder::default().header("wrapper.h"); + + // Suppress all clang warnings (deprecation warnings, etc.) + builder = builder.clang_arg("-w"); + + if env_var_is_truthy("PY_DEBUG") { + builder = builder.clang_arg("-D_DEBUG"); + } + if gil_disabled { + builder = builder.clang_arg("-DPy_GIL_DISABLED=1"); + } + + // Use the newest clang resource directory available on the system. + // Bindgen links against whatever libclang it finds (often an older + // system version), but the built-in headers in that version may be + // broken (e.g. libclang-18 on Ubuntu 24.04 has broken stdatomic.h + // and mmintrin.h). Overriding -resource-dir makes clang use a + // newer set of built-in headers without changing which libclang.so + // is loaded. + if let Some(resource_dir) = newest_clang_resource_dir() { + eprintln!( + "cpython-sys: using clang resource dir {}", + resource_dir.display() + ); + builder = builder.clang_arg(format!("-resource-dir={}", resource_dir.display())); + } + + // Tell clang the correct target triple for cross-compilation when we have + // an LLVM-specific triple. Otherwise let bindgen translate Cargo's TARGET + // itself (e.g. aarch64-apple-ios-sim -> arm64-apple-ios-simulator). + let cargo_target = env::var("TARGET").unwrap_or_default(); + let llvm_target = env::var("LLVM_TARGET").unwrap_or_default(); + if !llvm_target.is_empty() && llvm_target != cargo_target { + builder = builder.clang_arg(format!("--target={llvm_target}")); + } + + // Extract cross-compilation flags from the C compiler command (PY_CC), + // preprocessor flags (PY_CPPFLAGS), and compiler flags (PY_CFLAGS). + // These provide the sysroot, include paths, and defines that bindgen's + // clang needs when cross-compiling. + // + // - WASI: sysroot in CC, -D_WASI_EMULATED_SIGNAL in CFLAGS + // - iOS: -isysroot in CPPFLAGS + let mut have_sysroot = false; + for env_name in ["PY_CC", "PY_CPPFLAGS", "PY_CFLAGS"] { + if let Ok(value) = env::var(env_name) + && let Some(flags) = shlex::split(&value) + { + let mut iter = flags.iter().peekable(); + while let Some(flag) = iter.next() { + if flag.starts_with("--sysroot") || flag.starts_with("-isysroot") { + builder = builder.clang_arg(flag); + have_sysroot = true; + // Handle "-isysroot " (space-separated) + if (flag == "-isysroot" || flag == "--sysroot") + && let Some(path) = iter.next() + { + builder = builder.clang_arg(path); + } + } else if flag.starts_with("-I") + || flag.starts_with("-D") + || flag.starts_with("-std=") + || flag.starts_with("-isystem") + { + builder = builder.clang_arg(flag); + } + } + } + } + + // WASI SDK: WASI_SDK_PATH is set by Tools/wasm/wasi/__main__.py. + // The sysroot is at $WASI_SDK_PATH/share/wasi-sysroot. + if !have_sysroot + && cargo_target.contains("wasi") + && let Ok(sdk_path) = env::var("WASI_SDK_PATH") + { + let sysroot = PathBuf::from(&sdk_path).join("share").join("wasi-sysroot"); + if sysroot.is_dir() { + builder = builder.clang_arg(format!("--sysroot={}", sysroot.display())); + have_sysroot = true; + } + } + + // Android NDK: ANDROID_HOME is set by the CI/user environment, and + // Android/android-env.sh sets CC to the NDK clang binary at: + // $ANDROID_HOME/ndk//toolchains/llvm/prebuilt//bin/-clang + // The sysroot is a sibling of bin/: + // .../toolchains/llvm/prebuilt//sysroot + if !have_sysroot + && cargo_target.contains("android") + && let Ok(cc) = env::var("PY_CC") + && let Some(parts) = shlex::split(&cc) + && let Some(binary) = parts.first() + && let Some(bin_dir) = Path::new(binary).parent() + { + let sysroot = bin_dir.with_file_name("sysroot"); + if sysroot.is_dir() { + builder = builder.clang_arg(format!("--sysroot={}", sysroot.display())); + } + } + + // Include the build directory first so that cross-build pyconfig.h + // takes precedence over any pyconfig.h in the source tree (which may + // be from a native build with different settings like LONG_BIT). + let mut include_dirs = Vec::new(); + if let Some(build) = builddir { + include_dirs.push(PathBuf::from(build)); + } + include_dirs.push(srcdir.to_path_buf()); + include_dirs.push(srcdir.join("Include")); + + for dir in include_dirs { + builder = builder.clang_arg(format!("-I{}", dir.display())); + } + + builder = add_target_clang_args(builder, builddir); + + let bindings = builder + .allowlist_function("_?Py.*") + .allowlist_type("_?Py.*") + .allowlist_var("_?Py.*") + .blocklist_type("^PyMethodDef$") + .blocklist_type("PyObject") + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) + .generate() + .expect("Unable to generate bindings"); + + let dll_name = python_dll_name(srcdir, env_var_is_truthy("PY_DEBUG"), gil_disabled); + let bindings = patch_windows_imported_pointer_globals(bindings.to_string(), &dll_name); + + // Write the bindings to the $OUT_DIR/c_api.rs file. + std::fs::write(out_path.join("c_api.rs"), bindings).expect("Couldn't write bindings!"); +} + +/// Build the Windows DLL base name: `python{major}{minor}[t][_d]`. +fn python_dll_name(srcdir: &Path, debug: bool, gil_disabled: bool) -> String { + let patchlevel = srcdir.join("Include").join("patchlevel.h"); + let contents = + std::fs::read_to_string(&patchlevel).expect("failed to read Include/patchlevel.h"); + + let major = extract_define_int(&contents, "PY_MAJOR_VERSION"); + let minor = extract_define_int(&contents, "PY_MINOR_VERSION"); + + let mut name = format!("python{major}{minor}"); + if gil_disabled { + name.push('t'); + } + if debug { + name.push_str("_d"); + } + name +} + +fn extract_define_int(contents: &str, name: &str) -> u32 { + for line in contents.lines() { + let trimmed = line.trim(); + if let Some(rest) = trimmed.strip_prefix("#define") + && let Some(value) = rest.trim().strip_prefix(name) + && let Ok(n) = value.trim().parse() + { + return n; + } + } + panic!("could not find #define {name} in patchlevel.h"); +} + +fn patch_windows_imported_pointer_globals(bindings: String, dll_name: &str) -> String { + // On Windows/MSVC, exported data is imported through a synthetic + // "__imp_" pointer in the import address table (IAT). A plain + // `extern { pub static X: *mut T; }` linked via import library fails for + // data symbols because the import library only defines `__imp_X`, not `X`. + // + // Using `#[link_name = "__imp_X"]` links successfully but produces a + // single load — returning the *address* of the variable (the IAT slot + // value) rather than the variable's value. C's `__declspec(dllimport)` + // generates two loads to chase the indirection. + // + // The fix: annotate pointer-valued extern statics with `raw-dylib` on + // Windows so Rust generates the import thunk itself and handles the IAT + // indirection correctly — two loads, matching `__declspec(dllimport)`. + let lines: Vec<_> = bindings.lines().collect(); + let mut patched = String::with_capacity(bindings.len()); + let mut index = 0; + + while index < lines.len() { + if lines[index] == "unsafe extern \"C\" {" + && lines + .get(index + 1) + .and_then(|l| parse_pointer_static_decl(l)) + .is_some() + && lines.get(index + 2).is_some_and(|l| l.trim() == "}") + { + patched.push_str(&format!( + "#[cfg_attr(windows, link(name = \"{dll_name}\", kind = \"raw-dylib\"))]\n" + )); + // Keep the original extern block unchanged. + for i in index..index + 3 { + patched.push_str(lines[i]); + patched.push('\n'); + } + index += 3; + continue; + } + + patched.push_str(lines[index]); + patched.push('\n'); + index += 1; + } + + patched +} + +fn parse_pointer_static_decl(line: &str) -> Option<(&str, bool, &str)> { + let mut decl = line.trim().strip_prefix("pub static ")?; + let is_mut = decl.starts_with("mut "); + if is_mut { + decl = decl.strip_prefix("mut ")?; + } + + let (name, ty) = decl.split_once(':')?; + let ty = ty.trim().strip_suffix(';')?; + if !ty.starts_with('*') { + return None; + } + + Some((name.trim(), is_mut, ty)) +} + +fn add_target_clang_args( + mut builder: bindgen::Builder, + builddir: Option<&str>, +) -> bindgen::Builder { + let target = env::var("TARGET").unwrap_or_default(); + if !target.contains("apple-ios") { + return builder; + } + + // For iOS targets, bindgen may parse headers with an iOS simulator/device + // target but without a deployment minimum, which disables TLS support. + let deployment_target = ios_deployment_target(builddir).unwrap_or_else(|| "13.0".to_string()); + builder = builder.clang_arg(format!("-mios-version-min={deployment_target}")); + builder +} + +fn ios_deployment_target(builddir: Option<&str>) -> Option { + if let Ok(value) = env::var("IPHONEOS_DEPLOYMENT_TARGET") + && !value.is_empty() + { + return Some(value); + } + + let builddir = builddir?; + let makefile = Path::new(builddir).join("Makefile"); + let text = std::fs::read_to_string(makefile).ok()?; + for line in text.lines() { + if let Some(value) = line.strip_prefix("IPHONEOS_DEPLOYMENT_TARGET=") { + let value = value.trim(); + if !value.is_empty() { + return Some(value.to_string()); + } + } + } + None +} diff --git a/Modules/cpython-sys/parser.h b/Modules/cpython-sys/parser.h new file mode 100644 index 00000000000000..e58539b7236611 --- /dev/null +++ b/Modules/cpython-sys/parser.h @@ -0,0 +1,11 @@ +/* Private APIs */ +#define Py_BUILD_CORE + +// Parser +#include "Parser/pegen.h" +#include "Parser/string_parser.h" +#include "Parser/lexer/buffer.h" +#include "Parser/lexer/lexer.h" +#include "Parser/lexer/state.h" +#include "Parser/tokenizer/tokenizer.h" +#include "Parser/tokenizer/helpers.h" diff --git a/Modules/cpython-sys/src/lib.rs b/Modules/cpython-sys/src/lib.rs new file mode 100644 index 00000000000000..17924f0e5fb00a --- /dev/null +++ b/Modules/cpython-sys/src/lib.rs @@ -0,0 +1,154 @@ +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(unsafe_op_in_unsafe_fn)] +#![allow(unnecessary_transmutes)] +#![allow(clippy::approx_constant)] + +use std::ffi::{c_char, c_int, c_void}; + +include!(concat!(env!("OUT_DIR"), "/c_api.rs")); + +// TODO(emmatyping): include parser bindings (see build.rs) +//include!(concat!(env!("OUT_DIR"), "/parser.rs")); +/* Flag passed to newmethodobject */ +/* #define METH_OLDARGS 0x0000 -- unsupported now */ +pub const METH_VARARGS: c_int = 0x0001; +pub const METH_KEYWORDS: c_int = 0x0002; +/* METH_NOARGS and METH_O must not be combined with the flags above. */ +pub const METH_NOARGS: c_int = 0x0004; +pub const METH_O: c_int = 0x0008; + +/* METH_CLASS and METH_STATIC are a little different; these control +the construction of methods for a class. These cannot be used for +functions in modules. */ +pub const METH_CLASS: c_int = 0x0010; +pub const METH_STATIC: c_int = 0x0020; + +/* METH_COEXIST allows a method to be entered even though a slot has +already filled the entry. When defined, the flag allows a separate +method, "__contains__" for example, to coexist with a defined +slot like sq_contains. */ + +pub const METH_COEXIST: c_int = 0x0040; + +pub const METH_FASTCALL: c_int = 0x0080; + +/* This bit is preserved for Stackless Python +pub const METH_STACKLESS: c_int = 0x0100; +pub const METH_STACKLESS: c_int = 0x0000; +*/ + +/* METH_METHOD means the function stores an + * additional reference to the class that defines it; + * both self and class are passed to it. + * It uses PyCMethodObject instead of PyCFunctionObject. + * May not be combined with METH_NOARGS, METH_O, METH_CLASS or METH_STATIC. + */ + +pub const METH_METHOD: c_int = 0x0200; + +#[cfg(target_pointer_width = "64")] +pub const _Py_STATIC_FLAG_BITS: Py_ssize_t = + (_Py_STATICALLY_ALLOCATED_FLAG | _Py_IMMORTAL_FLAGS) as Py_ssize_t; +#[cfg(target_pointer_width = "64")] +pub const _Py_STATIC_IMMORTAL_INITIAL_REFCNT: Py_ssize_t = + (_Py_IMMORTAL_INITIAL_REFCNT as Py_ssize_t) | (_Py_STATIC_FLAG_BITS << 48); +#[cfg(not(target_pointer_width = "64"))] +pub const _Py_STATIC_IMMORTAL_INITIAL_REFCNT: Py_ssize_t = (7u32 << 28) as Py_ssize_t; + +#[repr(transparent)] +pub struct PyObject(std::cell::UnsafeCell<_object>); + +impl PyObject { + #[inline] + pub fn as_raw(&self) -> *mut Self { + self.0.get() as *mut Self + } +} + +#[repr(C)] +pub union PyMethodDefFuncPointer { + pub PyCFunction: unsafe extern "C" fn(slf: *mut PyObject, args: *mut PyObject) -> *mut PyObject, + pub PyCFunctionFast: unsafe extern "C" fn( + slf: *mut PyObject, + args: *mut *mut PyObject, + nargs: Py_ssize_t, + ) -> *mut PyObject, + pub PyCFunctionWithKeywords: unsafe extern "C" fn( + slf: *mut PyObject, + args: *mut PyObject, + kwargs: *mut PyObject, + ) -> *mut PyObject, + pub PyCFunctionFastWithKeywords: unsafe extern "C" fn( + slf: *mut PyObject, + args: *mut *mut PyObject, + nargs: Py_ssize_t, + kwargs: *mut PyObject, + ) -> *mut PyObject, + pub PyCMethod: unsafe extern "C" fn( + slf: *mut PyObject, + typ: *mut PyTypeObject, + args: *mut *mut PyObject, + nargs: Py_ssize_t, + kwargs: *mut PyObject, + ) -> *mut PyObject, + pub Void: *mut c_void, +} + +#[repr(C)] +pub struct PyMethodDef { + pub ml_name: *mut c_char, + pub ml_meth: PyMethodDefFuncPointer, + pub ml_flags: c_int, + pub ml_doc: *mut c_char, +} + +impl PyMethodDef { + pub const fn zeroed() -> Self { + Self { + ml_name: std::ptr::null_mut(), + ml_meth: PyMethodDefFuncPointer { + Void: std::ptr::null_mut(), + }, + ml_flags: 0, + ml_doc: std::ptr::null_mut(), + } + } +} + +// TODO: this is pretty unsafe, we should probably wrap this in a nicer +// abstraction +unsafe impl Sync for PyMethodDef {} +unsafe impl Send for PyMethodDef {} + +#[cfg(py_gil_disabled)] +pub const PyObject_HEAD_INIT: PyObject = { + let mut obj: _object = unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; + obj.ob_flags = _Py_STATICALLY_ALLOCATED_FLAG as _; + PyObject(std::cell::UnsafeCell::new(obj)) +}; + +#[cfg(all(not(py_gil_disabled), target_pointer_width = "64"))] +pub const PyObject_HEAD_INIT: PyObject = PyObject(std::cell::UnsafeCell::new(_object { + __bindgen_anon_1: _object__bindgen_ty_1 { + ob_refcnt_full: _Py_STATIC_IMMORTAL_INITIAL_REFCNT as i64, + }, + ob_type: std::ptr::null_mut(), +})); + +// On 32-bit platforms, the refcount union only has ob_refcnt (no ob_refcnt_split). +#[cfg(all(not(py_gil_disabled), not(target_pointer_width = "64")))] +pub const PyObject_HEAD_INIT: PyObject = PyObject(std::cell::UnsafeCell::new(_object { + __bindgen_anon_1: _object__bindgen_ty_1 { + ob_refcnt: _Py_STATIC_IMMORTAL_INITIAL_REFCNT, + }, + ob_type: std::ptr::null_mut(), +})); + +pub const PyModuleDef_HEAD_INIT: PyModuleDef_Base = PyModuleDef_Base { + ob_base: PyObject_HEAD_INIT, + m_init: None, + m_index: 0, + m_copy: std::ptr::null_mut(), +}; diff --git a/Modules/cpython-sys/wrapper.h b/Modules/cpython-sys/wrapper.h new file mode 100644 index 00000000000000..f79a44f0580633 --- /dev/null +++ b/Modules/cpython-sys/wrapper.h @@ -0,0 +1,177 @@ +/* Public APIs */ +#include "Python.h" + +// Misc + +// OS macros used in C, not necessary in Rust +//#include "osdefs.h" + +// Valgrind / analysis tools macros and functions +#include "dynamic_annotations.h" +// Macros for error codes +// #include "errcode.h" +// Macros to define symbol visibility +// #include "exports.h" +// Includes pyframe.h and cpython/frameobject.h +#include "frameobject.h" +// Includes cpython/marshal.h +#include "marshal.h" +// Macros defining opcodes +#include "opcode.h" +// More macros defining opcodes +#include "opcode_ids.h" +// Dtrace probes +#include "pydtrace.h" +//New code should use descrobject.h +//#include "structmember.h" + +// List of all stdlib names, autogenerated +#include "Python/stdlib_module_names.h" + +/* Private APIs */ +#define Py_BUILD_CORE + +// Internal +#include "internal/pycore_parser.h" +#include "internal/pycore_mimalloc.h" +#include "internal/mimalloc/mimalloc.h" +#include "internal/mimalloc/mimalloc/atomic.h" +#include "internal/mimalloc/mimalloc/internal.h" +#include "internal/mimalloc/mimalloc/prim.h" +#include "internal/mimalloc/mimalloc/track.h" +#include "internal/mimalloc/mimalloc/types.h" +#include "internal/pycore_abstract.h" +#include "internal/pycore_asdl.h" +#include "internal/pycore_ast.h" +#include "internal/pycore_ast_state.h" +#include "internal/pycore_atexit.h" +#include "internal/pycore_audit.h" +#include "internal/pycore_backoff.h" +#include "internal/pycore_bitutils.h" +#include "internal/pycore_blocks_output_buffer.h" +#include "internal/pycore_brc.h" +#include "internal/pycore_bytes_methods.h" +#include "internal/pycore_bytesobject.h" +#include "internal/pycore_call.h" +#include "internal/pycore_capsule.h" +#include "internal/pycore_cell.h" +#include "internal/pycore_ceval.h" +#include "internal/pycore_ceval_state.h" +#include "internal/pycore_code.h" +#include "internal/pycore_codecs.h" +#include "internal/pycore_compile.h" +#include "internal/pycore_complexobject.h" +#include "internal/pycore_condvar.h" +#include "internal/pycore_context.h" +#include "internal/pycore_critical_section.h" +#include "internal/pycore_crossinterp.h" +#include "internal/pycore_debug_offsets.h" +#include "internal/pycore_descrobject.h" +#include "internal/pycore_dict.h" +#include "internal/pycore_dict_state.h" +#include "internal/pycore_dtoa.h" +#include "internal/pycore_exceptions.h" +#include "internal/pycore_faulthandler.h" +#include "internal/pycore_fileutils.h" +#include "internal/pycore_floatobject.h" +#include "internal/pycore_flowgraph.h" +#include "internal/pycore_format.h" +#include "internal/pycore_frame.h" +#include "internal/pycore_freelist.h" +#include "internal/pycore_freelist_state.h" +#include "internal/pycore_function.h" +#include "internal/pycore_gc.h" +#include "internal/pycore_genobject.h" +#include "internal/pycore_getopt.h" +#include "internal/pycore_gil.h" +#include "internal/pycore_global_objects.h" +#include "internal/pycore_global_objects_fini_generated.h" +#include "internal/pycore_global_strings.h" +#include "internal/pycore_hamt.h" +#include "internal/pycore_hashtable.h" +#include "internal/pycore_import.h" +#include "internal/pycore_importdl.h" +#include "internal/pycore_index_pool.h" +#include "internal/pycore_initconfig.h" +#include "internal/pycore_instruments.h" +#include "internal/pycore_instruction_sequence.h" +#include "internal/pycore_interp.h" +#include "internal/pycore_interp_structs.h" +#include "internal/pycore_interpframe.h" +#include "internal/pycore_interpframe_structs.h" +#include "internal/pycore_interpolation.h" +#include "internal/pycore_intrinsics.h" +#include "internal/pycore_jit.h" +#include "internal/pycore_list.h" +#include "internal/pycore_llist.h" +#include "internal/pycore_lock.h" +#include "internal/pycore_long.h" +#include "internal/pycore_memoryobject.h" +#include "internal/pycore_mimalloc.h" +#include "internal/pycore_modsupport.h" +#include "internal/pycore_moduleobject.h" +#include "internal/pycore_namespace.h" +#include "internal/pycore_object.h" +#include "internal/pycore_object_alloc.h" +#include "internal/pycore_object_deferred.h" +#include "internal/pycore_object_stack.h" +#include "internal/pycore_object_state.h" +#include "internal/pycore_obmalloc.h" +#include "internal/pycore_obmalloc_init.h" +#include "internal/pycore_opcode_metadata.h" +#include "internal/pycore_opcode_utils.h" +#include "internal/pycore_optimizer.h" +#include "internal/pycore_parking_lot.h" +#include "internal/pycore_parser.h" +#include "internal/pycore_pathconfig.h" +#include "internal/pycore_pyarena.h" +#include "internal/pycore_pyatomic_ft_wrappers.h" +#include "internal/pycore_pybuffer.h" +#include "internal/pycore_pyerrors.h" +#include "internal/pycore_pyhash.h" +#include "internal/pycore_pylifecycle.h" +#include "internal/pycore_pymath.h" +#include "internal/pycore_pymem.h" +#include "internal/pycore_pymem_init.h" +#include "internal/pycore_pystate.h" +#include "internal/pycore_pystats.h" +#include "internal/pycore_pythonrun.h" +#include "internal/pycore_pythread.h" +#include "internal/pycore_qsbr.h" +#include "internal/pycore_range.h" +#include "internal/pycore_runtime.h" +#include "internal/pycore_runtime_init.h" +#include "internal/pycore_runtime_init_generated.h" +#include "internal/pycore_runtime_structs.h" +#include "internal/pycore_semaphore.h" +#include "internal/pycore_setobject.h" +#include "internal/pycore_signal.h" +#include "internal/pycore_sliceobject.h" +#include "internal/pycore_stats.h" +#include "internal/pycore_strhex.h" +#include "internal/pycore_stackref.h" +#include "internal/pycore_structs.h" +#include "internal/pycore_structseq.h" +#include "internal/pycore_symtable.h" +#include "internal/pycore_sysmodule.h" +#include "internal/pycore_template.h" +#include "internal/pycore_time.h" +#include "internal/pycore_token.h" +#include "internal/pycore_traceback.h" +#include "internal/pycore_tracemalloc.h" +#include "internal/pycore_tstate.h" +#include "internal/pycore_tuple.h" +#include "internal/pycore_typedefs.h" +#include "internal/pycore_typeobject.h" +#include "internal/pycore_typevarobject.h" +#include "internal/pycore_ucnhash.h" +#include "internal/pycore_unicodectype.h" +#include "internal/pycore_unicodeobject.h" +#include "internal/pycore_unicodeobject_generated.h" +#include "internal/pycore_unionobject.h" +#include "internal/pycore_uniqueid.h" +#include "internal/pycore_uop.h" +#include "internal/pycore_uop_ids.h" +#include "internal/pycore_uop_metadata.h" +#include "internal/pycore_warnings.h" +#include "internal/pycore_weakref.h" diff --git a/Modules/makesetup b/Modules/makesetup index 104c824b846540..76edc54988d845 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -83,6 +83,8 @@ case $makepre in '') makepre=Makefile.pre;; esac +UNAME_SYSTEM=`uname -s 2>/dev/null || echo unknown` + # Newline for sed i and a commands NL='\ ' @@ -144,6 +146,8 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | srcs= cpps= libs= + rust= + manifest= mods= mods_upper= skip= @@ -176,6 +180,8 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | *.c++) srcs="$srcs $arg";; *.cxx) srcs="$srcs $arg";; *.cpp) srcs="$srcs $arg";; + *.rs) srcs="$srcs $arg"; rust="true";; + *.toml) manifest="$arg";; \$\(*_CFLAGS\)) cpps="$cpps $arg";; \$\(*_INCLUDES\)) cpps="$cpps $arg";; \$\(*_LIBS\)) libs="$libs $arg";; @@ -226,60 +232,95 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | yes) continue;; esac objs='' - for src in $srcs - do - case $src in - *.c) obj=`basename $src .c`.o; cc='$(CC)';; - *.cc) obj=`basename $src .cc`.o; cc='$(CXX)';; - *.c++) obj=`basename $src .c++`.o; cc='$(CXX)';; - *.C) obj=`basename $src .C`.o; cc='$(CXX)';; - *.cxx) obj=`basename $src .cxx`.o; cc='$(CXX)';; - *.cpp) obj=`basename $src .cpp`.o; cc='$(CXX)';; - *.m) obj=`basename $src .m`.o; cc='$(CC)';; # Obj-C - *) continue;; - esac - case $src in - */*) obj="$srcdir/`dirname $src`/$obj";; - *) obj="$srcdir/$obj";; - esac - objs="$objs $obj" - case $src in - glmodule.c) ;; - /*) ;; - \$*) ;; - *) src='$(srcdir)/'"$srcdir/$src";; - esac - # custom flags first, PY_STDMODULE_CFLAGS may contain -I with system libmpdec + if test "x$rust" = "x"; then + for src in $srcs + do + case $src in + *.c) obj=`basename $src .c`.o; cc='$(CC)';; + *.cc) obj=`basename $src .cc`.o; cc='$(CXX)';; + *.c++) obj=`basename $src .c++`.o; cc='$(CXX)';; + *.C) obj=`basename $src .C`.o; cc='$(CXX)';; + *.cxx) obj=`basename $src .cxx`.o; cc='$(CXX)';; + *.cpp) obj=`basename $src .cpp`.o; cc='$(CXX)';; + *.m) obj=`basename $src .m`.o; cc='$(CC)';; # Obj-C + *) continue;; + esac + case $src in + */*) obj="$srcdir/`dirname $src`/$obj";; + *) obj="$srcdir/$obj";; + esac + objs="$objs $obj" + case $src in + glmodule.c) ;; + /*) ;; + \$*) ;; + *) src='$(srcdir)/'"$srcdir/$src";; + esac + # custom flags first, PY_STDMODULE_CFLAGS may contain -I with system libmpdec + case $doconfig in + no) + cc="$cc $cpps \$(PY_STDMODULE_CFLAGS) \$(CCSHARED)" + rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(MODULE_DEPS_SHARED) \$(PYTHON_HEADERS)" + rule="$rule; $cc -c $src -o $obj" + ;; + *) + cc="$cc $cpps \$(PY_BUILTIN_MODULE_CFLAGS)" + rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(MODULE_DEPS_STATIC) \$(PYTHON_HEADERS)" + rule="$rule; $cc -c $src -o $obj" + ;; + esac + echo "$rule" >>$rulesf + done case $doconfig in - no) - cc="$cc $cpps \$(PY_STDMODULE_CFLAGS) \$(CCSHARED)" - rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(MODULE_DEPS_SHARED) \$(PYTHON_HEADERS)" - rule="$rule; $cc -c $src -o $obj" - ;; - *) - cc="$cc $cpps \$(PY_BUILTIN_MODULE_CFLAGS)" - rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(MODULE_DEPS_STATIC) \$(PYTHON_HEADERS)" - rule="$rule; $cc -c $src -o $obj" - ;; + yes) OBJS="$OBJS $objs";; esac - echo "$rule" >>$rulesf - done - case $doconfig in - yes) OBJS="$OBJS $objs";; - esac - for mod in $mods - do - file="$srcdir/$mod\$(EXT_SUFFIX)" + for mod in $mods + do + file="$srcdir/$mod\$(EXT_SUFFIX)" + case $doconfig in + no) + SHAREDMODS="$SHAREDMODS $file" + BUILT_SHARED="$BUILT_SHARED $mod" + ;; + esac + rule="$file: $objs \$(MODULE_${mods_upper}_LDEPS)" + rule="$rule; \$(BLDSHARED_EXE) \$(BLDSHARED_ARGS) $objs $libs \$(LIBPYTHON) -o $file" + echo "$rule" >>$rulesf + done + else + prefixed_srcs= + for src in $srcs + do + prefixed_srcs="$prefixed_srcs $srcdir/$src" + done + objs= + libs= + case $doconfig in no) - SHAREDMODS="$SHAREDMODS $file" - BUILT_SHARED="$BUILT_SHARED $mod" + # Shared build: build each Rust module as a cdylib (.so) + for mod in $mods + do + rust_shared="target/\$(if \$(CARGO_TARGET),\$(CARGO_TARGET)/\$(CARGO_TARGET_DIR),\$(CARGO_TARGET_DIR))/lib$mod\$(CARGO_DYLIB_SUFFIX)" + file="$srcdir/$mod\$(EXT_SUFFIX)" + SHAREDMODS="$SHAREDMODS $file" + BUILT_SHARED="$BUILT_SHARED $mod" + # depends on the headers through cpython-sys + rule="$rust_shared: cpython-sys \$(srcdir)/Cargo.toml \$(srcdir)/Cargo.lock \$(srcdir)/$srcdir/$manifest $prefixed_srcs \$(PYTHON_HEADERS) \$(MODULE_${mods_upper}_LDEPS) \$(LIBRARY)" + rule="$rule; CARGO_TARGET_DIR=\$(abs_builddir)/target PYTHON_BUILD_DIR=\$(abs_builddir) PY_CC=\"\$(CC)\" PY_CPPFLAGS=\"\$(CPPFLAGS)\" PY_CFLAGS=\"\$(CFLAGS)\" LLVM_TARGET=\"\$(LLVM_TARGET)\" RUST_SHARED_BUILD=\$(PY_ENABLE_SHARED) IPHONEOS_DEPLOYMENT_TARGET=\$(IPHONEOS_DEPLOYMENT_TARGET) BLDSHARED_EXE=\"\$(BLDSHARED_EXE)\" BLDSHARED_ARGS=\"\$(BLDSHARED_ARGS)\" LIBPYTHON=\"\$(LIBPYTHON)\" \$(if \$(CARGO_TARGET_LINKER_ENV),\$(CARGO_TARGET_LINKER_ENV)=\$(CC)) \$(CARGO_HOME)/bin/cargo build -vvv --lib --locked --package ${mod} --profile \$(CARGO_PROFILE) \$(if \$(CARGO_TARGET),--target=\$(CARGO_TARGET)) --manifest-path \$(srcdir)/Cargo.toml" + echo "$rule" >>$rulesf + echo "$file: $rust_shared; mv $rust_shared $file" >>$rulesf + done + ;; + yes) + # Static build: Rust modules are built as part of cpython-rust-staticlib. + # The dependency on cpython-rust-staticlib is in Makefile.pre.in + # (via RUST_STATICLIB_DEP on the BUILDPYTHON rule). + rust_staticlib="target/\$(if \$(CARGO_TARGET),\$(CARGO_TARGET)/\$(CARGO_TARGET_DIR),\$(CARGO_TARGET_DIR))/libcpython_rust_staticlib.a" + LIBS="$LIBS $rust_staticlib" ;; esac - rule="$file: $objs \$(MODULE_${mods_upper}_LDEPS)" - rule="$rule; \$(BLDSHARED) $objs $libs \$(LIBPYTHON) -o $file" - echo "$rule" >>$rulesf - done + fi done case $SHAREDMODS in diff --git a/PCbuild/_base64.vcxproj b/PCbuild/_base64.vcxproj new file mode 100644 index 00000000000000..916069c1aea82a --- /dev/null +++ b/PCbuild/_base64.vcxproj @@ -0,0 +1,164 @@ + + + + + Debug + ARM + + + Debug + ARM64 + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + ARM + + + PGInstrument + ARM64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + ARM + + + PGUpdate + ARM64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + ARM + + + Release + ARM64 + + + Release + Win32 + + + Release + x64 + + + + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4} + _base64 + MakeFileProj + + + + + Makefile + NotSet + false + + + + $(PyStdlibPydExt) + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + llvm-21.1.4.0 + dev + release + debug + release + i686-pc-windows-msvc + x86_64-pc-windows-msvc + thumbv7a-pc-windows-msvc + aarch64-pc-windows-msvc + $(IntDir)cargo\ + $(CargoTargetDir.TrimEnd(`\`)) + $(BuildPath.TrimEnd(`\`)) + $(PySourcePath)PC + $(PySourcePath)Cargo.toml + $(LIBCLANG_PATH) + $(ExternalsDir)$(BundledLlvmTag)\bin + $(ExternalsDir)$(BundledLlvmTag)\bin + $(LLVMInstallDir)\bin + $(LLVMInstallDir)\bin + $(VCInstallDir)Tools\Llvm\x64\bin + $(VCInstallDir)Tools\Llvm\x64\bin + $(VCInstallDir)Tools\Llvm\bin + $(VCInstallDir)Tools\Llvm\bin + C:\Program Files\LLVM\bin + C:\Program Files\LLVM\bin + $(OutDir) + $(TargetDir)$(TargetName)$(TargetExt) + $(CargoTargetDirNoSlash)\$(CargoTarget)\$(CargoBuildSubdir)\_base64.dll + $(CargoTargetDirNoSlash)\$(CargoTarget)\$(CargoBuildSubdir)\_base64.pdb + $(OutDir)$(TargetName).pdb + setlocal +if "$(Configuration)"=="Debug" (set "PY_DEBUG=1") else (set "PY_DEBUG=") +if "$(DisableGil)"=="true" (set "PY_GIL_DISABLED=1") else (set "PY_GIL_DISABLED=") +where cargo >NUL 2>NUL || (echo WARNING: cargo was not found on PATH, skipping _base64 & exit /b 0) +if not "$(LibClangPath)"=="" set "LIBCLANG_PATH=$(LibClangPath)" +if "$(LibClangPath)"=="" if not defined LIBCLANG_PATH (echo WARNING: libclang was not found, skipping _base64 & exit /b 0) +if defined LIBCLANG_PATH if not exist "%LIBCLANG_PATH%\libclang.dll" if not exist "%LIBCLANG_PATH%\clang.dll" (echo WARNING: libclang.dll not found at %LIBCLANG_PATH%, skipping _base64 & exit /b 0) +if not exist "$(CargoTargetDirNoSlash)" mkdir "$(CargoTargetDirNoSlash)" +set "CARGO_TARGET_DIR=$(CargoTargetDirNoSlash)" +set "PYTHON_BUILD_DIR=$(PythonBuildDir)" +set "LIBPYTHON="/LIBPATH:$(BuildPathNoSlash)" "$(PyDllName).lib"" +cargo build --lib --locked --package _base64 --manifest-path "$(CargoWorkspaceManifest)" --profile $(CargoProfile) --target $(CargoTarget) +if errorlevel 1 exit /b 1 +if not exist "$(RustExtensionPath)" (echo ERROR: cargo did not produce $(RustExtensionPath) & exit /b 1) +copy /Y "$(RustExtensionPath)" "$(TargetPath)" >NUL +if errorlevel 1 exit /b 1 +if exist "$(RustExtensionPdb)" copy /Y "$(RustExtensionPdb)" "$(TargetPdb)" >NUL + if exist "$(TargetPath)" del /F /Q "$(TargetPath)" +if exist "$(TargetPdb)" del /F /Q "$(TargetPdb)" +if exist "$(CargoTargetDirNoSlash)" rmdir /S /Q "$(CargoTargetDirNoSlash)" + $(TargetPath) + + + + + + + + + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} + false + + + + + + + + + + diff --git a/PCbuild/_base64.vcxproj.filters b/PCbuild/_base64.vcxproj.filters new file mode 100644 index 00000000000000..0ba48e32c44518 --- /dev/null +++ b/PCbuild/_base64.vcxproj.filters @@ -0,0 +1,19 @@ + + + + + {86A93473-99A3-4B0B-9B78-4A09FF3A0A6F} + + + + + Rust Sources + + + Rust Sources + + + Rust Sources + + + diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 602357048867d6..43183fe06132cb 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -111,7 +111,7 @@ if "%IncludeExternals%"=="" set IncludeExternals=true if "%IncludeCTypes%"=="" set IncludeCTypes=true if "%IncludeSSL%"=="" set IncludeSSL=true if "%IncludeTkinter%"=="" set IncludeTkinter=true -if "%UseJIT%" NEQ "true" set IncludeLLVM=false +if "%IncludeLLVM%"=="" if "%UseJIT%"=="true" (set IncludeLLVM=true) else (set IncludeLLVM=false) if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 7a5327bf016cea..97a4e952e221b6 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -66,7 +66,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 7296ea75301157..2f441dd47eb9fe 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -23,6 +23,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python", "python.vcxproj", {28B5D777-DDF2-4B6B-B34F-31D938813856} = {28B5D777-DDF2-4B6B-B34F-31D938813856} {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781} = {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781} {384C224A-7474-476E-A01B-750EA7DE918C} = {384C224A-7474-476E-A01B-750EA7DE918C} + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4} = {D7B7A122-80A0-4AFB-9C06-18C29B3178E4} {447F05A8-F581-4CAC-A466-5AC7936E207E} = {447F05A8-F581-4CAC-A466-5AC7936E207E} {4946ECAC-2E69-4BF8-A90A-F5136F5094DF} = {4946ECAC-2E69-4BF8-A90A-F5136F5094DF} {494BAC80-A60C-43A9-99E7-ACB691CE2C4D} = {494BAC80-A60C-43A9-99E7-ACB691CE2C4D} @@ -134,6 +135,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testconsole", "_testconsol EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_asyncio", "_asyncio.vcxproj", "{384C224A-7474-476E-A01B-750EA7DE918C}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_base64", "_base64.vcxproj", "{D7B7A122-80A0-4AFB-9C06-18C29B3178E4}" +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_zoneinfo", "_zoneinfo.vcxproj", "{FCBE1EF2-E0F0-40B1-88B5-00A35D378742}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_queue", "_queue.vcxproj", "{78D80A15-BD8C-44E2-B49E-1F05B0A0A687}" @@ -1352,6 +1355,38 @@ Global {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.Build.0 = Release|Win32 {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.ActiveCfg = Release|x64 {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.Build.0 = Release|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|ARM.ActiveCfg = Debug|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|ARM.Build.0 = Debug|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|ARM64.Build.0 = Debug|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|Win32.ActiveCfg = Debug|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|Win32.Build.0 = Debug|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|x64.ActiveCfg = Debug|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|x64.Build.0 = Debug|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|ARM.Build.0 = PGInstrument|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|ARM.Build.0 = PGUpdate|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|ARM.ActiveCfg = Release|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|ARM.Build.0 = Release|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|ARM64.ActiveCfg = Release|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|ARM64.Build.0 = Release|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|Win32.ActiveCfg = Release|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|Win32.Build.0 = Release|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|x64.ActiveCfg = Release|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|x64.Build.0 = Release|x64 {FCBE1EF2-E0F0-40B1-88B5-00A35D378742}.Debug|ARM.ActiveCfg = Debug|ARM {FCBE1EF2-E0F0-40B1-88B5-00A35D378742}.Debug|ARM.Build.0 = Debug|ARM {FCBE1EF2-E0F0-40B1-88B5-00A35D378742}.Debug|ARM64.ActiveCfg = Debug|ARM64 diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 27c0d382281bdb..18ee141e7f8b8d 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -171,6 +171,7 @@ The following sub-projects are for individual modules of the standard library which are implemented in C; each one builds a DLL (renamed to .pyd) of the same name as the project: * _asyncio + * _base64 * _ctypes * _ctypes_test * _decimal diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 8937e666bbbdd5..54cbc7106069bc 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -10,6 +10,7 @@ static const char* _Py_stdlib_module_names[] = { "_ast", "_ast_unparse", "_asyncio", +"_base64", "_bisect", "_blake2", "_bz2", diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index bda72539640611..f7a4166415a4a5 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -3,6 +3,7 @@ from __future__ import annotations import _imp +import os import os.path import sys import sysconfig @@ -14,12 +15,17 @@ SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) STDLIB_PATH = os.path.join(SRC_DIR, 'Lib') +MODULES_PATH = os.path.join(SRC_DIR, 'Modules') IGNORE = { '__init__', '__pycache__', 'site-packages', + # Rust build-system support modules + 'cpython-build-helper', + 'cpython-rust-staticlib', + # Test modules and packages '__hello__', '__phello__', @@ -84,6 +90,19 @@ def list_modules_setup_extensions(names: set[str]) -> None: names.update(checker.list_module_names(all=True)) +def list_rust_modules(names: set[str]) -> None: + if not os.path.isdir(MODULES_PATH): + return + for entry in os.scandir(MODULES_PATH): + if not entry.is_dir(): + continue + if entry.name == "cpython-sys": + continue + cargo_toml = os.path.join(entry.path, "Cargo.toml") + if os.path.isfile(cargo_toml): + names.add(entry.name) + + # List frozen modules of the PyImport_FrozenModules list (Python/frozen.c). # Use the "./Programs/_testembed list_frozen" command. def list_frozen(names: set[str]) -> None: @@ -109,6 +128,7 @@ def list_modules() -> set[str]: list_builtin_modules(names) list_modules_setup_extensions(names) + list_rust_modules(names) list_packages(names) list_python_modules(names) list_frozen(names) diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs index 0707e77b5e9ab2..67ec4f911e2683 100644 --- a/Tools/msi/freethreaded/freethreaded_files.wxs +++ b/Tools/msi/freethreaded/freethreaded_files.wxs @@ -103,7 +103,7 @@ - + diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs index 4d44299f783909..dfb10f47c85398 100644 --- a/Tools/msi/lib/lib_files.wxs +++ b/Tools/msi/lib/lib_files.wxs @@ -1,6 +1,6 @@  - + diff --git a/configure b/configure index eeb24c1d844e86..16db4f9593cd47 100755 --- a/configure +++ b/configure @@ -687,6 +687,8 @@ MODULE_BINASCII_FALSE MODULE_BINASCII_TRUE MODULE_ZLIB_FALSE MODULE_ZLIB_TRUE +MODULE__BASE64_FALSE +MODULE__BASE64_TRUE MODULE__UUID_FALSE MODULE__UUID_TRUE MODULE__TKINTER_FALSE @@ -822,6 +824,7 @@ MODULE_TIME_FALSE MODULE_TIME_TRUE MODULE__IO_FALSE MODULE__IO_TRUE +RUST_STATICLIB_DEP MODULE_BUILDTYPE _PYTHREAD_NAME_MAXLEN TEST_MODULES @@ -885,6 +888,14 @@ TCLTK_LIBS TCLTK_CFLAGS LIBSQLITE3_LIBS LIBSQLITE3_CFLAGS +LLVM_TARGET +CARGO_DYLIB_SUFFIX +CARGO_TARGET_LINKER_ENV +CARGO_TARGET +CARGO_PROFILE +CARGO_TARGET_DIR +CARGO_HOME +HAVE_CARGO LIBMPDEC_INTERNAL LIBMPDEC_LIBS LIBMPDEC_CFLAGS @@ -901,7 +912,8 @@ SHLIBS CFLAGSFORSHARED LINKFORSHARED CCSHARED -BLDSHARED +BLDSHARED_ARGS +BLDSHARED_EXE LDCXXSHARED LDSHARED SHLIB_SUFFIX @@ -7166,6 +7178,13 @@ printf %s "checking for the platform triplet based on compiler characteristics.. if $CPP $CPPFLAGS $srcdir/Misc/platform_triplet.c >conftest.out 2>/dev/null; then PLATFORM_TRIPLET=`grep '^PLATFORM_TRIPLET=' conftest.out | tr -d ' '` PLATFORM_TRIPLET="${PLATFORM_TRIPLET#PLATFORM_TRIPLET=}" + RUST_TARGET=`grep '^RUST_TARGET=' conftest.out | tr -d ' '` + RUST_TARGET="${RUST_TARGET#RUST_TARGET=}" + LLVM_TARGET=`grep '^LLVM_TARGET=' conftest.out | tr -d ' '` + LLVM_TARGET="${LLVM_TARGET#LLVM_TARGET=}" + if test -z "$LLVM_TARGET"; then + LLVM_TARGET="$RUST_TARGET" + fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $PLATFORM_TRIPLET" >&5 printf "%s\n" "$PLATFORM_TRIPLET" >&6; } else @@ -13496,6 +13515,7 @@ fi + # SHLIB_SUFFIX is the extension of shared libraries `(including the dot!) # -- usually .so, .sl on HP-UX, .dll on Cygwin { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking the extension of shared libraries" >&5 @@ -13525,7 +13545,8 @@ if test -z "$LDSHARED" then case $ac_sys_system/$ac_sys_release in AIX*) - BLDSHARED="Modules/ld_so_aix \$(CC) -bI:Modules/python.exp" + BLDSHARED_EXE="Modules/ld_so_aix" + BLDSHARED_ARGS="\$(CC) -bI:Modules/python.exp" LDSHARED="\$(LIBPL)/ld_so_aix \$(CC) -bI:\$(LIBPL)/python.exp" ;; SunOS/5*) @@ -13549,7 +13570,8 @@ then LDCXXSHARED='$(CXX) -bundle' if test "$enable_framework" ; then # Link against the framework. All externals should be defined. - BLDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + BLDSHARED_EXE="$LDSHARED" + BLDSHARED_ARGS='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDCXXSHARED="$LDCXXSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' else @@ -13560,14 +13582,15 @@ then Darwin/1.4*|Darwin/5.*|Darwin/6.*) LDSHARED='$(CC) -bundle' LDCXXSHARED='$(CXX) -bundle' + BLDSHARED_EXE="$LDSHARED" if test "$enable_framework" ; then # Link against the framework. All externals should be defined. - BLDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + BLDSHARED_ARGS='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDCXXSHARED="$LDCXXSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' else # No framework, use the Python app as bundle-loader - BLDSHARED="$LDSHARED "'-bundle_loader $(BUILDPYTHON)' + BLDSHARED_ARGS='-bundle_loader $(BUILDPYTHON)' LDSHARED="$LDSHARED "'-bundle_loader $(BINDIR)/python$(VERSION)$(EXE)' LDCXXSHARED="$LDCXXSHARED "'-bundle_loader $(BINDIR)/python$(VERSION)$(EXE)' fi ;; @@ -13588,13 +13611,15 @@ then # building for OS X 10.3 and later LDSHARED='$(CC) -bundle -undefined dynamic_lookup' LDCXXSHARED='$(CXX) -bundle -undefined dynamic_lookup' - BLDSHARED="$LDSHARED" + BLDSHARED_EXE="$LDSHARED" + BLDSHARED_ARGS="" fi ;; iOS/*) LDSHARED='$(CC) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' - BLDSHARED="$LDSHARED" + BLDSHARED_EXE="$LDSHARED" + BLDSHARED_ARGS="" ;; Emscripten*|WASI*) LDSHARED='$(CC) -shared' @@ -13654,18 +13679,25 @@ fi;; fi if test "$enable_wasm_dynamic_linking" = "yes" -a "$ac_sys_system" = "Emscripten"; then - BLDSHARED='$(CC) -shared -sSIDE_MODULE=1' + BLDSHARED_EXE='$(CC) -shared' + BLDSHARED_ARGS='-sSIDE_MODULE=1' fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDSHARED" >&5 printf "%s\n" "$LDSHARED" >&6; } LDCXXSHARED=${LDCXXSHARED-$LDSHARED} -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking BLDSHARED flags" >&5 -printf %s "checking BLDSHARED flags... " >&6; } -BLDSHARED=${BLDSHARED-$LDSHARED} -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $BLDSHARED" >&5 -printf "%s\n" "$BLDSHARED" >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking BLDSHARED_EXE executable" >&5 +printf %s "checking BLDSHARED_EXE executable... " >&6; } +BLDSHARED_EXE=${BLDSHARED_EXE-$LDSHARED} +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $BLDSHARED_EXE" >&5 +printf "%s\n" "$BLDSHARED_EXE" >&6; } + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking BLDSHARED_ARGS flags" >&5 +printf %s "checking BLDSHARED_ARGS flags... " >&6; } +BLDSHARED_ARGS=${BLDSHARED_ARGS-} +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $BLDSHARED_ARGS" >&5 +printf "%s\n" "$BLDSHARED_ARGS" >&6; } # CCSHARED are the C *flags* used to create objects to go into a shared # library (module) -- this is only needed for a few systems @@ -16034,6 +16066,97 @@ fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for Rust" >&5 +printf %s "checking for Rust... " >&6; } +CARGO_TARGET_DIR= +CARGO_PROFILE= +if test "x$CARGO_HOME" = "x"; then + CARGO_HOME="$HOME/.cargo" +fi +# Extract the first word of "cargo", so it can be a program name with args. +set dummy cargo; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_HAVE_CARGO+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) if test -n "$HAVE_CARGO"; then + ac_cv_prog_HAVE_CARGO="$HAVE_CARGO" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in no +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_HAVE_CARGO=""$CARGO_HOME"" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + test -z "$ac_cv_prog_HAVE_CARGO" && ac_cv_prog_HAVE_CARGO="yes" +fi ;; +esac +fi +HAVE_CARGO=$ac_cv_prog_HAVE_CARGO +if test -n "$HAVE_CARGO"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $HAVE_CARGO" >&5 +printf "%s\n" "$HAVE_CARGO" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + + +if test $HAVE_CARGO = "no"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: n/a" >&5 +printf "%s\n" "n/a" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: Could not find the cargo executable. It can be installed via rustup" >&5 +printf "%s\n" "$as_me: WARNING: Could not find the cargo executable. It can be installed via rustup" >&2;} +else + if test "$Py_OPT" = 'true'; then + CARGO_TARGET_DIR='release' + CARGO_PROFILE='release' + else + CARGO_TARGET_DIR='debug' + CARGO_PROFILE='dev' + fi + # RUST_TARGET is detected from Misc/platform_triplet.c using the + # C preprocessor, giving us the correct Rust target triple for the + # platform being compiled for. + CARGO_TARGET="$RUST_TARGET" + if test -z "$CARGO_TARGET" || test "$CARGO_TARGET" = "unknown"; then + as_fn_error $? "Could not determine a Rust target triple for this platform. + Rust may not support this target. Set CARGO_TARGET manually or + disable Rust support." "$LINENO" 5 + fi +fi +if test -n "$CARGO_TARGET"; then + CARGO_TARGET_LINKER_ENV="CARGO_TARGET_$(echo "$CARGO_TARGET" | tr 'a-z-' 'A-Z_')_LINKER" +else + CARGO_TARGET_LINKER_ENV="" +fi +case "$host" in + *-apple-*) CARGO_DYLIB_SUFFIX=".dylib";; + *) CARGO_DYLIB_SUFFIX="$SHLIB_SUFFIX";; +esac + + + + + + + + @@ -31264,6 +31387,13 @@ case $host_cpu in #( esac +if test "$HAVE_CARGO" != "no" -a "$MODULE_BUILDTYPE" = "static"; then + RUST_STATICLIB_DEP=cpython-rust-staticlib +else + RUST_STATICLIB_DEP= +fi + + MODULE_BLOCK= @@ -33479,6 +33609,47 @@ printf "%s\n" "$py_cv_module__uuid" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _base64" >&5 +printf %s "checking for stdlib extension module _base64... " >&6; } + if test "$py_cv_module__base64" != "n/a" +then : + + if test "$HAVE_CARGO" = yes +then : + if test "$HAVE_CARGO" = yes +then : + py_cv_module__base64=yes +else case e in #( + e) py_cv_module__base64=missing ;; +esac +fi +else case e in #( + e) py_cv_module__base64=disabled ;; +esac +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__BASE64_STATE=$py_cv_module__base64$as_nl" + if test "x$py_cv_module__base64" = xyes +then : + + + + +fi + if test "$py_cv_module__base64" = yes; then + MODULE__BASE64_TRUE= + MODULE__BASE64_FALSE='#' +else + MODULE__BASE64_TRUE='#' + MODULE__BASE64_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__base64" >&5 +printf "%s\n" "$py_cv_module__base64" >&6; } + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module zlib" >&5 printf %s "checking for stdlib extension module zlib... " >&6; } if test "$py_cv_module_zlib" != "n/a" @@ -34715,6 +34886,10 @@ if test -z "${MODULE__UUID_TRUE}" && test -z "${MODULE__UUID_FALSE}"; then as_fn_error $? "conditional \"MODULE__UUID\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__BASE64_TRUE}" && test -z "${MODULE__BASE64_FALSE}"; then + as_fn_error $? "conditional \"MODULE__BASE64\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE_ZLIB_TRUE}" && test -z "${MODULE_ZLIB_FALSE}"; then as_fn_error $? "conditional \"MODULE_ZLIB\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.ac b/configure.ac index 92adc44da0d6fe..dd202d193a6da1 100644 --- a/configure.ac +++ b/configure.ac @@ -1154,6 +1154,14 @@ AC_MSG_CHECKING([for the platform triplet based on compiler characteristics]) if $CPP $CPPFLAGS $srcdir/Misc/platform_triplet.c >conftest.out 2>/dev/null; then PLATFORM_TRIPLET=`grep '^PLATFORM_TRIPLET=' conftest.out | tr -d ' '` PLATFORM_TRIPLET="${PLATFORM_TRIPLET@%:@PLATFORM_TRIPLET=}" + RUST_TARGET=`grep '^RUST_TARGET=' conftest.out | tr -d ' '` + RUST_TARGET="${RUST_TARGET@%:@RUST_TARGET=}" + dnl LLVM_TARGET is only emitted when it differs from RUST_TARGET + LLVM_TARGET=`grep '^LLVM_TARGET=' conftest.out | tr -d ' '` + LLVM_TARGET="${LLVM_TARGET@%:@LLVM_TARGET=}" + if test -z "$LLVM_TARGET"; then + LLVM_TARGET="$RUST_TARGET" + fi AC_MSG_RESULT([$PLATFORM_TRIPLET]) else AC_MSG_RESULT([none]) @@ -3379,7 +3387,8 @@ with_tsan="no" AC_SUBST([SHLIB_SUFFIX]) AC_SUBST([LDSHARED]) AC_SUBST([LDCXXSHARED]) -AC_SUBST([BLDSHARED]) +AC_SUBST([BLDSHARED_EXE]) +AC_SUBST([BLDSHARED_ARGS]) AC_SUBST([CCSHARED]) AC_SUBST([LINKFORSHARED]) @@ -3409,7 +3418,8 @@ if test -z "$LDSHARED" then case $ac_sys_system/$ac_sys_release in AIX*) - BLDSHARED="Modules/ld_so_aix \$(CC) -bI:Modules/python.exp" + BLDSHARED_EXE="Modules/ld_so_aix" + BLDSHARED_ARGS="\$(CC) -bI:Modules/python.exp" LDSHARED="\$(LIBPL)/ld_so_aix \$(CC) -bI:\$(LIBPL)/python.exp" ;; SunOS/5*) @@ -3433,7 +3443,8 @@ then LDCXXSHARED='$(CXX) -bundle' if test "$enable_framework" ; then # Link against the framework. All externals should be defined. - BLDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + BLDSHARED_EXE="$LDSHARED" + BLDSHARED_ARGS='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDCXXSHARED="$LDCXXSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' else @@ -3444,14 +3455,15 @@ then Darwin/1.4*|Darwin/5.*|Darwin/6.*) LDSHARED='$(CC) -bundle' LDCXXSHARED='$(CXX) -bundle' + BLDSHARED_EXE="$LDSHARED" if test "$enable_framework" ; then # Link against the framework. All externals should be defined. - BLDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + BLDSHARED_ARGS='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDCXXSHARED="$LDCXXSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' else # No framework, use the Python app as bundle-loader - BLDSHARED="$LDSHARED "'-bundle_loader $(BUILDPYTHON)' + BLDSHARED_ARGS='-bundle_loader $(BUILDPYTHON)' LDSHARED="$LDSHARED "'-bundle_loader $(BINDIR)/python$(VERSION)$(EXE)' LDCXXSHARED="$LDCXXSHARED "'-bundle_loader $(BINDIR)/python$(VERSION)$(EXE)' fi ;; @@ -3472,13 +3484,15 @@ then # building for OS X 10.3 and later LDSHARED='$(CC) -bundle -undefined dynamic_lookup' LDCXXSHARED='$(CXX) -bundle -undefined dynamic_lookup' - BLDSHARED="$LDSHARED" + BLDSHARED_EXE="$LDSHARED" + BLDSHARED_ARGS="" fi ;; iOS/*) LDSHARED='$(CC) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' - BLDSHARED="$LDSHARED" + BLDSHARED_EXE="$LDSHARED" + BLDSHARED_ARGS="" ;; Emscripten*|WASI*) LDSHARED='$(CC) -shared' @@ -3535,18 +3549,24 @@ then esac fi -dnl Emscripten's emconfigure sets LDSHARED. Set BLDSHARED outside the -dnl test -z $LDSHARED block to configure BLDSHARED for side module support. +dnl Emscripten's emconfigure sets LDSHARED. Set BLDSHARED_EXE and BLDSHARED_ARGS +dnl outside the test -z $LDSHARED block to configure BLDSHARED for side module +dnl support. if test "$enable_wasm_dynamic_linking" = "yes" -a "$ac_sys_system" = "Emscripten"; then - BLDSHARED='$(CC) -shared -sSIDE_MODULE=1' + BLDSHARED_EXE='$(CC) -shared' + BLDSHARED_ARGS='-sSIDE_MODULE=1' fi AC_MSG_RESULT([$LDSHARED]) LDCXXSHARED=${LDCXXSHARED-$LDSHARED} -AC_MSG_CHECKING([BLDSHARED flags]) -BLDSHARED=${BLDSHARED-$LDSHARED} -AC_MSG_RESULT([$BLDSHARED]) +AC_MSG_CHECKING([BLDSHARED_EXE executable]) +BLDSHARED_EXE=${BLDSHARED_EXE-$LDSHARED} +AC_MSG_RESULT([$BLDSHARED_EXE]) + +AC_MSG_CHECKING([BLDSHARED_ARGS flags]) +BLDSHARED_ARGS=${BLDSHARED_ARGS-} +AC_MSG_RESULT([$BLDSHARED_ARGS]) # CCSHARED are the C *flags* used to create objects to go into a shared # library (module) -- this is only needed for a few systems @@ -4307,6 +4327,59 @@ fi AC_SUBST([LIBMPDEC_CFLAGS]) AC_SUBST([LIBMPDEC_INTERNAL]) +dnl Try to detect cargo in the environment. Cargo and rustup +dnl install into CARGO_HOME and RUSTUP_HOME, so check for those initially +AC_MSG_CHECKING([for Rust]) +CARGO_TARGET_DIR= +CARGO_PROFILE= +if test "x$CARGO_HOME" = "x"; then + dnl try to guess the default UNIX value of ~/.cargo + CARGO_HOME="$HOME/.cargo" +fi +AC_CHECK_PROG(HAVE_CARGO, [cargo], ["$CARGO_HOME"], [yes], [no]) +if test $HAVE_CARGO = "no"; then + AC_MSG_RESULT([n/a]) + AC_MSG_WARN([Could not find the cargo executable. It can be installed via rustup]) +else + if test "$Py_OPT" = 'true'; then + CARGO_TARGET_DIR='release' + CARGO_PROFILE='release' + else + CARGO_TARGET_DIR='debug' + CARGO_PROFILE='dev' + fi + # RUST_TARGET is detected from Misc/platform_triplet.c using the + # C preprocessor, giving us the correct Rust target triple for the + # platform being compiled for. + CARGO_TARGET="$RUST_TARGET" + if test -z "$CARGO_TARGET" || test "$CARGO_TARGET" = "unknown"; then + AC_MSG_ERROR([Could not determine a Rust target triple for this platform. + Rust may not support this target. Set CARGO_TARGET manually or + disable Rust support.]) + fi +fi +dnl Compute the CARGO_TARGET__LINKER env var name so we can +dnl tell Cargo which linker to use for the target platform. +if test -n "$CARGO_TARGET"; then + CARGO_TARGET_LINKER_ENV="CARGO_TARGET_$(echo "$CARGO_TARGET" | tr 'a-z-' 'A-Z_')_LINKER" +else + CARGO_TARGET_LINKER_ENV="" +fi +dnl Cargo's cdylib crate type uses the platform's native shared library +dnl extension, which differs from CPython's SHLIB_SUFFIX on macOS/iOS +dnl (CPython uses .so, Cargo uses .dylib). +case "$host" in + *-apple-*) CARGO_DYLIB_SUFFIX=".dylib";; + *) CARGO_DYLIB_SUFFIX="$SHLIB_SUFFIX";; +esac +AC_SUBST([CARGO_HOME]) +AC_SUBST([CARGO_TARGET_DIR]) +AC_SUBST([CARGO_PROFILE]) +AC_SUBST([CARGO_TARGET]) +AC_SUBST([CARGO_TARGET_LINKER_ENV]) +AC_SUBST([CARGO_DYLIB_SUFFIX]) +AC_SUBST([LLVM_TARGET]) + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -7820,6 +7893,15 @@ AS_CASE([$host_cpu], ) AC_SUBST([MODULE_BUILDTYPE]) +dnl Set RUST_STATICLIB_DEP to cpython-rust-staticlib when building Rust +dnl modules statically so that the python binary depends on it. +if test "$HAVE_CARGO" != "no" -a "$MODULE_BUILDTYPE" = "static"; then + RUST_STATICLIB_DEP=cpython-rust-staticlib +else + RUST_STATICLIB_DEP= +fi +AC_SUBST([RUST_STATICLIB_DEP]) + dnl _MODULE_BLOCK_ADD([VAR], [VALUE]) dnl internal: adds $1=quote($2) to MODULE_BLOCK AC_DEFUN([_MODULE_BLOCK_ADD], [AS_VAR_APPEND([MODULE_BLOCK], ["$1=_AS_QUOTE([$2])$as_nl"])]) @@ -8156,6 +8238,10 @@ PY_STDLIB_MOD([_uuid], [], [test "$have_uuid" = "yes"], [$LIBUUID_CFLAGS], [$LIBUUID_LIBS]) +PY_STDLIB_MOD([_base64], + [test "$HAVE_CARGO" = yes], [test "$HAVE_CARGO" = yes], + [], []) + dnl compression libs PY_STDLIB_MOD([zlib], [], [test "$have_zlib" = yes], [$ZLIB_CFLAGS], [$ZLIB_LIBS]) diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 00000000000000..4f2204701d7903 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.91.1" +components = ["rustfmt", "clippy"]