From dd3694aa71c0da4a3964b3be468bc40a9b050f87 Mon Sep 17 00:00:00 2001 From: jwiltse Date: Wed, 5 Feb 2025 00:11:52 -0500 Subject: [PATCH 1/3] - Enable override of download URLs and ./configure flags - Normalize c_configure_args functions - Add more extensive comments to root yml file for posterity --- portable-python.yml | 27 +++ src/portable_python/__init__.py | 11 ++ src/portable_python/cpython.py | 3 + src/portable_python/external/xcpython.py | 222 ++++++++++++++--------- 4 files changed, 175 insertions(+), 88 deletions(-) diff --git a/portable-python.yml b/portable-python.yml index 1a46319..748c541 100644 --- a/portable-python.yml +++ b/portable-python.yml @@ -19,3 +19,30 @@ folders: cpython-additional-packages: # - Pillow==10.0.0 # - flake8==6.0.0 + +# Uncomment to download, compile and statically link Python module dependencies +# cpython-modules: libffi zlib xz bzip2 openssl uuid sqlite + +# Uncomment to override a dependency version +# libffi-version: 3.3 + +# Uncomment to override cpython or a dependency source URL +# Note: string "$version" will be replaced with version string (e.g. 1.2.3) +# cpython-url: https://my-cpython-mirror/cpython-$version.tar.gz +# zlib-url: https://my-zlib-mirror/zlib-$version.tar.gz + +# Uncomment to override the ./configure arguments for a dependency +# Note: this will replace the default arguments, not extend them +# Note: the string "$deps_lib" will be replaced with the output libs directory for the module +# openssl-configure: -v --openssldir=/etc/ssl no-shared no-idea no-tests no-dso + +# Note: It's also possible to set configure args per platform/arch +# linux: + # openssl-configure: --with-terminfo-dirs=/etc/terminfo:/lib/terminfo:/usr/share/terminfo +# macos: + # openssl-configure: --with-terminfo-dirs=/usr/share/terminfo + +# Note: you can also use one argument per line syntax +# openssl-configure: +# - -v +# - --openssldir=/etc/ssl diff --git a/src/portable_python/__init__.py b/src/portable_python/__init__.py index c854051..6c44678 100644 --- a/src/portable_python/__init__.py +++ b/src/portable_python/__init__.py @@ -16,6 +16,7 @@ import pathlib import re from typing import ClassVar, List +from string import Template import runez from runez.http import RestClient @@ -491,6 +492,16 @@ def is_usable_module(self, name): def cfg_version(self, default): return PPG.config.get_value("%s-version" % self.m_name) or default + def cfg_url(self, version): + if config_url := PPG.config.get_value("%s-url" % self.m_name): + url_template = Template(config_url) + return url_template.substitute(version=version) + + def cfg_configure(self, deps_lib): + if configure := PPG.config.get_value("%s-configure" % self.m_name): + configure_template = Template(configure) + return configure_template.substitute(deps_lib=deps_lib) + @property def url(self): """Url of source tarball, if any""" diff --git a/src/portable_python/cpython.py b/src/portable_python/cpython.py index 7c2ee36..628eb21 100644 --- a/src/portable_python/cpython.py +++ b/src/portable_python/cpython.py @@ -104,6 +104,9 @@ def url(self): if PPG.config.get_value("cpython-use-github"): return f"https://github.com/python/cpython/archive/refs/tags/v{self.version}.tar.gz" + if cfg_url := self.cfg_url(self.version): + return cfg_url + return f"https://www.python.org/ftp/python/{self.version.main}/Python-{self.version}.tar.xz" def xenv_LDFLAGS_NODIST(self): diff --git a/src/portable_python/external/xcpython.py b/src/portable_python/external/xcpython.py index a5e658c..8029f14 100644 --- a/src/portable_python/external/xcpython.py +++ b/src/portable_python/external/xcpython.py @@ -21,14 +21,24 @@ class Bdb(ModuleBuilder): @property def url(self): - return f"https://ftp.osuosl.org/pub/blfs/conglomeration/db/db-{self.version}.tar.gz" + return self.cfg_url(self.version) or f"https://ftp.osuosl.org/pub/blfs/conglomeration/db/db-{self.version}.tar.gz" @property def version(self): return self.cfg_version("6.2.32") + def c_configure_args(self): + if config_args := self.cfg_configure(self.deps_lib): + yield config_args + else: + yield "--enable-shared=no" + yield "--enable-static=yes" + yield "--enable-dbm" + yield "--with-pic=yes" + + def _do_linux_compile(self): - self.run_configure("../dist/configure", "--enable-shared=no", "--enable-static=yes", "--enable-dbm", "--with-pic=yes") + self.run_configure("../dist/configure", self.c_configure_args()) self.run_make() self.run_make("install") @@ -46,7 +56,7 @@ def auto_select_reason(self): @property def url(self): - return f"https://sourceware.org/pub/bzip2/bzip2-{self.version}.tar.gz" + return self.cfg_url(self.version) or f"https://sourceware.org/pub/bzip2/bzip2-{self.version}.tar.gz" @property def version(self): @@ -70,27 +80,29 @@ class Gdbm(ModuleBuilder): @property def url(self): - return f"https://ftp.gnu.org/gnu/gdbm/gdbm-{self.version}.tar.gz" + return self.cfg_url(self.version) or f"https://ftp.gnu.org/gnu/gdbm/gdbm-{self.version}.tar.gz" @property def version(self): return self.cfg_version("1.24") + + def c_configure_args(self): + if config_args := self.cfg_configure(self.deps_lib): + yield config_args + else: + yield "--enable-shared=no" + yield "--enable-static=yes" + yield "--with-pic=yes" + yield "--disable-nls" + yield "--disable-dependency-tracking" + yield "--disable-rpath" + yield "--disable-silent-rules" + yield "--without-libiconv-prefix" + yield "--without-libintl-prefix" + yield "--without-readline" def _do_linux_compile(self): - self.run_configure( - "./configure", - "--enable-shared=no", - "--enable-static=yes", - "--with-pic=yes", - "--enable-libgdbm-compat", - "--disable-dependency-tracking", - "--disable-nls", - "--disable-rpath", - "--disable-silent-rules", - "--without-libiconv-prefix", - "--without-libintl-prefix", - "--without-readline", - ) + self.run_configure("./configure", self.c_configure_args()) self.run_make() self.run_make("install") runez.move(self.deps / "include/ndbm.h", self.deps / "include/gdbm-ndbm.h") @@ -110,21 +122,25 @@ class LibFFI(ModuleBuilder): @property def url(self): - return f"https://github.com/libffi/libffi/releases/download/v{self.version}/libffi-{self.version}.tar.gz" + return self.cfg_url(self.version) or f"https://github.com/libffi/libffi/releases/download/v{self.version}/libffi-{self.version}.tar.gz" @property def version(self): return self.cfg_version("3.4.6") + + def c_configure_args(self): + if config_args := self.cfg_configure(self.deps_lib): + yield config_args + else: + yield "--enable-shared=no" + yield "--enable-static=yes" + yield "--with-pic=yes" + yield PPG.target.is_macos and "--disable-multi-os-directory" + yield "--disable-docs" + def _do_linux_compile(self): - self.run_configure( - "./configure", - "--enable-shared=no", - "--enable-static=yes", - "--with-pic=yes", - PPG.target.is_macos and "--disable-multi-os-directory", - "--disable-docs", - ) + self.run_configure("./configure", self.c_configure_args()) self.run_make() self.run_make("install") @@ -146,7 +162,7 @@ def url(self): if self.version and self.version.startswith("1.1.1"): # Not sure why URL suddenly changed for this on github... vfolder = self.version.replace(".", "_") - return f"https://github.com/openssl/openssl/releases/download/OpenSSL_{vfolder}/openssl-{self.version}.tar.gz" + return self.cfg_url(self.version) or f"https://github.com/openssl/openssl/releases/download/OpenSSL_{vfolder}/openssl-{self.version}.tar.gz" return f"https://github.com/openssl/openssl/releases/download/openssl-{self.version}/openssl-{self.version}.tar.gz" @@ -157,11 +173,15 @@ def version(self): return self.cfg_version("3.0.15") def c_configure_args(self): - yield "--openssldir=/etc/ssl" - yield "no-shared", "no-idea", "no-tests" + if config_args := self.cfg_configure(self.deps_lib): + yield config_args + else: + yield "-v" + yield "--openssldir=/etc/ssl" + yield "no-shared", "no-idea", "no-tests" def _do_linux_compile(self): - self.run_configure("./config", "-v", self.c_configure_args()) + self.run_configure("./config", self.c_configure_args()) self.run_make("depend") self.run_make() self.run_make("install_sw") # See https://github.com/openssl/openssl/issues/8170 @@ -174,33 +194,36 @@ class Ncurses(ModuleBuilder): @property def url(self): - return f"https://ftp.gnu.org/pub/gnu/ncurses/ncurses-{self.version}.tar.gz" + return self.cfg_url(self.version) or f"https://ftp.gnu.org/pub/gnu/ncurses/ncurses-{self.version}.tar.gz" @property def version(self): return self.cfg_version("6.5") def c_configure_args(self): - yield "--disable-shared" - yield "--enable-static" - yield "--without-ada" - yield "--disable-db-install" - yield "--without-manpages" - yield "--without-progs" - yield "--without-tests" - yield f"--with-pkg-config-libdir={self.deps_lib}/pkgconfig" - yield "--enable-pc-files" - yield "--with-debug=no" - yield "--with-gpm=no" - yield "--enable-widec" - yield "--enable-symlinks" - yield "--enable-sigwinch" - yield "--without-develop" - if PPG.target.is_linux: - yield "--with-terminfo-dirs=/etc/terminfo:/lib/terminfo:/usr/share/terminfo" - - if PPG.target.is_macos: - yield "--with-terminfo-dirs=/usr/share/terminfo" + if config_args := self.cfg_configure(self.deps_lib): + yield config_args + else: + yield "--disable-shared" + yield "--enable-static" + yield "--without-ada" + yield "--disable-db-install" + yield "--without-manpages" + yield "--without-progs" + yield "--without-tests" + yield f"--with-pkg-config-libdir={self.deps_lib}/pkgconfig" + yield "--enable-pc-files" + yield "--with-debug=no" + yield "--with-gpm=no" + yield "--enable-widec" + yield "--enable-symlinks" + yield "--enable-sigwinch" + yield "--without-develop" + if PPG.target.is_linux: + yield "--with-terminfo-dirs=/etc/terminfo:/lib/terminfo:/usr/share/terminfo" + + if PPG.target.is_macos: + yield "--with-terminfo-dirs=/usr/share/terminfo" def _do_linux_compile(self): self.run_configure("./configure", self.c_configure_args()) @@ -227,23 +250,26 @@ def candidate_modules(cls): @property def url(self): - return f"https://ftp.gnu.org/gnu/readline/readline-{self.version}.tar.gz" + return self.cfg_url(self.version) or f"https://ftp.gnu.org/gnu/readline/readline-{self.version}.tar.gz" @property def version(self): return self.cfg_version("8.2.13") + def c_configure_args(self): + if config_args := self.cfg_configure(self.deps_lib): + yield config_args + else: + yield "--enable-shared=no" + yield "--enable-static=yes" + yield "--with-curses" + yield "--enable-multibyte" + yield "--disable-install-examples" + yield "--disable-docs" + yield "--enable-portable-binary" + def _do_linux_compile(self): - self.run_configure( - "./configure", - "--disable-shared", - "--enable-static", - "--with-curses", - "--enable-multibyte", - "--disable-install-examples", - "--disable-docs", - "--enable-portable-binary", - ) + self.run_configure("./configure", self.c_configure_args()) self.run_make() self.run_make("install") @@ -267,21 +293,24 @@ def linker_outcome(self, is_selected): @property def url(self): - return f"https://github.com/sqlite/sqlite/archive/refs/tags/version-{self.version}.tar.gz" + return self.cfg_url(self.version) or f"https://github.com/sqlite/sqlite/archive/refs/tags/version-{self.version}.tar.gz" @property def version(self): return self.cfg_version("3.47.0") + + def c_configure_args(self): + if config_args := self.cfg_configure(self.deps_lib): + yield config_args + else: + yield "--enable-shared=no", + yield "--enable-static=yes", + yield "--disable-tcl", + yield "--disable-readline", + yield "--with-pic=yes", def _do_linux_compile(self): - self.run_configure( - "./configure", - "--enable-shared=no", - "--enable-static=yes", - "--disable-tcl", - "--disable-readline", - "--with-pic=yes", - ) + self.run_configure("./configure", self.c_configure_args()) self.run_make() self.run_make("install") @@ -300,14 +329,22 @@ class Uuid(ModuleBuilder): @property def url(self): - return f"https://sourceforge.net/projects/libuuid/files/libuuid-{self.version}.tar.gz" + return self.cfg_url(self.version) or f"https://sourceforge.net/projects/libuuid/files/libuuid-{self.version}.tar.gz" @property def version(self): return self.cfg_version("1.0.3") + def c_configure_args(self): + if config_args := self.cfg_configure(self.deps_lib): + yield config_args + else: + yield "--enable-shared=no" + yield "--enable-static=yes" + yield "--with-pic=yes" + def _do_linux_compile(self): - self.run_configure("./configure", "--enable-shared=no", "--enable-static=yes", "--with-pic=yes") + self.run_configure("./configure", self.c_configure_args()) self.run_make() self.run_make("install") @@ -321,24 +358,27 @@ def auto_select_reason(self): @property def url(self): - return f"https://downloads.sourceforge.net/project/lzmautils/xz-{self.version}.tar.gz" + return self.cfg_url(self.version) or f"https://downloads.sourceforge.net/project/lzmautils/xz-{self.version}.tar.gz" @property def version(self): return self.cfg_version("5.6.3") + + def c_configure_args(self): + if config_args := self.cfg_configure(self.deps_lib): + yield config_args + else: + yield "--enable-shared=no", + yield "--enable-static=yes", + yield "--with-pic=yes", + yield "--disable-rpath", + yield "--disable-dependency-tracking", + yield "--disable-doc", + yield "--disable-nls", + yield "--without-libintl-prefix", def _do_linux_compile(self): - self.run_configure( - "./configure", - "--enable-shared=no", - "--enable-static=yes", - "--with-pic=yes", - "--disable-rpath", - "--disable-dependency-tracking", - "--disable-doc", - "--disable-nls", - "--without-libintl-prefix", - ) + self.run_configure("./configure", self.c_configure_args()) self.run_make() self.run_make("install") @@ -360,13 +400,19 @@ def auto_select_reason(self): @property def url(self): - return f"https://zlib.net/fossils/zlib-{self.version}.tar.gz" + return self.cfg_url(self.version) or f"https://zlib.net/fossils/zlib-{self.version}.tar.gz" @property def version(self): return self.cfg_version("1.3.1") + + def c_configure_args(self): + if config_args := self.cfg_configure(self.deps_lib): + yield config_args + else: + yield "--static" def _do_linux_compile(self): - self.run_configure("./configure", "--static") + self.run_configure("./configure", self.c_configure_args()) self.run_make() self.run_make("install") From ebbf67a26c557fef586cdd9a49f85fa0d6d4c0d9 Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Thu, 13 Feb 2025 20:30:59 -0800 Subject: [PATCH 2/3] Bumped python_requires to 3.8 --- .github/workflows/tests.yml | 23 +---------------------- setup.py | 5 ++--- tox.ini | 2 +- 3 files changed, 4 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index edadb64..bff8813 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,28 +13,7 @@ jobs: strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - run: pip install -U pip tox - - run: tox -e py - - uses: codecov/codecov-action@v3 - with: - files: .tox/test-reports/coverage.xml - - test-eol: - # EOL-ed versions of python are exercised on older linux distros for a while longer - # Testing against these versions will eventually be retired - runs-on: ubuntu-20.04 - - strategy: - matrix: - python-version: ["3.6", "3.7"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 diff --git a/setup.py b/setup.py index 8246f13..e20724c 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ author="Zoran Simic zoran@simicweb.com", keywords="python, portable, binary", url="https://github.com/codrsquad/portable-python", - python_requires=">=3.6", + python_requires=">=3.8", entry_points={ "console_scripts": [ "portable-python = portable_python.__main__:main", @@ -22,13 +22,12 @@ "Operating System :: Unix", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Software Development :: Build Tools", "Topic :: System :: Installation/Setup", diff --git a/tox.ini b/tox.ini index 22e2f37..088414e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{310,311,312}, coverage, docs, style +envlist = py{310,311,312,313}, coverage, docs, style skip_missing_interpreters = true [testenv] From 494d796755d3a09d2efb096562d802f7f8450542 Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Thu, 13 Feb 2025 20:49:42 -0800 Subject: [PATCH 3/3] Satisfied linter --- src/portable_python/__init__.py | 2 +- src/portable_python/external/xcpython.py | 57 +++++++++++++++--------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/src/portable_python/__init__.py b/src/portable_python/__init__.py index 6c44678..8b9b429 100644 --- a/src/portable_python/__init__.py +++ b/src/portable_python/__init__.py @@ -15,8 +15,8 @@ import os import pathlib import re -from typing import ClassVar, List from string import Template +from typing import ClassVar, List import runez from runez.http import RestClient diff --git a/src/portable_python/external/xcpython.py b/src/portable_python/external/xcpython.py index 8029f14..1201d02 100644 --- a/src/portable_python/external/xcpython.py +++ b/src/portable_python/external/xcpython.py @@ -30,13 +30,13 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib): yield config_args + else: yield "--enable-shared=no" yield "--enable-static=yes" yield "--enable-dbm" yield "--with-pic=yes" - def _do_linux_compile(self): self.run_configure("../dist/configure", self.c_configure_args()) self.run_make() @@ -85,10 +85,11 @@ def url(self): @property def version(self): return self.cfg_version("1.24") - + def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib): yield config_args + else: yield "--enable-shared=no" yield "--enable-static=yes" @@ -122,22 +123,24 @@ class LibFFI(ModuleBuilder): @property def url(self): - return self.cfg_url(self.version) or f"https://github.com/libffi/libffi/releases/download/v{self.version}/libffi-{self.version}.tar.gz" + return ( + self.cfg_url(self.version) or f"https://github.com/libffi/libffi/releases/download/v{self.version}/libffi-{self.version}.tar.gz" + ) @property def version(self): return self.cfg_version("3.4.6") - + def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib): yield config_args + else: yield "--enable-shared=no" yield "--enable-static=yes" yield "--with-pic=yes" yield PPG.target.is_macos and "--disable-multi-os-directory" yield "--disable-docs" - def _do_linux_compile(self): self.run_configure("./configure", self.c_configure_args()) @@ -162,7 +165,10 @@ def url(self): if self.version and self.version.startswith("1.1.1"): # Not sure why URL suddenly changed for this on github... vfolder = self.version.replace(".", "_") - return self.cfg_url(self.version) or f"https://github.com/openssl/openssl/releases/download/OpenSSL_{vfolder}/openssl-{self.version}.tar.gz" + return ( + self.cfg_url(self.version) + or f"https://github.com/openssl/openssl/releases/download/OpenSSL_{vfolder}/openssl-{self.version}.tar.gz" + ) return f"https://github.com/openssl/openssl/releases/download/openssl-{self.version}/openssl-{self.version}.tar.gz" @@ -175,6 +181,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib): yield config_args + else: yield "-v" yield "--openssldir=/etc/ssl" @@ -203,6 +210,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib): yield config_args + else: yield "--disable-shared" yield "--enable-static" @@ -259,6 +267,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib): yield config_args + else: yield "--enable-shared=no" yield "--enable-static=yes" @@ -298,16 +307,17 @@ def url(self): @property def version(self): return self.cfg_version("3.47.0") - + def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib): yield config_args + else: - yield "--enable-shared=no", - yield "--enable-static=yes", - yield "--disable-tcl", - yield "--disable-readline", - yield "--with-pic=yes", + yield "--enable-shared=no" + yield "--enable-static=yes" + yield "--disable-tcl" + yield "--disable-readline" + yield "--with-pic=yes" def _do_linux_compile(self): self.run_configure("./configure", self.c_configure_args()) @@ -338,6 +348,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib): yield config_args + else: yield "--enable-shared=no" yield "--enable-static=yes" @@ -363,19 +374,20 @@ def url(self): @property def version(self): return self.cfg_version("5.6.3") - + def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib): yield config_args + else: - yield "--enable-shared=no", - yield "--enable-static=yes", - yield "--with-pic=yes", - yield "--disable-rpath", - yield "--disable-dependency-tracking", - yield "--disable-doc", - yield "--disable-nls", - yield "--without-libintl-prefix", + yield "--enable-shared=no" + yield "--enable-static=yes" + yield "--with-pic=yes" + yield "--disable-rpath" + yield "--disable-dependency-tracking" + yield "--disable-doc" + yield "--disable-nls" + yield "--without-libintl-prefix" def _do_linux_compile(self): self.run_configure("./configure", self.c_configure_args()) @@ -405,10 +417,11 @@ def url(self): @property def version(self): return self.cfg_version("1.3.1") - + def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib): yield config_args + else: yield "--static"