From 892e301bce46f7060e31acd6ab2c939ddcec06ad Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Thu, 19 Sep 2024 17:22:42 +0200 Subject: [PATCH 01/19] Add stubgen auto generaiton after bindings build --- bindings/CMakeLists.txt | 14 ++++++++++++++ scripts/generate_stubs.sh | 10 ++++++++++ 2 files changed, 24 insertions(+) create mode 100644 scripts/generate_stubs.sh diff --git a/bindings/CMakeLists.txt b/bindings/CMakeLists.txt index fbab2c18f..cfba155ca 100644 --- a/bindings/CMakeLists.txt +++ b/bindings/CMakeLists.txt @@ -49,6 +49,20 @@ sofa_create_component_in_package_with_targets( TARGETS ${PROJECT_NAME} ) +option(SP3_GENERATE_STUBS "Generate stub files at install step required for auto-completion" OFF) +if(SP3_GENERATE_STUBS) + if(CMAKE_SYSTEM_NAME STREQUAL Windows) + set(bash_name bash) + else () + set(bash_name /bin/bash) + endif () + add_custom_target(GenerateStubs ALL ${bash_name} ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/generate_stubs.sh ${CMAKE_BUILD_DIR} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/lib/${SP3_PYTHON_PACKAGES_DIRECTORY}/ + COMMENT "Generating stub files" + ) + add_dependencies(GenerateStubs ${PROJECT_NAME}) +endif() + #if (SP3_COMPILED_AS_SOFA_SUBPROJECT) # ## Python configuration file (build tree), the lib in the source dir (easier while developping .py files) # file(WRITE "${CMAKE_BINARY_DIR}/etc/sofa/python.d/plugin.SofaPython3.bindings" "${CMAKE_BINARY_DIR}/${SP3_PYTHON_PACKAGES_DIRECTORY}\n") diff --git a/scripts/generate_stubs.sh b/scripts/generate_stubs.sh new file mode 100644 index 000000000..d3260896a --- /dev/null +++ b/scripts/generate_stubs.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +for module_name in *; do + echo "Generating stubgen for $module_name" + stubgen -p $module_name --include-docstrings --inspect-mode + if [ -d "./out/$module_name/" ]; then + rsync -a ./out/$module_name/ ./$module_name/ + fi + rm -rf ./out +done From 61c270999e7eb16b4c2330d55f8bfaba29a9dec3 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Thu, 19 Sep 2024 18:12:47 +0200 Subject: [PATCH 02/19] Generating stubs through cmake does not work as expected with ninja with a in build startegy and isnt working in install because the install order is random --- .github/workflows/ci.yml | 18 +++++++++++++++++- bindings/CMakeLists.txt | 14 -------------- scripts/generate_stubs.sh | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4ca8d856..347e419f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,23 @@ jobs: echo ${CCACHE_BASEDIR} ccache -s fi - + + - name: Generate stubfiles + shell: bash + run: | + + #Install stubgen + ${{ steps.sofa.outputs.python_exe }} -m pip install mypy + + if [[ "$RUNNER_OS" == "Windows" ]]; then + cmd //c "${{ steps.sofa.outputs.vs_vsdevcmd }} \ + && cd /d ${{ env.WORKSPACE_INSTALL_PATH }}/lib/python3/site-packages/ \ + && bash ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.sh ${{ env.WORKSPACE_INSTALL_PATH }}/lib/python3/site-packages/ + else + cd ${{ env.WORKSPACE_INSTALL_PATH }}/lib/python3/site-packages/ + /bin/bash ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.sh ${{ env.WORKSPACE_INSTALL_PATH }}/lib/python3/site-packages/ + fi + - name: Set env vars for artifacts shell: bash run: | diff --git a/bindings/CMakeLists.txt b/bindings/CMakeLists.txt index cfba155ca..fbab2c18f 100644 --- a/bindings/CMakeLists.txt +++ b/bindings/CMakeLists.txt @@ -49,20 +49,6 @@ sofa_create_component_in_package_with_targets( TARGETS ${PROJECT_NAME} ) -option(SP3_GENERATE_STUBS "Generate stub files at install step required for auto-completion" OFF) -if(SP3_GENERATE_STUBS) - if(CMAKE_SYSTEM_NAME STREQUAL Windows) - set(bash_name bash) - else () - set(bash_name /bin/bash) - endif () - add_custom_target(GenerateStubs ALL ${bash_name} ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/generate_stubs.sh ${CMAKE_BUILD_DIR} - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/lib/${SP3_PYTHON_PACKAGES_DIRECTORY}/ - COMMENT "Generating stub files" - ) - add_dependencies(GenerateStubs ${PROJECT_NAME}) -endif() - #if (SP3_COMPILED_AS_SOFA_SUBPROJECT) # ## Python configuration file (build tree), the lib in the source dir (easier while developping .py files) # file(WRITE "${CMAKE_BINARY_DIR}/etc/sofa/python.d/plugin.SofaPython3.bindings" "${CMAKE_BINARY_DIR}/${SP3_PYTHON_PACKAGES_DIRECTORY}\n") diff --git a/scripts/generate_stubs.sh b/scripts/generate_stubs.sh index d3260896a..06ab69dbc 100644 --- a/scripts/generate_stubs.sh +++ b/scripts/generate_stubs.sh @@ -1,5 +1,19 @@ #!/bin/bash +usage() { + echo "Usage: generate_stubs.sh " + echo "This script automatically generates stub files for SofaPython3 modules" +} + +if [ "$#" -eq 1 ]; then + WORK_DIR=$1 +else + usage; exit 1 +fi + +cd $WORK_DIR + + for module_name in *; do echo "Generating stubgen for $module_name" stubgen -p $module_name --include-docstrings --inspect-mode From dbe30292fbcb10619fc7df764cdbe8668861a4c6 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Thu, 19 Sep 2024 18:24:23 +0200 Subject: [PATCH 03/19] fix missing quote --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 347e419f6..25e2511c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,7 +83,7 @@ jobs: if [[ "$RUNNER_OS" == "Windows" ]]; then cmd //c "${{ steps.sofa.outputs.vs_vsdevcmd }} \ && cd /d ${{ env.WORKSPACE_INSTALL_PATH }}/lib/python3/site-packages/ \ - && bash ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.sh ${{ env.WORKSPACE_INSTALL_PATH }}/lib/python3/site-packages/ + && bash ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.sh ${{ env.WORKSPACE_INSTALL_PATH }}/lib/python3/site-packages/" else cd ${{ env.WORKSPACE_INSTALL_PATH }}/lib/python3/site-packages/ /bin/bash ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.sh ${{ env.WORKSPACE_INSTALL_PATH }}/lib/python3/site-packages/ From bdea284b927e0a4abc93a76018c8571b83cba429 Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Fri, 20 Sep 2024 11:34:36 +0200 Subject: [PATCH 04/19] Add python path for stubgen --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25e2511c3..93ef10a59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,6 +79,9 @@ jobs: #Install stubgen ${{ steps.sofa.outputs.python_exe }} -m pip install mypy + + #Setup env + echo "PYTHONPATH=$WORKSPACE_ARTIFACT_PATH/lib/python3/site-packages" | tee -a $GITHUB_ENV if [[ "$RUNNER_OS" == "Windows" ]]; then cmd //c "${{ steps.sofa.outputs.vs_vsdevcmd }} \ From 13ba8e929250b2fe9a8504f1a0c738748b95e699 Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Fri, 20 Sep 2024 11:46:01 +0200 Subject: [PATCH 05/19] Try to find out what is wrong when generating --- scripts/generate_stubs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_stubs.sh b/scripts/generate_stubs.sh index 06ab69dbc..66e2cfcae 100644 --- a/scripts/generate_stubs.sh +++ b/scripts/generate_stubs.sh @@ -16,7 +16,7 @@ cd $WORK_DIR for module_name in *; do echo "Generating stubgen for $module_name" - stubgen -p $module_name --include-docstrings --inspect-mode + stubgen -v -p $module_name --include-docstrings --inspect-mode --ignore-errors if [ -d "./out/$module_name/" ]; then rsync -a ./out/$module_name/ ./$module_name/ fi From bf70e90551a42479a1f5dbb339e30531d8c77d78 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Mon, 16 Dec 2024 17:45:24 +0100 Subject: [PATCH 06/19] Change script to python , add Sofa.Component generation and allow the use of either pybind11 or mypy --- .github/workflows/ci.yml | 11 +- scripts/generate_stubs.py | 35 ++++ scripts/generate_stubs.sh | 24 --- scripts/sofaStubgen.py | 388 ++++++++++++++++++++++++++++++++++++++ scripts/utils.py | 96 ++++++++++ 5 files changed, 522 insertions(+), 32 deletions(-) create mode 100644 scripts/generate_stubs.py delete mode 100644 scripts/generate_stubs.sh create mode 100644 scripts/sofaStubgen.py create mode 100644 scripts/utils.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93ef10a59..16d3a1207 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,14 +83,9 @@ jobs: #Setup env echo "PYTHONPATH=$WORKSPACE_ARTIFACT_PATH/lib/python3/site-packages" | tee -a $GITHUB_ENV - if [[ "$RUNNER_OS" == "Windows" ]]; then - cmd //c "${{ steps.sofa.outputs.vs_vsdevcmd }} \ - && cd /d ${{ env.WORKSPACE_INSTALL_PATH }}/lib/python3/site-packages/ \ - && bash ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.sh ${{ env.WORKSPACE_INSTALL_PATH }}/lib/python3/site-packages/" - else - cd ${{ env.WORKSPACE_INSTALL_PATH }}/lib/python3/site-packages/ - /bin/bash ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.sh ${{ env.WORKSPACE_INSTALL_PATH }}/lib/python3/site-packages/ - fi + #For now use pybind11. This might be parametrized as an input of this action + ${{ steps.sofa.outputs.python_exe }} ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.py -d /home/paul/dev/build/sofa/lib/python3/site-packages -m Sofa --use_pybind11 + - name: Set env vars for artifacts shell: bash diff --git a/scripts/generate_stubs.py b/scripts/generate_stubs.py new file mode 100644 index 000000000..df41566f6 --- /dev/null +++ b/scripts/generate_stubs.py @@ -0,0 +1,35 @@ +import sys +import argparse +from utils import generate_module_stubs, generate_component_stubs + + +def main(site_package_dir,modules_name,use_pybind11 = False): + + work_dir = site_package_dir + modules = modules_name + + #Generate stubs using either pybind11-stubgen or mypy version of stubgen + + print(f"Generating stubgen for modules: {modules}") + + for module_name in modules: + generate_module_stubs(module_name, work_dir,use_pybind11) + + #Generate stubs for components using the factory + target_name="Sofa.Component" + generate_component_stubs(work_dir,target_name) + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog='generate_stubs', + description='Generates python stubs for SOFA modules') + + parser.add_argument('--use_pybind11',action='store_true',help='If flag is present, will use pybind11-stubgen instead of mypy stugen') + parser.add_argument('-d','--site_package_dir',nargs=1,help='Path to the site-package folder containing the SOFA modules') + parser.add_argument('-m','--modules_name',nargs='+',help='List of modules names to generate stubs for') + + args = parser.parse_args() + + main(args.site_package_dir,args.modules_name,args.use_pybind11) diff --git a/scripts/generate_stubs.sh b/scripts/generate_stubs.sh deleted file mode 100644 index 66e2cfcae..000000000 --- a/scripts/generate_stubs.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -usage() { - echo "Usage: generate_stubs.sh " - echo "This script automatically generates stub files for SofaPython3 modules" -} - -if [ "$#" -eq 1 ]; then - WORK_DIR=$1 -else - usage; exit 1 -fi - -cd $WORK_DIR - - -for module_name in *; do - echo "Generating stubgen for $module_name" - stubgen -v -p $module_name --include-docstrings --inspect-mode --ignore-errors - if [ -d "./out/$module_name/" ]; then - rsync -a ./out/$module_name/ ./$module_name/ - fi - rm -rf ./out -done diff --git a/scripts/sofaStubgen.py b/scripts/sofaStubgen.py new file mode 100644 index 000000000..74891766e --- /dev/null +++ b/scripts/sofaStubgen.py @@ -0,0 +1,388 @@ +# -*- coding: utf-8 -*- +""" +Autogenerate python wrapper for SOFA objects. + +Contributors: + damien.marchal@univ-lille.fr +""" +import re +import sys, os +import pprint +import argparse + +import Sofa +import Sofa.Simulation + +# some data field cannot have names as this is a python keywoards. +reserved = ["in", "with", "for", "if", "def", "class", "global"] + +def sofa_to_python_typename(name, short=False): + t = {"string":"str", + "bool": "bool", + "TagSet" : "object", + "BoundingBox" : "object", + "ComponentState" : "object", + "RGBAColor" : "object", + "OptionsGroup" : "object", + "SelectableItem" : "object", + "Material" : "object", + "DisplayFlags" : "object", + "d" : "float", + "f" : "float", + "i" : "int", + "I" : "int", + "L" : "int", + "l" : "int", + "b" : "int", + "LinkPath" : "LinkPath" + } + + SofaArray = "TypeHints.SofaArray" + if short: + SofaArray = "TypeHints.SofaArray" + + if name in t: + return t[name] + elif "Rigid" in name: + return SofaArray + elif "vector" in name: + return SofaArray + elif "set" in name: + return SofaArray + elif "Vec" in name: + return SofaArray + elif "Mat" in name: + return SofaArray + elif "Quat" in name: + return SofaArray + elif "map" in name: + return "object" + elif "fixed_array" in name: + return SofaArray + raise Exception("Missing type name,... ", name) + + +def sofa_datafields_to_constructor_arguments_list(data_fields, object_name, has_template=True): + #{'defaultValue': '0', + # 'group': '', + # 'help': 'if true, handle the events, otherwise ignore ' + # 'the events', + # 'name': 'listening', + # 'type': 'bool'}, + required_data_fields = [] + optional_data_fields = [] + + for data_field in data_fields: + name = data_field["name"] + if name in reserved: + print(f"Warning: {object_name} contains a the data field named '{name}' which is also a python keyword") + continue + + if " " in name: + print(f"Warning: this is an invalid arguments name: '{name}'") + continue + + if len(name) == 0: + print("Warning: empty data field empty name") + continue + + if data_field["isRequired"]: + required_data_fields.append(data_field) + else: + optional_data_fields.append(data_field) + + result_params = "" + if has_template: + result_params = "template: Optional[str] = None, " + result_params += ",".join([data["name"]+": "+sofa_to_python_typename(data["type"]) for data in required_data_fields]) + result_params += ",".join([data["name"]+": Optional["+sofa_to_python_typename(data["type"])+"] = None" for data in optional_data_fields]) + + ordered_fields = sorted(data_fields, key=lambda x: x["name"]) + + all = "\n ".join( [ data["name"]+" ("+sofa_to_python_typename(data["type"], short=True)+"): " + data["help"] for data in ordered_fields]) + all += "\n template (str): the type of degree of freedom" + return result_params, all + +def sofa_datafields_to_doc(data_fields): + p = "" + for data in data_fields: + name = data["name"] + help = data["help"] + if len(name) == 0: + continue + if " " in name: + continue + + if name in reserved: + p += "\t\t " + name + ": " + help + " (NB: use the kwargs syntax as name is a reserved word in python)\n\n" + else: + p += "\t\t " + name + ": " + help + "\n\n" + + return p + +def clean_sofa_text(t): + t = t.replace("\n","") + t = t.replace("'", "") + return t + +def sofa_datafields_to_typehints(data_fields, mode="Sofa"): + p = "" + p2 = "" + for data in data_fields: + name = data["name"] + help = clean_sofa_text(data["help"]) + type = sofa_to_python_typename(data["type"]) + if len(name) == 0: + continue + + if " " in name: + continue + + if name in reserved: + p += " " + name + ": " + help + " (NB: use the kwargs syntax as name is a reserved word in python)\n\n" + else: + p += f" {name}: Data[{type}] \n '{help}'\n\n" + p2 += f" {name}: Optional[{type} | LinkPath] = None \n '{help}'\n\n" + + return p, p2 + +def make_all_init_files(root_dir): + entries = {} + for dirpath, dirnames, filenames in os.walk(root_dir): + # Test if the root_dir and the dirpath are the same to skip first entry + # This could be implemented by using [os.walk(root_dir)][1:] but in that case + # python type hints get lost on my version 3.10. Maybe in a future python release + # this case will be handled + if os.path.abspath(root_dir) == os.path.abspath(dirpath): + continue + + initfile = open(dirpath + "/__init__.py", "wt") + res = [] + fres = [] + fres2 = [] + for d in dirnames: + res.append(d) + for f in filenames: + if f.endswith(".py") and f != "__init__.py": + fres.append(f[:-3]) + res.append(f[:-3]) + listc = "" + for r in res: + listc += " " + str(r) + "\n" + + initfile.write("# -*- coding: utf-8 -*-\n\n") + initfile.write(""" +\"\"\" +Sofa Component %s + +Summary: +======== + +%s + +\"\"\" +""" % (os.path.basename(dirpath), listc)) + +def wrapper_code(class_name, description, data_list, properties_doc, + class_typehints, params_typehints, + constructor_params_typehints, + constructor_params_docs): + properties_doc = "" + return f""" +class {class_name}(Object): + \"\"\"{description}\"\"\" + + def __init__(self, {constructor_params_typehints}): + \"\"\"{description} + + Args: + {constructor_params_docs} + \"\"\" + ... + +{class_typehints} + + @dataclasses.dataclass + class Parameters: + \"\"\"Parameter for the construction of the {class_name} component\"\"\" + +{params_typehints} + + def to_dict(self): + return dataclasses.asdict(self) + + @staticmethod + def new_parameters() -> Parameters: + return {class_name}.Parameters() +""" + +def documentation_code(class_name): + return """ +\"\"\" +Component %s + +.. autofunction:: %s + +Indices and tables +****************** + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` +\"\"\" +import Sofa +from Sofa.Core import Object, LinkPath +from Sofa.Component.TypeHints import Data, Optional, SofaArray +import dataclasses +""" % (class_name, class_name) + +def select_single_template(template_list): + """ returns a single template""" + for dim in ["Vec3", "Rigid3", "Vec2", "Rigid2", "Vec1", "Quatd"]: + for template in template_list: + if "Cuda" not in template and dim in template: + return template + + for template in template_list: + return template + + return "" + + +def create_typhint(pathname): + from pathlib import Path + path = Path(pathname).parent + if not os.path.exists(str(path)): + path.mkdir(parents=True) + c=""" +from typing import TypeVar, Generic, Optional as __optional__ +import numpy.typing +from Sofa.Core import Node, Object, LinkPath +import numpy +from numpy.typing import ArrayLike + +T = TypeVar("T", bound=object) + +# This is a generic type 'T' implemented without PEP 695 (as it needs python 3.12) +class Data(Generic[T]): + linkpath: LinkPath + value: T + +Optional = __optional__ +SofaArray = numpy.ndarray | list +""" + with open(pathname,"w") as w: + w.write(c) + +from pprint import pprint + +def load_component_list(target_name): + import json + import Sofa + import Sofa.Core + import SofaRuntime + + if target_name in ["Sofa"]: + # The binding is not a sofa plugin. + print("Loading a python module") + else: + print("Loading a sofa plugin") + SofaRuntime.importPlugin(target_name) + + json = json.loads(Sofa.Core.ObjectFactory.dump_json()) + + selected_entries = [] + for item in json: + selected_item = None + for type, entry in item["creator"].items(): + if entry["target"].startswith(target_name): + for data in entry["object"]["data"]: + data["isRequired"] = True + selected_item = item + + if selected_item: + selected_entries.append(selected_item) + + print("Number of objects ", len(json)) + print("Number of selected objects ", len(selected_entries)) + + return selected_entries + +def create_sofa_stubs(code_model, target_path): + blacklist = ["RequiredPlugin"] + + template_nodes = dict() + not_created_objects = list() + + create_typhint(target_path + "Sofa/Component/TypeHints.py") + + for entry in code_model: + data = {} + + class_name = entry["className"] + entry_templates = [n for n in entry["creator"].keys()] + description = entry["description"] + + selected_template = select_single_template(entry_templates) + selected_entry = entry["creator"][selected_template] + selected_class = selected_entry["class"] + selected_object = selected_entry["object"] + selected_target = selected_entry["target"] + + if len(selected_target) == 0: + selected_target = "unknown_target" + + object_name = class_name + "<" + selected_template + ">" + if object_name in blacklist or class_name in blacklist: + print("Skipping ", object_name) + continue + + + data = selected_object["data"] # obj.getDataFields() + links = selected_object["link"] + target = selected_target + + #load_existing_stub(selected_target, class_name) + + arguments_list, constructor_params_doc = sofa_datafields_to_constructor_arguments_list(data, object_name, len(entry_templates) > 0) + params_doc = sofa_datafields_to_doc(data) + class_typehint, params_typehint = sofa_datafields_to_typehints(data) + code = wrapper_code(class_name, description.strip(), arguments_list, params_doc, + class_typehint, params_typehint, + arguments_list, constructor_params_doc) + + pathname = target_path + target.replace(".","/") + "/" + full_component_name = target + "." + class_name + + # Creates the destination directory if needed. + os.makedirs(pathname, exist_ok=True) + + outfile = open(pathname + class_name + ".py", "wt") + outfile.write("# -*- coding: utf-8 -*-\n\n") + outfile.write(documentation_code(class_name)) + outfile.write(code) + outfile.close() + + if full_component_name in code_model: + raise Exception("Already existing entry") + + # In every directory, scan the object that are in and generates an init.py file + make_all_init_files(target_path) + return code_model + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog='sofa-component-stub-generator', + description='Generates python stubs that describes sofa components that have no binding') + parser.add_argument('--target_name', default="Sofa.Component") + parser.add_argument('--output_directory', default="out/") + args = parser.parse_args() + + target_name = args.target_name + output_directory = args.output_directory + + print(f"Generating SOFA's components python interfaces for {target_name}") + + components = load_component_list(target_name) + create_sofa_stubs(components, output_directory) + diff --git a/scripts/utils.py b/scripts/utils.py new file mode 100644 index 000000000..9e24356d2 --- /dev/null +++ b/scripts/utils.py @@ -0,0 +1,96 @@ +import sys +import os +import shutil +from sofaStubgen import load_component_list, create_sofa_stubs +from mypy.stubgen import parse_options, generate_stubs +import argparse + + + +#Method to use pybind11-stubgen +def pybind11_stub(module_name: str): + import logging + from pybind11_stubgen import CLIArgs, stub_parser_from_args, Printer, to_output_and_subdir, run, Writer + logging.basicConfig( + level=logging.INFO, + format="%(name)s - [%(levelname)7s] %(message)s", + ) + args = CLIArgs( + module_name=module_name, + output_dir='./out', + stub_extension="pyi", + # default ags: + root_suffix=None, + ignore_all_errors=False, + ignore_invalid_identifiers=None, + ignore_invalid_expressions=None, + ignore_unresolved_names=None, + exit_code=False, + numpy_array_wrap_with_annotated=False, + numpy_array_use_type_var=False, + numpy_array_remove_parameters=False, + enum_class_locations=[], + print_safe_value_reprs=None, + print_invalid_expressions_as_is=False, + dry_run=False) + + parser = stub_parser_from_args(args) + printer = Printer(invalid_expr_as_ellipses=not args.print_invalid_expressions_as_is) + + out_dir, sub_dir = to_output_and_subdir( + output_dir=args.output_dir, + module_name=args.module_name, + root_suffix=args.root_suffix, + ) + + run( + parser, + printer, + args.module_name, + out_dir, + sub_dir=sub_dir, + dry_run=args.dry_run, + writer=Writer(stub_ext=args.stub_extension), + ) + +#Generate stubs using either pybind11-stubgen or mypy version of stubgen +def generate_module_stubs(module_name, work_dir, usePybind11_stubgen = False): + print(f"Generating stubgen for {module_name} in {work_dir}") + + if(usePybind11_stubgen): + #Use pybind11 stubgen + #Could be replaced by an os call to + #subprocess.run(["pybind11-stubgen", module_name, "-o", "out"], check=True) + pybind11_stub(module_name) + else: + #Use mypy stubgen + options = parse_options(["-v","-p",module_name,"--include-docstrings","--no-analysis", "--ignore-errors"]) + generate_stubs(options) + + module_out_dir = os.path.join("out", module_name) + target_dir = os.path.join(work_dir, module_name) + + + if os.path.isdir(module_out_dir): + shutil.copytree(module_out_dir, target_dir, dirs_exist_ok=True) + print(f"Resync terminated for copying '{module_name}' to '{target_dir}'") + + shutil.rmtree("out", ignore_errors=True) + +#Generate stubs for components using the factory +def generate_component_stubs(work_dir,target_name): + print(f"Generating stubgen for all components in Sofa.Components using custom sofaStubgen.py") + + components = load_component_list(target_name) + create_sofa_stubs(components, "out/") + + + sofa_out_dir = os.path.join("out", "Sofa") + target_dir = os.path.join(work_dir, "Sofa") + + + if os.path.isdir(sofa_out_dir): + shutil.copytree(sofa_out_dir, target_dir, dirs_exist_ok=True) + print("Resync terminated.") + + shutil.rmtree("out", ignore_errors=True) From f3baf59f366d3d6554e1d4c5a1bb6f5222e45c0d Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Mon, 16 Dec 2024 18:02:59 +0100 Subject: [PATCH 07/19] fix action --- .github/workflows/ci.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16d3a1207..e270a580b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,10 +81,16 @@ jobs: ${{ steps.sofa.outputs.python_exe }} -m pip install mypy #Setup env - echo "PYTHONPATH=$WORKSPACE_ARTIFACT_PATH/lib/python3/site-packages" | tee -a $GITHUB_ENV - + OLD_PYTHONPATH=$PYTHONPATH + echo "PYTHONPATH=$WORKSPACE_INSTALL_PATH/lib/python3/site-packages" | tee -a $GITHUB_ENV + + #For now use pybind11. This might be parametrized as an input of this action - ${{ steps.sofa.outputs.python_exe }} ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.py -d /home/paul/dev/build/sofa/lib/python3/site-packages -m Sofa --use_pybind11 + ${{ steps.sofa.outputs.python_exe }} ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.py -d $WORKSPACE_INSTALL_PATH/lib/python3/site-packages -m Sofa --use_pybind11 + + #Go back to previous env + echo "PYTHONPATH=$OLD_PYTHONPATH" | tee -a $GITHUB_ENV + - name: Set env vars for artifacts From 2c7d65775c4c2cb05af3ed3de3dac4d1fb8dba59 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Mon, 16 Dec 2024 18:14:26 +0100 Subject: [PATCH 08/19] try fix env for stubgen --- .github/workflows/ci.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e270a580b..bb8be8557 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,6 +73,14 @@ jobs: ccache -s fi + + - name: Set env vars for stubfiles + shell: bash + run: | + #Setup env + echo "PYTHONPATH=$WORKSPACE_INSTALL_PATH/lib/python3/site-packages" | tee -a $GITHUB_ENV + + - name: Generate stubfiles shell: bash run: | @@ -80,16 +88,11 @@ jobs: #Install stubgen ${{ steps.sofa.outputs.python_exe }} -m pip install mypy - #Setup env - OLD_PYTHONPATH=$PYTHONPATH - echo "PYTHONPATH=$WORKSPACE_INSTALL_PATH/lib/python3/site-packages" | tee -a $GITHUB_ENV - - #For now use pybind11. This might be parametrized as an input of this action ${{ steps.sofa.outputs.python_exe }} ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.py -d $WORKSPACE_INSTALL_PATH/lib/python3/site-packages -m Sofa --use_pybind11 #Go back to previous env - echo "PYTHONPATH=$OLD_PYTHONPATH" | tee -a $GITHUB_ENV + echo "PYTHONPATH=" | tee -a $GITHUB_ENV From c951a590dd8e2566c4178e7456fe526aae7678b6 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 7 Jan 2025 14:02:31 +0100 Subject: [PATCH 09/19] try fix action --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb8be8557..b237a047d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,13 +80,16 @@ jobs: #Setup env echo "PYTHONPATH=$WORKSPACE_INSTALL_PATH/lib/python3/site-packages" | tee -a $GITHUB_ENV + echo $(ls $WORKSPACE_INSTALL_PATH/lib/) + echo $(ldd $WORKSPACE_INSTALL_PATH/lib/libSofa.Simulation.Graph.so) + - name: Generate stubfiles shell: bash run: | #Install stubgen - ${{ steps.sofa.outputs.python_exe }} -m pip install mypy + ${{ steps.sofa.outputs.python_exe }} -m pip install mypy pybind11-stubgen #For now use pybind11. This might be parametrized as an input of this action ${{ steps.sofa.outputs.python_exe }} ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.py -d $WORKSPACE_INSTALL_PATH/lib/python3/site-packages -m Sofa --use_pybind11 From fbf449347078f4f1cf023e7503eff3db3a2b3090 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 7 Jan 2025 14:14:50 +0100 Subject: [PATCH 10/19] Add env for SOFA libs --- .github/workflows/ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b237a047d..3e52396c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,16 @@ jobs: shell: bash run: | #Setup env + # Set env vars for tests + if [[ "$RUNNER_OS" == "Windows" ]]; then + echo "$WORKSPACE_INSTALL_PATH/lib" >> $GITHUB_PATH + echo "$WORKSPACE_INSTALL_PATH/bin" >> $GITHUB_PATH + elif [[ "$RUNNER_OS" == "macOS" ]]; then + echo "DYLD_LIBRARY_PATH=$WORKSPACE_ARTIFACT_PATH/lib:$SOFA_ROOT/lib:$DYLD_LIBRARY_PATH" | tee -a $GITHUB_ENV + else # Linux + echo "LD_LIBRARY_PATH=$WORKSPACE_ARTIFACT_PATH/lib:$SOFA_ROOT/lib:$LD_LIBRARY_PATH" | tee -a $GITHUB_ENV + fi + echo "PYTHONPATH=$WORKSPACE_INSTALL_PATH/lib/python3/site-packages" | tee -a $GITHUB_ENV echo $(ls $WORKSPACE_INSTALL_PATH/lib/) From b7742273eef0f34ace51070a37e4573457945775 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 7 Jan 2025 14:22:22 +0100 Subject: [PATCH 11/19] Remove output --- .github/workflows/ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e52396c1..0f90c2fed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,10 +90,6 @@ jobs: echo "PYTHONPATH=$WORKSPACE_INSTALL_PATH/lib/python3/site-packages" | tee -a $GITHUB_ENV - echo $(ls $WORKSPACE_INSTALL_PATH/lib/) - echo $(ldd $WORKSPACE_INSTALL_PATH/lib/libSofa.Simulation.Graph.so) - - - name: Generate stubfiles shell: bash run: | From 6f6dd5247a82c81a821c9cb75e853b21eb756d41 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 7 Jan 2025 14:30:46 +0100 Subject: [PATCH 12/19] Add echot osee line --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed717c725..1ccca0387 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,7 +98,9 @@ jobs: ${{ steps.sofa.outputs.python_exe }} -m pip install mypy pybind11-stubgen #For now use pybind11. This might be parametrized as an input of this action - ${{ steps.sofa.outputs.python_exe }} ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.py -d $WORKSPACE_INSTALL_PATH/lib/python3/site-packages -m Sofa --use_pybind11 + echo "Launching the stub generation with '${{ steps.sofa.outputs.python_exe }} ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.py -d $WORKSPACE_INSTALL_PATH/lib/python3/site-packages -m Sofa --use_pybind11'" + + ${{ steps.sofa.outputs.python_exe }} ${{ env.WORKSPACE_SRC_PATH }}/scripts/generate_stubs.py -d "$WORKSPACE_INSTALL_PATH/lib/python3/site-packages" -m Sofa --use_pybind11 #Go back to previous env echo "PYTHONPATH=" | tee -a $GITHUB_ENV From 98452f7a6cc6ca59cb02b723165e5d46b53073f2 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 7 Jan 2025 14:38:57 +0100 Subject: [PATCH 13/19] Add more informaiton --- scripts/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/utils.py b/scripts/utils.py index 9e24356d2..2216f39ad 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -68,6 +68,8 @@ def generate_module_stubs(module_name, work_dir, usePybind11_stubgen = False): generate_stubs(options) module_out_dir = os.path.join("out", module_name) + print(work_dir) + print(module_name) target_dir = os.path.join(work_dir, module_name) From 20f7dacaaac858513f02b7349567c4650eef92da Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 7 Jan 2025 14:46:41 +0100 Subject: [PATCH 14/19] Fix python script --- scripts/generate_stubs.py | 2 +- scripts/utils.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/generate_stubs.py b/scripts/generate_stubs.py index df41566f6..f221ea72f 100644 --- a/scripts/generate_stubs.py +++ b/scripts/generate_stubs.py @@ -5,7 +5,7 @@ def main(site_package_dir,modules_name,use_pybind11 = False): - work_dir = site_package_dir + work_dir = site_package_dir[0] modules = modules_name #Generate stubs using either pybind11-stubgen or mypy version of stubgen diff --git a/scripts/utils.py b/scripts/utils.py index 2216f39ad..9e24356d2 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -68,8 +68,6 @@ def generate_module_stubs(module_name, work_dir, usePybind11_stubgen = False): generate_stubs(options) module_out_dir = os.path.join("out", module_name) - print(work_dir) - print(module_name) target_dir = os.path.join(work_dir, module_name) From f34003a0e55f84b41bdc5d21e35460d825260fbf Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 7 Jan 2025 16:11:31 +0100 Subject: [PATCH 15/19] Try fix unit tests --- scripts/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/utils.py b/scripts/utils.py index 9e24356d2..9b93d941d 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -85,8 +85,8 @@ def generate_component_stubs(work_dir,target_name): create_sofa_stubs(components, "out/") - sofa_out_dir = os.path.join("out", "Sofa") - target_dir = os.path.join(work_dir, "Sofa") + sofa_out_dir = os.path.join("out",*target_name.split(.)) + target_dir = os.path.join(work_dir, *target_name.split(.)) if os.path.isdir(sofa_out_dir): From 27ddc89039251171005cb3353cdd311685d1179d Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 7 Jan 2025 16:18:15 +0100 Subject: [PATCH 16/19] Fix mistake in split --- scripts/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/utils.py b/scripts/utils.py index 9b93d941d..3ee71209d 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -7,6 +7,7 @@ + #Method to use pybind11-stubgen def pybind11_stub(module_name: str): import logging @@ -53,6 +54,7 @@ def pybind11_stub(module_name: str): writer=Writer(stub_ext=args.stub_extension), ) + #Generate stubs using either pybind11-stubgen or mypy version of stubgen def generate_module_stubs(module_name, work_dir, usePybind11_stubgen = False): print(f"Generating stubgen for {module_name} in {work_dir}") @@ -85,8 +87,8 @@ def generate_component_stubs(work_dir,target_name): create_sofa_stubs(components, "out/") - sofa_out_dir = os.path.join("out",*target_name.split(.)) - target_dir = os.path.join(work_dir, *target_name.split(.)) + sofa_out_dir = os.path.join("out",*target_name.split('.')) + target_dir = os.path.join(work_dir, *target_name.split('.')) if os.path.isdir(sofa_out_dir): From 325b68e3cb17afcc454a1825d5acbdacd76ae372 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 7 Jan 2025 17:09:06 +0100 Subject: [PATCH 17/19] add filter to skip synchronizing files that already exist in dest folder. This should be removed once create_sofa_stubs is able to merge the generated stubs with existing implementation --- scripts/utils.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/scripts/utils.py b/scripts/utils.py index 3ee71209d..5430b7beb 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -54,6 +54,34 @@ def pybind11_stub(module_name: str): writer=Writer(stub_ext=args.stub_extension), ) +# This class is used with shutil.copytree to skip already existing files in dest folder. +class noOverrideFilter: + def __init__(self,fromFolder,destFolder): + self.fromFolder = fromFolder + self.destFolder = destFolder + + def findDestPath(self,currentRelativeToFromPath): + + absoluteFromFolder = os.path.abspath(self.fromFolder) + + #here we remove the first folder because it is the output folder + destPath = '' + head = os.path.abspath(currentRelativeToFromPath) + while(head != absoluteFromFolder): + head,tail = os.path.split(head) + destPath = os.path.join(destPath,tail) + + return os.path.abspath(os.path.join(self.destFolder,destPath)) + + + def __call__(self,path, objectList): + + destPath = self.findDestPath(path) + returnList = [] + for obj in objectList: + if(os.path.isfile(os.path.join(destPath,obj))): + returnList.append(obj) + return returnList #Generate stubs using either pybind11-stubgen or mypy version of stubgen def generate_module_stubs(module_name, work_dir, usePybind11_stubgen = False): @@ -90,9 +118,10 @@ def generate_component_stubs(work_dir,target_name): sofa_out_dir = os.path.join("out",*target_name.split('.')) target_dir = os.path.join(work_dir, *target_name.split('.')) - if os.path.isdir(sofa_out_dir): - shutil.copytree(sofa_out_dir, target_dir, dirs_exist_ok=True) + #For now on we don't want no overriting of file, we thus use the parameter 'ignore=noOverrideFilter(sofa_out_dir,target_dir)' to skip files already existing in dest folder because it might brake the lib. + # When file merging is supported by create_sofa_stubs this will be removed because then, we will want to override the dest files because we've already did the merge + shutil.copytree(sofa_out_dir, target_dir, dirs_exist_ok=True, ignore=noOverrideFilter(sofa_out_dir,target_dir)) print("Resync terminated.") shutil.rmtree("out", ignore_errors=True) From 8162e6c7d56ad5460c617fafbca237a5ca457b39 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 7 Jan 2025 17:16:26 +0100 Subject: [PATCH 18/19] Fix windows stubgen by printing a warning instead of throwing an exception when input type is unknown --- scripts/sofaStubgen.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/sofaStubgen.py b/scripts/sofaStubgen.py index 74891766e..87d7bb65b 100644 --- a/scripts/sofaStubgen.py +++ b/scripts/sofaStubgen.py @@ -59,7 +59,9 @@ def sofa_to_python_typename(name, short=False): return "object" elif "fixed_array" in name: return SofaArray - raise Exception("Missing type name,... ", name) + + print(f"Warning: unknown type '${name}' encountered, falling back to generic object type") + return "object" def sofa_datafields_to_constructor_arguments_list(data_fields, object_name, has_template=True): From 793803e84ea333137c90d71e4de2b2943f71aef0 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 7 Jan 2025 17:33:11 +0100 Subject: [PATCH 19/19] Fix warning print --- scripts/sofaStubgen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sofaStubgen.py b/scripts/sofaStubgen.py index 87d7bb65b..6016501f8 100644 --- a/scripts/sofaStubgen.py +++ b/scripts/sofaStubgen.py @@ -60,7 +60,7 @@ def sofa_to_python_typename(name, short=False): elif "fixed_array" in name: return SofaArray - print(f"Warning: unknown type '${name}' encountered, falling back to generic object type") + print(f"Warning: unknown type '{name}' encountered, falling back to generic object type") return "object"