diff --git a/README.md b/README.md index e3cb191..de38361 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ and provides helpers to run (and replay) migrations until it works. * [Command ``init``](#command-init) * [Command ``pull-submodule``](#command-pull-submodule) * [Command ``get-code``](#command-get-code) + * [Command ``guess-requirement``](#command-guess-requirement) * [Command ``docker-build``](#command-docker-build) * [Command ``run``](#command-run) * [Command ``install-from-csv``](#command-install-from-csv) @@ -226,6 +227,32 @@ if you want to update the code of some given versions, you can provide an extra odoo-openupgrade-wizard get-code --versions 10.0,11.0 ``` + + +## Command: ``guess-requirement`` + +**Prerequites:** init + get-code + +```shell +odoo-openupgrade-wizard guess-requirement +``` + +Analyze the list of the modules defined in your ``modules.csv`` file. +For each module and each version, this command tries to parse the +according ``__manifest__.py`` file (and, if possible the according ``setup.py`` file). +Finally, it overwrite the requirements.txt files present in each env folder. (python and debian requirements). + +For exemple, here is the content of the ``extra_python_requirements.txt`` file, +when ``barcodes_generator_abstract`` and ``l10n_fr_siret`` are installed. + +``` +# Required by the module(s): barcodes_generator_abstract +python-barcode + +# Required by the module(s): l10n_fr_siret +python-stdnum>=1.18 +``` + ## Command: ``docker-build`` diff --git a/newsfragments/guess-requirement.feature b/newsfragments/guess-requirement.feature new file mode 100644 index 0000000..daeb99b --- /dev/null +++ b/newsfragments/guess-requirement.feature @@ -0,0 +1,4 @@ +add a new command ``odoo-openupgrade-wizard guess-requirement`` that +initialize the python and bin requirements files, based on the odoo +modules present in the modules.csv file, and analyzing code. +(``__manifest__.py`` and ``setup.py`` files) diff --git a/odoo_openupgrade_wizard/cli/cli.py b/odoo_openupgrade_wizard/cli/cli.py index 6282a50..0d73c0e 100644 --- a/odoo_openupgrade_wizard/cli/cli.py +++ b/odoo_openupgrade_wizard/cli/cli.py @@ -24,6 +24,7 @@ from odoo_openupgrade_wizard.cli.cli_generate_module_analysis import ( generate_module_analysis, ) from odoo_openupgrade_wizard.cli.cli_get_code import get_code +from odoo_openupgrade_wizard.cli.cli_guess_requirement import guess_requirement from odoo_openupgrade_wizard.cli.cli_init import init from odoo_openupgrade_wizard.cli.cli_install_from_csv import install_from_csv from odoo_openupgrade_wizard.cli.cli_psql import psql @@ -158,6 +159,7 @@ main.add_command(estimate_workload) main.add_command(execute_script_python) main.add_command(execute_script_sql) main.add_command(generate_module_analysis) +main.add_command(guess_requirement) main.add_command(get_code) main.add_command(init) main.add_command(install_from_csv) diff --git a/odoo_openupgrade_wizard/cli/cli_guess_requirement.py b/odoo_openupgrade_wizard/cli/cli_guess_requirement.py new file mode 100644 index 0000000..d024046 --- /dev/null +++ b/odoo_openupgrade_wizard/cli/cli_guess_requirement.py @@ -0,0 +1,50 @@ +from pathlib import Path + +import click + +from odoo_openupgrade_wizard.tools.tools_odoo import ( + get_odoo_env_path, + get_odoo_modules_from_csv, +) +from odoo_openupgrade_wizard.tools.tools_odoo_module import Analysis +from odoo_openupgrade_wizard.tools.tools_system import ( + ensure_file_exists_from_template, +) + + +@click.command() +@click.option( + "--extra-modules", + "extra_modules_list", + # TODO, add a callback to check the quality of the argument + help="Coma separated modules to analyse. If not set, the modules.csv" + " file will be used to define the list of module to analyse." + "Ex: 'account,product,base'", +) +@click.pass_context +def guess_requirement(ctx, extra_modules_list): + # Analyse + analysis = Analysis(ctx) + + if extra_modules_list: + module_list = extra_modules_list.split(",") + else: + module_list = get_odoo_modules_from_csv(ctx.obj["module_file_path"]) + + analysis.analyse_module_version(ctx, module_list) + analysis.analyse_missing_module() + result = analysis.get_requirements(ctx) + + for odoo_version in [x for x in ctx.obj["config"]["odoo_versions"]]: + path_version = get_odoo_env_path(ctx, odoo_version) + ensure_file_exists_from_template( + path_version / Path("extra_python_requirements.txt"), + "odoo/extra_python_requirements.txt.j2", + dependencies=result[odoo_version]["python"], + ) + + ensure_file_exists_from_template( + path_version / Path("extra_debian_requirements.txt"), + "odoo/extra_debian_requirements.txt.j2", + dependencies=result[odoo_version]["bin"], + ) diff --git a/odoo_openupgrade_wizard/cli/cli_init.py b/odoo_openupgrade_wizard/cli/cli_init.py index c3bfc44..806223b 100644 --- a/odoo_openupgrade_wizard/cli/cli_init.py +++ b/odoo_openupgrade_wizard/cli/cli_init.py @@ -156,12 +156,14 @@ def init( ensure_file_exists_from_template( path_version / Path("extra_python_requirements.txt"), "odoo/extra_python_requirements.txt.j2", + dependencies={}, ) # Create debian requirements file ensure_file_exists_from_template( path_version / Path("extra_debian_requirements.txt"), "odoo/extra_debian_requirements.txt.j2", + dependencies={}, ) # Create odoo config file diff --git a/odoo_openupgrade_wizard/templates/odoo/extra_debian_requirements.txt.j2 b/odoo_openupgrade_wizard/templates/odoo/extra_debian_requirements.txt.j2 index e69de29..3a9a936 100644 --- a/odoo_openupgrade_wizard/templates/odoo/extra_debian_requirements.txt.j2 +++ b/odoo_openupgrade_wizard/templates/odoo/extra_debian_requirements.txt.j2 @@ -0,0 +1,6 @@ +{% for bin_lib, module_list in dependencies.items() %} + +# Required by the module(s): {{ ','.join(module_list) }} +{{ bin_lib }} + +{% endfor %} diff --git a/odoo_openupgrade_wizard/templates/odoo/extra_python_requirements.txt.j2 b/odoo_openupgrade_wizard/templates/odoo/extra_python_requirements.txt.j2 index 75f29a2..ab64a92 100644 --- a/odoo_openupgrade_wizard/templates/odoo/extra_python_requirements.txt.j2 +++ b/odoo_openupgrade_wizard/templates/odoo/extra_python_requirements.txt.j2 @@ -7,3 +7,10 @@ git+https://github.com/OCA/openupgradelib@master#egg=openupgradelib # dependencies of the module OCA/server-tools 'upgrade_analysis' odoorpc mako + +{%- for python_lib, module_list in dependencies.items() %} + +# Required by the module(s): {{ ','.join(module_list) }} +{{ python_lib }} + +{%- endfor %} diff --git a/odoo_openupgrade_wizard/tools/tools_odoo_module.py b/odoo_openupgrade_wizard/tools/tools_odoo_module.py index 7d9e998..7d877b6 100644 --- a/odoo_openupgrade_wizard/tools/tools_odoo_module.py +++ b/odoo_openupgrade_wizard/tools/tools_odoo_module.py @@ -1,3 +1,4 @@ +import ast import importlib import os from functools import total_ordering @@ -110,6 +111,27 @@ class Analysis(object): for module_version in odoo_module.module_versions.values(): module_version.estimate_workload(ctx) + def get_requirements(self, ctx): + logger.info("Get requirements ...") + result = {x: {"python": {}, "bin": {}} for x in self.all_versions} + + for odoo_module in self.modules: + for module_version in odoo_module.module_versions.values(): + module_result = module_version.get_requirements(ctx) + for python_lib in module_result["python"]: + if ( + python_lib + not in result[module_result["version"]]["python"] + ): + result[module_result["version"]]["python"][ + python_lib + ] = [module_result["module_name"]] + else: + result[module_result["version"]]["python"][ + python_lib + ].append(module_result["module_name"]) + return result + def _generate_module_version_first_version(self, ctx, module_list): logger.info(f"Analyse version {self.initial_version}. (First version)") @@ -473,7 +495,7 @@ class OdooModuleVersion(object): "doc", "description", ] - _exclude_files = ["__openerp__.py", "__manifest__.py"] + _manifest_files = ["__openerp__.py", "__manifest__.py"] _file_extensions = [".py", ".xml", ".js"] @@ -515,6 +537,59 @@ class OdooModuleVersion(object): else: return False + def get_requirements(self, ctx): + result = { + "python": [], + "bin": [], + "module_name": self.odoo_module.name, + "version": self.version, + } + manifest_path = False + for manifest_name in self._manifest_files: + if not self.odoo_module.name or not self.addon_path: + continue + manifest_path = ( + self.addon_path / self.odoo_module.name / "__manifest__.py" + ) + if manifest_path.exists(): + break + if not manifest_path or not manifest_path.exists(): + return result + manifest = ast.literal_eval(open(manifest_path).read()) + python_libs = manifest.get("external_dependencies", {}).get( + "python", [] + ) + bin_libs = manifest.get("external_dependencies", {}).get("bin", []) + result["bin"] = bin_libs + # Handle specific replacement in the setup folder + setup_path = ( + self.addon_path / "setup" / self.odoo_module.name / "setup.py" + ) + if setup_path.exists(): + with setup_path.open(mode="r", encoding="utf-8") as f: + setup_content = f.read() + setup_content = ( + setup_content.replace("import setuptools", "") + .replace("setuptools.setup(", "{") + .replace("setup_requires=", "'setup_requires':") + .replace("odoo_addon=", "'odoo_addon':") + .replace(")", "}") + ) + setup_eval = ast.literal_eval(setup_content) + odoo_addon_value = setup_eval.get("odoo_addon", False) + if type(odoo_addon_value) is dict: + python_replacements = odoo_addon_value.get( + "external_dependencies_override", {} + ).get("python", {}) + for k, v in python_replacements.items(): + if k in python_libs: + python_libs.remove(k) + result["python"].append(v) + + result["python"] += python_libs + + return result + def estimate_workload(self, ctx): settings = ctx.obj["config"]["workload_settings"] port_minimal_time = settings["port_minimal_time"] @@ -605,7 +680,7 @@ class OdooModuleVersion(object): if set(Path(relative_path).parts) & set(self._exclude_directories): continue for name in files: - if name in self._exclude_files: + if name in self._manifest_files: continue filename, file_extension = os.path.splitext(name) if file_extension in self._file_extensions: diff --git a/tests/cli_09_guess_requirement_test.py b/tests/cli_09_guess_requirement_test.py new file mode 100644 index 0000000..c9cd96f --- /dev/null +++ b/tests/cli_09_guess_requirement_test.py @@ -0,0 +1,23 @@ +import filecmp +import unittest +from pathlib import Path + +from . import cli_runner_invoke, move_to_test_folder + + +class TestCliGuessRequirement(unittest.TestCase): + def test_cli_guess_requirement(self): + move_to_test_folder() + + expected_folder_path = Path("../output_expected").absolute() + + cli_runner_invoke( + ["guess-requirement", "--extra-modules=sentry"], + ) + + relative_path = Path("src/env_14.0/extra_python_requirements.txt") + + assert filecmp.cmp( + relative_path, + expected_folder_path / relative_path, + ) diff --git a/tests/data/output_expected/src/env_14.0/extra_python_requirements.txt b/tests/data/output_expected/src/env_14.0/extra_python_requirements.txt new file mode 100644 index 0000000..b091669 --- /dev/null +++ b/tests/data/output_expected/src/env_14.0/extra_python_requirements.txt @@ -0,0 +1,12 @@ +# Mandatory library used in all odoo-openupgrade-wizard +# Note: As the openupgradelib is not allways up to date in pypi, +# we use the github master url. +git+https://github.com/OCA/openupgradelib@master#egg=openupgradelib + +# Library used to run generate-module-analysis command +# dependencies of the module OCA/server-tools 'upgrade_analysis' +odoorpc +mako + +# Required by the module(s): sentry +sentry_sdk<=1.9.0