[ADD] guess requirements feature

README.md

@@ -33,6 +33,7 @@ and provides helpers to run (and replay) migrations until it works.
     * [Command ``init``](#command-init)
     * [Command ``pull-submodule``](#command-pull-submodule)
     * [Command ``get-code``](#command-get-code)
+    * [Command ``guess-requirement``](#command-guess-requirement)
     * [Command ``docker-build``](#command-docker-build)
     * [Command ``run``](#command-run)
     * [Command ``install-from-csv``](#command-install-from-csv)
@@ -226,6 +227,32 @@ if you want to update the code of some given versions, you can provide an extra
 odoo-openupgrade-wizard get-code --versions 10.0,11.0
 ```
 
+<a name="command-guess-requirement"/>
+
+## Command: ``guess-requirement``
+
+**Prerequites:** init + get-code
+
+```shell
+odoo-openupgrade-wizard guess-requirement
+```
+
+Analyze the list of the modules defined in your ``modules.csv`` file.
+For each module and each version, this command tries to parse the
+according ``__manifest__.py`` file (and, if possible the according ``setup.py`` file).
+Finally, it overwrite the requirements.txt files present in each env folder. (python and debian requirements).
+
+For exemple, here is the content of the ``extra_python_requirements.txt`` file,
+when ``barcodes_generator_abstract`` and ``l10n_fr_siret`` are installed.
+
+```
+# Required by the module(s): barcodes_generator_abstract
+python-barcode
+
+# Required by the module(s): l10n_fr_siret
+python-stdnum>=1.18
+```
+
 <a name="command-docker-build"/>
 
 ## Command: ``docker-build``

newsfragments/guess-requirement.feature

@@ -0,0 +1,4 @@
+add a new command ``odoo-openupgrade-wizard guess-requirement`` that
+initialize the python and bin requirements files, based on the odoo
+modules present in the modules.csv file, and analyzing code.
+(``__manifest__.py`` and ``setup.py`` files)

odoo_openupgrade_wizard/cli/cli.py

@@ -24,6 +24,7 @@ from odoo_openupgrade_wizard.cli.cli_generate_module_analysis import (
     generate_module_analysis,
 )
 from odoo_openupgrade_wizard.cli.cli_get_code import get_code
+from odoo_openupgrade_wizard.cli.cli_guess_requirement import guess_requirement
 from odoo_openupgrade_wizard.cli.cli_init import init
 from odoo_openupgrade_wizard.cli.cli_install_from_csv import install_from_csv
 from odoo_openupgrade_wizard.cli.cli_psql import psql
@@ -158,6 +159,7 @@ main.add_command(estimate_workload)
 main.add_command(execute_script_python)
 main.add_command(execute_script_sql)
 main.add_command(generate_module_analysis)
+main.add_command(guess_requirement)
 main.add_command(get_code)
 main.add_command(init)
 main.add_command(install_from_csv)

odoo_openupgrade_wizard/cli/cli_guess_requirement.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+
+import click
+
+from odoo_openupgrade_wizard.tools.tools_odoo import (
+    get_odoo_env_path,
+    get_odoo_modules_from_csv,
+)
+from odoo_openupgrade_wizard.tools.tools_odoo_module import Analysis
+from odoo_openupgrade_wizard.tools.tools_system import (
+    ensure_file_exists_from_template,
+)
+
+
+@click.command()
+@click.option(
+    "--extra-modules",
+    "extra_modules_list",
+    # TODO, add a callback to check the quality of the argument
+    help="Coma separated modules to analyse. If not set, the modules.csv"
+    " file will be used to define the list of module to analyse."
+    "Ex: 'account,product,base'",
+)
+@click.pass_context
+def guess_requirement(ctx, extra_modules_list):
+    # Analyse
+    analysis = Analysis(ctx)
+
+    if extra_modules_list:
+        module_list = extra_modules_list.split(",")
+    else:
+        module_list = get_odoo_modules_from_csv(ctx.obj["module_file_path"])
+
+    analysis.analyse_module_version(ctx, module_list)
+    analysis.analyse_missing_module()
+    result = analysis.get_requirements(ctx)
+
+    for odoo_version in [x for x in ctx.obj["config"]["odoo_versions"]]:
+        path_version = get_odoo_env_path(ctx, odoo_version)
+        ensure_file_exists_from_template(
+            path_version / Path("extra_python_requirements.txt"),
+            "odoo/extra_python_requirements.txt.j2",
+            dependencies=result[odoo_version]["python"],
+        )
+
+        ensure_file_exists_from_template(
+            path_version / Path("extra_debian_requirements.txt"),
+            "odoo/extra_debian_requirements.txt.j2",
+            dependencies=result[odoo_version]["bin"],
+        )

odoo_openupgrade_wizard/cli/cli_init.py

@@ -156,12 +156,14 @@ def init(
         ensure_file_exists_from_template(
             path_version / Path("extra_python_requirements.txt"),
             "odoo/extra_python_requirements.txt.j2",
+            dependencies={},
         )
 
         # Create debian requirements file
         ensure_file_exists_from_template(
             path_version / Path("extra_debian_requirements.txt"),
             "odoo/extra_debian_requirements.txt.j2",
+            dependencies={},
         )
 
         # Create odoo config file

odoo_openupgrade_wizard/templates/odoo/extra_debian_requirements.txt.j2

@@ -0,0 +1,6 @@
+{% for bin_lib, module_list in dependencies.items() %}
+
+# Required by the module(s): {{ ','.join(module_list) }}
+{{ bin_lib }}
+
+{% endfor %}

odoo_openupgrade_wizard/templates/odoo/extra_python_requirements.txt.j2

@@ -7,3 +7,10 @@ git+https://github.com/OCA/openupgradelib@master#egg=openupgradelib
 # dependencies of the module OCA/server-tools 'upgrade_analysis'
 odoorpc
 mako
+
+{%- for python_lib, module_list in dependencies.items() %}
+
+# Required by the module(s): {{ ','.join(module_list) }}
+{{ python_lib }}
+
+{%- endfor %}

odoo_openupgrade_wizard/tools/tools_odoo_module.py

@@ -1,3 +1,4 @@
+import ast
 import importlib
 import os
 from functools import total_ordering
@@ -110,6 +111,27 @@ class Analysis(object):
             for module_version in odoo_module.module_versions.values():
                 module_version.estimate_workload(ctx)
 
+    def get_requirements(self, ctx):
+        logger.info("Get requirements ...")
+        result = {x: {"python": {}, "bin": {}} for x in self.all_versions}
+
+        for odoo_module in self.modules:
+            for module_version in odoo_module.module_versions.values():
+                module_result = module_version.get_requirements(ctx)
+                for python_lib in module_result["python"]:
+                    if (
+                        python_lib
+                        not in result[module_result["version"]]["python"]
+                    ):
+                        result[module_result["version"]]["python"][
+                            python_lib
+                        ] = [module_result["module_name"]]
+                    else:
+                        result[module_result["version"]]["python"][
+                            python_lib
+                        ].append(module_result["module_name"])
+        return result
+
     def _generate_module_version_first_version(self, ctx, module_list):
         logger.info(f"Analyse version {self.initial_version}. (First version)")
 
@@ -473,7 +495,7 @@ class OdooModuleVersion(object):
         "doc",
         "description",
     ]
-    _exclude_files = ["__openerp__.py", "__manifest__.py"]
+    _manifest_files = ["__openerp__.py", "__manifest__.py"]
 
     _file_extensions = [".py", ".xml", ".js"]
 
@@ -515,6 +537,59 @@ class OdooModuleVersion(object):
         else:
             return False
 
+    def get_requirements(self, ctx):
+        result = {
+            "python": [],
+            "bin": [],
+            "module_name": self.odoo_module.name,
+            "version": self.version,
+        }
+        manifest_path = False
+        for manifest_name in self._manifest_files:
+            if not self.odoo_module.name or not self.addon_path:
+                continue
+            manifest_path = (
+                self.addon_path / self.odoo_module.name / "__manifest__.py"
+            )
+            if manifest_path.exists():
+                break
+        if not manifest_path or not manifest_path.exists():
+            return result
+        manifest = ast.literal_eval(open(manifest_path).read())
+        python_libs = manifest.get("external_dependencies", {}).get(
+            "python", []
+        )
+        bin_libs = manifest.get("external_dependencies", {}).get("bin", [])
+        result["bin"] = bin_libs
+        # Handle specific replacement in the setup folder
+        setup_path = (
+            self.addon_path / "setup" / self.odoo_module.name / "setup.py"
+        )
+        if setup_path.exists():
+            with setup_path.open(mode="r", encoding="utf-8") as f:
+                setup_content = f.read()
+            setup_content = (
+                setup_content.replace("import setuptools", "")
+                .replace("setuptools.setup(", "{")
+                .replace("setup_requires=", "'setup_requires':")
+                .replace("odoo_addon=", "'odoo_addon':")
+                .replace(")", "}")
+            )
+            setup_eval = ast.literal_eval(setup_content)
+            odoo_addon_value = setup_eval.get("odoo_addon", False)
+            if type(odoo_addon_value) is dict:
+                python_replacements = odoo_addon_value.get(
+                    "external_dependencies_override", {}
+                ).get("python", {})
+                for k, v in python_replacements.items():
+                    if k in python_libs:
+                        python_libs.remove(k)
+                        result["python"].append(v)
+
+        result["python"] += python_libs
+
+        return result
+
     def estimate_workload(self, ctx):
         settings = ctx.obj["config"]["workload_settings"]
         port_minimal_time = settings["port_minimal_time"]
@@ -605,7 +680,7 @@ class OdooModuleVersion(object):
             if set(Path(relative_path).parts) & set(self._exclude_directories):
                 continue
             for name in files:
-                if name in self._exclude_files:
+                if name in self._manifest_files:
                     continue
                 filename, file_extension = os.path.splitext(name)
                 if file_extension in self._file_extensions:

tests/cli_09_guess_requirement_test.py

@@ -0,0 +1,23 @@
+import filecmp
+import unittest
+from pathlib import Path
+
+from . import cli_runner_invoke, move_to_test_folder
+
+
+class TestCliGuessRequirement(unittest.TestCase):
+    def test_cli_guess_requirement(self):
+        move_to_test_folder()
+
+        expected_folder_path = Path("../output_expected").absolute()
+
+        cli_runner_invoke(
+            ["guess-requirement", "--extra-modules=sentry"],
+        )
+
+        relative_path = Path("src/env_14.0/extra_python_requirements.txt")
+
+        assert filecmp.cmp(
+            relative_path,
+            expected_folder_path / relative_path,
+        )

tests/data/output_expected/src/env_14.0/extra_python_requirements.txt

@@ -0,0 +1,12 @@
+# Mandatory library used in all odoo-openupgrade-wizard
+# Note: As the openupgradelib is not allways up to date in pypi,
+# we use the github master url.
+git+https://github.com/OCA/openupgradelib@master#egg=openupgradelib
+
+# Library used to run generate-module-analysis command
+# dependencies of the module OCA/server-tools 'upgrade_analysis'
+odoorpc
+mako
+
+# Required by the module(s): sentry
+sentry_sdk<=1.9.0