From 44fd15b45c09dc113c20550d0ed8ef8b064ae9dc Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 1 Jun 2022 13:10:12 +0200 Subject: [PATCH] [ADD] analyse-workload (2/2) WIP --- .../cli_estimate_workload.py | 4 +- odoo_openupgrade_wizard/cli_options.py | 4 +- .../configuration_version_dependant.py | 29 ++ odoo_openupgrade_wizard/templates.py | 67 ++- odoo_openupgrade_wizard/tools_odoo_module.py | 436 ++++++++++++++++-- tests/cli_01_init_test.py | 3 +- 6 files changed, 484 insertions(+), 59 deletions(-) diff --git a/odoo_openupgrade_wizard/cli_estimate_workload.py b/odoo_openupgrade_wizard/cli_estimate_workload.py index 7dff4a5..e88d10c 100644 --- a/odoo_openupgrade_wizard/cli_estimate_workload.py +++ b/odoo_openupgrade_wizard/cli_estimate_workload.py @@ -22,6 +22,8 @@ from odoo_openupgrade_wizard.tools_system import ( def estimate_workload(ctx, analysis_file_path): # Analyse analysis = Analysis(ctx) + analysis.analyse_module_version(ctx) + analysis.analyse_openupgrade_state(ctx) # Make some clean to display properly analysis.modules = sorted(analysis.modules) @@ -30,7 +32,7 @@ def estimate_workload(ctx, analysis_file_path): # TODO, make ensure_file_exists_from_template( Path(analysis_file_path), - templates.ANALYSIS_TEMPLATE, + templates.ANALYSIS_HTML_TEMPLATE, ctx=ctx, analysis=analysis, current_date=datetime.now().strftime("%d/%m/%Y %H:%M:%S"), diff --git a/odoo_openupgrade_wizard/cli_options.py b/odoo_openupgrade_wizard/cli_options.py index 2846b1a..f1fb468 100644 --- a/odoo_openupgrade_wizard/cli_options.py +++ b/odoo_openupgrade_wizard/cli_options.py @@ -7,7 +7,9 @@ def releases_options(function): "--releases", type=str, help="Coma-separated values of odoo releases for which" - " you want to perform the operation.", + " you want to perform the operation." + " Let empty to perform the operation on all the releases" + " of the project", )(function) return function diff --git a/odoo_openupgrade_wizard/configuration_version_dependant.py b/odoo_openupgrade_wizard/configuration_version_dependant.py index 321608a..54d8dde 100644 --- a/odoo_openupgrade_wizard/configuration_version_dependant.py +++ b/odoo_openupgrade_wizard/configuration_version_dependant.py @@ -228,3 +228,32 @@ def generate_analysis_files( logger.info("> Launch analysis. This can take a while ...") analysis.analyze() + + +def get_apriori_file_relative_path(migration_step: dict) -> (str, Path): + """Return the module name and the relative file path of + the apriori.py file that contains all the rename and + the merge information for a given upgrade.""" + if migration_step["release"] < 14.0: + return ("openupgrade_records", Path("lib/apriori.py")) + else: + return ("openupgrade_scripts", Path("apriori.py")) + + +def get_coverage_relative_path(migration_step: dict) -> (str, Path): + """Return the path of the coverage file.""" + if migration_step["release"] < 10.0: + base_path = Path("src/openupgrade/openerp/openupgrade/doc/source") + elif migration_step["release"] < 14.0: + base_path = Path("src/openupgrade/odoo/openupgrade/doc/source") + else: + base_path = Path("src/openupgrade/docsource") + + previous_release = migration_step["release"] - 1 + return base_path / Path( + "modules%s-%s.rst" + % ( + ("%.1f" % previous_release).replace(".", ""), + ("%.1f" % migration_step["release"]).replace(".", ""), + ) + ) diff --git a/odoo_openupgrade_wizard/templates.py b/odoo_openupgrade_wizard/templates.py index ffb0206..1b1421b 100644 --- a/odoo_openupgrade_wizard/templates.py +++ b/odoo_openupgrade_wizard/templates.py @@ -126,13 +126,18 @@ GIT_IGNORE_CONTENT = """ !.gitignore """ +# TODO, this value are usefull for test for analyse between 13 and 14. +# move that values in data/extra_script/modules.csv +# and let this template with only 'base' module. MODULES_CSV_TEMPLATE = """ base,Base account,Account Module web_responsive,Web Responsive Module +account_facturx,Account Factur X +account_bank_statement_import, Account Bank Statement Import """ -ANALYSIS_TEMPLATE = """ +ANALYSIS_HTML_TEMPLATE = """

Migration Analysis

@@ -154,19 +159,71 @@ ANALYSIS_TEMPLATE = """ - +

- + +{%- for odoo_version in ctx.obj["config"]["odoo_versions"] -%} + +{% endfor %} + -{%- for odoo_module in analysis.modules -%} +{% set ns = namespace( + current_repository='', + current_module_type='', +) %} +{% for odoo_module in analysis.modules %} + + + + + + {% if ( + ns.current_module_type != odoo_module.module_type + and odoo_module.module_type != 'odoo') %} + {% set ns.current_module_type = odoo_module.module_type %} - + + {% endif %} + + + + + + {% if ns.current_repository != odoo_module.repository %} + {% set ns.current_repository = odoo_module.repository %} + + + + {% endif %} + + + + + + + + {% for release in odoo_module.analyse.all_releases %} + {% set module_version = odoo_module.get_module_version(release) %} + {% if module_version %} + + {% else %} + + {% endif %} + {% endfor %} + {% endfor %} diff --git a/odoo_openupgrade_wizard/tools_odoo_module.py b/odoo_openupgrade_wizard/tools_odoo_module.py index 5fef688..73b117b 100644 --- a/odoo_openupgrade_wizard/tools_odoo_module.py +++ b/odoo_openupgrade_wizard/tools_odoo_module.py @@ -1,8 +1,14 @@ +import importlib from functools import total_ordering +from pathlib import Path from git import Repo from loguru import logger +from odoo_openupgrade_wizard.configuration_version_dependant import ( + get_apriori_file_relative_path, + get_coverage_relative_path, +) from odoo_openupgrade_wizard.tools_odoo import ( get_odoo_addons_path, get_odoo_env_path, @@ -11,78 +17,306 @@ from odoo_openupgrade_wizard.tools_odoo import ( class Analysis(object): - - modules = [] - def __init__(self, ctx): module_names = get_odoo_modules_from_csv(ctx.obj["module_file_path"]) - initial_release = ctx.obj["config"]["odoo_versions"][0]["release"] + self.modules = [] + self.initial_release = ctx.obj["config"]["odoo_versions"][0]["release"] + self.final_release = ctx.obj["config"]["odoo_versions"][-1]["release"] + self.all_releases = [ + x["release"] for x in ctx.obj["config"]["odoo_versions"] + ] # Instanciate a new odoo_module for module_name in module_names: - repository_name = OdooModule.find_repository( - ctx, module_name, initial_release + + addon_path = OdooModule.get_addon_path( + ctx, module_name, self.initial_release ) - if ( - repository_name - and "%s.%s" % (repository_name, module_name) - not in self.modules - ): + if addon_path: + repository_name = OdooModule.get_repository_name(addon_path) + if ( + "%s.%s" % (repository_name, module_name) + not in self.modules + ): + logger.debug( + "Discovering module '%s' in %s for release %s" + % (module_name, repository_name, self.initial_release) + ) + self.modules.append( + OdooModule(ctx, self, module_name, repository_name) + ) + else: + raise ValueError( + "The module %s has not been found in the release %s." + "Analyse can not be done." + % (module_name, self.initial_release) + ) + + def analyse_module_version(self, ctx): + self._generate_module_version_first_release(ctx) + + for count in range(len(self.all_releases) - 1): + previous_release = self.all_releases[count] + current_release = self.all_releases[count + 1] + self._generate_module_version_next_release( + ctx, previous_release, current_release + ) + + def analyse_openupgrade_state(self, ctx): + logger.info("Parsing openupgrade module coverage for each migration.") + coverage_analysis = {} + for release in self.all_releases[1:]: + coverage_analysis[release] = {} + relative_path = get_coverage_relative_path({"release": release}) + env_folder_path = get_odoo_env_path(ctx, {"release": release}) + coverage_path = env_folder_path / relative_path + with open(coverage_path) as f: + lines = f.readlines() + for line in [x for x in lines if "|" in x]: + clean_line = ( + line.replace("\n", "") + .replace("|del|", "") + .replace("|new|", "") + ) + splited_line = [x.strip() for x in clean_line.split("|") if x] + if len(splited_line) == 2: + coverage_analysis[release][splited_line[0]] = splited_line[ + 1 + ] + if len(splited_line) == 3: + coverage_analysis[release][splited_line[0]] = ( + splited_line[1] + " " + splited_line[2] + ).strip() + elif len(splited_line) > 3: + raise ValueError( + "Incorrect value in openupgrade analysis file %s" + " for line %s" % (coverage_path, line) + ) + + for odoo_module in filter( + lambda x: x.module_type == "odoo", self.modules + ): + odoo_module.analyse_openupgrade_state(coverage_analysis) + + def _generate_module_version_first_release(self, ctx): + logger.info( + "Analyse version %s. (First Release)" % self.initial_release + ) + for odoo_module in self.modules: + # Get new name of the module + new_name = odoo_module.name + + addon_path = OdooModule.get_addon_path( + ctx, new_name, self.initial_release + ) + + new_module_version = OdooModuleVersion( + self.initial_release, odoo_module, addon_path + ) + odoo_module.module_versions.update( + {self.initial_release: new_module_version} + ) + + def _generate_module_version_next_release( + self, ctx, previous_release, current_release + ): + logger.info( + "Analyse change between %s and %s" + % (previous_release, current_release) + ) + # Get changes between the two releases + ( + apriori_module_name, + apriori_relative_path, + ) = get_apriori_file_relative_path({"release": current_release}) + apriori_module_path = OdooModule.get_addon_path( + ctx, apriori_module_name, current_release + ) + apriori_absolute_path = ( + apriori_module_path + / Path(apriori_module_name) + / apriori_relative_path + ) + + module_spec = importlib.util.spec_from_file_location( + "package", str(apriori_absolute_path) + ) + module = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(module) + + renamed_modules = module.renamed_modules + merged_modules = module.merged_modules + + for odoo_module in self.modules: + state = False + new_module_name = False + if odoo_module.name in renamed_modules: + state = "renamed" + new_module_name = renamed_modules[odoo_module.name] logger.debug( - "Discovering module '%s' in %s for release %s" - % (module_name, repository_name, initial_release) + "%s -> %s : %s renamed into %s" + % ( + previous_release, + current_release, + odoo_module.name, + new_module_name, + ) ) - self.modules.append( - OdooModule(ctx, module_name, repository_name) + elif odoo_module.name in merged_modules: + state = "merged" + new_module_name = merged_modules[odoo_module.name] + logger.debug( + "%s -> %s : %s merged into %s" + % ( + previous_release, + current_release, + odoo_module.name, + new_module_name, + ) ) + # Handle new module + if state and new_module_name != odoo_module.name: + # Ensure that the module exists in self.modules + new_addon_path = OdooModule.get_addon_path( + ctx, new_module_name, current_release + ) + if not new_addon_path: + raise ValueError( + "The module %s has not been found in the release %s." + " Analyse can not be done." + % (new_module_name, current_release) + ) + else: + new_repository_name = OdooModule.get_repository_name( + new_addon_path + ) + if ( + "%s.%s" % (new_repository_name, new_module_name) + not in self.modules + ): + logger.debug( + "Discovering module '%s' in %s for release %s" + % ( + new_module_name, + new_repository_name, + current_release, + ) + ) + new_odoo_module = OdooModule( + ctx, self, new_module_name, new_repository_name + ) + self.modules.append(new_odoo_module) + new_odoo_module.module_versions.update( + { + current_release: OdooModuleVersion( + current_release, + new_odoo_module, + new_addon_path, + ) + } + ) + + # Get the previous release of the module + previous_module_version = odoo_module.get_module_version( + previous_release + ) + # if the previous release has been renamed or merged + # the loss is normal + if previous_module_version and previous_module_version.state in [ + "merged", + "renamed", + "normal_loss", + ]: + state = "normal_loss" + + new_addon_path = OdooModule.get_addon_path( + ctx, odoo_module.name, current_release + ) + odoo_module.module_versions.update( + { + current_release: OdooModuleVersion( + current_release, + odoo_module, + new_addon_path, + state=state, + target_module=new_module_name, + ) + } + ) + @total_ordering class OdooModule(object): - - active = True - name = False - repository = False - module_type = False - unique_name = False - - @classmethod - def find_repository(cls, ctx, module_name, current_release): - - # Try to find the repository that contains the module - main_path = get_odoo_env_path(ctx, {"release": current_release}) - addons_path = get_odoo_addons_path( - ctx, main_path, {"release": current_release, "action": "update"} - ) - for addon_path in addons_path: - if (addon_path / module_name).exists(): - - if str(addon_path).endswith("odoo/odoo/addons"): - path = addon_path.parent.parent - elif str(addon_path).endswith("odoo/addons"): - path = addon_path.parent - else: - path = addon_path - repo = Repo(str(path)) - repository_name = repo.remotes[0].url.replace( - "https://github.com/", "" - ) - - return repository_name - - return False - - def __init__(self, ctx, module_name, repository_name): + def __init__(self, ctx, analyse, module_name, repository_name): + self.analyse = analyse self.name = module_name self.repository = repository_name + self.unique_name = "%s.%s" % (repository_name, module_name) + self.module_versions = {} if repository_name == "odoo/odoo": self.module_type = "odoo" elif repository_name.startswith("OCA"): self.module_type = "OCA" else: self.module_type = "custom" - self.unique_name = "%s.%s" % (repository_name, module_name) + + def get_module_version(self, current_release): + res = self.module_versions.get(current_release, False) + return res + + def analyse_openupgrade_state(self, coverage_analysis): + for module_version in list(self.module_versions.values()): + module_version.analyse_openupgrade_state(coverage_analysis) + + @classmethod + def get_addon_path(cls, ctx, module_name, current_release): + """Search the module in all the addons path of the current release + and return the addon path of the module, or False if not found. + For exemple find_repository(ctx, 'web_responsive', 12.0) + '/PATH_TO_LOCAL_ENV/src/OCA/web' + """ + # Try to find the repository that contains the module + main_path = get_odoo_env_path(ctx, {"release": current_release}) + addons_path = get_odoo_addons_path( + ctx, main_path, {"release": current_release, "action": "upgrade"} + ) + for addon_path in addons_path: + if (addon_path / module_name).exists(): + return addon_path + return False + + @classmethod + def get_repository_name(cls, addon_path): + """Given an addons path that contains odoo modules in a folder + that has been checkouted via git, return a repository name with the + following format org_name/repo_name. + For exemple 'OCA/web' or 'odoo/odoo' + """ + # TODO, make the code cleaner and more resiliant + # for the time being, the code will fail for + # - github url set with git+http... + # - gitlab url + # - if odoo code is not in a odoo folder in the repos.yml file... + if str(addon_path).endswith("odoo/odoo/addons") or str( + addon_path + ).endswith("openupgrade/odoo/addons"): + path = addon_path.parent.parent + elif str(addon_path).endswith("odoo/addons") or str( + addon_path + ).endswith("openupgrade/addons"): + path = addon_path.parent + else: + path = addon_path + repo = Repo(str(path)) + repository_name = repo.remotes[0].url.replace( + "https://github.com/", "" + ) + if repository_name.lower() == "oca/openupgrade": + return "odoo/odoo" + else: + return repository_name def __eq__(self, other): if isinstance(other, str): @@ -94,9 +328,109 @@ class OdooModule(object): if self.module_type != other.module_type: if self.module_type == "odoo": return True - elif self.module_type == "OCA" and self.module_type == "custom": + elif self.module_type == "OCA" and other.module_type == "custom": return True else: return False + elif self.repository != other.repository: + return self.repository < other.repository else: return self.name < other.name + + def __str__(self): + return "%s - %s" % (self.unique_name, self.module_type) + + +class OdooModuleVersion(object): + def __init__( + self, + release, + odoo_module, + addon_path, + state=False, + target_module=False, + ): + self.release = release + self.odoo_module = odoo_module + self.addon_path = addon_path + self.state = state + self.target_module = target_module + self.openupgrade_state = "" + + def analyse_openupgrade_state(self, coverage_analysis): + if self.release == self.odoo_module.analyse.initial_release: + return + self.openupgrade_state = coverage_analysis[self.release].get( + self.odoo_module.name, False + ) + + def get_bg_color(self): + if self.addon_path: + if ( + self.odoo_module.module_type == "odoo" + and self.release != self.odoo_module.analyse.initial_release + ): + if self.openupgrade_state and ( + self.openupgrade_state.lower().startswith("done") + or self.openupgrade_state.lower().startswith( + "nothing to do" + ) + ): + return "lightgreen" + else: + return "orange" + return "lightgreen" + else: + # The module doesn't exist in the current release + if self.state in ["merged", "renamed", "normal_loss"]: + # Normal case, the previous version has been renamed + # or merged + return "lightgray" + + if self.odoo_module.module_type == "odoo": + # A core module disappeared and has not been merged + # or renamed + return "red" + elif self.release != self.odoo_module.analyse.final_release: + return "lightgray" + else: + return "orange" + + def get_text(self): + if self.addon_path: + if ( + self.odoo_module.module_type == "odoo" + and self.release != self.odoo_module.analyse.initial_release + ): + if self.openupgrade_state.lower().startswith( + "done" + ) or self.openupgrade_state.lower().startswith( + "nothing to do" + ): + return self.openupgrade_state + else: + return "To analyse" + return "" + else: + if self.state == "merged": + return "Merged into %s" % self.target_module + elif self.state == "renamed": + return "Renamed into %s" % self.target_module + elif self.state == "normal_loss": + return "" + + if self.odoo_module.module_type == "odoo": + # A core module disappeared and has not been merged + # or renamed + return "Module lost" + elif self.release != self.odoo_module.analyse.final_release: + return "Unported" + else: + return "To port" + + def __str__(self): + return "%s - %s - %s" % ( + self.odoo_module.name, + self.release, + self.addon_path, + ) diff --git a/tests/cli_01_init_test.py b/tests/cli_01_init_test.py index 01820ff..8113cf0 100644 --- a/tests/cli_01_init_test.py +++ b/tests/cli_01_init_test.py @@ -15,7 +15,8 @@ def test_cli_init(): "--project-name=test-cli", "--initial-release=13.0", "--final-release=14.0", - "--extra-repository=OCA/web,OCA/server-tools", + "--extra-repository=" + "OCA/web,OCA/server-tools,OCA/bank-statement-import", ] )
-  {{ odoo_version["release"] }}
{{odoo_module.name}} ({{odoo_module.module_type}}) + + {{ ns.current_module_type}} +
+ {{ ns.current_repository}} +
{{odoo_module.name}} + {{module_version.get_text()}} +