diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5f9e7a2..2b5c08a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,6 +52,7 @@ pytest: tests/cli_05_execute_script_python_test.py tests/cli_06_execute_script_sql_test.py tests/cli_07_upgrade_test.py + tests/cli_08_estimate_workload_test.py # Disabled test on gitlab-ci : # The following tests should work locally but doesn't on gitlab-ci diff --git a/odoo_openupgrade_wizard/cli.py b/odoo_openupgrade_wizard/cli.py index 4119203..fad64af 100644 --- a/odoo_openupgrade_wizard/cli.py +++ b/odoo_openupgrade_wizard/cli.py @@ -10,6 +10,7 @@ from loguru import logger import odoo_openupgrade_wizard from odoo_openupgrade_wizard.cli_docker_build import docker_build +from odoo_openupgrade_wizard.cli_estimate_workload import estimate_workload from odoo_openupgrade_wizard.cli_execute_script_python import ( execute_script_python, ) @@ -109,10 +110,9 @@ def main(ctx, env_folder, filestore_folder, log_level): elif ctx.invoked_subcommand != "init": raise - logger.debug("context %s: " % ctx.obj) - main.add_command(docker_build) +main.add_command(estimate_workload) main.add_command(execute_script_python) main.add_command(execute_script_sql) main.add_command(generate_module_analysis) diff --git a/odoo_openupgrade_wizard/cli_estimate_workload.py b/odoo_openupgrade_wizard/cli_estimate_workload.py new file mode 100644 index 0000000..7dff4a5 --- /dev/null +++ b/odoo_openupgrade_wizard/cli_estimate_workload.py @@ -0,0 +1,37 @@ +from datetime import datetime +from pathlib import Path + +import click + +from odoo_openupgrade_wizard import templates +from odoo_openupgrade_wizard.tools_odoo_module import Analysis +from odoo_openupgrade_wizard.tools_system import ( + ensure_file_exists_from_template, +) + + +@click.command() +@click.option( + "--analysis-file-path", + type=click.Path( + dir_okay=False, + ), + default="./analysis.html", +) +@click.pass_context +def estimate_workload(ctx, analysis_file_path): + # Analyse + analysis = Analysis(ctx) + + # Make some clean to display properly + analysis.modules = sorted(analysis.modules) + + # Render html file + # TODO, make + ensure_file_exists_from_template( + Path(analysis_file_path), + templates.ANALYSIS_TEMPLATE, + ctx=ctx, + analysis=analysis, + current_date=datetime.now().strftime("%d/%m/%Y %H:%M:%S"), + ) diff --git a/odoo_openupgrade_wizard/cli_install_from_csv.py b/odoo_openupgrade_wizard/cli_install_from_csv.py index 7b64d16..4c9d807 100644 --- a/odoo_openupgrade_wizard/cli_install_from_csv.py +++ b/odoo_openupgrade_wizard/cli_install_from_csv.py @@ -1,5 +1,3 @@ -import csv - import click from loguru import logger @@ -7,7 +5,11 @@ from odoo_openupgrade_wizard.cli_options import ( database_option, get_migration_step_from_options, ) -from odoo_openupgrade_wizard.tools_odoo import kill_odoo, run_odoo +from odoo_openupgrade_wizard.tools_odoo import ( + get_odoo_modules_from_csv, + kill_odoo, + run_odoo, +) from odoo_openupgrade_wizard.tools_odoo_instance import OdooInstance from odoo_openupgrade_wizard.tools_postgres import ensure_database @@ -20,23 +22,7 @@ def install_from_csv(ctx, database): ensure_database(ctx, database, state="present") # Get modules list from the CSV file - csv_path = ctx.obj["module_file_path"] - logger.info("Reading '%s' file ..." % csv_path) - module_names = [] - csvfile = open(csv_path, "r") - spamreader = csv.reader(csvfile, delimiter=",", quotechar='"') - for row in spamreader: - # Try to guess that a line is not correct - if not row: - continue - if not row[0]: - continue - if " " in row[0]: - continue - if any([x.isupper() for x in row[0]]): - continue - module_names.append(row[0]) - + module_names = get_odoo_modules_from_csv(ctx.obj["module_file_path"]) module_names.sort() logger.info("Found %d modules." % (len(module_names))) logger.debug(module_names) diff --git a/odoo_openupgrade_wizard/templates.py b/odoo_openupgrade_wizard/templates.py index 6161832..ffb0206 100644 --- a/odoo_openupgrade_wizard/templates.py +++ b/odoo_openupgrade_wizard/templates.py @@ -131,3 +131,46 @@ base,Base account,Account Module web_responsive,Web Responsive Module """ + +ANALYSIS_TEMPLATE = """ + + +

Migration Analysis

+ + + + + + + + + + + + + + + + + +
Initial ReleaseFinal ReleaseProject NameAnalysis Date
{{ ctx.obj["config"]["odoo_versions"][0]["release"] }}{{ ctx.obj["config"]["odoo_versions"][-1]["release"] }}{{ ctx.obj["config"]["project_name"] }}{{ current_date }}
+ + + + + + + + +{%- for odoo_module in analysis.modules -%} + + + +{% endfor %} + + +
-
{{odoo_module.name}} ({{odoo_module.module_type}}) +
+ + +""" diff --git a/odoo_openupgrade_wizard/tools_odoo.py b/odoo_openupgrade_wizard/tools_odoo.py index c2c2b76..52899c8 100644 --- a/odoo_openupgrade_wizard/tools_odoo.py +++ b/odoo_openupgrade_wizard/tools_odoo.py @@ -1,4 +1,5 @@ import configparser +import csv import os import sys import traceback @@ -41,7 +42,7 @@ def get_odoo_addons_path(ctx, root_path: Path, migration_step: dict) -> str: else: addons_path.append(path) - return ",".join([str(x) for x in addons_path]) + return addons_path def get_odoo_env_path(ctx, odoo_version: dict) -> Path: @@ -128,7 +129,14 @@ def generate_odoo_config_file(ctx, migration_step, log_file): parser.read(custom_odoo_config_file) # compute addons_path - addons_path = get_odoo_addons_path(ctx, Path("/odoo_env"), migration_step) + addons_path = ",".join( + [ + str(x) + for x in get_odoo_addons_path( + ctx, Path("/odoo_env"), migration_step + ) + ] + ) # compute server wides modules server_wide_modules = parser.get( @@ -297,3 +305,22 @@ def execute_click_odoo_python_files( raise e finally: kill_odoo(ctx, migration_step) + + +def get_odoo_modules_from_csv(module_file_path: Path) -> list: + logger.info("Reading '%s' file ..." % module_file_path) + module_names = [] + csvfile = open(module_file_path, "r") + spamreader = csv.reader(csvfile, delimiter=",", quotechar='"') + for row in spamreader: + # Try to guess that a line is not correct + if not row: + continue + if not row[0]: + continue + if " " in row[0]: + continue + if any([x.isupper() for x in row[0]]): + continue + module_names.append(row[0]) + return module_names diff --git a/odoo_openupgrade_wizard/tools_odoo_module.py b/odoo_openupgrade_wizard/tools_odoo_module.py new file mode 100644 index 0000000..5fef688 --- /dev/null +++ b/odoo_openupgrade_wizard/tools_odoo_module.py @@ -0,0 +1,102 @@ +from functools import total_ordering + +from git import Repo +from loguru import logger + +from odoo_openupgrade_wizard.tools_odoo import ( + get_odoo_addons_path, + get_odoo_env_path, + get_odoo_modules_from_csv, +) + + +class Analysis(object): + + modules = [] + + def __init__(self, ctx): + module_names = get_odoo_modules_from_csv(ctx.obj["module_file_path"]) + + initial_release = ctx.obj["config"]["odoo_versions"][0]["release"] + + # Instanciate a new odoo_module + for module_name in module_names: + repository_name = OdooModule.find_repository( + ctx, module_name, initial_release + ) + if ( + repository_name + and "%s.%s" % (repository_name, module_name) + not in self.modules + ): + logger.debug( + "Discovering module '%s' in %s for release %s" + % (module_name, repository_name, initial_release) + ) + self.modules.append( + OdooModule(ctx, module_name, repository_name) + ) + + +@total_ordering +class OdooModule(object): + + active = True + name = False + repository = False + module_type = False + unique_name = False + + @classmethod + def find_repository(cls, ctx, module_name, current_release): + + # Try to find the repository that contains the module + main_path = get_odoo_env_path(ctx, {"release": current_release}) + addons_path = get_odoo_addons_path( + ctx, main_path, {"release": current_release, "action": "update"} + ) + for addon_path in addons_path: + if (addon_path / module_name).exists(): + + if str(addon_path).endswith("odoo/odoo/addons"): + path = addon_path.parent.parent + elif str(addon_path).endswith("odoo/addons"): + path = addon_path.parent + else: + path = addon_path + repo = Repo(str(path)) + repository_name = repo.remotes[0].url.replace( + "https://github.com/", "" + ) + + return repository_name + + return False + + def __init__(self, ctx, module_name, repository_name): + self.name = module_name + self.repository = repository_name + if repository_name == "odoo/odoo": + self.module_type = "odoo" + elif repository_name.startswith("OCA"): + self.module_type = "OCA" + else: + self.module_type = "custom" + self.unique_name = "%s.%s" % (repository_name, module_name) + + def __eq__(self, other): + if isinstance(other, str): + return self.unique_name == other + elif isinstance(other, OdooModule): + return self.unique_name == other.unique_name + + def __lt__(self, other): + if self.module_type != other.module_type: + if self.module_type == "odoo": + return True + elif self.module_type == "OCA" and self.module_type == "custom": + return True + else: + return False + else: + return self.name < other.name diff --git a/poetry.lock b/poetry.lock index 128dddf..550c05b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -208,6 +208,29 @@ colorama = "*" kaptan = "*" requests = "*" +[[package]] +name = "gitdb" +version = "4.0.9" +description = "Git Object Database" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.20" +description = "Python Git Library" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +gitdb = ">=4.0.1,<5" +typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.10\""} + [[package]] name = "idna" version = "3.3" @@ -597,6 +620,14 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "smmap" +version = "5.0.0" +description = "A pure Python implementation of a sliding window memory map manager" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "toml" version = "0.10.2" @@ -750,7 +781,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "126762024024c25cf869fbd4a2c4e2dd5b58a9d1a198586f5071ab65a68e1f17" +content-hash = "c86a563043f2c105d46b393c93b6d10d67e35917d5dfbd0dd83daf42e62e3dcd" [metadata.files] aiocontextvars = [ @@ -868,6 +899,14 @@ git-aggregator = [ {file = "git-aggregator-2.1.0.tar.gz", hash = "sha256:efdc4d3f360fd63ef5b14e7064ce5edb14ea404c6a4047715cfc5b9384ff49cc"}, {file = "git_aggregator-2.1.0-py3-none-any.whl", hash = "sha256:59986c0ff7a1641849504dc4d86491872d9f65b46a076aac4bf21cd550ff61df"}, ] +gitdb = [ + {file = "gitdb-4.0.9-py3-none-any.whl", hash = "sha256:8033ad4e853066ba6ca92050b9df2f89301b8fc8bf7e9324d412a63f8bf1a8fd"}, + {file = "gitdb-4.0.9.tar.gz", hash = "sha256:bac2fd45c0a1c9cf619e63a90d62bdc63892ef92387424b855792a6cabe789aa"}, +] +gitpython = [ + {file = "GitPython-3.1.20-py3-none-any.whl", hash = "sha256:b1e1c269deab1b08ce65403cf14e10d2ef1f6c89e33ea7c5e5bb0222ea593b8a"}, + {file = "GitPython-3.1.20.tar.gz", hash = "sha256:df0e072a200703a65387b0cfdf0466e3bab729c0458cf6b7349d0e9877636519"}, +] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, @@ -1173,6 +1212,10 @@ six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +smmap = [ + {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, + {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, diff --git a/pyproject.toml b/pyproject.toml index 1d4d5ef..fa15860 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ single-source = "^0.3" git-aggregator = "^2.1" docker = "^5.0" pyyaml = "5.4.1" +GitPython = "^3.1" [tool.poetry.dev-dependencies] pytest = [ diff --git a/tests/cli_08_estimate_workload_test.py b/tests/cli_08_estimate_workload_test.py new file mode 100644 index 0000000..b599c9c --- /dev/null +++ b/tests/cli_08_estimate_workload_test.py @@ -0,0 +1,13 @@ +from . import cli_runner_invoke, move_to_test_folder + + +def test_cli_estimate_workload(): + move_to_test_folder() + + cli_runner_invoke( + [ + "--log-level=DEBUG", + "estimate-workload", + ] + ) + # TODO, write test