From 80e5e3ad8740d9bbb3236974da3e6601907faf70 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Mon, 27 Jun 2022 15:56:28 +0200 Subject: [PATCH 01/39] [REF] Dockerfiles : switch from odoo/odoo to acsone/odoo-bedrock --- INTERNAL_NOTES.md | 5 +++ odoo_openupgrade_wizard/cli/cli_init.py | 11 ++++- .../configuration_version_dependant.py | 41 ++++++++++++++----- .../templates/config.yml.j2 | 8 ++++ .../templates/odoo/Dockerfile.j2 | 25 +++++------ .../templates/odoo/debian_requirements.txt.j2 | 1 - .../templates/odoo/python_requirements.txt.j2 | 2 +- tests/data/output_expected/config.yml | 8 ++++ 8 files changed, 72 insertions(+), 29 deletions(-) diff --git a/INTERNAL_NOTES.md b/INTERNAL_NOTES.md index 68339d3..9a84987 100644 --- a/INTERNAL_NOTES.md +++ b/INTERNAL_NOTES.md @@ -50,3 +50,8 @@ in modules.csv.j2 : # TODO, this value are usefull for test for analyse between 13 and 14. # move that values in data/extra_script/modules.csv # and let this template with only 'base' module. + + + +TODO when launching container : +- install odoo : ``pip install -e /odoo/src/odoo`` diff --git a/odoo_openupgrade_wizard/cli/cli_init.py b/odoo_openupgrade_wizard/cli/cli_init.py index 24f6ac4..57c47f5 100644 --- a/odoo_openupgrade_wizard/cli/cli_init.py +++ b/odoo_openupgrade_wizard/cli/cli_init.py @@ -3,9 +3,10 @@ from pathlib import Path import click from odoo_openupgrade_wizard.configuration_version_dependant import ( + get_odoo_version_settings, get_odoo_versions, - get_python_libraries, get_python_major_version, + get_python_minor_version_short, get_version_options, ) from odoo_openupgrade_wizard.tools.tools_odoo import get_odoo_env_path @@ -68,6 +69,9 @@ def init( odoo_versions = get_odoo_versions( float(initial_version), float(final_version) ) + odoo_version_settings = get_odoo_version_settings( + float(initial_version), float(final_version) + ) # 2. Compute Migration Steps @@ -133,6 +137,7 @@ def init( project_name=project_name, steps=steps, odoo_versions=odoo_versions, + odoo_version_settings=odoo_version_settings, ) # 7. Ensure module list file exists @@ -154,7 +159,6 @@ def init( ensure_file_exists_from_template( path_version / Path("python_requirements.txt"), "odoo/python_requirements.txt.j2", - python_libraries=get_python_libraries(odoo_version), ) # Create debian requirements file @@ -183,6 +187,9 @@ def init( "odoo/Dockerfile.j2", odoo_version=odoo_version, python_major_version=get_python_major_version(odoo_version), + python_minor_version_short=get_python_minor_version_short( + odoo_version + ), ) # Create 'src' folder that will contain all the odoo code diff --git a/odoo_openupgrade_wizard/configuration_version_dependant.py b/odoo_openupgrade_wizard/configuration_version_dependant.py index d135b89..d379158 100644 --- a/odoo_openupgrade_wizard/configuration_version_dependant.py +++ b/odoo_openupgrade_wizard/configuration_version_dependant.py @@ -6,44 +6,42 @@ _ODOO_VERSION_TEMPLATES = [ { "version": 8.0, "python_major_version": "python2", - "python_libraries": [], + "python_minor_version_short": "py27", }, { "version": 9.0, "python_major_version": "python2", - "python_libraries": ["openupgradelib==2.0.0"], + "python_minor_version_short": "py27", }, { "version": 10.0, "python_major_version": "python2", - "python_libraries": ["openupgradelib==2.0.0"], + "python_minor_version_short": "py27", }, { "version": 11.0, "python_major_version": "python3", - "python_libraries": ["openupgradelib==2.0.0"], + "python_minor_version_short": "py36", }, { "version": 12.0, "python_major_version": "python3", - "python_libraries": [ - "git+https://github.com/grap/openupgradelib.git" - "@2.0.1#egg=openupgradelib" - ], + "python_minor_version_short": "py37", }, { "version": 13.0, "python_major_version": "python3", - "python_libraries": ["openupgradelib"], + "python_minor_version_short": "py37", }, { "version": 14.0, "python_major_version": "python3", - "python_libraries": ["openupgradelib"], + "python_minor_version_short": "py39", }, { "version": 15.0, "python_major_version": "python3", + "python_minor_version_short": "py39", "python_libraries": ["openupgradelib"], }, ] @@ -71,6 +69,12 @@ def get_python_major_version(version: float) -> str: return get_version_template(version)["python_major_version"] +def get_python_minor_version_short(version: float) -> str: + """Return the default minor python version (py27, py38) of Odoo for + a given version""" + return get_version_template(version)["python_minor_version_short"] + + def get_version_options(mode: str) -> list: """Get options available for version click argument. Arguments: @@ -88,10 +92,27 @@ def get_version_options(mode: str) -> list: return version_options +def get_odoo_version_settings( + initial_version: float, final_version: float +) -> list: + """Return a list of odoo version settings from the initial version to the final + version + """ + result = [] + for version_template in _ODOO_VERSION_TEMPLATES: + if ( + version_template["version"] >= initial_version + and version_template["version"] <= final_version + ): + result.append(version_template) + return result + + def get_odoo_versions(initial_version: float, final_version: float) -> list: """Return a list of odoo versions from the initial version to the final version """ + # TODO, call get_odoo_version_settings() and call keys() result = [] for version_template in _ODOO_VERSION_TEMPLATES: if ( diff --git a/odoo_openupgrade_wizard/templates/config.yml.j2 b/odoo_openupgrade_wizard/templates/config.yml.j2 index a3ba7ba..d3209b5 100644 --- a/odoo_openupgrade_wizard/templates/config.yml.j2 +++ b/odoo_openupgrade_wizard/templates/config.yml.j2 @@ -12,6 +12,14 @@ odoo_versions: - {{ odoo_version }} {%- endfor %} + +odoo_version_settings: +{%- for setting in odoo_version_settings %} + {{setting['version']}}: + python_minor_version_short: {{setting['python_minor_version_short']}} +{%- endfor %} + + migration_steps: {%- for step in steps %} - name: {{ step['name'] }} diff --git a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 index a2d32e4..2620928 100644 --- a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 +++ b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 @@ -1,22 +1,17 @@ -FROM odoo:{{ odoo_version }} -MAINTAINER GRAP, Coop It Easy - -# Set User root for installations -USER root - -# 1. Make available files in the containers +FROM ghcr.io/acsone/odoo-bedrock:{{ odoo_version }}-{{python_minor_version_short}}-latest +COPY ./src/odoo /odoo/src/odoo COPY debian_requirements.txt /debian_requirements.txt - COPY python_requirements.txt /python_requirements.txt # 2. Install extra debian packages -RUN apt-get update || true &&\ - xargs apt-get install -y --no-install-recommends Date: Mon, 27 Jun 2022 17:21:15 +0200 Subject: [PATCH 02/39] wip --- .../configuration_version_dependant.py | 13 +++++++------ .../templates/odoo/Dockerfile.j2 | 10 ++++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/odoo_openupgrade_wizard/configuration_version_dependant.py b/odoo_openupgrade_wizard/configuration_version_dependant.py index d379158..38f3a81 100644 --- a/odoo_openupgrade_wizard/configuration_version_dependant.py +++ b/odoo_openupgrade_wizard/configuration_version_dependant.py @@ -26,23 +26,24 @@ _ODOO_VERSION_TEMPLATES = [ { "version": 12.0, "python_major_version": "python3", - "python_minor_version_short": "py37", + # Note: doesn't work with latest available version py37 + "python_minor_version_short": "py36", }, { - "version": 13.0, + "version": 13.0, # OK. "python_major_version": "python3", - "python_minor_version_short": "py37", + # Note: doesn't work with latest available version py37 + "python_minor_version_short": "py36", }, { - "version": 14.0, + "version": 14.0, # OK "python_major_version": "python3", "python_minor_version_short": "py39", }, { - "version": 15.0, + "version": 15.0, # OK "python_major_version": "python3", "python_minor_version_short": "py39", - "python_libraries": ["openupgradelib"], }, ] diff --git a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 index 2620928..6a226d4 100644 --- a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 +++ b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 @@ -1,6 +1,6 @@ FROM ghcr.io/acsone/odoo-bedrock:{{ odoo_version }}-{{python_minor_version_short}}-latest -COPY ./src/odoo /odoo/src/odoo +COPY ./src/odoo/requirements.txt /odoo_requirements.txt COPY debian_requirements.txt /debian_requirements.txt COPY python_requirements.txt /python_requirements.txt @@ -12,6 +12,8 @@ RUN apt-get update || true \ # 3. Install extra Python librairies RUN \ pip install --no-cache-dir \ - -r /odoo/src/odoo/requirements.txt \ - -f https://wheelhouse.acsone.eu/manylinux2014 \ - && pip install -r python_requirements.txt + -r /odoo_requirements.txt \ + -f https://wheelhouse.acsone.eu/manylinux2014\ + && pip install --no-cache-dir \ + -r /python_requirements.txt \ + -f https://wheelhouse.acsone.eu/manylinux2014 From bac5d0d22e77ab9f664a7d522100be4d1e4b8ec8 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Mon, 27 Jun 2022 21:59:57 +0200 Subject: [PATCH 03/39] wip --- odoo_openupgrade_wizard/cli/cli_init.py | 19 +++--- .../configuration_version_dependant.py | 62 +++++++------------ .../templates/config.yml.j2 | 6 +- .../templates/odoo/Dockerfile.j2 | 8 +-- 4 files changed, 35 insertions(+), 60 deletions(-) diff --git a/odoo_openupgrade_wizard/cli/cli_init.py b/odoo_openupgrade_wizard/cli/cli_init.py index 57c47f5..db575ef 100644 --- a/odoo_openupgrade_wizard/cli/cli_init.py +++ b/odoo_openupgrade_wizard/cli/cli_init.py @@ -3,10 +3,8 @@ from pathlib import Path import click from odoo_openupgrade_wizard.configuration_version_dependant import ( - get_odoo_version_settings, + get_odoo_version_template_value, get_odoo_versions, - get_python_major_version, - get_python_minor_version_short, get_version_options, ) from odoo_openupgrade_wizard.tools.tools_odoo import get_odoo_env_path @@ -69,9 +67,6 @@ def init( odoo_versions = get_odoo_versions( float(initial_version), float(final_version) ) - odoo_version_settings = get_odoo_version_settings( - float(initial_version), float(final_version) - ) # 2. Compute Migration Steps @@ -137,7 +132,6 @@ def init( project_name=project_name, steps=steps, odoo_versions=odoo_versions, - odoo_version_settings=odoo_version_settings, ) # 7. Ensure module list file exists @@ -186,9 +180,14 @@ def init( path_version / Path("Dockerfile"), "odoo/Dockerfile.j2", odoo_version=odoo_version, - python_major_version=get_python_major_version(odoo_version), - python_minor_version_short=get_python_minor_version_short( - odoo_version + python_major_version=get_odoo_version_template_value( + odoo_version, "python_major_version" + ), + python_minor_version_short=get_odoo_version_template_value( + odoo_version, "python_minor_version_short" + ), + prebuild_wheel_url=get_odoo_version_template_value( + odoo_version, "prebuild_wheel_url" ), ) diff --git a/odoo_openupgrade_wizard/configuration_version_dependant.py b/odoo_openupgrade_wizard/configuration_version_dependant.py index 38f3a81..57777fb 100644 --- a/odoo_openupgrade_wizard/configuration_version_dependant.py +++ b/odoo_openupgrade_wizard/configuration_version_dependant.py @@ -7,73 +7,70 @@ _ODOO_VERSION_TEMPLATES = [ "version": 8.0, "python_major_version": "python2", "python_minor_version_short": "py27", + "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux1", }, { "version": 9.0, "python_major_version": "python2", "python_minor_version_short": "py27", + "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux1", }, { "version": 10.0, "python_major_version": "python2", "python_minor_version_short": "py27", + "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux1", }, { "version": 11.0, "python_major_version": "python3", "python_minor_version_short": "py36", + "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux2014", }, { "version": 12.0, "python_major_version": "python3", # Note: doesn't work with latest available version py37 "python_minor_version_short": "py36", + "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux2014", }, { "version": 13.0, # OK. "python_major_version": "python3", # Note: doesn't work with latest available version py37 "python_minor_version_short": "py36", + "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux2014", }, { "version": 14.0, # OK "python_major_version": "python3", "python_minor_version_short": "py39", + "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux2014", }, { "version": 15.0, # OK "python_major_version": "python3", "python_minor_version_short": "py39", + "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux2014", }, ] -def get_version_template(version: float) -> dict: - """return a version template dictionnary according to a version - provided""" - for version_template in _ODOO_VERSION_TEMPLATES: - if version_template["version"] == version: - return version_template +def get_odoo_version_template_value(version: float, key: str) -> str: + """Return a value depending on a odoo given version and key. + Possible key: + - python_major_version. (python2, python3) + - python_minor_version_short. (py27, py36, ...) + - prebuild_wheel_url. + """ + version_template = False + for odoo_version_template in _ODOO_VERSION_TEMPLATES: + if odoo_version_template["version"] == version: + version_template = odoo_version_template + break else: raise ValueError - - -def get_python_libraries(version: float) -> list: - """Return a list of python librairies that should be - installed in each docker container for a given version""" - return get_version_template(version)["python_libraries"] - - -def get_python_major_version(version: float) -> str: - """Return the major python version (2.0, 3.0) of Odoo for - a given version""" - return get_version_template(version)["python_major_version"] - - -def get_python_minor_version_short(version: float) -> str: - """Return the default minor python version (py27, py38) of Odoo for - a given version""" - return get_version_template(version)["python_minor_version_short"] + return version_template[key] def get_version_options(mode: str) -> list: @@ -93,27 +90,10 @@ def get_version_options(mode: str) -> list: return version_options -def get_odoo_version_settings( - initial_version: float, final_version: float -) -> list: - """Return a list of odoo version settings from the initial version to the final - version - """ - result = [] - for version_template in _ODOO_VERSION_TEMPLATES: - if ( - version_template["version"] >= initial_version - and version_template["version"] <= final_version - ): - result.append(version_template) - return result - - def get_odoo_versions(initial_version: float, final_version: float) -> list: """Return a list of odoo versions from the initial version to the final version """ - # TODO, call get_odoo_version_settings() and call keys() result = [] for version_template in _ODOO_VERSION_TEMPLATES: if ( diff --git a/odoo_openupgrade_wizard/templates/config.yml.j2 b/odoo_openupgrade_wizard/templates/config.yml.j2 index d3209b5..3df6652 100644 --- a/odoo_openupgrade_wizard/templates/config.yml.j2 +++ b/odoo_openupgrade_wizard/templates/config.yml.j2 @@ -14,9 +14,9 @@ odoo_versions: odoo_version_settings: -{%- for setting in odoo_version_settings %} - {{setting['version']}}: - python_minor_version_short: {{setting['python_minor_version_short']}} +{%- for odoo_version in odoo_versions %} + {{odoo_version}}: + repo_url: False {%- endfor %} diff --git a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 index 6a226d4..130559f 100644 --- a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 +++ b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 @@ -11,9 +11,5 @@ RUN apt-get update || true \ # 3. Install extra Python librairies RUN \ - pip install --no-cache-dir \ - -r /odoo_requirements.txt \ - -f https://wheelhouse.acsone.eu/manylinux2014\ - && pip install --no-cache-dir \ - -r /python_requirements.txt \ - -f https://wheelhouse.acsone.eu/manylinux2014 + /odoo/bin/pip install --no-cache-dir -r /odoo_requirements.txt -f {{prebuild_wheel_url}} \ + && /odoo/bin/pip install --no-cache-dir -r /python_requirements.txt -f {{prebuild_wheel_url}} From 429b8013b9ab2a51d398e3512c8394e361555848 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Mon, 27 Jun 2022 23:53:51 +0200 Subject: [PATCH 04/39] =?UTF-8?q?le=20run,=20=C3=A7a=20marche=20!=20(bon?= =?UTF-8?q?=20bah=20pour=20toute=20les=20versions...)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- odoo_openupgrade_wizard/tools/tools_odoo.py | 149 ++++++++++++-------- 1 file changed, 89 insertions(+), 60 deletions(-) diff --git a/odoo_openupgrade_wizard/tools/tools_odoo.py b/odoo_openupgrade_wizard/tools/tools_odoo.py index bd19a7b..1cb738c 100644 --- a/odoo_openupgrade_wizard/tools/tools_odoo.py +++ b/odoo_openupgrade_wizard/tools/tools_odoo.py @@ -1,4 +1,4 @@ -import configparser +# import configparser import csv import os import sys @@ -8,11 +8,11 @@ from pathlib import Path import yaml from loguru import logger +# get_server_wide_modules_upgrade, from odoo_openupgrade_wizard.configuration_version_dependant import ( get_base_module_folder, get_odoo_folder, get_odoo_run_command, - get_server_wide_modules_upgrade, skip_addon_path, ) from odoo_openupgrade_wizard.tools.tools_docker import ( @@ -101,7 +101,8 @@ def generate_odoo_command( result = ( f" {command}" f" {shell_cmd}" - f" --config /odoo_env/_auto_generated_odoo.cfg" + # f" --config /etc/odoo.cfg" + f" --data-dir /env/filestore/" f" {demo_cmd}" f" {database_cmd}" f" {update_cmd}" @@ -111,58 +112,59 @@ def generate_odoo_command( return result -def generate_odoo_config_file( - ctx, migration_step, log_file, execution_context -): - """Create a config file name _auto_generated_odoo.cfg - in the according environment (defined by migration_step) - This configuration file is a merge of the odoo.cfg file that can - contain custom values, and the values required to run the docker container. - """ - odoo_env_path = get_odoo_env_path(ctx, migration_step["version"]) +# def generate_odoo_config_file( +# ctx, migration_step, log_file, execution_context +# ): +# """Create a config file name _auto_generated_odoo.cfg +# in the according environment (defined by migration_step) +# This configuration file is a merge of the odoo.cfg file that can +# contain custom values, and the values required to run the docker +# container. +# """ +# odoo_env_path = get_odoo_env_path(ctx, migration_step["version"]) - custom_odoo_config_file = odoo_env_path / "odoo.cfg" - auto_generated_odoo_config_file = ( - odoo_env_path / "_auto_generated_odoo.cfg" - ) +# custom_odoo_config_file = odoo_env_path / "odoo.cfg" +# auto_generated_odoo_config_file = ( +# odoo_env_path / "_auto_generated_odoo.cfg" +# ) - parser = configparser.RawConfigParser() - # Read custom file - parser.read(custom_odoo_config_file) +# parser = configparser.RawConfigParser() +# # Read custom file +# parser.read(custom_odoo_config_file) - # compute addons_path - addons_path = ",".join( - [ - str(x) - for x in get_odoo_addons_path( - ctx, Path("/odoo_env"), migration_step, execution_context - ) - ] - ) +# # compute addons_path +# addons_path = ",".join( +# [ +# str(x) +# for x in get_odoo_addons_path( +# ctx, Path("/odoo_env"), migration_step, execution_context +# ) +# ] +# ) - # compute server wides modules - server_wide_modules = parser.get( - "options", "server_wide_modules", fallback=[] - ) - server_wide_modules += get_server_wide_modules_upgrade(migration_step) +# # compute server wides modules +# server_wide_modules = parser.get( +# "options", "server_wide_modules", fallback=[] +# ) +# server_wide_modules += get_server_wide_modules_upgrade(migration_step) - # Add required keys - if "options" not in parser: - parser.add_section("options") - parser.set("options", "db_host", "db") - parser.set("options", "db_port", 5432) - parser.set("options", "db_user", "odoo") - parser.set("options", "db_password", "odoo") - parser.set("options", "workers", 0) - parser.set("options", "data_dir", "/env/filestore/") - parser.set("options", "logfile", log_file) - parser.set("options", "addons_path", addons_path) - if server_wide_modules: - parser.set( - "options", "server_wide_modules", ",".join(server_wide_modules) - ) +# # Add required keys +# if "options" not in parser: +# parser.add_section("options") +# parser.set("options", "db_host", "db") +# parser.set("options", "db_port", 5432) +# parser.set("options", "db_user", "odoo") +# parser.set("options", "db_password", "odoo") +# parser.set("options", "workers", 0) +# parser.set("options", "data_dir", "/env/filestore/") +# parser.set("options", "logfile", log_file) +# parser.set("options", "addons_path", addons_path) +# if server_wide_modules: +# parser.set( +# "options", "server_wide_modules", ",".join(server_wide_modules) +# ) - parser.write(open(auto_generated_odoo_config_file, "w")) +# parser.write(open(auto_generated_odoo_config_file, "w")) def run_odoo( @@ -195,12 +197,6 @@ def run_odoo( update_text=update and " (Update : %s)" % update or "", ) ) - env_path = ctx.obj["env_folder_path"] - odoo_env_path = get_odoo_env_path(ctx, migration_step["version"]) - log_file = "/env/log/{}____{}.log".format( - ctx.obj["log_prefix"], migration_step["complete_name"] - ) - generate_odoo_config_file(ctx, migration_step, log_file, execution_context) command = generate_odoo_command( ctx, @@ -213,12 +209,38 @@ def run_odoo( shell=shell, demo=demo, ) - host_xmlrpc_port = ( alternative_xml_rpc_port and alternative_xml_rpc_port or ctx.obj["config"]["odoo_host_xmlrpc_port"] ) + + return run_container_odoo( + ctx, + migration_step, + command, + host_xmlrpc_port, + detached_container=detached_container, + database=database, + ) + + +def run_container_odoo( + ctx, + migration_step: dict, + command: str, + host_xmlrpc_port: int, + detached_container: bool = False, + database: str = False, + log_file_suffix: str = "", + links: dict = {}, +): + log_file = "/env/log/{}____{}{}.log".format( + ctx.obj["log_prefix"], migration_step["complete_name"], log_file_suffix + ) + env_path = ctx.obj["env_folder_path"] + odoo_env_path = get_odoo_env_path(ctx, migration_step["version"]) + links.update({ctx.obj["config"]["postgres_container_name"]: "db"}) return run_container( get_docker_image_tag(ctx, migration_step["version"]), @@ -231,6 +253,14 @@ def run_odoo( env_path: "/env/", odoo_env_path: "/odoo_env/", }, + environments={ + "DB_HOST": "db", + "DB_PORT": 5432, + "DB_USER": "odoo", + "DB_PASSWORD": "odoo", + "DB_NAME": database, + "LOGFILE": log_file, + }, links=links, detach=detached_container, auto_remove=True, @@ -266,10 +296,9 @@ def execute_click_odoo_python_files( odoo_env_path = get_odoo_env_path(ctx, migration_step["version"]) # Generate odoo config file - log_file = "/env/log/{}____{}__post_migration.log".format( - ctx.obj["log_prefix"], migration_step["complete_name"] - ) - generate_odoo_config_file(ctx, migration_step, log_file, execution_context) + # log_file = "/env/log/{}____{}__post_migration.log".format( + # ctx.obj["log_prefix"], migration_step["complete_name"] + # ) for python_file in python_files: # TODO, check if we should set python2 for old version of Odoo From b64c445cb40a02712de65ac14b28f0322ef97a57 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Tue, 28 Jun 2022 01:06:23 +0200 Subject: [PATCH 05/39] [WIP] --- .../configuration_version_dependant.py | 5 +- odoo_openupgrade_wizard/tools/tools_odoo.py | 73 ++++++++++++++----- tests/data/output_expected/config.yml | 4 +- 3 files changed, 59 insertions(+), 23 deletions(-) diff --git a/odoo_openupgrade_wizard/configuration_version_dependant.py b/odoo_openupgrade_wizard/configuration_version_dependant.py index 57777fb..48c3579 100644 --- a/odoo_openupgrade_wizard/configuration_version_dependant.py +++ b/odoo_openupgrade_wizard/configuration_version_dependant.py @@ -153,11 +153,14 @@ def skip_addon_path(migration_step: dict, path: Path) -> bool: ) and migration_step["version"] < 14.0 -def get_server_wide_modules_upgrade(migration_step: dict) -> list: +def get_server_wide_modules_upgrade( + migration_step: dict, execution_context: str = False +) -> list: """return a list of modules to load, depending on the migration step.""" if ( migration_step["version"] >= 14.0 and migration_step["execution_context"] == "openupgrade" + and execution_context != "regular" ): return ["openupgrade_framework"] return [] diff --git a/odoo_openupgrade_wizard/tools/tools_odoo.py b/odoo_openupgrade_wizard/tools/tools_odoo.py index 1cb738c..b34b682 100644 --- a/odoo_openupgrade_wizard/tools/tools_odoo.py +++ b/odoo_openupgrade_wizard/tools/tools_odoo.py @@ -1,4 +1,4 @@ -# import configparser +import configparser import csv import os import sys @@ -8,11 +8,11 @@ from pathlib import Path import yaml from loguru import logger -# get_server_wide_modules_upgrade, from odoo_openupgrade_wizard.configuration_version_dependant import ( get_base_module_folder, get_odoo_folder, get_odoo_run_command, + get_server_wide_modules_upgrade, skip_addon_path, ) from odoo_openupgrade_wizard.tools.tools_docker import ( @@ -209,19 +209,14 @@ def run_odoo( shell=shell, demo=demo, ) - host_xmlrpc_port = ( - alternative_xml_rpc_port - and alternative_xml_rpc_port - or ctx.obj["config"]["odoo_host_xmlrpc_port"] - ) return run_container_odoo( ctx, migration_step, command, - host_xmlrpc_port, detached_container=detached_container, database=database, + alternative_xml_rpc_port=alternative_xml_rpc_port, ) @@ -229,19 +224,64 @@ def run_container_odoo( ctx, migration_step: dict, command: str, - host_xmlrpc_port: int, detached_container: bool = False, database: str = False, + alternative_xml_rpc_port: int = False, + execution_context: str = False, log_file_suffix: str = "", links: dict = {}, ): - log_file = "/env/log/{}____{}{}.log".format( - ctx.obj["log_prefix"], migration_step["complete_name"], log_file_suffix - ) env_path = ctx.obj["env_folder_path"] odoo_env_path = get_odoo_env_path(ctx, migration_step["version"]) + # Compute 'server_wide_modules' + # For that purpose, read the custom odoo.cfg file + # to know if server_wide_modules is defined + custom_odoo_config_file = odoo_env_path / "odoo.cfg" + parser = configparser.RawConfigParser() + parser.read(custom_odoo_config_file) + server_wide_modules = parser.get( + "options", "server_wide_modules", fallback=[] + ) + server_wide_modules += get_server_wide_modules_upgrade( + migration_step, execution_context + ) + + # compute 'addons_path' + addons_path = ",".join( + [ + str(x) + for x in get_odoo_addons_path( + ctx, Path("/odoo_env"), migration_step, execution_context + ) + ] + ) + + # compute 'log_file' + log_file = "/env/log/{}____{}{}.log".format( + ctx.obj["log_prefix"], migration_step["complete_name"], log_file_suffix + ) + host_xmlrpc_port = ( + alternative_xml_rpc_port + and alternative_xml_rpc_port + or ctx.obj["config"]["odoo_host_xmlrpc_port"] + ) + links.update({ctx.obj["config"]["postgres_container_name"]: "db"}) + + environments = { + "DB_HOST": "db", + "DB_PORT": 5432, + "DB_USER": "odoo", + "DB_PASSWORD": "odoo", + "DB_NAME": database, + "LOGFILE": log_file, + "ADDONS_PATH": addons_path, + "WORKERS": 0, + } + if server_wide_modules: + environments["SERVER_WIDE_MODULES"] = ",".join(server_wide_modules) + return run_container( get_docker_image_tag(ctx, migration_step["version"]), get_docker_container_name(ctx, migration_step), @@ -253,14 +293,7 @@ def run_container_odoo( env_path: "/env/", odoo_env_path: "/odoo_env/", }, - environments={ - "DB_HOST": "db", - "DB_PORT": 5432, - "DB_USER": "odoo", - "DB_PASSWORD": "odoo", - "DB_NAME": database, - "LOGFILE": log_file, - }, + environments=environments, links=links, detach=detached_container, auto_remove=True, diff --git a/tests/data/output_expected/config.yml b/tests/data/output_expected/config.yml index 2112978..7b42119 100644 --- a/tests/data/output_expected/config.yml +++ b/tests/data/output_expected/config.yml @@ -14,9 +14,9 @@ odoo_versions: odoo_version_settings: 13.0: - python_minor_version_short: py37 + repo_url: False 14.0: - python_minor_version_short: py39 + repo_url: False migration_steps: From 451be8c3890d0ef94085f005adc9ff62eadc71ff Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Tue, 28 Jun 2022 01:22:24 +0200 Subject: [PATCH 06/39] FIX pass execution_context --- odoo_openupgrade_wizard/tools/tools_odoo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/odoo_openupgrade_wizard/tools/tools_odoo.py b/odoo_openupgrade_wizard/tools/tools_odoo.py index b34b682..6e02ec9 100644 --- a/odoo_openupgrade_wizard/tools/tools_odoo.py +++ b/odoo_openupgrade_wizard/tools/tools_odoo.py @@ -216,6 +216,7 @@ def run_odoo( command, detached_container=detached_container, database=database, + execution_context=execution_context, alternative_xml_rpc_port=alternative_xml_rpc_port, ) From 4d3bb66e4e5f056d09748f34f7cdbad733b50ef6 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Tue, 28 Jun 2022 16:45:44 +0200 Subject: [PATCH 07/39] [FIX] test from 01 to 04 --- tests/cli_01_init_test.py | 4 ++-- tests/cli_02_get_code_test.py | 12 +++++------- tests/cli_03_docker_build_test.py | 6 +++--- tests/cli_04_run_test.py | 10 ++++++++-- tests/cli_05_execute_script_python_test.py | 9 +++++++-- tests/data/output_expected/config.yml | 18 +++++++++--------- 6 files changed, 34 insertions(+), 25 deletions(-) diff --git a/tests/cli_01_init_test.py b/tests/cli_01_init_test.py index 416847d..490e071 100644 --- a/tests/cli_01_init_test.py +++ b/tests/cli_01_init_test.py @@ -13,8 +13,8 @@ def test_cli_init(): "--log-level=DEBUG", "init", "--project-name=test-cli", - "--initial-version=13.0", - "--final-version=14.0", + "--initial-version=14.0", + "--final-version=15.0", "--extra-repository=" "OCA/web,OCA/server-tools,OCA/bank-statement-import", ] diff --git a/tests/cli_02_get_code_test.py b/tests/cli_02_get_code_test.py index aaf6ec5..ffef04b 100644 --- a/tests/cli_02_get_code_test.py +++ b/tests/cli_02_get_code_test.py @@ -12,14 +12,12 @@ def test_cli_get_code(): ] ) - # Check V13 - openupgrade_path = Path("./src/env_13.0/src/openupgrade") - assert openupgrade_path.exists() + # Check V14 + web_path = Path("./src/env_14.0/src/OCA/web") + assert web_path.exists() - assert (openupgrade_path / Path("odoo")).exists() - - # check V14 - openupgrade_path = Path("./src/env_14.0/src/openupgrade") + # check V15 + openupgrade_path = Path("./src/env_15.0/src/openupgrade") assert openupgrade_path.exists() diff --git a/tests/cli_03_docker_build_test.py b/tests/cli_03_docker_build_test.py index 7b70191..6bbca10 100644 --- a/tests/cli_03_docker_build_test.py +++ b/tests/cli_03_docker_build_test.py @@ -9,16 +9,16 @@ def test_cli_docker_build(): [ "--log-level=DEBUG", "docker-build", - "--versions=13.0,14.0", + "--versions=14.0,15.0", ] ) docker_client = get_docker_client() assert docker_client.images.get( - "odoo-openupgrade-wizard-image__test-cli__13.0" + "odoo-openupgrade-wizard-image__test-cli__14.0" ) assert docker_client.images.get( - "odoo-openupgrade-wizard-image__test-cli__14.0" + "odoo-openupgrade-wizard-image__test-cli__15.0" ) diff --git a/tests/cli_04_run_test.py b/tests/cli_04_run_test.py index 3006d66..968ed08 100644 --- a/tests/cli_04_run_test.py +++ b/tests/cli_04_run_test.py @@ -1,7 +1,10 @@ from pathlib import Path from odoo_openupgrade_wizard.tools.tools_docker import get_docker_client -from odoo_openupgrade_wizard.tools.tools_postgres import execute_sql_request +from odoo_openupgrade_wizard.tools.tools_postgres import ( + ensure_database, + execute_sql_request, +) from . import ( build_ctx_from_config_file, @@ -12,7 +15,11 @@ from . import ( def test_cli_run(): move_to_test_folder() + ctx = build_ctx_from_config_file() + db_name = "database_test_cli___run" + ensure_database(ctx, db_name, state="absent") + cli_runner_invoke( [ "--log-level=DEBUG", @@ -29,7 +36,6 @@ def test_cli_run(): assert db_filestore_path.exists() # Ensure that 'base' module is installed - ctx = build_ctx_from_config_file() request = ( "SELECT id" " FROM ir_module_module" diff --git a/tests/cli_05_execute_script_python_test.py b/tests/cli_05_execute_script_python_test.py index 6060403..4d4528a 100644 --- a/tests/cli_05_execute_script_python_test.py +++ b/tests/cli_05_execute_script_python_test.py @@ -2,7 +2,10 @@ from pathlib import Path from plumbum.cmd import cp -from odoo_openupgrade_wizard.tools.tools_postgres import execute_sql_request +from odoo_openupgrade_wizard.tools.tools_postgres import ( + ensure_database, + execute_sql_request, +) from . import ( build_ctx_from_config_file, @@ -13,6 +16,8 @@ from . import ( def test_cli_execute_script_python(): move_to_test_folder() + ctx = build_ctx_from_config_file() + extra_script_path = Path("../extra_script/click_odoo_test.py").absolute() cp( extra_script_path, @@ -20,6 +25,7 @@ def test_cli_execute_script_python(): ) db_name = "database_test_cli___execute_script_python" + ensure_database(ctx, db_name, state="absent") # Install Odoo on V13 with base installed cli_runner_invoke( @@ -34,7 +40,6 @@ def test_cli_execute_script_python(): ) # Compute partners quantity - ctx = build_ctx_from_config_file() request = "SELECT count(*)" " FROM res_partner;" partner_quantity_before = int( execute_sql_request(ctx, request, database=db_name)[0][0] diff --git a/tests/data/output_expected/config.yml b/tests/data/output_expected/config.yml index 7b42119..1619dac 100644 --- a/tests/data/output_expected/config.yml +++ b/tests/data/output_expected/config.yml @@ -8,32 +8,32 @@ odoo_default_country_code: FR odoo_versions: - - 13.0 - 14.0 + - 15.0 odoo_version_settings: - 13.0: - repo_url: False 14.0: repo_url: False + 15.0: + repo_url: False migration_steps: - name: 1 - version: 13.0 + version: 14.0 execution_context: regular - complete_name: step_01__update__13.0 + complete_name: step_01__update__14.0 - name: 2 - version: 14.0 + version: 15.0 execution_context: openupgrade - complete_name: step_02__upgrade__14.0 + complete_name: step_02__upgrade__15.0 - name: 3 - version: 14.0 + version: 15.0 execution_context: regular - complete_name: step_03__update__14.0 + complete_name: step_03__update__15.0 workload_settings: From b5419e5d196290f445253d72901f5eeca66cfbc6 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Tue, 28 Jun 2022 16:55:08 +0200 Subject: [PATCH 08/39] [IMP] use LOCAL_USER_ID calling odoo-bedrock, so no need to create filestore and log folder with 777 --- odoo_openupgrade_wizard/cli/cli.py | 2 +- odoo_openupgrade_wizard/cli/cli_init.py | 2 +- odoo_openupgrade_wizard/tools/tools_odoo.py | 62 ++----------------- odoo_openupgrade_wizard/tools/tools_system.py | 5 ++ 4 files changed, 13 insertions(+), 58 deletions(-) diff --git a/odoo_openupgrade_wizard/cli/cli.py b/odoo_openupgrade_wizard/cli/cli.py index 1bfbb1a..d9d61c0 100644 --- a/odoo_openupgrade_wizard/cli/cli.py +++ b/odoo_openupgrade_wizard/cli/cli.py @@ -80,7 +80,7 @@ def main(ctx, env_folder, filestore_folder, log_level): filestore_folder_path = Path(filestore_folder) # ensure log folder exists - ensure_folder_exists(log_folder_path, mode="777", git_ignore_content=True) + ensure_folder_exists(log_folder_path, git_ignore_content=True) # Create log file log_prefix = "{}__{}".format( diff --git a/odoo_openupgrade_wizard/cli/cli_init.py b/odoo_openupgrade_wizard/cli/cli_init.py index db575ef..1e8b693 100644 --- a/odoo_openupgrade_wizard/cli/cli_init.py +++ b/odoo_openupgrade_wizard/cli/cli_init.py @@ -111,7 +111,7 @@ def init( # 4. ensure filestore folder exists ensure_folder_exists( - ctx.obj["filestore_folder_path"], mode="777", git_ignore_content=True + ctx.obj["filestore_folder_path"], git_ignore_content=True ) # 5. ensure postgres data folder exists diff --git a/odoo_openupgrade_wizard/tools/tools_odoo.py b/odoo_openupgrade_wizard/tools/tools_odoo.py index 6e02ec9..5eae1b1 100644 --- a/odoo_openupgrade_wizard/tools/tools_odoo.py +++ b/odoo_openupgrade_wizard/tools/tools_odoo.py @@ -20,7 +20,10 @@ from odoo_openupgrade_wizard.tools.tools_docker import ( run_container, ) from odoo_openupgrade_wizard.tools.tools_postgres import get_postgres_container -from odoo_openupgrade_wizard.tools.tools_system import get_script_folder +from odoo_openupgrade_wizard.tools.tools_system import ( + get_local_user_id, + get_script_folder, +) def get_odoo_addons_path( @@ -112,61 +115,6 @@ def generate_odoo_command( return result -# def generate_odoo_config_file( -# ctx, migration_step, log_file, execution_context -# ): -# """Create a config file name _auto_generated_odoo.cfg -# in the according environment (defined by migration_step) -# This configuration file is a merge of the odoo.cfg file that can -# contain custom values, and the values required to run the docker -# container. -# """ -# odoo_env_path = get_odoo_env_path(ctx, migration_step["version"]) - -# custom_odoo_config_file = odoo_env_path / "odoo.cfg" -# auto_generated_odoo_config_file = ( -# odoo_env_path / "_auto_generated_odoo.cfg" -# ) - -# parser = configparser.RawConfigParser() -# # Read custom file -# parser.read(custom_odoo_config_file) - -# # compute addons_path -# addons_path = ",".join( -# [ -# str(x) -# for x in get_odoo_addons_path( -# ctx, Path("/odoo_env"), migration_step, execution_context -# ) -# ] -# ) - -# # compute server wides modules -# server_wide_modules = parser.get( -# "options", "server_wide_modules", fallback=[] -# ) -# server_wide_modules += get_server_wide_modules_upgrade(migration_step) - -# # Add required keys -# if "options" not in parser: -# parser.add_section("options") -# parser.set("options", "db_host", "db") -# parser.set("options", "db_port", 5432) -# parser.set("options", "db_user", "odoo") -# parser.set("options", "db_password", "odoo") -# parser.set("options", "workers", 0) -# parser.set("options", "data_dir", "/env/filestore/") -# parser.set("options", "logfile", log_file) -# parser.set("options", "addons_path", addons_path) -# if server_wide_modules: -# parser.set( -# "options", "server_wide_modules", ",".join(server_wide_modules) -# ) - -# parser.write(open(auto_generated_odoo_config_file, "w")) - - def run_odoo( ctx, migration_step: dict, @@ -279,7 +227,9 @@ def run_container_odoo( "LOGFILE": log_file, "ADDONS_PATH": addons_path, "WORKERS": 0, + "LOCAL_USER_ID": get_local_user_id(), } + # TODO, handle custom config.cfg file if server_wide_modules: environments["SERVER_WIDE_MODULES"] = ",".join(server_wide_modules) diff --git a/odoo_openupgrade_wizard/tools/tools_system.py b/odoo_openupgrade_wizard/tools/tools_system.py index f309d8e..6d91d5a 100644 --- a/odoo_openupgrade_wizard/tools/tools_system.py +++ b/odoo_openupgrade_wizard/tools/tools_system.py @@ -91,3 +91,8 @@ def git_aggregate(folder_path: Path, config_path: Path, jobs: int): % config_path ) gitaggregate_cmd.run(args) + + +def get_local_user_id(): + # hum, could be improved, don't you think ? + return 1000 From db3d16e37ceb92c6c5a3f6dfec1a9a9810ec3ef9 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 10:34:58 +0200 Subject: [PATCH 09/39] [IMP] make cli_05_execute_script_python_test working --- .../templates/odoo/Dockerfile.j2 | 7 ++-- odoo_openupgrade_wizard/tools/tools_odoo.py | 34 +++++-------------- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 index 130559f..f9c3261 100644 --- a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 +++ b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 @@ -1,6 +1,6 @@ FROM ghcr.io/acsone/odoo-bedrock:{{ odoo_version }}-{{python_minor_version_short}}-latest -COPY ./src/odoo/requirements.txt /odoo_requirements.txt +COPY ./src/odoo/ /odoo/src/odoo COPY debian_requirements.txt /debian_requirements.txt COPY python_requirements.txt /python_requirements.txt @@ -11,5 +11,6 @@ RUN apt-get update || true \ # 3. Install extra Python librairies RUN \ - /odoo/bin/pip install --no-cache-dir -r /odoo_requirements.txt -f {{prebuild_wheel_url}} \ - && /odoo/bin/pip install --no-cache-dir -r /python_requirements.txt -f {{prebuild_wheel_url}} + /odoo/bin/pip install --no-cache-dir -r /odoo/src/odoo/requirements.txt -f https://wheelhouse.acsone.eu/manylinux2014 \ + && /odoo/bin/pip install --no-cache-dir -r /python_requirements.txt -f https://wheelhouse.acsone.eu/manylinux2014 \ + && /odoo/bin/pip install -e /odoo/src/odoo diff --git a/odoo_openupgrade_wizard/tools/tools_odoo.py b/odoo_openupgrade_wizard/tools/tools_odoo.py index 5eae1b1..271169f 100644 --- a/odoo_openupgrade_wizard/tools/tools_odoo.py +++ b/odoo_openupgrade_wizard/tools/tools_odoo.py @@ -274,24 +274,11 @@ def execute_click_odoo_python_files( ] python_files = sorted(python_files) - # Prepare data information for docker - links = {ctx.obj["config"]["postgres_container_name"]: "db"} - env_path = ctx.obj["env_folder_path"] - odoo_env_path = get_odoo_env_path(ctx, migration_step["version"]) - - # Generate odoo config file - # log_file = "/env/log/{}____{}__post_migration.log".format( - # ctx.obj["log_prefix"], migration_step["complete_name"] - # ) - for python_file in python_files: # TODO, check if we should set python2 for old version of Odoo # or just 'python' command = ( - "click-odoo" - " --database {database}" - " --config /odoo_env/_auto_generated_odoo.cfg" - " /env/{python_file}" + "click-odoo" " --database {database}" " /env/{python_file}" ).format( database=database, python_file=str(python_file), @@ -301,19 +288,14 @@ def execute_click_odoo_python_files( "Executing script %s / %s" % (migration_step["complete_name"], python_file) ) - run_container( - get_docker_image_tag(ctx, migration_step["version"]), - get_docker_container_name(ctx, migration_step), - command=command, - ports={}, - volumes={ - env_path: "/env/", - odoo_env_path: "/odoo_env/", - }, - links=links, - detach=False, - auto_remove=True, + return run_container_odoo( + ctx, + migration_step, + command, + detached_container=False, + database=database, ) + except Exception as e: traceback.print_exc() logger.error( From 01b2a94cb48c4b72eff29dca6c4052969271b23d Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 10:58:27 +0200 Subject: [PATCH 10/39] [FIX] do not crash if postgres container exists in a exited status --- .../tools/tools_postgres.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/odoo_openupgrade_wizard/tools/tools_postgres.py b/odoo_openupgrade_wizard/tools/tools_postgres.py index 944a67b..2bd84d0 100644 --- a/odoo_openupgrade_wizard/tools/tools_postgres.py +++ b/odoo_openupgrade_wizard/tools/tools_postgres.py @@ -15,9 +15,19 @@ def get_postgres_container(ctx): client = get_docker_client() image_name = ctx.obj["config"]["postgres_image_name"] container_name = ctx.obj["config"]["postgres_container_name"] - containers = client.containers.list(filters={"name": container_name}) + containers = client.containers.list( + all=True, filters={"name": container_name} + ) if containers: - return containers[0] + container = containers[0] + if container.status == "exited": + logger.warn( + "Found container %s in a exited status. Removing it..." + % container_name + ) + container.remove() + else: + return container logger.info("Launching Postgres Container. (Image %s)" % image_name) container = run_container( @@ -30,10 +40,10 @@ def get_postgres_container(ctx): "PGDATA": "/var/lib/postgresql/data/pgdata", }, volumes={ - ctx.obj["env_folder_path"]: "/env/", + ctx.obj["env_folder_path"].absolute(): "/env/", ctx.obj[ "postgres_folder_path" - ]: "/var/lib/postgresql/data/pgdata/", + ].absolute(): "/var/lib/postgresql/data/pgdata/", }, detach=True, ) From b22d9b5ad432a4a7b3c3331b8958793eba1d61e9 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 11:08:38 +0200 Subject: [PATCH 11/39] [FIX] cli_06_execute_script_sql_test.py --- tests/cli_05_execute_script_python_test.py | 4 ++-- tests/cli_06_execute_script_sql_test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cli_05_execute_script_python_test.py b/tests/cli_05_execute_script_python_test.py index 4d4528a..aa5ebd4 100644 --- a/tests/cli_05_execute_script_python_test.py +++ b/tests/cli_05_execute_script_python_test.py @@ -27,7 +27,7 @@ def test_cli_execute_script_python(): db_name = "database_test_cli___execute_script_python" ensure_database(ctx, db_name, state="absent") - # Install Odoo on V13 with base installed + # Install Odoo on V14 with base installed cli_runner_invoke( [ "--log-level=DEBUG", @@ -40,7 +40,7 @@ def test_cli_execute_script_python(): ) # Compute partners quantity - request = "SELECT count(*)" " FROM res_partner;" + request = "SELECT count(*) FROM res_partner;" partner_quantity_before = int( execute_sql_request(ctx, request, database=db_name)[0][0] ) diff --git a/tests/cli_06_execute_script_sql_test.py b/tests/cli_06_execute_script_sql_test.py index 185e8f4..b94cef0 100644 --- a/tests/cli_06_execute_script_sql_test.py +++ b/tests/cli_06_execute_script_sql_test.py @@ -21,7 +21,7 @@ def test_cli_execute_script_sql(): ).absolute() # Deploy SQL Script - destination_path = Path("scripts/step_01__update__13.0") + destination_path = Path("scripts/step_01__update__14.0") cp([extra_script_path, destination_path]) ctx = build_ctx_from_config_file() From 8035ba287137718077520a31779f3c85e2a7247d Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 11:25:29 +0200 Subject: [PATCH 12/39] [FIX] various test. Readd odoorpc in the requilrements.txt file to generate_module_analysis and add comments in the file --- .../templates/odoo/python_requirements.txt.j2 | 7 +++++++ tests/cli_07_upgrade_test.py | 8 ++++---- tests/cli_08_estimate_workload_test.py | 8 +++----- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/odoo_openupgrade_wizard/templates/odoo/python_requirements.txt.j2 b/odoo_openupgrade_wizard/templates/odoo/python_requirements.txt.j2 index f16e14e..e234fdc 100644 --- a/odoo_openupgrade_wizard/templates/odoo/python_requirements.txt.j2 +++ b/odoo_openupgrade_wizard/templates/odoo/python_requirements.txt.j2 @@ -1,5 +1,12 @@ {%- for python_librairy in python_libraries -%} {{ python_librairy }} {% endfor %} + +# Mandatory library used in all odoo-openupgrade-wizard openupgradelib + +# Library used to run 'post-migration.py' scripts click-odoo + +# Library used to run generate-module-analysis command +odoorpc diff --git a/tests/cli_07_upgrade_test.py b/tests/cli_07_upgrade_test.py index 37ee9b5..fc5ce82 100644 --- a/tests/cli_07_upgrade_test.py +++ b/tests/cli_07_upgrade_test.py @@ -29,7 +29,7 @@ def test_cli_upgrade(): ] ) - # Ensure that 'base' module is installed at 13.0 + # Ensure that 'base' module is installed at 14.0 request = ( "SELECT latest_version" " FROM ir_module_module" @@ -38,7 +38,7 @@ def test_cli_upgrade(): ) latest_version = execute_sql_request(ctx, request, database=db_name) - assert latest_version[0][0].startswith("13.") + assert latest_version[0][0].startswith("14.") cli_runner_invoke( [ @@ -50,7 +50,7 @@ def test_cli_upgrade(): ] ) - # Ensure that 'base' module is installed at 14.0 + # Ensure that 'base' module is installed at 15.0 request = ( "SELECT latest_version" " FROM ir_module_module" @@ -59,4 +59,4 @@ def test_cli_upgrade(): ) latest_version = execute_sql_request(ctx, request, database=db_name) - assert latest_version[0][0].startswith("14.") + assert latest_version[0][0].startswith("15.") diff --git a/tests/cli_08_estimate_workload_test.py b/tests/cli_08_estimate_workload_test.py index c9e8e5b..23ce71f 100644 --- a/tests/cli_08_estimate_workload_test.py +++ b/tests/cli_08_estimate_workload_test.py @@ -14,16 +14,14 @@ class TestCliEstimateWorkload(unittest.TestCase): "estimate-workload", "--extra-modules=" # Done Module - "account" + "base" # Deleted module (because merged) ",account_analytic_default" - # Deleted module (because renamed) - ",account_facturx" # Deleted module (because lost merge) - ",base_gengo" + ",account_edi_extended" # Some modules that are not ported (for the time being) ",l10n_be_invoice_bba,l10n_ch_qriban,l10n_latam_base" - # OCA Portted Modules + # OCA Ported Modules ",web_responsive" # OCA Unported modules ",web_boolean_button" From 1ead68bef07289b66018c7b3a1c88656416e0c91 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 14:12:12 +0200 Subject: [PATCH 13/39] [REF] remove click-odoo that requires to import odoo that is a mess --- .../templates/odoo/Dockerfile.j2 | 14 +++++++++----- .../templates/odoo/python_requirements.txt.j2 | 6 ++++-- odoo_openupgrade_wizard/tools/tools_postgres.py | 2 +- tests/cli_05_execute_script_python_test.py | 1 + tests/cli_07_upgrade_test.py | 1 + 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 index f9c3261..2c9baef 100644 --- a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 +++ b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 @@ -1,8 +1,8 @@ FROM ghcr.io/acsone/odoo-bedrock:{{ odoo_version }}-{{python_minor_version_short}}-latest -COPY ./src/odoo/ /odoo/src/odoo -COPY debian_requirements.txt /debian_requirements.txt +COPY ./src/odoo/requirements.txt /odoo_requirements.txt COPY python_requirements.txt /python_requirements.txt +COPY debian_requirements.txt /debian_requirements.txt # 2. Install extra debian packages RUN apt-get update || true \ @@ -11,6 +11,10 @@ RUN apt-get update || true \ # 3. Install extra Python librairies RUN \ - /odoo/bin/pip install --no-cache-dir -r /odoo/src/odoo/requirements.txt -f https://wheelhouse.acsone.eu/manylinux2014 \ - && /odoo/bin/pip install --no-cache-dir -r /python_requirements.txt -f https://wheelhouse.acsone.eu/manylinux2014 \ - && /odoo/bin/pip install -e /odoo/src/odoo + /odoo/bin/pip install --no-cache-dir -r /odoo_requirements.txt -f {{ prebuild_wheel_url }} \ + && /odoo/bin/pip install --no-cache-dir -r /python_requirements.txt + +# RUN \ +# /odoo/bin/pip install --no-cache-dir -r /odoo/src/odoo/requirements.txt -f {{ prebuild_wheel_url }} \ +# && /odoo/bin/pip install --no-cache-dir -r /python_requirements.txt -f {{ prebuild_wheel_url }} \ +# && /odoo/bin/pip install -e /odoo/src/odoo diff --git a/odoo_openupgrade_wizard/templates/odoo/python_requirements.txt.j2 b/odoo_openupgrade_wizard/templates/odoo/python_requirements.txt.j2 index e234fdc..5f03b8c 100644 --- a/odoo_openupgrade_wizard/templates/odoo/python_requirements.txt.j2 +++ b/odoo_openupgrade_wizard/templates/odoo/python_requirements.txt.j2 @@ -5,8 +5,10 @@ # Mandatory library used in all odoo-openupgrade-wizard openupgradelib -# Library used to run 'post-migration.py' scripts -click-odoo +# # Library used to run 'post-migration.py' scripts +# click-odoo # Library used to run generate-module-analysis command +# dependencies of the module OCA/server-tools 'upgrade_analysis' odoorpc +mako diff --git a/odoo_openupgrade_wizard/tools/tools_postgres.py b/odoo_openupgrade_wizard/tools/tools_postgres.py index 2bd84d0..97ca641 100644 --- a/odoo_openupgrade_wizard/tools/tools_postgres.py +++ b/odoo_openupgrade_wizard/tools/tools_postgres.py @@ -21,7 +21,7 @@ def get_postgres_container(ctx): if containers: container = containers[0] if container.status == "exited": - logger.warn( + logger.warning( "Found container %s in a exited status. Removing it..." % container_name ) diff --git a/tests/cli_05_execute_script_python_test.py b/tests/cli_05_execute_script_python_test.py index aa5ebd4..bb30661 100644 --- a/tests/cli_05_execute_script_python_test.py +++ b/tests/cli_05_execute_script_python_test.py @@ -15,6 +15,7 @@ from . import ( def test_cli_execute_script_python(): + return move_to_test_folder() ctx = build_ctx_from_config_file() diff --git a/tests/cli_07_upgrade_test.py b/tests/cli_07_upgrade_test.py index fc5ce82..d99ca0c 100644 --- a/tests/cli_07_upgrade_test.py +++ b/tests/cli_07_upgrade_test.py @@ -11,6 +11,7 @@ from . import ( def test_cli_upgrade(): + return move_to_test_folder() # Initialize database From ae3619e7667dab067a78346a20f979bdb3f518c8 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 15:13:13 +0200 Subject: [PATCH 14/39] [REF] replace click-odoo by calling shell command directly --- odoo_openupgrade_wizard/tools/tools_odoo.py | 30 ++++++----- tests/cli_05_execute_script_python_test.py | 9 ++-- tests/cli_07_upgrade_test.py | 1 - tests/cli_21_generate_module_analysis_test.py | 2 + tests/data/extra_script/click_odoo_test.py | 11 ---- .../post-migration-custom_test.py | 51 ++++--------------- 6 files changed, 34 insertions(+), 70 deletions(-) delete mode 100644 tests/data/extra_script/click_odoo_test.py diff --git a/odoo_openupgrade_wizard/tools/tools_odoo.py b/odoo_openupgrade_wizard/tools/tools_odoo.py index 271169f..bebf48b 100644 --- a/odoo_openupgrade_wizard/tools/tools_odoo.py +++ b/odoo_openupgrade_wizard/tools/tools_odoo.py @@ -84,11 +84,11 @@ def generate_odoo_command( migration_step: dict, execution_context: str, database: str, - update: str, - init: str, - stop_after_init: bool, - shell: bool, - demo: bool, + demo: bool = False, + update: str = False, + init: str = False, + stop_after_init: bool = False, + shell: bool = False, ) -> str: database_cmd = database and "--database %s" % database or "" update_cmd = update and "--update %s" % update or "" @@ -150,12 +150,12 @@ def run_odoo( ctx, migration_step, execution_context, - database=database, + database, + demo=demo, update=update, init=init, stop_after_init=stop_after_init, shell=shell, - demo=demo, ) return run_container_odoo( @@ -274,13 +274,17 @@ def execute_click_odoo_python_files( ] python_files = sorted(python_files) + command = generate_odoo_command( + ctx, + migration_step, + execution_context, + database, + shell=True, + ) + for python_file in python_files: - # TODO, check if we should set python2 for old version of Odoo - # or just 'python' - command = ( - "click-odoo" " --database {database}" " /env/{python_file}" - ).format( - database=database, + command = ("/bin/bash -c 'cat /env/{python_file} | {command}'").format( + command=command, python_file=str(python_file), ) try: diff --git a/tests/cli_05_execute_script_python_test.py b/tests/cli_05_execute_script_python_test.py index bb30661..ccf7ba2 100644 --- a/tests/cli_05_execute_script_python_test.py +++ b/tests/cli_05_execute_script_python_test.py @@ -15,14 +15,15 @@ from . import ( def test_cli_execute_script_python(): - return move_to_test_folder() ctx = build_ctx_from_config_file() - extra_script_path = Path("../extra_script/click_odoo_test.py").absolute() + extra_script_path = Path( + "../extra_script/post-migration-custom_test.py" + ).absolute() cp( extra_script_path, - Path("click_odoo_test.py"), + Path("post-migration-custom_test.py"), ) db_name = "database_test_cli___execute_script_python" @@ -53,7 +54,7 @@ def test_cli_execute_script_python(): "execute-script-python", "--step=1", "--database=%s" % db_name, - "--script-file-path=click_odoo_test.py", + "--script-file-path=post-migration-custom_test.py", ] ) partner_quantity_after = int( diff --git a/tests/cli_07_upgrade_test.py b/tests/cli_07_upgrade_test.py index d99ca0c..fc5ce82 100644 --- a/tests/cli_07_upgrade_test.py +++ b/tests/cli_07_upgrade_test.py @@ -11,7 +11,6 @@ from . import ( def test_cli_upgrade(): - return move_to_test_folder() # Initialize database diff --git a/tests/cli_21_generate_module_analysis_test.py b/tests/cli_21_generate_module_analysis_test.py index 99cc18c..013bd58 100644 --- a/tests/cli_21_generate_module_analysis_test.py +++ b/tests/cli_21_generate_module_analysis_test.py @@ -10,6 +10,8 @@ from . import ( def test_cli_generate_module_analysis(): + # hum... TODO fix me what this test is broken... + return move_to_test_folder() db_name = "database_test_cli___generate_module_analysis" diff --git a/tests/data/extra_script/click_odoo_test.py b/tests/data/extra_script/click_odoo_test.py deleted file mode 100644 index f83a671..0000000 --- a/tests/data/extra_script/click_odoo_test.py +++ /dev/null @@ -1,11 +0,0 @@ -import logging - -_logger = logging.getLogger(__name__) -_logger.info("click_odoo_test.py : Begin of script ...") - -env = env # noqa: F821 - -for i in range(0, 10): - env["res.partner"].create({"name": "Partner #%d" % (i)}) - -_logger.info("click_odoo_test.py : End of script.") diff --git a/tests/data/extra_script/post-migration-custom_test.py b/tests/data/extra_script/post-migration-custom_test.py index fffdd04..319cf96 100644 --- a/tests/data/extra_script/post-migration-custom_test.py +++ b/tests/data/extra_script/post-migration-custom_test.py @@ -1,46 +1,15 @@ -# Unused for the time being +import logging -# def _check_orm_usage(self): -# # Classic ORM usage Checks -# partners = self.browse_by_search("res.partner") +_logger = logging.getLogger(__name__) +_logger.info("post-migration-custom_test.py : Begin of script ...") -# self.browse_by_create("res.partner", {"name": "New Partner"}) +env = env # noqa: F821 -# new_partners = self.browse_by_search("res.partner") +for i in range(0, 10): + partner_name = "Partner #%d" % (i) + _logger.info("Create Partner %s") + env["res.partner"].create({"name": partner_name}) -# if len(partners) + 1 != len(new_partners): -# raise Exception("Creation of partner failed.") +_logger.info("post-migration-custom_test.py : End of script.") - -# def _check_modules(self): -# if self.check_modules_installed("sale"): -# self.uninstall_modules("sale") - -# self.install_modules("sale") - -# if not self.check_modules_installed("sale"): -# raise Exception("'sale' module should be installed") - -# self.uninstall_modules(["product"]) - -# if self.check_modules_installed("sale"): -# raise Exception( -# "'sale' module should not be installed" -# " after uninstallation of product" -# ) - - -# def _check_models(self): -# if not self.check_models_present("res.partner"): -# raise Exception("'res.partner' model should be present.") - -# if self.check_models_present("res.partner.unexisting.model"): -# raise Exception( -# "'res.partner.unexisting.model' model" " should not be present." -# ) - - -# def main(self): -# _check_orm_usage(self) -# _check_modules(self) -# _check_models(self) +env.cr.commit() From 7fca87bae7e837b9fe62afdffb1434622f73c770 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 15:50:17 +0200 Subject: [PATCH 15/39] [FIX] GREENIFY ALL THE TESTgit status! --- .../templates/scripts/post-migration.py.j2 | 4 ++++ odoo_openupgrade_wizard/tools/tools_odoo.py | 1 + tests/cli_21_generate_module_analysis_test.py | 6 ++---- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/odoo_openupgrade_wizard/templates/scripts/post-migration.py.j2 b/odoo_openupgrade_wizard/templates/scripts/post-migration.py.j2 index 40c3e2b..a4b3aa5 100644 --- a/odoo_openupgrade_wizard/templates/scripts/post-migration.py.j2 +++ b/odoo_openupgrade_wizard/templates/scripts/post-migration.py.j2 @@ -4,3 +4,7 @@ _logger = logging.getLogger(__name__) _logger.info("Executing post-migration.py script ...") env = env # noqa: F821 + +# Write custom script here + +env.cr.commit() diff --git a/odoo_openupgrade_wizard/tools/tools_odoo.py b/odoo_openupgrade_wizard/tools/tools_odoo.py index bebf48b..3d41e19 100644 --- a/odoo_openupgrade_wizard/tools/tools_odoo.py +++ b/odoo_openupgrade_wizard/tools/tools_odoo.py @@ -166,6 +166,7 @@ def run_odoo( database=database, execution_context=execution_context, alternative_xml_rpc_port=alternative_xml_rpc_port, + links=links, ) diff --git a/tests/cli_21_generate_module_analysis_test.py b/tests/cli_21_generate_module_analysis_test.py index 013bd58..eaf5ae5 100644 --- a/tests/cli_21_generate_module_analysis_test.py +++ b/tests/cli_21_generate_module_analysis_test.py @@ -10,16 +10,14 @@ from . import ( def test_cli_generate_module_analysis(): - # hum... TODO fix me what this test is broken... - return move_to_test_folder() db_name = "database_test_cli___generate_module_analysis" ctx = build_ctx_from_config_file() # identify main analysis file of openupgrade - analysis_file_path = get_odoo_env_path(ctx, 14.0) / Path( + analysis_file_path = get_odoo_env_path(ctx, 15.0) / Path( "src/openupgrade/openupgrade_scripts/scripts" - "/base/14.0.1.3/upgrade_general_log.txt" + "/base/15.0.1.3/upgrade_general_log.txt" ) # We remove this file and run the analysis From 424131e9b2c233382dc0a14b358e644fba995e4b Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 16:28:48 +0200 Subject: [PATCH 16/39] [REF] Remove bind fucking mounts (postgres) replaced by nice volumes --- odoo_openupgrade_wizard/cli/cli.py | 5 ---- odoo_openupgrade_wizard/cli/cli_init.py | 25 ++++++------------- .../templates/config.yml.j2 | 3 ++- .../templates/odoo/python_requirements.txt.j2 | 5 +--- .../tools/tools_postgres.py | 16 +++++++++--- tests/__init__.py | 3 --- tests/data/output_expected/config.yml | 3 ++- 7 files changed, 25 insertions(+), 35 deletions(-) diff --git a/odoo_openupgrade_wizard/cli/cli.py b/odoo_openupgrade_wizard/cli/cli.py index d9d61c0..b3cf4dd 100644 --- a/odoo_openupgrade_wizard/cli/cli.py +++ b/odoo_openupgrade_wizard/cli/cli.py @@ -68,10 +68,6 @@ def main(ctx, env_folder, filestore_folder, log_level): # Define all the folder required by the tools env_folder_path = Path(env_folder) src_folder_path = env_folder_path / Path("./src/") - # Note: postgres folder should be a subfolder, because - # the parent folder will contain a .gitignore file - # that the postgres docker image doesn't like - postgres_folder_path = env_folder_path / Path("./postgres_data/data") script_folder_path = env_folder_path / Path("./scripts/") log_folder_path = env_folder_path / Path("./log/") if not filestore_folder: @@ -95,7 +91,6 @@ def main(ctx, env_folder, filestore_folder, log_level): # Add all global values in the context ctx.obj["env_folder_path"] = env_folder_path ctx.obj["src_folder_path"] = src_folder_path - ctx.obj["postgres_folder_path"] = postgres_folder_path ctx.obj["script_folder_path"] = script_folder_path ctx.obj["log_folder_path"] = log_folder_path ctx.obj["log_prefix"] = log_prefix diff --git a/odoo_openupgrade_wizard/cli/cli_init.py b/odoo_openupgrade_wizard/cli/cli_init.py index 1e8b693..e538563 100644 --- a/odoo_openupgrade_wizard/cli/cli_init.py +++ b/odoo_openupgrade_wizard/cli/cli_init.py @@ -68,7 +68,7 @@ def init( float(initial_version), float(final_version) ) - # 2. Compute Migration Steps + # Compute Migration Steps # Create initial first step steps = [ @@ -106,26 +106,15 @@ def init( } ) - # 3. ensure src folder exists + # Ensure src folder exists ensure_folder_exists(ctx.obj["src_folder_path"]) - # 4. ensure filestore folder exists + # Ensure filestore folder exists ensure_folder_exists( ctx.obj["filestore_folder_path"], git_ignore_content=True ) - # 5. ensure postgres data folder exists - ensure_folder_exists( - ctx.obj["postgres_folder_path"].parent, - mode="777", - git_ignore_content=True, - ) - ensure_folder_exists( - ctx.obj["postgres_folder_path"], - mode="777", - ) - - # 6. ensure main configuration file exists + # Znsure main configuration file exists ensure_file_exists_from_template( ctx.obj["config_file_path"], "config.yml.j2", @@ -134,7 +123,7 @@ def init( odoo_versions=odoo_versions, ) - # 7. Ensure module list file exists + # Ensure module list file exists ensure_file_exists_from_template( ctx.obj["module_file_path"], "modules.csv.j2", @@ -143,7 +132,7 @@ def init( odoo_versions=odoo_versions, ) - # 8. Create one folder per version and add files + # Create one folder per version and add files for odoo_version in odoo_versions: # Create main path for each version path_version = get_odoo_env_path(ctx, odoo_version) @@ -196,7 +185,7 @@ def init( path_version / Path("src"), git_ignore_content=True ) - # 9. Create one folder per step and add files + # Create one folder per step and add files ensure_folder_exists(ctx.obj["script_folder_path"]) for step in steps: diff --git a/odoo_openupgrade_wizard/templates/config.yml.j2 b/odoo_openupgrade_wizard/templates/config.yml.j2 index 3df6652..611c5c3 100644 --- a/odoo_openupgrade_wizard/templates/config.yml.j2 +++ b/odoo_openupgrade_wizard/templates/config.yml.j2 @@ -1,7 +1,8 @@ project_name: {{ project_name }} postgres_image_name: postgres:13 -postgres_container_name: {{project_name}}-db +postgres_container_name: {{project_name}}-container-postgres +postgres_volume_name: {{project_name}}-volume-postgres odoo_host_xmlrpc_port: 9069 odoo_default_country_code: FR diff --git a/odoo_openupgrade_wizard/templates/odoo/python_requirements.txt.j2 b/odoo_openupgrade_wizard/templates/odoo/python_requirements.txt.j2 index 5f03b8c..5db3a9e 100644 --- a/odoo_openupgrade_wizard/templates/odoo/python_requirements.txt.j2 +++ b/odoo_openupgrade_wizard/templates/odoo/python_requirements.txt.j2 @@ -1,13 +1,10 @@ {%- for python_librairy in python_libraries -%} {{ python_librairy }} -{% endfor %} +{%- endfor -%} # Mandatory library used in all odoo-openupgrade-wizard openupgradelib -# # Library used to run 'post-migration.py' scripts -# click-odoo - # Library used to run generate-module-analysis command # dependencies of the module OCA/server-tools 'upgrade_analysis' odoorpc diff --git a/odoo_openupgrade_wizard/tools/tools_postgres.py b/odoo_openupgrade_wizard/tools/tools_postgres.py index 97ca641..de87fb5 100644 --- a/odoo_openupgrade_wizard/tools/tools_postgres.py +++ b/odoo_openupgrade_wizard/tools/tools_postgres.py @@ -2,6 +2,7 @@ import os import time from pathlib import Path +import docker from loguru import logger from odoo_openupgrade_wizard.tools.tools_docker import ( @@ -15,6 +16,9 @@ def get_postgres_container(ctx): client = get_docker_client() image_name = ctx.obj["config"]["postgres_image_name"] container_name = ctx.obj["config"]["postgres_container_name"] + volume_name = ctx.obj["config"]["postgres_volume_name"] + + # Check if container exists containers = client.containers.list( all=True, filters={"name": container_name} ) @@ -29,6 +33,14 @@ def get_postgres_container(ctx): else: return container + # Check if volume exists + try: + client.volumes.get(volume_name) + logger.debug("Recovering existing postgres volume: %s" % volume_name) + except docker.errors.NotFound: + logger.info("Creating Postgres volume: %s" % volume_name) + client.volumes.create(volume_name) + logger.info("Launching Postgres Container. (Image %s)" % image_name) container = run_container( image_name, @@ -41,9 +53,7 @@ def get_postgres_container(ctx): }, volumes={ ctx.obj["env_folder_path"].absolute(): "/env/", - ctx.obj[ - "postgres_folder_path" - ].absolute(): "/var/lib/postgresql/data/pgdata/", + volume_name: "/var/lib/postgresql/data/pgdata/", }, detach=True, ) diff --git a/tests/__init__.py b/tests/__init__.py index 217b466..632b9cd 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -58,7 +58,4 @@ def build_ctx_from_config_file() -> dict: ctx.obj["env_folder_path"] = env_folder_path ctx.obj["src_folder_path"] = env_folder_path / Path("src") - ctx.obj["postgres_folder_path"] = env_folder_path / Path( - "postgres_data/data" - ) return ctx diff --git a/tests/data/output_expected/config.yml b/tests/data/output_expected/config.yml index 1619dac..a553e4a 100644 --- a/tests/data/output_expected/config.yml +++ b/tests/data/output_expected/config.yml @@ -1,7 +1,8 @@ project_name: test-cli postgres_image_name: postgres:13 -postgres_container_name: test-cli-db +postgres_container_name: test-cli-container-postgres +postgres_volume_name: test-cli-volume-postgres odoo_host_xmlrpc_port: 9069 odoo_default_country_code: FR From c34715a3939d7b0a8d922fc6c65652513618a0b8 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 16:47:08 +0200 Subject: [PATCH 17/39] [FIX] add importlib-resources as a dependency --- poetry.lock | 10 +++++----- pyproject.toml | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index d960af0..200def8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -312,7 +312,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- name = "importlib-resources" version = "5.4.0" description = "Read resources from Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -797,7 +797,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.15.0" +version = "20.15.1" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -862,7 +862,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">=3.6.3,<4.0.0" -content-hash = "83c796afe3bead38187a2cd46deab78c3b6311c416d583b20fb077014d01f139" +content-hash = "cb073f5cf2ea8d6df3c7e5320419a3bcdd60ac83bf4a308ddea0286de59700b5" [metadata.files] aiocontextvars = [ @@ -1375,8 +1375,8 @@ urllib3 = [ {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, ] virtualenv = [ - {file = "virtualenv-20.15.0-py2.py3-none-any.whl", hash = "sha256:804cce4de5b8a322f099897e308eecc8f6e2951f1a8e7e2b3598dff865f01336"}, - {file = "virtualenv-20.15.0.tar.gz", hash = "sha256:4c44b1d77ca81f8368e2d7414f9b20c428ad16b343ac6d226206c5b84e2b4fcc"}, + {file = "virtualenv-20.15.1-py2.py3-none-any.whl", hash = "sha256:b30aefac647e86af6d82bfc944c556f8f1a9c90427b2fb4e3bfbf338cb82becf"}, + {file = "virtualenv-20.15.1.tar.gz", hash = "sha256:288171134a2ff3bfb1a2f54f119e77cd1b81c29fc1265a2356f3e8d14c7d58c4"}, ] websocket-client = [ {file = "websocket-client-1.3.1.tar.gz", hash = "sha256:6278a75065395418283f887de7c3beafb3aa68dada5cacbe4b214e8d26da499b"}, diff --git a/pyproject.toml b/pyproject.toml index 23d6122..67a0050 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ python = ">=3.6.3,<4.0.0" click = "^7.0" click-loglevel = "^0.4" docker = "^5.0" +importlib-resources = "^5.4" git-aggregator = "^2.1" GitPython = "^3.1" jinja2 = "^3.0" From 63c547f61a8e64aa8681a75934ca639d5c76f764 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 16:53:10 +0200 Subject: [PATCH 18/39] [DOC] update doc, removing click-odoo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7af2b23..b66ecb6 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ It works with Openupgrade OCA tools. (https://github.com/oca/openupgrade) this tool is useful for complex migrations: - migrate several versions - take advantage of the migration to install / uninstall modules -- execute sql requests or click-odoo scripts between each migration +- execute sql requests or odoo shell scripts between each migration - analyse workload It will create a migration environment (with all the code available) @@ -113,7 +113,7 @@ modules.csv before beginning the step. - ``post-migration.py`` can contains extra python command to execute after the execution of the step. - Script will be executed with ``click-odoo`` command. All the ORM is available + Script will be executed with ``odoo shell`` command. All the ORM is available via the ``env`` variable. * ``src`` folder contains a folder per Odoo version. In each environment folder: From 36db85e4b93b6197f41016478d3462851f7f73eb Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 17:17:17 +0200 Subject: [PATCH 19/39] [ADD] Log if container fails during odoo execution --- DEVELOP.md | 1 - odoo_openupgrade_wizard/cli/cli_init.py | 1 - odoo_openupgrade_wizard/tools/tools_odoo.py | 51 +++++++++++++-------- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/DEVELOP.md b/DEVELOP.md index e688d54..1501f57 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -10,7 +10,6 @@ to contribute to that project. * poetry (https://python-poetry.org/) * odoorpc (https://github.com/OCA/odoorpc) * git-aggregator (https://github.com/acsone/git-aggregator) -* click-odoo (https://github.com/acsone/click-odoo) # Extra Developper Requirements diff --git a/odoo_openupgrade_wizard/cli/cli_init.py b/odoo_openupgrade_wizard/cli/cli_init.py index e538563..382aba8 100644 --- a/odoo_openupgrade_wizard/cli/cli_init.py +++ b/odoo_openupgrade_wizard/cli/cli_init.py @@ -79,7 +79,6 @@ def init( "complete_name": "step_01__update__%s" % (odoo_versions[0]), } ] - # Add all upgrade steps step_nbr = 2 for odoo_version in odoo_versions[1:]: diff --git a/odoo_openupgrade_wizard/tools/tools_odoo.py b/odoo_openupgrade_wizard/tools/tools_odoo.py index 3d41e19..794d528 100644 --- a/odoo_openupgrade_wizard/tools/tools_odoo.py +++ b/odoo_openupgrade_wizard/tools/tools_odoo.py @@ -5,6 +5,7 @@ import sys import traceback from pathlib import Path +import docker import yaml from loguru import logger @@ -208,9 +209,10 @@ def run_container_odoo( ) # compute 'log_file' - log_file = "/env/log/{}____{}{}.log".format( + log_file_name = "{}____{}{}.log".format( ctx.obj["log_prefix"], migration_step["complete_name"], log_file_suffix ) + log_file_docker_path = "/env/log/%s" % log_file_name host_xmlrpc_port = ( alternative_xml_rpc_port and alternative_xml_rpc_port @@ -225,7 +227,7 @@ def run_container_odoo( "DB_USER": "odoo", "DB_PASSWORD": "odoo", "DB_NAME": database, - "LOGFILE": log_file, + "LOGFILE": log_file_docker_path, "ADDONS_PATH": addons_path, "WORKERS": 0, "LOCAL_USER_ID": get_local_user_id(), @@ -234,22 +236,35 @@ def run_container_odoo( if server_wide_modules: environments["SERVER_WIDE_MODULES"] = ",".join(server_wide_modules) - return run_container( - get_docker_image_tag(ctx, migration_step["version"]), - get_docker_container_name(ctx, migration_step), - command=command, - ports={ - host_xmlrpc_port: 8069, - }, - volumes={ - env_path: "/env/", - odoo_env_path: "/odoo_env/", - }, - environments=environments, - links=links, - detach=detached_container, - auto_remove=True, - ) + try: + return run_container( + get_docker_image_tag(ctx, migration_step["version"]), + get_docker_container_name(ctx, migration_step), + command=command[:-5], + ports={ + host_xmlrpc_port: 8069, + }, + volumes={ + env_path: "/env/", + odoo_env_path: "/odoo_env/", + }, + environments=environments, + links=links, + detach=detached_container, + auto_remove=True, + ) + except docker.errors.ContainerError as exception: + host_log_file_path = ctx.obj["log_folder_path"] / log_file_name + if host_log_file_path.exists(): + with open(host_log_file_path) as _log_file: + logger.debug("*" * 50) + logger.debug("*" * 50) + logger.debug("*" * 50) + logger.debug(_log_file.read()) + logger.debug("*" * 50) + logger.debug("*" * 50) + logger.debug("*" * 50) + raise exception def kill_odoo(ctx, migration_step: dict): From 1261dfaa2f23d5a8ec5b8559c3ada1615d07a655 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 17:37:08 +0200 Subject: [PATCH 20/39] [REF] Add logger in test --- odoo_openupgrade_wizard/cli/cli_init.py | 1 + tests/__init__.py | 38 ++++++++++++++----- tests/cli_01_init_test.py | 11 +++++- tests/cli_02_get_code_test.py | 11 +++++- tests/cli_03_docker_build_test.py | 11 +++++- tests/cli_04_run_test.py | 3 +- tests/cli_05_execute_script_python_test.py | 6 ++- tests/cli_06_execute_script_sql_test.py | 3 +- tests/cli_07_upgrade_test.py | 8 ++-- tests/cli_08_estimate_workload_test.py | 10 ++++- tests/cli_20_install_from_csv_test.py | 3 +- tests/cli_21_generate_module_analysis_test.py | 3 +- 12 files changed, 81 insertions(+), 27 deletions(-) diff --git a/odoo_openupgrade_wizard/cli/cli_init.py b/odoo_openupgrade_wizard/cli/cli_init.py index 382aba8..c630b72 100644 --- a/odoo_openupgrade_wizard/cli/cli_init.py +++ b/odoo_openupgrade_wizard/cli/cli_init.py @@ -51,6 +51,7 @@ def init( """Initialize OpenUpgrade Wizard Environment based on the initial and the final version of Odoo you want to migrate. """ + print(0 / 0) # Handle arguments if extra_repository_list: diff --git a/tests/__init__.py b/tests/__init__.py index 632b9cd..be5e39b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -28,16 +28,33 @@ def move_to_test_folder(): os.chdir(test_folder_path) -def cli_runner_invoke(cmd): - result = CliRunner().invoke( - main, - cmd, - catch_exceptions=False, - ) - if not result.exit_code == 0: - _logger.error("exit_code: %s" % result.exit_code) - _logger.error("output: %s" % result.output) - assert result.exit_code == 0 +def cli_runner_invoke(ctx, cmd): + try: + result = CliRunner().invoke( + main, + cmd, + catch_exceptions=False, + ) + if not result.exit_code == 0: + _logger.error("exit_code: %s" % result.exit_code) + _logger.error("output: %s" % result.output) + assert result.exit_code == 0 + except Exception as exception: + + log_files = [ + ctx.obj["log_folder_path"] / Path(f) + for f in os.listdir(ctx.obj["log_folder_path"]) + if f[-4:] == ".log" + ] + for log_file in log_files: + print("============================") + print(log_file) + print("============================") + _f = open(log_file) + print(_f.read()) + _f.close() + print("============================") + raise exception def build_ctx_from_config_file() -> dict: @@ -58,4 +75,5 @@ def build_ctx_from_config_file() -> dict: ctx.obj["env_folder_path"] = env_folder_path ctx.obj["src_folder_path"] = env_folder_path / Path("src") + ctx.obj["log_folder_path"] = env_folder_path / Path("log") return ctx diff --git a/tests/cli_01_init_test.py b/tests/cli_01_init_test.py index 490e071..eb20d9e 100644 --- a/tests/cli_01_init_test.py +++ b/tests/cli_01_init_test.py @@ -1,14 +1,21 @@ import filecmp from pathlib import Path -from . import cli_runner_invoke, move_to_test_folder +from . import ( + build_ctx_from_config_file, + cli_runner_invoke, + move_to_test_folder, +) def test_cli_init(): move_to_test_folder() + ctx = build_ctx_from_config_file() + expected_folder_path = Path("../output_expected").absolute() cli_runner_invoke( + ctx, [ "--log-level=DEBUG", "init", @@ -17,7 +24,7 @@ def test_cli_init(): "--final-version=15.0", "--extra-repository=" "OCA/web,OCA/server-tools,OCA/bank-statement-import", - ] + ], ) assert filecmp.cmp( diff --git a/tests/cli_02_get_code_test.py b/tests/cli_02_get_code_test.py index ffef04b..ef7d9b5 100644 --- a/tests/cli_02_get_code_test.py +++ b/tests/cli_02_get_code_test.py @@ -1,15 +1,22 @@ from pathlib import Path -from . import cli_runner_invoke, move_to_test_folder +from . import ( + build_ctx_from_config_file, + cli_runner_invoke, + move_to_test_folder, +) def test_cli_get_code(): move_to_test_folder() + ctx = build_ctx_from_config_file() + cli_runner_invoke( + ctx, [ "--log-level=DEBUG", "get-code", - ] + ], ) # Check V14 diff --git a/tests/cli_03_docker_build_test.py b/tests/cli_03_docker_build_test.py index 6bbca10..649ab7a 100644 --- a/tests/cli_03_docker_build_test.py +++ b/tests/cli_03_docker_build_test.py @@ -1,16 +1,23 @@ from odoo_openupgrade_wizard.tools.tools_docker import get_docker_client -from . import cli_runner_invoke, move_to_test_folder +from . import ( + build_ctx_from_config_file, + cli_runner_invoke, + move_to_test_folder, +) def test_cli_docker_build(): move_to_test_folder() + ctx = build_ctx_from_config_file() + cli_runner_invoke( + ctx, [ "--log-level=DEBUG", "docker-build", "--versions=14.0,15.0", - ] + ], ) docker_client = get_docker_client() diff --git a/tests/cli_04_run_test.py b/tests/cli_04_run_test.py index 968ed08..834256f 100644 --- a/tests/cli_04_run_test.py +++ b/tests/cli_04_run_test.py @@ -21,6 +21,7 @@ def test_cli_run(): ensure_database(ctx, db_name, state="absent") cli_runner_invoke( + ctx, [ "--log-level=DEBUG", "run", @@ -28,7 +29,7 @@ def test_cli_run(): "--database=%s" % db_name, "--init-modules=base", "--stop-after-init", - ] + ], ) # Ensure that a subfolder filestore/DB_NAME has been created diff --git a/tests/cli_05_execute_script_python_test.py b/tests/cli_05_execute_script_python_test.py index ccf7ba2..a151b8c 100644 --- a/tests/cli_05_execute_script_python_test.py +++ b/tests/cli_05_execute_script_python_test.py @@ -31,6 +31,7 @@ def test_cli_execute_script_python(): # Install Odoo on V14 with base installed cli_runner_invoke( + ctx, [ "--log-level=DEBUG", "run", @@ -38,7 +39,7 @@ def test_cli_execute_script_python(): "--database=%s" % db_name, "--init-modules=base", "--stop-after-init", - ] + ], ) # Compute partners quantity @@ -49,13 +50,14 @@ def test_cli_execute_script_python(): # Execute Custom Python Script cli_runner_invoke( + ctx, [ "--log-level=DEBUG", "execute-script-python", "--step=1", "--database=%s" % db_name, "--script-file-path=post-migration-custom_test.py", - ] + ], ) partner_quantity_after = int( execute_sql_request(ctx, request, database=db_name)[0][0] diff --git a/tests/cli_06_execute_script_sql_test.py b/tests/cli_06_execute_script_sql_test.py index b94cef0..fa71486 100644 --- a/tests/cli_06_execute_script_sql_test.py +++ b/tests/cli_06_execute_script_sql_test.py @@ -33,12 +33,13 @@ def test_cli_execute_script_sql(): # TODO call with script-file-path # to avoid to copy file in scripts/step_xxx folder cli_runner_invoke( + ctx, [ "--log-level=DEBUG", "execute-script-sql", "--step=1", "--database=%s" % db_name, - ] + ], ) # Ensure that the request has been done correctly diff --git a/tests/cli_07_upgrade_test.py b/tests/cli_07_upgrade_test.py index fc5ce82..e1b0c63 100644 --- a/tests/cli_07_upgrade_test.py +++ b/tests/cli_07_upgrade_test.py @@ -12,13 +12,14 @@ from . import ( def test_cli_upgrade(): move_to_test_folder() + ctx = build_ctx_from_config_file() # Initialize database db_name = "database_test_cli___upgrade" - ctx = build_ctx_from_config_file() ensure_database(ctx, db_name, state="absent") cli_runner_invoke( + ctx, [ "--log-level=DEBUG", "run", @@ -26,7 +27,7 @@ def test_cli_upgrade(): "--database=%s" % db_name, "--init-modules=base", "--stop-after-init", - ] + ], ) # Ensure that 'base' module is installed at 14.0 @@ -41,13 +42,14 @@ def test_cli_upgrade(): assert latest_version[0][0].startswith("14.") cli_runner_invoke( + ctx, [ "--log-level=DEBUG", "upgrade", "--database=%s" % db_name, "--first-step=1", "--last-step=3", - ] + ], ) # Ensure that 'base' module is installed at 15.0 diff --git a/tests/cli_08_estimate_workload_test.py b/tests/cli_08_estimate_workload_test.py index 23ce71f..3a88555 100644 --- a/tests/cli_08_estimate_workload_test.py +++ b/tests/cli_08_estimate_workload_test.py @@ -1,14 +1,20 @@ import unittest from pathlib import Path -from . import cli_runner_invoke, move_to_test_folder +from . import ( + build_ctx_from_config_file, + cli_runner_invoke, + move_to_test_folder, +) class TestCliEstimateWorkload(unittest.TestCase): def test_cli_estimate_workload(self): move_to_test_folder() + ctx = build_ctx_from_config_file() cli_runner_invoke( + ctx, [ "--log-level=DEBUG", "estimate-workload", @@ -32,7 +38,7 @@ class TestCliEstimateWorkload(unittest.TestCase): ",web_widget_one2many_tree_line_duplicate" ",web_widget_dropdown_dynamic_example" ",my_module_that_doesnt_exist", - ] + ], ) # We check file has been created diff --git a/tests/cli_20_install_from_csv_test.py b/tests/cli_20_install_from_csv_test.py index 222860c..c539a4f 100644 --- a/tests/cli_20_install_from_csv_test.py +++ b/tests/cli_20_install_from_csv_test.py @@ -19,11 +19,12 @@ def test_cli_install_from_csv(): ensure_database(ctx, db_name, state="absent") cli_runner_invoke( + ctx, [ "--log-level=DEBUG", "install-from-csv", "--database=%s" % db_name, - ] + ], ) # Ensure that 'base' is installed diff --git a/tests/cli_21_generate_module_analysis_test.py b/tests/cli_21_generate_module_analysis_test.py index eaf5ae5..f640aeb 100644 --- a/tests/cli_21_generate_module_analysis_test.py +++ b/tests/cli_21_generate_module_analysis_test.py @@ -28,13 +28,14 @@ def test_cli_generate_module_analysis(): analysis_file_path cli_runner_invoke( + ctx, [ "--log-level=DEBUG", "generate-module-analysis", "--step=2", "--database=%s" % db_name, "--modules=base", - ] + ], ) # The file should has been recreated by the analysis command From 9f5dea8a00c5b85bbf74620ae8e902a81421510a Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 17:40:38 +0200 Subject: [PATCH 21/39] [REF] test add absolute file path --- tests/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index be5e39b..a94b917 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -67,7 +67,9 @@ def build_ctx_from_config_file() -> dict: setattr(ctx, "obj", {}) config_file_path = env_folder_path / "config.yml" if not config_file_path.exists(): - raise Exception("Configuration file not found %s" % config_file_path) + raise Exception( + "Configuration file not found %s" % config_file_path.absolute() + ) with open(config_file_path) as file: config = yaml.safe_load(file) ctx.obj["config"] = config From 842e2ca3894054ce6b105f86993931e44e5c0ec5 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 17:45:21 +0200 Subject: [PATCH 22/39] various --- tests/__init__.py | 7 +++---- tests/cli_01_init_test.py | 8 +------- tests/cli_02_get_code_test.py | 8 +------- tests/cli_03_docker_build_test.py | 8 +------- tests/cli_04_run_test.py | 1 - tests/cli_05_execute_script_python_test.py | 2 -- tests/cli_06_execute_script_sql_test.py | 3 ++- tests/cli_07_upgrade_test.py | 2 -- tests/cli_08_estimate_workload_test.py | 8 +------- tests/cli_20_install_from_csv_test.py | 1 - tests/cli_21_generate_module_analysis_test.py | 4 ++-- 11 files changed, 11 insertions(+), 41 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index a94b917..d2616bb 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -28,7 +28,7 @@ def move_to_test_folder(): os.chdir(test_folder_path) -def cli_runner_invoke(ctx, cmd): +def cli_runner_invoke(cmd): try: result = CliRunner().invoke( main, @@ -42,8 +42,8 @@ def cli_runner_invoke(ctx, cmd): except Exception as exception: log_files = [ - ctx.obj["log_folder_path"] / Path(f) - for f in os.listdir(ctx.obj["log_folder_path"]) + Path("log") / Path(f) + for f in os.listdir(Path("log")) if f[-4:] == ".log" ] for log_file in log_files: @@ -77,5 +77,4 @@ def build_ctx_from_config_file() -> dict: ctx.obj["env_folder_path"] = env_folder_path ctx.obj["src_folder_path"] = env_folder_path / Path("src") - ctx.obj["log_folder_path"] = env_folder_path / Path("log") return ctx diff --git a/tests/cli_01_init_test.py b/tests/cli_01_init_test.py index eb20d9e..01e1b0c 100644 --- a/tests/cli_01_init_test.py +++ b/tests/cli_01_init_test.py @@ -1,21 +1,15 @@ import filecmp from pathlib import Path -from . import ( - build_ctx_from_config_file, - cli_runner_invoke, - move_to_test_folder, -) +from . import cli_runner_invoke, move_to_test_folder def test_cli_init(): move_to_test_folder() - ctx = build_ctx_from_config_file() expected_folder_path = Path("../output_expected").absolute() cli_runner_invoke( - ctx, [ "--log-level=DEBUG", "init", diff --git a/tests/cli_02_get_code_test.py b/tests/cli_02_get_code_test.py index ef7d9b5..a686c65 100644 --- a/tests/cli_02_get_code_test.py +++ b/tests/cli_02_get_code_test.py @@ -1,18 +1,12 @@ from pathlib import Path -from . import ( - build_ctx_from_config_file, - cli_runner_invoke, - move_to_test_folder, -) +from . import cli_runner_invoke, move_to_test_folder def test_cli_get_code(): move_to_test_folder() - ctx = build_ctx_from_config_file() cli_runner_invoke( - ctx, [ "--log-level=DEBUG", "get-code", diff --git a/tests/cli_03_docker_build_test.py b/tests/cli_03_docker_build_test.py index 649ab7a..a2b0be3 100644 --- a/tests/cli_03_docker_build_test.py +++ b/tests/cli_03_docker_build_test.py @@ -1,18 +1,12 @@ from odoo_openupgrade_wizard.tools.tools_docker import get_docker_client -from . import ( - build_ctx_from_config_file, - cli_runner_invoke, - move_to_test_folder, -) +from . import cli_runner_invoke, move_to_test_folder def test_cli_docker_build(): move_to_test_folder() - ctx = build_ctx_from_config_file() cli_runner_invoke( - ctx, [ "--log-level=DEBUG", "docker-build", diff --git a/tests/cli_04_run_test.py b/tests/cli_04_run_test.py index 834256f..e0a4a59 100644 --- a/tests/cli_04_run_test.py +++ b/tests/cli_04_run_test.py @@ -21,7 +21,6 @@ def test_cli_run(): ensure_database(ctx, db_name, state="absent") cli_runner_invoke( - ctx, [ "--log-level=DEBUG", "run", diff --git a/tests/cli_05_execute_script_python_test.py b/tests/cli_05_execute_script_python_test.py index a151b8c..4f44e4f 100644 --- a/tests/cli_05_execute_script_python_test.py +++ b/tests/cli_05_execute_script_python_test.py @@ -31,7 +31,6 @@ def test_cli_execute_script_python(): # Install Odoo on V14 with base installed cli_runner_invoke( - ctx, [ "--log-level=DEBUG", "run", @@ -50,7 +49,6 @@ def test_cli_execute_script_python(): # Execute Custom Python Script cli_runner_invoke( - ctx, [ "--log-level=DEBUG", "execute-script-python", diff --git a/tests/cli_06_execute_script_sql_test.py b/tests/cli_06_execute_script_sql_test.py index fa71486..54294df 100644 --- a/tests/cli_06_execute_script_sql_test.py +++ b/tests/cli_06_execute_script_sql_test.py @@ -16,6 +16,8 @@ from . import ( def test_cli_execute_script_sql(): move_to_test_folder() + ctx = build_ctx_from_config_file() + extra_script_path = Path( "../extra_script/pre-migration-custom_test.sql" ).absolute() @@ -23,7 +25,6 @@ def test_cli_execute_script_sql(): # Deploy SQL Script destination_path = Path("scripts/step_01__update__14.0") cp([extra_script_path, destination_path]) - ctx = build_ctx_from_config_file() # Reset database db_name = "database_test_cli___execute_script_sql" diff --git a/tests/cli_07_upgrade_test.py b/tests/cli_07_upgrade_test.py index e1b0c63..a8b610d 100644 --- a/tests/cli_07_upgrade_test.py +++ b/tests/cli_07_upgrade_test.py @@ -19,7 +19,6 @@ def test_cli_upgrade(): ensure_database(ctx, db_name, state="absent") cli_runner_invoke( - ctx, [ "--log-level=DEBUG", "run", @@ -42,7 +41,6 @@ def test_cli_upgrade(): assert latest_version[0][0].startswith("14.") cli_runner_invoke( - ctx, [ "--log-level=DEBUG", "upgrade", diff --git a/tests/cli_08_estimate_workload_test.py b/tests/cli_08_estimate_workload_test.py index 3a88555..7007611 100644 --- a/tests/cli_08_estimate_workload_test.py +++ b/tests/cli_08_estimate_workload_test.py @@ -1,20 +1,14 @@ import unittest from pathlib import Path -from . import ( - build_ctx_from_config_file, - cli_runner_invoke, - move_to_test_folder, -) +from . import cli_runner_invoke, move_to_test_folder class TestCliEstimateWorkload(unittest.TestCase): def test_cli_estimate_workload(self): move_to_test_folder() - ctx = build_ctx_from_config_file() cli_runner_invoke( - ctx, [ "--log-level=DEBUG", "estimate-workload", diff --git a/tests/cli_20_install_from_csv_test.py b/tests/cli_20_install_from_csv_test.py index c539a4f..8bc21fb 100644 --- a/tests/cli_20_install_from_csv_test.py +++ b/tests/cli_20_install_from_csv_test.py @@ -19,7 +19,6 @@ def test_cli_install_from_csv(): ensure_database(ctx, db_name, state="absent") cli_runner_invoke( - ctx, [ "--log-level=DEBUG", "install-from-csv", diff --git a/tests/cli_21_generate_module_analysis_test.py b/tests/cli_21_generate_module_analysis_test.py index f640aeb..3280e66 100644 --- a/tests/cli_21_generate_module_analysis_test.py +++ b/tests/cli_21_generate_module_analysis_test.py @@ -11,9 +11,10 @@ from . import ( def test_cli_generate_module_analysis(): move_to_test_folder() + ctx = build_ctx_from_config_file() + db_name = "database_test_cli___generate_module_analysis" - ctx = build_ctx_from_config_file() # identify main analysis file of openupgrade analysis_file_path = get_odoo_env_path(ctx, 15.0) / Path( "src/openupgrade/openupgrade_scripts/scripts" @@ -28,7 +29,6 @@ def test_cli_generate_module_analysis(): analysis_file_path cli_runner_invoke( - ctx, [ "--log-level=DEBUG", "generate-module-analysis", From 4427470a41297f86150d9b100fcaddafc31ea8f1 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 17:46:58 +0200 Subject: [PATCH 23/39] various --- odoo_openupgrade_wizard/cli/cli_init.py | 1 - tests/__init__.py | 30 ++++++++++++------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/odoo_openupgrade_wizard/cli/cli_init.py b/odoo_openupgrade_wizard/cli/cli_init.py index c630b72..382aba8 100644 --- a/odoo_openupgrade_wizard/cli/cli_init.py +++ b/odoo_openupgrade_wizard/cli/cli_init.py @@ -51,7 +51,6 @@ def init( """Initialize OpenUpgrade Wizard Environment based on the initial and the final version of Odoo you want to migrate. """ - print(0 / 0) # Handle arguments if extra_repository_list: diff --git a/tests/__init__.py b/tests/__init__.py index d2616bb..46945f8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -40,21 +40,21 @@ def cli_runner_invoke(cmd): _logger.error("output: %s" % result.output) assert result.exit_code == 0 except Exception as exception: - - log_files = [ - Path("log") / Path(f) - for f in os.listdir(Path("log")) - if f[-4:] == ".log" - ] - for log_file in log_files: - print("============================") - print(log_file) - print("============================") - _f = open(log_file) - print(_f.read()) - _f.close() - print("============================") - raise exception + if Path("log").exists(): + log_files = [ + Path("log") / Path(f) + for f in os.listdir(Path("log")) + if f[-4:] == ".log" + ] + for log_file in log_files: + print("============================") + print(log_file) + print("============================") + _f = open(log_file) + print(_f.read()) + _f.close() + print("============================") + raise exception def build_ctx_from_config_file() -> dict: From 80bd469095edf1d036b56867f9e7c7cdb034d507 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 17:52:58 +0200 Subject: [PATCH 24/39] [FIX] replace hardcoded 1000 value by os.getuid() --- odoo_openupgrade_wizard/tools/tools_system.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/odoo_openupgrade_wizard/tools/tools_system.py b/odoo_openupgrade_wizard/tools/tools_system.py index 6d91d5a..c710aed 100644 --- a/odoo_openupgrade_wizard/tools/tools_system.py +++ b/odoo_openupgrade_wizard/tools/tools_system.py @@ -94,5 +94,4 @@ def git_aggregate(folder_path: Path, config_path: Path, jobs: int): def get_local_user_id(): - # hum, could be improved, don't you think ? - return 1000 + return os.getuid() From 819ef6a41791b829de2dccf7fd9ca15333f16f44 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 18:02:21 +0200 Subject: [PATCH 25/39] [FIX] par exemple, introduire des bugs, c'est pas gentils --- odoo_openupgrade_wizard/tools/tools_odoo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odoo_openupgrade_wizard/tools/tools_odoo.py b/odoo_openupgrade_wizard/tools/tools_odoo.py index 794d528..381cb5b 100644 --- a/odoo_openupgrade_wizard/tools/tools_odoo.py +++ b/odoo_openupgrade_wizard/tools/tools_odoo.py @@ -240,7 +240,7 @@ def run_container_odoo( return run_container( get_docker_image_tag(ctx, migration_step["version"]), get_docker_container_name(ctx, migration_step), - command=command[:-5], + command=command, ports={ host_xmlrpc_port: 8069, }, From f44ac7f7cda729a1589997ad6053a06e43d5c4e1 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 18:15:14 +0200 Subject: [PATCH 26/39] [FIX] par exemple, introduire des bugs, c'est pas gentils (2) --- tests/cli_06_execute_script_sql_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/cli_06_execute_script_sql_test.py b/tests/cli_06_execute_script_sql_test.py index 54294df..81fd69e 100644 --- a/tests/cli_06_execute_script_sql_test.py +++ b/tests/cli_06_execute_script_sql_test.py @@ -34,7 +34,6 @@ def test_cli_execute_script_sql(): # TODO call with script-file-path # to avoid to copy file in scripts/step_xxx folder cli_runner_invoke( - ctx, [ "--log-level=DEBUG", "execute-script-sql", From 798e27ab1692d28fb37a6b3dd52a6f2b1a2f0e06 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 20:13:08 +0200 Subject: [PATCH 27/39] [IMP] add logger in execute_sql_file --- odoo_openupgrade_wizard/tools/tools_postgres.py | 3 ++- tests/data/extra_script/post-migration-custom_test.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/odoo_openupgrade_wizard/tools/tools_postgres.py b/odoo_openupgrade_wizard/tools/tools_postgres.py index de87fb5..d719944 100644 --- a/odoo_openupgrade_wizard/tools/tools_postgres.py +++ b/odoo_openupgrade_wizard/tools/tools_postgres.py @@ -83,7 +83,7 @@ def execute_sql_file(ctx, database, sql_file): container_path = Path("/env/") / relative_path docker_command = ( - "psql" " --username=odoo" " --dbname={database}" " --file {file_path}" + "psql --username=odoo --dbname={database} --file {file_path}" ).format(database=database, file_path=container_path) logger.info( "Executing the script '%s' in postgres container" @@ -91,6 +91,7 @@ def execute_sql_file(ctx, database, sql_file): ) docker_result = container.exec_run(docker_command) if docker_result.exit_code != 0: + logger.error(docker_result.output) raise Exception( "The script '%s' failed on database %s. Exit Code : %d" % (relative_path, database, docker_result.exit_code) diff --git a/tests/data/extra_script/post-migration-custom_test.py b/tests/data/extra_script/post-migration-custom_test.py index 319cf96..ba18852 100644 --- a/tests/data/extra_script/post-migration-custom_test.py +++ b/tests/data/extra_script/post-migration-custom_test.py @@ -7,7 +7,7 @@ env = env # noqa: F821 for i in range(0, 10): partner_name = "Partner #%d" % (i) - _logger.info("Create Partner %s") + _logger.info("Create Partner %s" % partner_name) env["res.partner"].create({"name": partner_name}) _logger.info("post-migration-custom_test.py : End of script.") From a9f7fa3568f39e2f0eeb19048f03f0171f177e69 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 21:03:06 +0200 Subject: [PATCH 28/39] improve log --- odoo_openupgrade_wizard/tools/tools_postgres.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/odoo_openupgrade_wizard/tools/tools_postgres.py b/odoo_openupgrade_wizard/tools/tools_postgres.py index d719944..d7973ae 100644 --- a/odoo_openupgrade_wizard/tools/tools_postgres.py +++ b/odoo_openupgrade_wizard/tools/tools_postgres.py @@ -91,10 +91,16 @@ def execute_sql_file(ctx, database, sql_file): ) docker_result = container.exec_run(docker_command) if docker_result.exit_code != 0: - logger.error(docker_result.output) raise Exception( - "The script '%s' failed on database %s. Exit Code : %d" - % (relative_path, database, docker_result.exit_code) + "The script '%s' failed on database %s.\n" + "- Exit Code : %d\n" + "- Output: %s" + % ( + relative_path, + database, + docker_result.exit_code, + docker_result.output, + ) ) From afcdeaa81d23f925c30ac62a59dea51d5277cdb0 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 21:16:27 +0200 Subject: [PATCH 29/39] ADD a lot of logger to try to understand why it works locally and not on the CI --- .../tools/tools_postgres.py | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/odoo_openupgrade_wizard/tools/tools_postgres.py b/odoo_openupgrade_wizard/tools/tools_postgres.py index d7973ae..fa33a57 100644 --- a/odoo_openupgrade_wizard/tools/tools_postgres.py +++ b/odoo_openupgrade_wizard/tools/tools_postgres.py @@ -82,13 +82,36 @@ def execute_sql_file(ctx, database, sql_file): ) container_path = Path("/env/") / relative_path - docker_command = ( - "psql --username=odoo --dbname={database} --file {file_path}" - ).format(database=database, file_path=container_path) - logger.info( - "Executing the script '%s' in postgres container" - " on database %s" % (relative_path, database) - ) + if container_path.exists(): + docker_command = ( + "psql --username=odoo --dbname={database} --file {file_path}" + ).format(database=database, file_path=container_path) + logger.info( + "Executing the script '%s' in postgres container" + " on database %s" % (relative_path, database) + ) + else: + logger.error("========================") + logger.error("========================") + logger.error("========================") + + def show_me_the_folders(path): + if not path.exists(): + return + else: + logger.info("path '%s' exist" % path) + logger.info("\n" + "\n -".join([f for f in os.listdir(path)])) + + show_me_the_folders(ctx.obj["env_folder_path"]) + show_me_the_folders(ctx.obj["env_folder_path"] / Path("scripts")) + show_me_the_folders( + ctx.obj["env_folder_path"] + / Path("scripts") + / Path("step_01__update__14") + ) + logger.error("========================") + + raise Exception("%s doesn't exist" % container_path) docker_result = container.exec_run(docker_command) if docker_result.exit_code != 0: raise Exception( From 4500d9fe43f47fe95a31f934192af880ddd4d96a Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 21:53:26 +0200 Subject: [PATCH 30/39] [REF] introduce exec_container command --- odoo_openupgrade_wizard/tools/tools_docker.py | 23 ++++++- .../tools/tools_postgres.py | 62 ++++--------------- 2 files changed, 34 insertions(+), 51 deletions(-) diff --git a/odoo_openupgrade_wizard/tools/tools_docker.py b/odoo_openupgrade_wizard/tools/tools_docker.py index fb9b6aa..c6212b8 100644 --- a/odoo_openupgrade_wizard/tools/tools_docker.py +++ b/odoo_openupgrade_wizard/tools/tools_docker.py @@ -66,7 +66,7 @@ def run_container( debug_docker_command += " %s" % (image_name) if command: debug_docker_command += " \\\n%s" % (command) - logger.debug("DOCKER COMMAND:\n %s" % debug_docker_command) + logger.debug("DOCKER COMMAND:\n%s" % debug_docker_command) container = client.containers.run( image_name, @@ -87,6 +87,27 @@ def run_container( return container +def exec_container(container, command): + debug_docker_command = "docker exec %s" % (container.name) + debug_docker_command += " \\\n%s" % (command) + logger.debug("DOCKER COMMAND:\n%s" % debug_docker_command) + docker_result = container.exec_run(command) + if docker_result.exit_code != 0: + raise Exception( + "The command failed in the container %s.\n" + "- Command : %s\n" + "- Exit Code : %d\n" + "- Output: %s" + % ( + container.name, + command, + docker_result.exit_code, + docker_result.output, + ) + ) + return docker_result + + def kill_container(container_name): client = get_docker_client() containers = client.containers.list( diff --git a/odoo_openupgrade_wizard/tools/tools_postgres.py b/odoo_openupgrade_wizard/tools/tools_postgres.py index fa33a57..9127c51 100644 --- a/odoo_openupgrade_wizard/tools/tools_postgres.py +++ b/odoo_openupgrade_wizard/tools/tools_postgres.py @@ -6,6 +6,7 @@ import docker from loguru import logger from odoo_openupgrade_wizard.tools.tools_docker import ( + exec_container, get_docker_client, run_container, ) @@ -82,54 +83,19 @@ def execute_sql_file(ctx, database, sql_file): ) container_path = Path("/env/") / relative_path - if container_path.exists(): - docker_command = ( - "psql --username=odoo --dbname={database} --file {file_path}" - ).format(database=database, file_path=container_path) - logger.info( - "Executing the script '%s' in postgres container" - " on database %s" % (relative_path, database) - ) - else: - logger.error("========================") - logger.error("========================") - logger.error("========================") - - def show_me_the_folders(path): - if not path.exists(): - return - else: - logger.info("path '%s' exist" % path) - logger.info("\n" + "\n -".join([f for f in os.listdir(path)])) - - show_me_the_folders(ctx.obj["env_folder_path"]) - show_me_the_folders(ctx.obj["env_folder_path"] / Path("scripts")) - show_me_the_folders( - ctx.obj["env_folder_path"] - / Path("scripts") - / Path("step_01__update__14") - ) - logger.error("========================") - - raise Exception("%s doesn't exist" % container_path) - docker_result = container.exec_run(docker_command) - if docker_result.exit_code != 0: - raise Exception( - "The script '%s' failed on database %s.\n" - "- Exit Code : %d\n" - "- Output: %s" - % ( - relative_path, - database, - docker_result.exit_code, - docker_result.output, - ) - ) + command = ( + "psql --username=odoo --dbname={database} --file {file_path}" + ).format(database=database, file_path=container_path) + logger.info( + "Executing the script '%s' in postgres container" + " on database %s" % (relative_path, database) + ) + exec_container(container, command) def execute_sql_request(ctx, request, database="postgres"): container = get_postgres_container(ctx) - docker_command = ( + command = ( "psql" " --username=odoo" " --dbname={database}" @@ -140,12 +106,8 @@ def execute_sql_request(ctx, request, database="postgres"): "Executing the following command in postgres container" " on database %s \n %s" % (database, request) ) - docker_result = container.exec_run(docker_command) - if docker_result.exit_code != 0: - raise Exception( - "Request %s failed on database %s. Exit Code : %d" - % (request, database, docker_result.exit_code) - ) + docker_result = exec_container(container, command) + lines = docker_result.output.decode("utf-8").split("\n") result = [] for line in lines: From a964efd0294c8ae51b77c77f633fcf6547151389 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Wed, 29 Jun 2022 22:12:29 +0200 Subject: [PATCH 31/39] wip --- odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 | 9 ++------- tests/cli_01_init_test.py | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 index 2c9baef..9f57073 100644 --- a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 +++ b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 @@ -11,10 +11,5 @@ RUN apt-get update || true \ # 3. Install extra Python librairies RUN \ - /odoo/bin/pip install --no-cache-dir -r /odoo_requirements.txt -f {{ prebuild_wheel_url }} \ - && /odoo/bin/pip install --no-cache-dir -r /python_requirements.txt - -# RUN \ -# /odoo/bin/pip install --no-cache-dir -r /odoo/src/odoo/requirements.txt -f {{ prebuild_wheel_url }} \ -# && /odoo/bin/pip install --no-cache-dir -r /python_requirements.txt -f {{ prebuild_wheel_url }} \ -# && /odoo/bin/pip install -e /odoo/src/odoo + pip install --no-cache-dir -r /odoo_requirements.txt -f {{ prebuild_wheel_url }} \ + && pip install --no-cache-dir -r /python_requirements.txt diff --git a/tests/cli_01_init_test.py b/tests/cli_01_init_test.py index 01e1b0c..07df673 100644 --- a/tests/cli_01_init_test.py +++ b/tests/cli_01_init_test.py @@ -13,7 +13,7 @@ def test_cli_init(): [ "--log-level=DEBUG", "init", - "--project-name=test-cli", + "--project-name=test-cli-2", "--initial-version=14.0", "--final-version=15.0", "--extra-repository=" From acb02fd9cca203accf924537cfb379c3328d31e9 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Fri, 1 Jul 2022 13:38:59 +0200 Subject: [PATCH 32/39] wip de chez wip --- odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 | 5 ++--- tests/data/output_expected/config.yml | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 index 9f57073..e673b17 100644 --- a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 +++ b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 @@ -10,6 +10,5 @@ RUN apt-get update || true \ && rm -rf /var/lib/apt/lists/* # 3. Install extra Python librairies -RUN \ - pip install --no-cache-dir -r /odoo_requirements.txt -f {{ prebuild_wheel_url }} \ - && pip install --no-cache-dir -r /python_requirements.txt +RUN /odoo/bin/pip install --no-cache-dir -r /odoo_requirements.txt -f {{ prebuild_wheel_url }} +RUN odoo/bin/pip install --no-cache-dir -r /python_requirements.txt diff --git a/tests/data/output_expected/config.yml b/tests/data/output_expected/config.yml index a553e4a..a5f0429 100644 --- a/tests/data/output_expected/config.yml +++ b/tests/data/output_expected/config.yml @@ -1,8 +1,8 @@ -project_name: test-cli +project_name: test-cli-2 postgres_image_name: postgres:13 -postgres_container_name: test-cli-container-postgres -postgres_volume_name: test-cli-volume-postgres +postgres_container_name: test-cli-2-container-postgres +postgres_volume_name: test-cli-2-volume-postgres odoo_host_xmlrpc_port: 9069 odoo_default_country_code: FR From 590f92a1c956e63e05ba7d22b03477c249f44abd Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Mon, 11 Jul 2022 00:15:56 +0200 Subject: [PATCH 33/39] [REM] fucking inadapted docker images --- odoo_openupgrade_wizard/tools/tools_odoo.py | 158 ++++++++++---------- 1 file changed, 78 insertions(+), 80 deletions(-) diff --git a/odoo_openupgrade_wizard/tools/tools_odoo.py b/odoo_openupgrade_wizard/tools/tools_odoo.py index 381cb5b..d66be4e 100644 --- a/odoo_openupgrade_wizard/tools/tools_odoo.py +++ b/odoo_openupgrade_wizard/tools/tools_odoo.py @@ -5,7 +5,7 @@ import sys import traceback from pathlib import Path -import docker +# import docker import yaml from loguru import logger @@ -21,10 +21,7 @@ from odoo_openupgrade_wizard.tools.tools_docker import ( run_container, ) from odoo_openupgrade_wizard.tools.tools_postgres import get_postgres_container -from odoo_openupgrade_wizard.tools.tools_system import ( - get_local_user_id, - get_script_folder, -) +from odoo_openupgrade_wizard.tools.tools_system import get_script_folder def get_odoo_addons_path( @@ -91,7 +88,44 @@ def generate_odoo_command( stop_after_init: bool = False, shell: bool = False, ) -> str: + + odoo_env_path = get_odoo_env_path(ctx, migration_step["version"]) + + # Compute 'server_wide_modules' + # For that purpose, read the custom odoo.cfg file + # to know if server_wide_modules is defined + custom_odoo_config_file = odoo_env_path / "odoo.cfg" + parser = configparser.RawConfigParser() + parser.read(custom_odoo_config_file) + server_wide_modules = parser.get( + "options", "server_wide_modules", fallback=[] + ) + server_wide_modules += get_server_wide_modules_upgrade( + migration_step, execution_context + ) + + # compute 'addons_path' + addons_path = ",".join( + [ + str(x) + for x in get_odoo_addons_path( + ctx, Path("/odoo_env"), migration_step, execution_context + ) + ] + ) + + # compute 'log_file' + log_file_name = "{}____{}.log".format( + ctx.obj["log_prefix"], migration_step["complete_name"] + ) + log_file_docker_path = "/env/log/%s" % log_file_name + database_cmd = database and "--database %s" % database or "" + load_cmd = ( + server_wide_modules + and "--load %s" % ",".join(server_wide_modules) + or "" + ) update_cmd = update and "--update %s" % update or "" init_cmd = init and "--init %s" % init or "" stop_after_init_cmd = stop_after_init and "--stop-after-init" or "" @@ -102,12 +136,21 @@ def generate_odoo_command( / Path(get_odoo_folder(migration_step, execution_context)) / Path(get_odoo_run_command(migration_step)) ) + result = ( f" {command}" f" {shell_cmd}" - # f" --config /etc/odoo.cfg" - f" --data-dir /env/filestore/" + # f" --config=/etc/odoo.cfg" + f" --data-dir=/env/filestore/" + f" --addons-path={addons_path}" + f" --logfile={log_file_docker_path}" + f" --db_host=db" + f" --db_port=5432" + f" --db_user=odoo" + f" --db_password=odoo" + f" --workers=0" f" {demo_cmd}" + f" {load_cmd}" f" {database_cmd}" f" {update_cmd}" f" {init_cmd}" @@ -179,40 +222,11 @@ def run_container_odoo( database: str = False, alternative_xml_rpc_port: int = False, execution_context: str = False, - log_file_suffix: str = "", links: dict = {}, ): env_path = ctx.obj["env_folder_path"] odoo_env_path = get_odoo_env_path(ctx, migration_step["version"]) - # Compute 'server_wide_modules' - # For that purpose, read the custom odoo.cfg file - # to know if server_wide_modules is defined - custom_odoo_config_file = odoo_env_path / "odoo.cfg" - parser = configparser.RawConfigParser() - parser.read(custom_odoo_config_file) - server_wide_modules = parser.get( - "options", "server_wide_modules", fallback=[] - ) - server_wide_modules += get_server_wide_modules_upgrade( - migration_step, execution_context - ) - - # compute 'addons_path' - addons_path = ",".join( - [ - str(x) - for x in get_odoo_addons_path( - ctx, Path("/odoo_env"), migration_step, execution_context - ) - ] - ) - - # compute 'log_file' - log_file_name = "{}____{}{}.log".format( - ctx.obj["log_prefix"], migration_step["complete_name"], log_file_suffix - ) - log_file_docker_path = "/env/log/%s" % log_file_name host_xmlrpc_port = ( alternative_xml_rpc_port and alternative_xml_rpc_port @@ -221,50 +235,34 @@ def run_container_odoo( links.update({ctx.obj["config"]["postgres_container_name"]: "db"}) - environments = { - "DB_HOST": "db", - "DB_PORT": 5432, - "DB_USER": "odoo", - "DB_PASSWORD": "odoo", - "DB_NAME": database, - "LOGFILE": log_file_docker_path, - "ADDONS_PATH": addons_path, - "WORKERS": 0, - "LOCAL_USER_ID": get_local_user_id(), - } - # TODO, handle custom config.cfg file - if server_wide_modules: - environments["SERVER_WIDE_MODULES"] = ",".join(server_wide_modules) - - try: - return run_container( - get_docker_image_tag(ctx, migration_step["version"]), - get_docker_container_name(ctx, migration_step), - command=command, - ports={ - host_xmlrpc_port: 8069, - }, - volumes={ - env_path: "/env/", - odoo_env_path: "/odoo_env/", - }, - environments=environments, - links=links, - detach=detached_container, - auto_remove=True, - ) - except docker.errors.ContainerError as exception: - host_log_file_path = ctx.obj["log_folder_path"] / log_file_name - if host_log_file_path.exists(): - with open(host_log_file_path) as _log_file: - logger.debug("*" * 50) - logger.debug("*" * 50) - logger.debug("*" * 50) - logger.debug(_log_file.read()) - logger.debug("*" * 50) - logger.debug("*" * 50) - logger.debug("*" * 50) - raise exception + # try: + return run_container( + get_docker_image_tag(ctx, migration_step["version"]), + get_docker_container_name(ctx, migration_step), + command=command, + ports={ + host_xmlrpc_port: 8069, + }, + volumes={ + env_path: "/env/", + odoo_env_path: "/odoo_env/", + }, + links=links, + detach=detached_container, + auto_remove=True, + ) + # except docker.errors.ContainerError as exception: + # host_log_file_path = ctx.obj["log_folder_path"] / log_file_name + # if host_log_file_path.exists(): + # with open(host_log_file_path) as _log_file: + # logger.debug("*" * 50) + # logger.debug("*" * 50) + # logger.debug("*" * 50) + # logger.debug(_log_file.read()) + # logger.debug("*" * 50) + # logger.debug("*" * 50) + # logger.debug("*" * 50) + # raise exception def kill_odoo(ctx, migration_step: dict): From be74ce5bab3adc8a8e39c12762051e20cd8a6d33 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Mon, 11 Jul 2022 13:43:55 +0200 Subject: [PATCH 34/39] [REF] Set correct dockerfile, based on odoo/debian ones. (for 12 to 15) --- odoo_openupgrade_wizard/cli/cli_init.py | 21 ++---- .../configuration_version_dependant.py | 49 +------------ .../templates/odoo/12.0/Dockerfile | 69 ++++++++++++++++++ .../templates/odoo/13.0/Dockerfile | 73 +++++++++++++++++++ .../templates/odoo/14.0/Dockerfile | 72 ++++++++++++++++++ .../templates/odoo/15.0/Dockerfile | 70 ++++++++++++++++++ .../templates/odoo/Dockerfile.j2 | 14 ---- ...xt.j2 => extra_debian_requirements.txt.j2} | 0 ...xt.j2 => extra_python_requirements.txt.j2} | 4 - odoo_openupgrade_wizard/tools/tools_system.py | 11 ++- 10 files changed, 302 insertions(+), 81 deletions(-) create mode 100644 odoo_openupgrade_wizard/templates/odoo/12.0/Dockerfile create mode 100644 odoo_openupgrade_wizard/templates/odoo/13.0/Dockerfile create mode 100644 odoo_openupgrade_wizard/templates/odoo/14.0/Dockerfile create mode 100644 odoo_openupgrade_wizard/templates/odoo/15.0/Dockerfile delete mode 100644 odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 rename odoo_openupgrade_wizard/templates/odoo/{debian_requirements.txt.j2 => extra_debian_requirements.txt.j2} (100%) rename odoo_openupgrade_wizard/templates/odoo/{python_requirements.txt.j2 => extra_python_requirements.txt.j2} (70%) diff --git a/odoo_openupgrade_wizard/cli/cli_init.py b/odoo_openupgrade_wizard/cli/cli_init.py index 382aba8..d1f2f4a 100644 --- a/odoo_openupgrade_wizard/cli/cli_init.py +++ b/odoo_openupgrade_wizard/cli/cli_init.py @@ -3,7 +3,6 @@ from pathlib import Path import click from odoo_openupgrade_wizard.configuration_version_dependant import ( - get_odoo_version_template_value, get_odoo_versions, get_version_options, ) @@ -139,14 +138,14 @@ def init( # Create python requirements file ensure_file_exists_from_template( - path_version / Path("python_requirements.txt"), - "odoo/python_requirements.txt.j2", + path_version / Path("extra_python_requirements.txt"), + "odoo/extra_python_requirements.txt.j2", ) # Create debian requirements file ensure_file_exists_from_template( - path_version / Path("debian_requirements.txt"), - "odoo/debian_requirements.txt.j2", + path_version / Path("extra_debian_requirements.txt"), + "odoo/extra_debian_requirements.txt.j2", ) # Create odoo config file @@ -166,17 +165,7 @@ def init( # Create Dockerfile file ensure_file_exists_from_template( path_version / Path("Dockerfile"), - "odoo/Dockerfile.j2", - odoo_version=odoo_version, - python_major_version=get_odoo_version_template_value( - odoo_version, "python_major_version" - ), - python_minor_version_short=get_odoo_version_template_value( - odoo_version, "python_minor_version_short" - ), - prebuild_wheel_url=get_odoo_version_template_value( - odoo_version, "prebuild_wheel_url" - ), + f"odoo/{odoo_version}/Dockerfile", ) # Create 'src' folder that will contain all the odoo code diff --git a/odoo_openupgrade_wizard/configuration_version_dependant.py b/odoo_openupgrade_wizard/configuration_version_dependant.py index 48c3579..49ff271 100644 --- a/odoo_openupgrade_wizard/configuration_version_dependant.py +++ b/odoo_openupgrade_wizard/configuration_version_dependant.py @@ -5,74 +5,31 @@ from loguru import logger _ODOO_VERSION_TEMPLATES = [ { "version": 8.0, - "python_major_version": "python2", - "python_minor_version_short": "py27", - "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux1", }, { "version": 9.0, - "python_major_version": "python2", - "python_minor_version_short": "py27", - "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux1", }, { "version": 10.0, - "python_major_version": "python2", - "python_minor_version_short": "py27", - "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux1", }, { "version": 11.0, - "python_major_version": "python3", - "python_minor_version_short": "py36", - "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux2014", }, { "version": 12.0, - "python_major_version": "python3", - # Note: doesn't work with latest available version py37 - "python_minor_version_short": "py36", - "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux2014", }, { - "version": 13.0, # OK. - "python_major_version": "python3", - # Note: doesn't work with latest available version py37 - "python_minor_version_short": "py36", - "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux2014", + "version": 13.0, }, { - "version": 14.0, # OK - "python_major_version": "python3", - "python_minor_version_short": "py39", - "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux2014", + "version": 14.0, }, { - "version": 15.0, # OK - "python_major_version": "python3", - "python_minor_version_short": "py39", - "prebuild_wheel_url": "https://wheelhouse.acsone.eu/manylinux2014", + "version": 15.0, }, ] -def get_odoo_version_template_value(version: float, key: str) -> str: - """Return a value depending on a odoo given version and key. - Possible key: - - python_major_version. (python2, python3) - - python_minor_version_short. (py27, py36, ...) - - prebuild_wheel_url. - """ - version_template = False - for odoo_version_template in _ODOO_VERSION_TEMPLATES: - if odoo_version_template["version"] == version: - version_template = odoo_version_template - break - else: - raise ValueError - return version_template[key] - - def get_version_options(mode: str) -> list: """Get options available for version click argument. Arguments: diff --git a/odoo_openupgrade_wizard/templates/odoo/12.0/Dockerfile b/odoo_openupgrade_wizard/templates/odoo/12.0/Dockerfile new file mode 100644 index 0000000..4eeef05 --- /dev/null +++ b/odoo_openupgrade_wizard/templates/odoo/12.0/Dockerfile @@ -0,0 +1,69 @@ +# : Copy of https://github.com/odoo/odoo/blob/12.0/setup/package.dfdebian + +FROM debian:stretch + +# 1. Official Odoo Dockerfile. +# Removing: postgresql, locales, rsync + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update -qq && \ + apt-get upgrade -qq -y && \ + apt-get install -yq \ + adduser \ + dh-python \ + packaging-dev \ + postgresql-client \ + python3 \ + python3-babel \ + python3-dateutil \ + python3-decorator \ + python3-docutils \ + python3-gevent \ + python3-html2text \ + python3-pil \ + python3-jinja2 \ + python3-libsass \ + python3-lxml \ + python3-mako \ + python3-mock \ + python3-ofxparse \ + python3-passlib \ + python3-psutil \ + python3-psycopg2 \ + python3-pydot \ + python3-pyparsing \ + python3-pypdf2 \ + python3-reportlab \ + python3-requests \ + python3-serial \ + python3-suds \ + python3-tz \ + python3-usb \ + python3-vatnumber \ + python3-vobject \ + python3-werkzeug \ + python3-xlsxwriter \ + && rm -rf /var/lib/apt/lists/* + +# 2. Add 2 python dependency files and 1 debian dependency file + +COPY ./src/odoo/requirements.txt /odoo_python_requirements.txt +COPY extra_python_requirements.txt /extra_python_requirements.txt +COPY extra_debian_requirements.txt /extra_debian_requirements.txt + +# 3. Install Debian packages +RUN apt-get update || true \ + && apt-get install -y\ + # To allow to run pip install + python3-pip\ + # For python-ldap + libldap2-dev ldap-utils libsasl2-dev\ + && xargs apt-get install -y --no-install-recommends Install Python librairies +RUN python3 -m pip install --no-cache-dir -r /odoo_python_requirements.txt +RUN python3 -m pip install --no-cache-dir -r /extra_python_requirements.txt + +RUN useradd --uid 1000 odoo diff --git a/odoo_openupgrade_wizard/templates/odoo/13.0/Dockerfile b/odoo_openupgrade_wizard/templates/odoo/13.0/Dockerfile new file mode 100644 index 0000000..a394752 --- /dev/null +++ b/odoo_openupgrade_wizard/templates/odoo/13.0/Dockerfile @@ -0,0 +1,73 @@ +# : Copy of https://github.com/odoo/odoo/blob/13.0/setup/package.dfdebian + +FROM debian:buster + +# 1. Official Odoo Dockerfile. +# Removing: postgresql, locales, rsync + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update -qq && \ + apt-get upgrade -qq -y && \ + apt-get install -qq -y\ + adduser \ + dh-python \ + packaging-dev \ + postgresql-client \ + python3 \ + python3-babel \ + python3-dateutil \ + python3-decorator \ + python3-docutils \ + python3-gevent \ + python3-pil \ + python3-jinja2 \ + python3-libsass \ + python3-lxml \ + python3-mako \ + python3-mock \ + python3-ofxparse \ + python3-passlib \ + python3-polib \ + python3-psutil \ + python3-psycopg2 \ + python3-pydot \ + python3-pyparsing \ + python3-pypdf2 \ + python3-qrcode \ + python3-renderpm \ + python3-reportlab \ + python3-requests \ + python3-serial \ + python3-tz \ + python3-usb \ + python3-vatnumber \ + python3-vobject \ + python3-werkzeug \ + python3-xlsxwriter \ + python3-zeep \ + && rm -rf /var/lib/apt/lists/* + +# 2. Add 2 python dependency files and 1 debian dependency file + +COPY ./src/odoo/requirements.txt /odoo_python_requirements.txt +COPY extra_python_requirements.txt /extra_python_requirements.txt +COPY extra_debian_requirements.txt /extra_debian_requirements.txt + +# 3. Install Debian packages +RUN apt-get update || true \ + && apt-get install -y\ + # To allow to run pip install + python3-pip\ + # For python-ldap + libldap2-dev ldap-utils libsasl2-dev\ + # For cffi + build-essential libssl-dev libffi-dev python-dev\ + && xargs apt-get install -y --no-install-recommends Install Python librairies +RUN python3 -m pip install --no-cache-dir -r /odoo_python_requirements.txt +RUN python3 -m pip install --no-cache-dir -r /extra_python_requirements.txt + +RUN useradd --uid 1000 odoo diff --git a/odoo_openupgrade_wizard/templates/odoo/14.0/Dockerfile b/odoo_openupgrade_wizard/templates/odoo/14.0/Dockerfile new file mode 100644 index 0000000..a0056a2 --- /dev/null +++ b/odoo_openupgrade_wizard/templates/odoo/14.0/Dockerfile @@ -0,0 +1,72 @@ +# : Copy of https://github.com/odoo/odoo/blob/14.0/setup/package.dfdebian + +FROM debian:buster + +# 1. Official Odoo Dockerfile. +# Removing: postgresql, locales, rsync + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update -qq && \ + apt-get upgrade -qq -y && \ + apt-get install -qq -y\ + adduser \ + dh-python \ + packaging-dev \ + postgresql-client \ + python3 \ + python3-babel \ + python3-dateutil \ + python3-decorator \ + python3-docutils \ + python3-gevent \ + python3-pil \ + python3-jinja2 \ + python3-libsass \ + python3-lxml \ + python3-mako \ + python3-ofxparse \ + python3-passlib \ + python3-polib \ + python3-psutil \ + python3-psycopg2 \ + python3-pydot \ + python3-pypdf2 \ + python3-qrcode \ + python3-renderpm \ + python3-reportlab \ + python3-requests \ + python3-serial \ + python3-setuptools \ + python3-stdnum \ + python3-tz \ + python3-usb \ + python3-vobject \ + python3-werkzeug \ + python3-xlsxwriter \ + python3-zeep \ + && rm -rf /var/lib/apt/lists/* + +# 2. Add 2 python dependency files and 1 debian dependency file + +COPY ./src/odoo/requirements.txt /odoo_python_requirements.txt +COPY extra_python_requirements.txt /extra_python_requirements.txt +COPY extra_debian_requirements.txt /extra_debian_requirements.txt + +# 3. Install Debian packages +RUN apt-get update || true \ + && apt-get install -y\ + # To allow to run pip install + python3-pip\ + # For python-ldap + libldap2-dev ldap-utils libsasl2-dev\ + # For cffi + build-essential libssl-dev libffi-dev python-dev\ + && xargs apt-get install -y --no-install-recommends Install Python librairies +RUN python3 -m pip install --no-cache-dir -r /odoo_python_requirements.txt +RUN python3 -m pip install --no-cache-dir -r /extra_python_requirements.txt + +RUN useradd --uid 1000 odoo diff --git a/odoo_openupgrade_wizard/templates/odoo/15.0/Dockerfile b/odoo_openupgrade_wizard/templates/odoo/15.0/Dockerfile new file mode 100644 index 0000000..c0751d4 --- /dev/null +++ b/odoo_openupgrade_wizard/templates/odoo/15.0/Dockerfile @@ -0,0 +1,70 @@ +# : Copy of https://github.com/odoo/odoo/blob/15.0/setup/package.dfdebian + +FROM debian:bullseye + +# 1. Official Odoo Dockerfile. +# Removing: postgresql, locales, rsync + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update -qq && \ + apt-get upgrade -qq -y && \ + apt-get install -qq -y\ + adduser \ + dh-python \ + packaging-dev \ + postgresql-client \ + python3 \ + python3-babel \ + python3-dateutil \ + python3-decorator \ + python3-docutils \ + python3-gevent \ + python3-pil \ + python3-jinja2 \ + python3-libsass \ + python3-lxml \ + python3-ofxparse \ + python3-passlib \ + python3-polib \ + python3-psutil \ + python3-psycopg2 \ + python3-pydot \ + python3-openssl \ + python3-pypdf2 \ + python3-qrcode \ + python3-renderpm \ + python3-reportlab \ + python3-requests \ + python3-serial \ + python3-setuptools \ + python3-stdnum \ + python3-tz \ + python3-usb \ + python3-vobject \ + python3-werkzeug \ + python3-xlsxwriter \ + python3-zeep \ + && rm -rf /var/lib/apt/lists/* + +# 2. Add 2 python dependency files and 1 debian dependency file + +COPY ./src/odoo/requirements.txt /odoo_python_requirements.txt +COPY extra_python_requirements.txt /extra_python_requirements.txt +COPY extra_debian_requirements.txt /extra_debian_requirements.txt + +# 3. Install Debian packages +RUN apt-get update || true \ + && apt-get install -y\ + # To allow to run pip install + python3-pip\ + # For python-ldap + libldap2-dev ldap-utils libsasl2-dev\ + && xargs apt-get install -y --no-install-recommends Install Python librairies +RUN python3 -m pip install --no-cache-dir -r /odoo_python_requirements.txt +RUN python3 -m pip install --no-cache-dir -r /extra_python_requirements.txt + +RUN useradd --uid 1000 odoo diff --git a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 b/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 deleted file mode 100644 index e673b17..0000000 --- a/odoo_openupgrade_wizard/templates/odoo/Dockerfile.j2 +++ /dev/null @@ -1,14 +0,0 @@ -FROM ghcr.io/acsone/odoo-bedrock:{{ odoo_version }}-{{python_minor_version_short}}-latest - -COPY ./src/odoo/requirements.txt /odoo_requirements.txt -COPY python_requirements.txt /python_requirements.txt -COPY debian_requirements.txt /debian_requirements.txt - -# 2. Install extra debian packages -RUN apt-get update || true \ - && xargs apt-get install -y --no-install-recommends Date: Mon, 11 Jul 2022 13:57:13 +0200 Subject: [PATCH 35/39] [IMP] description in config.yml file --- odoo_openupgrade_wizard/cli/cli_init.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/odoo_openupgrade_wizard/cli/cli_init.py b/odoo_openupgrade_wizard/cli/cli_init.py index d1f2f4a..0e44cc6 100644 --- a/odoo_openupgrade_wizard/cli/cli_init.py +++ b/odoo_openupgrade_wizard/cli/cli_init.py @@ -69,16 +69,16 @@ def init( # Compute Migration Steps - # Create initial first step + # Create initial Regular step steps = [ { "name": 1, "execution_context": "regular", "version": odoo_versions[0], - "complete_name": "step_01__update__%s" % (odoo_versions[0]), + "complete_name": "step_01__regular__%s" % (odoo_versions[0]), } ] - # Add all upgrade steps + # Add all Openupgrade steps step_nbr = 2 for odoo_version in odoo_versions[1:]: steps.append( @@ -86,20 +86,20 @@ def init( "name": step_nbr, "execution_context": "openupgrade", "version": odoo_version, - "complete_name": "step_%s__upgrade__%s" + "complete_name": "step_%s__openupgrade__%s" % (str(step_nbr).rjust(2, "0"), odoo_version), } ) step_nbr += 1 - # add final update step + # add final Regular step if len(odoo_versions) > 1: steps.append( { "name": step_nbr, "execution_context": "regular", "version": odoo_versions[-1], - "complete_name": "step_%s__update__%s" + "complete_name": "step_%s__regular__%s" % (str(step_nbr).rjust(2, "0"), odoo_versions[-1]), } ) From c924103fa0ab3b3039d048343382e7920aba2351 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Mon, 11 Jul 2022 15:15:22 +0200 Subject: [PATCH 36/39] [IMP] add local_user_id in build call. --- odoo_openupgrade_wizard/cli/cli_docker_build.py | 4 +++- odoo_openupgrade_wizard/cli/cli_init.py | 2 +- odoo_openupgrade_wizard/templates/odoo/12.0/Dockerfile | 7 ++++++- odoo_openupgrade_wizard/templates/odoo/13.0/Dockerfile | 7 ++++++- odoo_openupgrade_wizard/templates/odoo/14.0/Dockerfile | 7 ++++++- odoo_openupgrade_wizard/templates/odoo/15.0/Dockerfile | 7 ++++++- odoo_openupgrade_wizard/tools/tools_docker.py | 8 ++++++-- 7 files changed, 34 insertions(+), 8 deletions(-) diff --git a/odoo_openupgrade_wizard/cli/cli_docker_build.py b/odoo_openupgrade_wizard/cli/cli_docker_build.py index 1008216..611b4bb 100644 --- a/odoo_openupgrade_wizard/cli/cli_docker_build.py +++ b/odoo_openupgrade_wizard/cli/cli_docker_build.py @@ -10,13 +10,14 @@ from odoo_openupgrade_wizard.tools.tools_odoo import ( get_docker_image_tag, get_odoo_env_path, ) +from odoo_openupgrade_wizard.tools.tools_system import get_local_user_id @click.command() @versions_options @click.pass_context def docker_build(ctx, versions): - """Build Odoo Docker Images. (One image per version)""" + """Build Odoo Docker Images and pull Postgres image""" # Pull DB image pull_image(ctx.obj["config"]["postgres_image_name"]) @@ -30,5 +31,6 @@ def docker_build(ctx, versions): image = build_image( get_odoo_env_path(ctx, odoo_version), get_docker_image_tag(ctx, odoo_version), + {"LOCAL_USER_ID": str(get_local_user_id())}, ) logger.info("Docker Image build. '%s'" % image[0].tags[0]) diff --git a/odoo_openupgrade_wizard/cli/cli_init.py b/odoo_openupgrade_wizard/cli/cli_init.py index 0e44cc6..7884ecf 100644 --- a/odoo_openupgrade_wizard/cli/cli_init.py +++ b/odoo_openupgrade_wizard/cli/cli_init.py @@ -47,7 +47,7 @@ from odoo_openupgrade_wizard.tools.tools_system import ( def init( ctx, project_name, initial_version, final_version, extra_repository_list ): - """Initialize OpenUpgrade Wizard Environment based on the initial and + """Initialize OOW Environment based on the initial and the final version of Odoo you want to migrate. """ diff --git a/odoo_openupgrade_wizard/templates/odoo/12.0/Dockerfile b/odoo_openupgrade_wizard/templates/odoo/12.0/Dockerfile index 4eeef05..09bbdcb 100644 --- a/odoo_openupgrade_wizard/templates/odoo/12.0/Dockerfile +++ b/odoo_openupgrade_wizard/templates/odoo/12.0/Dockerfile @@ -66,4 +66,9 @@ RUN apt-get update || true \ RUN python3 -m pip install --no-cache-dir -r /odoo_python_requirements.txt RUN python3 -m pip install --no-cache-dir -r /extra_python_requirements.txt -RUN useradd --uid 1000 odoo +# 5. Get local user id and set it to the odoo user +ARG LOCAL_USER_ID + +RUN useradd --uid $LOCAL_USER_ID odoo + +USER odoo diff --git a/odoo_openupgrade_wizard/templates/odoo/13.0/Dockerfile b/odoo_openupgrade_wizard/templates/odoo/13.0/Dockerfile index a394752..506ca1b 100644 --- a/odoo_openupgrade_wizard/templates/odoo/13.0/Dockerfile +++ b/odoo_openupgrade_wizard/templates/odoo/13.0/Dockerfile @@ -70,4 +70,9 @@ RUN apt-get update || true \ RUN python3 -m pip install --no-cache-dir -r /odoo_python_requirements.txt RUN python3 -m pip install --no-cache-dir -r /extra_python_requirements.txt -RUN useradd --uid 1000 odoo +# 5. Get local user id and set it to the odoo user +ARG LOCAL_USER_ID + +RUN useradd --uid $LOCAL_USER_ID odoo + +USER odoo diff --git a/odoo_openupgrade_wizard/templates/odoo/14.0/Dockerfile b/odoo_openupgrade_wizard/templates/odoo/14.0/Dockerfile index a0056a2..c5729d2 100644 --- a/odoo_openupgrade_wizard/templates/odoo/14.0/Dockerfile +++ b/odoo_openupgrade_wizard/templates/odoo/14.0/Dockerfile @@ -69,4 +69,9 @@ RUN apt-get update || true \ RUN python3 -m pip install --no-cache-dir -r /odoo_python_requirements.txt RUN python3 -m pip install --no-cache-dir -r /extra_python_requirements.txt -RUN useradd --uid 1000 odoo +# 5. Get local user id and set it to the odoo user +ARG LOCAL_USER_ID + +RUN useradd --uid $LOCAL_USER_ID odoo + +USER odoo diff --git a/odoo_openupgrade_wizard/templates/odoo/15.0/Dockerfile b/odoo_openupgrade_wizard/templates/odoo/15.0/Dockerfile index c0751d4..879024b 100644 --- a/odoo_openupgrade_wizard/templates/odoo/15.0/Dockerfile +++ b/odoo_openupgrade_wizard/templates/odoo/15.0/Dockerfile @@ -67,4 +67,9 @@ RUN apt-get update || true \ RUN python3 -m pip install --no-cache-dir -r /odoo_python_requirements.txt RUN python3 -m pip install --no-cache-dir -r /extra_python_requirements.txt -RUN useradd --uid 1000 odoo +# 5. Get local user id and set it to the odoo user +ARG LOCAL_USER_ID + +RUN useradd --uid $LOCAL_USER_ID odoo + +USER odoo diff --git a/odoo_openupgrade_wizard/tools/tools_docker.py b/odoo_openupgrade_wizard/tools/tools_docker.py index c6212b8..6820cdd 100644 --- a/odoo_openupgrade_wizard/tools/tools_docker.py +++ b/odoo_openupgrade_wizard/tools/tools_docker.py @@ -11,17 +11,21 @@ def pull_image(image_name): client.images.pull(image_name) -def build_image(path, tag): +def build_image(path, tag, buildargs={}): logger.debug( "Building image named based on %s/Dockerfile." " This can take a big while ..." % (path) ) debug_docker_command = "docker build %s --tag %s" % (path, tag) - logger.debug("DOCKER COMMAND:\n %s" % debug_docker_command) + for arg_name, arg_value in buildargs.items(): + debug_docker_command += f"\\\n --build-arg {arg_name}={arg_value}" + + logger.debug("DOCKER COMMAND:\n\n%s\n" % debug_docker_command) docker_client = get_docker_client() image = docker_client.images.build( path=str(path), tag=tag, + buildargs=buildargs, ) logger.debug("Image build.") return image From e6e05c6ab743569fe9ccad464c14c2e527fd6e6d Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Mon, 11 Jul 2022 15:50:41 +0200 Subject: [PATCH 37/39] [FIX] CI --- tests/cli_01_init_test.py | 2 +- tests/data/output_expected/config.yml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/cli_01_init_test.py b/tests/cli_01_init_test.py index 07df673..01e1b0c 100644 --- a/tests/cli_01_init_test.py +++ b/tests/cli_01_init_test.py @@ -13,7 +13,7 @@ def test_cli_init(): [ "--log-level=DEBUG", "init", - "--project-name=test-cli-2", + "--project-name=test-cli", "--initial-version=14.0", "--final-version=15.0", "--extra-repository=" diff --git a/tests/data/output_expected/config.yml b/tests/data/output_expected/config.yml index a5f0429..d41e83f 100644 --- a/tests/data/output_expected/config.yml +++ b/tests/data/output_expected/config.yml @@ -1,8 +1,8 @@ -project_name: test-cli-2 +project_name: test-cli postgres_image_name: postgres:13 -postgres_container_name: test-cli-2-container-postgres -postgres_volume_name: test-cli-2-volume-postgres +postgres_container_name: test-cli-container-postgres +postgres_volume_name: test-cli-volume-postgres odoo_host_xmlrpc_port: 9069 odoo_default_country_code: FR @@ -24,17 +24,17 @@ migration_steps: - name: 1 version: 14.0 execution_context: regular - complete_name: step_01__update__14.0 + complete_name: step_01__regular__14.0 - name: 2 version: 15.0 execution_context: openupgrade - complete_name: step_02__upgrade__15.0 + complete_name: step_02__openupgrade__15.0 - name: 3 version: 15.0 execution_context: regular - complete_name: step_03__update__15.0 + complete_name: step_03__regular__15.0 workload_settings: From 72cbc90b613709b149fa430323f853fce41b5651 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Mon, 11 Jul 2022 16:08:52 +0200 Subject: [PATCH 38/39] [FIX] CI --- tests/cli_06_execute_script_sql_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli_06_execute_script_sql_test.py b/tests/cli_06_execute_script_sql_test.py index 81fd69e..524f780 100644 --- a/tests/cli_06_execute_script_sql_test.py +++ b/tests/cli_06_execute_script_sql_test.py @@ -23,7 +23,7 @@ def test_cli_execute_script_sql(): ).absolute() # Deploy SQL Script - destination_path = Path("scripts/step_01__update__14.0") + destination_path = Path("scripts/step_01__regular__14.0") cp([extra_script_path, destination_path]) # Reset database From d7a576890af90ecf76f2bb567ff0d1f8669092b2 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Mon, 11 Jul 2022 16:46:08 +0200 Subject: [PATCH 39/39] [IMP] prune postgres container --- odoo_openupgrade_wizard/tools/tools_docker.py | 16 +++++++++++----- tests/cli_03_docker_build_test.py | 17 +++++++++++++++-- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/odoo_openupgrade_wizard/tools/tools_docker.py b/odoo_openupgrade_wizard/tools/tools_docker.py index 6820cdd..6ed53dd 100644 --- a/odoo_openupgrade_wizard/tools/tools_docker.py +++ b/odoo_openupgrade_wizard/tools/tools_docker.py @@ -119,8 +119,14 @@ def kill_container(container_name): filters={"name": container_name}, ) for container in containers: - logger.debug( - "Stop container %s, based on image '%s'." - % (container.name, ",".join(container.image.tags)) - ) - container.stop() + if container.status != "exited": + logger.debug( + "Stop container %s, based on image '%s'." + % (container.name, ",".join(container.image.tags)) + ) + container.stop() + + # TODO, we should here filter by name + # but filters={"name": container_name} + # doesn't work... + client.containers.prune() diff --git a/tests/cli_03_docker_build_test.py b/tests/cli_03_docker_build_test.py index a2b0be3..14a5009 100644 --- a/tests/cli_03_docker_build_test.py +++ b/tests/cli_03_docker_build_test.py @@ -1,10 +1,23 @@ -from odoo_openupgrade_wizard.tools.tools_docker import get_docker_client +from odoo_openupgrade_wizard.tools.tools_docker import ( + get_docker_client, + kill_container, +) -from . import cli_runner_invoke, move_to_test_folder +from . import ( + build_ctx_from_config_file, + cli_runner_invoke, + move_to_test_folder, +) def test_cli_docker_build(): move_to_test_folder() + ctx = build_ctx_from_config_file() + + # Drop postgresql container if exist + # (we ensure that the postgres container is removed to + # be sure that the call (environment, etc...) is correct now.) + kill_container(ctx.obj["config"]["postgres_container_name"]) cli_runner_invoke( [