From bdc9b606ecd54ac83c4b83597b2b543edd94a361 Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Thu, 14 Jul 2022 23:42:53 +0200 Subject: [PATCH] [IMP] Add postgresql custom configuration ; [ADD] extra setting into odoo_default_company key. (in config.yml file) ; [ADD] odoo_rpc_timeout extra settings ; [FIX] restore call to custom odoo.cfg file --- DEVELOP.md | 129 +++++++++++------- INTERNAL_NOTES.md | 16 ++- odoo_openupgrade_wizard/cli/cli.py | 6 - odoo_openupgrade_wizard/cli/cli_init.py | 7 +- .../cli/cli_install_from_csv.py | 24 ++-- odoo_openupgrade_wizard/cli/cli_run.py | 1 - .../templates/config.yml.j2 | 7 +- .../templates/custom-postgres.conf.j2 | 0 odoo_openupgrade_wizard/tools/tools_odoo.py | 2 +- .../tools/tools_odoo_instance.py | 5 +- .../tools/tools_postgres.py | 12 +- tests/data/output_expected/config.yml | 7 +- 12 files changed, 138 insertions(+), 78 deletions(-) create mode 100644 odoo_openupgrade_wizard/templates/custom-postgres.conf.j2 diff --git a/DEVELOP.md b/DEVELOP.md index 1501f57..d3aab4a 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -1,18 +1,18 @@ -# Tools to understand +# Installation to develop -The library is using many tools. It is recommanded to understand that tools -to contribute to that project. +## Basic installation -* Docker (https://www.docker.com/) -* Dind (Docker In Docker) for running docker in gitlab-ci. (https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker) -* Gitlab CI (https://docs.gitlab.com/ee/ci/quick_start/index.html) -* openupgrade project (https://github.com/oca/openupgrade) and related openupgradelib (https://github.com/oca/openupgradelib) -* poetry (https://python-poetry.org/) -* odoorpc (https://github.com/OCA/odoorpc) -* git-aggregator (https://github.com/acsone/git-aggregator) +``` +git clone https://gitlab.com/odoo-openupgrade-wizard/odoo-openupgrade-wizard/ +cd odoo-openupgrade-wizard +virtualenv env --python=python3 +. ./env/bin/activate +poetry install +``` +``odoo-openupgrade-wizard`` commands are now available in your virutalenv. -# Extra Developper Requirements +## Advanced installation If you want to use this library without installing anything in your system, execute the following steps, otherwise, go to 'Installation' part. @@ -37,29 +37,9 @@ pipx install virtualenv pipx install poetry ``` -# Installation - -``` -git clone https://gitlab.com/odoo-openupgrade-wizard/odoo-openupgrade-wizard/ -cd odoo-openupgrade-wizard -virtualenv env --python=python3 -. ./env/bin/activate -poetry install -``` - -``odoo-openupgrade-wizard`` commands are now available in your virutalenv. - -# Add python dependencies - -If you add new dependencies, you have to: - -- add the reference in the file ``pyproject.toml`` - -- run the following command in your virtualenv : ``poetry update`` - # Run tests -## Via pytest +## Via pytest (simple) This will run tests only for the current ``python3.X`` version. @@ -68,17 +48,7 @@ This will run tests only for the current ``python3.X`` version. poetry run pytest --cov odoo_openupgrade_wizard --verbosity=2 --exitfirst ``` -Debug Test -``` -poetry run pytest --verbosity=2 --exitfirst --cov odoo_openupgrade_wizard\ - tests/cli_01_init_test.py\ - tests/cli_02_get_code_test.py\ - tests/cli_03_docker_build_test.py\ - tests/cli_04_run_test.py\ - tests/cli_06_execute_script_sql_test.py -``` - -## Via Tox +## Via Tox (advanced) This will run tests for all the python versions put in the ``tox.ini`` folder. @@ -108,14 +78,50 @@ pipx install gitlabci-local gitlabci-local ``` -# Publish on pyPI +# Debugging + +Some docker command could help you when using / developping this tools. + +**Enter the postgres container** + +docker exec -it POSTGRES_CONTAINER_NAME /bin/bash + +# Contribute + +## Add python dependencies + +If you add new dependencies, you have to: + +- add the reference in the file ``pyproject.toml`` + +- run the following command in your virtualenv : ``poetry update`` + +## Publish on pyPI ``` - +poetry publish ``` +# Understanding the library -# Réferences +## Tools to understand + +The library is using many tools. It is recommanded to understand that tools +to contribute to that project: + +* Docker (https://www.docker.com/) +* Gitlab CI (https://docs.gitlab.com/ee/ci/quick_start/index.html) +* openupgrade project (https://github.com/oca/openupgrade) and related openupgradelib (https://github.com/oca/openupgradelib) +* poetry (https://python-poetry.org/) +* odoorpc (https://github.com/OCA/odoorpc) +* git-aggregator (https://github.com/acsone/git-aggregator) + +Also this project is inspired by the following tools: + +* click-odoo-contrib (https://github.com/acsone/click-odoo-contrib) + + +## Réferences - how to install gitlab runner locally: @@ -126,3 +132,34 @@ https://docs.gitlab.com/runner/install/linux-manually.html https://blog.stephane-robert.info/post/gitlab-valider-ci-yml/ https://blog.callr.tech/building-docker-images-with-gitlab-ci-best-practices/ + + +## Python version settings depending on the debian version + +This part can help you if you want to change your autogenerated Dockerfiles. + +See (https://github.com/odoo/odoo/blob/ODOO_VERSION/setup/package.dfdebian) + +### debian:jessie (V8) (for Odoo: 9.0, 10.0) +- Ubuntu release : 14.04 trusty, 14.10 utopic, 15.04 vivid, 15.10 wily +- python2.7 and ??? +- First release : 26/04/2015 +- End LTS : June 2020 + +### debian:stretch (V9) (for Odoo: 11.0, 12.0) +- Ubuntu releases : 16.04 xenial, 16.10 yakkety, 17.04 zesty, 17.10 artful +- python2.7 and python3.5 +- First release : 17/06/2017 +- End LTS : June 2022 + +### debian:buster (13.0, 14.0) +- Ubuntu release : 18.04 bionic, 18.10 cosmic, 19.04 disco, 19.10 eoan +- python2.7 and python3.7 +- First release : 06/07/2019 +- End LTS : Undefined. + +## debian:bullseye (15.0) +- Ubuntu release : 20.04 focal, 20.10 groovy, 21.04 hirsute, 21.10 impish +- python3.9 +- First release : 14/07/2021 +- End LTS : Undefined. diff --git a/INTERNAL_NOTES.md b/INTERNAL_NOTES.md index 9a84987..20dcffc 100644 --- a/INTERNAL_NOTES.md +++ b/INTERNAL_NOTES.md @@ -53,5 +53,17 @@ in modules.csv.j2 : -TODO when launching container : -- install odoo : ``pip install -e /odoo/src/odoo`` +## Without postgres optimization +2022-07-13 19:42 +2022-07-13 21:20 + +Duration : 1:37 (107) + +## With postgres optimization + +2022-07-13 21:52 +2022-07-14 23:11 + +duration : 1:19 (79) + +16% diff --git a/odoo_openupgrade_wizard/cli/cli.py b/odoo_openupgrade_wizard/cli/cli.py index 570f9e5..92d58ef 100644 --- a/odoo_openupgrade_wizard/cli/cli.py +++ b/odoo_openupgrade_wizard/cli/cli.py @@ -71,10 +71,6 @@ def main(ctx, env_folder, filestore_folder, log_level): src_folder_path = env_folder_path / Path("./src/") script_folder_path = env_folder_path / Path("./scripts/") log_folder_path = env_folder_path / Path("./log/") - if not filestore_folder: - filestore_folder_path = env_folder_path / Path("./filestore/") - else: - filestore_folder_path = Path(filestore_folder) # ensure log folder exists ensure_folder_exists(log_folder_path, git_ignore_content=True) @@ -95,8 +91,6 @@ def main(ctx, env_folder, filestore_folder, log_level): ctx.obj["script_folder_path"] = script_folder_path ctx.obj["log_folder_path"] = log_folder_path ctx.obj["log_prefix"] = log_prefix - ctx.obj["filestore_folder_path"] = filestore_folder_path - ctx.obj["config_file_path"] = config_file_path ctx.obj["module_file_path"] = module_file_path diff --git a/odoo_openupgrade_wizard/cli/cli_init.py b/odoo_openupgrade_wizard/cli/cli_init.py index 7884ecf..d207cb4 100644 --- a/odoo_openupgrade_wizard/cli/cli_init.py +++ b/odoo_openupgrade_wizard/cli/cli_init.py @@ -107,12 +107,7 @@ def init( # Ensure src folder exists ensure_folder_exists(ctx.obj["src_folder_path"]) - # Ensure filestore folder exists - ensure_folder_exists( - ctx.obj["filestore_folder_path"], git_ignore_content=True - ) - - # Znsure main configuration file exists + # Ensure main configuration file exists ensure_file_exists_from_template( ctx.obj["config_file_path"], "config.yml.j2", diff --git a/odoo_openupgrade_wizard/cli/cli_install_from_csv.py b/odoo_openupgrade_wizard/cli/cli_install_from_csv.py index 3ebdeff..f218e42 100644 --- a/odoo_openupgrade_wizard/cli/cli_install_from_csv.py +++ b/odoo_openupgrade_wizard/cli/cli_install_from_csv.py @@ -37,11 +37,10 @@ def install_from_csv(ctx, database): init="base", ) odoo_instance = OdooInstance(ctx, database) - - default_country_code = ctx.obj["config"].get( - "odoo_default_country_code", False + odoo_default_company = ctx.obj["config"].get( + "odoo_default_company", False ) - if "account" in module_names and default_country_code: + if odoo_default_company: # Then, set correct country to the company of the current user # Otherwise, due to poor design of Odoo, when installing account # the US localization will be installed. @@ -49,22 +48,29 @@ def install_from_csv(ctx, database): countries = odoo_instance.browse_by_search( "res.country", - [("code", "=", default_country_code)], + [("code", "=", odoo_default_company["country_code"])], ) if len(countries) != 1: raise Exception( "Unable to find a country, based on the code %s." " countries found : %s " % ( - default_country_code, + odoo_default_company["country_code"], ", ".join([x.name for x in countries]), ) ) + vals = { + "country_id": countries[0].id, + "currency_id": countries[0].currency_id.id, + "phone": odoo_default_company.get("phone"), + "email": odoo_default_company.get("email"), + } logger.info( - "Configuring country of the main company with #%d - %s" - % (countries[0].id, countries[0].name) + f"Configuring main company with values {vals}" + f" (country {countries[0].name}" ) - odoo_instance.env.user.company_id.country_id = countries[0].id + + odoo_instance.env.user.company_id.write(vals) # Install modules odoo_instance.install_modules(module_names) diff --git a/odoo_openupgrade_wizard/cli/cli_run.py b/odoo_openupgrade_wizard/cli/cli_run.py index 444f61b..6d06dc1 100644 --- a/odoo_openupgrade_wizard/cli/cli_run.py +++ b/odoo_openupgrade_wizard/cli/cli_run.py @@ -37,7 +37,6 @@ from odoo_openupgrade_wizard.tools.tools_postgres import ensure_database ) @click.pass_context def run(ctx, step, database, stop_after_init, init_modules, execution_context): - migration_step = get_migration_step_from_options(ctx, step) ensure_database(ctx, database, state="present") try: diff --git a/odoo_openupgrade_wizard/templates/config.yml.j2 b/odoo_openupgrade_wizard/templates/config.yml.j2 index 509d317..33f9941 100644 --- a/odoo_openupgrade_wizard/templates/config.yml.j2 +++ b/odoo_openupgrade_wizard/templates/config.yml.j2 @@ -1,11 +1,16 @@ project_name: {{ project_name }} + postgres_image_name: postgres:13 postgres_container_name: {{project_name}}-container-postgres postgres_volume_name: {{project_name}}-volume-postgres +postgres_extra_settings: + +odoo_rpc_timeout: 3600 odoo_host_xmlrpc_port: 9069 -odoo_default_country_code: FR +odoo_default_company: + country_code: FR odoo_versions: diff --git a/odoo_openupgrade_wizard/templates/custom-postgres.conf.j2 b/odoo_openupgrade_wizard/templates/custom-postgres.conf.j2 new file mode 100644 index 0000000..e69de29 diff --git a/odoo_openupgrade_wizard/tools/tools_odoo.py b/odoo_openupgrade_wizard/tools/tools_odoo.py index 7fbab7e..b36ddcd 100644 --- a/odoo_openupgrade_wizard/tools/tools_odoo.py +++ b/odoo_openupgrade_wizard/tools/tools_odoo.py @@ -160,7 +160,7 @@ def generate_odoo_command( result = ( f" {command}" f" {shell_cmd}" - # f" --config=/etc/odoo.cfg" + f" --config=/odoo_env/odoo.cfg" f" --data-dir=/env/filestore/" f" --addons-path={addons_path}" f" --logfile={log_file_docker_path}" diff --git a/odoo_openupgrade_wizard/tools/tools_odoo_instance.py b/odoo_openupgrade_wizard/tools/tools_odoo_instance.py index a9cf61a..71d1b76 100644 --- a/odoo_openupgrade_wizard/tools/tools_odoo_instance.py +++ b/odoo_openupgrade_wizard/tools/tools_odoo_instance.py @@ -7,9 +7,6 @@ from loguru import logger # Wait for the launch of odoo instance 60 seconds _ODOO_RPC_MAX_TRY = 60 -# Timeout for odoorpc call is 24 hours -_ODOO_RPC_TIMEOUT = 86400 - class OdooInstance: @@ -34,7 +31,7 @@ class OdooInstance: "0.0.0.0", "jsonrpc", port=port, - timeout=_ODOO_RPC_TIMEOUT, + timeout=ctx.obj["config"]["odoo_rpc_timeout"], ) # connexion is OK break diff --git a/odoo_openupgrade_wizard/tools/tools_postgres.py b/odoo_openupgrade_wizard/tools/tools_postgres.py index 9127c51..34da7d3 100644 --- a/odoo_openupgrade_wizard/tools/tools_postgres.py +++ b/odoo_openupgrade_wizard/tools/tools_postgres.py @@ -42,10 +42,18 @@ def get_postgres_container(ctx): logger.info("Creating Postgres volume: %s" % volume_name) client.volumes.create(volume_name) + command = None + postgres_extra_settings = ctx.obj["config"].get("postgres_extra_settings") + if postgres_extra_settings: + command = "postgres" + for key, value in postgres_extra_settings.items(): + command += f" -c {key}={value}" + logger.info("Launching Postgres Container. (Image %s)" % image_name) container = run_container( image_name, container_name, + command=command, environments={ "POSTGRES_USER": "odoo", "POSTGRES_PASSWORD": "odoo", @@ -53,8 +61,10 @@ def get_postgres_container(ctx): "PGDATA": "/var/lib/postgresql/data/pgdata", }, volumes={ - ctx.obj["env_folder_path"].absolute(): "/env/", + # Data volume volume_name: "/var/lib/postgresql/data/pgdata/", + # main folder path (to pass files) + ctx.obj["env_folder_path"].absolute(): "/env/", }, detach=True, ) diff --git a/tests/data/output_expected/config.yml b/tests/data/output_expected/config.yml index a31773e..1d97dd0 100644 --- a/tests/data/output_expected/config.yml +++ b/tests/data/output_expected/config.yml @@ -1,11 +1,16 @@ project_name: test-cli + postgres_image_name: postgres:13 postgres_container_name: test-cli-container-postgres postgres_volume_name: test-cli-volume-postgres +postgres_extra_settings: + +odoo_rpc_timeout: 3600 odoo_host_xmlrpc_port: 9069 -odoo_default_country_code: FR +odoo_default_company: + country_code: FR odoo_versions: