From 5d6905ebaa919e89b0a639e2168a914c70b7696c Mon Sep 17 00:00:00 2001 From: Sylvain LE GAL Date: Thu, 5 May 2022 12:35:52 +0200 Subject: [PATCH] [ADD] ensure_database function ; [IMP] tests with SQL request --- README.md | 83 +++++++++++++++++-- ROADMAP.md | 27 ++++++ .../cli_install_from_csv.py | 4 +- odoo_openupgrade_wizard/cli_run.py | 4 +- odoo_openupgrade_wizard/templates.py | 8 +- odoo_openupgrade_wizard/tools_postgres.py | 33 +++++--- tests/cli_B_03_run_test.py | 19 +++++ tests/cli_B_04_execute_script_test.py | 1 + tests/cli_B_05_upgrade_test.py | 34 +++++++- tests/output_A_expected/config.yml | 4 + 10 files changed, 189 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index e3fb016..2664ed6 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ This will generate the following structure : ``` config.yml +modules.csv log/ 2022_03_25__23_12_41__init.log ... @@ -67,6 +68,12 @@ src/ ``` +* ``config.xml`` is the main configuration file of your project. + +* ``modules.csv`` file is an optional file. You can fill it with the list + of your modules installed on your production. The first column of this + file should contain the technical name of the module. + * ``log`` folder will contains all the log of the ``odoo-openupgrade-wizard`` and the logs of the odoo instance that will be executed. @@ -103,10 +110,13 @@ extra repositories, or dependencies... odoo-openupgrade-wizard get-code ``` -This command will simply get all the Odoo code required to run all the steps for your migration. +This command will simply get all the Odoo code required to run all the steps for your migration with the ``gitaggregate`` tools. + The code is defined in the ``repos.yml`` of each sub folders. -Note : This step could take a big while ! +**Note** + +* This step could take a big while ! **Optional arguments** @@ -124,11 +134,12 @@ This script will pull official odoo docker images, defined in the ``Dockerfile`` each folder, and build a custom images on top the official one, installing inside custom librairies defined in ``debian_requirements.txt``, ``python_requirements.txt``. -At this end of this step executing the following command -``docker images --filter "reference=odoo-openupgrade-wizard-*"`` should show a docker image per version. +At this end of this step executing the following command should show a docker image per version. ``` +$ docker images --filter "reference=odoo-openupgrade-wizard-*" + REPOSITORY TAG IMAGE ID CREATED SIZE odoo-openupgrade-wizard-image---my-customer-10-12---12.0 latest ef664c366208 2 weeks ago 1.39GB odoo-openupgrade-wizard-image---my-customer-10-12---11.0 latest 24e283fe4ae4 2 weeks ago 1.16GB @@ -137,11 +148,67 @@ odoo-openupgrade-wizard-image---my-customer-10-12---10.0 latest 9d94dce2bd4 **Optional arguments** -if you want to build an image for some given releases, you can provide an extra parameter: +* if you want to (re)build an image for some given releases, you can provide + an extra parameter: ``--releases 10.0,12.0`` -``` -odoo-openupgrade-wizard docker-build --releases 10.0,12.0 -``` +**Note** +* This step could take a big while also ! ## ``odoo-openupgrade-wizard run`` + +``` +odoo-openupgrade-wizard run\ + --step 1\ + --database DB_NAME +``` + +Run an Odoo instance with the environment defined by the step argument. + +The database will be created, if it doesn't exists. + +if ``stop-after-init`` is disabled, the odoo instance will be available +at your host, at the following url : http://localhost:9069 +(Port depends on your ``host_odoo_xmlrpc_port`` setting of your ``config.yml`` file) + +**Optional arguments** + +* You can add ``--init-modules=purchase,sale`` to install modules. + +* You can add ``stop-after-init`` flag to turn off the process at the end + of the installation. + +## ``odoo-openupgrade-wizard install-from-csv`` + +``` +odoo-openupgrade-wizard install-from-csv\ + --database DB_NAME +``` + +Install the list of the modules defined in your ``modules.csv`` files on the +given database. + +The database will be created, if it doesn't exists. + +## ``odoo-openupgrade-wizard upgrade`` + +``` +odoo-openupgrade-wizard upgrade\ + --database DB_NAME +``` + +Realize an upgrade of the database from the initial release to +the final release, following the different steps. + +For each step, it will : + +1. Execute the ``pre-migration.sql`` of the step. +2. Realize an "update all" (in an upgrade or update context) +3. Execute the scripts via XML-RPC (via ``odoorpc``) defined in + the ``post-migration.py`` file. + +**Optional arguments** + +* You can add ``--first-step=2`` to start at the second step. + +* You can add ``--last-step=3`` to end at the third step. diff --git a/ROADMAP.md b/ROADMAP.md index 90d75c2..54794a0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -45,3 +45,30 @@ that raise an error : # execute sql request in postgres docker docker exec db psql --username=odoo --dbname=test_v12 -c "update res_partner set ""email"" = 'bib@bqsdfqsdf.txt';" ``` + + +# TODO Must Have + +- Fix via another way the problem of old ``openupgradelib``. + (it makes the upgrade failing for old revision (V8, etc...)) + +- Fix gitlab CI. tests are working locally but there is a network problem + to use ``odoorpc`` on gitlab-ci. + +# TODO Features + +- select ``without-demo all`` depending on if the database + is created or not (, and if current database contains demo data ?!?) + +- add a tools to analyze workload. + +- execute ``pre-migration.sql`` files. + +# TODO Nice To have + +- Fix gitlabci-local. For the time being, it is not possible to debug + locally. (there are extra bugs locally that doesn't occures on gitlab, + in ``cli_B_03_run_test.py``... + +- ``.absolute()`` has been added in test to try to fix some things, + but maybe it's not necessary. diff --git a/odoo_openupgrade_wizard/cli_install_from_csv.py b/odoo_openupgrade_wizard/cli_install_from_csv.py index c752c95..f9f14d0 100644 --- a/odoo_openupgrade_wizard/cli_install_from_csv.py +++ b/odoo_openupgrade_wizard/cli_install_from_csv.py @@ -9,7 +9,7 @@ from odoo_openupgrade_wizard.cli_options import ( ) from odoo_openupgrade_wizard.tools_odoo import kill_odoo, run_odoo from odoo_openupgrade_wizard.tools_odoo_instance import OdooInstance -from odoo_openupgrade_wizard.tools_postgres import ensure_database_exists +from odoo_openupgrade_wizard.tools_postgres import ensure_database @click.command() @@ -17,7 +17,7 @@ from odoo_openupgrade_wizard.tools_postgres import ensure_database_exists @click.pass_context def install_from_csv(ctx, database): migration_step = get_migration_step_from_options(ctx, 1) - ensure_database_exists(database) + ensure_database(database, state="present") # Get modules list from the CSV file csv_path = ctx.obj["module_file_path"] diff --git a/odoo_openupgrade_wizard/cli_run.py b/odoo_openupgrade_wizard/cli_run.py index 05d956f..0a66ecc 100644 --- a/odoo_openupgrade_wizard/cli_run.py +++ b/odoo_openupgrade_wizard/cli_run.py @@ -8,7 +8,7 @@ from odoo_openupgrade_wizard.cli_options import ( ) from odoo_openupgrade_wizard.tools_odoo import kill_odoo, run_odoo from odoo_openupgrade_wizard.tools_odoo_instance import get_odoo_url -from odoo_openupgrade_wizard.tools_postgres import ensure_database_exists +from odoo_openupgrade_wizard.tools_postgres import ensure_database @click.command() @@ -31,7 +31,7 @@ from odoo_openupgrade_wizard.tools_postgres import ensure_database_exists def run(ctx, step, database, stop_after_init, init_modules): migration_step = get_migration_step_from_options(ctx, step) - ensure_database_exists(database) + ensure_database(database, state="present") try: run_odoo( ctx, diff --git a/odoo_openupgrade_wizard/templates.py b/odoo_openupgrade_wizard/templates.py index d7cc5db..35d6459 100644 --- a/odoo_openupgrade_wizard/templates.py +++ b/odoo_openupgrade_wizard/templates.py @@ -1,7 +1,10 @@ -CONFIG_YML_TEMPLATE = """project_name: {{ project_name }} +CONFIG_YML_TEMPLATE = """ +project_name: {{ project_name }} host_odoo_xmlrpc_port: 9069 + host_postgres_port: 9432 + default_country_code: FR odoo_versions: @@ -116,7 +119,8 @@ GIT_IGNORE_CONTENT = """ !.gitignore """ -MODULES_CSV_TEMPLATE = """base,Base +MODULES_CSV_TEMPLATE = """ +base,Base account,Account Module product,Product web_responsive,Web Responsive Module diff --git a/odoo_openupgrade_wizard/tools_postgres.py b/odoo_openupgrade_wizard/tools_postgres.py index d6022f6..4798ae1 100644 --- a/odoo_openupgrade_wizard/tools_postgres.py +++ b/odoo_openupgrade_wizard/tools_postgres.py @@ -5,7 +5,10 @@ from odoo_openupgrade_wizard.tools_docker import get_docker_client def get_postgres_container(): client = get_docker_client() - return client.containers.list(filters={"name": "db"})[0] + containers = client.containers.list(filters={"name": "db"}) + if not containers: + raise Exception("Postgresql container not found with name 'db'.") + return containers[0] def execute_sql_request(request, database="postgres"): @@ -33,20 +36,30 @@ def execute_sql_request(request, database="postgres"): return result -def ensure_database_exists(database: str): +def ensure_database(database: str, state="present"): """ - Connect to postgres container. - Check if the database exist. - - if not, create it. + - if doesn't exists and state == 'present', create it. + - if exists and state == 'absent', drop it. """ request = "select datname FROM pg_database WHERE datistemplate = false;" result = execute_sql_request(request) - if [database] in result: - return - logger.info("Create database '%s' ..." % database) - request = "CREATE DATABASE {database} owner odoo;".format( - database=database - ) - execute_sql_request(request) + if state == "present": + if [database] in result: + return + + logger.info("Create database '%s' ..." % database) + request = "CREATE DATABASE {database} owner odoo;".format( + database=database + ) + execute_sql_request(request) + else: + if [database] not in result: + return + + logger.info("Drop database '%s' ..." % database) + request = "DROP DATABASE {database};".format(database=database) + execute_sql_request(request) diff --git a/tests/cli_B_03_run_test.py b/tests/cli_B_03_run_test.py index b814643..1f1d055 100644 --- a/tests/cli_B_03_run_test.py +++ b/tests/cli_B_03_run_test.py @@ -1,6 +1,7 @@ from pathlib import Path from odoo_openupgrade_wizard.tools_docker import get_docker_client +from odoo_openupgrade_wizard.tools_postgres import execute_sql_request from . import cli_runner_invoke @@ -27,6 +28,24 @@ def test_cli_run(): ) assert db_filestore_path.exists() + # Ensure that 'base' module is installed + request = ( + "SELECT id" + " FROM ir_module_module" + " WHERE state ='installed'" + " AND name='base';" + ) + assert execute_sql_request(request, database=db_name) + + # Ensure that 'point_of_sale' module is not installed + request = ( + "SELECT id" + " FROM ir_module_module" + " WHERE state ='installed'" + " AND name='point_of_sale';" + ) + assert not execute_sql_request(request, database=db_name) + # Ensure that all the containers are removed docker_client = get_docker_client() assert not docker_client.containers.list( diff --git a/tests/cli_B_04_execute_script_test.py b/tests/cli_B_04_execute_script_test.py index 59a2275..46d55cf 100644 --- a/tests/cli_B_04_execute_script_test.py +++ b/tests/cli_B_04_execute_script_test.py @@ -6,6 +6,7 @@ from . import cli_runner_invoke def test_cli_execute_script(): # TODO: FIXME # This test works locally, but doesn't work on gitlabci + output_folder_path = Path("./tests/output_B").absolute() extra_script_path = Path( diff --git a/tests/cli_B_05_upgrade_test.py b/tests/cli_B_05_upgrade_test.py index 7a58794..9c18504 100644 --- a/tests/cli_B_05_upgrade_test.py +++ b/tests/cli_B_05_upgrade_test.py @@ -1,13 +1,20 @@ from pathlib import Path +from odoo_openupgrade_wizard.tools_postgres import ( + ensure_database, + execute_sql_request, +) + from . import cli_runner_invoke def test_cli_upgrade(): - return output_folder_path = Path("./tests/output_B").absolute() db_name = "database_test_cli_upgrade" + + ensure_database(db_name, state="absent") + cli_runner_invoke( [ "--log-level=DEBUG", @@ -20,6 +27,17 @@ def test_cli_upgrade(): ] ) + # Ensure that 'base' module is installed at 13.0 + request = ( + "SELECT latest_version" + " FROM ir_module_module" + " WHERE state ='installed'" + " AND name='base';" + ) + latest_version = execute_sql_request(request, database=db_name) + + assert latest_version[0][0].startswith("13.") + cli_runner_invoke( [ "--log-level=DEBUG", @@ -27,9 +45,17 @@ def test_cli_upgrade(): "upgrade", "--database=%s" % db_name, "--first-step=1", - # TODO : set 3 when dropping database will be done - "--last-step=1", + "--last-step=3", ] ) - # TODO, write test + # Ensure that 'base' module is installed at 14.0 + request = ( + "SELECT latest_version" + " FROM ir_module_module" + " WHERE state ='installed'" + " AND name='base';" + ) + latest_version = execute_sql_request(request, database=db_name) + + assert latest_version[0][0].startswith("14.") diff --git a/tests/output_A_expected/config.yml b/tests/output_A_expected/config.yml index 8767fa3..79674b8 100644 --- a/tests/output_A_expected/config.yml +++ b/tests/output_A_expected/config.yml @@ -1,8 +1,12 @@ + project_name: test-cli host_odoo_xmlrpc_port: 9069 + host_postgres_port: 9432 +default_country_code: FR + odoo_versions: - release: 9.0