[ADD] ensure_database function ; [IMP] tests with SQL request

README.md

@@ -34,6 +34,7 @@ This will generate the following structure :
 
 ```
 config.yml
+modules.csv
 log/
     2022_03_25__23_12_41__init.log
     ...
@@ -67,6 +68,12 @@ src/
 
 ```
 
+* ``config.xml`` is the main configuration file of your project.
+
+* ``modules.csv`` file is an optional file. You can fill it with the list
+  of your modules installed on your production. The first column of this
+  file should contain the technical name of the module.
+
 * ``log`` folder will contains all the log of the ``odoo-openupgrade-wizard``
   and the logs of the odoo instance that will be executed.
 
@@ -103,10 +110,13 @@ extra repositories, or dependencies...
 odoo-openupgrade-wizard get-code
 ```
 
-This command will simply get all the Odoo code required to run all the steps for your migration.
+This command will simply get all the Odoo code required to run all the steps for your migration with the ``gitaggregate`` tools.
+
 The code is defined in the ``repos.yml`` of each sub folders.
 
-Note : This step could take a big while !
+**Note**
+
+* This step could take a big while !
 
 **Optional arguments**
 
@@ -124,11 +134,12 @@ This script will pull official odoo docker images, defined in the ``Dockerfile``
 each folder, and build a custom images on top the official one, installing inside
 custom librairies defined in ``debian_requirements.txt``, ``python_requirements.txt``.
 
-At this end of this step executing the following command
-``docker images --filter "reference=odoo-openupgrade-wizard-*"`` should show a docker image per version.
+At this end of this step executing the following command should show a docker image per version.
 
 
 ```
+$ docker images --filter "reference=odoo-openupgrade-wizard-*"
+
 REPOSITORY                                                 TAG       IMAGE ID       CREATED       SIZE
 odoo-openupgrade-wizard-image---my-customer-10-12---12.0   latest    ef664c366208   2 weeks ago   1.39GB
 odoo-openupgrade-wizard-image---my-customer-10-12---11.0   latest    24e283fe4ae4   2 weeks ago   1.16GB
@@ -137,11 +148,67 @@ odoo-openupgrade-wizard-image---my-customer-10-12---10.0   latest    9d94dce2bd4
 
 **Optional arguments**
 
-if you want to build an image for some given releases, you can provide an extra parameter:
+* if you want to (re)build an image for some given releases, you can provide
+  an extra parameter: ``--releases 10.0,12.0``
 
-```
-odoo-openupgrade-wizard docker-build --releases 10.0,12.0
-```
+**Note**
 
+* This step could take a big while also !
 
 ## ``odoo-openupgrade-wizard run``
+
+```
+odoo-openupgrade-wizard run\
+    --step 1\
+    --database DB_NAME
+```
+
+Run an Odoo instance with the environment defined by the step argument.
+
+The database will be created, if it doesn't exists.
+
+if ``stop-after-init`` is disabled, the odoo instance will be available
+at your host, at the following url : http://localhost:9069
+(Port depends on your ``host_odoo_xmlrpc_port`` setting of your ``config.yml`` file)
+
+**Optional arguments**
+
+* You can add ``--init-modules=purchase,sale`` to install modules.
+
+* You can add ``stop-after-init`` flag to turn off the process at the end
+  of the installation.
+
+## ``odoo-openupgrade-wizard install-from-csv``
+
+```
+odoo-openupgrade-wizard install-from-csv\
+    --database DB_NAME
+```
+
+Install the list of the modules defined in your ``modules.csv`` files on the
+given database.
+
+The database will be created, if it doesn't exists.
+
+## ``odoo-openupgrade-wizard upgrade``
+
+```
+odoo-openupgrade-wizard upgrade\
+    --database DB_NAME
+```
+
+Realize an upgrade of the database from the initial release to
+the final release, following the different steps.
+
+For each step, it will :
+
+1. Execute the ``pre-migration.sql`` of the step.
+2. Realize an "update all" (in an upgrade or update context)
+3. Execute the scripts via XML-RPC (via ``odoorpc``) defined in
+   the ``post-migration.py`` file.
+
+**Optional arguments**
+
+* You can add ``--first-step=2`` to start at the second step.
+
+* You can add ``--last-step=3`` to end at the third step.

ROADMAP.md

@@ -45,3 +45,30 @@ that raise an error :
 # execute sql request in postgres docker
 docker exec db psql --username=odoo --dbname=test_v12 -c "update res_partner set ""email"" = 'bib@bqsdfqsdf.txt';"
 ```
+
+
+# TODO Must Have
+
+- Fix via another way the problem of old ``openupgradelib``.
+  (it makes the upgrade failing for old revision (V8, etc...))
+
+- Fix gitlab CI. tests are working locally but there is a network problem
+  to use ``odoorpc`` on gitlab-ci.
+
+# TODO Features
+
+- select ``without-demo all`` depending on if the database
+  is created or not (, and if current database contains demo data ?!?)
+
+- add a tools to analyze workload.
+
+- execute ``pre-migration.sql`` files.
+
+# TODO Nice To have
+
+- Fix gitlabci-local. For the time being, it is not possible to debug
+  locally. (there are extra bugs locally that doesn't occures on gitlab,
+  in ``cli_B_03_run_test.py``...
+
+- ``.absolute()`` has been added in test to try to fix some things,
+  but maybe it's not necessary.

odoo_openupgrade_wizard/cli_install_from_csv.py

@@ -9,7 +9,7 @@ from odoo_openupgrade_wizard.cli_options import (
 )
 from odoo_openupgrade_wizard.tools_odoo import kill_odoo, run_odoo
 from odoo_openupgrade_wizard.tools_odoo_instance import OdooInstance
-from odoo_openupgrade_wizard.tools_postgres import ensure_database_exists
+from odoo_openupgrade_wizard.tools_postgres import ensure_database
 
 
 @click.command()
@@ -17,7 +17,7 @@ from odoo_openupgrade_wizard.tools_postgres import ensure_database_exists
 @click.pass_context
 def install_from_csv(ctx, database):
     migration_step = get_migration_step_from_options(ctx, 1)
-    ensure_database_exists(database)
+    ensure_database(database, state="present")
 
     # Get modules list from the CSV file
     csv_path = ctx.obj["module_file_path"]

odoo_openupgrade_wizard/cli_run.py

@@ -8,7 +8,7 @@ from odoo_openupgrade_wizard.cli_options import (
 )
 from odoo_openupgrade_wizard.tools_odoo import kill_odoo, run_odoo
 from odoo_openupgrade_wizard.tools_odoo_instance import get_odoo_url
-from odoo_openupgrade_wizard.tools_postgres import ensure_database_exists
+from odoo_openupgrade_wizard.tools_postgres import ensure_database
 
 
 @click.command()
@@ -31,7 +31,7 @@ from odoo_openupgrade_wizard.tools_postgres import ensure_database_exists
 def run(ctx, step, database, stop_after_init, init_modules):
 
     migration_step = get_migration_step_from_options(ctx, step)
-    ensure_database_exists(database)
+    ensure_database(database, state="present")
     try:
         run_odoo(
             ctx,

odoo_openupgrade_wizard/templates.py

@@ -1,7 +1,10 @@
-CONFIG_YML_TEMPLATE = """project_name: {{ project_name }}
+CONFIG_YML_TEMPLATE = """
+project_name: {{ project_name }}
 
 host_odoo_xmlrpc_port: 9069
+
 host_postgres_port: 9432
+
 default_country_code: FR
 
 odoo_versions:
@@ -116,7 +119,8 @@ GIT_IGNORE_CONTENT = """
 !.gitignore
 """
 
-MODULES_CSV_TEMPLATE = """base,Base
+MODULES_CSV_TEMPLATE = """
+base,Base
 account,Account Module
 product,Product
 web_responsive,Web Responsive Module

odoo_openupgrade_wizard/tools_postgres.py

@@ -5,7 +5,10 @@ from odoo_openupgrade_wizard.tools_docker import get_docker_client
 
 def get_postgres_container():
     client = get_docker_client()
-    return client.containers.list(filters={"name": "db"})[0]
+    containers = client.containers.list(filters={"name": "db"})
+    if not containers:
+        raise Exception("Postgresql container not found with name 'db'.")
+    return containers[0]
 
 
 def execute_sql_request(request, database="postgres"):
@@ -33,15 +36,18 @@ def execute_sql_request(request, database="postgres"):
     return result
 
 
-def ensure_database_exists(database: str):
+def ensure_database(database: str, state="present"):
     """
     - Connect to postgres container.
     - Check if the database exist.
-    - if not, create it.
+    - if doesn't exists and state == 'present', create it.
+    - if exists and state == 'absent', drop it.
     """
     request = "select datname FROM pg_database WHERE datistemplate = false;"
 
     result = execute_sql_request(request)
+
+    if state == "present":
         if [database] in result:
             return
 
@@ -50,3 +56,10 @@ def ensure_database_exists(database: str):
             database=database
         )
         execute_sql_request(request)
+    else:
+        if [database] not in result:
+            return
+
+        logger.info("Drop database '%s' ..." % database)
+        request = "DROP DATABASE {database};".format(database=database)
+        execute_sql_request(request)

tests/cli_B_03_run_test.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
 from odoo_openupgrade_wizard.tools_docker import get_docker_client
+from odoo_openupgrade_wizard.tools_postgres import execute_sql_request
 
 from . import cli_runner_invoke
 
@@ -27,6 +28,24 @@ def test_cli_run():
     )
     assert db_filestore_path.exists()
 
+    # Ensure that 'base' module is installed
+    request = (
+        "SELECT id"
+        " FROM ir_module_module"
+        " WHERE state ='installed'"
+        " AND name='base';"
+    )
+    assert execute_sql_request(request, database=db_name)
+
+    # Ensure that 'point_of_sale' module is not installed
+    request = (
+        "SELECT id"
+        " FROM ir_module_module"
+        " WHERE state ='installed'"
+        " AND name='point_of_sale';"
+    )
+    assert not execute_sql_request(request, database=db_name)
+
     # Ensure that all the containers are removed
     docker_client = get_docker_client()
     assert not docker_client.containers.list(

tests/cli_B_04_execute_script_test.py

@@ -6,6 +6,7 @@ from . import cli_runner_invoke
 def test_cli_execute_script():
     # TODO: FIXME
     # This test works locally, but doesn't work on gitlabci
+
     output_folder_path = Path("./tests/output_B").absolute()
 
     extra_script_path = Path(

tests/cli_B_05_upgrade_test.py

@@ -1,13 +1,20 @@
 from pathlib import Path
 
+from odoo_openupgrade_wizard.tools_postgres import (
+    ensure_database,
+    execute_sql_request,
+)
+
 from . import cli_runner_invoke
 
 
 def test_cli_upgrade():
-    return
     output_folder_path = Path("./tests/output_B").absolute()
 
     db_name = "database_test_cli_upgrade"
+
+    ensure_database(db_name, state="absent")
+
     cli_runner_invoke(
         [
             "--log-level=DEBUG",
@@ -20,6 +27,17 @@ def test_cli_upgrade():
         ]
     )
 
+    # Ensure that 'base' module is installed at 13.0
+    request = (
+        "SELECT latest_version"
+        " FROM ir_module_module"
+        " WHERE state ='installed'"
+        " AND name='base';"
+    )
+    latest_version = execute_sql_request(request, database=db_name)
+
+    assert latest_version[0][0].startswith("13.")
+
     cli_runner_invoke(
         [
             "--log-level=DEBUG",
@@ -27,9 +45,17 @@ def test_cli_upgrade():
             "upgrade",
             "--database=%s" % db_name,
             "--first-step=1",
-            # TODO : set 3 when dropping database will be done
-            "--last-step=1",
+            "--last-step=3",
         ]
     )
 
-    # TODO, write test
+    # Ensure that 'base' module is installed at 14.0
+    request = (
+        "SELECT latest_version"
+        " FROM ir_module_module"
+        " WHERE state ='installed'"
+        " AND name='base';"
+    )
+    latest_version = execute_sql_request(request, database=db_name)
+
+    assert latest_version[0][0].startswith("14.")

tests/output_A_expected/config.yml

@@ -1,8 +1,12 @@
+
 project_name: test-cli
 
 host_odoo_xmlrpc_port: 9069
+
 host_postgres_port: 9432
 
+default_country_code: FR
+
 odoo_versions:
 
   - release: 9.0