Merge branch 'update_ci' into 'main'

New CI with docker-in-docker

See merge request odoo-openupgrade-wizard/odoo-openupgrade-wizard!48

.gitlab-ci.yml

@@ -5,9 +5,8 @@ stages:
   - release
 
 pre-commit:
+  image: python
   stage: lint
-  tags:
-    - shell
   only:
     - merge_requests
   before_script:
@@ -16,24 +15,35 @@ pre-commit:
     - pre-commit run --all --show-diff-on-failure --verbose --color always
 
 pytest:
+  image:
+    name: python:$PYTHON_VERSION
   stage: test
   tags:
-    - shell
+    - cie-oow-dind-runner
   only:
     - merge_requests
     - "main"
+  services:
+    - name: docker:dind
+      alias: dind
+  variables:
+    ODOO_RPC_URL: dind
+    DOCKER_HOST: tcp://dind:2375/
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: ""
   coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+)\%/'
-  script:
+  before_script:
-    - python3 -m venv ./.gitlab-ci-venv
-    - source ./.gitlab-ci-venv/bin/activate
-    # - pip install --upgrade pip
     - pip install poetry
-    - poetry --version
+    - export PATH="$HOME/.local/bin:$PATH"
-    - poetry install -v
+    - poetry install --all-extras
-    - echo $PATH
+  script:
-    - echo $PYTHONPATH
+    - poetry run pytest -vv -x --cov=odoo_openupgrade_wizard
-    - poetry run pytest --version
+  parallel:
-    - poetry run pytest --verbosity=2 --exitfirst --cov odoo_openupgrade_wizard
+    matrix:
+      - PYTHON_VERSION:
+        - "3.7"
+        - "3.8"
+        - "3.9"
 
 build:
   stage: build

.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 exclude: '^tests/data/output_expected/'
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -19,15 +19,15 @@ repos:
       - id: check-merge-conflict
       - id: check-symlinks
   - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.7.0
+    rev: v5.10.1
     hooks:
       - id: isort
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 23.11.0
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/flake8
-    rev: "3.9.2"
+    rev: "6.1.0"
     hooks:
       - id: flake8
   # - repo: https://gitlab.com/smop/pre-commit-hooks

odoo_openupgrade_wizard/cli/cli_docker_build.py

@@ -27,7 +27,6 @@ def docker_build(ctx, versions):
 
     # Build images for each odoo version
     for odoo_version in get_odoo_versions_from_options(ctx, versions):
-
         odoo_requirement_file_path = (
             get_odoo_env_path(ctx, odoo_version) / "src/odoo/requirements.txt"
         )

odoo_openupgrade_wizard/cli/cli_generate_module_analysis.py

@@ -33,7 +33,6 @@ from odoo_openupgrade_wizard.tools.tools_system import ensure_folder_writable
 )
 @click.pass_context
 def generate_module_analysis(ctx, step, database, modules):
-
     migration_steps = get_migration_steps_from_options(ctx, step - 1, step)
 
     initial_step = migration_steps[0].copy()

odoo_openupgrade_wizard/cli/cli_options.py

@@ -67,7 +67,6 @@ def database_option_required(function):
 
 
 def get_odoo_versions_from_options(ctx, versions_arg):
-
     if not versions_arg:
         return ctx.obj["config"]["odoo_versions"]
     else:

odoo_openupgrade_wizard/cli/cli_upgrade.py

@@ -23,7 +23,6 @@ from odoo_openupgrade_wizard.tools.tools_postgres import (
 @database_option_required
 @click.pass_context
 def upgrade(ctx, first_step, last_step, database):
-
     migration_steps = get_migration_steps_from_options(
         ctx, first_step, last_step
     )

odoo_openupgrade_wizard/templates/odoo/11.0/Dockerfile

@@ -55,6 +55,6 @@ RUN pip3 install --upgrade pip \
 # <OOW> Get local user id and set it to the odoo user
 ARG LOCAL_USER_ID
 
-RUN useradd --uid $LOCAL_USER_ID odoo
+RUN useradd --uid $LOCAL_USER_ID --non-unique odoo
 
 USER odoo

odoo_openupgrade_wizard/templates/odoo/12.0/Dockerfile

@@ -55,6 +55,6 @@ RUN pip3 install --upgrade pip \
 # <OOW> Get local user id and set it to the odoo user
 ARG LOCAL_USER_ID
 
-RUN useradd --uid $LOCAL_USER_ID odoo
+RUN useradd --uid $LOCAL_USER_ID --non-unique odoo
 
 USER odoo

odoo_openupgrade_wizard/templates/odoo/13.0/Dockerfile

@@ -51,6 +51,6 @@ RUN pip3 install --upgrade pip \
 # <OOW> Get local user id and set it to the odoo user
 ARG LOCAL_USER_ID
 
-RUN useradd --uid $LOCAL_USER_ID odoo
+RUN useradd --uid $LOCAL_USER_ID --non-unique odoo
 
 USER odoo

odoo_openupgrade_wizard/templates/odoo/14.0/Dockerfile

@@ -44,6 +44,6 @@ RUN pip3 install --upgrade pip \
 # <OOW> Get local user id and set it to the odoo user
 ARG LOCAL_USER_ID
 
-RUN useradd --uid $LOCAL_USER_ID odoo
+RUN useradd --uid $LOCAL_USER_ID --non-unique odoo
 
 USER odoo

odoo_openupgrade_wizard/templates/odoo/15.0/Dockerfile

@@ -44,6 +44,6 @@ RUN pip3 install --upgrade pip \
 # <OOW> Get local user id and set it to the odoo user
 ARG LOCAL_USER_ID
 
-RUN useradd --uid $LOCAL_USER_ID odoo
+RUN useradd --uid $LOCAL_USER_ID --non-unique odoo
 
 USER odoo

odoo_openupgrade_wizard/templates/odoo/16.0/Dockerfile

@@ -44,6 +44,6 @@ RUN pip3 install --upgrade pip \
 # <OOW> Get local user id and set it to the odoo user
 ARG LOCAL_USER_ID
 
-RUN useradd --uid $LOCAL_USER_ID odoo
+RUN useradd --uid $LOCAL_USER_ID --non-unique odoo
 
 USER odoo

odoo_openupgrade_wizard/tools/tools_docker.py

@@ -1,5 +1,3 @@
-import time
-
 import docker
 from loguru import logger
 
@@ -88,15 +86,6 @@ def run_container(
     if detach:
         logger.debug("Container %s launched." % image_name)
     elif auto_remove:
-        # Workaround to avoid
-        # "Conflict. The container name xxx is already in use"
-        # Because it seems that the container name is not freed when the
-        # run command completes
-        while client.containers.list(
-            all=True, filters={"name": container_name}
-        ):
-            time.sleep(1)
-
         logger.debug("Container closed.")
 
     return container
@@ -130,6 +119,7 @@ def kill_container(container_name):
         containers = client.containers.list(
             all=True,
             filters={"name": container_name},
+            ignore_removed=True,
         )
     except docker.errors.NotFound as err:
         logger.debug(f"Cannot kill container {container_name}: " + str(err))

odoo_openupgrade_wizard/tools/tools_odoo.py

@@ -107,7 +107,6 @@ def generate_odoo_command(
     stop_after_init: bool = False,
     shell: bool = False,
 ) -> str:
-
     odoo_env_path = get_odoo_env_path(ctx, migration_step["version"])
 
     # Compute 'server_wide_modules'
@@ -295,7 +294,6 @@ def execute_click_odoo_python_files(
     python_files: list = [],
     execution_context: str = False,
 ):
-
     if not python_files:
         # Get post-migration python scripts to execute
         script_folder = get_script_folder(ctx, migration_step)

odoo_openupgrade_wizard/tools/tools_odoo_instance.py

@@ -6,10 +6,10 @@ from loguru import logger
 
 # Wait for the launch of odoo instance 60 seconds
 _ODOO_RPC_MAX_TRY = 60
+_ODOO_RPC_URL = "0.0.0.0"
 
 
 class OdooInstance:
-
     env = False
     version = False
 
@@ -28,7 +28,7 @@ class OdooInstance:
             # Connection
             try:
                 rpc_connexion = odoorpc.ODOO(
-                    "0.0.0.0",
+                    _ODOO_RPC_URL,
                     "jsonrpc",
                     port=port,
                     timeout=ctx.obj["config"]["odoo_rpc_timeout"],
@@ -83,7 +83,7 @@ class OdooInstance:
         return model.browse(model.create(vals))
 
     def install_modules(self, module_names):
-        if type(module_names) == str:
+        if type(module_names) is str:
             module_names = [module_names]
         installed_modules = []
         i = 0

odoo_openupgrade_wizard/tools/tools_odoo_module.py

@@ -117,7 +117,6 @@ class Analysis(object):
 
         # Instanciate a new odoo_module
         for module_name in module_list:
-
             addon_path = OdooModule.get_addon_path(
                 ctx, module_name, self.initial_version
             )
@@ -476,7 +475,6 @@ class OdooModule(object):
 
 
 class OdooModuleVersion(object):
-
     _exclude_directories = [
         "lib",
         "demo",

odoo_openupgrade_wizard/tools/tools_postgres.py

@@ -21,7 +21,9 @@ def get_postgres_container(ctx):
 
     # Check if container exists
     containers = client.containers.list(
-        all=True, filters={"name": container_name}
+        all=True,
+        filters={"name": container_name},
+        ignore_removed=True,
     )
     if containers:
         container = containers[0]

odoo_openupgrade_wizard/tools/tools_system.py

@@ -84,7 +84,6 @@ def ensure_file_exists_from_template(
 
 
 def git_aggregate(folder_path: Path, config_path: Path, jobs: int):
-
     args = argparse.Namespace(
         command="aggregate",
         config=str(config_path),

pyproject.toml

@@ -43,11 +43,12 @@ pygount = "^1.4"
 pyyaml = "5.4.1"
 single-source = "^0.3"
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
 pytest = [
     {version = "<=6.1.2", python = "<3.10"},
     {version = ">=6.2.5", python = ">=3.10"}
 ]
+pytest-mock = "3.6.1"
 pytest-cov = "*"
 safety = "*"
 pylint = "*"

tests/__init__.py

@@ -81,3 +81,13 @@ def build_ctx_from_config_file() -> dict:
     ctx.obj["env_folder_path"] = env_folder_path
     ctx.obj["src_folder_path"] = env_folder_path / Path("src")
     return ctx
+
+
+def mock_odoo_rpc_url(mocker):
+    """Mock the _ODOO_RPC_URL for testing purpose"""
+    odoo_rpc_url = os.environ.get("ODOO_RPC_URL")
+    if odoo_rpc_url:
+        mocker.patch(
+            "odoo_openupgrade_wizard.tools.tools_odoo_instance._ODOO_RPC_URL",
+            odoo_rpc_url,
+        )

tests/cli_20_install_from_csv_test.py

@@ -6,16 +6,19 @@ from odoo_openupgrade_wizard.tools.tools_postgres import (
 from . import (
     build_ctx_from_config_file,
     cli_runner_invoke,
+    mock_odoo_rpc_url,
     move_to_test_folder,
 )
 
 
-def test_cli_install_from_csv():
+def test_cli_install_from_csv(mocker):
     move_to_test_folder()
+    mock_odoo_rpc_url(mocker)
 
     # Initialize database
     db_name = "database_test_cli___install_from_csv"
     ctx = build_ctx_from_config_file()
+
     ensure_database(ctx, db_name, state="absent")
 
     cli_runner_invoke(

tests/cli_21_generate_module_analysis_test.py

@@ -5,12 +5,14 @@ from odoo_openupgrade_wizard.tools.tools_odoo import get_odoo_env_path
 from . import (
     build_ctx_from_config_file,
     cli_runner_invoke,
+    mock_odoo_rpc_url,
     move_to_test_folder,
 )
 
 
-def test_cli_generate_module_analysis():
+def test_cli_generate_module_analysis(mocker):
     move_to_test_folder()
+    mock_odoo_rpc_url(mocker)
     ctx = build_ctx_from_config_file()
 
     db_name = "database_test_cli___generate_module_analysis"

tests/cli_22_dumpdb_test.py

@@ -6,12 +6,14 @@ from odoo_openupgrade_wizard.tools.tools_postgres import ensure_database
 from . import (
     build_ctx_from_config_file,
     cli_runner_invoke,
+    mock_odoo_rpc_url,
     move_to_test_folder,
 )
 
 
-def test_cli_dumpdb():
+def test_cli_dumpdb(mocker):
     move_to_test_folder()
+    mock_odoo_rpc_url(mocker)
 
     # Initialize database
     db_name = "database_test_cli___dumpdb"
@@ -62,8 +64,9 @@ def test_cli_dumpdb():
             filestore_path.unlink()
 
 
-def test_cli_dumpdb_failure():
+def test_cli_dumpdb_failure(mocker):
     move_to_test_folder()
+    mock_odoo_rpc_url(mocker)
 
     # Initialize database
     db_name = "database_test_cli___dumpdb"

tests/cli_31_copydb_test.py

@@ -6,12 +6,14 @@ from odoo_openupgrade_wizard.tools.tools_postgres import ensure_database
 from . import (
     build_ctx_from_config_file,
     cli_runner_invoke,
+    mock_odoo_rpc_url,
     move_to_test_folder,
 )
 
 
-def test_cli_copydb():
+def test_cli_copydb(mocker):
     move_to_test_folder()
+    mock_odoo_rpc_url(mocker)
 
     db_name = "database_test_cli___copydb"
     db_dest_name = "database_test_cli___copydb__copy"

tox.ini

@@ -1,10 +0,0 @@
-[tox]
-isolated_build = true
-skipsdist = True
-envlist = py36, py37, py38, py39
-
-[testenv]
-whitelist_externals = poetry
-commands =
-    poetry install -v
-    poetry run pytest -vv