From 72a5dcf4347bcd5e0971b768c59f0c3237d55646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Taymans?= Date: Sat, 3 Feb 2024 22:38:13 +0100 Subject: [PATCH] New CI to auto build and publish Using a commit message to trigger the build does not work when merging a PR because last commit is the merge commit and not the commit edited with the right name. Given that, the jobs that will run, are defined at the creation of the pipeline, publishing and creating a release cannot be done based on the sate of the code. A way to trigger publication and release is the git tags. So with theses changes: - linting is done only on a merge request - testing and building are performed on a merge request and on the main branch When a tag is pushed: - check are done to ensure that the tag is the same as the version of the program, in order to not publish and release someting that is not coherent. - the program is published on pypi. - a release is created, but only if the tag is for a major, minor or patch version. No release created for an alpha, beta or pre-release version. So all versions of the program are published on PyPI, but only the important ones are published via the release mechanism. Because the release mechanism will warn user for a new version. Version that are not major, minor or patch are not intended to be used by end users. The idea of auto publishing and releasing every time a commit is pushed on the main branch does not work with semantic versioning. For doing that maybe a calversioning will be better. The idea of using the CI to push a tag for a new release lead to security risk. Because the CI will contains credential for writing to the repository, any contributor can read this token by editing the gitlab-ci file and use token for bad purposes. Gitlab does not provide token for writing to a repository owned by the project. So for now, we control the publication and release of a new version with two actions: - updating the version on the pyproject.toml file. - creating a tag with the same version as in the pyproject.toml file. --- .gitlab-ci.yml | 101 ++++++++++++++++++++++++++++++++++--------------- CHANGES.rst | 0 DEVELOP.md | 35 ++++++++++++++++- 3 files changed, 104 insertions(+), 32 deletions(-) create mode 100644 CHANGES.rst diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cea8cbb..0e14c4c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,27 +2,59 @@ stages: - lint - test - build + - publish - release pre-commit: image: python stage: lint - only: - - merge_requests + rules: + # Run only if merge request + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + # Run if commit on default branch + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH before_script: - pip install pre-commit script: - pre-commit run --all --show-diff-on-failure --verbose --color always +check_version: + stage: lint + image: python:latest + rules: + # Run if commit that start with a version number is pushed + - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+.*/ + before_script: + - pip install poetry + - poetry --version + script: + # Ensure tag is the same as the program version + - test $(poetry version --short) = $CI_COMMIT_TAG + +check_changelog: + stage: lint + image: python:latest + rules: + # Run if commit that start with a version number is pushed + - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/ + before_script: + - pip install poetry + - poetry --version + script: + # Ensure change log is completed correctly + - cat CHANGES.rst | grep $CI_COMMIT_TAG + pytest: image: name: python:$PYTHON_VERSION stage: test tags: - cie-oow-dind-runner - only: - - merge_requests - - "main" + rules: + # Run if merge request + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + # Run if commit on default branch + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH services: - name: docker:dind alias: dind @@ -49,40 +81,49 @@ build: stage: build image: python:latest rules: - # Do not run this job when a tag is created manually - - if: $CI_COMMIT_TAG - when: never - # Run this job when commits are pushed or merged to the default branch - # and the commit message contains "Bump to version" - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE =~ /^Bump to version.+/ - script: + # Run if merge request + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + # Run if commit on default branch + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + # Run if commit that start with a version number is pushed + - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+.*/ + before_script: - pip install poetry - poetry --version + script: - poetry build - - poetry publish --username $PYPI_USER --password $PYPI_TOKEN - - echo "VERSION=$(poetry version --short)" >> variables.env artifacts: - reports: - # Use artifacts:reports:dotenv to expose the variables to other jobs - dotenv: variables.env + untracked: true + paths: + - dist/ + +publish: + stage: publish + image: python:latest + rules: + # Run if commit that start with a version number is pushed + - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+.*/ + before_script: + - pip install poetry + - poetry --version + - ls -l dist + # Uncomment for testing build publication on test.pypi.org + #- poetry config repo.pypitest https://test.pypi.org/legacy/ + script: + - poetry publish --skip-existing --username $PYPI_USER --password $PYPI_TOKEN release: stage: release image: registry.gitlab.com/gitlab-org/release-cli:latest - needs: - - job: build - artifacts: true rules: - # Do not run this job when a tag is created manually - - if: $CI_COMMIT_TAG - when: never - # Run this job when commits are pushed or merged to the default branch - # and the commit message contains "Bump to version" - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE =~ /^Bump to version.+/ + # Run only for a patch, minor or major release + # This avoid creating a release for alpha, beta, or other special + # releases + - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/ script: - - echo "running release_job for $VERSION" + - echo "running release_job for $CI_COMMIT_TAG" release: - name: "$VERSION" - description: "$CI_COMMIT_DESCRIPTION" - tag_name: "$VERSION" + name: "$CI_COMMIT_TAG" + description: "Change log here: ${CI_PROJECT_URL}/-/blob/main/CHANGES.rst" + tag_name: "$CI_COMMIT_TAG" ref: "$CI_COMMIT_SHA" diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 0000000..e69de29 diff --git a/DEVELOP.md b/DEVELOP.md index 593d514..51f46cd 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -96,12 +96,43 @@ If you add new dependencies, you have to: - run the following command in your virtualenv : ``poetry update`` -## Publish on pyPI +## Release on Gitlab and publish on PyPI + +Building, releasing and publishing a new version works with tags. + +Tags that trigger a build and a publication on PyPI must have a name +equal to the version of the program found in `pyproject.toml`. + +Tags that matches an change in major, minor or patch version will +trigger a release on gitlab. + +Tags that are alpha, beta, pre-release, etc does not trigger a release +on gitlab, but they trigger a publication on PyPI. + +Before creating a tag, ensure that the version of the program is +updated. To update the program version you can use the command: ``` -poetry publish +poetry version {major,minor,patch} ``` +Ensure that the `CHANGES.rst` file contains information about this new +release, if it is a major, minor or patch release. For alpha, beta, etc +release information will be published with the next major, minor or +patch release. + +Then push a commit with the version and the changlog updated on the main +branch. + +When everything is good on the main branch and that tests succeed, +create a new tag with the same name as the version in `pyproject.toml` +file. Tags can be created via Code > Tags > New tag. + +To see if the publication on PyPI and the release on gitlab were done +correctly, go in Build > Pipelines. You will find a pipeline for the tag +you just created. + + # Understanding the library ## Tools to understand