# Autogenerated by ~/code/xcookie/xcookie/builders/gitlab_ci.py stages: # TEMPLATES - lint - build - test - gpgsign - deploy .common_template: &common_template tags: # Tags define which runners will accept which jobs - docker - linux-x86_64 - build - kitware-python-stack variables: # Change pip's cache directory to be inside the project directory # since we can only cache local items. PIP_CACHE_DIR: $CI_PROJECT_DIR/.cache/pip #rules: # # Dependent jobs run automatically # - if: '$CI_COMMIT_TAG != ""' # when: never # - if: '$CI_MERGE_REQUEST_ID && $RUN_DEPENDENT == "true"' # when: on_success # # Release jobs never run on MRs # - if: '$CI_MERGE_REQUEST_ID && $IS_FOR_RELEASE == "true"' # when: never # # MRs use manual triggers. # - if: '$CI_MERGE_REQUEST_ID' # when: manual # # Non-protected branches without tags have no pipelines (MR source branches). # - if: '$CI_COMMIT_REF_PROTECTED != "true" && $CI_COMMIT_TAG == ""' # when: never # # Otherwise, if it's a matching tag on the main project, run the job. # - if: '$CI_PROJECT_PATH == "computer-vision/geowatch" && $CI_COMMIT_TAG =~ $RELEASE_TAG_RE' # when: on_success # # If there is a tag and it doesn't match, skip the job. # - if: '$CI_PROJECT_PATH == "computer-vision/geowatch" && $CI_COMMIT_TAG != ""' # when: never # # If it's a release job and in the project, skip the job. # - if: '$CI_PROJECT_PATH == "computer-vision/geowatch" && $IS_FOR_RELEASE == "true"' # when: never # # Otherwise, if it is in the project, run the job. # - if: '$CI_PROJECT_PATH == "computer-vision/geowatch"' # when: on_success # # Ignore everything else. # - when: never except: # Don't run the pipeline for new tags - tags cache: paths: - .cache/pip .build_template: &build_template <<: *common_template stage: build before_script: - 'python -V # Print out python version for debugging' - df -h script: - python -m pip install setuptools>=0.8 wheel build twine - python -m build --wheel --outdir dist - python -m twine check ./dist/geowatch*.whl artifacts: paths: - ./dist/geowatch*.whl .common_test_template: &common_test_template <<: *common_template stage: test coverage: /TOTAL.+ ([0-9]{1,3}%)/ .test_full-loose_template: &test_full-loose_template <<: *common_test_template before_script: - |- # Setup the correct version of python (which should be the same as this instance) python --version # Print out python version for debugging export PYVER=$(python -c "import sys; print('{}{}'.format(*sys.version_info[0:2]))") python -m pip install virtualenv python -m virtualenv venv$PYVER source venv$PYVER/bin/activate pip install pip -U pip install pip setuptools -U pip install pygments python --version # Print out python version for debugging script: - export INSTALL_EXTRAS="tests,optional,headless" - echo "Finding the path to the wheel" - ls dist || echo "dist does not exist" - echo "Installing helpers" - pip install setuptools>=0.8 setuptools_scm wheel build -U - pip install tomli pkginfo - export WHEEL_FPATH=$(python -c "import pathlib; print(str(sorted(pathlib.Path('dist').glob('geowatch*.whl'))[-1]).replace(chr(92), chr(47)))") - export MOD_VERSION=$(python -c "from pkginfo import Wheel; print(Wheel('$WHEEL_FPATH').version)") - pip install -r requirements/gdal.txt - pip install --prefer-binary "geowatch[$INSTALL_EXTRAS]==$MOD_VERSION" -f dist - echo "Install finished." - echo "Creating test sandbox directory" - export WORKSPACE_DNAME="sandbox" - echo "WORKSPACE_DNAME=$WORKSPACE_DNAME" - mkdir -p $WORKSPACE_DNAME - echo "cd-ing into the workspace" - cd $WORKSPACE_DNAME - pwd - ls -altr - '# Get the path to the installed package and run the tests' - export MOD_DPATH=$(python -c "import geowatch, os; print(os.path.dirname(geowatch.__file__))") - export MOD_NAME=geowatch - |- echo " --- MOD_DPATH = $MOD_DPATH --- running the pytest command inside the workspace --- " - |- python -m pytest --verbose -p pytester -p no:doctest --xdoctest --cov-config ../pyproject.toml --cov-report term --cov="$MOD_NAME" "$MOD_DPATH" ../tests --durations=100 .test_full-strict_template: &test_full-strict_template <<: *common_test_template before_script: - |- # Setup the correct version of python (which should be the same as this instance) python --version # Print out python version for debugging export PYVER=$(python -c "import sys; print('{}{}'.format(*sys.version_info[0:2]))") python -m pip install virtualenv python -m virtualenv venv$PYVER source venv$PYVER/bin/activate pip install pip -U pip install pip setuptools -U pip install pygments python --version # Print out python version for debugging script: - export INSTALL_EXTRAS="tests-strict,runtime-strict,optional-strict,headless-strict" - echo "Finding the path to the wheel" - ls dist || echo "dist does not exist" - echo "Installing helpers" - pip install setuptools>=0.8 setuptools_scm wheel build -U - pip install tomli pkginfo - export WHEEL_FPATH=$(python -c "import pathlib; print(str(sorted(pathlib.Path('dist').glob('geowatch*.whl'))[-1]).replace(chr(92), chr(47)))") - export MOD_VERSION=$(python -c "from pkginfo import Wheel; print(Wheel('$WHEEL_FPATH').version)") - pip install -r requirements/gdal.txt - pip install --prefer-binary "geowatch[$INSTALL_EXTRAS]==$MOD_VERSION" -f dist - echo "Install finished." - echo "Creating test sandbox directory" - export WORKSPACE_DNAME="sandbox" - echo "WORKSPACE_DNAME=$WORKSPACE_DNAME" - mkdir -p $WORKSPACE_DNAME - echo "cd-ing into the workspace" - cd $WORKSPACE_DNAME - pwd - ls -altr - '# Get the path to the installed package and run the tests' - export MOD_DPATH=$(python -c "import geowatch, os; print(os.path.dirname(geowatch.__file__))") - export MOD_NAME=geowatch - |- echo " --- MOD_DPATH = $MOD_DPATH --- running the pytest command inside the workspace --- " - |- python -m pytest --verbose -p pytester -p no:doctest --xdoctest --cov-config ../pyproject.toml --cov-report term --cov="$MOD_NAME" "$MOD_DPATH" ../tests --durations=100 build/cp310-linux-x86_64: <<: *build_template image: python:3.10 test/full-loose/cp310-linux-x86_64: <<: *test_full-loose_template image: python:3.10 needs: - build/cp310-linux-x86_64 test/full-strict/cp310-linux-x86_64: <<: *test_full-strict_template image: python:3.10 needs: - build/cp310-linux-x86_64 build/cp311-linux-x86_64: <<: *build_template image: python:3.11 test/full-loose/cp311-linux-x86_64: <<: *test_full-loose_template image: python:3.11 needs: - build/cp311-linux-x86_64 test/full-strict/cp311-linux-x86_64: <<: *test_full-strict_template image: python:3.11 needs: - build/cp311-linux-x86_64 lint: <<: *common_template image: python:3.11 stage: lint before_script: - df -h script: - pip install -r requirements/linting.txt - ./run_linter.sh allow_failure: true gpgsign/wheels: <<: *common_template image: python:3.11 stage: gpgsign artifacts: paths: - dist/*.asc - dist/*.whl - dist/*.ots #variables: # IS_FOR_RELEASE: "true" #rules: # - if: '$CI_COMMIT_BRANCH == "main"' # when: on_success # - if: '$CI_COMMIT_BRANCH == "release"' # when: on_success only: refs: # Gitlab will only expose protected variables on protected branches # (which I've set to be main and release), so only run this stage # there. - master - main - release needs: - job: build/cp310-linux-x86_64 artifacts: true - job: build/cp311-linux-x86_64 artifacts: true script: - ls dist - export GPG_EXECUTABLE=gpg - export GPG_KEYID=$(cat dev/public_gpg_key) - echo "GPG_KEYID = $GPG_KEYID" # Decrypt and import GPG Keys / trust # note the variable pointed to by VARNAME_CI_SECRET is a protected variables only available on main and release branch - source dev/secrets_configuration.sh - CI_SECRET=${!VARNAME_CI_SECRET} - $GPG_EXECUTABLE --version - openssl version - $GPG_EXECUTABLE --list-keys # note CI_KITWARE_SECRET is a protected variables only available on main and release branch - CIS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:CIS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - CIS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:CIS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust - CIS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:CIS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import - GPG_SIGN_CMD="$GPG_EXECUTABLE --batch --yes --detach-sign --armor --local-user $GPG_KEYID" - |- WHEEL_PATHS=(dist/*.whl) WHEEL_PATHS_STR=$(printf '"%s" ' "${WHEEL_PATHS[@]}") echo "$WHEEL_PATHS_STR" for WHEEL_PATH in "${WHEEL_PATHS[@]}" do echo "------" echo "WHEEL_PATH = $WHEEL_PATH" $GPG_SIGN_CMD --output $WHEEL_PATH.asc $WHEEL_PATH $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH || echo "hack, the first run of gpg very fails" $GPG_EXECUTABLE --verify $WHEEL_PATH.asc $WHEEL_PATH done - ls dist - python -m pip install opentimestamps-client - ots stamp dist/*.whl dist/*.asc deploy/wheels: <<: *common_template image: python:3.11 stage: deploy #variables: # IS_FOR_RELEASE: "true" #rules: # #- if: '$CI_COMMIT_BRANCH == "main"' # # when: on_success # - if: '$CI_COMMIT_BRANCH == "release"' # when: on_success only: refs: - release script: - pip install pyopenssl ndg-httpsclient pyasn1 requests[security] twine -U - ls dist - |- WHEEL_PATHS=(dist/*.whl) WHEEL_PATHS_STR=$(printf '"%s" ' "${WHEEL_PATHS[@]}") source dev/secrets_configuration.sh TWINE_PASSWORD=${!VARNAME_TWINE_PASSWORD} TWINE_USERNAME=${!VARNAME_TWINE_USERNAME} echo "$WHEEL_PATHS_STR" for WHEEL_PATH in "${WHEEL_PATHS[@]}" do twine check $WHEEL_PATH.asc $WHEEL_PATH twine upload --username $TWINE_USERNAME --password $TWINE_PASSWORD $WHEEL_PATH.asc $WHEEL_PATH || echo "upload already exists" done - |- # Have the server git-tag the release and push the tags export VERSION=$(python -c "import setup; print(setup.VERSION)") # do sed twice to handle the case of https clone with and without a read token URL_HOST=$(git remote get-url origin | sed -e 's|https\?://.*@||g' | sed -e 's|https\?://||g' | sed -e 's|git@||g' | sed -e 's|:|/|g') source dev/secrets_configuration.sh CI_SECRET=${!VARNAME_CI_SECRET} PUSH_TOKEN=${!VARNAME_PUSH_TOKEN} echo "URL_HOST = $URL_HOST" # A git config user name and email is required. Set if needed. if [[ "$(git config user.email)" == "" ]]; then git config user.email "ci@gitlab.org.com" git config user.name "Gitlab-CI" fi TAG_NAME="v${VERSION}" echo "TAG_NAME = $TAG_NAME" if [ $(git tag -l "$TAG_NAME") ]; then echo "Tag already exists" else # if we messed up we can delete the tag # git push origin :refs/tags/$TAG_NAME # and then tag with -f git tag $TAG_NAME -m "tarball tag $VERSION" git push --tags "https://git-push-token:${PUSH_TOKEN}@${URL_HOST}" fi - |- # https://docs.gitlab.com/ee/ci/variables/predefined_variables.html echo "CI_PROJECT_URL=$CI_PROJECT_URL" echo "CI_PROJECT_ID=$CI_PROJECT_ID" echo "CI_PROJECT_NAME=$CI_PROJECT_NAME" echo "CI_PROJECT_NAMESPACE=$CI_PROJECT_NAMESPACE" echo "CI_API_V4_URL=$CI_API_V4_URL" export PROJECT_VERSION=$(python -c "import setup; print(setup.VERSION)") echo "PROJECT_VERSION=$PROJECT_VERSION" # --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" \ for FPATH in "dist"/*; do FNAME=$(basename $FPATH) echo $FNAME curl \ --header "JOB-TOKEN: $CI_JOB_TOKEN" \ --upload-file $FPATH \ "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/$CI_PROJECT_NAME/$PROJECT_VERSION/$FNAME" done # end