#!/usr/bin/env bash __doc__=' Script to publish a new version of this library on PyPI. If your script has binary dependencies then we assume that you have built a proper binary wheel with auditwheel and it exists in the wheelhouse directory. Otherwise, for source tarballs and wheels this script runs the setup.py script to create the wheels as well. Running this script with the default arguments will perform any builds and gpg signing, but nothing will be uploaded to pypi unless the user explicitly sets DO_UPLOAD=True or answers yes to the prompts. Args: TWINE_USERNAME (str) : username for pypi. This must be set if uploading to pypi. Defaults to "". TWINE_PASSWORD (str) : password for pypi. This must be set if uploading to pypi. Defaults to "". DO_GPG (bool) : If True, sign the packages with a GPG key specified by `GPG_KEYID`. defaults to auto. DO_OTS (bool) : If True, make an opentimestamp for the package and signature (if available) DO_UPLOAD (bool) : If True, upload the packages to the pypi server specified by `TWINE_REPOSITORY_URL`. DO_BUILD (bool) : If True, will execute the setup.py build script, which is expected to use setuptools. In the future we may add support for other build systems. If False, this script will expect the pre-built packages to exist in "wheelhouse/{NAME}-{VERSION}-{SUFFIX}.{EXT}". Defaults to "auto". DO_TAG (bool) : if True, will "git tag" the current HEAD with TWINE_REPOSITORY_URL (url) : The URL of the pypi server to upload to. Defaults to "auto", which if on the release branch, this will default to the live pypi server `https://upload.pypi.org/legacy` otherwise this will default to the test.pypi server: `https://test.pypi.org/legacy` GPG_KEYID (str) : The keyid of the gpg key to sign with. (if DO_GPG=True). Defaults to the local git config user.signingkey DEPLOY_REMOTE (str) : The git remote to push any tags to. Defaults to "origin" GPG_EXECUTABLE (path) : Path to the GPG executable. Defaults to "auto", which chooses "gpg2" if it exists, otherwise "gpg". MODE (str): Can be pure, binary, or all. Defaults to pure unless a CMakeLists.txt exists in which case it defaults to binary. Requirements: twine >= 1.13.0 gpg2 >= 2.2.4 OpenSSL >= 1.1.1c Notes: # NEW API TO UPLOAD TO PYPI # https://docs.travis-ci.com/user/deployment/pypi/ # https://packaging.python.org/tutorials/distributing-packages/ # https://stackoverflow.com/questions/45188811/how-to-gpg-sign-a-file-that-is-built-by-travis-ci Based on template in # github.com/Erotemic/xcookie/ ~/code/xcookie/publish.sh Usage: load_secrets # TODO: set a trap to unload secrets? cd # Set your variables or load your secrets export TWINE_USERNAME= export TWINE_PASSWORD= TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/" ' DEBUG=${DEBUG:=''} if [[ "${DEBUG}" != "" ]]; then set -x fi check_variable(){ KEY=$1 HIDE=$2 VAL=${!KEY} if [[ "$HIDE" == "" ]]; then echo "[DEBUG] CHECK VARIABLE: $KEY=\"$VAL\"" else echo "[DEBUG] CHECK VARIABLE: $KEY=" fi if [[ "$VAL" == "" ]]; then echo "[ERROR] UNSET VARIABLE: $KEY=\"$VAL\"" exit 1; fi } normalize_boolean(){ ARG=$1 ARG=$(echo "$ARG" | awk '{print tolower($0)}') if [ "$ARG" = "true" ] || [ "$ARG" = "1" ] || [ "$ARG" = "yes" ] || [ "$ARG" = "y" ] || [ "$ARG" = "on" ]; then echo "True" elif [ "$ARG" = "false" ] || [ "$ARG" = "0" ] || [ "$ARG" = "no" ] || [ "$ARG" = "n" ] || [ "$ARG" = "off" ]; then echo "False" else echo "$ARG" fi } #### # Parameters ### # Options DEPLOY_REMOTE=${DEPLOY_REMOTE:=origin} NAME=${NAME:=$(python -c "import setup; print(setup.NAME)")} VERSION=$(python -c "import setup; print(setup.VERSION)") check_variable DEPLOY_REMOTE ARG_1=$1 DO_UPLOAD=${DO_UPLOAD:=$ARG_1} DO_TAG=${DO_TAG:=$ARG_1} DO_GPG=${DO_GPG:="auto"} if [ "$DO_GPG" == "auto" ]; then DO_GPG="True" fi DO_OTS=${DO_OTS:="auto"} if [ "$DO_OTS" == "auto" ]; then # Do opentimestamp if it is available # python -m pip install opentimestamps-client if type ots ; then DO_OTS="True" else DO_OTS="False" fi fi DO_BUILD=${DO_BUILD:="auto"} # Verify that we want to build if [ "$DO_BUILD" == "auto" ]; then DO_BUILD="True" fi DO_GPG=$(normalize_boolean "$DO_GPG") DO_OTS=$(normalize_boolean "$DO_OTS") DO_BUILD=$(normalize_boolean "$DO_BUILD") DO_UPLOAD=$(normalize_boolean "$DO_UPLOAD") DO_TAG=$(normalize_boolean "$DO_TAG") TWINE_USERNAME=${TWINE_USERNAME:=""} TWINE_PASSWORD=${TWINE_PASSWORD:=""} DEFAULT_TEST_TWINE_REPO_URL="https://test.pypi.org/legacy/" DEFAULT_LIVE_TWINE_REPO_URL="https://upload.pypi.org/legacy/" TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL:="auto"} if [[ "${TWINE_REPOSITORY_URL}" == "auto" ]]; then #if [[ "$(cat .git/HEAD)" != "ref: refs/heads/release" ]]; then # # If we are not on release, then default to the test pypi upload repo # TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL:="https://test.pypi.org/legacy/"} #else if [[ "$DEBUG" == "" ]]; then TWINE_REPOSITORY_URL="live" else TWINE_REPOSITORY_URL="test" fi fi if [[ "${TWINE_REPOSITORY_URL}" == "live" ]]; then TWINE_REPOSITORY_URL=$DEFAULT_LIVE_TWINE_REPO_URL elif [[ "${TWINE_REPOSITORY_URL}" == "test" ]]; then TWINE_REPOSITORY_URL=$DEFAULT_TEST_TWINE_REPO_URL fi GPG_EXECUTABLE=${GPG_EXECUTABLE:="auto"} if [[ "$GPG_EXECUTABLE" == "auto" ]]; then if [[ "$(which gpg2)" != "" ]]; then GPG_EXECUTABLE="gpg2" else GPG_EXECUTABLE="gpg" fi fi GPG_KEYID=${GPG_KEYID:="auto"} if [[ "$GPG_KEYID" == "auto" ]]; then GPG_KEYID=$(git config --local user.signingkey) if [[ "$GPG_KEYID" == "" ]]; then GPG_KEYID=$(git config --global user.signingkey) fi fi if [ -f CMakeLists.txt ] ; then DEFAULT_MODE="binary" else DEFAULT_MODE="pure" fi # TODO: parameterize # The default should change depending on the application MODE=${MODE:=$DEFAULT_MODE} if [[ "$MODE" == "all" ]]; then MODE_LIST=("sdist" "native" "bdist") elif [[ "$MODE" == "pure" ]]; then MODE_LIST=("sdist" "native") elif [[ "$MODE" == "binary" ]]; then MODE_LIST=("sdist" "bdist") else MODE_LIST=("$MODE") fi MODE_LIST_STR=$(printf '"%s" ' "${MODE_LIST[@]}") #echo "MODE_LIST_STR = $MODE_LIST_STR" #### # Logic ### WAS_INTERACTION="False" echo " === PYPI BUILDING SCRIPT == NAME='$NAME' VERSION='$VERSION' TWINE_USERNAME='$TWINE_USERNAME' TWINE_REPOSITORY_URL = $TWINE_REPOSITORY_URL GPG_KEYID = '$GPG_KEYID' DO_UPLOAD=${DO_UPLOAD} DO_TAG=${DO_TAG} DO_GPG=${DO_GPG} DO_OTS=${DO_OTS} DO_BUILD=${DO_BUILD} MODE_LIST_STR=${MODE_LIST_STR} " # Verify that we want to tag if [[ "$DO_TAG" == "True" ]]; then echo "About to tag VERSION='$VERSION'" else if [[ "$DO_TAG" == "False" ]]; then echo "We are NOT about to tag VERSION='$VERSION'" else # shellcheck disable=SC2162 read -p "Do you want to git tag and push version='$VERSION'? (input 'yes' to confirm)" ANS echo "ANS = $ANS" WAS_INTERACTION="True" DO_TAG="$ANS" DO_TAG=$(normalize_boolean "$DO_TAG") if [ "$DO_BUILD" == "auto" ]; then DO_BUILD="" DO_GPG="" fi fi fi if [[ "$DO_BUILD" == "True" ]]; then echo "About to build wheels" else if [[ "$DO_BUILD" == "False" ]]; then echo "We are NOT about to build wheels" else # shellcheck disable=SC2162 read -p "Do you need to build wheels? (input 'yes' to confirm)" ANS echo "ANS = $ANS" WAS_INTERACTION="True" DO_BUILD="$ANS" DO_BUILD=$(normalize_boolean "$DO_BUILD") fi fi # Verify that we want to publish if [[ "$DO_UPLOAD" == "True" ]]; then echo "About to directly publish VERSION='$VERSION'" else if [[ "$DO_UPLOAD" == "False" ]]; then echo "We are NOT about to directly publish VERSION='$VERSION'" else # shellcheck disable=SC2162 read -p "Are you ready to directly publish version='$VERSION'? ('yes' will twine upload)" ANS echo "ANS = $ANS" WAS_INTERACTION="True" DO_UPLOAD="$ANS" DO_UPLOAD=$(normalize_boolean "$DO_UPLOAD") fi fi if [[ "$WAS_INTERACTION" == "True" ]]; then echo " === PYPI BUILDING SCRIPT == VERSION='$VERSION' TWINE_USERNAME='$TWINE_USERNAME' TWINE_REPOSITORY_URL = $TWINE_REPOSITORY_URL GPG_KEYID = '$GPG_KEYID' DO_UPLOAD=${DO_UPLOAD} DO_TAG=${DO_TAG} DO_GPG=${DO_GPG} DO_BUILD=${DO_BUILD} MODE_LIST_STR='${MODE_LIST_STR}' " # shellcheck disable=SC2162 read -p "Look good? Ready to build? Enter any text to continue" ANS fi if [ "$DO_BUILD" == "True" ]; then echo " === === " echo "LIVE BUILDING" # Build wheel and source distribution for _MODE in "${MODE_LIST[@]}" do echo "_MODE = $_MODE" if [[ "$_MODE" == "sdist" ]]; then python setup.py sdist || { echo 'failed to build sdist wheel' ; exit 1; } elif [[ "$_MODE" == "native" ]]; then python setup.py bdist_wheel || { echo 'failed to build native wheel' ; exit 1; } elif [[ "$_MODE" == "bdist" ]]; then echo "Assume wheel has already been built" else echo "ERROR: bad mode" exit 1 fi done echo " === === " else echo "DO_BUILD=False, Skipping build" fi ls_array(){ __doc__=' Read the results of a glob pattern into an array Args: arr_name glob_pattern Example: arr_name="myarray" glob_pattern="*" pass ' local arr_name="$1" local glob_pattern="$2" shopt -s nullglob # shellcheck disable=SC2206 array=($glob_pattern) shopt -u nullglob # Turn off nullglob to make sure it doesn't interfere with anything later # FIXME; for some reason this doesnt always work properly # Copy the array into the dynamically named variable # shellcheck disable=SC2086 readarray -t $arr_name < <(printf '%s\n' "${array[@]}") } WHEEL_FPATHS=() for _MODE in "${MODE_LIST[@]}" do if [[ "$_MODE" == "sdist" ]]; then ls_array "_NEW_WHEEL_PATHS" "dist/${NAME}-${VERSION}*.tar.gz" elif [[ "$_MODE" == "native" ]]; then ls_array "_NEW_WHEEL_PATHS" "dist/${NAME}-${VERSION}*.whl" elif [[ "$_MODE" == "bdist" ]]; then ls_array "_NEW_WHEEL_PATHS" "wheelhouse/${NAME}-${VERSION}-*.whl" else echo "ERROR: bad mode" exit 1 fi # hacky CONCAT because for some reason ls_array will return # something that looks empty but has one empty element for new_item in "${_NEW_WHEEL_PATHS[@]}" do if [[ "$new_item" != "" ]]; then WHEEL_FPATHS+=("$new_item") fi done done # Dedup the paths readarray -t WHEEL_FPATHS < <(printf '%s\n' "${WHEEL_FPATHS[@]}" | sort -u) WHEEL_PATHS_STR=$(printf '"%s" ' "${WHEEL_FPATHS[@]}") echo "WHEEL_PATHS_STR = $WHEEL_PATHS_STR" echo " MODE=$MODE VERSION='$VERSION' WHEEL_FPATHS='$WHEEL_PATHS_STR' " WHEEL_SIGNATURE_FPATHS=() if [ "$DO_GPG" == "True" ]; then echo " === === " for WHEEL_FPATH in "${WHEEL_FPATHS[@]}" do echo "WHEEL_FPATH = $WHEEL_FPATH" check_variable WHEEL_FPATH # https://stackoverflow.com/questions/45188811/how-to-gpg-sign-a-file-that-is-built-by-travis-ci # secure gpg --export-secret-keys > all.gpg # REQUIRES GPG >= 2.2 check_variable GPG_EXECUTABLE || { echo 'failed no gpg exe' ; exit 1; } check_variable GPG_KEYID || { echo 'failed no gpg key' ; exit 1; } echo "Signing wheels" GPG_SIGN_CMD="$GPG_EXECUTABLE --batch --yes --detach-sign --armor --local-user $GPG_KEYID" echo "GPG_SIGN_CMD = $GPG_SIGN_CMD" $GPG_SIGN_CMD --output "$WHEEL_FPATH".asc "$WHEEL_FPATH" echo "Checking wheels" twine check "$WHEEL_FPATH".asc "$WHEEL_FPATH" || { echo 'could not check wheels' ; exit 1; } echo "Verifying wheels" $GPG_EXECUTABLE --verify "$WHEEL_FPATH".asc "$WHEEL_FPATH" || { echo 'could not verify wheels' ; exit 1; } WHEEL_SIGNATURE_FPATHS+=("$WHEEL_FPATH".asc) done echo " === === " else echo "DO_GPG=False, Skipping GPG sign" fi if [ "$DO_OTS" == "True" ]; then echo " === === " if [ "$DO_GPG" == "True" ]; then # Stamp the wheels and the signatures ots stamp "${WHEEL_FPATHS[@]}" "${WHEEL_SIGNATURE_FPATHS[@]}" else # Stamp only the wheels ots stamp "${WHEEL_FPATHS[@]}" fi echo " === === " else echo "DO_OTS=False, Skipping OTS sign" fi if [[ "$DO_TAG" == "True" ]]; then TAG_NAME="v${VERSION}" # if we messed up we can delete the tag # git push origin :refs/tags/$TAG_NAME # and then tag with -f # git tag "$TAG_NAME" -m "tarball tag $VERSION" git push --tags "$DEPLOY_REMOTE" echo "Should also do a: git push $DEPLOY_REMOTE main:release" echo "For github should draft a new release: https://github.com/PyUtils/line_profiler/releases/new" else echo "Not tagging" fi if [[ "$DO_UPLOAD" == "True" ]]; then check_variable TWINE_USERNAME check_variable TWINE_PASSWORD "hide" for WHEEL_FPATH in "${WHEEL_FPATHS[@]}" do twine upload --username "$TWINE_USERNAME" "--password=$TWINE_PASSWORD" \ --repository-url "$TWINE_REPOSITORY_URL" \ "$WHEEL_FPATH" --skip-existing --verbose || { echo 'failed to twine upload' ; exit 1; } done echo """ !!! FINISH: LIVE RUN !!! """ else echo """ DRY RUN ... Skipping upload DEPLOY_REMOTE = '$DEPLOY_REMOTE' DO_UPLOAD = '$DO_UPLOAD' WHEEL_FPATH = '$WHEEL_FPATH' WHEEL_PATHS_STR = '$WHEEL_PATHS_STR' MODE_LIST_STR = '$MODE_LIST_STR' VERSION='$VERSION' NAME='$NAME' TWINE_USERNAME='$TWINE_USERNAME' GPG_KEYID = '$GPG_KEYID' To do live run set DO_UPLOAD=1 and ensure deploy and current branch are the same !!! FINISH: DRY RUN !!! """ fi __devel__=' # Checking to see how easy it is to upload packages to gitlab. # This logic should go in the CI script, not sure if it belongs here. export HOST=https://gitlab.kitware.com export GROUP_NAME=computer-vision export PROJECT_NAME=geowatch PROJECT_VERSION=$(geowatch --version) echo "$PROJECT_VERSION" load_secrets export PRIVATE_GITLAB_TOKEN=$(git_token_for "$HOST") TMP_DIR=$(mktemp -d -t ci-XXXXXXXXXX) curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/groups" > "$TMP_DIR/all_group_info" GROUP_ID=$(cat "$TMP_DIR/all_group_info" | jq ". | map(select(.name==\"$GROUP_NAME\")) | .[0].id") echo "GROUP_ID = $GROUP_ID" curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/groups/$GROUP_ID" > "$TMP_DIR/group_info" PROJ_ID=$(cat "$TMP_DIR/group_info" | jq ".projects | map(select(.name==\"$PROJECT_NAME\")) | .[0].id") echo "PROJ_ID = $PROJ_ID" ls_array DIST_FPATHS "dist/*" for FPATH in "${DIST_FPATHS[@]}" do FNAME=$(basename $FPATH) echo $FNAME curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" \ --upload-file $FPATH \ "https://gitlab.kitware.com/api/v4/projects/$PROJ_ID/packages/generic/$PROJECT_NAME/$PROJECT_VERSION/$FNAME" done '