From 91979285ebe562f085fba92a45616a43a97e01fb Mon Sep 17 00:00:00 2001 From: "Xinwei Xiong(cubxxw-openim)" <3293172751nss@gmail.com> Date: Tue, 4 Jul 2023 10:28:19 +0800 Subject: [PATCH] feat: add scripts lib Signed-off-by: Xinwei Xiong(cubxxw-openim) <3293172751nss@gmail.com> --- scripts/lib/color.sh | 38 ++ scripts/lib/golang.sh | 193 ++++++++++ scripts/lib/init.sh | 28 ++ scripts/lib/logging.sh | 161 +++++++++ scripts/lib/release.sh | 597 ++++++++++++++++++++++++++++++ scripts/lib/util.sh | 683 +++++++++++++++++++++++++++++++++++ scripts/lib/version.sh | 137 +++++++ scripts/make-rules/golang.mk | 11 +- 8 files changed, 1839 insertions(+), 9 deletions(-) create mode 100755 scripts/lib/color.sh create mode 100755 scripts/lib/golang.sh create mode 100755 scripts/lib/init.sh create mode 100755 scripts/lib/logging.sh create mode 100755 scripts/lib/release.sh create mode 100755 scripts/lib/util.sh create mode 100755 scripts/lib/version.sh diff --git a/scripts/lib/color.sh b/scripts/lib/color.sh new file mode 100755 index 000000000..bdbe0e843 --- /dev/null +++ b/scripts/lib/color.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Copyright 2020 Lingfei Kong . All rights reserved. +# Use of this source code is governed by a MIT style +# license that can be found in the LICENSE file. + +#Define color variables +#Feature +COLOR_NORMAL='\033[0m';COLOR_BOLD='\033[1m';COLOR_DIM='\033[2m';COLOR_UNDER='\033[4m'; +COLOR_ITALIC='\033[3m';COLOR_NOITALIC='\033[23m';COLOR_BLINK='\033[5m'; +COLOR_REVERSE='\033[7m';COLOR_CONCEAL='\033[8m';COLOR_NOBOLD='\033[22m'; +COLOR_NOUNDER='\033[24m';COLOR_NOBLINK='\033[25m'; + +#Front color +COLOR_BLACK='\033[30m';COLOR_RED='\033[31m';COLOR_GREEN='\033[32m';COLOR_YELLOW='\033[33m'; +COLOR_BLUE='\033[34m';COLOR_MAGENTA='\033[35m';COLOR_CYAN='\033[36m';COLOR_WHITE='\033[37m'; + +#background color +COLOR_BBLACK='\033[40m';COLOR_BRED='\033[41m'; +COLOR_BGREEN='\033[42m';COLOR_BYELLOW='\033[43m'; +COLOR_BBLUE='\033[44m';COLOR_BMAGENTA='\033[45m'; +COLOR_BCYAN='\033[46m';COLOR_BWHITE='\033[47m'; + +# Print colors you can use +iam::color::print_color() +{ + echo + echo -e ${bmagenta}--back-color:${normal} + echo "bblack; bgreen; bblue; bcyan; bred; byellow; bmagenta; bwhite" + echo + echo -e ${red}--font-color:${normal} + echo "black; red; green; yellow; blue; magenta; cyan; white" + echo + echo -e ${bold}--font:${normal} + echo "normal; italic; reverse; nounder; bold; noitalic; conceal; noblink; + dim; blink; nobold; under" + echo +} diff --git a/scripts/lib/golang.sh b/scripts/lib/golang.sh new file mode 100755 index 000000000..f58e08649 --- /dev/null +++ b/scripts/lib/golang.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash + +# Copyright 2020 Lingfei Kong . All rights reserved. +# Use of this source code is governed by a MIT style +# license that can be found in the LICENSE file. + +# shellcheck disable=SC2034 # Variables sourced in other scripts. + +# The server platform we are building on. +readonly IAM_SUPPORTED_SERVER_PLATFORMS=( + linux/amd64 + linux/arm64 +) + +# If we update this we should also update the set of platforms whose standard +# library is precompiled for in build/build-image/cross/Dockerfile +readonly IAM_SUPPORTED_CLIENT_PLATFORMS=( + linux/amd64 + linux/arm64 +) + +# The set of server targets that we are only building for Linux +# If you update this list, please also update build/BUILD. +iam::golang::server_targets() { + local targets=( + iam-apiserver + iam-authz-server + iam-pump + iam-watcher + ) + echo "${targets[@]}" +} + +IFS=" " read -ra IAM_SERVER_TARGETS <<< "$(iam::golang::server_targets)" +readonly IAM_SERVER_TARGETS +readonly IAM_SERVER_BINARIES=("${IAM_SERVER_TARGETS[@]##*/}") + +# The set of server targets we build docker images for +iam::golang::server_image_targets() { + # NOTE: this contains cmd targets for iam::build::get_docker_wrapped_binaries + local targets=( + cmd/iam-apiserver + cmd/iam-authz-server + cmd/iam-pump + cmd/iam-watcher + ) + echo "${targets[@]}" +} + +IFS=" " read -ra IAM_SERVER_IMAGE_TARGETS <<< "$(iam::golang::server_image_targets)" +readonly IAM_SERVER_IMAGE_TARGETS +readonly IAM_SERVER_IMAGE_BINARIES=("${IAM_SERVER_IMAGE_TARGETS[@]##*/}") + +# ------------ +# NOTE: All functions that return lists should use newlines. +# bash functions can't return arrays, and spaces are tricky, so newline +# separators are the preferred pattern. +# To transform a string of newline-separated items to an array, use iam::util::read-array: +# iam::util::read-array FOO < <(iam::golang::dups a b c a) +# +# ALWAYS remember to quote your subshells. Not doing so will break in +# bash 4.3, and potentially cause other issues. +# ------------ + +# Returns a sorted newline-separated list containing only duplicated items. +iam::golang::dups() { + # We use printf to insert newlines, which are required by sort. + printf "%s\n" "$@" | sort | uniq -d +} + +# Returns a sorted newline-separated list with duplicated items removed. +iam::golang::dedup() { + # We use printf to insert newlines, which are required by sort. + printf "%s\n" "$@" | sort -u +} + +# Depends on values of user-facing IAM_BUILD_PLATFORMS, IAM_FASTBUILD, +# and IAM_BUILDER_OS. +# Configures IAM_SERVER_PLATFORMS and IAM_CLIENT_PLATFORMS, then sets them +# to readonly. +# The configured vars will only contain platforms allowed by the +# IAM_SUPPORTED* vars at the top of this file. +declare -a IAM_SERVER_PLATFORMS +declare -a IAM_CLIENT_PLATFORMS +iam::golang::setup_platforms() { + if [[ -n "${IAM_BUILD_PLATFORMS:-}" ]]; then + # IAM_BUILD_PLATFORMS needs to be read into an array before the next + # step, or quoting treats it all as one element. + local -a platforms + IFS=" " read -ra platforms <<< "${IAM_BUILD_PLATFORMS}" + + # Deduplicate to ensure the intersection trick with iam::golang::dups + # is not defeated by duplicates in user input. + iam::util::read-array platforms < <(iam::golang::dedup "${platforms[@]}") + + # Use iam::golang::dups to restrict the builds to the platforms in + # IAM_SUPPORTED_*_PLATFORMS. Items should only appear at most once in each + # set, so if they appear twice after the merge they are in the intersection. + iam::util::read-array IAM_SERVER_PLATFORMS < <(iam::golang::dups \ + "${platforms[@]}" \ + "${IAM_SUPPORTED_SERVER_PLATFORMS[@]}" \ + ) + readonly IAM_SERVER_PLATFORMS + + iam::util::read-array IAM_CLIENT_PLATFORMS < <(iam::golang::dups \ + "${platforms[@]}" \ + "${IAM_SUPPORTED_CLIENT_PLATFORMS[@]}" \ + ) + readonly IAM_CLIENT_PLATFORMS + + elif [[ "${IAM_FASTBUILD:-}" == "true" ]]; then + IAM_SERVER_PLATFORMS=(linux/amd64) + readonly IAM_SERVER_PLATFORMS + IAM_CLIENT_PLATFORMS=(linux/amd64) + readonly IAM_CLIENT_PLATFORMS + else + IAM_SERVER_PLATFORMS=("${IAM_SUPPORTED_SERVER_PLATFORMS[@]}") + readonly IAM_SERVER_PLATFORMS + + IAM_CLIENT_PLATFORMS=("${IAM_SUPPORTED_CLIENT_PLATFORMS[@]}") + readonly IAM_CLIENT_PLATFORMS + fi +} + +iam::golang::setup_platforms + +# The set of client targets that we are building for all platforms +# If you update this list, please also update build/BUILD. +readonly IAM_CLIENT_TARGETS=( + iamctl +) +readonly IAM_CLIENT_BINARIES=("${IAM_CLIENT_TARGETS[@]##*/}") + +readonly IAM_ALL_TARGETS=( + "${IAM_SERVER_TARGETS[@]}" + "${IAM_CLIENT_TARGETS[@]}" +) +readonly IAM_ALL_BINARIES=("${IAM_ALL_TARGETS[@]##*/}") + +# Asks golang what it thinks the host platform is. The go tool chain does some +# slightly different things when the target platform matches the host platform. +iam::golang::host_platform() { + echo "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" +} + +# Ensure the go tool exists and is a viable version. +iam::golang::verify_go_version() { + if [[ -z "$(command -v go)" ]]; then + iam::log::usage_from_stdin <. All rights reserved. +# Use of this source code is governed by a MIT style +# license that can be found in the LICENSE file. + +set -o errexit +set +o nounset +set -o pipefail + +# Unset CDPATH so that path interpolation can work correctly +# https://github.com/iamrnetes/iamrnetes/issues/52255 +unset CDPATH + +# Default use go modules +export GO111MODULE=on + +# The root of the build/dist directory +IAM_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" + +source "${IAM_ROOT}/scripts/lib/util.sh" +source "${IAM_ROOT}/scripts/lib/logging.sh" +source "${IAM_ROOT}/scripts/lib/color.sh" + +iam::log::install_errexit + +source "${IAM_ROOT}/scripts/lib/version.sh" +source "${IAM_ROOT}/scripts/lib/golang.sh" diff --git a/scripts/lib/logging.sh b/scripts/lib/logging.sh new file mode 100755 index 000000000..39b805c11 --- /dev/null +++ b/scripts/lib/logging.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash + +# Copyright 2020 Lingfei Kong . All rights reserved. +# Use of this source code is governed by a MIT style +# license that can be found in the LICENSE file. + +# Controls verbosity of the script output and logging. +IAM_VERBOSE="${IAM_VERBOSE:-5}" + +# Handler for when we exit automatically on an error. +# Borrowed from https://gist.github.com/ahendrix/7030300 +iam::log::errexit() { + local err="${PIPESTATUS[*]}" + + # If the shell we are in doesn't have errexit set (common in subshells) then + # don't dump stacks. + set +o | grep -qe "-o errexit" || return + + set +o xtrace + local code="${1:-1}" + # Print out the stack trace described by $function_stack + if [ ${#FUNCNAME[@]} -gt 2 ] + then + iam::log::error "Call tree:" + for ((i=1;i<${#FUNCNAME[@]}-1;i++)) + do + iam::log::error " ${i}: ${BASH_SOURCE[${i}+1]}:${BASH_LINENO[${i}]} ${FUNCNAME[${i}]}(...)" + done + fi + iam::log::error_exit "Error in ${BASH_SOURCE[1]}:${BASH_LINENO[0]}. '${BASH_COMMAND}' exited with status ${err}" "${1:-1}" 1 +} + +iam::log::install_errexit() { + # trap ERR to provide an error handler whenever a command exits nonzero this + # is a more verbose version of set -o errexit + trap 'iam::log::errexit' ERR + + # setting errtrace allows our ERR trap handler to be propagated to functions, + # expansions and subshells + set -o errtrace +} + +# Print out the stack trace +# +# Args: +# $1 The number of stack frames to skip when printing. +iam::log::stack() { + local stack_skip=${1:-0} + stack_skip=$((stack_skip + 1)) + if [[ ${#FUNCNAME[@]} -gt ${stack_skip} ]]; then + echo "Call stack:" >&2 + local i + for ((i=1 ; i <= ${#FUNCNAME[@]} - stack_skip ; i++)) + do + local frame_no=$((i - 1 + stack_skip)) + local source_file=${BASH_SOURCE[${frame_no}]} + local source_lineno=${BASH_LINENO[$((frame_no - 1))]} + local funcname=${FUNCNAME[${frame_no}]} + echo " ${i}: ${source_file}:${source_lineno} ${funcname}(...)" >&2 + done + fi +} + +# Log an error and exit. +# Args: +# $1 Message to log with the error +# $2 The error code to return +# $3 The number of stack frames to skip when printing. +iam::log::error_exit() { + local message="${1:-}" + local code="${2:-1}" + local stack_skip="${3:-0}" + stack_skip=$((stack_skip + 1)) + + if [[ ${IAM_VERBOSE} -ge 4 ]]; then + local source_file=${BASH_SOURCE[${stack_skip}]} + local source_line=${BASH_LINENO[$((stack_skip - 1))]} + echo "!!! Error in ${source_file}:${source_line}" >&2 + [[ -z ${1-} ]] || { + echo " ${1}" >&2 + } + + iam::log::stack ${stack_skip} + + echo "Exiting with status ${code}" >&2 + fi + + exit "${code}" +} + +# Log an error but keep going. Don't dump the stack or exit. +iam::log::error() { + timestamp=$(date +"[%m%d %H:%M:%S]") + echo "!!! ${timestamp} ${1-}" >&2 + shift + for message; do + echo " ${message}" >&2 + done +} + +# Print an usage message to stderr. The arguments are printed directly. +iam::log::usage() { + echo >&2 + local message + for message; do + echo "${message}" >&2 + done + echo >&2 +} + +iam::log::usage_from_stdin() { + local messages=() + while read -r line; do + messages+=("${line}") + done + + iam::log::usage "${messages[@]}" +} + +# Print out some info that isn't a top level status line +iam::log::info() { + local V="${V:-0}" + if [[ ${IAM_VERBOSE} < ${V} ]]; then + return + fi + + for message; do + echo "${message}" + done +} + +# Just like iam::log::info, but no \n, so you can make a progress bar +iam::log::progress() { + for message; do + echo -e -n "${message}" + done +} + +iam::log::info_from_stdin() { + local messages=() + while read -r line; do + messages+=("${line}") + done + + iam::log::info "${messages[@]}" +} + +# Print a status line. Formatted to show up in a stream of output. +iam::log::status() { + local V="${V:-0}" + if [[ ${IAM_VERBOSE} < ${V} ]]; then + return + fi + + timestamp=$(date +"[%m%d %H:%M:%S]") + echo "+++ ${timestamp} ${1}" + shift + for message; do + echo " ${message}" + done +} diff --git a/scripts/lib/release.sh b/scripts/lib/release.sh new file mode 100755 index 000000000..acc3730ef --- /dev/null +++ b/scripts/lib/release.sh @@ -0,0 +1,597 @@ +#!/usr/bin/env bash + +# Copyright 2020 Lingfei Kong . All rights reserved. +# Use of this source code is governed by a MIT style +# license that can be found in the LICENSE file. + +# This file creates release artifacts (tar files, container images) that are +# ready to distribute to install or distribute to end users. + +############################################################################### +# Most of the ::release:: namespace functions have been moved to +# github.com/iam/release. Have a look in that repo and specifically in +# lib/releaselib.sh for ::release::-related functionality. +############################################################################### + +# Tencent cos configuration +readonly BUCKET="marmotedu-1254073058" +readonly REGION="ap-beijing" +readonly COS_RELEASE_DIR="iam-release" +readonly COSTOOL="coscmd" + +# This is where the final release artifacts are created locally +readonly RELEASE_STAGE="${LOCAL_OUTPUT_ROOT}/release-stage" +readonly RELEASE_TARS="${LOCAL_OUTPUT_ROOT}/release-tars" +readonly RELEASE_IMAGES="${LOCAL_OUTPUT_ROOT}/release-images" + +# IAM github account info +readonly IAM_GITHUB_ORG=marmotedu +readonly IAM_GITHUB_REPO=iam + +readonly ARTIFACT=iam.tar.gz +readonly CHECKSUM=${ARTIFACT}.sha1sum + +IAM_BUILD_CONFORMANCE=${IAM_BUILD_CONFORMANCE:-y} +IAM_BUILD_PULL_LATEST_IMAGES=${IAM_BUILD_PULL_LATEST_IMAGES:-y} + +# Validate a ci version +# +# Globals: +# None +# Arguments: +# version +# Returns: +# If version is a valid ci version +# Sets: (e.g. for '1.2.3-alpha.4.56+abcdef12345678') +# VERSION_MAJOR (e.g. '1') +# VERSION_MINOR (e.g. '2') +# VERSION_PATCH (e.g. '3') +# VERSION_PRERELEASE (e.g. 'alpha') +# VERSION_PRERELEASE_REV (e.g. '4') +# VERSION_BUILD_INFO (e.g. '.56+abcdef12345678') +# VERSION_COMMITS (e.g. '56') +function iam::release::parse_and_validate_ci_version() { + # Accept things like "v1.2.3-alpha.4.56+abcdef12345678" or "v1.2.3-beta.4" + local -r version_regex="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-([a-zA-Z0-9]+)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*)\\+[0-9a-f]{7,40})?$" + local -r version="${1-}" + [[ "${version}" =~ ${version_regex} ]] || { + iam::log::error "Invalid ci version: '${version}', must match regex ${version_regex}" + return 1 + } + + # The VERSION variables are used when this file is sourced, hence + # the shellcheck SC2034 'appears unused' warning is to be ignored. + + # shellcheck disable=SC2034 + VERSION_MAJOR="${BASH_REMATCH[1]}" + # shellcheck disable=SC2034 + VERSION_MINOR="${BASH_REMATCH[2]}" + # shellcheck disable=SC2034 + VERSION_PATCH="${BASH_REMATCH[3]}" + # shellcheck disable=SC2034 + VERSION_PRERELEASE="${BASH_REMATCH[4]}" + # shellcheck disable=SC2034 + VERSION_PRERELEASE_REV="${BASH_REMATCH[5]}" + # shellcheck disable=SC2034 + VERSION_BUILD_INFO="${BASH_REMATCH[6]}" + # shellcheck disable=SC2034 + VERSION_COMMITS="${BASH_REMATCH[7]}" +} + +# --------------------------------------------------------------------------- +# Build final release artifacts +function iam::release::clean_cruft() { + # Clean out cruft + find "${RELEASE_STAGE}" -name '*~' -exec rm {} \; + find "${RELEASE_STAGE}" -name '#*#' -exec rm {} \; + find "${RELEASE_STAGE}" -name '.DS*' -exec rm {} \; +} + +function iam::release::package_tarballs() { + # Clean out any old releases + rm -rf "${RELEASE_STAGE}" "${RELEASE_TARS}" "${RELEASE_IMAGES}" + mkdir -p "${RELEASE_TARS}" + iam::release::package_src_tarball & + iam::release::package_client_tarballs & + iam::release::package_iam_manifests_tarball & + iam::release::package_server_tarballs & + iam::util::wait-for-jobs || { iam::log::error "previous tarball phase failed"; return 1; } + + iam::release::package_final_tarball & # _final depends on some of the previous phases + iam::util::wait-for-jobs || { iam::log::error "previous tarball phase failed"; return 1; } +} + +function iam::release::updload_tarballs() { + iam::log::info "upload ${RELEASE_TARS}/* to cos bucket ${BUCKET}." + for file in $(ls ${RELEASE_TARS}/*) + do + if [ "${COSTOOL}" == "coscli" ];then + coscli cp "${file}" "cos://${BUCKET}/${COS_RELEASE_DIR}/${IAM_GIT_VERSION}/${file##*/}" + coscli cp "${file}" "cos://${BUCKET}/${COS_RELEASE_DIR}/latest/${file##*/}" + else + coscmd upload "${file}" "${COS_RELEASE_DIR}/${IAM_GIT_VERSION}/" + coscmd upload "${file}" "${COS_RELEASE_DIR}/latest/" + fi + done +} + +# Package the source code we built, for compliance/licensing/audit/yadda. +function iam::release::package_src_tarball() { + local -r src_tarball="${RELEASE_TARS}/iam-src.tar.gz" + iam::log::status "Building tarball: src" + if [[ "${IAM_GIT_TREE_STATE-}" = 'clean' ]]; then + git archive -o "${src_tarball}" HEAD + else + find "${IAM_ROOT}" -mindepth 1 -maxdepth 1 \ + ! \( \ + \( -path "${IAM_ROOT}"/_\* -o \ + -path "${IAM_ROOT}"/.git\* -o \ + -path "${IAM_ROOT}"/.gitignore\* -o \ + -path "${IAM_ROOT}"/.gsemver.yaml\* -o \ + -path "${IAM_ROOT}"/.config\* -o \ + -path "${IAM_ROOT}"/.chglog\* -o \ + -path "${IAM_ROOT}"/.gitlint -o \ + -path "${IAM_ROOT}"/.golangci.yaml -o \ + -path "${IAM_ROOT}"/.goreleaser.yml -o \ + -path "${IAM_ROOT}"/.note.md -o \ + -path "${IAM_ROOT}"/.todo.md \ + \) -prune \ + \) -print0 \ + | "${TAR}" czf "${src_tarball}" --transform "s|${IAM_ROOT#/*}|iam|" --null -T - + fi +} + +# Package up all of the server binaries +function iam::release::package_server_tarballs() { + # Find all of the built client binaries + local long_platforms=("${LOCAL_OUTPUT_BINPATH}"/*/*) + if [[ -n ${IAM_BUILD_PLATFORMS-} ]]; then + read -ra long_platforms <<< "${IAM_BUILD_PLATFORMS}" + fi + + for platform_long in "${long_platforms[@]}"; do + local platform + local platform_tag + platform=${platform_long##${LOCAL_OUTPUT_BINPATH}/} # Strip LOCAL_OUTPUT_BINPATH + platform_tag=${platform/\//-} # Replace a "/" for a "-" + iam::log::status "Starting tarball: server $platform_tag" + + ( + local release_stage="${RELEASE_STAGE}/server/${platform_tag}/iam" + rm -rf "${release_stage}" + mkdir -p "${release_stage}/server/bin" + + local server_bins=("${IAM_SERVER_BINARIES[@]}") + + # This fancy expression will expand to prepend a path + # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the + # server_bins array. + cp "${server_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ + "${release_stage}/server/bin/" + + iam::release::clean_cruft + + local package_name="${RELEASE_TARS}/iam-server-${platform_tag}.tar.gz" + iam::release::create_tarball "${package_name}" "${release_stage}/.." + ) & + done + + iam::log::status "Waiting on tarballs" + iam::util::wait-for-jobs || { iam::log::error "server tarball creation failed"; exit 1; } + } + +# Package up all of the cross compiled clients. Over time this should grow into +# a full SDK +function iam::release::package_client_tarballs() { + # Find all of the built client binaries + local long_platforms=("${LOCAL_OUTPUT_BINPATH}"/*/*) + if [[ -n ${IAM_BUILD_PLATFORMS-} ]]; then + read -ra long_platforms <<< "${IAM_BUILD_PLATFORMS}" + fi + + for platform_long in "${long_platforms[@]}"; do + local platform + local platform_tag + platform=${platform_long##${LOCAL_OUTPUT_BINPATH}/} # Strip LOCAL_OUTPUT_BINPATH + platform_tag=${platform/\//-} # Replace a "/" for a "-" + iam::log::status "Starting tarball: client $platform_tag" + + ( + local release_stage="${RELEASE_STAGE}/client/${platform_tag}/iam" + rm -rf "${release_stage}" + mkdir -p "${release_stage}/client/bin" + + local client_bins=("${IAM_CLIENT_BINARIES[@]}") + + # This fancy expression will expand to prepend a path + # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the + # client_bins array. + cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ + "${release_stage}/client/bin/" + + iam::release::clean_cruft + + local package_name="${RELEASE_TARS}/iam-client-${platform_tag}.tar.gz" + iam::release::create_tarball "${package_name}" "${release_stage}/.." + ) & + done + + iam::log::status "Waiting on tarballs" + iam::util::wait-for-jobs || { iam::log::error "client tarball creation failed"; exit 1; } +} + +# Package up all of the server binaries in docker images +function iam::release::build_server_images() { + # Clean out any old images + rm -rf "${RELEASE_IMAGES}" + local platform + for platform in "${IAM_SERVER_PLATFORMS[@]}"; do + local platform_tag + local arch + platform_tag=${platform/\//-} # Replace a "/" for a "-" + arch=$(basename "${platform}") + iam::log::status "Building images: $platform_tag" + + local release_stage + release_stage="${RELEASE_STAGE}/server/${platform_tag}/iam" + rm -rf "${release_stage}" + mkdir -p "${release_stage}/server/bin" + + # This fancy expression will expand to prepend a path + # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the + # IAM_SERVER_IMAGE_BINARIES array. + cp "${IAM_SERVER_IMAGE_BINARIES[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ + "${release_stage}/server/bin/" + + iam::release::create_docker_images_for_server "${release_stage}/server/bin" "${arch}" + done +} + +function iam::release::md5() { + if which md5 >/dev/null 2>&1; then + md5 -q "$1" + else + md5sum "$1" | awk '{ print $1 }' + fi +} + +function iam::release::sha1() { + if which sha1sum >/dev/null 2>&1; then + sha1sum "$1" | awk '{ print $1 }' + else + shasum -a1 "$1" | awk '{ print $1 }' + fi +} + +function iam::release::build_conformance_image() { + local -r arch="$1" + local -r registry="$2" + local -r version="$3" + local -r save_dir="${4-}" + iam::log::status "Building conformance image for arch: ${arch}" + ARCH="${arch}" REGISTRY="${registry}" VERSION="${version}" \ + make -C cluster/images/conformance/ build >/dev/null + + local conformance_tag + conformance_tag="${registry}/conformance-${arch}:${version}" + if [[ -n "${save_dir}" ]]; then + "${DOCKER[@]}" save "${conformance_tag}" > "${save_dir}/conformance-${arch}.tar" + fi + iam::log::status "Deleting conformance image ${conformance_tag}" + "${DOCKER[@]}" rmi "${conformance_tag}" &>/dev/null || true +} + +# This builds all the release docker images (One docker image per binary) +# Args: +# $1 - binary_dir, the directory to save the tared images to. +# $2 - arch, architecture for which we are building docker images. +function iam::release::create_docker_images_for_server() { + # Create a sub-shell so that we don't pollute the outer environment + ( + local binary_dir + local arch + local binaries + local images_dir + binary_dir="$1" + arch="$2" + binaries=$(iam::build::get_docker_wrapped_binaries "${arch}") + images_dir="${RELEASE_IMAGES}/${arch}" + mkdir -p "${images_dir}" + + # k8s.gcr.io is the constant tag in the docker archives, this is also the default for config scripts in GKE. + # We can use IAM_DOCKER_REGISTRY to include and extra registry in the docker archive. + # If we use IAM_DOCKER_REGISTRY="k8s.gcr.io", then the extra tag (same) is ignored, see release_docker_image_tag below. + local -r docker_registry="k8s.gcr.io" + # Docker tags cannot contain '+' + local docker_tag="${IAM_GIT_VERSION/+/_}" + if [[ -z "${docker_tag}" ]]; then + iam::log::error "git version information missing; cannot create Docker tag" + return 1 + fi + + # provide `--pull` argument to `docker build` if `IAM_BUILD_PULL_LATEST_IMAGES` + # is set to y or Y; otherwise try to build the image without forcefully + # pulling the latest base image. + local docker_build_opts + docker_build_opts= + if [[ "${IAM_BUILD_PULL_LATEST_IMAGES}" =~ [yY] ]]; then + docker_build_opts='--pull' + fi + + for wrappable in $binaries; do + + local binary_name=${wrappable%%,*} + local base_image=${wrappable##*,} + local binary_file_path="${binary_dir}/${binary_name}" + local docker_build_path="${binary_file_path}.dockerbuild" + local docker_file_path="${docker_build_path}/Dockerfile" + local docker_image_tag="${docker_registry}/${binary_name}-${arch}:${docker_tag}" + + iam::log::status "Starting docker build for image: ${binary_name}-${arch}" + ( + rm -rf "${docker_build_path}" + mkdir -p "${docker_build_path}" + ln "${binary_file_path}" "${docker_build_path}/${binary_name}" + ln "${IAM_ROOT}/build/nsswitch.conf" "${docker_build_path}/nsswitch.conf" + chmod 0644 "${docker_build_path}/nsswitch.conf" + cat < "${docker_file_path}" +FROM ${base_image} +COPY ${binary_name} /usr/local/bin/${binary_name} +EOF + # ensure /etc/nsswitch.conf exists so go's resolver respects /etc/hosts + if [[ "${base_image}" =~ busybox ]]; then + echo "COPY nsswitch.conf /etc/" >> "${docker_file_path}" + fi + + "${DOCKER[@]}" build ${docker_build_opts:+"${docker_build_opts}"} -q -t "${docker_image_tag}" "${docker_build_path}" >/dev/null + # If we are building an official/alpha/beta release we want to keep + # docker images and tag them appropriately. + local -r release_docker_image_tag="${IAM_DOCKER_REGISTRY-$docker_registry}/${binary_name}-${arch}:${IAM_DOCKER_IMAGE_TAG-$docker_tag}" + if [[ "${release_docker_image_tag}" != "${docker_image_tag}" ]]; then + iam::log::status "Tagging docker image ${docker_image_tag} as ${release_docker_image_tag}" + "${DOCKER[@]}" rmi "${release_docker_image_tag}" 2>/dev/null || true + "${DOCKER[@]}" tag "${docker_image_tag}" "${release_docker_image_tag}" 2>/dev/null + fi + "${DOCKER[@]}" save -o "${binary_file_path}.tar" "${docker_image_tag}" "${release_docker_image_tag}" + echo "${docker_tag}" > "${binary_file_path}.docker_tag" + rm -rf "${docker_build_path}" + ln "${binary_file_path}.tar" "${images_dir}/" + + iam::log::status "Deleting docker image ${docker_image_tag}" + "${DOCKER[@]}" rmi "${docker_image_tag}" &>/dev/null || true + ) & + done + + if [[ "${IAM_BUILD_CONFORMANCE}" =~ [yY] ]]; then + iam::release::build_conformance_image "${arch}" "${docker_registry}" \ + "${docker_tag}" "${images_dir}" & + fi + + iam::util::wait-for-jobs || { iam::log::error "previous Docker build failed"; return 1; } + iam::log::status "Docker builds done" + ) + +} + +# This will pack iam-system manifests files for distros such as COS. +function iam::release::package_iam_manifests_tarball() { + iam::log::status "Building tarball: manifests" + + local src_dir="${IAM_ROOT}/deployments" + + local release_stage="${RELEASE_STAGE}/manifests/iam" + rm -rf "${release_stage}" + + local dst_dir="${release_stage}" + mkdir -p "${dst_dir}" + cp -r ${src_dir}/* "${dst_dir}" + #cp "${src_dir}/iam-apiserver.yaml" "${dst_dir}" + #cp "${src_dir}/iam-authz-server.yaml" "${dst_dir}" + #cp "${src_dir}/iam-pump.yaml" "${dst_dir}" + #cp "${src_dir}/iam-watcher.yaml" "${dst_dir}" + #cp "${IAM_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh" + + iam::release::clean_cruft + + local package_name="${RELEASE_TARS}/iam-manifests.tar.gz" + iam::release::create_tarball "${package_name}" "${release_stage}/.." +} + +# This is all the platform-independent stuff you need to run/install iam. +# Arch-specific binaries will need to be downloaded separately (possibly by +# using the bundled cluster/get-iam-binaries.sh script). +# Included in this tarball: +# - Cluster spin up/down scripts and configs for various cloud providers +# - Tarballs for manifest configs that are ready to be uploaded +# - Examples (which may or may not still work) +# - The remnants of the docs/ directory +function iam::release::package_final_tarball() { + iam::log::status "Building tarball: final" + + # This isn't a "full" tarball anymore, but the release lib still expects + # artifacts under "full/iam/" + local release_stage="${RELEASE_STAGE}/full/iam" + rm -rf "${release_stage}" + mkdir -p "${release_stage}" + + mkdir -p "${release_stage}/client" + cat < "${release_stage}/client/README" +Client binaries are no longer included in the IAM final tarball. + +Run release/get-iam-binaries.sh to download client and server binaries. +EOF + + # We want everything in /scripts. + mkdir -p "${release_stage}/release" + cp -R "${IAM_ROOT}/scripts/release" "${release_stage}/" + cat < "${release_stage}/release/get-iam-binaries.sh" +#!/usr/bin/env bash + +# Copyright 2020 Lingfei Kong . All rights reserved. +# Use of this source code is governed by a MIT style +# license that can be found in the LICENSE file. + +# This file download iam client and server binaries from tencent cos bucket. + +os=linux arch=amd64 version=${IAM_GIT_VERSION} && wget https://${BUCKET}.cos.${REGION}.myqcloud.com/${COS_RELEASE_DIR}/\$version/{iam-client-\$os-\$arch.tar.gz,iam-server-\$os-\$arch.tar.gz} +EOF + chmod +x ${release_stage}/release/get-iam-binaries.sh + + mkdir -p "${release_stage}/server" + cp "${RELEASE_TARS}/iam-manifests.tar.gz" "${release_stage}/server/" + cat < "${release_stage}/server/README" +Server binary tarballs are no longer included in the IAM final tarball. + +Run release/get-iam-binaries.sh to download client and server binaries. +EOF + + # Include hack/lib as a dependency for the cluster/ scripts + #mkdir -p "${release_stage}/hack" + #cp -R "${IAM_ROOT}/hack/lib" "${release_stage}/hack/" + + cp -R ${IAM_ROOT}/{docs,configs,scripts,deployments,init,README.md,LICENSE} "${release_stage}/" + + echo "${IAM_GIT_VERSION}" > "${release_stage}/version" + + iam::release::clean_cruft + + local package_name="${RELEASE_TARS}/${ARTIFACT}" + iam::release::create_tarball "${package_name}" "${release_stage}/.." +} + +# Build a release tarball. $1 is the output tar name. $2 is the base directory +# of the files to be packaged. This assumes that ${2}/iamis what is +# being packaged. +function iam::release::create_tarball() { + iam::build::ensure_tar + + local tarfile=$1 + local stagingdir=$2 + + "${TAR}" czf "${tarfile}" -C "${stagingdir}" iam --owner=0 --group=0 +} + +function iam::release::install_github_release(){ + GO111MODULE=on go install github.com/github-release/github-release@latest +} + +# Require the following tools: +# - github-release +# - gsemver +# - git-chglog +# - coscmd or coscli +function iam::release::verify_prereqs(){ + if [ -z "$(which github-release 2>/dev/null)" ]; then + iam::log::info "'github-release' tool not installed, try to install it." + + if ! iam::release::install_github_release; then + iam::log::error "failed to install 'github-release'" + return 1 + fi + fi + + if [ -z "$(which git-chglog 2>/dev/null)" ]; then + iam::log::info "'git-chglog' tool not installed, try to install it." + + if ! go install github.com/git-chglog/git-chglog/cmd/git-chglog@latest &>/dev/null; then + iam::log::error "failed to install 'git-chglog'" + return 1 + fi + fi + + if [ -z "$(which gsemver 2>/dev/null)" ]; then + iam::log::info "'gsemver' tool not installed, try to install it." + + if ! go install github.com/arnaud-deprez/gsemver@latest &>/dev/null; then + iam::log::error "failed to install 'gsemver'" + return 1 + fi + fi + + + if [ -z "$(which ${COSTOOL} 2>/dev/null)" ]; then + iam::log::info "${COSTOOL} tool not installed, try to install it." + + if ! make -C "${IAM_ROOT}" tools.install.${COSTOOL}; then + iam::log::error "failed to install ${COSTOOL}" + return 1 + fi + fi + + if [ -z "${TENCENT_SECRET_ID}" -o -z "${TENCENT_SECRET_KEY}" ];then + iam::log::error "can not find env: TENCENT_SECRET_ID and TENCENT_SECRET_KEY" + return 1 + fi + + if [ "${COSTOOL}" == "coscli" ];then + if [ ! -f "${HOME}/.cos.yaml" ];then + cat << EOF > "${HOME}/.cos.yaml" +cos: + base: + secretid: ${TENCENT_SECRET_ID} + secretkey: ${TENCENT_SECRET_KEY} + sessiontoken: "" + buckets: + - name: ${BUCKET} + alias: ${BUCKET} + region: ${REGION} +EOF + fi + else + if [ ! -f "${HOME}/.cos.conf" ];then + cat << EOF > "${HOME}/.cos.conf" +[common] +secret_id = ${TENCENT_SECRET_ID} +secret_key = ${TENCENT_SECRET_KEY} +bucket = ${BUCKET} +region =${REGION} +max_thread = 5 +part_size = 1 +schema = https +EOF + fi + fi +} + +# Create a github release with specified tarballs. +# NOTICE: Must export 'GITHUB_TOKEN' env in the shell, details: +# https://github.com/github-release/github-release +function iam::release::github_release() { + # create a github release + iam::log::info "create a new github release with tag ${IAM_GIT_VERSION}" + github-release release \ + --user ${IAM_GITHUB_ORG} \ + --repo ${IAM_GITHUB_REPO} \ + --tag ${IAM_GIT_VERSION} \ + --description "" \ + --pre-release + + # update iam tarballs + iam::log::info "upload ${ARTIFACT} to release ${IAM_GIT_VERSION}" + github-release upload \ + --user ${IAM_GITHUB_ORG} \ + --repo ${IAM_GITHUB_REPO} \ + --tag ${IAM_GIT_VERSION} \ + --name ${ARTIFACT} \ + --file ${RELEASE_TARS}/${ARTIFACT} + + iam::log::info "upload iam-src.tar.gz to release ${IAM_GIT_VERSION}" + github-release upload \ + --user ${IAM_GITHUB_ORG} \ + --repo ${IAM_GITHUB_REPO} \ + --tag ${IAM_GIT_VERSION} \ + --name "iam-src.tar.gz" \ + --file ${RELEASE_TARS}/iam-src.tar.gz +} + +function iam::release::generate_changelog() { + iam::log::info "generate CHANGELOG-${IAM_GIT_VERSION#v}.md and commit it" + + git-chglog ${IAM_GIT_VERSION} > ${IAM_ROOT}/CHANGELOG/CHANGELOG-${IAM_GIT_VERSION#v}.md + + set +o errexit + git add ${IAM_ROOT}/CHANGELOG/CHANGELOG-${IAM_GIT_VERSION#v}.md + git commit -a -m "docs(changelog): add CHANGELOG-${IAM_GIT_VERSION#v}.md" + git push -f origin master # 最后将 CHANGELOG 也 push 上去 +} + diff --git a/scripts/lib/util.sh b/scripts/lib/util.sh new file mode 100755 index 000000000..86c32a69e --- /dev/null +++ b/scripts/lib/util.sh @@ -0,0 +1,683 @@ +#!/usr/bin/env bash + +# Copyright 2020 Lingfei Kong . All rights reserved. +# Use of this source code is governed by a MIT style +# license that can be found in the LICENSE file. + +function iam::util::sourced_variable { + # Call this function to tell shellcheck that a variable is supposed to + # be used from other calling context. This helps quiet an "unused + # variable" warning from shellcheck and also document your code. + true +} + +iam::util::sortable_date() { + date "+%Y%m%d-%H%M%S" +} + +# arguments: target, item1, item2, item3, ... +# returns 0 if target is in the given items, 1 otherwise. +iam::util::array_contains() { + local search="$1" + local element + shift + for element; do + if [[ "${element}" == "${search}" ]]; then + return 0 + fi + done + return 1 +} + +iam::util::wait_for_url() { + local url=$1 + local prefix=${2:-} + local wait=${3:-1} + local times=${4:-30} + local maxtime=${5:-1} + + command -v curl >/dev/null || { + iam::log::usage "curl must be installed" + exit 1 + } + + local i + for i in $(seq 1 "${times}"); do + local out + if out=$(curl --max-time "${maxtime}" -gkfs "${url}" 2>/dev/null); then + iam::log::status "On try ${i}, ${prefix}: ${out}" + return 0 + fi + sleep "${wait}" + done + iam::log::error "Timed out waiting for ${prefix} to answer at ${url}; tried ${times} waiting ${wait} between each" + return 1 +} + +# Example: iam::util::wait_for_success 120 5 "iamctl get nodes|grep localhost" +# arguments: wait time, sleep time, shell command +# returns 0 if the shell command get output, 1 otherwise. +iam::util::wait_for_success(){ + local wait_time="$1" + local sleep_time="$2" + local cmd="$3" + while [ "$wait_time" -gt 0 ]; do + if eval "$cmd"; then + return 0 + else + sleep "$sleep_time" + wait_time=$((wait_time-sleep_time)) + fi + done + return 1 +} + +# Example: iam::util::trap_add 'echo "in trap DEBUG"' DEBUG +# See: http://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal +iam::util::trap_add() { + local trap_add_cmd + trap_add_cmd=$1 + shift + + for trap_add_name in "$@"; do + local existing_cmd + local new_cmd + + # Grab the currently defined trap commands for this trap + existing_cmd=$(trap -p "${trap_add_name}" | awk -F"'" '{print $2}') + + if [[ -z "${existing_cmd}" ]]; then + new_cmd="${trap_add_cmd}" + else + new_cmd="${trap_add_cmd};${existing_cmd}" + fi + + # Assign the test. Disable the shellcheck warning telling that trap + # commands should be single quoted to avoid evaluating them at this + # point instead evaluating them at run time. The logic of adding new + # commands to a single trap requires them to be evaluated right away. + # shellcheck disable=SC2064 + trap "${new_cmd}" "${trap_add_name}" + done +} + +# Opposite of iam::util::ensure-temp-dir() +iam::util::cleanup-temp-dir() { + rm -rf "${IAM_TEMP}" +} + +# Create a temp dir that'll be deleted at the end of this bash session. +# +# Vars set: +# IAM_TEMP +iam::util::ensure-temp-dir() { + if [[ -z ${IAM_TEMP-} ]]; then + IAM_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t iamrnetes.XXXXXX) + iam::util::trap_add iam::util::cleanup-temp-dir EXIT + fi +} + +iam::util::host_os() { + local host_os + case "$(uname -s)" in + Darwin) + host_os=darwin + ;; + Linux) + host_os=linux + ;; + *) + iam::log::error "Unsupported host OS. Must be Linux or Mac OS X." + exit 1 + ;; + esac + echo "${host_os}" +} + +iam::util::host_arch() { + local host_arch + case "$(uname -m)" in + x86_64*) + host_arch=amd64 + ;; + i?86_64*) + host_arch=amd64 + ;; + amd64*) + host_arch=amd64 + ;; + aarch64*) + host_arch=arm64 + ;; + arm64*) + host_arch=arm64 + ;; + arm*) + host_arch=arm + ;; + i?86*) + host_arch=x86 + ;; + s390x*) + host_arch=s390x + ;; + ppc64le*) + host_arch=ppc64le + ;; + *) + iam::log::error "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x or ppc64le." + exit 1 + ;; + esac + echo "${host_arch}" +} + +# This figures out the host platform without relying on golang. We need this as +# we don't want a golang install to be a prerequisite to building yet we need +# this info to figure out where the final binaries are placed. +iam::util::host_platform() { + echo "$(iam::util::host_os)/$(iam::util::host_arch)" +} + +# looks for $1 in well-known output locations for the platform ($2) +# $IAM_ROOT must be set +iam::util::find-binary-for-platform() { + local -r lookfor="$1" + local -r platform="$2" + local locations=( + "${IAM_ROOT}/_output/bin/${lookfor}" + "${IAM_ROOT}/_output/${platform}/${lookfor}" + "${IAM_ROOT}/_output/local/bin/${platform}/${lookfor}" + "${IAM_ROOT}/_output/platforms/${platform}/${lookfor}" + ) + + # List most recently-updated location. + local -r bin=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 ) + echo -n "${bin}" +} + +# looks for $1 in well-known output locations for the host platform +# $IAM_ROOT must be set +iam::util::find-binary() { + iam::util::find-binary-for-platform "$1" "$(iam::util::host_platform)" +} + +# Run all known doc generators (today gendocs and genman for iamctl) +# $1 is the directory to put those generated documents +iam::util::gen-docs() { + local dest="$1" + + # Find binary + gendocs=$(iam::util::find-binary "gendocs") + geniamdocs=$(iam::util::find-binary "geniamdocs") + genman=$(iam::util::find-binary "genman") + genyaml=$(iam::util::find-binary "genyaml") + genfeddocs=$(iam::util::find-binary "genfeddocs") + + # TODO: If ${genfeddocs} is not used from anywhere (it isn't used at + # least from k/k tree), remove it completely. + iam::util::sourced_variable "${genfeddocs}" + + mkdir -p "${dest}/docs/guide/en-US/cmd/iamctl/" + "${gendocs}" "${dest}/docs/guide/en-US/cmd/iamctl/" + + mkdir -p "${dest}/docs/guide/en-US/cmd/" + "${geniamdocs}" "${dest}/docs/guide/en-US/cmd/" "iam-apiserver" + "${geniamdocs}" "${dest}/docs/guide/en-US/cmd/" "iam-authz-server" + "${geniamdocs}" "${dest}/docs/guide/en-US/cmd/" "iam-pump" + "${geniamdocs}" "${dest}/docs/guide/en-US/cmd/" "iam-watcher" + "${geniamdocs}" "${dest}/docs/guide/en-US/cmd/iamctl" "iamctl" + + mkdir -p "${dest}/docs/man/man1/" + "${genman}" "${dest}/docs/man/man1/" "iam-apiserver" + "${genman}" "${dest}/docs/man/man1/" "iam-authz-server" + "${genman}" "${dest}/docs/man/man1/" "iam-pump" + "${genman}" "${dest}/docs/man/man1/" "iam-watcher" + "${genman}" "${dest}/docs/man/man1/" "iamctl" + + mkdir -p "${dest}/docs/guide/en-US/yaml/iamctl/" + "${genyaml}" "${dest}/docs/guide/en-US/yaml/iamctl/" + + # create the list of generated files + pushd "${dest}" > /dev/null || return 1 + touch docs/.generated_docs + find . -type f | cut -sd / -f 2- | LC_ALL=C sort > docs/.generated_docs + popd > /dev/null || return 1 +} + +# Removes previously generated docs-- we don't want to check them in. $IAM_ROOT +# must be set. +iam::util::remove-gen-docs() { + if [ -e "${IAM_ROOT}/docs/.generated_docs" ]; then + # remove all of the old docs; we don't want to check them in. + while read -r file; do + rm "${IAM_ROOT}/${file}" 2>/dev/null || true + done <"${IAM_ROOT}/docs/.generated_docs" + # The docs/.generated_docs file lists itself, so we don't need to explicitly + # delete it. + fi +} + +# Returns the name of the upstream remote repository name for the local git +# repo, e.g. "upstream" or "origin". +iam::util::git_upstream_remote_name() { + git remote -v | grep fetch |\ + grep -E 'github.com[/:]marmotedu/iam|marmotedu.io/iam' |\ + head -n 1 | awk '{print $1}' +} + +# Exits script if working directory is dirty. If it's run interactively in the terminal +# the user can commit changes in a second terminal. This script will wait. +iam::util::ensure_clean_working_dir() { + while ! git diff HEAD --exit-code &>/dev/null; do + echo -e "\nUnexpected dirty working directory:\n" + if tty -s; then + git status -s + else + git diff -a # be more verbose in log files without tty + exit 1 + fi | sed 's/^/ /' + echo -e "\nCommit your changes in another terminal and then continue here by pressing enter." + read -r + done 1>&2 +} + +# Find the base commit using: +# $PULL_BASE_SHA if set (from Prow) +# current ref from the remote upstream branch +iam::util::base_ref() { + local -r git_branch=$1 + + if [[ -n ${PULL_BASE_SHA:-} ]]; then + echo "${PULL_BASE_SHA}" + return + fi + + full_branch="$(iam::util::git_upstream_remote_name)/${git_branch}" + + # make sure the branch is valid, otherwise the check will pass erroneously. + if ! git describe "${full_branch}" >/dev/null; then + # abort! + exit 1 + fi + + echo "${full_branch}" +} + +# Checks whether there are any files matching pattern $2 changed between the +# current branch and upstream branch named by $1. +# Returns 1 (false) if there are no changes +# 0 (true) if there are changes detected. +iam::util::has_changes() { + local -r git_branch=$1 + local -r pattern=$2 + local -r not_pattern=${3:-totallyimpossiblepattern} + + local base_ref + base_ref=$(iam::util::base_ref "${git_branch}") + echo "Checking for '${pattern}' changes against '${base_ref}'" + + # notice this uses ... to find the first shared ancestor + if git diff --name-only "${base_ref}...HEAD" | grep -v -E "${not_pattern}" | grep "${pattern}" > /dev/null; then + return 0 + fi + # also check for pending changes + if git status --porcelain | grep -v -E "${not_pattern}" | grep "${pattern}" > /dev/null; then + echo "Detected '${pattern}' uncommitted changes." + return 0 + fi + echo "No '${pattern}' changes detected." + return 1 +} + +iam::util::download_file() { + local -r url=$1 + local -r destination_file=$2 + + rm "${destination_file}" 2&> /dev/null || true + + for i in $(seq 5) + do + if ! curl -fsSL --retry 3 --keepalive-time 2 "${url}" -o "${destination_file}"; then + echo "Downloading ${url} failed. $((5-i)) retries left." + sleep 1 + else + echo "Downloading ${url} succeed" + return 0 + fi + done + return 1 +} + +# Test whether openssl is installed. +# Sets: +# OPENSSL_BIN: The path to the openssl binary to use +function iam::util::test_openssl_installed { + if ! openssl version >& /dev/null; then + echo "Failed to run openssl. Please ensure openssl is installed" + exit 1 + fi + + OPENSSL_BIN=$(command -v openssl) +} + +# creates a client CA, args are sudo, dest-dir, ca-id, purpose +# purpose is dropped in after "key encipherment", you usually want +# '"client auth"' +# '"server auth"' +# '"client auth","server auth"' +function iam::util::create_signing_certkey { + local sudo=$1 + local dest_dir=$2 + local id=$3 + local purpose=$4 + # Create client ca + ${sudo} /usr/bin/env bash -e < "${dest_dir}/${id}-ca-config.json" +EOF +} + +# signs a client certificate: args are sudo, dest-dir, CA, filename (roughly), username, groups... +function iam::util::create_client_certkey { + local sudo=$1 + local dest_dir=$2 + local ca=$3 + local id=$4 + local cn=${5:-$4} + local groups="" + local SEP="" + shift 5 + while [ -n "${1:-}" ]; do + groups+="${SEP}{\"O\":\"$1\"}" + SEP="," + shift 1 + done + ${sudo} /usr/bin/env bash -e < /dev/null +apiVersion: v1 +kind: Config +clusters: + - cluster: + certificate-authority: ${ca_file} + server: https://${api_host}:${api_port}/ + name: local-up-cluster +users: + - user: + token: ${token} + client-certificate: ${dest_dir}/client-${client_id}.crt + client-key: ${dest_dir}/client-${client_id}.key + name: local-up-cluster +contexts: + - context: + cluster: local-up-cluster + user: local-up-cluster + name: local-up-cluster +current-context: local-up-cluster +EOF + + # flatten the iamconfig files to make them self contained + username=$(whoami) + ${sudo} /usr/bin/env bash -e < "/tmp/${client_id}.iamconfig" + mv -f "/tmp/${client_id}.iamconfig" "${dest_dir}/${client_id}.iamconfig" + chown ${username} "${dest_dir}/${client_id}.iamconfig" +EOF +} + +# Determines if docker can be run, failures may simply require that the user be added to the docker group. +function iam::util::ensure_docker_daemon_connectivity { + IFS=" " read -ra DOCKER <<< "${DOCKER_OPTS}" + # Expand ${DOCKER[@]} only if it's not unset. This is to work around + # Bash 3 issue with unbound variable. + DOCKER=(docker ${DOCKER[@]:+"${DOCKER[@]}"}) + if ! "${DOCKER[@]}" info > /dev/null 2>&1 ; then + cat <<'EOF' >&2 +Can't connect to 'docker' daemon. please fix and retry. + +Possible causes: + - Docker Daemon not started + - Linux: confirm via your init system + - macOS w/ docker-machine: run `docker-machine ls` and `docker-machine start ` + - macOS w/ Docker for Mac: Check the menu bar and start the Docker application + - DOCKER_HOST hasn't been set or is set incorrectly + - Linux: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}` + - macOS w/ docker-machine: run `eval "$(docker-machine env )"` + - macOS w/ Docker for Mac: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}` + - Other things to check: + - Linux: User isn't in 'docker' group. Add and relogin. + - Something like 'sudo usermod -a -G docker ${USER}' + - RHEL7 bug and workaround: https://bugzilla.redhat.com/show_bug.cgi?id=1119282#c8 +EOF + return 1 + fi +} + +# Wait for background jobs to finish. Return with +# an error status if any of the jobs failed. +iam::util::wait-for-jobs() { + local fail=0 + local job + for job in $(jobs -p); do + wait "${job}" || fail=$((fail + 1)) + done + return ${fail} +} + +# iam::util::join +# Concatenates the list elements with the delimiter passed as first parameter +# +# Ex: iam::util::join , a b c +# -> a,b,c +function iam::util::join { + local IFS="$1" + shift + echo "$*" +} + +# Downloads cfssl/cfssljson/cfssl-certinfo into $1 directory if they do not already exist in PATH +# +# Assumed vars: +# $1 (cfssl directory) (optional) +# +# Sets: +# CFSSL_BIN: The path of the installed cfssl binary +# CFSSLJSON_BIN: The path of the installed cfssljson binary +# CFSSLCERTINFO_BIN: The path of the installed cfssl-certinfo binary +# +function iam::util::ensure-cfssl { + if command -v cfssl &>/dev/null && command -v cfssljson &>/dev/null && command -v cfssl-certinfo &>/dev/null; then + CFSSL_BIN=$(command -v cfssl) + CFSSLJSON_BIN=$(command -v cfssljson) + CFSSLCERTINFO_BIN=$(command -v cfssl-certinfo) + return 0 + fi + + host_arch=$(iam::util::host_arch) + + if [[ "${host_arch}" != "amd64" ]]; then + echo "Cannot download cfssl on non-amd64 hosts and cfssl does not appear to be installed." + echo "Please install cfssl, cfssljson and cfssl-certinfo and verify they are in \$PATH." + echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..." + exit 1 + fi + + # Create a temp dir for cfssl if no directory was given + local cfssldir=${1:-} + if [[ -z "${cfssldir}" ]]; then + cfssldir="$HOME/bin" + fi + + mkdir -p "${cfssldir}" + pushd "${cfssldir}" > /dev/null || return 1 + + echo "Unable to successfully run 'cfssl' from ${PATH}; downloading instead..." + kernel=$(uname -s) + case "${kernel}" in + Linux) + curl --retry 10 -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 + curl --retry 10 -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 + curl --retry 10 -L -o cfssl-certinfo https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 + ;; + Darwin) + curl --retry 10 -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_darwin-amd64 + curl --retry 10 -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_darwin-amd64 + curl --retry 10 -L -o cfssl-certinfo https://pkg.cfssl.org/R1.2/cfssl-certinfo_darwin-amd64 + ;; + *) + echo "Unknown, unsupported platform: ${kernel}." >&2 + echo "Supported platforms: Linux, Darwin." >&2 + exit 2 + esac + + chmod +x cfssl || true + chmod +x cfssljson || true + chmod +x cfssl-certinfo || true + + CFSSL_BIN="${cfssldir}/cfssl" + CFSSLJSON_BIN="${cfssldir}/cfssljson" + CFSSLCERTINFO_BIN="${cfssldir}/cfssl-certinfo" + if [[ ! -x ${CFSSL_BIN} || ! -x ${CFSSLJSON_BIN} || ! -x ${CFSSLCERTINFO_BIN} ]]; then + echo "Failed to download 'cfssl'." + echo "Please install cfssl, cfssljson and cfssl-certinfo and verify they are in \$PATH." + echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..." + exit 1 + fi + popd > /dev/null || return 1 +} + +# iam::util::ensure-gnu-sed +# Determines which sed binary is gnu-sed on linux/darwin +# +# Sets: +# SED: The name of the gnu-sed binary +# +function iam::util::ensure-gnu-sed { + # NOTE: the echo below is a workaround to ensure sed is executed before the grep. + # see: https://github.com/iamrnetes/iamrnetes/issues/87251 + sed_help="$(LANG=C sed --help 2>&1 || true)" + if echo "${sed_help}" | grep -q "GNU\|BusyBox"; then + SED="sed" + elif command -v gsed &>/dev/null; then + SED="gsed" + else + iam::log::error "Failed to find GNU sed as sed or gsed. If you are on Mac: brew install gnu-sed." >&2 + return 1 + fi + iam::util::sourced_variable "${SED}" +} + +# iam::util::check-file-in-alphabetical-order +# Check that the file is in alphabetical order +# +function iam::util::check-file-in-alphabetical-order { + local failure_file="$1" + if ! diff -u "${failure_file}" <(LC_ALL=C sort "${failure_file}"); then + { + echo + echo "${failure_file} is not in alphabetical order. Please sort it:" + echo + echo " LC_ALL=C sort -o ${failure_file} ${failure_file}" + echo + } >&2 + false + fi +} + +# iam::util::require-jq +# Checks whether jq is installed. +function iam::util::require-jq { + if ! command -v jq &>/dev/null; then + echo "jq not found. Please install." 1>&2 + return 1 + fi +} + +# outputs md5 hash of $1, works on macOS and Linux +function iam::util::md5() { + if which md5 >/dev/null 2>&1; then + md5 -q "$1" + else + md5sum "$1" | awk '{ print $1 }' + fi +} + +# iam::util::read-array +# Reads in stdin and adds it line by line to the array provided. This can be +# used instead of "mapfile -t", and is bash 3 compatible. +# +# Assumed vars: +# $1 (name of array to create/modify) +# +# Example usage: +# iam::util::read-array files < <(ls -1) +# +function iam::util::read-array { + local i=0 + unset -v "$1" + while IFS= read -r "$1[i++]"; do :; done + eval "[[ \${$1[--i]} ]]" || unset "$1[i]" # ensures last element isn't empty +} + +# Some useful colors. +if [[ -z "${color_start-}" ]]; then + declare -r color_start="\033[" + declare -r color_red="${color_start}0;31m" + declare -r color_yellow="${color_start}0;33m" + declare -r color_green="${color_start}0;32m" + declare -r color_blue="${color_start}1;34m" + declare -r color_cyan="${color_start}1;36m" + declare -r color_norm="${color_start}0m" + + iam::util::sourced_variable "${color_start}" + iam::util::sourced_variable "${color_red}" + iam::util::sourced_variable "${color_yellow}" + iam::util::sourced_variable "${color_green}" + iam::util::sourced_variable "${color_blue}" + iam::util::sourced_variable "${color_cyan}" + iam::util::sourced_variable "${color_norm}" +fi + +# ex: ts=2 sw=2 et filetype=sh diff --git a/scripts/lib/version.sh b/scripts/lib/version.sh new file mode 100755 index 000000000..0edeac114 --- /dev/null +++ b/scripts/lib/version.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash + +# Copyright 2020 Lingfei Kong . All rights reserved. +# Use of this source code is governed by a MIT style +# license that can be found in the LICENSE file. + +# ----------------------------------------------------------------------------- +# Version management helpers. These functions help to set, save and load the +# following variables: +# +# IAM_GIT_COMMIT - The git commit id corresponding to this +# source code. +# IAM_GIT_TREE_STATE - "clean" indicates no changes since the git commit id +# "dirty" indicates source code changes after the git commit id +# "archive" indicates the tree was produced by 'git archive' +# IAM_GIT_VERSION - "vX.Y" used to indicate the last release version. +# IAM_GIT_MAJOR - The major part of the version +# IAM_GIT_MINOR - The minor component of the version + +# Grovels through git to set a set of env variables. +# +# If IAM_GIT_VERSION_FILE, this function will load from that file instead of +# querying git. +iam::version::get_version_vars() { + if [[ -n ${IAM_GIT_VERSION_FILE-} ]]; then + iam::version::load_version_vars "${IAM_GIT_VERSION_FILE}" + return + fi + + # If the iamrnetes source was exported through git archive, then + # we likely don't have a git tree, but these magic values may be filled in. + # shellcheck disable=SC2016,SC2050 + # Disabled as we're not expanding these at runtime, but rather expecting + # that another tool may have expanded these and rewritten the source (!) + if [[ '$Format:%%$' == "%" ]]; then + IAM_GIT_COMMIT='$Format:%H$' + IAM_GIT_TREE_STATE="archive" + # When a 'git archive' is exported, the '$Format:%D$' below will look + # something like 'HEAD -> release-1.8, tag: v1.8.3' where then 'tag: ' + # can be extracted from it. + if [[ '$Format:%D$' =~ tag:\ (v[^ ,]+) ]]; then + IAM_GIT_VERSION="${BASH_REMATCH[1]}" + fi + fi + + local git=(git --work-tree "${IAM_ROOT}") + + if [[ -n ${IAM_GIT_COMMIT-} ]] || IAM_GIT_COMMIT=$("${git[@]}" rev-parse "HEAD^{commit}" 2>/dev/null); then + if [[ -z ${IAM_GIT_TREE_STATE-} ]]; then + # Check if the tree is dirty. default to dirty + if git_status=$("${git[@]}" status --porcelain 2>/dev/null) && [[ -z ${git_status} ]]; then + IAM_GIT_TREE_STATE="clean" + else + IAM_GIT_TREE_STATE="dirty" + fi + fi + + # Use git describe to find the version based on tags. + if [[ -n ${IAM_GIT_VERSION-} ]] || IAM_GIT_VERSION=$("${git[@]}" describe --tags --always --match='v*' 2>/dev/null); then + # This translates the "git describe" to an actual semver.org + # compatible semantic version that looks something like this: + # v1.1.0-alpha.0.6+84c76d1142ea4d + # + # TODO: We continue calling this "git version" because so many + # downstream consumers are expecting it there. + # + # These regexes are painful enough in sed... + # We don't want to do them in pure shell, so disable SC2001 + # shellcheck disable=SC2001 + DASHES_IN_VERSION=$(echo "${IAM_GIT_VERSION}" | sed "s/[^-]//g") + if [[ "${DASHES_IN_VERSION}" == "---" ]] ; then + # shellcheck disable=SC2001 + # We have distance to subversion (v1.1.0-subversion-1-gCommitHash) + IAM_GIT_VERSION=$(echo "${IAM_GIT_VERSION}" | sed "s/-\([0-9]\{1,\}\)-g\([0-9a-f]\{14\}\)$/.\1\+\2/") + elif [[ "${DASHES_IN_VERSION}" == "--" ]] ; then + # shellcheck disable=SC2001 + # We have distance to base tag (v1.1.0-1-gCommitHash) + IAM_GIT_VERSION=$(echo "${IAM_GIT_VERSION}" | sed "s/-g\([0-9a-f]\{14\}\)$/+\1/") + fi + if [[ "${IAM_GIT_TREE_STATE}" == "dirty" ]]; then + # git describe --dirty only considers changes to existing files, but + # that is problematic since new untracked .go files affect the build, + # so use our idea of "dirty" from git status instead. + # TODO? + #IAM_GIT_VERSION+="-dirty" + : + fi + + + # Try to match the "git describe" output to a regex to try to extract + # the "major" and "minor" versions and whether this is the exact tagged + # version or whether the tree is between two tagged versions. + if [[ "${IAM_GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?([-].*)?([+].*)?$ ]]; then + IAM_GIT_MAJOR=${BASH_REMATCH[1]} + IAM_GIT_MINOR=${BASH_REMATCH[2]} + if [[ -n "${BASH_REMATCH[4]}" ]]; then + IAM_GIT_MINOR+="+" + fi + fi + + # If IAM_GIT_VERSION is not a valid Semantic Version, then refuse to build. + if ! [[ "${IAM_GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then + echo "IAM_GIT_VERSION should be a valid Semantic Version. Current value: ${IAM_GIT_VERSION}" + echo "Please see more details here: https://semver.org" + exit 1 + fi + fi + fi +} + +# Saves the environment flags to $1 +iam::version::save_version_vars() { + local version_file=${1-} + [[ -n ${version_file} ]] || { + echo "!!! Internal error. No file specified in iam::version::save_version_vars" + return 1 + } + + cat <"${version_file}" +IAM_GIT_COMMIT='${IAM_GIT_COMMIT-}' +IAM_GIT_TREE_STATE='${IAM_GIT_TREE_STATE-}' +IAM_GIT_VERSION='${IAM_GIT_VERSION-}' +IAM_GIT_MAJOR='${IAM_GIT_MAJOR-}' +IAM_GIT_MINOR='${IAM_GIT_MINOR-}' +EOF +} + +# Loads up the version variables from file $1 +iam::version::load_version_vars() { + local version_file=${1-} + [[ -n ${version_file} ]] || { + echo "!!! Internal error. No file specified in iam::version::load_version_vars" + return 1 + } + + source "${version_file}" +} diff --git a/scripts/make-rules/golang.mk b/scripts/make-rules/golang.mk index 255306062..bf503db63 100644 --- a/scripts/make-rules/golang.mk +++ b/scripts/make-rules/golang.mk @@ -132,24 +132,17 @@ go.build.multiarch: go.build.verify $(foreach p,$(PLATFORMS),$(addprefix go.buil go.lint: tools.verify.golangci-lint @echo "===========> Run golangci to lint source codes" @$(TOOLS_DIR)/golangci-lint run --color always -c $(ROOT_DIR)/.golangci.yml $(ROOT_DIR)/... + ## go.test: Run unit test .PHONY: go.test go.test: @$(GO) test ./... -# ## go.test.junit-report: Run unit test -# .PHONY: go.test.junit-report -# go.test.junit-report: tools.verify.go-junit-report -# @echo "===========> Run unit test > $(TMP_DIR)/report.xml" -# @$(GO) test -v -coverprofile=$(TMP_DIR)/coverage.out 2>&1 $(GO_BUILD_FLAGS) ./... | $(TOOLS_DIR)/go-junit-report -set-exit-code > $(TMP_DIR)/report.xml -# @sed -i '/mock_.*.go/d' $(TMP_DIR)/coverage.out -# @echo "===========> Test coverage of Go code is reported to $(TMP_DIR)/coverage.html by generating HTML" -# @$(GO) tool cover -html=$(TMP_DIR)/coverage.out -o $(TMP_DIR)/coverage.html - ## go.test.junit-report: Run unit test .PHONY: go.test.junit-report go.test.junit-report: tools.verify.go-junit-report @echo "===========> Run unit test > $(TMP_DIR)/report.xml" +# @$(GO) test -v -coverprofile=$(TMP_DIR)/coverage.out 2>&1 $(GO_BUILD_FLAGS) ./... | $(TOOLS_DIR)/go-junit-report -set-exit-code > $(TMP_DIR)/report.xml @$(GO) test -v -coverprofile=$(TMP_DIR)/coverage.out 2>&1 ./... | $(TOOLS_DIR)/go-junit-report -set-exit-code > $(TMP_DIR)/report.xml @sed -i '/mock_.*.go/d' $(TMP_DIR)/coverage.out @echo "===========> Test coverage of Go code is reported to $(TMP_DIR)/coverage.html by generating HTML"