msb-public
/
Open-IM-Server
mirror of https://github.com/OpenIMSDK/Open-IM-Server
2
0
Code Issues Projects Releases Wiki Activity

feat: add scripts lib

Browse Source 
Signed-off-by: Xinwei Xiong(cubxxw-openim) <3293172751nss@gmail.com>
pull/462/head
Xinwei Xiong(cubxxw-openim) 1 year ago
parent dc318bbdb5
commit 91979285eb
8 changed files with 1839 additions and 9 deletions
Show Stats Download Patch File Download Diff File

38
scripts/lib/color.sh
Unescape View File

@ -0,0 +1,38 @@
#!/usr/bin/env bash
# Copyright 2020 Lingfei Kong <colin404@foxmail.com>. All rights reserved.
# Use of this source code is governed by a MIT style
# license that can be found in the LICENSE file.
#Define color variables
#Feature
COLOR_NORMAL='\033[0m';COLOR_BOLD='\033[1m';COLOR_DIM='\033[2m';COLOR_UNDER='\033[4m';
COLOR_ITALIC='\033[3m';COLOR_NOITALIC='\033[23m';COLOR_BLINK='\033[5m';
COLOR_REVERSE='\033[7m';COLOR_CONCEAL='\033[8m';COLOR_NOBOLD='\033[22m';
COLOR_NOUNDER='\033[24m';COLOR_NOBLINK='\033[25m';
#Front color
COLOR_BLACK='\033[30m';COLOR_RED='\033[31m';COLOR_GREEN='\033[32m';COLOR_YELLOW='\033[33m';
COLOR_BLUE='\033[34m';COLOR_MAGENTA='\033[35m';COLOR_CYAN='\033[36m';COLOR_WHITE='\033[37m';
#background color
COLOR_BBLACK='\033[40m';COLOR_BRED='\033[41m';
COLOR_BGREEN='\033[42m';COLOR_BYELLOW='\033[43m';
COLOR_BBLUE='\033[44m';COLOR_BMAGENTA='\033[45m';
COLOR_BCYAN='\033[46m';COLOR_BWHITE='\033[47m';
# Print colors you can use
iam::color::print_color()
{
echo
echo -e ${bmagenta}--back-color:${normal}
echo "bblack; bgreen; bblue; bcyan; bred; byellow; bmagenta; bwhite"
echo
echo -e ${red}--font-color:${normal}
echo "black; red; green; yellow; blue; magenta; cyan; white"
echo
echo -e ${bold}--font:${normal}
echo "normal; italic; reverse; nounder; bold; noitalic; conceal; noblink;
dim; blink; nobold; under"
echo
}

193
scripts/lib/golang.sh
Unescape View File

@ -0,0 +1,193 @@
#!/usr/bin/env bash
# Copyright 2020 Lingfei Kong <colin404@foxmail.com>. All rights reserved.
# Use of this source code is governed by a MIT style
# license that can be found in the LICENSE file.
# shellcheck disable=SC2034 # Variables sourced in other scripts.
# The server platform we are building on.
readonly IAM_SUPPORTED_SERVER_PLATFORMS=(
linux/amd64
linux/arm64
)
# If we update this we should also update the set of platforms whose standard
# library is precompiled for in build/build-image/cross/Dockerfile
readonly IAM_SUPPORTED_CLIENT_PLATFORMS=(
linux/amd64
linux/arm64
)
# The set of server targets that we are only building for Linux
# If you update this list, please also update build/BUILD.
iam::golang::server_targets() {
local targets=(
iam-apiserver
iam-authz-server
iam-pump
iam-watcher
)
echo "${targets[@]}"
}
IFS=" " read -ra IAM_SERVER_TARGETS <<< "$(iam::golang::server_targets)"
readonly IAM_SERVER_TARGETS
readonly IAM_SERVER_BINARIES=("${IAM_SERVER_TARGETS[@]##*/}")
# The set of server targets we build docker images for
iam::golang::server_image_targets() {
# NOTE: this contains cmd targets for iam::build::get_docker_wrapped_binaries
local targets=(
cmd/iam-apiserver
cmd/iam-authz-server
cmd/iam-pump
cmd/iam-watcher
)
echo "${targets[@]}"
}
IFS=" " read -ra IAM_SERVER_IMAGE_TARGETS <<< "$(iam::golang::server_image_targets)"
readonly IAM_SERVER_IMAGE_TARGETS
readonly IAM_SERVER_IMAGE_BINARIES=("${IAM_SERVER_IMAGE_TARGETS[@]##*/}")
# ------------
# NOTE: All functions that return lists should use newlines.
# bash functions can't return arrays, and spaces are tricky, so newline
# separators are the preferred pattern.
# To transform a string of newline-separated items to an array, use iam::util::read-array:
# iam::util::read-array FOO < <(iam::golang::dups a b c a)
#
# ALWAYS remember to quote your subshells. Not doing so will break in
# bash 4.3, and potentially cause other issues.
# ------------
# Returns a sorted newline-separated list containing only duplicated items.
iam::golang::dups() {
# We use printf to insert newlines, which are required by sort.
printf "%s\n" "$@" | sort | uniq -d
}
# Returns a sorted newline-separated list with duplicated items removed.
iam::golang::dedup() {
# We use printf to insert newlines, which are required by sort.
printf "%s\n" "$@" | sort -u
}
# Depends on values of user-facing IAM_BUILD_PLATFORMS, IAM_FASTBUILD,
# and IAM_BUILDER_OS.
# Configures IAM_SERVER_PLATFORMS and IAM_CLIENT_PLATFORMS, then sets them
# to readonly.
# The configured vars will only contain platforms allowed by the
# IAM_SUPPORTED* vars at the top of this file.
declare -a IAM_SERVER_PLATFORMS
declare -a IAM_CLIENT_PLATFORMS
iam::golang::setup_platforms() {
if [[ -n "${IAM_BUILD_PLATFORMS:-}" ]]; then
# IAM_BUILD_PLATFORMS needs to be read into an array before the next
# step, or quoting treats it all as one element.
local -a platforms
IFS=" " read -ra platforms <<< "${IAM_BUILD_PLATFORMS}"
# Deduplicate to ensure the intersection trick with iam::golang::dups
# is not defeated by duplicates in user input.
iam::util::read-array platforms < <(iam::golang::dedup "${platforms[@]}")
# Use iam::golang::dups to restrict the builds to the platforms in
# IAM_SUPPORTED_*_PLATFORMS. Items should only appear at most once in each
# set, so if they appear twice after the merge they are in the intersection.
iam::util::read-array IAM_SERVER_PLATFORMS < <(iam::golang::dups \
"${platforms[@]}" \
"${IAM_SUPPORTED_SERVER_PLATFORMS[@]}" \
)
readonly IAM_SERVER_PLATFORMS
iam::util::read-array IAM_CLIENT_PLATFORMS < <(iam::golang::dups \
"${platforms[@]}" \
"${IAM_SUPPORTED_CLIENT_PLATFORMS[@]}" \
)
readonly IAM_CLIENT_PLATFORMS
elif [[ "${IAM_FASTBUILD:-}" == "true" ]]; then
IAM_SERVER_PLATFORMS=(linux/amd64)
readonly IAM_SERVER_PLATFORMS
IAM_CLIENT_PLATFORMS=(linux/amd64)
readonly IAM_CLIENT_PLATFORMS
else
IAM_SERVER_PLATFORMS=("${IAM_SUPPORTED_SERVER_PLATFORMS[@]}")
readonly IAM_SERVER_PLATFORMS
IAM_CLIENT_PLATFORMS=("${IAM_SUPPORTED_CLIENT_PLATFORMS[@]}")
readonly IAM_CLIENT_PLATFORMS
fi
}
iam::golang::setup_platforms
# The set of client targets that we are building for all platforms
# If you update this list, please also update build/BUILD.
readonly IAM_CLIENT_TARGETS=(
iamctl
)
readonly IAM_CLIENT_BINARIES=("${IAM_CLIENT_TARGETS[@]##*/}")
readonly IAM_ALL_TARGETS=(
"${IAM_SERVER_TARGETS[@]}"
"${IAM_CLIENT_TARGETS[@]}"
)
readonly IAM_ALL_BINARIES=("${IAM_ALL_TARGETS[@]##*/}")
# Asks golang what it thinks the host platform is. The go tool chain does some
# slightly different things when the target platform matches the host platform.
iam::golang::host_platform() {
echo "$(go env GOHOSTOS)/$(go env GOHOSTARCH)"
}
# Ensure the go tool exists and is a viable version.
iam::golang::verify_go_version() {
if [[ -z "$(command -v go)" ]]; then
iam::log::usage_from_stdin <<EOF
Can't find 'go' in PATH, please fix and retry.
See http://golang.org/doc/install for installation instructions.
EOF
return 2
fi
local go_version
IFS=" " read -ra go_version <<< "$(go version)"
local minimum_go_version
minimum_go_version=go1.13.4
if [[ "${minimum_go_version}" != $(echo -e "${minimum_go_version}\n${go_version[2]}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) && "${go_version[2]}" != "devel" ]]; then
iam::log::usage_from_stdin <<EOF
Detected go version: ${go_version[*]}.
IAM requires ${minimum_go_version} or greater.
Please install ${minimum_go_version} or later.
EOF
return 2
fi
}
# iam::golang::setup_env will check that the `go` commands is available in
# ${PATH}. It will also check that the Go version is good enough for the
# IAM build.
#
# Outputs:
# env-var GOBIN is unset (we want binaries in a predictable place)
# env-var GO15VENDOREXPERIMENT=1
# env-var GO111MODULE=on
iam::golang::setup_env() {
iam::golang::verify_go_version
# Unset GOBIN in case it already exists in the current session.
unset GOBIN
# This seems to matter to some tools
export GO15VENDOREXPERIMENT=1
# Open go module feature
export GO111MODULE=on
# This is for sanity. Without it, user umasks leak through into release
# artifacts.
umask 0022
}

28
scripts/lib/init.sh
Unescape View File

@ -0,0 +1,28 @@
#!/usr/bin/env bash
# Copyright 2020 Lingfei Kong <colin404@foxmail.com>. All rights reserved.
# Use of this source code is governed by a MIT style
# license that can be found in the LICENSE file.
set -o errexit
set +o nounset
set -o pipefail
# Unset CDPATH so that path interpolation can work correctly
# https://github.com/iamrnetes/iamrnetes/issues/52255
unset CDPATH
# Default use go modules
export GO111MODULE=on
# The root of the build/dist directory
IAM_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
source "${IAM_ROOT}/scripts/lib/util.sh"
source "${IAM_ROOT}/scripts/lib/logging.sh"
source "${IAM_ROOT}/scripts/lib/color.sh"
iam::log::install_errexit
source "${IAM_ROOT}/scripts/lib/version.sh"
source "${IAM_ROOT}/scripts/lib/golang.sh"

161
scripts/lib/logging.sh
Unescape View File

@ -0,0 +1,161 @@
#!/usr/bin/env bash
# Copyright 2020 Lingfei Kong <colin404@foxmail.com>. All rights reserved.
# Use of this source code is governed by a MIT style
# license that can be found in the LICENSE file.
# Controls verbosity of the script output and logging.
IAM_VERBOSE="${IAM_VERBOSE:-5}"
# Handler for when we exit automatically on an error.
# Borrowed from https://gist.github.com/ahendrix/7030300
iam::log::errexit() {
local err="${PIPESTATUS[*]}"
# If the shell we are in doesn't have errexit set (common in subshells) then
# don't dump stacks.
set +o | grep -qe "-o errexit" || return
set +o xtrace
local code="${1:-1}"
# Print out the stack trace described by $function_stack
if [ ${#FUNCNAME[@]} -gt 2 ]
then
iam::log::error "Call tree:"
for ((i=1;i<${#FUNCNAME[@]}-1;i++))
do
iam::log::error " ${i}: ${BASH_SOURCE[${i}+1]}:${BASH_LINENO[${i}]} ${FUNCNAME[${i}]}(...)"
done
fi
iam::log::error_exit "Error in ${BASH_SOURCE[1]}:${BASH_LINENO[0]}. '${BASH_COMMAND}' exited with status ${err}" "${1:-1}" 1
}
iam::log::install_errexit() {
# trap ERR to provide an error handler whenever a command exits nonzero this
# is a more verbose version of set -o errexit
trap 'iam::log::errexit' ERR
# setting errtrace allows our ERR trap handler to be propagated to functions,
# expansions and subshells
set -o errtrace
}
# Print out the stack trace
#
# Args:
# $1 The number of stack frames to skip when printing.
iam::log::stack() {
local stack_skip=${1:-0}
stack_skip=$((stack_skip + 1))
if [[ ${#FUNCNAME[@]} -gt ${stack_skip} ]]; then
echo "Call stack:" >&2
local i
for ((i=1 ; i <= ${#FUNCNAME[@]} - stack_skip ; i++))
do
local frame_no=$((i - 1 + stack_skip))
local source_file=${BASH_SOURCE[${frame_no}]}
local source_lineno=${BASH_LINENO[$((frame_no - 1))]}
local funcname=${FUNCNAME[${frame_no}]}
echo " ${i}: ${source_file}:${source_lineno} ${funcname}(...)" >&2
done
fi
}
# Log an error and exit.
# Args:
# $1 Message to log with the error
# $2 The error code to return
# $3 The number of stack frames to skip when printing.
iam::log::error_exit() {
local message="${1:-}"
local code="${2:-1}"
local stack_skip="${3:-0}"
stack_skip=$((stack_skip + 1))
if [[ ${IAM_VERBOSE} -ge 4 ]]; then
local source_file=${BASH_SOURCE[${stack_skip}]}
local source_line=${BASH_LINENO[$((stack_skip - 1))]}
echo "!!! Error in ${source_file}:${source_line}" >&2
[[ -z ${1-} ]] || {
echo " ${1}" >&2
}
iam::log::stack ${stack_skip}
echo "Exiting with status ${code}" >&2
fi
exit "${code}"
}
# Log an error but keep going. Don't dump the stack or exit.
iam::log::error() {
timestamp=$(date +"[%m%d %H:%M:%S]")
echo "!!! ${timestamp} ${1-}" >&2
shift
for message; do
echo " ${message}" >&2
done
}
# Print an usage message to stderr. The arguments are printed directly.
iam::log::usage() {
echo >&2
local message
for message; do
echo "${message}" >&2
done
echo >&2
}
iam::log::usage_from_stdin() {
local messages=()
while read -r line; do
messages+=("${line}")
done
iam::log::usage "${messages[@]}"
}
# Print out some info that isn't a top level status line
iam::log::info() {
local V="${V:-0}"
if [[ ${IAM_VERBOSE} < ${V} ]]; then
return
fi
for message; do
echo "${message}"
done
}
# Just like iam::log::info, but no \n, so you can make a progress bar
iam::log::progress() {
for message; do
echo -e -n "${message}"
done
}
iam::log::info_from_stdin() {
local messages=()
while read -r line; do
messages+=("${line}")
done
iam::log::info "${messages[@]}"
}
# Print a status line. Formatted to show up in a stream of output.
iam::log::status() {
local V="${V:-0}"
if [[ ${IAM_VERBOSE} < ${V} ]]; then
return
fi
timestamp=$(date +"[%m%d %H:%M:%S]")
echo "+++ ${timestamp} ${1}"
shift
for message; do
echo " ${message}"
done
}

597
scripts/lib/release.sh
Unescape View File

@ -0,0 +1,597 @@
#!/usr/bin/env bash
# Copyright 2020 Lingfei Kong <colin404@foxmail.com>. All rights reserved.
# Use of this source code is governed by a MIT style
# license that can be found in the LICENSE file.
# This file creates release artifacts (tar files, container images) that are
# ready to distribute to install or distribute to end users.
###############################################################################
# Most of the ::release:: namespace functions have been moved to
# github.com/iam/release. Have a look in that repo and specifically in
# lib/releaselib.sh for ::release::-related functionality.
###############################################################################
# Tencent cos configuration
readonly BUCKET="marmotedu-1254073058"
readonly REGION="ap-beijing"
readonly COS_RELEASE_DIR="iam-release"
readonly COSTOOL="coscmd"
# This is where the final release artifacts are created locally
readonly RELEASE_STAGE="${LOCAL_OUTPUT_ROOT}/release-stage"
readonly RELEASE_TARS="${LOCAL_OUTPUT_ROOT}/release-tars"
readonly RELEASE_IMAGES="${LOCAL_OUTPUT_ROOT}/release-images"
# IAM github account info
readonly IAM_GITHUB_ORG=marmotedu
readonly IAM_GITHUB_REPO=iam
readonly ARTIFACT=iam.tar.gz
readonly CHECKSUM=${ARTIFACT}.sha1sum
IAM_BUILD_CONFORMANCE=${IAM_BUILD_CONFORMANCE:-y}
IAM_BUILD_PULL_LATEST_IMAGES=${IAM_BUILD_PULL_LATEST_IMAGES:-y}
# Validate a ci version
#
# Globals:
# None
# Arguments:
# version
# Returns:
# If version is a valid ci version
# Sets: (e.g. for '1.2.3-alpha.4.56+abcdef12345678')
# VERSION_MAJOR (e.g. '1')
# VERSION_MINOR (e.g. '2')
# VERSION_PATCH (e.g. '3')
# VERSION_PRERELEASE (e.g. 'alpha')
# VERSION_PRERELEASE_REV (e.g. '4')
# VERSION_BUILD_INFO (e.g. '.56+abcdef12345678')
# VERSION_COMMITS (e.g. '56')
function iam::release::parse_and_validate_ci_version() {
# Accept things like "v1.2.3-alpha.4.56+abcdef12345678" or "v1.2.3-beta.4"
local -r version_regex="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-([a-zA-Z0-9]+)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*)\\+[0-9a-f]{7,40})?$"
local -r version="${1-}"
[[ "${version}" =~ ${version_regex} ]] || {
iam::log::error "Invalid ci version: '${version}', must match regex ${version_regex}"
return 1
}
# The VERSION variables are used when this file is sourced, hence
# the shellcheck SC2034 'appears unused' warning is to be ignored.
# shellcheck disable=SC2034
VERSION_MAJOR="${BASH_REMATCH[1]}"
# shellcheck disable=SC2034
VERSION_MINOR="${BASH_REMATCH[2]}"
# shellcheck disable=SC2034
VERSION_PATCH="${BASH_REMATCH[3]}"
# shellcheck disable=SC2034
VERSION_PRERELEASE="${BASH_REMATCH[4]}"
# shellcheck disable=SC2034
VERSION_PRERELEASE_REV="${BASH_REMATCH[5]}"
# shellcheck disable=SC2034
VERSION_BUILD_INFO="${BASH_REMATCH[6]}"
# shellcheck disable=SC2034
VERSION_COMMITS="${BASH_REMATCH[7]}"
}
# ---------------------------------------------------------------------------
# Build final release artifacts
function iam::release::clean_cruft() {
# Clean out cruft
find "${RELEASE_STAGE}" -name '*~' -exec rm {} \;
find "${RELEASE_STAGE}" -name '#*#' -exec rm {} \;
find "${RELEASE_STAGE}" -name '.DS*' -exec rm {} \;
}
function iam::release::package_tarballs() {
# Clean out any old releases
rm -rf "${RELEASE_STAGE}" "${RELEASE_TARS}" "${RELEASE_IMAGES}"
mkdir -p "${RELEASE_TARS}"
iam::release::package_src_tarball &
iam::release::package_client_tarballs &
iam::release::package_iam_manifests_tarball &
iam::release::package_server_tarballs &
iam::util::wait-for-jobs || { iam::log::error "previous tarball phase failed"; return 1; }
iam::release::package_final_tarball & # _final depends on some of the previous phases
iam::util::wait-for-jobs || { iam::log::error "previous tarball phase failed"; return 1; }
}
function iam::release::updload_tarballs() {
iam::log::info "upload ${RELEASE_TARS}/* to cos bucket ${BUCKET}."
for file in $(ls ${RELEASE_TARS}/*)
do
if [ "${COSTOOL}" == "coscli" ];then
coscli cp "${file}" "cos://${BUCKET}/${COS_RELEASE_DIR}/${IAM_GIT_VERSION}/${file##*/}"
coscli cp "${file}" "cos://${BUCKET}/${COS_RELEASE_DIR}/latest/${file##*/}"
else
coscmd upload "${file}" "${COS_RELEASE_DIR}/${IAM_GIT_VERSION}/"
coscmd upload "${file}" "${COS_RELEASE_DIR}/latest/"
fi
done
}
# Package the source code we built, for compliance/licensing/audit/yadda.
function iam::release::package_src_tarball() {
local -r src_tarball="${RELEASE_TARS}/iam-src.tar.gz"
iam::log::status "Building tarball: src"
if [[ "${IAM_GIT_TREE_STATE-}" = 'clean' ]]; then
git archive -o "${src_tarball}" HEAD
else
find "${IAM_ROOT}" -mindepth 1 -maxdepth 1 \
! \( \
\( -path "${IAM_ROOT}"/_\* -o \
-path "${IAM_ROOT}"/.git\* -o \
-path "${IAM_ROOT}"/.gitignore\* -o \
-path "${IAM_ROOT}"/.gsemver.yaml\* -o \
-path "${IAM_ROOT}"/.config\* -o \
-path "${IAM_ROOT}"/.chglog\* -o \
-path "${IAM_ROOT}"/.gitlint -o \
-path "${IAM_ROOT}"/.golangci.yaml -o \
-path "${IAM_ROOT}"/.goreleaser.yml -o \
-path "${IAM_ROOT}"/.note.md -o \
-path "${IAM_ROOT}"/.todo.md \
\) -prune \
\) -print0 \
| "${TAR}" czf "${src_tarball}" --transform "s|${IAM_ROOT#/*}|iam|" --null -T -
fi
}
# Package up all of the server binaries
function iam::release::package_server_tarballs() {
# Find all of the built client binaries
local long_platforms=("${LOCAL_OUTPUT_BINPATH}"/*/*)
if [[ -n ${IAM_BUILD_PLATFORMS-} ]]; then
read -ra long_platforms <<< "${IAM_BUILD_PLATFORMS}"
fi
for platform_long in "${long_platforms[@]}"; do
local platform
local platform_tag
platform=${platform_long##${LOCAL_OUTPUT_BINPATH}/} # Strip LOCAL_OUTPUT_BINPATH
platform_tag=${platform/\//-} # Replace a "/" for a "-"
iam::log::status "Starting tarball: server $platform_tag"
(
local release_stage="${RELEASE_STAGE}/server/${platform_tag}/iam"
rm -rf "${release_stage}"
mkdir -p "${release_stage}/server/bin"
local server_bins=("${IAM_SERVER_BINARIES[@]}")
# This fancy expression will expand to prepend a path
# (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
# server_bins array.
cp "${server_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
"${release_stage}/server/bin/"
iam::release::clean_cruft
local package_name="${RELEASE_TARS}/iam-server-${platform_tag}.tar.gz"
iam::release::create_tarball "${package_name}" "${release_stage}/.."
) &
done
iam::log::status "Waiting on tarballs"
iam::util::wait-for-jobs || { iam::log::error "server tarball creation failed"; exit 1; }
}
# Package up all of the cross compiled clients. Over time this should grow into
# a full SDK
function iam::release::package_client_tarballs() {
# Find all of the built client binaries
local long_platforms=("${LOCAL_OUTPUT_BINPATH}"/*/*)
if [[ -n ${IAM_BUILD_PLATFORMS-} ]]; then
read -ra long_platforms <<< "${IAM_BUILD_PLATFORMS}"
fi
for platform_long in "${long_platforms[@]}"; do
local platform
local platform_tag
platform=${platform_long##${LOCAL_OUTPUT_BINPATH}/} # Strip LOCAL_OUTPUT_BINPATH
platform_tag=${platform/\//-} # Replace a "/" for a "-"
iam::log::status "Starting tarball: client $platform_tag"
(
local release_stage="${RELEASE_STAGE}/client/${platform_tag}/iam"
rm -rf "${release_stage}"
mkdir -p "${release_stage}/client/bin"
local client_bins=("${IAM_CLIENT_BINARIES[@]}")
# This fancy expression will expand to prepend a path
# (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
# client_bins array.
cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
"${release_stage}/client/bin/"
iam::release::clean_cruft
local package_name="${RELEASE_TARS}/iam-client-${platform_tag}.tar.gz"
iam::release::create_tarball "${package_name}" "${release_stage}/.."
) &
done
iam::log::status "Waiting on tarballs"
iam::util::wait-for-jobs || { iam::log::error "client tarball creation failed"; exit 1; }
}
# Package up all of the server binaries in docker images
function iam::release::build_server_images() {
# Clean out any old images
rm -rf "${RELEASE_IMAGES}"
local platform
for platform in "${IAM_SERVER_PLATFORMS[@]}"; do
local platform_tag
local arch
platform_tag=${platform/\//-} # Replace a "/" for a "-"
arch=$(basename "${platform}")
iam::log::status "Building images: $platform_tag"
local release_stage
release_stage="${RELEASE_STAGE}/server/${platform_tag}/iam"
rm -rf "${release_stage}"
mkdir -p "${release_stage}/server/bin"
# This fancy expression will expand to prepend a path
# (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
# IAM_SERVER_IMAGE_BINARIES array.
cp "${IAM_SERVER_IMAGE_BINARIES[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
"${release_stage}/server/bin/"
iam::release::create_docker_images_for_server "${release_stage}/server/bin" "${arch}"
done
}
function iam::release::md5() {
if which md5 >/dev/null 2>&1; then
md5 -q "$1"
else
md5sum "$1" | awk '{ print $1 }'
fi
}
function iam::release::sha1() {
if which sha1sum >/dev/null 2>&1; then
sha1sum "$1" | awk '{ print $1 }'
else
shasum -a1 "$1" | awk '{ print $1 }'
fi
}
function iam::release::build_conformance_image() {
local -r arch="$1"
local -r registry="$2"
local -r version="$3"
local -r save_dir="${4-}"
iam::log::status "Building conformance image for arch: ${arch}"
ARCH="${arch}" REGISTRY="${registry}" VERSION="${version}" \
make -C cluster/images/conformance/ build >/dev/null
local conformance_tag
conformance_tag="${registry}/conformance-${arch}:${version}"
if [[ -n "${save_dir}" ]]; then
"${DOCKER[@]}" save "${conformance_tag}" > "${save_dir}/conformance-${arch}.tar"
fi
iam::log::status "Deleting conformance image ${conformance_tag}"
"${DOCKER[@]}" rmi "${conformance_tag}" &>/dev/null || true
}
# This builds all the release docker images (One docker image per binary)
# Args:
# $1 - binary_dir, the directory to save the tared images to.
# $2 - arch, architecture for which we are building docker images.
function iam::release::create_docker_images_for_server() {
# Create a sub-shell so that we don't pollute the outer environment
(
local binary_dir
local arch
local binaries
local images_dir
binary_dir="$1"
arch="$2"
binaries=$(iam::build::get_docker_wrapped_binaries "${arch}")
images_dir="${RELEASE_IMAGES}/${arch}"
mkdir -p "${images_dir}"
# k8s.gcr.io is the constant tag in the docker archives, this is also the default for config scripts in GKE.
# We can use IAM_DOCKER_REGISTRY to include and extra registry in the docker archive.
# If we use IAM_DOCKER_REGISTRY="k8s.gcr.io", then the extra tag (same) is ignored, see release_docker_image_tag below.
local -r docker_registry="k8s.gcr.io"
# Docker tags cannot contain '+'
local docker_tag="${IAM_GIT_VERSION/+/_}"
if [[ -z "${docker_tag}" ]]; then
iam::log::error "git version information missing; cannot create Docker tag"
return 1
fi
# provide `--pull` argument to `docker build` if `IAM_BUILD_PULL_LATEST_IMAGES`
# is set to y or Y; otherwise try to build the image without forcefully
# pulling the latest base image.
local docker_build_opts
docker_build_opts=
if [[ "${IAM_BUILD_PULL_LATEST_IMAGES}" =~ [yY] ]]; then
docker_build_opts='--pull'
fi
for wrappable in $binaries; do
local binary_name=${wrappable%%,*}
local base_image=${wrappable##*,}
local binary_file_path="${binary_dir}/${binary_name}"
local docker_build_path="${binary_file_path}.dockerbuild"
local docker_file_path="${docker_build_path}/Dockerfile"
local docker_image_tag="${docker_registry}/${binary_name}-${arch}:${docker_tag}"
iam::log::status "Starting docker build for image: ${binary_name}-${arch}"
(
rm -rf "${docker_build_path}"
mkdir -p "${docker_build_path}"
ln "${binary_file_path}" "${docker_build_path}/${binary_name}"
ln "${IAM_ROOT}/build/nsswitch.conf" "${docker_build_path}/nsswitch.conf"
chmod 0644 "${docker_build_path}/nsswitch.conf"
cat <<EOF > "${docker_file_path}"
FROM ${base_image}
COPY ${binary_name} /usr/local/bin/${binary_name}
EOF
# ensure /etc/nsswitch.conf exists so go's resolver respects /etc/hosts
if [[ "${base_image}" =~ busybox ]]; then
echo "COPY nsswitch.conf /etc/" >> "${docker_file_path}"
fi
"${DOCKER[@]}" build ${docker_build_opts:+"${docker_build_opts}"} -q -t "${docker_image_tag}" "${docker_build_path}" >/dev/null
# If we are building an official/alpha/beta release we want to keep
# docker images and tag them appropriately.
local -r release_docker_image_tag="${IAM_DOCKER_REGISTRY-$docker_registry}/${binary_name}-${arch}:${IAM_DOCKER_IMAGE_TAG-$docker_tag}"
if [[ "${release_docker_image_tag}" != "${docker_image_tag}" ]]; then
iam::log::status "Tagging docker image ${docker_image_tag} as ${release_docker_image_tag}"
"${DOCKER[@]}" rmi "${release_docker_image_tag}" 2>/dev/null || true
"${DOCKER[@]}" tag "${docker_image_tag}" "${release_docker_image_tag}" 2>/dev/null
fi
"${DOCKER[@]}" save -o "${binary_file_path}.tar" "${docker_image_tag}" "${release_docker_image_tag}"
echo "${docker_tag}" > "${binary_file_path}.docker_tag"
rm -rf "${docker_build_path}"
ln "${binary_file_path}.tar" "${images_dir}/"
iam::log::status "Deleting docker image ${docker_image_tag}"
"${DOCKER[@]}" rmi "${docker_image_tag}" &>/dev/null || true
) &
done
if [[ "${IAM_BUILD_CONFORMANCE}" =~ [yY] ]]; then
iam::release::build_conformance_image "${arch}" "${docker_registry}" \
"${docker_tag}" "${images_dir}" &
fi
iam::util::wait-for-jobs || { iam::log::error "previous Docker build failed"; return 1; }
iam::log::status "Docker builds done"
)
}
# This will pack iam-system manifests files for distros such as COS.
function iam::release::package_iam_manifests_tarball() {
iam::log::status "Building tarball: manifests"
local src_dir="${IAM_ROOT}/deployments"
local release_stage="${RELEASE_STAGE}/manifests/iam"
rm -rf "${release_stage}"
local dst_dir="${release_stage}"
mkdir -p "${dst_dir}"
cp -r ${src_dir}/* "${dst_dir}"
#cp "${src_dir}/iam-apiserver.yaml" "${dst_dir}"
#cp "${src_dir}/iam-authz-server.yaml" "${dst_dir}"
#cp "${src_dir}/iam-pump.yaml" "${dst_dir}"
#cp "${src_dir}/iam-watcher.yaml" "${dst_dir}"
#cp "${IAM_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh"
iam::release::clean_cruft
local package_name="${RELEASE_TARS}/iam-manifests.tar.gz"
iam::release::create_tarball "${package_name}" "${release_stage}/.."
}
# This is all the platform-independent stuff you need to run/install iam.
# Arch-specific binaries will need to be downloaded separately (possibly by
# using the bundled cluster/get-iam-binaries.sh script).
# Included in this tarball:
# - Cluster spin up/down scripts and configs for various cloud providers
# - Tarballs for manifest configs that are ready to be uploaded
# - Examples (which may or may not still work)
# - The remnants of the docs/ directory
function iam::release::package_final_tarball() {
iam::log::status "Building tarball: final"
# This isn't a "full" tarball anymore, but the release lib still expects
# artifacts under "full/iam/"
local release_stage="${RELEASE_STAGE}/full/iam"
rm -rf "${release_stage}"
mkdir -p "${release_stage}"
mkdir -p "${release_stage}/client"
cat <<EOF > "${release_stage}/client/README"
Client binaries are no longer included in the IAM final tarball.
Run release/get-iam-binaries.sh to download client and server binaries.
EOF
# We want everything in /scripts.
mkdir -p "${release_stage}/release"
cp -R "${IAM_ROOT}/scripts/release" "${release_stage}/"
cat <<EOF > "${release_stage}/release/get-iam-binaries.sh"
#!/usr/bin/env bash
# Copyright 2020 Lingfei Kong <colin404@foxmail.com>. All rights reserved.
# Use of this source code is governed by a MIT style
# license that can be found in the LICENSE file.
# This file download iam client and server binaries from tencent cos bucket.
os=linux arch=amd64 version=${IAM_GIT_VERSION} && wget https://${BUCKET}.cos.${REGION}.myqcloud.com/${COS_RELEASE_DIR}/\$version/{iam-client-\$os-\$arch.tar.gz,iam-server-\$os-\$arch.tar.gz}
EOF
chmod +x ${release_stage}/release/get-iam-binaries.sh
mkdir -p "${release_stage}/server"
cp "${RELEASE_TARS}/iam-manifests.tar.gz" "${release_stage}/server/"
cat <<EOF > "${release_stage}/server/README"
Server binary tarballs are no longer included in the IAM final tarball.
Run release/get-iam-binaries.sh to download client and server binaries.
EOF
# Include hack/lib as a dependency for the cluster/ scripts
#mkdir -p "${release_stage}/hack"
#cp -R "${IAM_ROOT}/hack/lib" "${release_stage}/hack/"
cp -R ${IAM_ROOT}/{docs,configs,scripts,deployments,init,README.md,LICENSE} "${release_stage}/"
echo "${IAM_GIT_VERSION}" > "${release_stage}/version"
iam::release::clean_cruft
local package_name="${RELEASE_TARS}/${ARTIFACT}"
iam::release::create_tarball "${package_name}" "${release_stage}/.."
}
# Build a release tarball. $1 is the output tar name. $2 is the base directory
# of the files to be packaged. This assumes that ${2}/iamis what is
# being packaged.
function iam::release::create_tarball() {
iam::build::ensure_tar
local tarfile=$1
local stagingdir=$2
"${TAR}" czf "${tarfile}" -C "${stagingdir}" iam --owner=0 --group=0
}
function iam::release::install_github_release(){
GO111MODULE=on go install github.com/github-release/github-release@latest
}
# Require the following tools:
# - github-release
# - gsemver
# - git-chglog
# - coscmd or coscli
function iam::release::verify_prereqs(){
if [ -z "$(which github-release 2>/dev/null)" ]; then
iam::log::info "'github-release' tool not installed, try to install it."
if ! iam::release::install_github_release; then
iam::log::error "failed to install 'github-release'"
return 1
fi
fi
if [ -z "$(which git-chglog 2>/dev/null)" ]; then
iam::log::info "'git-chglog' tool not installed, try to install it."
if ! go install github.com/git-chglog/git-chglog/cmd/git-chglog@latest &>/dev/null; then
iam::log::error "failed to install 'git-chglog'"
return 1
fi
fi
if [ -z "$(which gsemver 2>/dev/null)" ]; then
iam::log::info "'gsemver' tool not installed, try to install it."
if ! go install github.com/arnaud-deprez/gsemver@latest &>/dev/null; then
iam::log::error "failed to install 'gsemver'"
return 1
fi
fi
if [ -z "$(which ${COSTOOL} 2>/dev/null)" ]; then
iam::log::info "${COSTOOL} tool not installed, try to install it."
if ! make -C "${IAM_ROOT}" tools.install.${COSTOOL}; then
iam::log::error "failed to install ${COSTOOL}"
return 1
fi
fi
if [ -z "${TENCENT_SECRET_ID}" -o -z "${TENCENT_SECRET_KEY}" ];then
iam::log::error "can not find env: TENCENT_SECRET_ID and TENCENT_SECRET_KEY"
return 1
fi
if [ "${COSTOOL}" == "coscli" ];then
if [ ! -f "${HOME}/.cos.yaml" ];then
cat << EOF > "${HOME}/.cos.yaml"
cos:
base:
secretid: ${TENCENT_SECRET_ID}
secretkey: ${TENCENT_SECRET_KEY}
sessiontoken: ""
buckets:
- name: ${BUCKET}
alias: ${BUCKET}
region: ${REGION}
EOF
fi
else
if [ ! -f "${HOME}/.cos.conf" ];then
cat << EOF > "${HOME}/.cos.conf"
[common]
secret_id = ${TENCENT_SECRET_ID}
secret_key = ${TENCENT_SECRET_KEY}
bucket = ${BUCKET}
region =${REGION}
max_thread = 5
part_size = 1
schema = https
EOF
fi
fi
}
# Create a github release with specified tarballs.
# NOTICE: Must export 'GITHUB_TOKEN' env in the shell, details:
# https://github.com/github-release/github-release
function iam::release::github_release() {
# create a github release
iam::log::info "create a new github release with tag ${IAM_GIT_VERSION}"
github-release release \
--user ${IAM_GITHUB_ORG} \
--repo ${IAM_GITHUB_REPO} \
--tag ${IAM_GIT_VERSION} \
--description "" \
--pre-release
# update iam tarballs
iam::log::info "upload ${ARTIFACT} to release ${IAM_GIT_VERSION}"
github-release upload \
--user ${IAM_GITHUB_ORG} \
--repo ${IAM_GITHUB_REPO} \
--tag ${IAM_GIT_VERSION} \
--name ${ARTIFACT} \
--file ${RELEASE_TARS}/${ARTIFACT}
iam::log::info "upload iam-src.tar.gz to release ${IAM_GIT_VERSION}"
github-release upload \
--user ${IAM_GITHUB_ORG} \
--repo ${IAM_GITHUB_REPO} \
--tag ${IAM_GIT_VERSION} \
--name "iam-src.tar.gz" \
--file ${RELEASE_TARS}/iam-src.tar.gz
}
function iam::release::generate_changelog() {
iam::log::info "generate CHANGELOG-${IAM_GIT_VERSION#v}.md and commit it"
git-chglog ${IAM_GIT_VERSION} > ${IAM_ROOT}/CHANGELOG/CHANGELOG-${IAM_GIT_VERSION#v}.md
set +o errexit
git add ${IAM_ROOT}/CHANGELOG/CHANGELOG-${IAM_GIT_VERSION#v}.md
git commit -a -m "docs(changelog): add CHANGELOG-${IAM_GIT_VERSION#v}.md"
git push -f origin master # 最后将 CHANGELOG 也 push 上去
}

683
scripts/lib/util.sh
Unescape View File

@ -0,0 +1,683 @@
#!/usr/bin/env bash
# Copyright 2020 Lingfei Kong <colin404@foxmail.com>. All rights reserved.
# Use of this source code is governed by a MIT style
# license that can be found in the LICENSE file.
function iam::util::sourced_variable {
# Call this function to tell shellcheck that a variable is supposed to
# be used from other calling context. This helps quiet an "unused
# variable" warning from shellcheck and also document your code.
true
}
iam::util::sortable_date() {
date "+%Y%m%d-%H%M%S"
}
# arguments: target, item1, item2, item3, ...
# returns 0 if target is in the given items, 1 otherwise.
iam::util::array_contains() {
local search="$1"
local element
shift
for element; do
if [[ "${element}" == "${search}" ]]; then
return 0
fi
done
return 1
}
iam::util::wait_for_url() {
local url=$1
local prefix=${2:-}
local wait=${3:-1}
local times=${4:-30}
local maxtime=${5:-1}
command -v curl >/dev/null || {
iam::log::usage "curl must be installed"
exit 1
}
local i
for i in $(seq 1 "${times}"); do
local out
if out=$(curl --max-time "${maxtime}" -gkfs "${url}" 2>/dev/null); then
iam::log::status "On try ${i}, ${prefix}: ${out}"
return 0
fi
sleep "${wait}"
done
iam::log::error "Timed out waiting for ${prefix} to answer at ${url}; tried ${times} waiting ${wait} between each"
return 1
}
# Example: iam::util::wait_for_success 120 5 "iamctl get nodes|grep localhost"
# arguments: wait time, sleep time, shell command
# returns 0 if the shell command get output, 1 otherwise.
iam::util::wait_for_success(){
local wait_time="$1"
local sleep_time="$2"
local cmd="$3"
while [ "$wait_time" -gt 0 ]; do
if eval "$cmd"; then
return 0
else
sleep "$sleep_time"
wait_time=$((wait_time-sleep_time))
fi
done
return 1
}
# Example: iam::util::trap_add 'echo "in trap DEBUG"' DEBUG
# See: http://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal
iam::util::trap_add() {
local trap_add_cmd
trap_add_cmd=$1
shift
for trap_add_name in "$@"; do
local existing_cmd
local new_cmd
# Grab the currently defined trap commands for this trap
existing_cmd=$(trap -p "${trap_add_name}" | awk -F"'" '{print $2}')
if [[ -z "${existing_cmd}" ]]; then
new_cmd="${trap_add_cmd}"
else
new_cmd="${trap_add_cmd};${existing_cmd}"
fi
# Assign the test. Disable the shellcheck warning telling that trap
# commands should be single quoted to avoid evaluating them at this
# point instead evaluating them at run time. The logic of adding new
# commands to a single trap requires them to be evaluated right away.
# shellcheck disable=SC2064
trap "${new_cmd}" "${trap_add_name}"
done
}
# Opposite of iam::util::ensure-temp-dir()
iam::util::cleanup-temp-dir() {
rm -rf "${IAM_TEMP}"
}
# Create a temp dir that'll be deleted at the end of this bash session.
#
# Vars set:
# IAM_TEMP
iam::util::ensure-temp-dir() {
if [[ -z ${IAM_TEMP-} ]]; then
IAM_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t iamrnetes.XXXXXX)
iam::util::trap_add iam::util::cleanup-temp-dir EXIT
fi
}
iam::util::host_os() {
local host_os
case "$(uname -s)" in
Darwin)
host_os=darwin
;;
Linux)
host_os=linux
;;
*)
iam::log::error "Unsupported host OS. Must be Linux or Mac OS X."
exit 1
;;
esac
echo "${host_os}"
}
iam::util::host_arch() {
local host_arch
case "$(uname -m)" in
x86_64*)
host_arch=amd64
;;
i?86_64*)
host_arch=amd64
;;
amd64*)
host_arch=amd64
;;
aarch64*)
host_arch=arm64
;;
arm64*)
host_arch=arm64
;;
arm*)
host_arch=arm
;;
i?86*)
host_arch=x86
;;
s390x*)
host_arch=s390x
;;
ppc64le*)
host_arch=ppc64le
;;
*)
iam::log::error "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x or ppc64le."
exit 1
;;
esac
echo "${host_arch}"
}
# This figures out the host platform without relying on golang. We need this as
# we don't want a golang install to be a prerequisite to building yet we need
# this info to figure out where the final binaries are placed.
iam::util::host_platform() {
echo "$(iam::util::host_os)/$(iam::util::host_arch)"
}
# looks for $1 in well-known output locations for the platform ($2)
# $IAM_ROOT must be set
iam::util::find-binary-for-platform() {
local -r lookfor="$1"
local -r platform="$2"
local locations=(
"${IAM_ROOT}/_output/bin/${lookfor}"
"${IAM_ROOT}/_output/${platform}/${lookfor}"
"${IAM_ROOT}/_output/local/bin/${platform}/${lookfor}"
"${IAM_ROOT}/_output/platforms/${platform}/${lookfor}"
)
# List most recently-updated location.
local -r bin=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 )
echo -n "${bin}"
}
# looks for $1 in well-known output locations for the host platform
# $IAM_ROOT must be set
iam::util::find-binary() {
iam::util::find-binary-for-platform "$1" "$(iam::util::host_platform)"
}
# Run all known doc generators (today gendocs and genman for iamctl)
# $1 is the directory to put those generated documents
iam::util::gen-docs() {
local dest="$1"
# Find binary
gendocs=$(iam::util::find-binary "gendocs")
geniamdocs=$(iam::util::find-binary "geniamdocs")
genman=$(iam::util::find-binary "genman")
genyaml=$(iam::util::find-binary "genyaml")
genfeddocs=$(iam::util::find-binary "genfeddocs")
# TODO: If ${genfeddocs} is not used from anywhere (it isn't used at
# least from k/k tree), remove it completely.
iam::util::sourced_variable "${genfeddocs}"
mkdir -p "${dest}/docs/guide/en-US/cmd/iamctl/"
"${gendocs}" "${dest}/docs/guide/en-US/cmd/iamctl/"
mkdir -p "${dest}/docs/guide/en-US/cmd/"
"${geniamdocs}" "${dest}/docs/guide/en-US/cmd/" "iam-apiserver"
"${geniamdocs}" "${dest}/docs/guide/en-US/cmd/" "iam-authz-server"
"${geniamdocs}" "${dest}/docs/guide/en-US/cmd/" "iam-pump"
"${geniamdocs}" "${dest}/docs/guide/en-US/cmd/" "iam-watcher"
"${geniamdocs}" "${dest}/docs/guide/en-US/cmd/iamctl" "iamctl"
mkdir -p "${dest}/docs/man/man1/"
"${genman}" "${dest}/docs/man/man1/" "iam-apiserver"
"${genman}" "${dest}/docs/man/man1/" "iam-authz-server"
"${genman}" "${dest}/docs/man/man1/" "iam-pump"
"${genman}" "${dest}/docs/man/man1/" "iam-watcher"
"${genman}" "${dest}/docs/man/man1/" "iamctl"
mkdir -p "${dest}/docs/guide/en-US/yaml/iamctl/"
"${genyaml}" "${dest}/docs/guide/en-US/yaml/iamctl/"
# create the list of generated files
pushd "${dest}" > /dev/null || return 1
touch docs/.generated_docs
find . -type f | cut -sd / -f 2- | LC_ALL=C sort > docs/.generated_docs
popd > /dev/null || return 1
}
# Removes previously generated docs-- we don't want to check them in. $IAM_ROOT
# must be set.
iam::util::remove-gen-docs() {
if [ -e "${IAM_ROOT}/docs/.generated_docs" ]; then
# remove all of the old docs; we don't want to check them in.
while read -r file; do
rm "${IAM_ROOT}/${file}" 2>/dev/null || true
done <"${IAM_ROOT}/docs/.generated_docs"
# The docs/.generated_docs file lists itself, so we don't need to explicitly
# delete it.
fi
}
# Returns the name of the upstream remote repository name for the local git
# repo, e.g. "upstream" or "origin".
iam::util::git_upstream_remote_name() {
git remote -v | grep fetch |\
grep -E 'github.com[/:]marmotedu/iam|marmotedu.io/iam' |\
head -n 1 | awk '{print $1}'
}
# Exits script if working directory is dirty. If it's run interactively in the terminal
# the user can commit changes in a second terminal. This script will wait.
iam::util::ensure_clean_working_dir() {
while ! git diff HEAD --exit-code &>/dev/null; do
echo -e "\nUnexpected dirty working directory:\n"
if tty -s; then
git status -s
else
git diff -a # be more verbose in log files without tty
exit 1
fi | sed 's/^/ /'
echo -e "\nCommit your changes in another terminal and then continue here by pressing enter."
read -r
done 1>&2
}
# Find the base commit using:
# $PULL_BASE_SHA if set (from Prow)
# current ref from the remote upstream branch
iam::util::base_ref() {
local -r git_branch=$1
if [[ -n ${PULL_BASE_SHA:-} ]]; then
echo "${PULL_BASE_SHA}"
return
fi
full_branch="$(iam::util::git_upstream_remote_name)/${git_branch}"
# make sure the branch is valid, otherwise the check will pass erroneously.
if ! git describe "${full_branch}" >/dev/null; then
# abort!
exit 1
fi
echo "${full_branch}"
}
# Checks whether there are any files matching pattern $2 changed between the
# current branch and upstream branch named by $1.
# Returns 1 (false) if there are no changes
# 0 (true) if there are changes detected.
iam::util::has_changes() {
local -r git_branch=$1
local -r pattern=$2
local -r not_pattern=${3:-totallyimpossiblepattern}
local base_ref
base_ref=$(iam::util::base_ref "${git_branch}")
echo "Checking for '${pattern}' changes against '${base_ref}'"
# notice this uses ... to find the first shared ancestor
if git diff --name-only "${base_ref}...HEAD" | grep -v -E "${not_pattern}" | grep "${pattern}" > /dev/null; then
return 0
fi
# also check for pending changes
if git status --porcelain | grep -v -E "${not_pattern}" | grep "${pattern}" > /dev/null; then
echo "Detected '${pattern}' uncommitted changes."
return 0
fi
echo "No '${pattern}' changes detected."
return 1
}
iam::util::download_file() {
local -r url=$1
local -r destination_file=$2
rm "${destination_file}" 2&> /dev/null || true
for i in $(seq 5)
do
if ! curl -fsSL --retry 3 --keepalive-time 2 "${url}" -o "${destination_file}"; then
echo "Downloading ${url} failed. $((5-i)) retries left."
sleep 1
else
echo "Downloading ${url} succeed"
return 0
fi
done
return 1
}
# Test whether openssl is installed.
# Sets:
# OPENSSL_BIN: The path to the openssl binary to use
function iam::util::test_openssl_installed {
if ! openssl version >& /dev/null; then
echo "Failed to run openssl. Please ensure openssl is installed"
exit 1
fi
OPENSSL_BIN=$(command -v openssl)
}
# creates a client CA, args are sudo, dest-dir, ca-id, purpose
# purpose is dropped in after "key encipherment", you usually want
# '"client auth"'
# '"server auth"'
# '"client auth","server auth"'
function iam::util::create_signing_certkey {
local sudo=$1
local dest_dir=$2
local id=$3
local purpose=$4
# Create client ca
${sudo} /usr/bin/env bash -e <<EOF
rm -f "${dest_dir}/${id}-ca.crt" "${dest_dir}/${id}-ca.key"
${OPENSSL_BIN} req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout "${dest_dir}/${id}-ca.key" -out "${dest_dir}/${id}-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/"
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment",${purpose}]}}}' > "${dest_dir}/${id}-ca-config.json"
EOF
}
# signs a client certificate: args are sudo, dest-dir, CA, filename (roughly), username, groups...
function iam::util::create_client_certkey {
local sudo=$1
local dest_dir=$2
local ca=$3
local id=$4
local cn=${5:-$4}
local groups=""
local SEP=""
shift 5
while [ -n "${1:-}" ]; do
groups+="${SEP}{\"O\":\"$1\"}"
SEP=","
shift 1
done
${sudo} /usr/bin/env bash -e <<EOF
cd ${dest_dir}
echo '{"CN":"${cn}","names":[${groups}],"hosts":[""],"key":{"algo":"rsa","size":2048}}' | ${CFSSL_BIN} gencert -ca=${ca}.crt -ca-key=${ca}.key -config=${ca}-config.json - | ${CFSSLJSON_BIN} -bare client-${id}
mv "client-${id}-key.pem" "client-${id}.key"
mv "client-${id}.pem" "client-${id}.crt"
rm -f "client-${id}.csr"
EOF
}
# signs a serving certificate: args are sudo, dest-dir, ca, filename (roughly), subject, hosts...
function iam::util::create_serving_certkey {
local sudo=$1
local dest_dir=$2
local ca=$3
local id=$4
local cn=${5:-$4}
local hosts=""
local SEP=""
shift 5
while [ -n "${1:-}" ]; do
hosts+="${SEP}\"$1\""
SEP=","
shift 1
done
${sudo} /usr/bin/env bash -e <<EOF
cd ${dest_dir}
echo '{"CN":"${cn}","hosts":[${hosts}],"key":{"algo":"rsa","size":2048}}' | ${CFSSL_BIN} gencert -ca=${ca}.crt -ca-key=${ca}.key -config=${ca}-config.json - | ${CFSSLJSON_BIN} -bare serving-${id}
mv "serving-${id}-key.pem" "serving-${id}.key"
mv "serving-${id}.pem" "serving-${id}.crt"
rm -f "serving-${id}.csr"
EOF
}
# creates a self-contained iamconfig: args are sudo, dest-dir, ca file, host, port, client id, token(optional)
function iam::util::write_client_iamconfig {
local sudo=$1
local dest_dir=$2
local ca_file=$3
local api_host=$4
local api_port=$5
local client_id=$6
local token=${7:-}
cat <<EOF | ${sudo} tee "${dest_dir}"/"${client_id}".iamconfig > /dev/null
apiVersion: v1
kind: Config
clusters:
- cluster:
certificate-authority: ${ca_file}
server: https://${api_host}:${api_port}/
name: local-up-cluster
users:
- user:
token: ${token}
client-certificate: ${dest_dir}/client-${client_id}.crt
client-key: ${dest_dir}/client-${client_id}.key
name: local-up-cluster
contexts:
- context:
cluster: local-up-cluster
user: local-up-cluster
name: local-up-cluster
current-context: local-up-cluster
EOF
# flatten the iamconfig files to make them self contained
username=$(whoami)
${sudo} /usr/bin/env bash -e <<EOF
$(iam::util::find-binary iamctl) --iamconfig="${dest_dir}/${client_id}.iamconfig" config view --minify --flatten > "/tmp/${client_id}.iamconfig"
mv -f "/tmp/${client_id}.iamconfig" "${dest_dir}/${client_id}.iamconfig"
chown ${username} "${dest_dir}/${client_id}.iamconfig"
EOF
}
# Determines if docker can be run, failures may simply require that the user be added to the docker group.
function iam::util::ensure_docker_daemon_connectivity {
IFS=" " read -ra DOCKER <<< "${DOCKER_OPTS}"
# Expand ${DOCKER[@]} only if it's not unset. This is to work around
# Bash 3 issue with unbound variable.
DOCKER=(docker ${DOCKER[@]:+"${DOCKER[@]}"})
if ! "${DOCKER[@]}" info > /dev/null 2>&1 ; then
cat <<'EOF' >&2
Can't connect to 'docker' daemon. please fix and retry.
Possible causes:
- Docker Daemon not started
- Linux: confirm via your init system
- macOS w/ docker-machine: run `docker-machine ls` and `docker-machine start <name>`
- macOS w/ Docker for Mac: Check the menu bar and start the Docker application
- DOCKER_HOST hasn't been set or is set incorrectly
- Linux: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}`
- macOS w/ docker-machine: run `eval "$(docker-machine env <name>)"`
- macOS w/ Docker for Mac: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}`
- Other things to check:
- Linux: User isn't in 'docker' group. Add and relogin.
- Something like 'sudo usermod -a -G docker ${USER}'
- RHEL7 bug and workaround: https://bugzilla.redhat.com/show_bug.cgi?id=1119282#c8
EOF
return 1
fi
}
# Wait for background jobs to finish. Return with
# an error status if any of the jobs failed.
iam::util::wait-for-jobs() {
local fail=0
local job
for job in $(jobs -p); do
wait "${job}" || fail=$((fail + 1))
done
return ${fail}
}
# iam::util::join <delim> <list...>
# Concatenates the list elements with the delimiter passed as first parameter
#
# Ex: iam::util::join , a b c
# -> a,b,c
function iam::util::join {
local IFS="$1"
shift
echo "$*"
}
# Downloads cfssl/cfssljson/cfssl-certinfo into $1 directory if they do not already exist in PATH
#
# Assumed vars:
# $1 (cfssl directory) (optional)
#
# Sets:
# CFSSL_BIN: The path of the installed cfssl binary
# CFSSLJSON_BIN: The path of the installed cfssljson binary
# CFSSLCERTINFO_BIN: The path of the installed cfssl-certinfo binary
#
function iam::util::ensure-cfssl {
if command -v cfssl &>/dev/null && command -v cfssljson &>/dev/null && command -v cfssl-certinfo &>/dev/null; then
CFSSL_BIN=$(command -v cfssl)
CFSSLJSON_BIN=$(command -v cfssljson)
CFSSLCERTINFO_BIN=$(command -v cfssl-certinfo)
return 0
fi
host_arch=$(iam::util::host_arch)
if [[ "${host_arch}" != "amd64" ]]; then
echo "Cannot download cfssl on non-amd64 hosts and cfssl does not appear to be installed."
echo "Please install cfssl, cfssljson and cfssl-certinfo and verify they are in \$PATH."
echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..."
exit 1
fi
# Create a temp dir for cfssl if no directory was given
local cfssldir=${1:-}
if [[ -z "${cfssldir}" ]]; then
cfssldir="$HOME/bin"
fi
mkdir -p "${cfssldir}"
pushd "${cfssldir}" > /dev/null || return 1
echo "Unable to successfully run 'cfssl' from ${PATH}; downloading instead..."
kernel=$(uname -s)
case "${kernel}" in
Linux)
curl --retry 10 -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
curl --retry 10 -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
curl --retry 10 -L -o cfssl-certinfo https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
;;
Darwin)
curl --retry 10 -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_darwin-amd64
curl --retry 10 -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_darwin-amd64
curl --retry 10 -L -o cfssl-certinfo https://pkg.cfssl.org/R1.2/cfssl-certinfo_darwin-amd64
;;
*)
echo "Unknown, unsupported platform: ${kernel}." >&2
echo "Supported platforms: Linux, Darwin." >&2
exit 2
esac
chmod +x cfssl || true
chmod +x cfssljson || true
chmod +x cfssl-certinfo || true
CFSSL_BIN="${cfssldir}/cfssl"
CFSSLJSON_BIN="${cfssldir}/cfssljson"
CFSSLCERTINFO_BIN="${cfssldir}/cfssl-certinfo"
if [[ ! -x ${CFSSL_BIN} || ! -x ${CFSSLJSON_BIN} || ! -x ${CFSSLCERTINFO_BIN} ]]; then
echo "Failed to download 'cfssl'."
echo "Please install cfssl, cfssljson and cfssl-certinfo and verify they are in \$PATH."
echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..."
exit 1
fi
popd > /dev/null || return 1
}
# iam::util::ensure-gnu-sed
# Determines which sed binary is gnu-sed on linux/darwin
#
# Sets:
# SED: The name of the gnu-sed binary
#
function iam::util::ensure-gnu-sed {
# NOTE: the echo below is a workaround to ensure sed is executed before the grep.
# see: https://github.com/iamrnetes/iamrnetes/issues/87251
sed_help="$(LANG=C sed --help 2>&1 || true)"
if echo "${sed_help}" | grep -q "GNU\|BusyBox"; then
SED="sed"
elif command -v gsed &>/dev/null; then
SED="gsed"
else
iam::log::error "Failed to find GNU sed as sed or gsed. If you are on Mac: brew install gnu-sed." >&2
return 1
fi
iam::util::sourced_variable "${SED}"
}
# iam::util::check-file-in-alphabetical-order <file>
# Check that the file is in alphabetical order
#
function iam::util::check-file-in-alphabetical-order {
local failure_file="$1"
if ! diff -u "${failure_file}" <(LC_ALL=C sort "${failure_file}"); then
{
echo
echo "${failure_file} is not in alphabetical order. Please sort it:"
echo
echo " LC_ALL=C sort -o ${failure_file} ${failure_file}"
echo
} >&2
false
fi
}
# iam::util::require-jq
# Checks whether jq is installed.
function iam::util::require-jq {
if ! command -v jq &>/dev/null; then
echo "jq not found. Please install." 1>&2
return 1
fi
}
# outputs md5 hash of $1, works on macOS and Linux
function iam::util::md5() {
if which md5 >/dev/null 2>&1; then
md5 -q "$1"
else
md5sum "$1" | awk '{ print $1 }'
fi
}
# iam::util::read-array
# Reads in stdin and adds it line by line to the array provided. This can be
# used instead of "mapfile -t", and is bash 3 compatible.
#
# Assumed vars:
# $1 (name of array to create/modify)
#
# Example usage:
# iam::util::read-array files < <(ls -1)
#
function iam::util::read-array {
local i=0
unset -v "$1"
while IFS= read -r "$1[i++]"; do :; done
eval "[[ \${$1[--i]} ]]" || unset "$1[i]" # ensures last element isn't empty
}
# Some useful colors.
if [[ -z "${color_start-}" ]]; then
declare -r color_start="\033["
declare -r color_red="${color_start}0;31m"
declare -r color_yellow="${color_start}0;33m"
declare -r color_green="${color_start}0;32m"
declare -r color_blue="${color_start}1;34m"
declare -r color_cyan="${color_start}1;36m"
declare -r color_norm="${color_start}0m"
iam::util::sourced_variable "${color_start}"
iam::util::sourced_variable "${color_red}"
iam::util::sourced_variable "${color_yellow}"
iam::util::sourced_variable "${color_green}"
iam::util::sourced_variable "${color_blue}"
iam::util::sourced_variable "${color_cyan}"
iam::util::sourced_variable "${color_norm}"
fi
# ex: ts=2 sw=2 et filetype=sh

137
scripts/lib/version.sh
Unescape View File

@ -0,0 +1,137 @@
#!/usr/bin/env bash
# Copyright 2020 Lingfei Kong <colin404@foxmail.com>. All rights reserved.
# Use of this source code is governed by a MIT style
# license that can be found in the LICENSE file.
# -----------------------------------------------------------------------------
# Version management helpers. These functions help to set, save and load the
# following variables:
#
# IAM_GIT_COMMIT - The git commit id corresponding to this
# source code.
# IAM_GIT_TREE_STATE - "clean" indicates no changes since the git commit id
# "dirty" indicates source code changes after the git commit id
# "archive" indicates the tree was produced by 'git archive'
# IAM_GIT_VERSION - "vX.Y" used to indicate the last release version.
# IAM_GIT_MAJOR - The major part of the version
# IAM_GIT_MINOR - The minor component of the version
# Grovels through git to set a set of env variables.
#
# If IAM_GIT_VERSION_FILE, this function will load from that file instead of
# querying git.
iam::version::get_version_vars() {
if [[ -n ${IAM_GIT_VERSION_FILE-} ]]; then
iam::version::load_version_vars "${IAM_GIT_VERSION_FILE}"
return
fi
# If the iamrnetes source was exported through git archive, then
# we likely don't have a git tree, but these magic values may be filled in.
# shellcheck disable=SC2016,SC2050
# Disabled as we're not expanding these at runtime, but rather expecting
# that another tool may have expanded these and rewritten the source (!)
if [[ '$Format:%%$' == "%" ]]; then
IAM_GIT_COMMIT='$Format:%H$'
IAM_GIT_TREE_STATE="archive"
# When a 'git archive' is exported, the '$Format:%D$' below will look
# something like 'HEAD -> release-1.8, tag: v1.8.3' where then 'tag: '
# can be extracted from it.
if [[ '$Format:%D$' =~ tag:\ (v[^ ,]+) ]]; then
IAM_GIT_VERSION="${BASH_REMATCH[1]}"
fi
fi
local git=(git --work-tree "${IAM_ROOT}")
if [[ -n ${IAM_GIT_COMMIT-} ]] || IAM_GIT_COMMIT=$("${git[@]}" rev-parse "HEAD^{commit}" 2>/dev/null); then
if [[ -z ${IAM_GIT_TREE_STATE-} ]]; then
# Check if the tree is dirty. default to dirty
if git_status=$("${git[@]}" status --porcelain 2>/dev/null) && [[ -z ${git_status} ]]; then
IAM_GIT_TREE_STATE="clean"
else
IAM_GIT_TREE_STATE="dirty"
fi
fi
# Use git describe to find the version based on tags.
if [[ -n ${IAM_GIT_VERSION-} ]] || IAM_GIT_VERSION=$("${git[@]}" describe --tags --always --match='v*' 2>/dev/null); then
# This translates the "git describe" to an actual semver.org
# compatible semantic version that looks something like this:
# v1.1.0-alpha.0.6+84c76d1142ea4d
#
# TODO: We continue calling this "git version" because so many
# downstream consumers are expecting it there.
#
# These regexes are painful enough in sed...
# We don't want to do them in pure shell, so disable SC2001
# shellcheck disable=SC2001
DASHES_IN_VERSION=$(echo "${IAM_GIT_VERSION}" | sed "s/[^-]//g")
if [[ "${DASHES_IN_VERSION}" == "---" ]] ; then
# shellcheck disable=SC2001
# We have distance to subversion (v1.1.0-subversion-1-gCommitHash)
IAM_GIT_VERSION=$(echo "${IAM_GIT_VERSION}" | sed "s/-\([0-9]\{1,\}\)-g\([0-9a-f]\{14\}\)$/.\1\+\2/")
elif [[ "${DASHES_IN_VERSION}" == "--" ]] ; then
# shellcheck disable=SC2001
# We have distance to base tag (v1.1.0-1-gCommitHash)
IAM_GIT_VERSION=$(echo "${IAM_GIT_VERSION}" | sed "s/-g\([0-9a-f]\{14\}\)$/+\1/")
fi
if [[ "${IAM_GIT_TREE_STATE}" == "dirty" ]]; then
# git describe --dirty only considers changes to existing files, but
# that is problematic since new untracked .go files affect the build,
# so use our idea of "dirty" from git status instead.
# TODO?
#IAM_GIT_VERSION+="-dirty"
:
fi
# Try to match the "git describe" output to a regex to try to extract
# the "major" and "minor" versions and whether this is the exact tagged
# version or whether the tree is between two tagged versions.
if [[ "${IAM_GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?([-].*)?([+].*)?$ ]]; then
IAM_GIT_MAJOR=${BASH_REMATCH[1]}
IAM_GIT_MINOR=${BASH_REMATCH[2]}
if [[ -n "${BASH_REMATCH[4]}" ]]; then
IAM_GIT_MINOR+="+"
fi
fi
# If IAM_GIT_VERSION is not a valid Semantic Version, then refuse to build.
if ! [[ "${IAM_GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then
echo "IAM_GIT_VERSION should be a valid Semantic Version. Current value: ${IAM_GIT_VERSION}"
echo "Please see more details here: https://semver.org"
exit 1
fi
fi
fi
}
# Saves the environment flags to $1
iam::version::save_version_vars() {
local version_file=${1-}
[[ -n ${version_file} ]] || {
echo "!!! Internal error. No file specified in iam::version::save_version_vars"
return 1
}
cat <<EOF >"${version_file}"
IAM_GIT_COMMIT='${IAM_GIT_COMMIT-}'
IAM_GIT_TREE_STATE='${IAM_GIT_TREE_STATE-}'
IAM_GIT_VERSION='${IAM_GIT_VERSION-}'
IAM_GIT_MAJOR='${IAM_GIT_MAJOR-}'
IAM_GIT_MINOR='${IAM_GIT_MINOR-}'
EOF
}
# Loads up the version variables from file $1
iam::version::load_version_vars() {
local version_file=${1-}
[[ -n ${version_file} ]] || {
echo "!!! Internal error. No file specified in iam::version::load_version_vars"
return 1
}
source "${version_file}"
}

11
scripts/make-rules/golang.mk
Unescape View File

@ -132,24 +132,17 @@ go.build.multiarch: go.build.verify $(foreach p,$(PLATFORMS),$(addprefix go.buil
go.lint: tools.verify.golangci-lint go.lint: tools.verify.golangci-lint
@echo "===========> Run golangci to lint source codes" @echo "===========> Run golangci to lint source codes"
@$(TOOLS_DIR)/golangci-lint run --color always -c $(ROOT_DIR)/.golangci.yml $(ROOT_DIR)/... @$(TOOLS_DIR)/golangci-lint run --color always -c $(ROOT_DIR)/.golangci.yml $(ROOT_DIR)/...
## go.test: Run unit test ## go.test: Run unit test
.PHONY: go.test .PHONY: go.test
go.test: go.test:
@$(GO) test ./... @$(GO) test ./...
# ## go.test.junit-report: Run unit test
# .PHONY: go.test.junit-report
# go.test.junit-report: tools.verify.go-junit-report
# @echo "===========> Run unit test > $(TMP_DIR)/report.xml"
# @$(GO) test -v -coverprofile=$(TMP_DIR)/coverage.out 2>&1 $(GO_BUILD_FLAGS) ./... | $(TOOLS_DIR)/go-junit-report -set-exit-code > $(TMP_DIR)/report.xml
# @sed -i '/mock_.*.go/d' $(TMP_DIR)/coverage.out
# @echo "===========> Test coverage of Go code is reported to $(TMP_DIR)/coverage.html by generating HTML"
# @$(GO) tool cover -html=$(TMP_DIR)/coverage.out -o $(TMP_DIR)/coverage.html
## go.test.junit-report: Run unit test ## go.test.junit-report: Run unit test
.PHONY: go.test.junit-report .PHONY: go.test.junit-report
go.test.junit-report: tools.verify.go-junit-report go.test.junit-report: tools.verify.go-junit-report
@echo "===========> Run unit test > $(TMP_DIR)/report.xml" @echo "===========> Run unit test > $(TMP_DIR)/report.xml"
# @$(GO) test -v -coverprofile=$(TMP_DIR)/coverage.out 2>&1 $(GO_BUILD_FLAGS) ./... | $(TOOLS_DIR)/go-junit-report -set-exit-code > $(TMP_DIR)/report.xml
@$(GO) test -v -coverprofile=$(TMP_DIR)/coverage.out 2>&1 ./... | $(TOOLS_DIR)/go-junit-report -set-exit-code > $(TMP_DIR)/report.xml @$(GO) test -v -coverprofile=$(TMP_DIR)/coverage.out 2>&1 ./... | $(TOOLS_DIR)/go-junit-report -set-exit-code > $(TMP_DIR)/report.xml
@sed -i '/mock_.*.go/d' $(TMP_DIR)/coverage.out @sed -i '/mock_.*.go/d' $(TMP_DIR)/coverage.out
@echo "===========> Test coverage of Go code is reported to $(TMP_DIR)/coverage.html by generating HTML" @echo "===========> Test coverage of Go code is reported to $(TMP_DIR)/coverage.html by generating HTML"

Write Preview
Loading…
Cancel
Save