From 6f3d17b9271df0e9105a58c42e9204ecef2daf71 Mon Sep 17 00:00:00 2001 From: kubbot & kubecub <3293172751ysy@gmail.com> Date: Fri, 30 Jun 2023 09:56:10 +0800 Subject: [PATCH] feat: migrate the original main branch code Signed-off-by: kubbot & kubecub <3293172751ysy@gmail.com> --- .github/workflows/.mlc_config.json | 10 -- .github/workflows/check-coverage.yml | 1 + .github/workflows/e2e-test.yml | 1 + .github/workflows/link-pr.yml | 47 ++++++++ .github/workflows/markdown.yml | 27 ----- script/githooks/commit-msg | 17 ++- script/githooks/pre-commit | 71 ++++++++--- script/githooks/pre-push | 16 +++ script/make-rules/common.mk | 7 +- script/make-rules/copyright.mk | 6 +- script/make-rules/gen.mk | 57 +++++++++ script/make-rules/golang.mk | 172 +++++++++++++++++++++++++++ script/make-rules/image.mk | 128 ++++++++++++++++++++ script/make-rules/release.mk | 42 +++++++ script/make-rules/tools.mk | 39 ++++-- 15 files changed, 568 insertions(+), 73 deletions(-) delete mode 100644 .github/workflows/.mlc_config.json create mode 100644 .github/workflows/link-pr.yml delete mode 100644 .github/workflows/markdown.yml create mode 100644 script/make-rules/release.mk diff --git a/.github/workflows/.mlc_config.json b/.github/workflows/.mlc_config.json deleted file mode 100644 index 37f6b0516..000000000 --- a/.github/workflows/.mlc_config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "ignorePatterns": [ - { - "pattern": "^http://127.0.0.1" - }, - { - "pattern": "^http://localhost" - } - ] -} \ No newline at end of file diff --git a/.github/workflows/check-coverage.yml b/.github/workflows/check-coverage.yml index 9534ee6de..21c383076 100644 --- a/.github/workflows/check-coverage.yml +++ b/.github/workflows/check-coverage.yml @@ -33,6 +33,7 @@ # uses: magnetikonline/action-golang-cache@v3 # with: # go-version: ${{ env.GO_VERSION }} +# token: ${{ secrets.BOT_GITHUB_TOKEN }} # - name: Install Dependencies # run: sudo apt update && sudo apt install -y libgpgme-dev libbtrfs-dev libdevmapper-dev diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index e69de29bb..4e768b56d 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -0,0 +1 @@ +# \ No newline at end of file diff --git a/.github/workflows/link-pr.yml b/.github/workflows/link-pr.yml new file mode 100644 index 000000000..844dd5fde --- /dev/null +++ b/.github/workflows/link-pr.yml @@ -0,0 +1,47 @@ +# name: Github Rebot for Link check error + +# on: +# pull_request: +# branches: [ main ] +# paths: +# - '**.md' +# - 'docs/**' +# - '.lycheeignore' +# push: +# branches: [ main ] + +# schedule: +# - cron: '0 11 * * *' + +# jobs: +# linkChecker: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 + +# - name: Link Checker +# id: lychee +# uses: lycheeverse/lychee-action@v1.7.0 +# with: +# # For parameter description, see https://github.com/lycheeverse/lychee#commandline-parameters +# # Actions Link address -> https://github.com/lycheeverse/lychee-action +# # -E, --exclude-all-private Exclude all private IPs from checking. +# # -i, --insecure Proceed for server connections considered insecure (invalid TLS) +# # -n, --no-progress Do not show progress bar. +# # -t, --timeout Website timeout in seconds from connect to response finished [default:20] +# # --max-concurrency Maximum number of concurrent network requests [default: 128] +# # -a --accept Comma-separated list of accepted status codes for valid links +# # docs/.vitepress/dist the site directory to check +# # ./*.md all markdown files in the root directory +# args: --verbose -E -i --no-progress --exclude-path './CHANGELOG' './**/*.md' +# env: +# GITHUB_TOKEN: ${{secrets.GH_PAT}} + +# - name: Create Issue From File +# if: env.lychee_exit_code != 0 +# uses: peter-evans/create-issue-from-file@v4 +# with: +# title: Bug reports for links in OpenIM docs +# content-filepath: ./lychee/out.md +# labels: kind/documentation, triage/unresolved, report +# token: ${{ secrets.BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/markdown.yml b/.github/workflows/markdown.yml deleted file mode 100644 index 05cb6e8db..000000000 --- a/.github/workflows/markdown.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Markdown-related check - -on: - pull_request: - branches: "*" - push: - branches: "*" - -jobs: - markdown-related-check: - name: markdown-related check - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Markdown Link Check - uses: docker://dragonflyoss/linter:v0.2.7 - with: - # validate all links in markdown files excluding vendor folder - args: bash -c "code=0;for mdFile in $(find ./ -name '*.md' | grep -v vendor | grep -v .github); do markdown-link-check -q --config .github/workflows/.mlc_config.json $mdFile; if [ $? -ne 0 ]; then code=1; fi;done; echo $code; exit $code" - - - name: Markdown Lint - uses: docker://dragonflyoss/linter:v0.2.7 - with: - # lint all markdown files excluding vendor folder - args: bash -c "find ./ -name '*.md' | grep -v vendor | grep -v .github | xargs mdl -r ~MD010,~MD013,~MD022,~MD024,~MD026,~MD029,~MD033,~MD036" diff --git a/script/githooks/commit-msg b/script/githooks/commit-msg index add13ac52..eede298a0 100644 --- a/script/githooks/commit-msg +++ b/script/githooks/commit-msg @@ -1,5 +1,4 @@ #!/usr/bin/env bash - # Copyright © 2023 OpenIMSDK. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -56,9 +55,12 @@ test "" = "$(grep '^Signed-off-by: ' "$1" | exit 1 } -./tools/go-gitlint \ - --msg-file=$1 \ - --subject-regex="^(build|chore|ci|docs|feat|feature|fix|perf|refactor|revert|style|test)(.*)?:\s?.*" \ +# TODO: go-gitlint dir set +GITLINT_DIR="./_output/tools/go-gitlint" + +$GITLINT_DIR \ + --msg-file=$1 \ + --subject-regex="^(build|chore|ci|docs|feat|feature|fix|perf|refactor|revert|style|test)(.*)?:\s?.*" \ --subject-maxlen=150 \ --subject-minlen=10 \ --body-regex=".*" \ @@ -66,7 +68,10 @@ test "" = "$(grep '^Signed-off-by: ' "$1" | if [ $? -ne 0 ] then - printError "Please fix your commit message to match OpenIM coding standards" - printError "https://OpenIM.gitbook.io/docs/essential-documentation/contribute-to-OpenIM/software-contributions/submitting-code/code-submission-guidelines#commit-message-guidelines" + if ! command -v $GITLINT_DIR &>/dev/null; then + printError "$GITLINT_DIR not found. Please run 'make tools' OR 'make tools.verify.go-gitlint' make verto install it." + fi + printError "Please fix your commit message to match kubecub coding standards" + printError "https://gist.github.com/cubxxw/126b72104ac0b0ca484c9db09c3e5694#file-githook-md" exit 1 fi \ No newline at end of file diff --git a/script/githooks/pre-commit b/script/githooks/pre-commit index a52d77f5c..db9d7cf17 100644 --- a/script/githooks/pre-commit +++ b/script/githooks/pre-commit @@ -1,5 +1,4 @@ #!/usr/bin/env bash - # Copyright © 2023 OpenIMSDK. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# ============================================================================== +# This is a pre-commit hook that ensures attempts to commit files that are +# are larger than $limit to your _local_ repo fail, with a helpful error message. + +# You can override the default limit of 2MB by supplying the environment variable: +# GIT_FILE_SIZE_LIMIT=50000000 git commit -m "test: this commit is allowed file sizes up to 50MB" +# +# ============================================================================== +# + +LC_ALL=C + +local_branch="$(git rev-parse --abbrev-ref HEAD)" +valid_branch_regex="^(main|master|develop)$|(feature|feat|release|hotfix|test|bug|ci|style|)\/[a-z0-9._-]+$|^HEAD$" + YELLOW="\e[93m" GREEN="\e[32m" @@ -21,23 +35,25 @@ RED="\e[31m" ENDCOLOR="\e[0m" printMessage() { - printf "${YELLOW}OpenIM : $1${ENDCOLOR}\n" + printf "${YELLOW}openim : $1${ENDCOLOR}\n" } printSuccess() { - printf "${GREEN}OpenIM : $1${ENDCOLOR}\n" + printf "${GREEN}openim : $1${ENDCOLOR}\n" } printError() { - printf "${RED}OpenIM : $1${ENDCOLOR}\n" + printf "${RED}openim : $1${ENDCOLOR}\n" } -printMessage "Running local OpenIM pre-commit hook." +printMessage "Running local openim pre-commit hook." -#flutter format . -##https://gist.github.com/benmccallum/28e4f216d9d72f5965133e6c43aaff6e -# TODO! GIT_FILE_SIZE_LIMIT=42000000 git commit -m "This commit is allowed file sizes up to 42MB" -limit=$(( 2 * 2**20 )) # 2MB +# flutter format . +# https://gist.github.com/cubxxw/126b72104ac0b0ca484c9db09c3e5694#file-githook-md +# TODO! GIT_FILE_SIZE_LIMIT=50000000 git commit -m "test: this commit is allowed file sizes up to 50MB" +# Maximum file size limit in bytes +limit=${GIT_FILE_SIZE_LIMIT:-2000000} # Default 2MB +limitInMB=$(( $limit / 1000000 )) function file_too_large(){ filename=$0 @@ -46,24 +62,51 @@ function file_too_large(){ cat < /dev/null 2>&1 then against=HEAD else - against=empty_tree + against="$empty_tree" fi +# Set split so that for loop below can handle spaces in file names by splitting on line breaks +IFS=' +' + +shouldFail=false for file in $( git diff-index --cached --name-only $against ); do - file_size=$( ls -la $file | awk '{ print $5 }') + file_size=$(([ ! -f $file ] && echo 0) || (ls -la $file | awk '{ print $5 }')) if [ "$file_size" -gt "$limit" ]; then - file_too_large $filename $file_size - exit 1; + printError "File $file is $(( $file_size / 10**6 )) MB, which is larger than our configured limit of $limitInMB MB" + shouldFail=true fi done + +if $shouldFail +then + printMessage "If you really need to commit this file, you can override the size limit by setting the GIT_FILE_SIZE_LIMIT environment variable, e.g. GIT_FILE_SIZE_LIMIT=42000000 for 42MB. Or, commit with the --no-verify switch to skip the check entirely." + printError "Commit aborted" + exit 1; +fi + +if [[ ! $local_branch =~ $valid_branch_regex ]] +then + printError "There is something wrong with your branch name. Branch names in this project must adhere to this contract: $valid_branch_regex. +Your commit will be rejected. You should rename your branch to a valid name(feat/name OR bug/name) and try again." + printError "For more on this, read on: https://gist.github.com/cubxxw/126b72104ac0b0ca484c9db09c3e5694" + exit 1 +fi \ No newline at end of file diff --git a/script/githooks/pre-push b/script/githooks/pre-push index 3402ac656..ef1b5b1fe 100644 --- a/script/githooks/pre-push +++ b/script/githooks/pre-push @@ -1,4 +1,20 @@ #!/usr/bin/env bash +# Copyright © 2023 OpenIMSDK. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== +# YELLOW="\e[93m" GREEN="\e[32m" diff --git a/script/make-rules/common.mk b/script/make-rules/common.mk index e79102d89..18e97f8b6 100644 --- a/script/make-rules/common.mk +++ b/script/make-rules/common.mk @@ -48,13 +48,13 @@ endif # TOOLS_DIR: The directory where tools are stored for build and testing. ifeq ($(origin TOOLS_DIR),undefined) -TOOLS_DIR := $(ROOT_DIR)/tools +TOOLS_DIR := $(OUTPUT_DIR)/tools $(shell mkdir -p $(TOOLS_DIR)) endif # TMP_DIR: directory where temporary files are stored. ifeq ($(origin TMP_DIR),undefined) -TMP_DIR := $(ROOT_DIR)/tmp +TMP_DIR := $(OUTPUT_DIR)/tmp $(shell mkdir -p $(TMP_DIR)) endif @@ -72,6 +72,7 @@ endif GIT_COMMIT:=$(shell git rev-parse HEAD) # Minimum test coverage +# can u use make cover COVERAGE=90 ifeq ($(origin COVERAGE),undefined) COVERAGE := 60 endif @@ -88,6 +89,8 @@ PLATFORMS ?= linux_amd64 linux_arm64 # The OS can be linux/windows/darwin when building binaries # PLATFORMS ?= darwin_amd64 windows_amd64 linux_amd64 linux_arm64 +# only support linux +GOOS=linux # set a specific PLATFORM, defaults to the host platform ifeq ($(origin PLATFORM), undefined) diff --git a/script/make-rules/copyright.mk b/script/make-rules/copyright.mk index 95b8dbe76..4b38e74da 100644 --- a/script/make-rules/copyright.mk +++ b/script/make-rules/copyright.mk @@ -19,20 +19,18 @@ LICENSE_TEMPLATE ?= $(ROOT_DIR)/script/LICENSE/LICENSE_TEMPLATES -# TODO: GOBIN -> TOOLS_DIR -# Questions about go mod instead of go path: https://github.com/kubernetes/kubernetes/issues/117181 ## copyright.verify: Validate boilerplate headers for assign files .PHONY: copyright.verify copyright.verify: tools.verify.addlicense @echo "===========> Validate boilerplate headers for assign files starting in the $(ROOT_DIR) directory" - @$(GOBIN)/addlicense -v -check -ignore **/test/** -f $(LICENSE_TEMPLATE) $(CODE_DIRS) + @$(TOOLS_DIR)/addlicense -v -check -ignore **/test/** -f $(LICENSE_TEMPLATE) $(CODE_DIRS) @echo "===========> End of boilerplate headers check..." ## copyright.add: Add the boilerplate headers for all files .PHONY: copyright.add copyright.add: tools.verify.addlicense @echo "===========> Adding $(LICENSE_TEMPLATE) the boilerplate headers for all files" - @$(GOBIN)/addlicense -y $(shell date +"%Y") -v -c "OpenIM." -f $(LICENSE_TEMPLATE) $(CODE_DIRS) + @$(TOOLS_DIR)/addlicense -y $(shell date +"%Y") -v -c "OpenIM." -f $(LICENSE_TEMPLATE) $(CODE_DIRS) @echo "===========> End the copyright is added..." # Addlicense Flags: diff --git a/script/make-rules/gen.mk b/script/make-rules/gen.mk index c875223c4..e6fe3e63c 100644 --- a/script/make-rules/gen.mk +++ b/script/make-rules/gen.mk @@ -17,7 +17,64 @@ # https://cloud.redhat.com/blog/kubernetes-deep-dive-code-generation-customresources # ! The stock of code generated by `make gen` should be idempotent # +# Questions about go mod instead of go path: https://github.com/kubernetes/kubernetes/issues/117181 +# ============================================================================== +# Makefile helper functions for generate necessary files +# + +.PHONY: gen.run +#gen.run: gen.errcode gen.docgo +gen.run: gen.clean gen.errcode gen.docgo.doc + +.PHONY: gen.errcode +gen.errcode: gen.errcode.code gen.errcode.doc + +.PHONY: gen.errcode.code +gen.errcode.code: tools.verify.codegen + @echo "===========> Generating iam error code go source files" + @codegen -type=int ${ROOT_DIR}/internal/pkg/code + +.PHONY: gen.errcode.doc +gen.errcode.doc: tools.verify.codegen + @echo "===========> Generating error code markdown documentation" + @codegen -type=int -doc \ + -output ${ROOT_DIR}/docs/guide/zh-CN/api/error_code_generated.md ${ROOT_DIR}/internal/pkg/code + +.PHONY: gen.ca.% +gen.ca.%: + $(eval CA := $(word 1,$(subst ., ,$*))) + @echo "===========> Generating CA files for $(CA)" + @${ROOT_DIR}/scripts/gencerts.sh generate-iam-cert $(OUTPUT_DIR)/cert $(CA) + +.PHONY: gen.ca +gen.ca: $(addprefix gen.ca., $(CERTIFICATES)) + +.PHONY: gen.docgo.doc +gen.docgo.doc: + @echo "===========> Generating missing doc.go for go packages" + @${ROOT_DIR}/scripts/gendoc.sh + +.PHONY: gen.docgo.check +gen.docgo.check: gen.docgo.doc + @n="$$(git ls-files --others '*/doc.go' | wc -l)"; \ + if test "$$n" -gt 0; then \ + git ls-files --others '*/doc.go' | sed -e 's/^/ /'; \ + echo "$@: untracked doc.go file(s) exist in working directory" >&2 ; \ + false ; \ + fi + +.PHONY: gen.docgo.add +gen.docgo.add: + @git ls-files --others '*/doc.go' | $(XARGS) -- git add + +.PHONY: gen.defaultconfigs +gen.defaultconfigs: + @${ROOT_DIR}/scripts/gen_default_config.sh +.PHONY: gen.clean +gen.clean: + @rm -rf ./api/client/{clientset,informers,listers} + @$(FIND) -type f -name '*_generated.go' -delete ## gen.help: show help for gen .PHONY: gen.help diff --git a/script/make-rules/golang.mk b/script/make-rules/golang.mk index 1eb0b1aba..8d36620b7 100644 --- a/script/make-rules/golang.mk +++ b/script/make-rules/golang.mk @@ -16,6 +16,178 @@ # Build management helpers. These functions help to set, save and load the # +GO := go +GO_SUPPORTED_VERSIONS ?= |1.15|1.16|1.17|1.18|1.19|1.20| + +GO_LDFLAGS += -X $(VERSION_PACKAGE).gitVersion=$(GIT_TAG) \ + -X $(VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) \ + -X $(VERSION_PACKAGE).gitTreeState=$(GIT_TREE_STATE) \ + -X $(VERSION_PACKAGE).buildDate=$(BUILD_DATE) \ + -s -w # -s -w deletes debugging information and symbol tables +ifeq ($(DEBUG), 1) + GO_BUILD_FLAGS += -gcflags "all=-N -l" + GO_LDFLAGS= +endif + +GO_BUILD_FLAGS += -ldflags "$(GO_LDFLAGS)" + +ifeq ($(GOOS),windows) + GO_OUT_EXT := .exe +endif + +ifeq ($(ROOT_PACKAGE),) + $(error the variable ROOT_PACKAGE must be set prior to including golang.mk, ->/Makefile) +endif + +GOPATH ?= $(shell go env GOPATH) +ifeq ($(origin GOBIN), undefined) + GOBIN := $(GOPATH)/bin +endif + +# COMMANDS is Specify all files under ${ROOT_DIR}/cmd/ except those ending in.md +COMMANDS ?= $(filter-out %.md, $(wildcard ${ROOT_DIR}/cmd/*)) +ifeq (${COMMANDS},) + $(error Could not determine COMMANDS, set ROOT_DIR or run in source dir) +endif + +# BINS is the name of each file in ${COMMANDS}, excluding the directory path +# If there are no files in ${COMMANDS}, or if all files end in.md, ${BINS} will be empty +BINS ?= $(foreach cmd,${COMMANDS},$(notdir ${cmd})) +ifeq (${BINS},) + $(error Could not determine BINS, set ROOT_DIR or run in source dir) +endif + +ifeq (${COMMANDS},) + $(error Could not determine COMMANDS, set ROOT_DIR or run in source dir) +endif +ifeq (${BINS},) + $(error Could not determine BINS, set ROOT_DIR or run in source dir) +endif + +# TODO: EXCLUDE_TESTS variable, which contains the name of the package to be excluded from the test +EXCLUDE_TESTS=github.com/OpenIMSDK/Open-IM-Server/test github.com/OpenIMSDK/Open-IM-Server/pkg/log github.com/OpenIMSDK/Open-IM-Server/db github.com/OpenIMSDK/Open-IM-Server/script github.com/OpenIMSDK/Open-IM-Server/deploy_k8s github.com/OpenIMSDK/Open-IM-Server/deploy github.com/OpenIMSDK/Open-IM-Server/config + +# ============================================================================== +# ❯ tree -L 1 cmd +# cmd +# ├── openim-sdk-core/ - main.go +# ├── open_im_api +# ├── open_im_cms_api +# ├── open_im_cron_task +# ├── open_im_demo +# ├── open_im_msg_gateway +# ├── open_im_msg_transfer +# ├── open_im_push +# ├── rpc/open_im_admin_cms/ - main.go +# └── test/ - main.go +# COMMAND=openim +# PLATFORM=linux_amd64 +# OS=linux +# ARCH=amd64 +# BINS=open_im_api open_im_cms_api open_im_cron_task open_im_demo open_im_msg_gateway open_im_msg_transfer open_im_push +# BIN_DIR=/root/workspaces/OpenIM/_output/bin +# ============================================================================== + +## go.build: Build binaries +.PHONY: go.build +go.build: go.build.verify $(addprefix go.build., $(addprefix $(PLATFORM)., $(BINS))) + @echo "===========> Building binary $(BINS) $(VERSION) for $(PLATFORM)" + +## go.build.verify: Verify that a suitable version of Go exists +.PHONY: go.build.verify +go.build.verify: +ifneq ($(shell $(GO) version | grep -q -E '\bgo($(GO_SUPPORTED_VERSIONS))\b' && echo 0 || echo 1), 0) + $(error unsupported go version. Please make install one of the following supported version: '$(GO_SUPPORTED_VERSIONS)') +endif + +.PHONY: go.build.% +go.build.%: + $(eval COMMAND := $(word 2,$(subst ., ,$*))) + $(eval PLATFORM := $(word 1,$(subst ., ,$*))) + $(eval OS := $(word 1,$(subst _, ,$(PLATFORM)))) + $(eval ARCH := $(word 2,$(subst _, ,$(PLATFORM)))) + @echo "=====> COMMAND=$(COMMAND)" + @echo "=====> PLATFORM=$(PLATFORM)" + @echo "=====> BIN_DIR=$(BIN_DIR)" + @echo "===========> Building binary $(COMMAND) $(VERSION) for $(OS)_$(ARCH)" + @mkdir -p $(BIN_DIR)/platforms/$(OS)/$(ARCH) + @if [ "$(COMMAND)" == "openim-sdk-core" ]; then \ + echo "===========> DEBUG: Compilation is not yet supported $(COMMAND)"; \ + elif [ "$(COMMAND)" == "rpc" ]; then \ + for d in $(wildcard $(ROOT_DIR)/cmd/rpc/*/); do \ + cd $$d && CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) $(GO) build $(GO_BUILD_FLAGS) -o\ + $(BIN_DIR)/platforms/$(OS)/$(ARCH)/$$(basename $$d)$(GO_OUT_EXT) .; \ + done; \ + else \ + CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) $(GO) build $(GO_BUILD_FLAGS) -o \ + $(BIN_DIR)/platforms/$(OS)/$(ARCH)/$(COMMAND)$(GO_OUT_EXT) $(ROOT_DIR)/cmd/$(COMMAND)/main.go; \ + fi + +## go.multiarch: Build multi-arch binaries +.PHONY: go.build.multiarch +go.build.multiarch: go.build.verify $(foreach p,$(PLATFORMS),$(addprefix go.build., $(addprefix $(p)., $(BINS)))) + +## go.lint: Run golangci to lint source codes +.PHONY: go.lint +go.lint: tools.verify.golangci-lint + @echo "===========> Run golangci to lint source codes" + @$(BIN_DIR)/golangci-lint run -c $(ROOT_DIR)/.golangci.yml $(ROOT_DIR)/... + +## go.test: Run unit test +.PHONY: go.test +go.test: + @$(GO) test ./... + +# ## go.test.junit-report: Run unit test +# .PHONY: go.test.junit-report +# go.test.junit-report: tools.verify.go-junit-report +# @echo "===========> Run unit test > $(TMP_DIR)/report.xml" +# @$(GO) test -v -coverprofile=$(TMP_DIR)/coverage.out 2>&1 $(GO_BUILD_FLAGS) ./... | $(TOOLS_DIR)/go-junit-report -set-exit-code > $(TMP_DIR)/report.xml +# @sed -i '/mock_.*.go/d' $(TMP_DIR)/coverage.out +# @echo "===========> Test coverage of Go code is reported to $(TMP_DIR)/coverage.html by generating HTML" +# @$(GO) tool cover -html=$(TMP_DIR)/coverage.out -o $(TMP_DIR)/coverage.html + +## go.test.junit-report: Run unit test +.PHONY: go.test.junit-report +go.test.junit-report: tools.verify.go-junit-report + @echo "===========> Run unit test > $(TMP_DIR)/report.xml" + @$(GO) test -v -coverprofile=$(TMP_DIR)/coverage.out 2>&1 ./... | $(TOOLS_DIR)/go-junit-report -set-exit-code > $(OUTPUT_DIR)/report.xml + @sed -i '/mock_.*.go/d' $(TMP_DIR)/coverage.out + @echo "===========> Test coverage of Go code is reported to $(TMP_DIR)/coverage.html by generating HTML" + @$(GO) tool cover -html=$(TMP_DIR)/coverage.out -o $(TMP_DIR)/coverage.html + +## go.test.cover: Run unit test with coverage +.PHONY: go.test.cover +go.test.cover: go.test.junit-report + @touch $(TMP_DIR)/coverage.out + @$(GO) tool cover -func=$(TMP_DIR)/coverage.out | \ + awk -v target=$(COVERAGE) -f $(ROOT_DIR)/scripts/coverage.awk + +## go.format: Run unit test and format codes +.PHONY: go.format +go.format: tools.verify.golines tools.verify.goimports + @echo "===========> Formating codes" + @$(FIND) -type f -name '*.go' | $(XARGS) gofmt -s -w + @$(FIND) -type f -name '*.go' | $(XARGS) $(TOOLS_DIR)/goimports -w -local $(ROOT_PACKAGE) + @$(FIND) -type f -name '*.go' | $(XARGS) $(TOOLS_DIR)/golines -w --max-len=120 --reformat-tags --shorten-comments --ignore-generated . + @$(GO) mod edit -fmt + +## imports: task to automatically handle import packages in Go files using goimports tool +.PHONY: go.imports +go.imports: tools.verify.goimports + @$(TOOLS_DIR)/goimports -l -w $(SRC) + +## go.updates: Check for updates to go.mod dependencies +.PHONY: go.updates +go.updates: tools.verify.go-mod-outdated + @$(GO) list -u -m -json all | go-mod-outdated -update -direct + +## go.clean: Clean all builds directories and files +.PHONY: go.clean +go.clean: + @echo "===========> Cleaning all builds TMP_DIR($(TMP_DIR)) AND BIN_DIR($(BIN_DIR))" + @-rm -vrf $(TMP_DIR) $(BIN_DIR) + @echo "===========> End clean..." ## copyright.help: Show copyright help .PHONY: go.help diff --git a/script/make-rules/image.mk b/script/make-rules/image.mk index 25c2a9c07..8a886ea6c 100644 --- a/script/make-rules/image.mk +++ b/script/make-rules/image.mk @@ -20,6 +20,134 @@ # docker registry: registry.example.com/namespace/image:tag as: registry.hub.docker.com/cubxxw/: # +DOCKER := docker +DOCKER_SUPPORTED_API_VERSION ?= 1.32|1.40|1.41 + +REGISTRY_PREFIX ?= cubxxw +BASE_IMAGE = centos:centos8 + +EXTRA_ARGS ?= --no-cache +_DOCKER_BUILD_EXTRA_ARGS := + +ifdef HTTP_PROXY +_DOCKER_BUILD_EXTRA_ARGS += --build-arg HTTP_PROXY=${HTTP_PROXY} +endif + +ifneq ($(EXTRA_ARGS), ) +_DOCKER_BUILD_EXTRA_ARGS += $(EXTRA_ARGS) +endif + +# Determine image files by looking into build/docker/*/Dockerfile +IMAGES_DIR ?= $(wildcard ${ROOT_DIR}/build/docker/*) +# Determine images names by stripping out the dir names +IMAGES ?= $(filter-out tools,$(foreach image,${IMAGES_DIR},$(notdir ${image}))) + +# ifeq (${IMAGES},) +# $(error Could not determine IMAGES, set ROOT_DIR or run in source dir) +# endif + +# ============================================================================== +# Image targets +# ============================================================================== + +## image.verify: Verify docker version +.PHONY: image.verify +image.verify: + $(eval API_VERSION := $(shell $(DOCKER) version | grep -E 'API version: {1,6}[0-9]' | head -n1 | awk '{print $$3} END { if (NR==0) print 0}' )) + $(eval PASS := $(shell echo "$(API_VERSION) > $(DOCKER_SUPPORTED_API_VERSION)" | bc)) + @if [ $(PASS) -ne 1 ]; then \ + $(DOCKER) -v ;\ + echo "Unsupported docker version. Docker API version should be greater than $(DOCKER_SUPPORTED_API_VERSION)"; \ + exit 1; \ + fi + +## image.daemon.verify: Verify docker daemon experimental features +.PHONY: image.daemon.verify +image.daemon.verify: + $(eval PASS := $(shell $(DOCKER) version | grep -q -E 'Experimental: {1,5}true' && echo 1 || echo 0)) + @if [ $(PASS) -ne 1 ]; then \ + echo "Experimental features of Docker daemon is not enabled. Please add \"experimental\": true in '/etc/docker/daemon.json' and then restart Docker daemon."; \ + exit 1; \ + fi + +## image.build: Build docker images +.PHONY: image.build +image.build: image.verify go.build.verify $(addprefix image.build., $(addprefix $(IMAGE_PLAT)., $(IMAGES))) + +## image.build.multiarch: Build docker images for all platforms +.PHONY: image.build.multiarch +image.build.multiarch: image.verify go.build.verify $(foreach p,$(PLATFORMS),$(addprefix image.build., $(addprefix $(p)., $(IMAGES)))) + +## image.build.%: Build docker image for a specific platform +.PHONY: image.build.% +image.build.%: go.build.% + $(eval IMAGE := $(COMMAND)) + $(eval IMAGE_PLAT := $(subst _,/,$(PLATFORM))) + @echo "===========> Building docker image $(IMAGE) $(VERSION) for $(IMAGE_PLAT)" + @mkdir -p $(TMP_DIR)/$(IMAGE) + @cat $(ROOT_DIR)/build/docker/$(IMAGE)/Dockerfile\ + | sed "s#BASE_IMAGE#$(BASE_IMAGE)#g" >$(TMP_DIR)/$(IMAGE)/Dockerfile + @cp $(OUTPUT_DIR)/platforms/$(IMAGE_PLAT)/$(IMAGE) $(TMP_DIR)/$(IMAGE)/ + @DST_DIR=$(TMP_DIR)/$(IMAGE) $(ROOT_DIR)/build/docker/$(IMAGE)/build.sh 2>/dev/null || true + $(eval BUILD_SUFFIX := $(_DOCKER_BUILD_EXTRA_ARGS) --pull -t $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION) $(TMP_DIR)/$(IMAGE)) + @if [ $(shell $(GO) env GOARCH) != $(ARCH) ] ; then \ + $(MAKE) image.daemon.verify ;\ + $(DOCKER) build --platform $(IMAGE_PLAT) $(BUILD_SUFFIX) ; \ + else \ + $(DOCKER) build $(BUILD_SUFFIX) ; \ + fi + @rm -rf $(TMP_DIR)/$(IMAGE) + +## image.push: Push docker images +.PHONY: image.push +image.push: image.verify go.build.verify $(addprefix image.push., $(addprefix $(IMAGE_PLAT)., $(IMAGES))) + +## image.push.multiarch: Push docker images for all platforms +.PHONY: image.push.multiarch +image.push.multiarch: image.verify go.build.verify $(foreach p,$(PLATFORMS),$(addprefix image.push., $(addprefix $(p)., $(IMAGES)))) + +## image.push.%: Push docker image for a specific platform +.PHONY: image.push.% +image.push.%: image.build.% + @echo "===========> Pushing image $(IMAGE) $(VERSION) to $(REGISTRY_PREFIX)" + $(DOCKER) push $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION) + +## image.manifest.push: Push manifest list for multi-arch images +.PHONY: image.manifest.push +image.manifest.push: export DOCKER_CLI_EXPERIMENTAL := enabled +image.manifest.push: image.verify go.build.verify \ +$(addprefix image.manifest.push., $(addprefix $(IMAGE_PLAT)., $(IMAGES))) + +## image.manifest.push.%: Push manifest list for multi-arch images for a specific platform +.PHONY: image.manifest.push.% +image.manifest.push.%: image.push.% image.manifest.remove.% + @echo "===========> Pushing manifest $(IMAGE) $(VERSION) to $(REGISTRY_PREFIX) and then remove the local manifest list" + @$(DOCKER) manifest create $(REGISTRY_PREFIX)/$(IMAGE):$(VERSION) \ + $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION) + @$(DOCKER) manifest annotate $(REGISTRY_PREFIX)/$(IMAGE):$(VERSION) \ + $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION) \ + --os $(OS) --arch ${ARCH} + @$(DOCKER) manifest push --purge $(REGISTRY_PREFIX)/$(IMAGE):$(VERSION) + +# Docker cli has a bug: https://github.com/docker/cli/issues/954 +# If you find your manifests were not updated, +# Please manually delete them in $HOME/.docker/manifests/ +# and re-run. +## image.manifest.remove.%: Remove local manifest list +.PHONY: image.manifest.remove.% +image.manifest.remove.%: + @rm -rf ${HOME}/.docker/manifests/docker.io_$(REGISTRY_PREFIX)_$(IMAGE)-$(VERSION) + +## image.manifest.push.multiarch: Push manifest list for multi-arch images for all platforms +.PHONY: image.manifest.push.multiarch +image.manifest.push.multiarch: image.push.multiarch $(addprefix image.manifest.push.multiarch., $(IMAGES)) + +## image.manifest.push.multiarch.%: Push manifest list for multi-arch images for all platforms for a specific image +.PHONY: image.manifest.push.multiarch.% +image.manifest.push.multiarch.%: + @echo "===========> Pushing manifest $* $(VERSION) to $(REGISTRY_PREFIX) and then remove the local manifest list" + REGISTRY_PREFIX=$(REGISTRY_PREFIX) PLATFROMS="$(PLATFORMS)" IMAGE=$* VERSION=$(VERSION) DOCKER_CLI_EXPERIMENTAL=enabled \ + $(ROOT_DIR)/build/lib/create-manifest.sh ## image.help: Print help for image targets .PHONY: image.help diff --git a/script/make-rules/release.mk b/script/make-rules/release.mk new file mode 100644 index 000000000..5f250b48e --- /dev/null +++ b/script/make-rules/release.mk @@ -0,0 +1,42 @@ +# Copyright © 2023 OpenIMSDK. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================== +# Makefile helper functions for release +# Versions are used after merging +# + +## release: release the project +.PHONY: release.run +release.run: release.verify release.ensure-tag + @scripts/release.sh + +## release.verify: Check if a tool is installed and install it +.PHONY: release.verify +release.verify: tools.verify.git-chglog tools.verify.github-release tools.verify.coscmd + +## release.tag: release the project +.PHONY: release.tag +release.tag: tools.verify.gsemver release.ensure-tag + @git push origin `git describe --tags --abbrev=0` + +## release.ensure-tag: ensure tag +.PHONY: release.ensure-tag +release.ensure-tag: tools.verify.gsemver + @scripts/ensure_tag.sh + +## release.help: Display help information about the release package +.PHONY: release.help +release.help: script/make-rules/release.mk + $(call smallhelp) \ No newline at end of file diff --git a/script/make-rules/tools.mk b/script/make-rules/tools.mk index 398f5372f..2a56ac3b7 100644 --- a/script/make-rules/tools.mk +++ b/script/make-rules/tools.mk @@ -17,7 +17,7 @@ # Why download to the tools directory, thinking we might often switch Go versions using gvm. # -# sealer build use BUILD_TOOLS +# openim build use BUILD_TOOLS BUILD_TOOLS ?= golangci-lint goimports addlicense deepcopy-gen conversion-gen ginkgo go-junit-report go-gitlint # Code analysis tools ANALYSIS_TOOLS = golangci-lint goimports golines go-callvis kube-score @@ -25,16 +25,18 @@ ANALYSIS_TOOLS = golangci-lint goimports golines go-callvis kube-score GENERATION_TOOLS = deepcopy-gen conversion-gen protoc-gen-go cfssl rts codegen # Testing tools TEST_TOOLS = ginkgo go-junit-report gotests +# tenxun cos tools +COS_TOOLS = coscli coscmd # Version control tools VERSION_CONTROL_TOOLS = addlicense go-gitlint git-chglog github-release gsemver # Utility tools UTILITY_TOOLS = go-mod-outdated mockgen gothanks richgo kubeconform # All tools -ALL_TOOLS ?= $(ANALYSIS_TOOLS) $(GENERATION_TOOLS) $(TEST_TOOLS) $(VERSION_CONTROL_TOOLS) $(UTILITY_TOOLS) +ALL_TOOLS ?= $(ANALYSIS_TOOLS) $(GENERATION_TOOLS) $(TEST_TOOLS) $(VERSION_CONTROL_TOOLS) $(UTILITY_TOOLS) $(COS_TOOLS) ## tools.install: Install a must tools .PHONY: tools.install -tools.install: $(addprefix tools.install., $(BUILD_TOOLS)) +tools.install: $(addprefix tools.verify., $(BUILD_TOOLS)) ## tools.install-all: Install all tools .PHONY: tools.install-all @@ -59,8 +61,8 @@ tools.verify.%: @if [ ! -f $(TOOLS_DIR)/$* ]; then GOBIN=$(TOOLS_DIR) $(MAKE) tools.install.$*; fi @echo "===========> $* is install in $(TOOLS_DIR)/$*" -.PHONY: ## install.golangci-lint: Install golangci-lint +.PHONY: install.golangci-lint install.golangci-lint: @$(GO) install github.com/golangci/golangci-lint/cmd/golangci-lint@latest @@ -69,7 +71,6 @@ install.golangci-lint: install.goimports: @$(GO) install golang.org/x/tools/cmd/goimports@latest -# Actions path: https://github.com/sealerio/sealer/tree/main/.github/workflows/go.yml#L37-L50 ## install.addlicense: Install addlicense, used to add license header to source files .PHONY: install.addlicense install.addlicense: @@ -129,6 +130,29 @@ install.git-chglog: install.github-release: @$(GO) install github.com/github-release/github-release@latest +## install.coscli: Install coscli, used to upload files to cos +# example: ./coscli cp/sync -r /home/off-line/docker-off-line/ cos://openim-1306374445/openim/image/amd/off-line/off-line/ -e cos.ap-guangzhou.myqcloud.com +# https://cloud.tencent.com/document/product/436/71763 +.PHONY: install.coscli +install.coscli: + @wget -q https://ghproxy.com/https://github.com/tencentyun/coscli/releases/download/v0.13.0-beta/coscli-linux -O ${TOOLS_DIR}/coscli + @chmod +x ${TOOLS_DIR}/coscli + +## install.coscmd: Install coscmd, used to upload files to cos +.PHONY: install.coscmd +install.coscmd: + @if which pip &>/dev/null; then pip install coscmd; else pip3 install coscmd; fi + +## install.delve: Install delve, used to debug go program +.PHONY: install.delve +install.delve: + @$(GO) install github.com/go-delve/delve/cmd/dlv@latest + +## install.air: Install air, used to hot reload go program +.PHONY: install.air +install.air: + @$(GO) install github.com/cosmtrek/air@latest + ## install.gvm: Install gvm, gvm is a Go version manager, built on top of the official go tool. # github: https://github.com/moovweb/gvm .PHONY: install.gvm @@ -192,11 +216,6 @@ install.richgo: install.rts: @$(GO) install github.com/galeone/rts/cmd/rts@latest -## install.codegen: Install code generator, used to generate code -.PHONY: install.codegen -install.codegen: - @$(GO) install ${ROOT_DIR}/tools/codegen/codegen.go - ## tools.help: Display help information about the tools package .PHONY: tools.help tools.help: script/make-rules/tools.mk