diff --git a/Makefile b/Makefile index 496c9166d..a6badd827 100644 --- a/Makefile +++ b/Makefile @@ -42,11 +42,11 @@ Options: BINS Binaries to build. Default is all binaries under cmd. This option is available when using: make {build}(.multiarch) - Example: make build BINS="open_im_api open_im_cms_api" + Example: make build BINS="open_im_api open_im_cms_api". PLATFORMS Platform to build for. Default is linux_arm64 and linux_amd64. This option is available when using: make {build}.multiarch - Example: make build.multiarch PLATFORMS="linux_arm64 linux_amd64" + Example: make build.multiarch PLATFORMS="linux_arm64 linux_amd64". V Set to 1 enable verbose build. Default is 0. endef diff --git a/script/make-rules/gen.mk b/script/make-rules/gen.mk index 2955d344e..7dc5a5c28 100644 --- a/script/make-rules/gen.mk +++ b/script/make-rules/gen.mk @@ -21,6 +21,64 @@ # Questions about go mod instead of go path: https://github.com/kubernetes/kubernetes/issues/117181 +# ============================================================================== +# Makefile helper functions for generate necessary files +# + +.PHONY: gen.run +#gen.run: gen.errcode gen.docgo +gen.run: gen.clean gen.errcode gen.docgo.doc + +.PHONY: gen.errcode +gen.errcode: gen.errcode.code gen.errcode.doc + +.PHONY: gen.errcode.code +gen.errcode.code: tools.verify.codegen + @echo "===========> Generating iam error code go source files" + @codegen -type=int ${ROOT_DIR}/internal/pkg/code + +.PHONY: gen.errcode.doc +gen.errcode.doc: tools.verify.codegen + @echo "===========> Generating error code markdown documentation" + @codegen -type=int -doc \ + -output ${ROOT_DIR}/docs/guide/zh-CN/api/error_code_generated.md ${ROOT_DIR}/internal/pkg/code + +.PHONY: gen.ca.% +gen.ca.%: + $(eval CA := $(word 1,$(subst ., ,$*))) + @echo "===========> Generating CA files for $(CA)" + @${ROOT_DIR}/scripts/gencerts.sh generate-iam-cert $(OUTPUT_DIR)/cert $(CA) + +.PHONY: gen.ca +gen.ca: $(addprefix gen.ca., $(CERTIFICATES)) + +.PHONY: gen.docgo.doc +gen.docgo.doc: + @echo "===========> Generating missing doc.go for go packages" + @${ROOT_DIR}/scripts/gendoc.sh + +.PHONY: gen.docgo.check +gen.docgo.check: gen.docgo.doc + @n="$$(git ls-files --others '*/doc.go' | wc -l)"; \ + if test "$$n" -gt 0; then \ + git ls-files --others '*/doc.go' | sed -e 's/^/ /'; \ + echo "$@: untracked doc.go file(s) exist in working directory" >&2 ; \ + false ; \ + fi + +.PHONY: gen.docgo.add +gen.docgo.add: + @git ls-files --others '*/doc.go' | $(XARGS) -- git add + +.PHONY: gen.defaultconfigs +gen.defaultconfigs: + @${ROOT_DIR}/scripts/gen_default_config.sh + +.PHONY: gen.clean +gen.clean: + @rm -rf ./api/client/{clientset,informers,listers} + @$(FIND) -type f -name '*_generated.go' -delete + ## gen.help: show help for gen .PHONY: gen.help gen.help: script/make-rules/gen.mk diff --git a/script/make-rules/golang.mk b/script/make-rules/golang.mk index 50be587d2..e0ae4e399 100644 --- a/script/make-rules/golang.mk +++ b/script/make-rules/golang.mk @@ -16,8 +16,171 @@ # Build management helpers. These functions help to set, save and load the # +GO := go +GO_SUPPORTED_VERSIONS ?= |1.15|1.16|1.17|1.18|1.19|1.20| +GO_LDFLAGS += -X $(VERSION_PACKAGE).gitVersion=$(GIT_TAG) \ + -X $(VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) \ + -X $(VERSION_PACKAGE).gitTreeState=$(GIT_TREE_STATE) \ + -X $(VERSION_PACKAGE).buildDate=$(BUILD_DATE) \ + -s -w # -s -w deletes debugging information and symbol tables +ifeq ($(DEBUG), 1) + GO_BUILD_FLAGS += -gcflags "all=-N -l" + GO_LDFLAGS= +endif +GO_BUILD_FLAGS += -ldflags "$(GO_LDFLAGS)" + +ifeq ($(GOOS),windows) + GO_OUT_EXT := .exe +endif + +ifeq ($(ROOT_PACKAGE),) + $(error the variable ROOT_PACKAGE must be set prior to including golang.mk, ->/Makefile) +endif + +GOPATH ?= $(shell go env GOPATH) +ifeq ($(origin GOBIN), undefined) + GOBIN := $(GOPATH)/bin +endif + +# COMMANDS is Specify all files under ${ROOT_DIR}/cmd/ except those ending in.md +COMMANDS ?= $(filter-out %.md, $(wildcard ${ROOT_DIR}/cmd/*)) +ifeq (${COMMANDS},) + $(error Could not determine COMMANDS, set ROOT_DIR or run in source dir) +endif + +# BINS is the name of each file in ${COMMANDS}, excluding the directory path +# If there are no files in ${COMMANDS}, or if all files end in.md, ${BINS} will be empty +BINS ?= $(foreach cmd,${COMMANDS},$(notdir ${cmd})) +ifeq (${BINS},) + $(error Could not determine BINS, set ROOT_DIR or run in source dir) +endif + +ifeq (${COMMANDS},) + $(error Could not determine COMMANDS, set ROOT_DIR or run in source dir) +endif +ifeq (${BINS},) + $(error Could not determine BINS, set ROOT_DIR or run in source dir) +endif + +# TODO: EXCLUDE_TESTS variable, which contains the name of the package to be excluded from the test +EXCLUDE_TESTS=github.com/OpenIMSDK/Open-IM-Server/test github.com/OpenIMSDK/Open-IM-Server/pkg/log github.com/OpenIMSDK/Open-IM-Server/db github.com/OpenIMSDK/Open-IM-Server/script github.com/OpenIMSDK/Open-IM-Server/deploy_k8s github.com/OpenIMSDK/Open-IM-Server/deploy github.com/OpenIMSDK/Open-IM-Server/config + +# ============================================================================== +# ❯ tree -L 1 cmd +# cmd +# ├── Open-IM-SDK-Core/ - main.go +# ├── open_im_api +# ├── open_im_cms_api +# ├── open_im_cron_task +# ├── open_im_demo +# ├── open_im_msg_gateway +# ├── open_im_msg_transfer +# ├── open_im_push +# ├── rpc/open_im_admin_cms/ - main.go +# └── test/ - main.go +# COMMAND=openim +# PLATFORM=linux_amd64 +# OS=linux +# ARCH=amd64 +# BINS=open_im_api open_im_cms_api open_im_cron_task open_im_demo open_im_msg_gateway open_im_msg_transfer open_im_push +# BIN_DIR=/root/workspaces/OpenIM/_output/bin +# ============================================================================== + +## go.build.verify: Verify that a suitable version of Go exists +.PHONY: go.build.verify +go.build.verify: +ifneq ($(shell $(GO) version | grep -q -E '\bgo($(GO_SUPPORTED_VERSIONS))\b' && echo 0 || echo 1), 0) + $(error unsupported go version. Please make install one of the following supported version: '$(GO_SUPPORTED_VERSIONS)') +endif + +.PHONY: go.build.% +go.build.%: + $(eval COMMAND := $(word 2,$(subst ., ,$*))) + $(eval PLATFORM := $(word 1,$(subst ., ,$*))) + $(eval OS := $(word 1,$(subst _, ,$(PLATFORM)))) + $(eval ARCH := $(word 2,$(subst _, ,$(PLATFORM)))) + @echo "=====> COMMAND=$(COMMAND)" + @echo "=====> PLATFORM=$(PLATFORM)" + @echo "=====> BIN_DIR=$(BIN_DIR)" + @echo "===========> Building binary $(COMMAND) $(VERSION) for $(OS)_$(ARCH)" + @mkdir -p $(OUTPUT_DIR)/platforms/$(OS)/$(ARCH) + @CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) $(GO) build $(GO_BUILD_FLAGS) -o $(OUTPUT_DIR)/platforms/$(OS)/$(ARCH)/$(COMMAND)$(GO_OUT_EXT) $(ROOT_PACKAGE)/cmd/$(COMMAND) + +## go.build: Build binaries +.PHONY: go.build +go.build: go.build.verify $(addprefix go.build., $(addprefix $(PLATFORM)., $(BINS))) + @echo "===========> Building binary $(BINS) $(VERSION) for $(PLATFORM)" + +## go.multiarch: Build multi-arch binaries +.PHONY: go.build.multiarch +go.build.multiarch: go.build.verify $(foreach p,$(PLATFORMS),$(addprefix go.build., $(addprefix $(p)., $(BINS)))) + +## go.lint: Run golangci to lint source codes +.PHONY: go.lint +go.lint: tools.verify.golangci-lint + @echo "===========> Run golangci to lint source codes" + @$(TOOLS_DIR)/golangci-lint run -c $(ROOT_DIR)/.golangci.yml $(ROOT_DIR)/... + +# # @$(GO) test $(GO_BUILD_FLAGS) ./... +# ## go.test: Run unit test +# .PHONY: go.test +# go.test: tools.verify.go-junit-report +# @echo "===========> Run unit test" +# @set -o pipefail;$(GO) test -race -cover -coverprofile=$(OUTPUT_DIR)/coverage.out \ +# -timeout=10m -shuffle=on -short -v `go list ./...|\ +# egrep -v $(subst $(SPACE),'|',$(sort $(EXCLUDE_TESTS)))` 2>&1 | \ +# tee >(go-junit-report --set-exit-code >$(OUTPUT_DIR)/report.xml) +# @sed -i '/mock_.*.go/d' $(OUTPUT_DIR)/coverage.out # remove mock_.*.go files from test coverage +# @$(GO) tool cover -html=$(OUTPUT_DIR)/coverage.out -o $(OUTPUT_DIR)/coverage.html + +## go.test: Run unit test +.PHONY: go.test +go.test: + @$(GO) test $(GO_BUILD_FLAGS) ./... + +## go.test.junit-report: Run unit test +.PHONY: go.test.junit-report +go.test.junit-report: tools.verify.go-junit-report + @echo "===========> Run unit test > $(TMP_DIR)/report.xml" + @$(GO) test -v -coverprofile=$(TMP_DIR)/coverage.out 2>&1 $(GO_BUILD_FLAGS) ./... | $(TOOLS_DIR)/go-junit-report -set-exit-code > $(TMP_DIR)/report.xml + @sed -i '/mock_.*.go/d' $(TMP_DIR)/coverage.out + @echo "===========> Test coverage of Go code is reported to $(TMP_DIR)/coverage.html by generating HTML" + @$(GO) tool cover -html=$(TMP_DIR)/coverage.out -o $(TMP_DIR)/coverage.html + +## go.test.cover: Run unit test with coverage +.PHONY: go.test.cover +go.test.cover: go.test.junit-report + @touch $(TMP_DIR)/coverage.out + @$(GO) tool cover -func=$(TMP_DIR)/coverage.out | \ + awk -v target=$(COVERAGE) -f $(ROOT_DIR)/scripts/coverage.awk + +## go.format: Run unit test and format codes +.PHONY: go.format +go.format: tools.verify.golines tools.verify.goimports + @echo "===========> Formating codes" + @$(FIND) -type f -name '*.go' | $(XARGS) gofmt -s -w + @$(FIND) -type f -name '*.go' | $(XARGS) $(TOOLS_DIR)/goimports -w -local $(ROOT_PACKAGE) + @$(FIND) -type f -name '*.go' | $(XARGS) $(TOOLS_DIR)/golines -w --max-len=120 --reformat-tags --shorten-comments --ignore-generated . + @$(GO) mod edit -fmt + +## imports: task to automatically handle import packages in Go files using goimports tool +.PHONY: go.imports +go.imports: tools.verify.goimports + @$(TOOLS_DIR)/goimports -l -w $(SRC) + +## go.updates: Check for updates to go.mod dependencies +.PHONY: go.updates +go.updates: tools.verify.go-mod-outdated + @$(GO) list -u -m -json all | go-mod-outdated -update -direct + +## go.clean: Clean all builds directories and files +.PHONY: go.clean +go.clean: + @echo "===========> Cleaning all builds OUTPUT_DIR($(OUTPUT_DIR)) AND BIN_DIR($(BIN_DIR))" + @-rm -vrf $(OUTPUT_DIR) $(BIN_DIR) + @echo "===========> End clean..." ## copyright.help: Show copyright help .PHONY: go.help diff --git a/script/make-rules/image.mk b/script/make-rules/image.mk index 25c2a9c07..c584e5220 100644 --- a/script/make-rules/image.mk +++ b/script/make-rules/image.mk @@ -21,6 +21,118 @@ # +DOCKER := docker +DOCKER_SUPPORTED_API_VERSION ?= 1.32 + +REGISTRY_PREFIX ?= marmotedu +BASE_IMAGE = centos:centos8 + +EXTRA_ARGS ?= --no-cache +_DOCKER_BUILD_EXTRA_ARGS := + +ifdef HTTP_PROXY +_DOCKER_BUILD_EXTRA_ARGS += --build-arg HTTP_PROXY=${HTTP_PROXY} +endif + +ifneq ($(EXTRA_ARGS), ) +_DOCKER_BUILD_EXTRA_ARGS += $(EXTRA_ARGS) +endif + +# Determine image files by looking into build/docker/*/Dockerfile +IMAGES_DIR ?= $(wildcard ${ROOT_DIR}/build/docker/*) +# Determine images names by stripping out the dir names +IMAGES ?= $(filter-out tools,$(foreach image,${IMAGES_DIR},$(notdir ${image}))) + +ifeq (${IMAGES},) + $(error Could not determine IMAGES, set ROOT_DIR or run in source dir) +endif + +.PHONY: image.verify +image.verify: + $(eval API_VERSION := $(shell $(DOCKER) version | grep -E 'API version: {1,6}[0-9]' | head -n1 | awk '{print $$3} END { if (NR==0) print 0}' )) + $(eval PASS := $(shell echo "$(API_VERSION) > $(DOCKER_SUPPORTED_API_VERSION)" | bc)) + @if [ $(PASS) -ne 1 ]; then \ + $(DOCKER) -v ;\ + echo "Unsupported docker version. Docker API version should be greater than $(DOCKER_SUPPORTED_API_VERSION)"; \ + exit 1; \ + fi + +.PHONY: image.daemon.verify +image.daemon.verify: + $(eval PASS := $(shell $(DOCKER) version | grep -q -E 'Experimental: {1,5}true' && echo 1 || echo 0)) + @if [ $(PASS) -ne 1 ]; then \ + echo "Experimental features of Docker daemon is not enabled. Please add \"experimental\": true in '/etc/docker/daemon.json' and then restart Docker daemon."; \ + exit 1; \ + fi + +.PHONY: image.build +image.build: image.verify go.build.verify $(addprefix image.build., $(addprefix $(IMAGE_PLAT)., $(IMAGES))) + +.PHONY: image.build.multiarch +image.build.multiarch: image.verify go.build.verify $(foreach p,$(PLATFORMS),$(addprefix image.build., $(addprefix $(p)., $(IMAGES)))) + +.PHONY: image.build.% +image.build.%: go.build.% + $(eval IMAGE := $(COMMAND)) + $(eval IMAGE_PLAT := $(subst _,/,$(PLATFORM))) + @echo "===========> Building docker image $(IMAGE) $(VERSION) for $(IMAGE_PLAT)" + @mkdir -p $(TMP_DIR)/$(IMAGE) + @cat $(ROOT_DIR)/build/docker/$(IMAGE)/Dockerfile\ + | sed "s#BASE_IMAGE#$(BASE_IMAGE)#g" >$(TMP_DIR)/$(IMAGE)/Dockerfile + @cp $(OUTPUT_DIR)/platforms/$(IMAGE_PLAT)/$(IMAGE) $(TMP_DIR)/$(IMAGE)/ + @DST_DIR=$(TMP_DIR)/$(IMAGE) $(ROOT_DIR)/build/docker/$(IMAGE)/build.sh 2>/dev/null || true + $(eval BUILD_SUFFIX := $(_DOCKER_BUILD_EXTRA_ARGS) --pull -t $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION) $(TMP_DIR)/$(IMAGE)) + @if [ $(shell $(GO) env GOARCH) != $(ARCH) ] ; then \ + $(MAKE) image.daemon.verify ;\ + $(DOCKER) build --platform $(IMAGE_PLAT) $(BUILD_SUFFIX) ; \ + else \ + $(DOCKER) build $(BUILD_SUFFIX) ; \ + fi + @rm -rf $(TMP_DIR)/$(IMAGE) + +.PHONY: image.push +image.push: image.verify go.build.verify $(addprefix image.push., $(addprefix $(IMAGE_PLAT)., $(IMAGES))) + +.PHONY: image.push.multiarch +image.push.multiarch: image.verify go.build.verify $(foreach p,$(PLATFORMS),$(addprefix image.push., $(addprefix $(p)., $(IMAGES)))) + +.PHONY: image.push.% +image.push.%: image.build.% + @echo "===========> Pushing image $(IMAGE) $(VERSION) to $(REGISTRY_PREFIX)" + $(DOCKER) push $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION) + +.PHONY: image.manifest.push +image.manifest.push: export DOCKER_CLI_EXPERIMENTAL := enabled +image.manifest.push: image.verify go.build.verify \ +$(addprefix image.manifest.push., $(addprefix $(IMAGE_PLAT)., $(IMAGES))) + +.PHONY: image.manifest.push.% +image.manifest.push.%: image.push.% image.manifest.remove.% + @echo "===========> Pushing manifest $(IMAGE) $(VERSION) to $(REGISTRY_PREFIX) and then remove the local manifest list" + @$(DOCKER) manifest create $(REGISTRY_PREFIX)/$(IMAGE):$(VERSION) \ + $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION) + @$(DOCKER) manifest annotate $(REGISTRY_PREFIX)/$(IMAGE):$(VERSION) \ + $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION) \ + --os $(OS) --arch ${ARCH} + @$(DOCKER) manifest push --purge $(REGISTRY_PREFIX)/$(IMAGE):$(VERSION) + +# Docker cli has a bug: https://github.com/docker/cli/issues/954 +# If you find your manifests were not updated, +# Please manually delete them in $HOME/.docker/manifests/ +# and re-run. +.PHONY: image.manifest.remove.% +image.manifest.remove.%: + @rm -rf ${HOME}/.docker/manifests/docker.io_$(REGISTRY_PREFIX)_$(IMAGE)-$(VERSION) + +.PHONY: image.manifest.push.multiarch +image.manifest.push.multiarch: image.push.multiarch $(addprefix image.manifest.push.multiarch., $(IMAGES)) + +.PHONY: image.manifest.push.multiarch.% +image.manifest.push.multiarch.%: + @echo "===========> Pushing manifest $* $(VERSION) to $(REGISTRY_PREFIX) and then remove the local manifest list" + REGISTRY_PREFIX=$(REGISTRY_PREFIX) PLATFROMS="$(PLATFORMS)" IMAGE=$* VERSION=$(VERSION) DOCKER_CLI_EXPERIMENTAL=enabled \ + $(ROOT_DIR)/build/lib/create-manifest.sh + ## image.help: Print help for image targets .PHONY: image.help image.help: script/make-rules/image.mk