From d30131166db208ee6d507267a48e265d75d43190 Mon Sep 17 00:00:00 2001 From: "Xinwei Xiong(cubxxw-openim)" <3293172751nss@gmail.com> Date: Wed, 5 Jul 2023 22:12:48 +0800 Subject: [PATCH] feat: add make file Signed-off-by: Xinwei Xiong(cubxxw-openim) <3293172751nss@gmail.com> --- .github/sync.yml | 12 +++ Makefile | 61 +++++++-------- docs/contrib/cicd-actions.md | 127 +++++++++++++++++++++++++++++++ docs/contrib/code_conventions.md | 38 +++++++++ scripts/common.sh | 18 ++--- scripts/lib/release.sh | 8 +- scripts/lib/util.sh | 1 + scripts/make-rules/common.mk | 2 +- scripts/make-rules/image.mk | 7 ++ 9 files changed, 230 insertions(+), 44 deletions(-) create mode 100644 docs/contrib/cicd-actions.md create mode 100644 docs/contrib/code_conventions.md diff --git a/.github/sync.yml b/.github/sync.yml index 859096c6f..fdc003a0f 100644 --- a/.github/sync.yml +++ b/.github/sync.yml @@ -28,6 +28,9 @@ OpenIMSDK/openim-sdk-core: - source: .github/workflows/stale.yml dest: .github/workflows/stale.yml replace: false + - source: .github/.codecov.yml + dest: .github/.codecov.yml + replace: false OpenIMSDK/OpenIM-Docs: - source: .github/workflows/ @@ -38,6 +41,9 @@ OpenIMSDK/OpenIM-Docs: - source: scripts/githooks/ dest: scripts/githooks/ replace: true + - source: .github/.codecov.yml + dest: .github/.codecov.yml + replace: false OpenIMSDK/OpenKF: - source: LICENSE @@ -51,6 +57,9 @@ OpenIMSDK/OpenKF: - source: .github/workflows/stale.yml dest: .github/workflows/stale.yml replace: false + - source: .github/.codecov.yml + dest: .github/.codecov.yml + replace: false group: # first group：common to all warehouses @@ -75,6 +84,9 @@ group: - source: .github/workflows/help-comment-issue.yml dest: .github/workflows/help-comment-issue.yml replace: false + - source: .github/.codecov.yml + dest: .github/.codecov.yml + replace: false - source: ./scripts/githooks/ dest: ./scripts/githooks/ replace: true \ No newline at end of file diff --git a/Makefile b/Makefile index 179f986dc..8b827e91b 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ .DEFAULT_GOAL := help +## all: Run tidy, gen, add-copyright, format, lint, cover, build ✨ .PHONY: all all: tidy gen add-copyright format lint cover build @@ -52,135 +53,135 @@ export USAGE_OPTIONS # ============================================================================== # Targets -## build: Build binaries by default +## build: Build binaries by default ✨ .PHONY: build build: @$(MAKE) go.build -## multiarch: Build binaries for multiple platforms. See option PLATFORMS. +## multiarch: Build binaries for multiple platforms. See option PLATFORMS. ✨ .PHONY: multiarch multiarch: @$(MAKE) go.build.multiarch -## tidy: tidy go.mod +## tidy: tidy go.mod ✨ .PHONY: tidy tidy: @$(GO) mod tidy -## vendor: vendor go.mod +## vendor: vendor go.mod ✨ .PHONY: vendor vendor: @$(GO) mod vendor -## style: code style -> fmt,vet,lint +## style: code style -> fmt,vet,lint ✨ .PHONY: style style: fmt vet lint -## fmt: Run go fmt against code. +## fmt: Run go fmt against code. ✨ .PHONY: fmt fmt: @$(GO) fmt ./... -## vet: Run go vet against code. +## vet: Run go vet against code. ✨ .PHONY: vet vet: @$(GO) vet ./... -## lint: Check syntax and styling of go sources. +lint: Check syntax and styling of go sources. ✨ .PHONY: lint lint: @$(MAKE) go.lint -## format: Gofmt (reformat) package sources (exclude vendor dir if existed). +## format: Gofmt (reformat) package sources (exclude vendor dir if existed). ✨ .PHONY: format format: @$(MAKE) go.format -## test: Run unit test. +## test: Run unit test. ✨ .PHONY: test test: @$(MAKE) go.test -## cover: Run unit test and get test coverage. -.PHONY: cover +## cover: Run unit test and get test coverage. ✨ +.PHONY: cover cover: @$(MAKE) go.test.cover -## updates: Check for updates to go.mod dependencies +## updates: Check for updates to go.mod dependencies ✨ .PHONY: updates @$(MAKE) go.updates -## imports: task to automatically handle import packages in Go files using goimports tool +## imports: task to automatically handle import packages in Go files using goimports tool ✨ .PHONY: imports imports: @$(MAKE) go.imports -## clean: Remove all files that are created by building. +## clean: Remove all files that are created by building. ✨ .PHONY: clean clean: @$(MAKE) go.clean -## image: Build docker images for host arch. +## image: Build docker images for host arch. ✨ .PHONY: image image: @$(MAKE) image.build -## image.multiarch: Build docker images for multiple platforms. See option PLATFORMS. +## image.multiarch: Build docker images for multiple platforms. See option PLATFORMS. ✨ .PHONY: image.multiarch image.multiarch: @$(MAKE) image.build.multiarch -## push: Build docker images for host arch and push images to registry. +## push: Build docker images for host arch and push images to registry. ✨ .PHONY: push push: @$(MAKE) image.push -## push.multiarch: Build docker images for multiple platforms and push images to registry. +## push.multiarch: Build docker images for multiple platforms and push images to registry. ✨ .PHONY: push.multiarch push.multiarch: @$(MAKE) image.push.multiarch -## tools: Install dependent tools. +## tools: Install dependent tools. ✨ .PHONY: tools tools: @$(MAKE) tools.install -## gen: Generate all necessary files. +## gen: Generate all necessary files. ✨ .PHONY: gen gen: @$(MAKE) gen.run -## swagger: Generate swagger document. +## swagger: Generate swagger document. ✨ .PHONY: swagger swagger: @$(MAKE) swagger.run -## serve-swagger: Serve swagger spec and docs. +## serve-swagger: Serve swagger spec and docs. ✨ .PHONY: swagger.serve serve-swagger: @$(MAKE) swagger.serve -## verify-copyright: Verify the license headers for all files. +## verify-copyright: Verify the license headers for all files. ✨ .PHONY: verify-copyright verify-copyright: @$(MAKE) copyright.verify -## add-copyright: Add copyright ensure source code files have license headers. +## add-copyright: Add copyright ensure source code files have license headers. ✨ .PHONY: add-copyright add-copyright: @$(MAKE) copyright.add -## release: release the project +## release: release the project ✨ .PHONY: release release: release.verify release.ensure-tag @scripts/release.sh -## help: Show this help info. +## help: Show this help info. ✨ .PHONY: help help: Makefile $(call makehelp) -## help-all: Show all help details info. +## help-all: Show all help details info. ✨ .PHONY: help-all -help-all: go.help copyright.help tools.help image.help help - $(call makeallhelp) +help-all: go.help copyright.help tools.help image.help dependencies.help gen.help release.help swagger.help help + $(call makeallhelp) \ No newline at end of file diff --git a/docs/contrib/cicd-actions.md b/docs/contrib/cicd-actions.md new file mode 100644 index 000000000..7a581d3fa --- /dev/null +++ b/docs/contrib/cicd-actions.md @@ -0,0 +1,127 @@ +# Continuous Integration and Automation + +Every change on the K3s repository, either made through a pull request or direct push, triggers the continuous integration pipelines defined within the same repository. Needless to say, all the K3s contributions can be merged until all the checks pass (AKA having green builds). + +- [Continuous Integration and Automation](#continuous-integration-and-automation) + - [CI Platforms](#ci-platforms) + - [GitHub Actions](#github-actions) + - [Running locally](#running-locally) + +## CI Platforms + +Currently, there are two different platforms involved in running the CI processes: + +- GitHub actions +- Drone pipelines on CNCF infrastructure + +### GitHub Actions + +All the existing GitHub Actions are defined as YAML files under the `.github/workflows` directory. These can be grouped into: + +- **PR Checks**. These actions run all the required validations upon PR creation and update. Covering the DCO compliance check, `x86_64` test batteries (unit, integration, smoke), and code coverage. +- **Repository automation**. Currently, it only covers issues and epic grooming. + +Everything runs on GitHub's provided runners; thus, the tests are limited to run in `x86_64` architectures. + + +## Running locally + +A contributor should verify their changes locally to speed up the pull request process. Fortunately, all the CI steps can be on local environments, except for the publishing ones, through either of the following methods: + +**User Makefile:** +```bash +root@PS2023EVRHNCXG:~/workspaces/openim/Open-IM-Server# make help 😊 + +Usage: make ... + +Targets: + +all Run tidy, gen, add-copyright, format, lint, cover, build 🚀 +build Build binaries by default 🛠️ +multiarch Build binaries for multiple platforms. See option PLATFORMS. 🌍 +tidy tidy go.mod ✨ +vendor vendor go.mod 📦 +style code style -> fmt,vet,lint 💅 +fmt Run go fmt against code. ✨ +vet Run go vet against code. ✅ +lint Check syntax and styling of go sources. ✔️ +format Gofmt (reformat) package sources (exclude vendor dir if existed). 🔄 +test Run unit test. 🧪 +cover Run unit test and get test coverage. 📊 +updates Check for updates to go.mod dependencies 🆕 +imports task to automatically handle import packages in Go files using goimports tool 📥 +clean Remove all files that are created by building. 🗑️ +image Build docker images for host arch. 🐳 +image.multiarch Build docker images for multiple platforms. See option PLATFORMS. 🌍🐳 +push Build docker images for host arch and push images to registry. 📤🐳 +push.multiarch Build docker images for multiple platforms and push images to registry. 🌍📤🐳 +tools Install dependent tools. 🧰 +gen Generate all necessary files. 🧩 +swagger Generate swagger document. 📖 +serve-swagger Serve swagger spec and docs. 🚀📚 +verify-copyright Verify the license headers for all files. ✅ +add-copyright Add copyright ensure source code files have license headers. 📄 +release release the project 🎉 +help Show this help info. ℹ️ +help-all Show all help details info. ℹ️📚 + +Options: + +DEBUG Whether or not to generate debug symbols. Default is 0. ❓ + +BINS Binaries to build. Default is all binaries under cmd. 🛠️ +This option is available when using: make {build}(.multiarch) 🧰 +Example: make build BINS="openim_api openim_cms_api". + +PLATFORMS Platform to build for. Default is linux_arm64 and linux_amd64. 🌍 +This option is available when using: make {build}.multiarch 🌍 +Example: make multiarch PLATFORMS="linux_s390x linux_mips64 +linux_mips64le darwin_amd64 windows_amd64 linux_amd64 linux_arm64". + +V Set to 1 enable verbose build. Default is 0. 📝 +``` +How to Use Makefile to Help Contributors Build Projects Quickly 😊 + +The `make help` command is a handy tool that provides useful information on how to utilize the Makefile effectively. By running this command, contributors will gain insights into various targets and options available for building projects swiftly. + +Here's a breakdown of the targets and options provided by the Makefile: + +Targets 😃 + +1. `all`: This target runs multiple tasks like `tidy`, `gen`, `add-copyright`, `format`, `lint`, `cover`, and `build`. It ensures comprehensive project building. +2. `build`: The primary target that compiles binaries by default. It is particularly useful for creating the necessary executable files. +3. `multiarch`: A target that builds binaries for multiple platforms. Contributors can specify the desired platforms using the `PLATFORMS` option. +4. `tidy`: This target cleans up the `go.mod` file, ensuring its consistency. +5. `vendor`: A target that updates the project dependencies based on the `go.mod` file. +6. `style`: Checks the code style using tools like `fmt`, `vet`, and `lint`. It ensures a consistent coding style throughout the project. +7. `fmt`: Formats the code using the `go fmt` command, ensuring proper indentation and formatting. +8. `vet`: Runs the `go vet` command to identify common errors in the code. +9. `lint`: Validates the syntax and styling of Go source files using a linter. +10. `format`: Reformats the package sources using `gofmt`. It excludes the vendor directory if it exists. +11. `test`: Executes unit tests to ensure the functionality and stability of the code. +12. `cover`: Performs unit tests and calculates the test coverage of the code. +13. `updates`: Checks for updates to the project's dependencies specified in the `go.mod` file. +14. `imports`: Automatically handles import packages in Go files using the `goimports` tool. +15. `clean`: Removes all files generated during the build process, effectively cleaning up the project directory. +16. `image`: Builds Docker images for the host architecture. +17. `image.multiarch`: Similar to the `image` target, but it builds Docker images for multiple platforms. Contributors can specify the desired platforms using the `PLATFORMS` option. +18. `push`: Builds Docker images for the host architecture and pushes them to a registry. +19. `push.multiarch`: Builds Docker images for multiple platforms and pushes them to a registry. Contributors can specify the desired platforms using the `PLATFORMS` option. +20. `tools`: Installs the necessary tools or dependencies required by the project. +21. `gen`: Generates all the required files automatically. +22. `swagger`: Generates the swagger document for the project. +23. `serve-swagger`: Serves the swagger specification and documentation. +24. `verify-copyright`: Verifies the license headers for all project files. +25. `add-copyright`: Adds copyright headers to the source code files. +26. `release`: Releases the project, presumably for distribution. +27. `help`: Displays information about available targets and options. +28. `help-all`: Shows detailed information about all available targets and options. + +Options 😄 + +1. `DEBUG`: A boolean option that determines whether or not to generate debug symbols. The default value is 0 (false). +2. `BINS`: Specifies the binaries to build. By default, it builds all binaries under the `cmd` directory. Contributors can provide a list of specific binaries using this option. +3. `PLATFORMS`: Specifies the platforms to build for. The default platforms are `linux_arm64` and `linux_amd64`. Contributors can specify multiple platforms by providing a space-separated list of platform names. +4. `V`: A boolean option that enables verbose build output when set to 1 (true). The default value is 0 (false). + +With these targets and options in place, contributors can efficiently build projects using the Makefile. Happy coding! 🚀😊 diff --git a/docs/contrib/code_conventions.md b/docs/contrib/code_conventions.md new file mode 100644 index 000000000..1387da2f7 --- /dev/null +++ b/docs/contrib/code_conventions.md @@ -0,0 +1,38 @@ +# Code conventions + +- [Code conventions](#code-conventions) + - [POSIX shell](#posix-shell) + - [Go](#go) + - [Directory and file conventions](#directory-and-file-conventions) + - [Testing conventions](#testing-conventions) + +## POSIX shell + +- [Style guide](https://google.github.io/styleguide/shell.xml) + +## Go + +- [Go Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments) +- [Effective Go](https://golang.org/doc/effective_go.html) +- Know and avoid [Go landmines](https://gist.github.com/lavalamp/4bd23295a9f32706a48f) +- Comment your code. + - [Go's commenting conventions](http://blog.golang.org/godoc-documenting-go-code) + - If reviewers ask questions about why the code is the way it is, that's a sign that comments might be helpful. +- Command-line flags should use dashes, not underscores +- Naming + - Please consider package name when selecting an interface name, and avoid redundancy. For example, `storage.Interface` is better than `storage.StorageInterface`. + - Do not use uppercase characters, underscores, or dashes in package names. + - Please consider parent directory name when choosing a package name. For example, `pkg/controllers/autoscaler/foo.go` should say `package autoscaler` not `package autoscalercontroller`. + - Unless there's a good reason, the `package foo` line should match the name of the directory in which the `.go` file exists. + - Importers can use a different name if they need to disambiguate. + +## Directory and file conventions + +- Avoid general utility packages. Packages called "util" are suspect. Instead, derive a name that describes your desired function. For example, the utility functions dealing with waiting for operations are in the `wait` package and include functionality like `Poll`. The full name is `wait.Poll`. +- All filenames should be lowercase. +- All source files and directories should use underscores, not dashes. + - Package directories should generally avoid using separators as much as possible. When package names are multiple words, they usually should be in nested subdirectories. + +## Testing conventions + +Please refer to [TESTING.md](../../tests/TESTING.md) document. diff --git a/scripts/common.sh b/scripts/common.sh index 396f1c4a2..8ae2abfa5 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -203,9 +203,9 @@ function openim::build::prepare_docker_machine() { openim::log::status "Creating a machine to build OPENIM" docker-machine create --driver "${DOCKER_MACHINE_DRIVER}" \ --virtualbox-memory "${virtualbox_memory_mb}" \ - --engine-env HTTP_PROXY="${IAMRNETES_HTTP_PROXY:-}" \ - --engine-env HTTPS_PROXY="${IAMRNETES_HTTPS_PROXY:-}" \ - --engine-env NO_PROXY="${IAMRNETES_NO_PROXY:-127.0.0.1}" \ + --engine-env HTTP_PROXY="${OPENIMRNETES_HTTP_PROXY:-}" \ + --engine-env HTTPS_PROXY="${OPENIMRNETES_HTTPS_PROXY:-}" \ + --engine-env NO_PROXY="${OPENIMRNETES_NO_PROXY:-127.0.0.1}" \ "${DOCKER_MACHINE_NAME}" > /dev/null || { openim::log::error "Something went wrong creating a machine." openim::log::error "Try the following: " @@ -250,14 +250,14 @@ function openim::build::update_dockerfile() { } function openim::build::set_proxy() { - if [[ -n "${IAMRNETES_HTTPS_PROXY:-}" ]]; then - echo "ENV https_proxy $IAMRNETES_HTTPS_PROXY" >> "${LOCAL_OUTPUT_BUILD_CONTEXT}/Dockerfile" + if [[ -n "${OPENIMRNETES_HTTPS_PROXY:-}" ]]; then + echo "ENV https_proxy $OPENIMRNETES_HTTPS_PROXY" >> "${LOCAL_OUTPUT_BUILD_CONTEXT}/Dockerfile" fi - if [[ -n "${IAMRNETES_HTTP_PROXY:-}" ]]; then - echo "ENV http_proxy $IAMRNETES_HTTP_PROXY" >> "${LOCAL_OUTPUT_BUILD_CONTEXT}/Dockerfile" + if [[ -n "${OPENIMRNETES_HTTP_PROXY:-}" ]]; then + echo "ENV http_proxy $OPENIMRNETES_HTTP_PROXY" >> "${LOCAL_OUTPUT_BUILD_CONTEXT}/Dockerfile" fi - if [[ -n "${IAMRNETES_NO_PROXY:-}" ]]; then - echo "ENV no_proxy $IAMRNETES_NO_PROXY" >> "${LOCAL_OUTPUT_BUILD_CONTEXT}/Dockerfile" + if [[ -n "${OPENIMRNETES_NO_PROXY:-}" ]]; then + echo "ENV no_proxy $OPENIMRNETES_NO_PROXY" >> "${LOCAL_OUTPUT_BUILD_CONTEXT}/Dockerfile" fi } diff --git a/scripts/lib/release.sh b/scripts/lib/release.sh index a299a8943..d6cdcdfcb 100755 --- a/scripts/lib/release.sh +++ b/scripts/lib/release.sh @@ -24,7 +24,7 @@ # Tencent cos configuration readonly BUCKET="openim-1306374445" -readonly REGION="ap-beijing" +readonly REGION="ap-guangzhou" readonly COS_RELEASE_DIR="openim-release" # default cos command tool coscli or coscmd @@ -178,7 +178,7 @@ function openim::release::package_server_tarballs() { # This fancy expression will expand to prepend a path # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the # server_bins array. - cp "${server_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ + cp "${server_bins[@]/bin/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ "${release_stage}/server/bin/" openim::release::clean_cruft @@ -218,7 +218,7 @@ function openim::release::package_client_tarballs() { # This fancy expression will expand to prepend a path # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the # client_bins array. - cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ + cp "${client_bins[@]/bin/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ "${release_stage}/client/bin/" openim::release::clean_cruft @@ -252,7 +252,7 @@ function openim::release::build_server_images() { # This fancy expression will expand to prepend a path # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the # OPENIM_SERVER_IMAGE_BINARIES array. - cp "${OPENIM_SERVER_IMAGE_BINARIES[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ + cp "${OPENIM_SERVER_IMAGE_BINARIES[@]/bin/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ "${release_stage}/server/bin/" openim::release::create_docker_images_for_server "${release_stage}/server/bin" "${arch}" diff --git a/scripts/lib/util.sh b/scripts/lib/util.sh index dee1cad74..63d32eb79 100755 --- a/scripts/lib/util.sh +++ b/scripts/lib/util.sh @@ -199,6 +199,7 @@ openim::util::find-binary-for-platform() { "${OPENIM_ROOT}/_output/${platform}/${lookfor}" "${OPENIM_ROOT}/_output/local/bin/${platform}/${lookfor}" "${OPENIM_ROOT}/_output/platforms/${platform}/${lookfor}" + "${OPENIM_ROOT}/_output/platforms/bin/${platform}/${lookfor}" ) # List most recently-updated location. diff --git a/scripts/make-rules/common.mk b/scripts/make-rules/common.mk index 179643b12..e99585a76 100644 --- a/scripts/make-rules/common.mk +++ b/scripts/make-rules/common.mk @@ -87,7 +87,7 @@ endif # The OS must be linux when building docker images # PLATFORMS ?= linux_amd64 linux_arm64 # The OS can be linux/windows/darwin when building binaries -PLATFORMS ?= linux_s390x linux_mips64 linux_mips64le darwin_amd64 windows_amd64 linux_amd64 linux_arm64 +PLATFORMS ?= linux_s390x linux_mips64 linux_mips64le darwin_amd64 windows_amd64 linux_amd64 linux_arm64 linux_ppc64le # only support linux GOOS=linux diff --git a/scripts/make-rules/image.mk b/scripts/make-rules/image.mk index adcffa1cc..e70583c34 100644 --- a/scripts/make-rules/image.mk +++ b/scripts/make-rules/image.mk @@ -98,6 +98,13 @@ image.build.%: go.build.% fi @rm -rf $(TMP_DIR)/$(IMAGE) +## image.buildx.%: Build docker images with buildx +.PHONY: image.buildx.% +image.buildx.%: + $(eval IMAGE := $(word 1,$(subst ., ,$*))) + echo "===========> Building docker image $(IMAGE) $(VERSION)" + $(DOCKER) buildx build -f $(ROOT_DIR)/Dockerfile --pull --no-cache --platform=$(PLATFORMS) --push . -t $(REGISTRY_PREFIX)/$(IMAGE)-$(ARCH):$(VERSION) + ## image.push: Push docker images .PHONY: image.push image.push: image.verify go.build.verify $(addprefix image.push., $(addprefix $(IMAGE_PLAT)., $(IMAGES)))