diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 000000000..bd16a84d1 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,25 @@ +# Notes about go workspace + +As openim is using go1.18's [workspace feature](https://go.dev/doc/tutorial/workspaces), once you add a new module, you need to run `go work use -r .` at root directory to update the workspace synced. + +### Create a new extensions + +1. Create your tools_name directory in pkg `/tools` first and cd into it. +2. Init the project. +3. Then `go work use -r .` at current directory to update the workspace. +4. Create your tools + +You can execute the following commands to do things above: + +```bash +# edit the CRD_NAME and CRD_GROUP to your own +export OPENIM_TOOLS_NAME= + +# copy and paste to create a new CRD and Controller +mkdir tools/${OPENIM_TOOLS_NAME} +cd tools/${OPENIM_TOOLS_NAME} +go mod init github.com/openimsdk/open-im-server/tools/${OPENIM_TOOLS_NAME} +go mod tidy +go work use -r . +cd ../.. +``` \ No newline at end of file diff --git a/tools/changelog/changelog.go b/tools/changelog/changelog.go new file mode 100644 index 000000000..17a9e5404 --- /dev/null +++ b/tools/changelog/changelog.go @@ -0,0 +1,308 @@ +// Copyright © 2023 OpenIM. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "log" + "os" + "os/exec" + "regexp" + "sort" + "strings" +) + +var ( + mergeRequest = regexp.MustCompile(`Merge pull request #([\d]+)`) + webconsoleBump = regexp.MustCompile(regexp.QuoteMeta("bump(github.com/openshift/origin-web-console): ") + `([\w]+)`) + upstreamKube = regexp.MustCompile(`^UPSTREAM: (\d+)+:(.+)`) + upstreamRepo = regexp.MustCompile(`^UPSTREAM: ([\w/-]+): (\d+)+:(.+)`) + prefix = regexp.MustCompile(`^[\w-]: `) + + assignments = []prefixAssignment{ + {"cluster up", "cluster"}, + {" pv ", "storage"}, + {"haproxy", "router"}, + {"router", "router"}, + {"route", "route"}, + {"authoriz", "auth"}, + {"rbac", "auth"}, + {"authent", "auth"}, + {"reconcil", "auth"}, + {"auth", "auth"}, + {"role", "auth"}, + {" dc ", "deploy"}, + {"deployment", "deploy"}, + {"rolling", "deploy"}, + {"security context constr", "security"}, + {"scc", "security"}, + {"pipeline", "build"}, + {"build", "build"}, + {"registry", "registry"}, + {"registries", "image"}, + {"image", "image"}, + {" arp ", "network"}, + {" cni ", "network"}, + {"egress", "network"}, + {"network", "network"}, + {"oc ", "cli"}, + {"template", "template"}, + {"etcd", "server"}, + {"pod", "node"}, + {"hack/", "hack"}, + {"e2e", "test"}, + {"integration", "test"}, + {"cluster", "cluster"}, + {"master", "server"}, + {"packages", "hack"}, + {"api", "server"}, + } +) + +type prefixAssignment struct { + term string + prefix string +} + +type commit struct { + short string + parents []string + message string +} + +func contains(arr []string, value string) bool { + for _, s := range arr { + if s == value { + return true + } + } + return false +} + +func main() { + log.SetFlags(0) + if len(os.Args) != 3 { + log.Fatalf("Must specify two arguments, FROM and TO") + } + from := os.Args[1] + to := os.Args[2] + + out, err := exec.Command("git", "log", "--topo-order", "--pretty=tformat:%h %p|%s", "--reverse", fmt.Sprintf("%s..%s", from, to)).CombinedOutput() + if err != nil { + log.Fatal(err) + } + + hide := make(map[string]struct{}) + var apiChanges []string + var webconsole []string + var commits []commit + var upstreams []commit + var bumps []commit + for _, line := range strings.Split(string(out), "\n") { + if len(strings.TrimSpace(line)) == 0 { + continue + } + parts := strings.SplitN(line, "|", 2) + hashes := strings.Split(parts[0], " ") + c := commit{short: hashes[0], parents: hashes[1:], message: parts[1]} + + if strings.HasPrefix(c.message, "UPSTREAM: ") { + hide[c.short] = struct{}{} + upstreams = append(upstreams, c) + } + if strings.HasPrefix(c.message, "bump(") { + hide[c.short] = struct{}{} + bumps = append(bumps, c) + } + + if len(c.parents) == 1 { + commits = append(commits, c) + continue + } + + matches := mergeRequest.FindStringSubmatch(line) + if len(matches) == 0 { + // this may have been a human pressing the merge button, we'll just record this as a direct push + continue + } + + // split the accumulated commits into any that are force merges (assumed to be the initial set due + // to --topo-order) from the PR commits as soon as we see any of our merge parents. Then print + // any of the force merges + var first int + for i := range commits { + first = i + if contains(c.parents, commits[i].short) { + first++ + break + } + } + individual := commits[:first] + merged := commits[first:] + for _, commit := range individual { + if len(commit.parents) > 1 { + continue + } + if _, ok := hide[commit.short]; ok { + continue + } + fmt.Printf("force-merge: %s %s\n", commit.message, commit.short) + } + + // try to find either the PR title or the first commit title from the merge commit + out, err := exec.Command("git", "show", "--pretty=tformat:%b", c.short).CombinedOutput() + if err != nil { + log.Fatal(err) + } + var message string + para := strings.Split(string(out), "\n\n") + if len(para) > 0 && strings.HasPrefix(para[0], "Automatic merge from submit-queue") { + para = para[1:] + } + // this is no longer necessary with the submit queue in place + if len(para) > 0 && strings.HasPrefix(para[0], "Merged by ") { + para = para[1:] + } + // post submit-queue, the merge bot will add the PR title, which is usually pretty good + if len(para) > 0 { + message = strings.Split(para[0], "\n")[0] + } + if len(message) == 0 && len(merged) > 0 { + message = merged[0].message + } + if len(message) > 0 && len(merged) == 1 && message == merged[0].message { + merged = nil + } + + // try to calculate a prefix based on the diff + if len(message) > 0 && !prefix.MatchString(message) { + prefix, ok := findPrefixFor(message, merged) + if ok { + message = prefix + ": " + message + } + } + + // github merge + + // has api changes + display := fmt.Sprintf("%s [\\#%s](https://github.com/openimsdk/Open-IM-Server/pull/%s)", message, matches[1], matches[1]) + if hasFileChanges(c.short, "pkg/apistruct/") { + apiChanges = append(apiChanges, display) + } + + var filtered []commit + for _, commit := range merged { + if _, ok := hide[commit.short]; ok { + continue + } + filtered = append(filtered, commit) + } + if len(filtered) > 0 { + fmt.Printf("- %s\n", display) + for _, commit := range filtered { + fmt.Printf(" - %s (%s)\n", commit.message, commit.short) + } + } + + // stick the merge commit in at the beginning of the next list so we can anchor the previous parent + commits = []commit{c} + } + + // chunk the bumps + var lines []string + for _, commit := range bumps { + if m := webconsoleBump.FindStringSubmatch(commit.message); len(m) > 0 { + webconsole = append(webconsole, m[1]) + continue + } + lines = append(lines, commit.message) + } + lines = sortAndUniq(lines) + for _, line := range lines { + fmt.Printf("- %s\n", line) + } + + // chunk the upstreams + lines = nil + for _, commit := range upstreams { + lines = append(lines, commit.message) + } + lines = sortAndUniq(lines) + for _, line := range lines { + fmt.Printf("- %s\n", upstreamLinkify(line)) + } + + if len(webconsole) > 0 { + fmt.Printf("- web: from %s^..%s\n", webconsole[0], webconsole[len(webconsole)-1]) + } + + for _, apiChange := range apiChanges { + fmt.Printf(" - %s\n", apiChange) + } +} + +func findPrefixFor(message string, commits []commit) (string, bool) { + message = strings.ToLower(message) + for _, m := range assignments { + if strings.Contains(message, m.term) { + return m.prefix, true + } + } + for _, c := range commits { + if prefix, ok := findPrefixFor(c.message, nil); ok { + return prefix, ok + } + } + return "", false +} + +func hasFileChanges(commit string, prefixes ...string) bool { + out, err := exec.Command("git", "diff", "--name-only", fmt.Sprintf("%s^..%s", commit, commit)).CombinedOutput() + if err != nil { + log.Fatal(err) + } + for _, file := range strings.Split(string(out), "\n") { + for _, prefix := range prefixes { + if strings.HasPrefix(file, prefix) { + return true + } + } + } + return false +} + +func sortAndUniq(lines []string) []string { + sort.Strings(lines) + out := make([]string, 0, len(lines)) + last := "" + for _, s := range lines { + if last == s { + continue + } + last = s + out = append(out, s) + } + return out +} + +func upstreamLinkify(line string) string { + if m := upstreamKube.FindStringSubmatch(line); len(m) > 0 { + return fmt.Sprintf("UPSTREAM: [#%s](https://github.com/openimsdk/open-im-server/pull/%s):%s", m[1], m[1], m[2]) + } + if m := upstreamRepo.FindStringSubmatch(line); len(m) > 0 { + return fmt.Sprintf("UPSTREAM: [%s#%s](https://github.com/%s/pull/%s):%s", m[1], m[2], m[1], m[2], m[3]) + } + return line +} diff --git a/tools/changelog/go.mod b/tools/changelog/go.mod new file mode 100644 index 000000000..924f6709a --- /dev/null +++ b/tools/changelog/go.mod @@ -0,0 +1,3 @@ +module github.com/openimsdk/open-im-server/v3/tools/changelog + +go 1.18 diff --git a/tools/component/component.go b/tools/component/component.go new file mode 100644 index 000000000..04c6ba9a5 --- /dev/null +++ b/tools/component/component.go @@ -0,0 +1,311 @@ +// Copyright © 2023 OpenIM. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "database/sql" + "flag" + "fmt" + "net" + "net/url" + "os" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/redis/go-redis/v9" + "gopkg.in/yaml.v3" + + "github.com/IBM/sarama" + "github.com/OpenIMSDK/tools/errs" + "github.com/OpenIMSDK/tools/utils" + "github.com/go-zookeeper/zk" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readpref" + "gorm.io/driver/mysql" + "gorm.io/gorm" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/kafka" + + "github.com/minio/minio-go/v7/pkg/credentials" +) + +const ( + // defaultCfgPath is the default path of the configuration file + defaultCfgPath = "../../../../../config/config.yaml" + minioHealthCheckDuration = 1 + maxRetry = 100 + componentStartErrCode = 6000 + configErrCode = 6001 +) + +var ( + cfgPath = flag.String("c", defaultCfgPath, "Path to the configuration file") + + ErrComponentStart = errs.NewCodeError(componentStartErrCode, "ComponentStartErr") + ErrConfig = errs.NewCodeError(configErrCode, "Config file is incorrect") +) + +func initCfg() error { + data, err := os.ReadFile(*cfgPath) + if err != nil { + return err + } + return yaml.Unmarshal(data, &config.Config) +} + +type checkFunc struct { + name string + function func() error +} + +func main() { + flag.Parse() + + if err := initCfg(); err != nil { + fmt.Printf("Read config failed: %v\n", err) + return + } + + checks := []checkFunc{ + {name: "Mysql", function: checkMysql}, + {name: "Mongo", function: checkMongo}, + {name: "Minio", function: checkMinio}, + {name: "Redis", function: checkRedis}, + {name: "Zookeeper", function: checkZookeeper}, + {name: "Kafka", function: checkKafka}, + } + + for i := 0; i < maxRetry; i++ { + if i != 0 { + time.Sleep(3 * time.Second) + } + fmt.Printf("Checking components Round %v...\n", i+1) + + allSuccess := true + for _, check := range checks { + err := check.function() + if err != nil { + errorPrint(fmt.Sprintf("Starting %s failed: %v", check.name, err)) + allSuccess = false + break + } else { + successPrint(fmt.Sprintf("%s starts successfully", check.name)) + } + } + + if allSuccess { + successPrint("All components started successfully!") + return + } + } + os.Exit(1) +} + +func exactIP(urll string) string { + u, _ := url.Parse(urll) + host, _, err := net.SplitHostPort(u.Host) + if err != nil { + host = u.Host + } + if strings.HasSuffix(host, ":") { + host = host[0 : len(host)-1] + } + return host +} + +func checkMysql() error { + var sqlDB *sql.DB + defer func() { + if sqlDB != nil { + sqlDB.Close() + } + }() + dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=true&loc=Local", + config.Config.Mysql.Username, config.Config.Mysql.Password, config.Config.Mysql.Address[0], "mysql") + db, err := gorm.Open(mysql.Open(dsn), nil) + if err != nil { + return errs.Wrap(err) + } else { + sqlDB, err = db.DB() + err = sqlDB.Ping() + if err != nil { + return errs.Wrap(err) + } + } + return nil +} + +func checkMongo() error { + var client *mongo.Client + defer func() { + if client != nil { + client.Disconnect(context.TODO()) + } + }() + mongodbHosts := "" + for i, v := range config.Config.Mongo.Address { + if i == len(config.Config.Mongo.Address)-1 { + mongodbHosts += v + } else { + mongodbHosts += v + "," + } + } + client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI( + fmt.Sprintf("mongodb://%v:%v@%v/?authSource=admin", + config.Config.Mongo.Username, config.Config.Mongo.Password, mongodbHosts))) + if err != nil { + return errs.Wrap(err) + } else { + err = client.Ping(context.TODO(), &readpref.ReadPref{}) + if err != nil { + return errs.Wrap(err) + } + } + return nil +} + +func checkMinio() error { + if config.Config.Object.Enable == "minio" { + conf := config.Config.Object.Minio + u, _ := url.Parse(conf.Endpoint) + minioClient, err := minio.New(u.Host, &minio.Options{ + Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, ""), + Secure: u.Scheme == "https", + }) + if err != nil { + return errs.Wrap(err) + } + + cancel, err := minioClient.HealthCheck(time.Duration(minioHealthCheckDuration) * time.Second) + defer func() { + if cancel != nil { + cancel() + } + }() + if err != nil { + return errs.Wrap(err) + } else { + if minioClient.IsOffline() { + return ErrComponentStart.Wrap("Minio server is offline") + } + } + if exactIP(config.Config.Object.ApiURL) == "127.0.0.1" || exactIP(config.Config.Object.Minio.SignEndpoint) == "127.0.0.1" { + return ErrConfig.Wrap("apiURL or Minio SignEndpoint endpoint contain 127.0.0.1") + } + } + return nil +} + +func checkRedis() error { + var redisClient redis.UniversalClient + defer func() { + if redisClient != nil { + redisClient.Close() + } + }() + if len(config.Config.Redis.Address) > 1 { + redisClient = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: config.Config.Redis.Address, + Username: config.Config.Redis.Username, + Password: config.Config.Redis.Password, + }) + } else { + redisClient = redis.NewClient(&redis.Options{ + Addr: config.Config.Redis.Address[0], + Username: config.Config.Redis.Username, + Password: config.Config.Redis.Password, + }) + } + _, err := redisClient.Ping(context.Background()).Result() + if err != nil { + return errs.Wrap(err) + } + return nil +} + +func checkZookeeper() error { + var c *zk.Conn + defer func() { + if c != nil { + c.Close() + } + }() + c, _, err := zk.Connect(config.Config.Zookeeper.ZkAddr, time.Second) + if err != nil { + return errs.Wrap(err) + } else { + if config.Config.Zookeeper.Username != "" && config.Config.Zookeeper.Password != "" { + if err := c.AddAuth("digest", []byte(config.Config.Zookeeper.Username+":"+config.Config.Zookeeper.Password)); err != nil { + return errs.Wrap(err) + } + } + _, _, err = c.Get("/") + if err != nil { + return errs.Wrap(err) + } + } + return nil +} + +func checkKafka() error { + var kafkaClient sarama.Client + defer func() { + if kafkaClient != nil { + kafkaClient.Close() + } + }() + cfg := sarama.NewConfig() + if config.Config.Kafka.Username != "" && config.Config.Kafka.Password != "" { + cfg.Net.SASL.Enable = true + cfg.Net.SASL.User = config.Config.Kafka.Username + cfg.Net.SASL.Password = config.Config.Kafka.Password + } + kafka.SetupTLSConfig(cfg) + kafkaClient, err := sarama.NewClient(config.Config.Kafka.Addr, cfg) + if err != nil { + return errs.Wrap(err) + } else { + topics, err := kafkaClient.Topics() + if err != nil { + return err + } + if !utils.IsContain(config.Config.Kafka.MsgToMongo.Topic, topics) { + return ErrComponentStart.Wrap(fmt.Sprintf("kafka doesn't contain topic:%v", config.Config.Kafka.MsgToMongo.Topic)) + } + if !utils.IsContain(config.Config.Kafka.MsgToPush.Topic, topics) { + return ErrComponentStart.Wrap(fmt.Sprintf("kafka doesn't contain topic:%v", config.Config.Kafka.MsgToPush.Topic)) + } + if !utils.IsContain(config.Config.Kafka.LatestMsgToRedis.Topic, topics) { + return ErrComponentStart.Wrap(fmt.Sprintf("kafka doesn't contain topic:%v", config.Config.Kafka.LatestMsgToRedis.Topic)) + } + } + return nil +} + +func errorPrint(s string) { + fmt.Printf("\x1b[%dm%v\x1b[0m\n", 31, s) +} + +func successPrint(s string) { + fmt.Printf("\x1b[%dm%v\x1b[0m\n", 32, s) +} + +func warningPrint(s string) { + fmt.Printf("\x1b[%dmWarning: But %v\x1b[0m\n", 33, s) +} diff --git a/tools/component/component_test.go b/tools/component/component_test.go new file mode 100644 index 000000000..afa51ef2c --- /dev/null +++ b/tools/component/component_test.go @@ -0,0 +1,42 @@ +// Copyright © 2023 OpenIM. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" +) + +func TestCheckMysql(t *testing.T) { + err := mockInitCfg() + assert.NoError(t, err, "Initialization should not produce errors") + + err = checkMysql() + if err != nil { + // You might expect an error if MySQL isn't running locally with the mock credentials. + t.Logf("Expected error due to mock configuration: %v", err) + } +} + +// Mock for initCfg for testing purpose +func mockInitCfg() error { + config.Config.Mysql.Username = "root" + config.Config.Mysql.Password = "openIM123" + config.Config.Mysql.Address = []string{"127.0.0.1:13306"} + return nil +} diff --git a/tools/component/go.mod b/tools/component/go.mod new file mode 100644 index 000000000..8d5548ee8 --- /dev/null +++ b/tools/component/go.mod @@ -0,0 +1,3 @@ +module github.com/openimsdk/open-im-server/v3/tools/component + +go 1.18 diff --git a/tools/infra/go.mod b/tools/infra/go.mod new file mode 100644 index 000000000..6ee10304b --- /dev/null +++ b/tools/infra/go.mod @@ -0,0 +1,3 @@ +module github.com/openimsdk/open-im-server/v3/tools/infra + +go 1.18 diff --git a/tools/infra/infra.go b/tools/infra/infra.go new file mode 100644 index 000000000..cc20a17c7 --- /dev/null +++ b/tools/infra/infra.go @@ -0,0 +1,40 @@ +// Copyright © 2023 OpenIM. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "log" +) + +func main() { + log.Println("Current module is still under development.") + message := ` +Current module is still under development. + +____ _____ __ __ +/ __ \ |_ _|| \/ | +| | | | _ __ ___ _ __ | | | \ / | +| | | || '_ \ / _ \| '_ \ | | | |\/| | +| |__| || |_) || __/| | | | _| |_ | | | | +\____/ | .__/ \___||_| |_||_____||_| |_| + | | + |_| + +Keep checking for updates! +` + + fmt.Println(message) +} diff --git a/tools/ncpu/README.md b/tools/ncpu/README.md new file mode 100644 index 000000000..f7c05d583 --- /dev/null +++ b/tools/ncpu/README.md @@ -0,0 +1,39 @@ +# ncpu + +**ncpu** is a simple utility to fetch the number of CPU cores across different operating systems. + +## Introduction + +In various scenarios, especially while compiling code, it's beneficial to know the number of available CPU cores to optimize the build process. However, the command to fetch the CPU core count differs between operating systems. For example, on Linux, we use `nproc`, while on macOS, it's `sysctl -n hw.ncpu`. The `ncpu` utility provides a unified way to obtain this number, regardless of the platform. + +## Usage + +To retrieve the number of CPU cores, simply use the `ncpu` command: + +```bash +$ ncpu +``` + +This will return an integer representing the number of available CPU cores. + +### Example: + +Let's say you're compiling a project using `make`. To utilize all the CPU cores for the compilation process, you can use: + +```bash +$ make -j $(ncpu) build # or any other build command +``` + +The above command will ensure the build process takes advantage of all the available CPU cores, thereby potentially speeding up the compilation. + +## Why use `ncpu`? + +- **Cross-platform compatibility**: No need to remember or detect which OS-specific command to use. Just use `ncpu`! + +- **Ease of use**: A simple and intuitive command that's easy to incorporate into scripts or command-line operations. + +- **Consistency**: Ensures consistent behavior and output across different systems and environments. + +## Installation + +(Include installation steps here, e.g., how to clone the repo, build the tool, or install via package manager.) diff --git a/tools/ncpu/go.mod b/tools/ncpu/go.mod new file mode 100644 index 000000000..dfccea27a --- /dev/null +++ b/tools/ncpu/go.mod @@ -0,0 +1,5 @@ +module github.com/openimsdk/open-im-server/v3/tools/ncpu + +go 1.18 + +require go.uber.org/automaxprocs v1.5.3 diff --git a/tools/ncpu/go.sum b/tools/ncpu/go.sum new file mode 100644 index 000000000..804f593b5 --- /dev/null +++ b/tools/ncpu/go.sum @@ -0,0 +1,7 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/tools/ncpu/ncpu.go b/tools/ncpu/ncpu.go new file mode 100644 index 000000000..7ca3dff5e --- /dev/null +++ b/tools/ncpu/ncpu.go @@ -0,0 +1,27 @@ +// Copyright © 2023 OpenIM. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "runtime" + + "go.uber.org/automaxprocs/maxprocs" +) + +func main() { + maxprocs.Set() + fmt.Print(runtime.GOMAXPROCS(0)) +} diff --git a/tools/ncpu/ncpu_test.go b/tools/ncpu/ncpu_test.go new file mode 100644 index 000000000..f24203226 --- /dev/null +++ b/tools/ncpu/ncpu_test.go @@ -0,0 +1,35 @@ +// Copyright © 2023 OpenIM. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "testing" + +func Test_main(t *testing.T) { + tests := []struct { + name string + }{ + { + name: "Test_main", + }, + { + name: "Test_main2", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + main() + }) + } +} diff --git a/tools/openim-web/Dockerfile b/tools/openim-web/Dockerfile new file mode 100644 index 000000000..2e0cd9e8e --- /dev/null +++ b/tools/openim-web/Dockerfile @@ -0,0 +1,47 @@ +# Copyright © 2023 OpenIM. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 使用官方Go镜像作为基础镜像 +FROM golang:1.21 AS build-env +ENV CGO_ENABLED=0 +# 设置工作目录 +WORKDIR /app + +# 安装curl和unzip工具 +#RUN apt-get update && apt-get install -y curl unzip + +# 从GitHub下载并解压dist.zip +#RUN curl -LO https://github.com/OpenIMSDK/dist.zip \ + # && unzip dist.zip -d ./ \ + # && rm dist.zip + +# 复制Go代码到容器 +COPY . . + +# 编译Go代码 +RUN go build -o openim-web + +# 使用轻量级的基础镜像 +FROM debian:buster-slim + +# 将编译好的二进制文件和dist资源复制到新的容器 +WORKDIR /app +COPY --from=build-env /app/openim-web /app/openim-web +COPY --from=build-env /app/dist /app/dist + +# 开放容器的20001端口 +EXPOSE 20001 + +# 指定容器启动命令 +ENTRYPOINT ["/app/openim-web"] \ No newline at end of file diff --git a/tools/openim-web/README.md b/tools/openim-web/README.md new file mode 100644 index 000000000..afd5e9a96 --- /dev/null +++ b/tools/openim-web/README.md @@ -0,0 +1,79 @@ +# OpenIM Web Service + +- [OpenIM Web Service](#openim-web-service) + - [Overview](#overview) + - [User](#user) + - [Docker Deployment](#docker-deployment) + - [Build the Docker Image](#build-the-docker-image) + - [Run the Docker Container](#run-the-docker-container) + - [Configuration](#configuration) + - [Contributions](#contributions) + + +OpenIM Web Service is a lightweight containerized service built with Go. The service serves static files and allows customization via environment variables. + +## Overview + +- Built using Go. +- Deployed as a Docker container. +- Serves static files from a directory which can be set via an environment variable. +- The default port for the service is `20001`, but it can be customized using an environment variable. + +## User + +example: + +```bash +$ ./openim-web -h +Usage of ./openim-web: + -distPath string + Path to the distribution (default "/app/dist") + -port string + Port to run the server on (default "20001") +``` + +Variables can be set as above, Environment variables can also be set + +example: + +```bash +$ export OPENIM_WEB_DIST_PATH="/app/dist" +$ export OPENIM_WEB_PPRT="11001" +``` + +Initialize the env configuration file: + +```bash +$ make init +``` + +## Docker Deployment + +### Build the Docker Image + +Even though we've implemented automation, it's to make the developer experience easier: + +To build the Docker image for OpenIM Web Service: + +```bash +$ docker build -t openim-web . +``` + +### Run the Docker Container + +To run the service: + +```bash +$ docker run -e DIST_PATH=/app/dist -e PORT=20001 -p 20001:20001 openim-web +``` + +## Configuration + +You can configure the OpenIM Web Service using the following environment variables: + +- **DIST_PATH**: The path to the directory containing the static files. Default: `/app/dist`. +- **PORT**: The port on which the service listens. Default: `11001`. + +## Contributions + +We welcome contributions from the community. If you find any bugs or have feature suggestions, please create an issue or send a pull request. \ No newline at end of file diff --git a/tools/openim-web/go.mod b/tools/openim-web/go.mod new file mode 100644 index 000000000..6f30c673c --- /dev/null +++ b/tools/openim-web/go.mod @@ -0,0 +1,7 @@ +module github.com/openimsdk/open-im-server/v3/tools/openim-web + +go 1.18 + +require gopkg.in/yaml.v2 v2.4.0 + +require github.com/NYTimes/gziphandler v1.1.1 // indirect diff --git a/tools/openim-web/go.sum b/tools/openim-web/go.sum new file mode 100644 index 000000000..54ca3deb1 --- /dev/null +++ b/tools/openim-web/go.sum @@ -0,0 +1,10 @@ +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/tools/openim-web/openim-web.go b/tools/openim-web/openim-web.go new file mode 100644 index 000000000..c913e35e7 --- /dev/null +++ b/tools/openim-web/openim-web.go @@ -0,0 +1,63 @@ +// Copyright © 2023 OpenIM. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "log" + "net/http" + "os" + + "github.com/NYTimes/gziphandler" +) + +var ( + distPathFlag string + portFlag string +) + +func init() { + flag.StringVar(&distPathFlag, "distPath", "/app/dist", "Path to the distribution") + flag.StringVar(&portFlag, "port", "11001", "Port to run the server on") +} + +func main() { + flag.Parse() + + distPath := getConfigValue("DIST_PATH", distPathFlag, "/app/dist") + fs := http.FileServer(http.Dir(distPath)) + + withGzip := gziphandler.GzipHandler(fs) + + http.Handle("/", withGzip) + + port := getConfigValue("PORT", portFlag, "11001") + log.Printf("Server listening on port %s in %s...", port, distPath) + err := http.ListenAndServe(":"+port, nil) + if err != nil { + log.Fatal(err) + } +} + +func getConfigValue(envKey, flagValue, fallback string) string { + envVal := os.Getenv(envKey) + if envVal != "" { + return envVal + } + if flagValue != "" { + return flagValue + } + return fallback +} diff --git a/tools/openim-web/openim-web_test.go b/tools/openim-web/openim-web_test.go new file mode 100644 index 000000000..dd1c93316 --- /dev/null +++ b/tools/openim-web/openim-web_test.go @@ -0,0 +1,71 @@ +// Copyright © 2023 OpenIM. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "testing" +) + +func TestGetConfigValue(t *testing.T) { + tests := []struct { + name string + envKey string + envValue string + flagValue string + fallback string + wantResult string + }{ + { + name: "environment variable set", + envKey: "TEST_KEY", + envValue: "envValue", + flagValue: "", + fallback: "default", + wantResult: "envValue", + }, + { + name: "flag set and environment variable not set", + envKey: "TEST_KEY", + envValue: "", + flagValue: "flagValue", + fallback: "default", + wantResult: "flagValue", + }, + { + name: "nothing set, use fallback", + envKey: "TEST_KEY", + envValue: "", + flagValue: "", + fallback: "default", + wantResult: "default", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.envValue != "" { + os.Setenv(tt.envKey, tt.envValue) + defer os.Unsetenv(tt.envKey) + } + + got := getConfigValue(tt.envKey, tt.flagValue, tt.fallback) + + if got != tt.wantResult { + t.Errorf("getConfigValue(%s, %s, %s) = %s; want %s", tt.envKey, tt.flagValue, tt.fallback, got, tt.wantResult) + } + }) + } +} diff --git a/tools/url2im/go.mod b/tools/url2im/go.mod new file mode 100644 index 000000000..b1d0a5bac --- /dev/null +++ b/tools/url2im/go.mod @@ -0,0 +1,20 @@ +module github.com/openimsdk/open-im-server/v3/tools/url2im + +go 1.20 + +require ( + github.com/OpenIMSDK/protocol v0.0.21 + github.com/kelindar/bitmap v1.5.1 +) + +require ( + github.com/golang/protobuf v1.5.3 // indirect + github.com/kelindar/simd v1.1.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/grpc v1.56.2 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/tools/url2im/go.sum b/tools/url2im/go.sum new file mode 100644 index 000000000..9985d4828 --- /dev/null +++ b/tools/url2im/go.sum @@ -0,0 +1,33 @@ +github.com/OpenIMSDK/protocol v0.0.21 h1:5H6H+hJ9d/VgRqttvxD/zfK9Asd+4M8Eknk5swSbUVY= +github.com/OpenIMSDK/protocol v0.0.21/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/kelindar/bitmap v1.5.1 h1:+ZmZdwHbJ+CGE+q/aAJ74KJSnp0vOlGD7KY5x51mVzk= +github.com/kelindar/bitmap v1.5.1/go.mod h1:j3qZjxH9s4OtvsnFTP2bmPkjqil9Y2xQlxPYHexasEA= +github.com/kelindar/simd v1.1.2 h1:KduKb+M9cMY2HIH8S/cdJyD+5n5EGgq+Aeeleos55To= +github.com/kelindar/simd v1.1.2/go.mod h1:inq4DFudC7W8L5fhxoeZflLRNpWSs0GNx6MlWFvuvr0= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= +google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/tools/url2im/main.go b/tools/url2im/main.go new file mode 100644 index 000000000..ee159b5e8 --- /dev/null +++ b/tools/url2im/main.go @@ -0,0 +1,84 @@ +package main + +import ( + "flag" + "log" + "os" + "path/filepath" + "time" + + "github.com/openimsdk/open-im-server/v3/tools/url2im/pkg" +) + +/*take.txt +{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"} +{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"} +{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"} +*/ + +func main() { + var conf pkg.Config // 后面带*的为必填项 + flag.StringVar(&conf.TaskPath, "task", "take.txt", "task path") // 任务日志文件* + flag.StringVar(&conf.ProgressPath, "progress", "", "progress path") // 进度日志文件 + flag.IntVar(&conf.Concurrency, "concurrency", 1, "concurrency num") // 并发数 + flag.IntVar(&conf.Retry, "retry", 1, "retry num") // 重试次数 + flag.StringVar(&conf.TempDir, "temp", "", "temp dir") // 临时文件夹 + flag.Int64Var(&conf.CacheSize, "cache", 1024*1024*100, "cache size") // 缓存大小(超过时,下载到磁盘) + flag.Int64Var((*int64)(&conf.Timeout), "timeout", 5000, "timeout") // 请求超时时间(毫秒) + flag.StringVar(&conf.Api, "api", "http://127.0.0.1:10002", "api") // im地址* + flag.StringVar(&conf.UserID, "userID", "openIM123456", "userID") // im管理员 + flag.StringVar(&conf.Secret, "secret", "openIM123", "secret") // im config secret + flag.Parse() + if !filepath.IsAbs(conf.TaskPath) { + var err error + conf.TaskPath, err = filepath.Abs(conf.TaskPath) + if err != nil { + log.Println("get abs path err:", err) + return + } + } + if conf.ProgressPath == "" { + conf.ProgressPath = conf.TaskPath + ".progress.txt" + } else if !filepath.IsAbs(conf.ProgressPath) { + var err error + conf.ProgressPath, err = filepath.Abs(conf.ProgressPath) + if err != nil { + log.Println("get abs path err:", err) + return + } + } + if conf.TempDir == "" { + conf.TempDir = conf.TaskPath + ".temp" + } + if info, err := os.Stat(conf.TempDir); err == nil { + if !info.IsDir() { + log.Printf("temp dir %s is not dir\n", err) + return + } + } else if os.IsNotExist(err) { + if err := os.MkdirAll(conf.TempDir, os.ModePerm); err != nil { + log.Printf("mkdir temp dir %s err %+v\n", conf.TempDir, err) + return + } + defer os.RemoveAll(conf.TempDir) + } else { + log.Println("get temp dir err:", err) + return + } + if conf.Concurrency <= 0 { + conf.Concurrency = 1 + } + if conf.Retry <= 0 { + conf.Retry = 1 + } + if conf.CacheSize <= 0 { + conf.CacheSize = 1024 * 1024 * 100 // 100M + } + if conf.Timeout <= 0 { + conf.Timeout = 5000 + } + conf.Timeout = conf.Timeout * time.Millisecond + if err := pkg.Run(conf); err != nil { + log.Println("main err:", err) + } +} diff --git a/tools/url2im/pkg/api.go b/tools/url2im/pkg/api.go new file mode 100644 index 000000000..1fc3813bb --- /dev/null +++ b/tools/url2im/pkg/api.go @@ -0,0 +1,112 @@ +package pkg + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + "github.com/OpenIMSDK/protocol/auth" + "github.com/OpenIMSDK/protocol/constant" + "github.com/OpenIMSDK/protocol/third" +) + +type Api struct { + Api string + UserID string + Secret string + Token string + Client *http.Client +} + +func (a *Api) apiPost(ctx context.Context, path string, req any, resp any) error { + operationID, _ := ctx.Value("operationID").(string) + if operationID == "" { + return errors.New("call api operationID is empty") + } + reqBody, err := json.Marshal(req) + if err != nil { + return err + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, a.Api+path, bytes.NewReader(reqBody)) + if err != nil { + return err + } + DefaultRequestHeader(request.Header) + request.ContentLength = int64(len(reqBody)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("operationID", operationID) + if a.Token != "" { + request.Header.Set("token", a.Token) + } + response, err := a.Client.Do(request) + if err != nil { + return err + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("api %s status %s body %s", path, response.Status, body) + } + var baseResponse struct { + ErrCode int `json:"errCode"` + ErrMsg string `json:"errMsg"` + ErrDlt string `json:"errDlt"` + Data json.RawMessage `json:"data"` + } + if err := json.Unmarshal(body, &baseResponse); err != nil { + return err + } + if baseResponse.ErrCode != 0 { + return fmt.Errorf("api %s errCode %d errMsg %s errDlt %s", path, baseResponse.ErrCode, baseResponse.ErrMsg, baseResponse.ErrDlt) + } + if resp != nil { + if err := json.Unmarshal(baseResponse.Data, resp); err != nil { + return err + } + } + return nil +} + +func (a *Api) GetToken(ctx context.Context) (string, error) { + req := auth.UserTokenReq{ + UserID: a.UserID, + Secret: a.Secret, + PlatformID: constant.AdminPlatformID, + } + var resp auth.UserTokenResp + if err := a.apiPost(ctx, "/auth/user_token", &req, &resp); err != nil { + return "", err + } + return resp.Token, nil +} + +func (a *Api) GetPartLimit(ctx context.Context) (*third.PartLimitResp, error) { + var resp third.PartLimitResp + if err := a.apiPost(ctx, "/object/part_limit", &third.PartLimitReq{}, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (a *Api) InitiateMultipartUpload(ctx context.Context, req *third.InitiateMultipartUploadReq) (*third.InitiateMultipartUploadResp, error) { + var resp third.InitiateMultipartUploadResp + if err := a.apiPost(ctx, "/object/initiate_multipart_upload", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (a *Api) CompleteMultipartUpload(ctx context.Context, req *third.CompleteMultipartUploadReq) (string, error) { + var resp third.CompleteMultipartUploadResp + if err := a.apiPost(ctx, "/object/complete_multipart_upload", req, &resp); err != nil { + return "", err + } + return resp.Url, nil +} diff --git a/tools/url2im/pkg/buffer.go b/tools/url2im/pkg/buffer.go new file mode 100644 index 000000000..008400926 --- /dev/null +++ b/tools/url2im/pkg/buffer.go @@ -0,0 +1,96 @@ +package pkg + +import ( + "bytes" + "io" + "os" +) + +type ReadSeekSizeCloser interface { + io.ReadSeekCloser + Size() int64 +} + +func NewReader(r io.Reader, max int64, path string) (ReadSeekSizeCloser, error) { + buf := make([]byte, max+1) + n, err := io.ReadFull(r, buf) + if err == nil { + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0666) + if err != nil { + return nil, err + } + var ok bool + defer func() { + if !ok { + _ = f.Close() + _ = os.Remove(path) + } + }() + if _, err := f.Write(buf[:n]); err != nil { + return nil, err + } + cn, err := io.Copy(f, r) + if err != nil { + return nil, err + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return nil, err + } + ok = true + return &fileBuffer{ + f: f, + n: cn + int64(n), + }, nil + } else if err == io.EOF || err == io.ErrUnexpectedEOF { + return &memoryBuffer{ + r: bytes.NewReader(buf[:n]), + }, nil + } else { + return nil, err + } +} + +type fileBuffer struct { + n int64 + f *os.File +} + +func (r *fileBuffer) Read(p []byte) (n int, err error) { + return r.f.Read(p) +} + +func (r *fileBuffer) Seek(offset int64, whence int) (int64, error) { + return r.f.Seek(offset, whence) +} + +func (r *fileBuffer) Size() int64 { + return r.n +} + +func (r *fileBuffer) Close() error { + name := r.f.Name() + if err := r.f.Close(); err != nil { + return err + } + return os.Remove(name) +} + +type memoryBuffer struct { + r *bytes.Reader +} + +func (r *memoryBuffer) Read(p []byte) (n int, err error) { + return r.r.Read(p) +} + +func (r *memoryBuffer) Seek(offset int64, whence int) (int64, error) { + return r.r.Seek(offset, whence) +} + +func (r *memoryBuffer) Close() error { + return nil +} + +func (r *memoryBuffer) Size() int64 { + return r.r.Size() +} diff --git a/tools/url2im/pkg/config.go b/tools/url2im/pkg/config.go new file mode 100644 index 000000000..020395262 --- /dev/null +++ b/tools/url2im/pkg/config.go @@ -0,0 +1,16 @@ +package pkg + +import "time" + +type Config struct { + TaskPath string + ProgressPath string + Concurrency int + Retry int + Timeout time.Duration + Api string + UserID string + Secret string + TempDir string + CacheSize int64 +} diff --git a/tools/url2im/pkg/http.go b/tools/url2im/pkg/http.go new file mode 100644 index 000000000..32e128524 --- /dev/null +++ b/tools/url2im/pkg/http.go @@ -0,0 +1,7 @@ +package pkg + +import "net/http" + +func DefaultRequestHeader(header http.Header) { + header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36") +} diff --git a/tools/url2im/pkg/manage.go b/tools/url2im/pkg/manage.go new file mode 100644 index 000000000..a68078f85 --- /dev/null +++ b/tools/url2im/pkg/manage.go @@ -0,0 +1,385 @@ +package pkg + +import ( + "bufio" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/OpenIMSDK/protocol/third" +) + +type Upload struct { + URL string `json:"url"` + Name string `json:"name"` + ContentType string `json:"contentType"` +} + +type Task struct { + Index int + Upload Upload +} + +type PartInfo struct { + ContentType string + PartSize int64 + PartNum int + FileMd5 string + PartMd5 string + PartSizes []int64 + PartMd5s []string +} + +func Run(conf Config) error { + m := &Manage{ + prefix: time.Now().Format("20060102150405"), + conf: &conf, + ctx: context.Background(), + } + return m.Run() +} + +type Manage struct { + conf *Config + ctx context.Context + api *Api + partLimit *third.PartLimitResp + prefix string + tasks chan Task + id uint64 + success int64 + failed int64 +} + +func (m *Manage) tempFilePath() string { + return filepath.Join(m.conf.TempDir, fmt.Sprintf("%s_%d", m.prefix, atomic.AddUint64(&m.id, 1))) +} + +func (m *Manage) Run() error { + defer func(start time.Time) { + log.Printf("run time %s\n", time.Since(start)) + }(time.Now()) + m.api = &Api{ + Api: m.conf.Api, + UserID: m.conf.UserID, + Secret: m.conf.Secret, + Client: &http.Client{Timeout: m.conf.Timeout}, + } + var err error + ctx := context.WithValue(m.ctx, "operationID", fmt.Sprintf("%s_init", m.prefix)) + m.api.Token, err = m.api.GetToken(ctx) + if err != nil { + return err + } + m.partLimit, err = m.api.GetPartLimit(ctx) + if err != nil { + return err + } + progress, err := ReadProgress(m.conf.ProgressPath) + if err != nil { + return err + } + progressFile, err := os.OpenFile(m.conf.ProgressPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) + if err != nil { + return err + } + var mutex sync.Mutex + writeSuccessIndex := func(index int) { + mutex.Lock() + defer mutex.Unlock() + if _, err := progressFile.Write([]byte(strconv.Itoa(index) + "\n")); err != nil { + log.Printf("write progress err: %v\n", err) + } + } + file, err := os.Open(m.conf.TaskPath) + if err != nil { + return err + } + m.tasks = make(chan Task, m.conf.Concurrency*2) + go func() { + defer file.Close() + defer close(m.tasks) + scanner := bufio.NewScanner(file) + var ( + index int + num int + ) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + index++ + if progress.IsUploaded(index) { + log.Printf("index: %d already uploaded %s\n", index, line) + continue + } + var upload Upload + if err := json.Unmarshal([]byte(line), &upload); err != nil { + log.Printf("index: %d json.Unmarshal(%s) err: %v", index, line, err) + continue + } + num++ + m.tasks <- Task{ + Index: index, + Upload: upload, + } + } + if num == 0 { + log.Println("mark all completed") + } + }() + var wg sync.WaitGroup + wg.Add(m.conf.Concurrency) + for i := 0; i < m.conf.Concurrency; i++ { + go func(tid int) { + defer wg.Done() + for task := range m.tasks { + var success bool + for n := 0; n < m.conf.Retry; n++ { + ctx := context.WithValue(m.ctx, "operationID", fmt.Sprintf("%s_%d_%d_%d", m.prefix, tid, task.Index, n+1)) + if urlRaw, err := m.RunTask(ctx, task); err == nil { + writeSuccessIndex(task.Index) + log.Println("index:", task.Index, "upload success", "urlRaw", urlRaw) + success = true + break + } else { + log.Printf("index: %d upload: %+v err: %v", task.Index, task.Upload, err) + } + } + if success { + atomic.AddInt64(&m.success, 1) + } else { + atomic.AddInt64(&m.failed, 1) + log.Printf("index: %d upload: %+v failed", task.Index, task.Upload) + } + } + }(i + 1) + } + wg.Wait() + log.Printf("execution completed success %d failed %d\n", m.success, m.failed) + return nil +} + +func (m *Manage) RunTask(ctx context.Context, task Task) (string, error) { + resp, err := m.HttpGet(ctx, task.Upload.URL) + if err != nil { + return "", err + } + defer resp.Body.Close() + reader, err := NewReader(resp.Body, m.conf.CacheSize, m.tempFilePath()) + if err != nil { + return "", err + } + defer reader.Close() + part, err := m.getPartInfo(ctx, reader, reader.Size()) + if err != nil { + return "", err + } + var contentType string + if task.Upload.ContentType == "" { + contentType = part.ContentType + } else { + contentType = task.Upload.ContentType + } + initiateMultipartUploadResp, err := m.api.InitiateMultipartUpload(ctx, &third.InitiateMultipartUploadReq{ + Hash: part.PartMd5, + Size: reader.Size(), + PartSize: part.PartSize, + MaxParts: -1, + Cause: "batch-import", + Name: task.Upload.Name, + ContentType: contentType, + }) + if err != nil { + return "", err + } + if initiateMultipartUploadResp.Upload == nil { + return initiateMultipartUploadResp.Url, nil + } + if _, err := reader.Seek(0, io.SeekStart); err != nil { + return "", err + } + uploadParts := make([]*third.SignPart, part.PartNum) + for _, part := range initiateMultipartUploadResp.Upload.Sign.Parts { + uploadParts[part.PartNumber-1] = part + } + for i, currentPartSize := range part.PartSizes { + md5Reader := NewMd5Reader(io.LimitReader(reader, currentPartSize)) + if m.doPut(ctx, m.api.Client, initiateMultipartUploadResp.Upload.Sign, uploadParts[i], md5Reader, currentPartSize); err != nil { + return "", err + } + if md5val := md5Reader.Md5(); md5val != part.PartMd5s[i] { + return "", fmt.Errorf("upload part %d failed, md5 not match, expect %s, got %s", i, part.PartMd5s[i], md5val) + } + } + urlRaw, err := m.api.CompleteMultipartUpload(ctx, &third.CompleteMultipartUploadReq{ + UploadID: initiateMultipartUploadResp.Upload.UploadID, + Parts: part.PartMd5s, + Name: task.Upload.Name, + ContentType: contentType, + Cause: "batch-import", + }) + if err != nil { + return "", err + } + return urlRaw, nil +} + +func (m *Manage) partSize(size int64) (int64, error) { + if size <= 0 { + return 0, errors.New("size must be greater than 0") + } + if size > m.partLimit.MaxPartSize*int64(m.partLimit.MaxNumSize) { + return 0, fmt.Errorf("size must be less than %db", m.partLimit.MaxPartSize*int64(m.partLimit.MaxNumSize)) + } + if size <= m.partLimit.MinPartSize*int64(m.partLimit.MaxNumSize) { + return m.partLimit.MinPartSize, nil + } + partSize := size / int64(m.partLimit.MaxNumSize) + if size%int64(m.partLimit.MaxNumSize) != 0 { + partSize++ + } + return partSize, nil +} + +func (m *Manage) partMD5(parts []string) string { + s := strings.Join(parts, ",") + md5Sum := md5.Sum([]byte(s)) + return hex.EncodeToString(md5Sum[:]) +} + +func (m *Manage) getPartInfo(ctx context.Context, r io.Reader, fileSize int64) (*PartInfo, error) { + partSize, err := m.partSize(fileSize) + if err != nil { + return nil, err + } + partNum := int(fileSize / partSize) + if fileSize%partSize != 0 { + partNum++ + } + partSizes := make([]int64, partNum) + for i := 0; i < partNum; i++ { + partSizes[i] = partSize + } + partSizes[partNum-1] = fileSize - partSize*(int64(partNum)-1) + partMd5s := make([]string, partNum) + buf := make([]byte, 1024*8) + fileMd5 := md5.New() + var contentType string + for i := 0; i < partNum; i++ { + h := md5.New() + r := io.LimitReader(r, partSize) + for { + if n, err := r.Read(buf); err == nil { + if contentType == "" { + contentType = http.DetectContentType(buf[:n]) + } + h.Write(buf[:n]) + fileMd5.Write(buf[:n]) + } else if err == io.EOF { + break + } else { + return nil, err + } + } + partMd5s[i] = hex.EncodeToString(h.Sum(nil)) + } + partMd5Val := m.partMD5(partMd5s) + fileMd5val := hex.EncodeToString(fileMd5.Sum(nil)) + return &PartInfo{ + ContentType: contentType, + PartSize: partSize, + PartNum: partNum, + FileMd5: fileMd5val, + PartMd5: partMd5Val, + PartSizes: partSizes, + PartMd5s: partMd5s, + }, nil +} + +func (m *Manage) doPut(ctx context.Context, client *http.Client, sign *third.AuthSignParts, part *third.SignPart, reader io.Reader, size int64) error { + rawURL := part.Url + if rawURL == "" { + rawURL = sign.Url + } + if len(sign.Query)+len(part.Query) > 0 { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + query := u.Query() + for i := range sign.Query { + v := sign.Query[i] + query[v.Key] = v.Values + } + for i := range part.Query { + v := part.Query[i] + query[v.Key] = v.Values + } + u.RawQuery = query.Encode() + rawURL = u.String() + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, reader) + if err != nil { + return err + } + for i := range sign.Header { + v := sign.Header[i] + req.Header[v.Key] = v.Values + } + for i := range part.Header { + v := part.Header[i] + req.Header[v.Key] = v.Values + } + req.ContentLength = size + resp, err := client.Do(req) + if err != nil { + return err + } + defer func() { + _ = resp.Body.Close() + }() + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode/200 != 1 { + return fmt.Errorf("PUT %s part %d failed, status code %d, body %s", rawURL, part.PartNumber, resp.StatusCode, string(body)) + } + return nil +} + +func (m *Manage) HttpGet(ctx context.Context, url string) (*http.Response, error) { + reqUrl := url + for { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, reqUrl, nil) + if err != nil { + return nil, err + } + DefaultRequestHeader(request.Header) + response, err := m.api.Client.Do(request) + if err != nil { + return nil, err + } + if response.StatusCode != http.StatusOK { + _ = response.Body.Close() + return nil, fmt.Errorf("http get %s status %s", url, response.Status) + } + return response, nil + } +} diff --git a/tools/url2im/pkg/md5.go b/tools/url2im/pkg/md5.go new file mode 100644 index 000000000..0db5ba000 --- /dev/null +++ b/tools/url2im/pkg/md5.go @@ -0,0 +1,29 @@ +package pkg + +import ( + "crypto/md5" + "encoding/hex" + "hash" + "io" +) + +func NewMd5Reader(r io.Reader) *Md5Reader { + return &Md5Reader{h: md5.New(), r: r} +} + +type Md5Reader struct { + h hash.Hash + r io.Reader +} + +func (r *Md5Reader) Read(p []byte) (n int, err error) { + n, err = r.r.Read(p) + if err == nil && n > 0 { + r.h.Write(p[:n]) + } + return +} + +func (r *Md5Reader) Md5() string { + return hex.EncodeToString(r.h.Sum(nil)) +} diff --git a/tools/url2im/pkg/progress.go b/tools/url2im/pkg/progress.go new file mode 100644 index 000000000..2d6ef3891 --- /dev/null +++ b/tools/url2im/pkg/progress.go @@ -0,0 +1,41 @@ +package pkg + +import ( + "bufio" + "os" + "strconv" + + "github.com/kelindar/bitmap" +) + +func ReadProgress(path string) (*Progress, error) { + file, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return &Progress{}, nil + } + return nil, err + } + defer file.Close() + scanner := bufio.NewScanner(file) + var upload bitmap.Bitmap + for scanner.Scan() { + index, err := strconv.Atoi(scanner.Text()) + if err != nil || index < 0 { + continue + } + upload.Set(uint32(index)) + } + return &Progress{upload: upload}, nil +} + +type Progress struct { + upload bitmap.Bitmap +} + +func (p *Progress) IsUploaded(index int) bool { + if p == nil { + return false + } + return p.upload.Contains(uint32(index)) +} diff --git a/tools/versionchecker/go.mod b/tools/versionchecker/go.mod new file mode 100644 index 000000000..5858649e8 --- /dev/null +++ b/tools/versionchecker/go.mod @@ -0,0 +1,3 @@ +module github.com/openimsdk/open-im-server/v3/tools/versionchecker + +go 1.18 diff --git a/tools/versionchecker/versionchecker.go b/tools/versionchecker/versionchecker.go new file mode 100644 index 000000000..e7bc3e379 --- /dev/null +++ b/tools/versionchecker/versionchecker.go @@ -0,0 +1,115 @@ +// Copyright © 2023 OpenIM. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "fmt" + "os/exec" + "runtime" + "time" +) + +func executeCommand(cmdName string, args ...string) (string, error) { + cmd := exec.Command(cmdName, args...) + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + return "", fmt.Errorf("Error executing %s: %v", cmdName, err) + } + return out.String(), nil +} + +func printTime() string { + currentTime := time.Now() + + // 使用 Format 函数以优雅的方式格式化日期和时间 + // 2006-01-02 15:04:05 是 Go 中的标准时间格式 + formattedTime := currentTime.Format("2006-01-02 15:04:05") + + return fmt.Sprintf("Current Date & Time:", formattedTime) +} + +func getGoVersion() string { + version := runtime.Version() + goos := runtime.GOOS + goarch := runtime.GOARCH + return fmt.Sprintf("Go Version: %s\nOS: %s\nArchitecture: %s", version, goos, goarch) +} + +func getDockerVersion() string { + version, err := executeCommand("docker", "--version") + if err != nil { + return "Docker is not installed. Please install it to get the version." + } + return version +} + +func getDockerComposeVersion() string { + version, err := executeCommand("docker-compose", "--version") + if err != nil { + return "Docker Compose is not installed. Please install it to get the version." + } + return version +} + +func getKubernetesVersion() string { + version, err := executeCommand("kubectl", "version", "--client", "--short") + if err != nil { + return "Kubernetes is not installed. Please install it to get the version." + } + return version +} + +func getGitVersion() string { + version, err := executeCommand("git", "branch", "--show-current") + if err != nil { + return "Git is not installed. Please install it to get the version." + } + return version +} + +// NOTE: You'll need to provide appropriate commands for OpenIM versions. +func getOpenIMServerVersion() string { + // Placeholder + return "OpenIM Server: v3.2" +} + +func getOpenIMClientVersion() string { + // Placeholder + return "OpenIM Client: v3.2" +} + +func main() { + fmt.Println(printTime()) + fmt.Println("# Diagnostic Tool Result\n") + fmt.Println("## Go Version") + fmt.Println(getGoVersion()) + fmt.Println("## Branch Type") + fmt.Println(getGitVersion()) + fmt.Println("## Docker Version") + fmt.Println(getDockerVersion()) + fmt.Println("## Docker Compose Version") + fmt.Println(getDockerComposeVersion()) + fmt.Println("## Kubernetes Version") + fmt.Println(getKubernetesVersion()) + fmt.Println("## OpenIM Versions") + fmt.Println(getOpenIMServerVersion()) + fmt.Println(getOpenIMClientVersion()) +} diff --git a/tools/yamlfmt/OWNERS b/tools/yamlfmt/OWNERS new file mode 100644 index 000000000..b7a5428e7 --- /dev/null +++ b/tools/yamlfmt/OWNERS @@ -0,0 +1,10 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +reviewers: + - cubxxw + - kubbot +approvers: + - cubxxw +labels: + - sig/testing + - sig/contributor-experience \ No newline at end of file diff --git a/tools/yamlfmt/go.mod b/tools/yamlfmt/go.mod new file mode 100644 index 000000000..5b3581efc --- /dev/null +++ b/tools/yamlfmt/go.mod @@ -0,0 +1,8 @@ +module github.com/openimsdk/open-im-server/v3/tools/yamlfmt + +go 1.18 + +require ( + github.com/likexian/gokit v0.25.13 + gopkg.in/yaml.v3 v3.0.1 +) diff --git a/tools/yamlfmt/go.sum b/tools/yamlfmt/go.sum new file mode 100644 index 000000000..0cf090cc8 --- /dev/null +++ b/tools/yamlfmt/go.sum @@ -0,0 +1,6 @@ +github.com/likexian/gokit v0.25.13 h1:p2Uw3+6fGG53CwdU2Dz0T6bOycdb2+bAFAa3ymwWVkM= +github.com/likexian/gokit v0.25.13/go.mod h1:qQhEWFBEfqLCO3/vOEo2EDKd+EycekVtUK4tex+l2H4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/yamlfmt/yamlfmt.go b/tools/yamlfmt/yamlfmt.go new file mode 100644 index 000000000..38004bfde --- /dev/null +++ b/tools/yamlfmt/yamlfmt.go @@ -0,0 +1,72 @@ +// Copyright © 2023 OpenIM. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// OPENIM plan on prow tools +package main + +import ( + "flag" + "fmt" + "io" + "os" + + "gopkg.in/yaml.v3" +) + +func main() { + // Prow OWNERs file defines the default indent as 2 spaces. + indent := flag.Int("indent", 2, "default indent") + flag.Parse() + for _, path := range flag.Args() { + sourceYaml, err := os.ReadFile(path) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", path, err) + continue + } + rootNode, err := fetchYaml(sourceYaml) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", path, err) + continue + } + writer, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", path, err) + continue + } + err = streamYaml(writer, indent, rootNode) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", path, err) + continue + } + } +} + +func fetchYaml(sourceYaml []byte) (*yaml.Node, error) { + rootNode := yaml.Node{} + err := yaml.Unmarshal(sourceYaml, &rootNode) + if err != nil { + return nil, err + } + return &rootNode, nil +} + +func streamYaml(writer io.Writer, indent *int, in *yaml.Node) error { + encoder := yaml.NewEncoder(writer) + encoder.SetIndent(*indent) + err := encoder.Encode(in) + if err != nil { + return err + } + return encoder.Close() +} diff --git a/tools/yamlfmt/yamlfmt_test.go b/tools/yamlfmt/yamlfmt_test.go new file mode 100644 index 000000000..0a72e496b --- /dev/null +++ b/tools/yamlfmt/yamlfmt_test.go @@ -0,0 +1,158 @@ +// Copyright © 2023 OpenIM. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "bytes" + "reflect" + "testing" + + "github.com/likexian/gokit/assert" + "gopkg.in/yaml.v3" +) + +func Test_main(t *testing.T) { + sourceYaml := ` # See the OWNERS docs at https://go.k8s.io/owners +approvers: +- dep-approvers +- thockin # Network +- liggitt + +labels: +- sig/architecture +` + + outputYaml := `# See the OWNERS docs at https://go.k8s.io/owners +approvers: + - dep-approvers + - thockin # Network + - liggitt +labels: + - sig/architecture +` + node, _ := fetchYaml([]byte(sourceYaml)) + var output bytes.Buffer + indent := 2 + writer := bufio.NewWriter(&output) + _ = streamYaml(writer, &indent, node) + _ = writer.Flush() + assert.Equal(t, outputYaml, string(output.Bytes()), "yaml was not formatted correctly") +} + +func Test_fetchYaml(t *testing.T) { + type args struct { + sourceYaml []byte + } + tests := []struct { + name string + args args + want *yaml.Node + wantErr bool + }{ + { + name: "Valid YAML", + args: args{sourceYaml: []byte("key: value")}, + want: &yaml.Node{ + Kind: yaml.MappingNode, + Tag: "!!map", + Value: "", + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "key", + }, + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "value", + }, + }, + }, + wantErr: false, + }, + { + name: "Invalid YAML", + args: args{sourceYaml: []byte("key:")}, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := fetchYaml(tt.args.sourceYaml) + if (err != nil) != tt.wantErr { + t.Errorf("fetchYaml() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("fetchYaml() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_streamYaml(t *testing.T) { + type args struct { + indent *int + in *yaml.Node + } + defaultIndent := 2 + tests := []struct { + name string + args args + wantWriter string + wantErr bool + }{ + { + name: "Valid YAML node with default indent", + args: args{ + indent: &defaultIndent, + in: &yaml.Node{ + Kind: yaml.MappingNode, + Tag: "!!map", + Value: "", + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "key", + }, + { + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "value", + }, + }, + }, + }, + wantWriter: "key: value\n", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + writer := &bytes.Buffer{} + if err := streamYaml(writer, tt.args.indent, tt.args.in); (err != nil) != tt.wantErr { + t.Errorf("streamYaml() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotWriter := writer.String(); gotWriter != tt.wantWriter { + t.Errorf("streamYaml() = %v, want %v", gotWriter, tt.wantWriter) + } + }) + } +}