msb-public
/
Open-IM-Server
mirror of https://github.com/OpenIMSDK/Open-IM-Server
2
0
Code Issues Projects Releases Wiki Activity

feat: remote tools imctl

Browse Source
pull/1055/head
Xinwei Xiong(cubxxw) 2 years ago
parent 355223f494
commit 3e10fec755
36 changed files with 2320 additions and 0 deletions
Show Stats Download Patch File Download Diff File

25
tools/README.md
Unescape View File

@ -0,0 +1,25 @@
# Notes about go workspace
As openim is using go1.18's [workspace feature](https://go.dev/doc/tutorial/workspaces), once you add a new module, you need to run `go work use -r .` at root directory to update the workspace synced.
### Create a new extensions
1. Create your tools_name directory in pkg `/tools` first and cd into it.
2. Init the project.
3. Then `go work use -r .` at current directory to update the workspace.
4. Create your tools
You can execute the following commands to do things above:
```bash
# edit the CRD_NAME and CRD_GROUP to your own
export OPENIM_TOOLS_NAME=<Changeme>
# copy and paste to create a new CRD and Controller
mkdir tools/${OPENIM_TOOLS_NAME}
cd tools/${OPENIM_TOOLS_NAME}
go mod init github.com/openimsdk/open-im-server/tools/${OPENIM_TOOLS_NAME}
go mod tidy
go work use -r .
cd ../..
```

308
tools/changelog/changelog.go
Unescape View File

@ -0,0 +1,308 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"log"
"os"
"os/exec"
"regexp"
"sort"
"strings"
)
var (
mergeRequest = regexp.MustCompile(`Merge pull request #([\d]+)`)
webconsoleBump = regexp.MustCompile(regexp.QuoteMeta("bump(github.com/openshift/origin-web-console): ") + `([\w]+)`)
upstreamKube = regexp.MustCompile(`^UPSTREAM: (\d+)+:(.+)`)
upstreamRepo = regexp.MustCompile(`^UPSTREAM: ([\w/-]+): (\d+)+:(.+)`)
prefix = regexp.MustCompile(`^[\w-]: `)
assignments = []prefixAssignment{
{"cluster up", "cluster"},
{" pv ", "storage"},
{"haproxy", "router"},
{"router", "router"},
{"route", "route"},
{"authoriz", "auth"},
{"rbac", "auth"},
{"authent", "auth"},
{"reconcil", "auth"},
{"auth", "auth"},
{"role", "auth"},
{" dc ", "deploy"},
{"deployment", "deploy"},
{"rolling", "deploy"},
{"security context constr", "security"},
{"scc", "security"},
{"pipeline", "build"},
{"build", "build"},
{"registry", "registry"},
{"registries", "image"},
{"image", "image"},
{" arp ", "network"},
{" cni ", "network"},
{"egress", "network"},
{"network", "network"},
{"oc ", "cli"},
{"template", "template"},
{"etcd", "server"},
{"pod", "node"},
{"hack/", "hack"},
{"e2e", "test"},
{"integration", "test"},
{"cluster", "cluster"},
{"master", "server"},
{"packages", "hack"},
{"api", "server"},
}
)
type prefixAssignment struct {
term string
prefix string
}
type commit struct {
short string
parents []string
message string
}
func contains(arr []string, value string) bool {
for _, s := range arr {
if s == value {
return true
}
}
return false
}
func main() {
log.SetFlags(0)
if len(os.Args) != 3 {
log.Fatalf("Must specify two arguments, FROM and TO")
}
from := os.Args[1]
to := os.Args[2]
out, err := exec.Command("git", "log", "--topo-order", "--pretty=tformat:%h %p|%s", "--reverse", fmt.Sprintf("%s..%s", from, to)).CombinedOutput()
if err != nil {
log.Fatal(err)
}
hide := make(map[string]struct{})
var apiChanges []string
var webconsole []string
var commits []commit
var upstreams []commit
var bumps []commit
for _, line := range strings.Split(string(out), "\n") {
if len(strings.TrimSpace(line)) == 0 {
continue
}
parts := strings.SplitN(line, "|", 2)
hashes := strings.Split(parts[0], " ")
c := commit{short: hashes[0], parents: hashes[1:], message: parts[1]}
if strings.HasPrefix(c.message, "UPSTREAM: ") {
hide[c.short] = struct{}{}
upstreams = append(upstreams, c)
}
if strings.HasPrefix(c.message, "bump(") {
hide[c.short] = struct{}{}
bumps = append(bumps, c)
}
if len(c.parents) == 1 {
commits = append(commits, c)
continue
}
matches := mergeRequest.FindStringSubmatch(line)
if len(matches) == 0 {
// this may have been a human pressing the merge button, we'll just record this as a direct push
continue
}
// split the accumulated commits into any that are force merges (assumed to be the initial set due
// to --topo-order) from the PR commits as soon as we see any of our merge parents. Then print
// any of the force merges
var first int
for i := range commits {
first = i
if contains(c.parents, commits[i].short) {
first++
break
}
}
individual := commits[:first]
merged := commits[first:]
for _, commit := range individual {
if len(commit.parents) > 1 {
continue
}
if _, ok := hide[commit.short]; ok {
continue
}
fmt.Printf("force-merge: %s %s\n", commit.message, commit.short)
}
// try to find either the PR title or the first commit title from the merge commit
out, err := exec.Command("git", "show", "--pretty=tformat:%b", c.short).CombinedOutput()
if err != nil {
log.Fatal(err)
}
var message string
para := strings.Split(string(out), "\n\n")
if len(para) > 0 && strings.HasPrefix(para[0], "Automatic merge from submit-queue") {
para = para[1:]
}
// this is no longer necessary with the submit queue in place
if len(para) > 0 && strings.HasPrefix(para[0], "Merged by ") {
para = para[1:]
}
// post submit-queue, the merge bot will add the PR title, which is usually pretty good
if len(para) > 0 {
message = strings.Split(para[0], "\n")[0]
}
if len(message) == 0 && len(merged) > 0 {
message = merged[0].message
}
if len(message) > 0 && len(merged) == 1 && message == merged[0].message {
merged = nil
}
// try to calculate a prefix based on the diff
if len(message) > 0 && !prefix.MatchString(message) {
prefix, ok := findPrefixFor(message, merged)
if ok {
message = prefix + ": " + message
}
}
// github merge
// has api changes
display := fmt.Sprintf("%s [\\#%s](https://github.com/openimsdk/Open-IM-Server/pull/%s)", message, matches[1], matches[1])
if hasFileChanges(c.short, "pkg/apistruct/") {
apiChanges = append(apiChanges, display)
}
var filtered []commit
for _, commit := range merged {
if _, ok := hide[commit.short]; ok {
continue
}
filtered = append(filtered, commit)
}
if len(filtered) > 0 {
fmt.Printf("- %s\n", display)
for _, commit := range filtered {
fmt.Printf(" - %s (%s)\n", commit.message, commit.short)
}
}
// stick the merge commit in at the beginning of the next list so we can anchor the previous parent
commits = []commit{c}
}
// chunk the bumps
var lines []string
for _, commit := range bumps {
if m := webconsoleBump.FindStringSubmatch(commit.message); len(m) > 0 {
webconsole = append(webconsole, m[1])
continue
}
lines = append(lines, commit.message)
}
lines = sortAndUniq(lines)
for _, line := range lines {
fmt.Printf("- %s\n", line)
}
// chunk the upstreams
lines = nil
for _, commit := range upstreams {
lines = append(lines, commit.message)
}
lines = sortAndUniq(lines)
for _, line := range lines {
fmt.Printf("- %s\n", upstreamLinkify(line))
}
if len(webconsole) > 0 {
fmt.Printf("- web: from %s^..%s\n", webconsole[0], webconsole[len(webconsole)-1])
}
for _, apiChange := range apiChanges {
fmt.Printf(" - %s\n", apiChange)
}
}
func findPrefixFor(message string, commits []commit) (string, bool) {
message = strings.ToLower(message)
for _, m := range assignments {
if strings.Contains(message, m.term) {
return m.prefix, true
}
}
for _, c := range commits {
if prefix, ok := findPrefixFor(c.message, nil); ok {
return prefix, ok
}
}
return "", false
}
func hasFileChanges(commit string, prefixes ...string) bool {
out, err := exec.Command("git", "diff", "--name-only", fmt.Sprintf("%s^..%s", commit, commit)).CombinedOutput()
if err != nil {
log.Fatal(err)
}
for _, file := range strings.Split(string(out), "\n") {
for _, prefix := range prefixes {
if strings.HasPrefix(file, prefix) {
return true
}
}
}
return false
}
func sortAndUniq(lines []string) []string {
sort.Strings(lines)
out := make([]string, 0, len(lines))
last := ""
for _, s := range lines {
if last == s {
continue
}
last = s
out = append(out, s)
}
return out
}
func upstreamLinkify(line string) string {
if m := upstreamKube.FindStringSubmatch(line); len(m) > 0 {
return fmt.Sprintf("UPSTREAM: [#%s](https://github.com/openimsdk/open-im-server/pull/%s):%s", m[1], m[1], m[2])
}
if m := upstreamRepo.FindStringSubmatch(line); len(m) > 0 {
return fmt.Sprintf("UPSTREAM: [%s#%s](https://github.com/%s/pull/%s):%s", m[1], m[2], m[1], m[2], m[3])
}
return line
}

3
tools/changelog/go.mod
Unescape View File

@ -0,0 +1,3 @@
module github.com/openimsdk/open-im-server/v3/tools/changelog
go 1.18

311
tools/component/component.go
Unescape View File

@ -0,0 +1,311 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"database/sql"
"flag"
"fmt"
"net"
"net/url"
"os"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/redis/go-redis/v9"
"gopkg.in/yaml.v3"
"github.com/IBM/sarama"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/utils"
"github.com/go-zookeeper/zk"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readpref"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/kafka"
"github.com/minio/minio-go/v7/pkg/credentials"
)
const (
// defaultCfgPath is the default path of the configuration file
defaultCfgPath = "../../../../../config/config.yaml"
minioHealthCheckDuration = 1
maxRetry = 100
componentStartErrCode = 6000
configErrCode = 6001
)
var (
cfgPath = flag.String("c", defaultCfgPath, "Path to the configuration file")
ErrComponentStart = errs.NewCodeError(componentStartErrCode, "ComponentStartErr")
ErrConfig = errs.NewCodeError(configErrCode, "Config file is incorrect")
)
func initCfg() error {
data, err := os.ReadFile(*cfgPath)
if err != nil {
return err
}
return yaml.Unmarshal(data, &config.Config)
}
type checkFunc struct {
name string
function func() error
}
func main() {
flag.Parse()
if err := initCfg(); err != nil {
fmt.Printf("Read config failed: %v\n", err)
return
}
checks := []checkFunc{
{name: "Mysql", function: checkMysql},
{name: "Mongo", function: checkMongo},
{name: "Minio", function: checkMinio},
{name: "Redis", function: checkRedis},
{name: "Zookeeper", function: checkZookeeper},
{name: "Kafka", function: checkKafka},
}
for i := 0; i < maxRetry; i++ {
if i != 0 {
time.Sleep(3 * time.Second)
}
fmt.Printf("Checking components Round %v...\n", i+1)
allSuccess := true
for _, check := range checks {
err := check.function()
if err != nil {
errorPrint(fmt.Sprintf("Starting %s failed: %v", check.name, err))
allSuccess = false
break
} else {
successPrint(fmt.Sprintf("%s starts successfully", check.name))
}
}
if allSuccess {
successPrint("All components started successfully!")
return
}
}
os.Exit(1)
}
func exactIP(urll string) string {
u, _ := url.Parse(urll)
host, _, err := net.SplitHostPort(u.Host)
if err != nil {
host = u.Host
}
if strings.HasSuffix(host, ":") {
host = host[0 : len(host)-1]
}
return host
}
func checkMysql() error {
var sqlDB *sql.DB
defer func() {
if sqlDB != nil {
sqlDB.Close()
}
}()
dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=true&loc=Local",
config.Config.Mysql.Username, config.Config.Mysql.Password, config.Config.Mysql.Address[0], "mysql")
db, err := gorm.Open(mysql.Open(dsn), nil)
if err != nil {
return errs.Wrap(err)
} else {
sqlDB, err = db.DB()
err = sqlDB.Ping()
if err != nil {
return errs.Wrap(err)
}
}
return nil
}
func checkMongo() error {
var client *mongo.Client
defer func() {
if client != nil {
client.Disconnect(context.TODO())
}
}()
mongodbHosts := ""
for i, v := range config.Config.Mongo.Address {
if i == len(config.Config.Mongo.Address)-1 {
mongodbHosts += v
} else {
mongodbHosts += v + ","
}
}
client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(
fmt.Sprintf("mongodb://%v:%v@%v/?authSource=admin",
config.Config.Mongo.Username, config.Config.Mongo.Password, mongodbHosts)))
if err != nil {
return errs.Wrap(err)
} else {
err = client.Ping(context.TODO(), &readpref.ReadPref{})
if err != nil {
return errs.Wrap(err)
}
}
return nil
}
func checkMinio() error {
if config.Config.Object.Enable == "minio" {
conf := config.Config.Object.Minio
u, _ := url.Parse(conf.Endpoint)
minioClient, err := minio.New(u.Host, &minio.Options{
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, ""),
Secure: u.Scheme == "https",
})
if err != nil {
return errs.Wrap(err)
}
cancel, err := minioClient.HealthCheck(time.Duration(minioHealthCheckDuration) * time.Second)
defer func() {
if cancel != nil {
cancel()
}
}()
if err != nil {
return errs.Wrap(err)
} else {
if minioClient.IsOffline() {
return ErrComponentStart.Wrap("Minio server is offline")
}
}
if exactIP(config.Config.Object.ApiURL) == "127.0.0.1" || exactIP(config.Config.Object.Minio.SignEndpoint) == "127.0.0.1" {
return ErrConfig.Wrap("apiURL or Minio SignEndpoint endpoint contain 127.0.0.1")
}
}
return nil
}
func checkRedis() error {
var redisClient redis.UniversalClient
defer func() {
if redisClient != nil {
redisClient.Close()
}
}()
if len(config.Config.Redis.Address) > 1 {
redisClient = redis.NewClusterClient(&redis.ClusterOptions{
Addrs: config.Config.Redis.Address,
Username: config.Config.Redis.Username,
Password: config.Config.Redis.Password,
})
} else {
redisClient = redis.NewClient(&redis.Options{
Addr: config.Config.Redis.Address[0],
Username: config.Config.Redis.Username,
Password: config.Config.Redis.Password,
})
}
_, err := redisClient.Ping(context.Background()).Result()
if err != nil {
return errs.Wrap(err)
}
return nil
}
func checkZookeeper() error {
var c *zk.Conn
defer func() {
if c != nil {
c.Close()
}
}()
c, _, err := zk.Connect(config.Config.Zookeeper.ZkAddr, time.Second)
if err != nil {
return errs.Wrap(err)
} else {
if config.Config.Zookeeper.Username != "" && config.Config.Zookeeper.Password != "" {
if err := c.AddAuth("digest", []byte(config.Config.Zookeeper.Username+":"+config.Config.Zookeeper.Password)); err != nil {
return errs.Wrap(err)
}
}
_, _, err = c.Get("/")
if err != nil {
return errs.Wrap(err)
}
}
return nil
}
func checkKafka() error {
var kafkaClient sarama.Client
defer func() {
if kafkaClient != nil {
kafkaClient.Close()
}
}()
cfg := sarama.NewConfig()
if config.Config.Kafka.Username != "" && config.Config.Kafka.Password != "" {
cfg.Net.SASL.Enable = true
cfg.Net.SASL.User = config.Config.Kafka.Username
cfg.Net.SASL.Password = config.Config.Kafka.Password
}
kafka.SetupTLSConfig(cfg)
kafkaClient, err := sarama.NewClient(config.Config.Kafka.Addr, cfg)
if err != nil {
return errs.Wrap(err)
} else {
topics, err := kafkaClient.Topics()
if err != nil {
return err
}
if !utils.IsContain(config.Config.Kafka.MsgToMongo.Topic, topics) {
return ErrComponentStart.Wrap(fmt.Sprintf("kafka doesn't contain topic:%v", config.Config.Kafka.MsgToMongo.Topic))
}
if !utils.IsContain(config.Config.Kafka.MsgToPush.Topic, topics) {
return ErrComponentStart.Wrap(fmt.Sprintf("kafka doesn't contain topic:%v", config.Config.Kafka.MsgToPush.Topic))
}
if !utils.IsContain(config.Config.Kafka.LatestMsgToRedis.Topic, topics) {
return ErrComponentStart.Wrap(fmt.Sprintf("kafka doesn't contain topic:%v", config.Config.Kafka.LatestMsgToRedis.Topic))
}
}
return nil
}
func errorPrint(s string) {
fmt.Printf("\x1b[%dm%v\x1b[0m\n", 31, s)
}
func successPrint(s string) {
fmt.Printf("\x1b[%dm%v\x1b[0m\n", 32, s)
}
func warningPrint(s string) {
fmt.Printf("\x1b[%dmWarning: But %v\x1b[0m\n", 33, s)
}

42
tools/component/component_test.go
Unescape View File

@ -0,0 +1,42 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
)
func TestCheckMysql(t *testing.T) {
err := mockInitCfg()
assert.NoError(t, err, "Initialization should not produce errors")
err = checkMysql()
if err != nil {
// You might expect an error if MySQL isn't running locally with the mock credentials.
t.Logf("Expected error due to mock configuration: %v", err)
}
}
// Mock for initCfg for testing purpose
func mockInitCfg() error {
config.Config.Mysql.Username = "root"
config.Config.Mysql.Password = "openIM123"
config.Config.Mysql.Address = []string{"127.0.0.1:13306"}
return nil
}

3
tools/component/go.mod
Unescape View File

@ -0,0 +1,3 @@
module github.com/openimsdk/open-im-server/v3/tools/component
go 1.18

3
tools/infra/go.mod
Unescape View File

@ -0,0 +1,3 @@
module github.com/openimsdk/open-im-server/v3/tools/infra
go 1.18

40
tools/infra/infra.go
Unescape View File

@ -0,0 +1,40 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"log"
)
func main() {
log.Println("Current module is still under development.")
message := `
Current module is still under development.
____ _____ __ __
/ __ \ |_ _|| \/ |
| | | | _ __ ___ _ __ | | | \ / |
| | | || '_ \ / _ \| '_ \ | | | |\/| |
| |__| || |_) || __/| | | | _| |_ | | | |
\____/ | .__/ \___||_| |_||_____||_| |_|
| |
|_|
Keep checking for updates!
`
fmt.Println(message)
}

39
tools/ncpu/README.md
Unescape View File

@ -0,0 +1,39 @@
# ncpu
**ncpu** is a simple utility to fetch the number of CPU cores across different operating systems.
## Introduction
In various scenarios, especially while compiling code, it's beneficial to know the number of available CPU cores to optimize the build process. However, the command to fetch the CPU core count differs between operating systems. For example, on Linux, we use `nproc`, while on macOS, it's `sysctl -n hw.ncpu`. The `ncpu` utility provides a unified way to obtain this number, regardless of the platform.
## Usage
To retrieve the number of CPU cores, simply use the `ncpu` command:
```bash
$ ncpu
```
This will return an integer representing the number of available CPU cores.
### Example:
Let's say you're compiling a project using `make`. To utilize all the CPU cores for the compilation process, you can use:
```bash
$ make -j $(ncpu) build # or any other build command
```
The above command will ensure the build process takes advantage of all the available CPU cores, thereby potentially speeding up the compilation.
## Why use `ncpu`?
- **Cross-platform compatibility**: No need to remember or detect which OS-specific command to use. Just use `ncpu`!
- **Ease of use**: A simple and intuitive command that's easy to incorporate into scripts or command-line operations.
- **Consistency**: Ensures consistent behavior and output across different systems and environments.
## Installation
(Include installation steps here, e.g., how to clone the repo, build the tool, or install via package manager.)

5
tools/ncpu/go.mod
Unescape View File

@ -0,0 +1,5 @@
module github.com/openimsdk/open-im-server/v3/tools/ncpu
go 1.18
require go.uber.org/automaxprocs v1.5.3

7
tools/ncpu/go.sum
Unescape View File

@ -0,0 +1,7 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8=
go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

27
tools/ncpu/ncpu.go
Unescape View File

@ -0,0 +1,27 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"runtime"
"go.uber.org/automaxprocs/maxprocs"
)
func main() {
maxprocs.Set()
fmt.Print(runtime.GOMAXPROCS(0))
}

35
tools/ncpu/ncpu_test.go
Unescape View File

@ -0,0 +1,35 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import "testing"
func Test_main(t *testing.T) {
tests := []struct {
name string
}{
{
name: "Test_main",
},
{
name: "Test_main2",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
main()
})
}
}

47
tools/openim-web/Dockerfile
Unescape View File

@ -0,0 +1,47 @@
# Copyright © 2023 OpenIM. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# 使用官方Go镜像作为基础镜像
FROM golang:1.21 AS build-env
ENV CGO_ENABLED=0
# 设置工作目录
WORKDIR /app
# 安装curl和unzip工具
#RUN apt-get update && apt-get install -y curl unzip
# 从GitHub下载并解压dist.zip
#RUN curl -LO https://github.com/OpenIMSDK/dist.zip \
# && unzip dist.zip -d ./ \
# && rm dist.zip
# 复制Go代码到容器
COPY . .
# 编译Go代码
RUN go build -o openim-web
# 使用轻量级的基础镜像
FROM debian:buster-slim
# 将编译好的二进制文件和dist资源复制到新的容器
WORKDIR /app
COPY --from=build-env /app/openim-web /app/openim-web
COPY --from=build-env /app/dist /app/dist
# 开放容器的20001端口
EXPOSE 20001
# 指定容器启动命令
ENTRYPOINT ["/app/openim-web"]

79
tools/openim-web/README.md
Unescape View File

@ -0,0 +1,79 @@
# OpenIM Web Service
- [OpenIM Web Service](#openim-web-service)
- [Overview](#overview)
- [User](#user)
- [Docker Deployment](#docker-deployment)
- [Build the Docker Image](#build-the-docker-image)
- [Run the Docker Container](#run-the-docker-container)
- [Configuration](#configuration)
- [Contributions](#contributions)
OpenIM Web Service is a lightweight containerized service built with Go. The service serves static files and allows customization via environment variables.
## Overview
- Built using Go.
- Deployed as a Docker container.
- Serves static files from a directory which can be set via an environment variable.
- The default port for the service is `20001`, but it can be customized using an environment variable.
## User
example：
```bash
$ ./openim-web -h
Usage of ./openim-web:
-distPath string
Path to the distribution (default "/app/dist")
-port string
Port to run the server on (default "20001")
```
Variables can be set as above, Environment variables can also be set
example:
```bash
$ export OPENIM_WEB_DIST_PATH="/app/dist"
$ export OPENIM_WEB_PPRT="11001"
```
Initialize the env configuration file:
```bash
$ make init
```
## Docker Deployment
### Build the Docker Image
Even though we've implemented automation, it's to make the developer experience easier:
To build the Docker image for OpenIM Web Service:
```bash
$ docker build -t openim-web .
```
### Run the Docker Container
To run the service:
```bash
$ docker run -e DIST_PATH=/app/dist -e PORT=20001 -p 20001:20001 openim-web
```
## Configuration
You can configure the OpenIM Web Service using the following environment variables:
- **DIST_PATH**: The path to the directory containing the static files. Default: `/app/dist`.
- **PORT**: The port on which the service listens. Default: `11001`.
## Contributions
We welcome contributions from the community. If you find any bugs or have feature suggestions, please create an issue or send a pull request.

7
tools/openim-web/go.mod
Unescape View File

@ -0,0 +1,7 @@
module github.com/openimsdk/open-im-server/v3/tools/openim-web
go 1.18
require gopkg.in/yaml.v2 v2.4.0
require github.com/NYTimes/gziphandler v1.1.1 // indirect

10
tools/openim-web/go.sum
Unescape View File

@ -0,0 +1,10 @@
github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

63
tools/openim-web/openim-web.go
Unescape View File

@ -0,0 +1,63 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"flag"
"log"
"net/http"
"os"
"github.com/NYTimes/gziphandler"
)
var (
distPathFlag string
portFlag string
)
func init() {
flag.StringVar(&distPathFlag, "distPath", "/app/dist", "Path to the distribution")
flag.StringVar(&portFlag, "port", "11001", "Port to run the server on")
}
func main() {
flag.Parse()
distPath := getConfigValue("DIST_PATH", distPathFlag, "/app/dist")
fs := http.FileServer(http.Dir(distPath))
withGzip := gziphandler.GzipHandler(fs)
http.Handle("/", withGzip)
port := getConfigValue("PORT", portFlag, "11001")
log.Printf("Server listening on port %s in %s...", port, distPath)
err := http.ListenAndServe(":"+port, nil)
if err != nil {
log.Fatal(err)
}
}
func getConfigValue(envKey, flagValue, fallback string) string {
envVal := os.Getenv(envKey)
if envVal != "" {
return envVal
}
if flagValue != "" {
return flagValue
}
return fallback
}

71
tools/openim-web/openim-web_test.go
Unescape View File

@ -0,0 +1,71 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"os"
"testing"
)
func TestGetConfigValue(t *testing.T) {
tests := []struct {
name string
envKey string
envValue string
flagValue string
fallback string
wantResult string
}{
{
name: "environment variable set",
envKey: "TEST_KEY",
envValue: "envValue",
flagValue: "",
fallback: "default",
wantResult: "envValue",
},
{
name: "flag set and environment variable not set",
envKey: "TEST_KEY",
envValue: "",
flagValue: "flagValue",
fallback: "default",
wantResult: "flagValue",
},
{
name: "nothing set, use fallback",
envKey: "TEST_KEY",
envValue: "",
flagValue: "",
fallback: "default",
wantResult: "default",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.envValue != "" {
os.Setenv(tt.envKey, tt.envValue)
defer os.Unsetenv(tt.envKey)
}
got := getConfigValue(tt.envKey, tt.flagValue, tt.fallback)
if got != tt.wantResult {
t.Errorf("getConfigValue(%s, %s, %s) = %s; want %s", tt.envKey, tt.flagValue, tt.fallback, got, tt.wantResult)
}
})
}
}

20
tools/url2im/go.mod
Unescape View File

@ -0,0 +1,20 @@
module github.com/openimsdk/open-im-server/v3/tools/url2im
go 1.20
require (
github.com/OpenIMSDK/protocol v0.0.21
github.com/kelindar/bitmap v1.5.1
)
require (
github.com/golang/protobuf v1.5.3 // indirect
github.com/kelindar/simd v1.1.2 // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/text v0.9.0 // indirect
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
google.golang.org/grpc v1.56.2 // indirect
google.golang.org/protobuf v1.31.0 // indirect
)

33
tools/url2im/go.sum
Unescape View File

@ -0,0 +1,33 @@
github.com/OpenIMSDK/protocol v0.0.21 h1:5H6H+hJ9d/VgRqttvxD/zfK9Asd+4M8Eknk5swSbUVY=
github.com/OpenIMSDK/protocol v0.0.21/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/kelindar/bitmap v1.5.1 h1:+ZmZdwHbJ+CGE+q/aAJ74KJSnp0vOlGD7KY5x51mVzk=
github.com/kelindar/bitmap v1.5.1/go.mod h1:j3qZjxH9s4OtvsnFTP2bmPkjqil9Y2xQlxPYHexasEA=
github.com/kelindar/simd v1.1.2 h1:KduKb+M9cMY2HIH8S/cdJyD+5n5EGgq+Aeeleos55To=
github.com/kelindar/simd v1.1.2/go.mod h1:inq4DFudC7W8L5fhxoeZflLRNpWSs0GNx6MlWFvuvr0=
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A=
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

84
tools/url2im/main.go
Unescape View File

@ -0,0 +1,84 @@
package main
import (
"flag"
"log"
"os"
"path/filepath"
"time"
"github.com/openimsdk/open-im-server/v3/tools/url2im/pkg"
)
/*take.txt
{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"}
{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"}
{"url":"http://xxx/xxxx","name":"xxxx","contentType":"image/jpeg"}
*/
func main() {
var conf pkg.Config // 后面带*的为必填项
flag.StringVar(&conf.TaskPath, "task", "take.txt", "task path") // 任务日志文件*
flag.StringVar(&conf.ProgressPath, "progress", "", "progress path") // 进度日志文件
flag.IntVar(&conf.Concurrency, "concurrency", 1, "concurrency num") // 并发数
flag.IntVar(&conf.Retry, "retry", 1, "retry num") // 重试次数
flag.StringVar(&conf.TempDir, "temp", "", "temp dir") // 临时文件夹
flag.Int64Var(&conf.CacheSize, "cache", 1024*1024*100, "cache size") // 缓存大小(超过时,下载到磁盘)
flag.Int64Var((*int64)(&conf.Timeout), "timeout", 5000, "timeout") // 请求超时时间(毫秒)
flag.StringVar(&conf.Api, "api", "http://127.0.0.1:10002", "api") // im地址*
flag.StringVar(&conf.UserID, "userID", "openIM123456", "userID") // im管理员
flag.StringVar(&conf.Secret, "secret", "openIM123", "secret") // im config secret
flag.Parse()
if !filepath.IsAbs(conf.TaskPath) {
var err error
conf.TaskPath, err = filepath.Abs(conf.TaskPath)
if err != nil {
log.Println("get abs path err:", err)
return
}
}
if conf.ProgressPath == "" {
conf.ProgressPath = conf.TaskPath + ".progress.txt"
} else if !filepath.IsAbs(conf.ProgressPath) {
var err error
conf.ProgressPath, err = filepath.Abs(conf.ProgressPath)
if err != nil {
log.Println("get abs path err:", err)
return
}
}
if conf.TempDir == "" {
conf.TempDir = conf.TaskPath + ".temp"
}
if info, err := os.Stat(conf.TempDir); err == nil {
if !info.IsDir() {
log.Printf("temp dir %s is not dir\n", err)
return
}
} else if os.IsNotExist(err) {
if err := os.MkdirAll(conf.TempDir, os.ModePerm); err != nil {
log.Printf("mkdir temp dir %s err %+v\n", conf.TempDir, err)
return
}
defer os.RemoveAll(conf.TempDir)
} else {
log.Println("get temp dir err:", err)
return
}
if conf.Concurrency <= 0 {
conf.Concurrency = 1
}
if conf.Retry <= 0 {
conf.Retry = 1
}
if conf.CacheSize <= 0 {
conf.CacheSize = 1024 * 1024 * 100 // 100M
}
if conf.Timeout <= 0 {
conf.Timeout = 5000
}
conf.Timeout = conf.Timeout * time.Millisecond
if err := pkg.Run(conf); err != nil {
log.Println("main err:", err)
}
}

112
tools/url2im/pkg/api.go
Unescape View File

@ -0,0 +1,112 @@
package pkg
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"github.com/OpenIMSDK/protocol/auth"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/protocol/third"
)
type Api struct {
Api string
UserID string
Secret string
Token string
Client *http.Client
}
func (a *Api) apiPost(ctx context.Context, path string, req any, resp any) error {
operationID, _ := ctx.Value("operationID").(string)
if operationID == "" {
return errors.New("call api operationID is empty")
}
reqBody, err := json.Marshal(req)
if err != nil {
return err
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost, a.Api+path, bytes.NewReader(reqBody))
if err != nil {
return err
}
DefaultRequestHeader(request.Header)
request.ContentLength = int64(len(reqBody))
request.Header.Set("Content-Type", "application/json")
request.Header.Set("operationID", operationID)
if a.Token != "" {
request.Header.Set("token", a.Token)
}
response, err := a.Client.Do(request)
if err != nil {
return err
}
defer response.Body.Close()
body, err := io.ReadAll(response.Body)
if err != nil {
return err
}
if response.StatusCode != http.StatusOK {
return fmt.Errorf("api %s status %s body %s", path, response.Status, body)
}
var baseResponse struct {
ErrCode int `json:"errCode"`
ErrMsg string `json:"errMsg"`
ErrDlt string `json:"errDlt"`
Data json.RawMessage `json:"data"`
}
if err := json.Unmarshal(body, &baseResponse); err != nil {
return err
}
if baseResponse.ErrCode != 0 {
return fmt.Errorf("api %s errCode %d errMsg %s errDlt %s", path, baseResponse.ErrCode, baseResponse.ErrMsg, baseResponse.ErrDlt)
}
if resp != nil {
if err := json.Unmarshal(baseResponse.Data, resp); err != nil {
return err
}
}
return nil
}
func (a *Api) GetToken(ctx context.Context) (string, error) {
req := auth.UserTokenReq{
UserID: a.UserID,
Secret: a.Secret,
PlatformID: constant.AdminPlatformID,
}
var resp auth.UserTokenResp
if err := a.apiPost(ctx, "/auth/user_token", &req, &resp); err != nil {
return "", err
}
return resp.Token, nil
}
func (a *Api) GetPartLimit(ctx context.Context) (*third.PartLimitResp, error) {
var resp third.PartLimitResp
if err := a.apiPost(ctx, "/object/part_limit", &third.PartLimitReq{}, &resp); err != nil {
return nil, err
}
return &resp, nil
}
func (a *Api) InitiateMultipartUpload(ctx context.Context, req *third.InitiateMultipartUploadReq) (*third.InitiateMultipartUploadResp, error) {
var resp third.InitiateMultipartUploadResp
if err := a.apiPost(ctx, "/object/initiate_multipart_upload", req, &resp); err != nil {
return nil, err
}
return &resp, nil
}
func (a *Api) CompleteMultipartUpload(ctx context.Context, req *third.CompleteMultipartUploadReq) (string, error) {
var resp third.CompleteMultipartUploadResp
if err := a.apiPost(ctx, "/object/complete_multipart_upload", req, &resp); err != nil {
return "", err
}
return resp.Url, nil
}

96
tools/url2im/pkg/buffer.go
Unescape View File

@ -0,0 +1,96 @@
package pkg
import (
"bytes"
"io"
"os"
)
type ReadSeekSizeCloser interface {
io.ReadSeekCloser
Size() int64
}
func NewReader(r io.Reader, max int64, path string) (ReadSeekSizeCloser, error) {
buf := make([]byte, max+1)
n, err := io.ReadFull(r, buf)
if err == nil {
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0666)
if err != nil {
return nil, err
}
var ok bool
defer func() {
if !ok {
_ = f.Close()
_ = os.Remove(path)
}
}()
if _, err := f.Write(buf[:n]); err != nil {
return nil, err
}
cn, err := io.Copy(f, r)
if err != nil {
return nil, err
}
if _, err := f.Seek(0, io.SeekStart); err != nil {
return nil, err
}
ok = true
return &fileBuffer{
f: f,
n: cn + int64(n),
}, nil
} else if err == io.EOF || err == io.ErrUnexpectedEOF {
return &memoryBuffer{
r: bytes.NewReader(buf[:n]),
}, nil
} else {
return nil, err
}
}
type fileBuffer struct {
n int64
f *os.File
}
func (r *fileBuffer) Read(p []byte) (n int, err error) {
return r.f.Read(p)
}
func (r *fileBuffer) Seek(offset int64, whence int) (int64, error) {
return r.f.Seek(offset, whence)
}
func (r *fileBuffer) Size() int64 {
return r.n
}
func (r *fileBuffer) Close() error {
name := r.f.Name()
if err := r.f.Close(); err != nil {
return err
}
return os.Remove(name)
}
type memoryBuffer struct {
r *bytes.Reader
}
func (r *memoryBuffer) Read(p []byte) (n int, err error) {
return r.r.Read(p)
}
func (r *memoryBuffer) Seek(offset int64, whence int) (int64, error) {
return r.r.Seek(offset, whence)
}
func (r *memoryBuffer) Close() error {
return nil
}
func (r *memoryBuffer) Size() int64 {
return r.r.Size()
}

16
tools/url2im/pkg/config.go
Unescape View File

@ -0,0 +1,16 @@
package pkg
import "time"
type Config struct {
TaskPath string
ProgressPath string
Concurrency int
Retry int
Timeout time.Duration
Api string
UserID string
Secret string
TempDir string
CacheSize int64
}

7
tools/url2im/pkg/http.go
Unescape View File

@ -0,0 +1,7 @@
package pkg
import "net/http"
func DefaultRequestHeader(header http.Header) {
header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36")
}

385
tools/url2im/pkg/manage.go
Unescape View File

@ -0,0 +1,385 @@
package pkg
import (
"bufio"
"context"
"crypto/md5"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/OpenIMSDK/protocol/third"
)
type Upload struct {
URL string `json:"url"`
Name string `json:"name"`
ContentType string `json:"contentType"`
}
type Task struct {
Index int
Upload Upload
}
type PartInfo struct {
ContentType string
PartSize int64
PartNum int
FileMd5 string
PartMd5 string
PartSizes []int64
PartMd5s []string
}
func Run(conf Config) error {
m := &Manage{
prefix: time.Now().Format("20060102150405"),
conf: &conf,
ctx: context.Background(),
}
return m.Run()
}
type Manage struct {
conf *Config
ctx context.Context
api *Api
partLimit *third.PartLimitResp
prefix string
tasks chan Task
id uint64
success int64
failed int64
}
func (m *Manage) tempFilePath() string {
return filepath.Join(m.conf.TempDir, fmt.Sprintf("%s_%d", m.prefix, atomic.AddUint64(&m.id, 1)))
}
func (m *Manage) Run() error {
defer func(start time.Time) {
log.Printf("run time %s\n", time.Since(start))
}(time.Now())
m.api = &Api{
Api: m.conf.Api,
UserID: m.conf.UserID,
Secret: m.conf.Secret,
Client: &http.Client{Timeout: m.conf.Timeout},
}
var err error
ctx := context.WithValue(m.ctx, "operationID", fmt.Sprintf("%s_init", m.prefix))
m.api.Token, err = m.api.GetToken(ctx)
if err != nil {
return err
}
m.partLimit, err = m.api.GetPartLimit(ctx)
if err != nil {
return err
}
progress, err := ReadProgress(m.conf.ProgressPath)
if err != nil {
return err
}
progressFile, err := os.OpenFile(m.conf.ProgressPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
if err != nil {
return err
}
var mutex sync.Mutex
writeSuccessIndex := func(index int) {
mutex.Lock()
defer mutex.Unlock()
if _, err := progressFile.Write([]byte(strconv.Itoa(index) + "\n")); err != nil {
log.Printf("write progress err: %v\n", err)
}
}
file, err := os.Open(m.conf.TaskPath)
if err != nil {
return err
}
m.tasks = make(chan Task, m.conf.Concurrency*2)
go func() {
defer file.Close()
defer close(m.tasks)
scanner := bufio.NewScanner(file)
var (
index int
num int
)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
index++
if progress.IsUploaded(index) {
log.Printf("index: %d already uploaded %s\n", index, line)
continue
}
var upload Upload
if err := json.Unmarshal([]byte(line), &upload); err != nil {
log.Printf("index: %d json.Unmarshal(%s) err: %v", index, line, err)
continue
}
num++
m.tasks <- Task{
Index: index,
Upload: upload,
}
}
if num == 0 {
log.Println("mark all completed")
}
}()
var wg sync.WaitGroup
wg.Add(m.conf.Concurrency)
for i := 0; i < m.conf.Concurrency; i++ {
go func(tid int) {
defer wg.Done()
for task := range m.tasks {
var success bool
for n := 0; n < m.conf.Retry; n++ {
ctx := context.WithValue(m.ctx, "operationID", fmt.Sprintf("%s_%d_%d_%d", m.prefix, tid, task.Index, n+1))
if urlRaw, err := m.RunTask(ctx, task); err == nil {
writeSuccessIndex(task.Index)
log.Println("index:", task.Index, "upload success", "urlRaw", urlRaw)
success = true
break
} else {
log.Printf("index: %d upload: %+v err: %v", task.Index, task.Upload, err)
}
}
if success {
atomic.AddInt64(&m.success, 1)
} else {
atomic.AddInt64(&m.failed, 1)
log.Printf("index: %d upload: %+v failed", task.Index, task.Upload)
}
}
}(i + 1)
}
wg.Wait()
log.Printf("execution completed success %d failed %d\n", m.success, m.failed)
return nil
}
func (m *Manage) RunTask(ctx context.Context, task Task) (string, error) {
resp, err := m.HttpGet(ctx, task.Upload.URL)
if err != nil {
return "", err
}
defer resp.Body.Close()
reader, err := NewReader(resp.Body, m.conf.CacheSize, m.tempFilePath())
if err != nil {
return "", err
}
defer reader.Close()
part, err := m.getPartInfo(ctx, reader, reader.Size())
if err != nil {
return "", err
}
var contentType string
if task.Upload.ContentType == "" {
contentType = part.ContentType
} else {
contentType = task.Upload.ContentType
}
initiateMultipartUploadResp, err := m.api.InitiateMultipartUpload(ctx, &third.InitiateMultipartUploadReq{
Hash: part.PartMd5,
Size: reader.Size(),
PartSize: part.PartSize,
MaxParts: -1,
Cause: "batch-import",
Name: task.Upload.Name,
ContentType: contentType,
})
if err != nil {
return "", err
}
if initiateMultipartUploadResp.Upload == nil {
return initiateMultipartUploadResp.Url, nil
}
if _, err := reader.Seek(0, io.SeekStart); err != nil {
return "", err
}
uploadParts := make([]*third.SignPart, part.PartNum)
for _, part := range initiateMultipartUploadResp.Upload.Sign.Parts {
uploadParts[part.PartNumber-1] = part
}
for i, currentPartSize := range part.PartSizes {
md5Reader := NewMd5Reader(io.LimitReader(reader, currentPartSize))
if m.doPut(ctx, m.api.Client, initiateMultipartUploadResp.Upload.Sign, uploadParts[i], md5Reader, currentPartSize); err != nil {
return "", err
}
if md5val := md5Reader.Md5(); md5val != part.PartMd5s[i] {
return "", fmt.Errorf("upload part %d failed, md5 not match, expect %s, got %s", i, part.PartMd5s[i], md5val)
}
}
urlRaw, err := m.api.CompleteMultipartUpload(ctx, &third.CompleteMultipartUploadReq{
UploadID: initiateMultipartUploadResp.Upload.UploadID,
Parts: part.PartMd5s,
Name: task.Upload.Name,
ContentType: contentType,
Cause: "batch-import",
})
if err != nil {
return "", err
}
return urlRaw, nil
}
func (m *Manage) partSize(size int64) (int64, error) {
if size <= 0 {
return 0, errors.New("size must be greater than 0")
}
if size > m.partLimit.MaxPartSize*int64(m.partLimit.MaxNumSize) {
return 0, fmt.Errorf("size must be less than %db", m.partLimit.MaxPartSize*int64(m.partLimit.MaxNumSize))
}
if size <= m.partLimit.MinPartSize*int64(m.partLimit.MaxNumSize) {
return m.partLimit.MinPartSize, nil
}
partSize := size / int64(m.partLimit.MaxNumSize)
if size%int64(m.partLimit.MaxNumSize) != 0 {
partSize++
}
return partSize, nil
}
func (m *Manage) partMD5(parts []string) string {
s := strings.Join(parts, ",")
md5Sum := md5.Sum([]byte(s))
return hex.EncodeToString(md5Sum[:])
}
func (m *Manage) getPartInfo(ctx context.Context, r io.Reader, fileSize int64) (*PartInfo, error) {
partSize, err := m.partSize(fileSize)
if err != nil {
return nil, err
}
partNum := int(fileSize / partSize)
if fileSize%partSize != 0 {
partNum++
}
partSizes := make([]int64, partNum)
for i := 0; i < partNum; i++ {
partSizes[i] = partSize
}
partSizes[partNum-1] = fileSize - partSize*(int64(partNum)-1)
partMd5s := make([]string, partNum)
buf := make([]byte, 1024*8)
fileMd5 := md5.New()
var contentType string
for i := 0; i < partNum; i++ {
h := md5.New()
r := io.LimitReader(r, partSize)
for {
if n, err := r.Read(buf); err == nil {
if contentType == "" {
contentType = http.DetectContentType(buf[:n])
}
h.Write(buf[:n])
fileMd5.Write(buf[:n])
} else if err == io.EOF {
break
} else {
return nil, err
}
}
partMd5s[i] = hex.EncodeToString(h.Sum(nil))
}
partMd5Val := m.partMD5(partMd5s)
fileMd5val := hex.EncodeToString(fileMd5.Sum(nil))
return &PartInfo{
ContentType: contentType,
PartSize: partSize,
PartNum: partNum,
FileMd5: fileMd5val,
PartMd5: partMd5Val,
PartSizes: partSizes,
PartMd5s: partMd5s,
}, nil
}
func (m *Manage) doPut(ctx context.Context, client *http.Client, sign *third.AuthSignParts, part *third.SignPart, reader io.Reader, size int64) error {
rawURL := part.Url
if rawURL == "" {
rawURL = sign.Url
}
if len(sign.Query)+len(part.Query) > 0 {
u, err := url.Parse(rawURL)
if err != nil {
return err
}
query := u.Query()
for i := range sign.Query {
v := sign.Query[i]
query[v.Key] = v.Values
}
for i := range part.Query {
v := part.Query[i]
query[v.Key] = v.Values
}
u.RawQuery = query.Encode()
rawURL = u.String()
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, rawURL, reader)
if err != nil {
return err
}
for i := range sign.Header {
v := sign.Header[i]
req.Header[v.Key] = v.Values
}
for i := range part.Header {
v := part.Header[i]
req.Header[v.Key] = v.Values
}
req.ContentLength = size
resp, err := client.Do(req)
if err != nil {
return err
}
defer func() {
_ = resp.Body.Close()
}()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode/200 != 1 {
return fmt.Errorf("PUT %s part %d failed, status code %d, body %s", rawURL, part.PartNumber, resp.StatusCode, string(body))
}
return nil
}
func (m *Manage) HttpGet(ctx context.Context, url string) (*http.Response, error) {
reqUrl := url
for {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, reqUrl, nil)
if err != nil {
return nil, err
}
DefaultRequestHeader(request.Header)
response, err := m.api.Client.Do(request)
if err != nil {
return nil, err
}
if response.StatusCode != http.StatusOK {
_ = response.Body.Close()
return nil, fmt.Errorf("http get %s status %s", url, response.Status)
}
return response, nil
}
}

29
tools/url2im/pkg/md5.go
Unescape View File

@ -0,0 +1,29 @@
package pkg
import (
"crypto/md5"
"encoding/hex"
"hash"
"io"
)
func NewMd5Reader(r io.Reader) *Md5Reader {
return &Md5Reader{h: md5.New(), r: r}
}
type Md5Reader struct {
h hash.Hash
r io.Reader
}
func (r *Md5Reader) Read(p []byte) (n int, err error) {
n, err = r.r.Read(p)
if err == nil && n > 0 {
r.h.Write(p[:n])
}
return
}
func (r *Md5Reader) Md5() string {
return hex.EncodeToString(r.h.Sum(nil))
}

41
tools/url2im/pkg/progress.go
Unescape View File

@ -0,0 +1,41 @@
package pkg
import (
"bufio"
"os"
"strconv"
"github.com/kelindar/bitmap"
)
func ReadProgress(path string) (*Progress, error) {
file, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return &Progress{}, nil
}
return nil, err
}
defer file.Close()
scanner := bufio.NewScanner(file)
var upload bitmap.Bitmap
for scanner.Scan() {
index, err := strconv.Atoi(scanner.Text())
if err != nil || index < 0 {
continue
}
upload.Set(uint32(index))
}
return &Progress{upload: upload}, nil
}
type Progress struct {
upload bitmap.Bitmap
}
func (p *Progress) IsUploaded(index int) bool {
if p == nil {
return false
}
return p.upload.Contains(uint32(index))
}

3
tools/versionchecker/go.mod
Unescape View File

@ -0,0 +1,3 @@
module github.com/openimsdk/open-im-server/v3/tools/versionchecker
go 1.18

115
tools/versionchecker/versionchecker.go
Unescape View File

@ -0,0 +1,115 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"fmt"
"os/exec"
"runtime"
"time"
)
func executeCommand(cmdName string, args ...string) (string, error) {
cmd := exec.Command(cmdName, args...)
var out bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &stderr
err := cmd.Run()
if err != nil {
return "", fmt.Errorf("Error executing %s: %v", cmdName, err)
}
return out.String(), nil
}
func printTime() string {
currentTime := time.Now()
// 使用 Format 函数以优雅的方式格式化日期和时间
// 2006-01-02 15:04:05 是 Go 中的标准时间格式
formattedTime := currentTime.Format("2006-01-02 15:04:05")
return fmt.Sprintf("Current Date & Time:", formattedTime)
}
func getGoVersion() string {
version := runtime.Version()
goos := runtime.GOOS
goarch := runtime.GOARCH
return fmt.Sprintf("Go Version: %s\nOS: %s\nArchitecture: %s", version, goos, goarch)
}
func getDockerVersion() string {
version, err := executeCommand("docker", "--version")
if err != nil {
return "Docker is not installed. Please install it to get the version."
}
return version
}
func getDockerComposeVersion() string {
version, err := executeCommand("docker-compose", "--version")
if err != nil {
return "Docker Compose is not installed. Please install it to get the version."
}
return version
}
func getKubernetesVersion() string {
version, err := executeCommand("kubectl", "version", "--client", "--short")
if err != nil {
return "Kubernetes is not installed. Please install it to get the version."
}
return version
}
func getGitVersion() string {
version, err := executeCommand("git", "branch", "--show-current")
if err != nil {
return "Git is not installed. Please install it to get the version."
}
return version
}
// NOTE: You'll need to provide appropriate commands for OpenIM versions.
func getOpenIMServerVersion() string {
// Placeholder
return "OpenIM Server: v3.2"
}
func getOpenIMClientVersion() string {
// Placeholder
return "OpenIM Client: v3.2"
}
func main() {
fmt.Println(printTime())
fmt.Println("# Diagnostic Tool Result\n")
fmt.Println("## Go Version")
fmt.Println(getGoVersion())
fmt.Println("## Branch Type")
fmt.Println(getGitVersion())
fmt.Println("## Docker Version")
fmt.Println(getDockerVersion())
fmt.Println("## Docker Compose Version")
fmt.Println(getDockerComposeVersion())
fmt.Println("## Kubernetes Version")
fmt.Println(getKubernetesVersion())
fmt.Println("## OpenIM Versions")
fmt.Println(getOpenIMServerVersion())
fmt.Println(getOpenIMClientVersion())
}

10
tools/yamlfmt/OWNERS
Unescape View File

@ -0,0 +1,10 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- cubxxw
- kubbot
approvers:
- cubxxw
labels:
- sig/testing
- sig/contributor-experience

8
tools/yamlfmt/go.mod
Unescape View File

@ -0,0 +1,8 @@
module github.com/openimsdk/open-im-server/v3/tools/yamlfmt
go 1.18
require (
github.com/likexian/gokit v0.25.13
gopkg.in/yaml.v3 v3.0.1
)

6
tools/yamlfmt/go.sum
Unescape View File

@ -0,0 +1,6 @@
github.com/likexian/gokit v0.25.13 h1:p2Uw3+6fGG53CwdU2Dz0T6bOycdb2+bAFAa3ymwWVkM=
github.com/likexian/gokit v0.25.13/go.mod h1:qQhEWFBEfqLCO3/vOEo2EDKd+EycekVtUK4tex+l2H4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

72
tools/yamlfmt/yamlfmt.go
Unescape View File

@ -0,0 +1,72 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// OPENIM plan on prow tools
package main
import (
"flag"
"fmt"
"io"
"os"
"gopkg.in/yaml.v3"
)
func main() {
// Prow OWNERs file defines the default indent as 2 spaces.
indent := flag.Int("indent", 2, "default indent")
flag.Parse()
for _, path := range flag.Args() {
sourceYaml, err := os.ReadFile(path)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
continue
}
rootNode, err := fetchYaml(sourceYaml)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
continue
}
writer, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
continue
}
err = streamYaml(writer, indent, rootNode)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
continue
}
}
}
func fetchYaml(sourceYaml []byte) (*yaml.Node, error) {
rootNode := yaml.Node{}
err := yaml.Unmarshal(sourceYaml, &rootNode)
if err != nil {
return nil, err
}
return &rootNode, nil
}
func streamYaml(writer io.Writer, indent *int, in *yaml.Node) error {
encoder := yaml.NewEncoder(writer)
encoder.SetIndent(*indent)
err := encoder.Encode(in)
if err != nil {
return err
}
return encoder.Close()
}

158
tools/yamlfmt/yamlfmt_test.go
Unescape View File

@ -0,0 +1,158 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bufio"
"bytes"
"reflect"
"testing"
"github.com/likexian/gokit/assert"
"gopkg.in/yaml.v3"
)
func Test_main(t *testing.T) {
sourceYaml := ` # See the OWNERS docs at https://go.k8s.io/owners
approvers:
- dep-approvers
- thockin # Network
- liggitt
labels:
- sig/architecture
`
outputYaml := `# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- dep-approvers
- thockin # Network
- liggitt
labels:
- sig/architecture
`
node, _ := fetchYaml([]byte(sourceYaml))
var output bytes.Buffer
indent := 2
writer := bufio.NewWriter(&output)
_ = streamYaml(writer, &indent, node)
_ = writer.Flush()
assert.Equal(t, outputYaml, string(output.Bytes()), "yaml was not formatted correctly")
}
func Test_fetchYaml(t *testing.T) {
type args struct {
sourceYaml []byte
}
tests := []struct {
name string
args args
want *yaml.Node
wantErr bool
}{
{
name: "Valid YAML",
args: args{sourceYaml: []byte("key: value")},
want: &yaml.Node{
Kind: yaml.MappingNode,
Tag: "!!map",
Value: "",
Content: []*yaml.Node{
{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: "key",
},
{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: "value",
},
},
},
wantErr: false,
},
{
name: "Invalid YAML",
args: args{sourceYaml: []byte("key:")},
want: nil,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := fetchYaml(tt.args.sourceYaml)
if (err != nil) != tt.wantErr {
t.Errorf("fetchYaml() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("fetchYaml() = %v, want %v", got, tt.want)
}
})
}
}
func Test_streamYaml(t *testing.T) {
type args struct {
indent *int
in *yaml.Node
}
defaultIndent := 2
tests := []struct {
name string
args args
wantWriter string
wantErr bool
}{
{
name: "Valid YAML node with default indent",
args: args{
indent: &defaultIndent,
in: &yaml.Node{
Kind: yaml.MappingNode,
Tag: "!!map",
Value: "",
Content: []*yaml.Node{
{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: "key",
},
{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: "value",
},
},
},
},
wantWriter: "key: value\n",
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
writer := &bytes.Buffer{}
if err := streamYaml(writer, tt.args.indent, tt.args.in); (err != nil) != tt.wantErr {
t.Errorf("streamYaml() error = %v, wantErr %v", err, tt.wantErr)
return
}
if gotWriter := writer.String(); gotWriter != tt.wantWriter {
t.Errorf("streamYaml() = %v, want %v", gotWriter, tt.wantWriter)
}
})
}
}
Write Preview
Loading…
Cancel
Save