From 9c244e4c10a1e76e2563af7aa933e8d0cdbe118d Mon Sep 17 00:00:00 2001 From: Yaxian Date: Thu, 11 Nov 2021 18:45:40 +0800 Subject: [PATCH] feat: improve develop experience --- .gitignore | 2 +- README.md | 45 ++- app.yaml | 41 +++ cmd/rpc/open_im_auth/main.go | 2 +- config/config.dev.yaml | 147 +++++++++ dev.Dockerfile | 11 + docker-compose.dev.yaml | 381 ++++++++++++++++++++++++ docker-compose.local.yaml | 85 ++++++ internal/api/auth/user_register_test.go | 25 ++ pkg/common/config/config.go | 10 +- 10 files changed, 738 insertions(+), 11 deletions(-) create mode 100644 app.yaml create mode 100644 config/config.dev.yaml create mode 100644 dev.Dockerfile create mode 100644 docker-compose.dev.yaml create mode 100644 docker-compose.local.yaml create mode 100644 internal/api/auth/user_register_test.go diff --git a/.gitignore b/.gitignore index 532c7da35..f75d03c97 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ logs out-test .github - +tmp diff --git a/README.md b/README.md index ad3aa2129..b6bf34c92 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Open-IM-Server +# Open-IM-Server ![avatar](https://github.com/OpenIMSDK/Open-IM-Server/blob/main/docs/open-im-logo.png) @@ -51,7 +51,7 @@ By deployment of the Open-IM-Server on the customer's server, developers can int ``` #cd Open-IM-server/script - + chmod +x *.sh ``` @@ -98,11 +98,48 @@ All images are available at https://hub.docker.com/r/lyt1123/open_im_server 5. Check service ``` - ./docker_check_service.sh + ./docker_check_service.sh ``` ![OpenIMServersondockerpng](https://github.com/OpenIMSDK/Open-IM-Server/blob/main/docs/Open-IM-Servers-on-docker.png) +#### How to develop + +1. Install [Go environment](https://golang.org/doc/install). Make sure Go version is at least 1.15. + +2. Install `Nodejs` and `pm2` + +``` + curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash - + npm install pm2 -g +``` + +2. Clone the Open-IM project to your server. + + ``` + git clone https://github.com/OpenIMSDK/Open-IM-Server.git --recursive + ``` + +3. Start Service. + + ``` + # run etcd/mongodb/mysql/redis/kafka + docker-compose -f docker-compose.local.yaml up -d + + # run open-im services + pm2 start app.yaml --watch + pm2 ls + pm2 logs + ``` + +or try with `docker-compose` + +``` + docker-compose -f docker-compose.dev.yaml up -d + docker-compose -f docker-compose.dev.yaml ps + docker-compose -f docker-compose.dev.yaml logs -f +``` + ### CONFIGURATION INSTRUCTIONS > Open-IM configuration is divided into basic component configuration and business internal service configuration. Developers need to fill in the address of each component as the address of their server component when using the product, and ensure that the internal service port of the business is not occupied @@ -144,7 +181,7 @@ All images are available at https://hub.docker.com/r/lyt1123/open_im_server - start_all.sh&&stop_all.sh - Total script, start all services and close all services -## Authentication Clow Chart +## Authentication Clow Chart ![avatar](https://github.com/OpenIMSDK/Open-IM-Server/blob/main/docs/open-im-server.png) diff --git a/app.yaml b/app.yaml new file mode 100644 index 000000000..52b14d2d7 --- /dev/null +++ b/app.yaml @@ -0,0 +1,41 @@ + +apps: +- name: open_im_api + script: "go run main.go" + cwd: "cmd/open_im_api" + +- name: open_im_auth + script: "go run main.go" + cwd: "cmd/rpc/open_im_auth" + +- name: open_im_offline_msg + script: "go run main.go" + cwd: "cmd/rpc/open_im_msg" + +- name: open_im_friend + script: "go run main.go" + cwd: "cmd/rpc/open_im_friend" + +- name: open_im_group + script: "go run main.go" + cwd: "cmd/rpc/open_im_group" + +- name: open_im_user + script: "go run main.go" + cwd: "cmd/rpc/open_im_user" + +- name: open_im_push + script: "go run main.go" + cwd: "cmd/open_im_push" + +- name: open_im_timed_task + script: "go run main.go" + cwd: "cmd/open_im_timer_task" + +- name: open_im_msg_transfer + script: "go run main.go" + cwd: "cmd/open_im_msg_transfer" + +- name: open_im_msg_gateway + script: "go run main.go" + cwd: "cmd/open_im_msg_gateway" \ No newline at end of file diff --git a/cmd/rpc/open_im_auth/main.go b/cmd/rpc/open_im_auth/main.go index ed4656d35..6c6f3da1c 100644 --- a/cmd/rpc/open_im_auth/main.go +++ b/cmd/rpc/open_im_auth/main.go @@ -6,7 +6,7 @@ import ( ) func main() { - rpcPort := flag.Int("port", 10600, "RpcToken default listen port 10800") + rpcPort := flag.Int("port", 10600, "RpcToken default listen port 10600") flag.Parse() rpcServer := rpcAuth.NewRpcAuthServer(*rpcPort) rpcServer.Run() diff --git a/config/config.dev.yaml b/config/config.dev.yaml new file mode 100644 index 000000000..2b9537bef --- /dev/null +++ b/config/config.dev.yaml @@ -0,0 +1,147 @@ + +# The class cannot be named by Pascal or camel case. +# If it is not used, the corresponding structure will not be set, +# and it will not be read naturally. +serverversion: 1.0.0 +#---------------Infrastructure configuration---------------------# +etcd: + etcdSchema: openIM + etcdAddr: [ etcd:2379 ] + +mysql: + dbMysqlAddress: [ mysql:3306 ] + dbMysqlUserName: root + dbMysqlPassword: openIM + dbMysqlDatabaseName: openIM + dbTableName: eMsg + dbMsgTableNum: 1 + dbMaxOpenConns: 20 + dbMaxIdleConns: 10 + dbMaxLifeTime: 120 + +mongo: + dbAddress: [ mongodb:27017 ] + dbDirect: false + dbTimeout: 10 + dbDatabase: openIM + dbSource: admin + dbUserName: + dbPassword: + dbMaxPoolSize: 20 + dbRetainChatRecords: 7 + +redis: + dbAddress: redis:6379 + dbMaxIdle: 128 + dbMaxActive: 0 + dbIdleTimeout: 120 + dbPassWord: openIM + +kafka: + ws2mschat: + addr: [ kafka:9092 ] + topic: "ws2ms_chat" + ms2pschat: + addr: [ kafka:9092 ] + topic: "ms2ps_chat" + consumergroupid: + msgToMongo: mongo + msgToMySql: mysql + msgToPush: push + + + +#---------------Internal service configuration---------------------# + +# The service ip default is empty, +# automatically obtain the machine's valid network card ip as the service ip, +# otherwise the configuration ip is preferred +serverip: + +api: + openImApiPort: [ 10000 ] +sdk: + openImSdkWsPort: [ 30000 ] + +credential: + tencent: + appID: 1302656840 + region: ap-chengdu + bucket: echat-1302656840 + secretID: AKIDGNYVChzIQinu7QEgtNp0hnNgqcV8vZTC + secretKey: kz15vW83qM6dBUWIq681eBZA0c0vlIbe + + +rpcport: + openImUserPort: [ 10100 ] + openImFriendPort: [ 10200 ] + openImOfflineMessagePort: [ 10300 ] + openImOnlineRelayPort: [ 10400 ] + openImGroupPort: [ 10500 ] + openImAuthPort: [ 10600 ] + openImPushPort: [ 10700 ] + + +rpcregistername: + openImUserName: User + openImFriendName: Friend + openImOfflineMessageName: OfflineMessage + openImPushName: Push + openImOnlineMessageRelayName: OnlineMessageRelay + openImGroupName: Group + openImAuthName: Auth + +log: + storageLocation: ../logs/ + rotationTime: 24 + remainRotationCount: 5 + remainLogLevel: 6 + elasticSearchSwitch: false + elasticSearchAddr: [ 127.0.0.1:9201 ] + elasticSearchUser: "" + elasticSearchPassword: "" + +modulename: + longConnSvrName: msg_gateway + msgTransferName: msg_transfer + pushName: push + +longconnsvr: + openImWsPort: [ 17778 ] + websocketMaxConnNum: 10000 + websocketMaxMsgLen: 4096 + websocketTimeOut: 10 + +push: + tpns: + ios: + accessID: 1600018281 + secretKey: 3cd68a77a95b89e5089a1aca523f318f + android: + accessID: 111 + secretKey: 111 + jpns: + appKey: 2783339cee4de379cc798fe1 + masterSecret: 66e5f309e032c68cc668c28a + pushUrl: "https://api.jpush.cn/v3/push" +manager: + appManagerUid: ["openIM123456","openIM654321"] + secrets: ["openIM1","openIM2"] + +secret: tuoyun + +multiloginpolicy: + onlyOneTerminalAccess: false + mobileAndPCTerminalAccessButOtherTerminalKickEachOther: true + allTerminalAccess: false + +#token config +tokenpolicy: + accessSecret: "open_im_server" + # Token effective time seconds as a unit + #Seven days 7*24*60*60 + accessExpire: 604800 + +messagecallback: + callbackSwitch: false + callbackUrl: "http://www.xxx.com/msg/judge" \ No newline at end of file diff --git a/dev.Dockerfile b/dev.Dockerfile new file mode 100644 index 000000000..4b20d744f --- /dev/null +++ b/dev.Dockerfile @@ -0,0 +1,11 @@ +FROM golang:1.16 as base + +FROM base as dev + +ENV GO111MODULE=on +ENV GOPROXY=https://goproxy.cn,direct + + +RUN curl -sSfL https://raw.githubusercontent.com/cosmtrek/air/master/install.sh | sh -s -- -b $(go env GOPATH)/bin + +CMD ["air"] \ No newline at end of file diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml new file mode 100644 index 000000000..202450a33 --- /dev/null +++ b/docker-compose.dev.yaml @@ -0,0 +1,381 @@ +version: "3" +services: + mysql: + image: mysql:5.7 + ports: + - 3306:3306 + container_name: mysql + volumes: + - ./components/mysql/data:/var/lib/mysql + - /etc/localtime:/etc/localtime + environment: + MYSQL_ROOT_PASSWORD: openIM + restart: always + + mongodb: + image: mongo:4.0 + ports: + - 27017:27017 + container_name: mongo + volumes: + - ./components/mongodb/data:/data/db + environment: + TZ: Asia/Shanghai + restart: always + + redis: + image: redis + ports: + - 6379:6379 + container_name: redis + volumes: + - ./components/redis/data:/data + #redis config file + #- ./components/redis/config/redis.conf:/usr/local/redis/config/redis.conf + environment: + TZ: Asia/Shanghai + restart: always + sysctls: + net.core.somaxconn: 1024 + command: redis-server --requirepass openIM --appendonly yes + + + zookeeper: + image: wurstmeister/zookeeper + ports: + - 2181:2181 + container_name: zookeeper + volumes: + - /etc/localtime:/etc/localtime + environment: + TZ: Asia/Shanghai + restart: always + + + kafka: + image: wurstmeister/kafka + container_name: kafka + restart: always + environment: + TZ: Asia/Shanghai + KAFKA_BROKER_ID: 0 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_LISTENERS: PLAINTEXT://kafka:9092 + depends_on: + - zookeeper + links: + - zookeeper + ports: + - 9092:9092 + + etcd: + image: quay.io/coreos/etcd + ports: + - 2379:2379 + - 2380:2380 + container_name: etcd + volumes: + - /etc/timezone:/etc/timezone + - /etc/localtime:/etc/localtime + environment: + ETCDCTL_API: 3 + restart: always + command: /usr/local/bin/etcd --name etcd0 --data-dir /etcd-data --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://0.0.0.0:2379 --listen-peer-urls http://0.0.0.0:2380 --initial-advertise-peer-urls http://0.0.0.0:2380 --initial-cluster etcd0=http://0.0.0.0:2380 --initial-cluster-token tkn --initial-cluster-state new + + open_im_api: + build: + dockerfile: dev.Dockerfile + context: . + target: dev + network: host + depends_on: + - kafka + - mysql + - mongodb + - redis + - etcd + container_name: open_im_api + volumes: + - ./cmd:/Open-IM-Server/cmd + - ./config/config.dev.yaml:/Open-IM-Server/config/config.yaml + - ./internal:/Open-IM-Server/internal + - ./pkg:/Open-IM-Server/pkg + - ./go.mod:/Open-IM-Server/go.mod + - ./go.sum:/Open-IM-Server/go.sum + working_dir: /Open-IM-Server/cmd/open_im_api + ports: + - 10000:10000 + links: + - kafka + - mysql + - mongodb + - redis + - etcd + + open_im_auth: + build: + dockerfile: dev.Dockerfile + context: . + target: dev + network: host + depends_on: + - kafka + - mysql + - mongodb + - redis + - etcd + links: + - kafka + - mysql + - mongodb + - redis + - etcd + container_name: open_im_auth + volumes: + - ./cmd:/Open-IM-Server/cmd + - ./config/config.dev.yaml:/Open-IM-Server/config/config.yaml + - ./internal:/Open-IM-Server/internal + - ./pkg:/Open-IM-Server/pkg + - ./go.mod:/Open-IM-Server/go.mod + - ./go.sum:/Open-IM-Server/go.sum + working_dir: /Open-IM-Server/cmd/rpc/open_im_auth + ports: + - 10600:10600 + + open_im_user: + build: + dockerfile: dev.Dockerfile + context: . + target: dev + network: host + depends_on: + - kafka + - mysql + - mongodb + - redis + - etcd + links: + - kafka + - mysql + - mongodb + - redis + - etcd + container_name: open_im_user + volumes: + - ./cmd:/Open-IM-Server/cmd + - ./config/config.dev.yaml:/Open-IM-Server/config/config.yaml + - ./internal:/Open-IM-Server/internal + - ./pkg:/Open-IM-Server/pkg + - ./go.mod:/Open-IM-Server/go.mod + - ./go.sum:/Open-IM-Server/go.sum + working_dir: /Open-IM-Server/cmd/rpc/open_im_user + ports: + - 10100:10100 + + open_im_friend: + build: + dockerfile: dev.Dockerfile + context: . + target: dev + network: host + depends_on: + - kafka + - mysql + - mongodb + - redis + - etcd + links: + - kafka + - mysql + - mongodb + - redis + - etcd + container_name: open_im_friend + volumes: + - ./cmd:/Open-IM-Server/cmd + - ./config/config.dev.yaml:/Open-IM-Server/config/config.yaml + - ./internal:/Open-IM-Server/internal + - ./pkg:/Open-IM-Server/pkg + - ./go.mod:/Open-IM-Server/go.mod + - ./go.sum:/Open-IM-Server/go.sum + working_dir: /Open-IM-Server/cmd/rpc/open_im_friend + ports: + - 10200:10200 + + open_im_group: + build: + dockerfile: dev.Dockerfile + context: . + target: dev + network: host + depends_on: + - kafka + - mysql + - mongodb + - redis + - etcd + links: + - kafka + - mysql + - mongodb + - redis + - etcd + container_name: open_im_group + volumes: + - ./cmd:/Open-IM-Server/cmd + - ./config/config.dev.yaml:/Open-IM-Server/config/config.yaml + - ./internal:/Open-IM-Server/internal + - ./pkg:/Open-IM-Server/pkg + - ./go.mod:/Open-IM-Server/go.mod + - ./go.sum:/Open-IM-Server/go.sum + working_dir: /Open-IM-Server/cmd/rpc/open_im_group + ports: + - 10500:10500 + + open_im_push: + build: + dockerfile: dev.Dockerfile + context: . + target: dev + network: host + depends_on: + - kafka + - mysql + - mongodb + - redis + - etcd + links: + - kafka + - mysql + - mongodb + - redis + - etcd + container_name: open_im_push + volumes: + - ./cmd:/Open-IM-Server/cmd + - ./config/config.dev.yaml:/Open-IM-Server/config/config.yaml + - ./internal:/Open-IM-Server/internal + - ./pkg:/Open-IM-Server/pkg + - ./go.mod:/Open-IM-Server/go.mod + - ./go.sum:/Open-IM-Server/go.sum + working_dir: /Open-IM-Server/cmd/open_im_push + ports: + - 10700:10700 + + open_im_timed_task: + build: + dockerfile: dev.Dockerfile + context: . + target: dev + network: host + depends_on: + - kafka + - mysql + - mongodb + - redis + - etcd + links: + - kafka + - mysql + - mongodb + - redis + - etcd + container_name: open_im_timed_task + volumes: + - ./cmd:/Open-IM-Server/cmd + - ./config/config.dev.yaml:/Open-IM-Server/config/config.yaml + - ./internal:/Open-IM-Server/internal + - ./pkg:/Open-IM-Server/pkg + - ./go.mod:/Open-IM-Server/go.mod + - ./go.sum:/Open-IM-Server/go.sum + working_dir: /Open-IM-Server/cmd/open_im_timer_task + + open_im_offline_msg: + build: + dockerfile: dev.Dockerfile + context: . + target: dev + network: host + depends_on: + - kafka + - mysql + - mongodb + - redis + - etcd + links: + - kafka + - mysql + - mongodb + - redis + - etcd + container_name: open_im_offline_msg + volumes: + - ./cmd:/Open-IM-Server/cmd + - ./config/config.dev.yaml:/Open-IM-Server/config/config.yaml + - ./internal:/Open-IM-Server/internal + - ./pkg:/Open-IM-Server/pkg + - ./go.mod:/Open-IM-Server/go.mod + - ./go.sum:/Open-IM-Server/go.sum + working_dir: /Open-IM-Server/cmd/rpc/open_im_msg + ports: + - 10300:10300 + + open_im_msg_transfer: + build: + dockerfile: dev.Dockerfile + context: . + target: dev + network: host + depends_on: + - kafka + - mysql + - mongodb + - redis + - etcd + links: + - kafka + - mysql + - mongodb + - redis + - etcd + container_name: open_im_msg_transfer + volumes: + - ./cmd:/Open-IM-Server/cmd + - ./config/config.dev.yaml:/Open-IM-Server/config/config.yaml + - ./internal:/Open-IM-Server/internal + - ./pkg:/Open-IM-Server/pkg + - ./go.mod:/Open-IM-Server/go.mod + - ./go.sum:/Open-IM-Server/go.sum + working_dir: /Open-IM-Server/cmd/open_im_msg_transfer + + open_im_msg_gateway: + build: + dockerfile: dev.Dockerfile + context: . + target: dev + network: host + depends_on: + - kafka + - mysql + - mongodb + - redis + - etcd + links: + - kafka + - mysql + - mongodb + - redis + - etcd + container_name: open_im_msg_gateway + volumes: + - ./cmd:/Open-IM-Server/cmd + - ./config/config.dev.yaml:/Open-IM-Server/config/config.yaml + - ./internal:/Open-IM-Server/internal + - ./pkg:/Open-IM-Server/pkg + - ./go.mod:/Open-IM-Server/go.mod + - ./go.sum:/Open-IM-Server/go.sum + working_dir: /Open-IM-Server/cmd/open_im_msg_gateway + ports: + - 10400:10400 + - 17778:17778 \ No newline at end of file diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml new file mode 100644 index 000000000..4755f4118 --- /dev/null +++ b/docker-compose.local.yaml @@ -0,0 +1,85 @@ +version: "3" +services: + mysql: + image: mysql:5.7 + ports: + - 3306:3306 + container_name: mysql + volumes: + - ./components/mysql/data:/var/lib/mysql + - /etc/localtime:/etc/localtime + environment: + MYSQL_ROOT_PASSWORD: openIM + restart: always + + mongodb: + image: mongo:4.0 + ports: + - 27017:27017 + container_name: mongo + volumes: + - ./components/mongodb/data:/data/db + environment: + TZ: Asia/Shanghai + restart: always + + redis: + image: redis + ports: + - 6379:6379 + container_name: redis + volumes: + - ./components/redis/data:/data + #redis config file + #- ./components/redis/config/redis.conf:/usr/local/redis/config/redis.conf + environment: + TZ: Asia/Shanghai + restart: always + sysctls: + net.core.somaxconn: 1024 + command: redis-server --requirepass openIM --appendonly yes + + + zookeeper: + image: wurstmeister/zookeeper + ports: + - 2181:2181 + container_name: zookeeper + volumes: + - /etc/localtime:/etc/localtime + environment: + TZ: Asia/Shanghai + restart: always + + + kafka: + image: wurstmeister/kafka + container_name: kafka + restart: always + environment: + TZ: Asia/Shanghai + KAFKA_BROKER_ID: 0 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092 + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 + # network_mode: "host" + depends_on: + - zookeeper + links: + - zookeeper + ports: + - 9092:9092 + + etcd: + image: quay.io/coreos/etcd + ports: + - 2379:2379 + - 2380:2380 + container_name: etcd + volumes: + - /etc/timezone:/etc/timezone + - /etc/localtime:/etc/localtime + environment: + ETCDCTL_API: 3 + restart: always + command: /usr/local/bin/etcd --name etcd0 --data-dir /etcd-data --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://0.0.0.0:2379 --listen-peer-urls http://0.0.0.0:2380 --initial-advertise-peer-urls http://0.0.0.0:2380 --initial-cluster etcd0=http://0.0.0.0:2380 --initial-cluster-token tkn --initial-cluster-state new diff --git a/internal/api/auth/user_register_test.go b/internal/api/auth/user_register_test.go new file mode 100644 index 000000000..edebbbf79 --- /dev/null +++ b/internal/api/auth/user_register_test.go @@ -0,0 +1,25 @@ +package apiAuth + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" +) + +func init() { + gin.SetMode(gin.TestMode) +} + +func Test_UserRegister(t *testing.T) { + res := httptest.NewRecorder() + c, _ := gin.CreateTestContext(res) + c.Request, _ = http.NewRequest("POST", "/", bytes.NewBufferString(`{"secret": "tuoyun", "platform": 1, "uid": "3", "name": "1"}`)) + + UserRegister(c) + + assert.Equal(t, res.Code, 200) +} diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 6f5723591..f10abdc0d 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -1,20 +1,22 @@ package config import ( - "gopkg.in/yaml.v3" "io/ioutil" + + "gopkg.in/yaml.v3" + "path/filepath" "runtime" ) +var Config config + var ( _, b, _, _ = runtime.Caller(0) // Root folder of this project Root = filepath.Join(filepath.Dir(b), "../../..") ) -var Config config - type config struct { ServerIP string `yaml:"serverip"` ServerVersion string `yaml:"serverversion"` @@ -160,8 +162,6 @@ type config struct { } func init() { - //path, _ := os.Getwd() - //bytes, err := ioutil.ReadFile(path + "/config/config.yaml") // if we cd Open-IM-Server/src/utils and run go test // it will panic cannot find config/config.yaml bytes, err := ioutil.ReadFile(Root + "/config/config.yaml")