feat: k8s

k8s/deployment/clusterRole.yml

@@ -5,7 +5,7 @@ metadata:
   name: service-reader
 rules:
   - apiGroups: [""]
-    resources: ["services", "endpoints"]
+    resources: ["services", "endpoints", "pods"]
     verbs: ["get", "list", "watch"]
 
 ---

k8s/deployment/docker-secret.yml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: docker-secret
+  namespace: default
+type: kubernetes.io/dockerconfigjson
+data:
+  # {
+  #  "auths": {
+  #    "your.registry.com": {
+  #      "username": "username",
+  #      "password": "psw",
+  #      "email":    "openim@example.com",
+  #      "auth":     "dXNlcm5hbWU6cHN3"
+  #    }
+  #  }
+  #}
+  # Base64 encoded .docerconfigjson content
+  .dockerconfigjson: "ewogICJhdXRocyI6IHsKICAgICJ5b3VyLnJlZ2lzdHJ5LmNvbSI6IHsKICAgICAgInVzZXJuYW1lIjogInVzZXJuYW1lIiwKICAgICAgInBhc3N3b3JkIjogInBzdyIsCiAgICAgICJlbWFpbCI6ICAgICJvcGVuaW1AZXhhbXBsZS5jb20iLAogICAgICAiYXV0aCI6ICAgICAiZFhObGNtNWhiV1U2Y0hOMyIKICAgIH0KICB9Cn0="

k8s/deployment/http-ingress.yml

@@ -3,23 +3,24 @@ kind: Ingress
 metadata:
   name: openim-ingress
   annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: /
+    nginx.ingress.kubernetes.io/rewrite-target: /$1
 spec:
-  ingressClassName: openim-nginx
+  ingressClassName: nginx
   rules:
-    - http:
+    - host: openim-api.localtest.me
+      http:
         paths:
-          - path: /openim-api
+          - path: /api/(.*)
-            pathType: Prefix
+            pathType: ImplementationSpecific
             backend:
               service:
                 name: openim-api-service
                 port:
                   number: 10002
-          - path: /openim-msggateway
+          - path: /msg_gateway/(.*)
-            pathType: Prefix
+            pathType: ImplementationSpecific
             backend:
               service:
-                name: openim-msggateway-service
+                name: messagegateway-rpc-service
                 port:
                   number: 10001

k8s/deployment/https-ingress.yml

@@ -0,0 +1,35 @@
+# HTTPS Ingress (TLS + cert-manager)
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: openim-ingress-https
+  namespace: default
+  annotations:
+    nginx.ingress.kubernetes.io/use-regex: "true"
+    nginx.ingress.kubernetes.io/rewrite-target: /$1
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - openim-api.localtest.me
+      secretName: openim-api-tls
+  rules:
+    - host: openim-api.localtest.me
+      http:
+        paths:
+          - path: /api/(.*)
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: openim-api-service
+                port:
+                  number: 10002
+          - path: /msg_gateway/(.*)
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: messagegateway-rpc-service
+                port:
+                  number: 10001

k8s/deployment/kafka-statefulset.yml

@@ -5,7 +5,7 @@ metadata:
   labels:
     app: kafka
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: kafka

k8s/deployment/minio-statefulset.yml

@@ -5,7 +5,7 @@ metadata:
   labels:
     app: minio
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: minio

k8s/deployment/mongo-statefulset.yml

@@ -4,7 +4,7 @@ metadata:
   name: mongo-statefulset
 spec:
   serviceName: "mongo"
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: mongo

k8s/deployment/openim-api-deployment.yml

@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: openim-api
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: openim-api
@@ -12,9 +12,11 @@ spec:
       labels:
         app: openim-api
     spec:
+      imagePullSecrets:
+        - name: docker-secret
       containers:
         - name: openim-api-container
-          image: openim/openim-api:v3.8.3
+          image: openim-api:test
           env:
             - name: CONFIG_PATH
               value: "/config"

k8s/deployment/openim-crontask-deployment.yml

@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: openim-crontask
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: crontask
@@ -12,9 +12,11 @@ spec:
       labels:
         app: crontask
     spec:
+      imagePullSecrets:
+        - name: docker-secret
       containers:
         - name: crontask-container
-          image: openim/openim-crontask:v3.8.3
+          image: openim-crontask:test
           env:
             - name: CONFIG_PATH
               value: "/config"

k8s/deployment/openim-msggateway-deployment.yml

@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: messagegateway-rpc-server
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: messagegateway-rpc-server
@@ -12,9 +12,11 @@ spec:
       labels:
         app: messagegateway-rpc-server
     spec:
+      imagePullSecrets:
+        - name: docker-secret
       containers:
         - name: openim-msggateway-container
-          image: openim/openim-msggateway:v3.8.3
+          image: openim-msggateway:test
           env:
             - name: CONFIG_PATH
               value: "/config"

k8s/deployment/openim-msggateway-service.yml

@@ -6,7 +6,7 @@ spec:
   selector:
     app: messagegateway-rpc-server
   ports:
-    - name: longConnServer-10001
+    - name: longconnserver-10001
       protocol: TCP
       port: 10001
       targetPort: 10001

k8s/deployment/openim-msgtransfer-deployment.yml

@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: openim-msgtransfer-server
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: openim-msgtransfer-server
@@ -12,9 +12,11 @@ spec:
       labels:
         app: openim-msgtransfer-server
     spec:
+      imagePullSecrets:
+        - name: docker-secret
       containers:
         - name: openim-msgtransfer-container
-          image: openim/openim-msgtransfer:v3.8.3
+          image: openim-msgtransfer:test
           env:
             - name: CONFIG_PATH
               value: "/config"

k8s/deployment/openim-push-deployment.yml

@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: push-rpc-server
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: push-rpc-server
@@ -12,9 +12,11 @@ spec:
       labels:
         app: push-rpc-server
     spec:
+      imagePullSecrets:
+        - name: docker-secret
       containers:
         - name: push-rpc-server-container
-          image: openim/openim-push:v3.8.3
+          image: openim-push:test
           env:
             - name: CONFIG_PATH
               value: "/config"

k8s/deployment/openim-rpc-auth-deployment.yml

@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: auth-rpc-server
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: auth-rpc-server
@@ -12,10 +12,12 @@ spec:
       labels:
         app: auth-rpc-server
     spec:
+      imagePullSecrets:
+        - name: docker-secret
       containers:
         - name: auth-rpc-server-container
-          image: openim/openim-rpc-auth:v3.8.3
+          image: openim-rpc-auth:test
-          imagePullPolicy: Never
+          imagePullPolicy: IfNotPresent
           env:
             - name: CONFIG_PATH
               value: "/config"

k8s/deployment/openim-rpc-conversation-deployment.yml

@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: conversation-rpc-server
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: conversation-rpc-server
@@ -12,9 +12,11 @@ spec:
       labels:
         app: conversation-rpc-server
     spec:
+      imagePullSecrets:
+        - name: docker-secret
       containers:
         - name: conversation-rpc-server-container
-          image: openim/openim-rpc-conversation:v3.8.3
+          image: openim-rpc-conversation:test
           env:
             - name: CONFIG_PATH
               value: "/config"

k8s/deployment/openim-rpc-friend-deployment.yml

@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: friend-rpc-server
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: friend-rpc-server
@@ -12,9 +12,11 @@ spec:
       labels:
         app: friend-rpc-server
     spec:
+      imagePullSecrets:
+        - name: docker-secret
       containers:
         - name: friend-rpc-server-container
-          image: openim/openim-rpc-friend:v3.8.3
+          image: openim-rpc-friend:test
           env:
             - name: CONFIG_PATH
               value: "/config"

k8s/deployment/openim-rpc-group-deployment.yml

@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: group-rpc-server
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: group-rpc-server
@@ -12,9 +12,11 @@ spec:
       labels:
         app: group-rpc-server
     spec:
+      imagePullSecrets:
+        - name: docker-secret
       containers:
         - name: group-rpc-server-container
-          image: openim/openim-rpc-group:v3.8.3
+          image: openim-rpc-group:test
           env:
             - name: CONFIG_PATH
               value: "/config"

k8s/deployment/openim-rpc-msg-deployment.yml

@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: msg-rpc-server
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: msg-rpc-server
@@ -12,9 +12,11 @@ spec:
       labels:
         app: msg-rpc-server
     spec:
+      imagePullSecrets:
+        - name: docker-secret
       containers:
         - name: msg-rpc-server-container
-          image: openim/openim-rpc-msg:v3.8.3
+          image: openim-rpc-msg:test
           env:
             - name: CONFIG_PATH
               value: "/config"

k8s/deployment/openim-rpc-third-deployment.yml

@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: third-rpc-server
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: third-rpc-server
@@ -12,9 +12,11 @@ spec:
       labels:
         app: third-rpc-server
     spec:
+      imagePullSecrets:
+        - name: docker-secret
       containers:
         - name: third-rpc-server-container
-          image: openim/openim-rpc-third:v3.8.3
+          image: openim-rpc-third:test
           env:
             - name: CONFIG_PATH
               value: "/config"

k8s/deployment/openim-rpc-user-deployment.yml

@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: user-rpc-server
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: user-rpc-server
@@ -12,9 +12,11 @@ spec:
       labels:
         app: user-rpc-server
     spec:
+      imagePullSecrets:
+        - name: docker-secret
       containers:
         - name: user-rpc-server-container
-          image: openim/openim-rpc-user:v3.8.3
+          image: openim-rpc-user:test
           env:
             - name: CONFIG_PATH
               value: "/config"

k8s/deployment/redis-statefulset.yml

@@ -2,9 +2,11 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: redis-statefulset
+  labels:
+    app: redis
 spec:
-  serviceName: "redis"
+  serviceName: "redis-service"
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: redis
@@ -18,38 +20,31 @@ spec:
           image: redis:7.0.0
           ports:
             - containerPort: 6379
+              name: redis
           env:
             - name: TZ
               value: "Asia/Shanghai"
             - name: REDIS_PASSWORD
               valueFrom:
                 secretKeyRef:
-                  name: redis-secret
+                  name: openim-redis-secret
                   key: redis-password
           volumeMounts:
             - name: redis-data
               mountPath: /data
           command:
-            [
+            - /bin/sh
-              "/bin/sh",
+            - -c
-              "-c",
+            - |
-              'redis-server  --requirepass "$REDIS_PASSWORD" --appendonly yes',
+              redis-server \
-            ]
+                --requirepass "$REDIS_PASSWORD" \
-      volumes:
+                --appendonly yes \
-        - name: redis-config-volume
+                --dir /data
-          configMap:
+  volumeClaimTemplates:
-            name: openim-config
+    - metadata:
-        - name: redis-data
+        name: redis-data
-          persistentVolumeClaim:
+      spec:
-            claimName: redis-pvc
+        accessModes: ["ReadWriteOnce"]
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: redis-pvc
-spec:
-  accessModes:
-    - ReadWriteOnce
         resources:
           requests:
             storage: 5Gi

k8s/deployment/tls-secret.yml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: openim-api-tls
+  namespace: default
+type: kubernetes.io/tls
+data:
+  tls.crt: <base64-encoded-tls.crt>
+  tls.key: <base64-encoded-tls.key>

pkg/common/cmd/msg_gateway.go

@@ -58,10 +58,9 @@ func (m *MsgGatewayCmd) Exec() error {
 func (m *MsgGatewayCmd) runE() error {
 	m.msgGatewayConfig.Index = config.Index(m.Index())
 	rpc := m.msgGatewayConfig.MsgGateway.RPC
-	var prometheus config.Prometheus
 	return startrpc.Start(
 		m.ctx, &m.msgGatewayConfig.Discovery,
-		&prometheus,
+		&m.msgGatewayConfig.MsgGateway.Prometheus,
 		rpc.ListenIP, rpc.RegisterIP,
 		rpc.AutoSetPorts,
 		rpc.Ports, int(m.msgGatewayConfig.Index),

pkg/common/cmd/root.go

@@ -110,9 +110,13 @@ func (r *RootCmd) persistentPreRun(cmd *cobra.Command, opts ...func(*CmdOpts)) e
 	if err := r.initializeLogger(cmdOpts); err != nil {
 		return errs.WrapMsg(err, "failed to initialize logger")
 	}
+
+	if r.etcdClient != nil {
 		if err := r.etcdClient.Close(); err != nil {
 			return errs.WrapMsg(err, "failed to close etcd client")
 		}
+	}
+
 	return nil
 }