build: improve kubernetes deployment.

9 months ago · 4cd8914bf5
parent fbaa7792ca
commit 4cd8914bf5
14 changed files with 146 additions and 245 deletions
--- a/deployments/deploy/clusterRole.yml
+++ b/deployments/deploy/clusterRole.yml
@ -2,11 +2,10 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  # role name
  name: service-reader
 rules:
  - apiGroups: [""]
-    resources: ["services"]
+    resources: ["services", "endpoints"]
    verbs: ["get", "list", "watch"]

 ---
--- a/deployments/deploy/minio-statefulset.yml
+++ b/deployments/deploy/minio-statefulset.yml
@ -17,7 +17,6 @@ spec:
      containers:
        - name: minio
          image: minio/minio:RELEASE.2024-01-11T07-46-16Z
-          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 9000 # MinIO service port
            - containerPort: 9090 # MinIO console port
@ -29,23 +28,22 @@ spec:
          env:
            - name: TZ
              value: "Asia/Shanghai"
-            - name: MINIO_ACCESS_KEY
+            - name: MINIO_ROOT_USER
              valueFrom:
                secretKeyRef:
                  name: minio-secret
                  key: minio-root-user
-            - name: MINIO_SECRET_KEY
+            - name: MINIO_ROOT_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: minio-secret
                  key: minio-root-password
-
          command:
-            - minio
-            - server
-            - /data
-            - "--console-address"
-            - ":9090"
+            - "/bin/sh"
+            - "-c"
+            - |
+              mkdir -p /data && \
+              minio server /data --console-address ":9090"
      volumes:
        - name: minio-data
          persistentVolumeClaim:
--- a/deployments/deploy/mongo-service.yml
+++ b/deployments/deploy/mongo-service.yml
@ -10,4 +10,4 @@ spec:
      protocol: TCP
      port: 37017
      targetPort: 27017
-  type: ClusterIP
+  type: NodePort
--- a/deployments/deploy/openim-api-deployment.yml
+++ b/deployments/deploy/openim-api-deployment.yml
@ -17,7 +17,25 @@ spec:
          image: openim/openim-api:v3.8.3
          env:
            - name: CONFIG_PATH
-              value: "/config"          
+              value: "/config"
+            - name: IMENV_REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: redis-secret
+                  key: redis-password
+
+            - name: IMENV_MONGODB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mongo-secret
+                  key: mongo_openim_password
+
+            - name: IMENV_MONGODB_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: mongo-secret
+                  key: mongo_openim_username
+
          volumeMounts:
            - name: openim-config
              mountPath: "/config"
--- a/deployments/deploy/openim-api-service.yml
+++ b/deployments/deploy/openim-api-service.yml
@ -6,11 +6,11 @@ spec:
  selector:
    app: openim-api
  ports:
-    - name: http-10302
+    - name: http-10002
      protocol: TCP
      port: 10002
      targetPort: 10002
-    - name: prometheus-20113
+    - name: prometheus-12002
      protocol: TCP
      port: 12002
      targetPort: 12002
--- a/deployments/deploy/openim-config.yml
+++ b/deployments/deploy/openim-config.yml
@ -14,11 +14,11 @@ data:
      password: ''

    rpcService:
-      messageGateway: messageGateway-rpc-service
      user: user-rpc-service
      friend: friend-rpc-service
      msg: msg-rpc-service
      push: push-rpc-service
+      messageGateway: messagegateway-rpc-service
      group: group-rpc-service
      auth: auth-rpc-service
      conversation: conversation-rpc-service
@ -438,9 +438,9 @@ data:
    # Name of the bucket in MinIO
    bucket: openim
    # Access key ID for MinIO authentication
-    accessKeyID: root   # ${MINIO_ACCESS_KEY}
+    accessKeyID: root   
    # Secret access key for MinIO authentication
-    secretAccessKey:    # openIM123 # ${MINIO_SECRET_KEY}
+    secretAccessKey:    # openIM123
    # Session token for MinIO authentication (optional)
    sessionToken:
    # Internal address of the MinIO server
@ -775,94 +775,6 @@ data:
        desc: burn after reading
        ext: burn after reading

-  prometheus.yml: |
-    # my global config
-    global:
-      scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
-      evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
-      # scrape_timeout is set to the global default (10s).
-
-    # Alertmanager configuration
-    alerting:
-      alertmanagers:
-        - static_configs:
-            - targets: [internal_ip:19093]
-
-    # Load rules once and periodically evaluate them according to the global evaluation_interval.
-    rule_files:
-      - instance-down-rules.yml
-    # - first_rules.yml
-    # - second_rules.yml
-
-    # A scrape configuration containing exactly one endpoint to scrape:
-    # Here it's Prometheus itself.
-    scrape_configs:
-      # The job name is added as a label "job=job_name" to any timeseries scraped from this config.
-      # Monitored information captured by prometheus
-
-      # prometheus fetches application services
-      - job_name: node_exporter
-        static_configs:
-          - targets: [ internal_ip:20500 ]
-      - job_name: openimserver-openim-api
-        static_configs:
-          - targets: [ internal_ip:12002 ]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-msggateway
-        static_configs:
-          - targets: [ internal_ip:12140 ]
-    #      - targets: [ internal_ip:12140, internal_ip:12141, internal_ip:12142, internal_ip:12143, internal_ip:12144, internal_ip:12145, internal_ip:12146, internal_ip:12147, internal_ip:12148, internal_ip:12149, internal_ip:12150, internal_ip:12151, internal_ip:12152, internal_ip:12153, internal_ip:12154, internal_ip:12155 ]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-msgtransfer
-        static_configs:
-          - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027 ]
-    #      - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027, internal_ip:12028, internal_ip:12029, internal_ip:12030, internal_ip:12031, internal_ip:12032, internal_ip:12033, internal_ip:12034, internal_ip:12035 ]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-push
-        static_configs:
-          - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177 ]
-    #      - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180,  internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185, internal_ip:12186 ]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-auth
-        static_configs:
-          - targets: [ internal_ip:12200 ]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-conversation
-        static_configs:
-          - targets: [ internal_ip:12220 ]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-friend
-        static_configs:
-          - targets: [ internal_ip:12240 ]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-group
-        static_configs:
-          - targets: [ internal_ip:12260 ]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-msg
-        static_configs:
-          - targets: [ internal_ip:12280 ]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-third
-        static_configs:
-          - targets: [ internal_ip:12300 ]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-user
-        static_configs:
-          - targets: [ internal_ip:12320 ]
-            labels:
-              namespace: default
-
  webhooks.yml: |
    url: http://127.0.0.1:10006/callbackExample
    beforeSendSingleMsg:
@ -1056,6 +968,94 @@ data:
      enable: false
      timeout: 5

+  prometheus.yml: |
+    # my global config
+    global:
+      scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+      evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+      # scrape_timeout is set to the global default (10s).
+
+    # Alertmanager configuration
+    alerting:
+      alertmanagers:
+        - static_configs:
+            - targets: [internal_ip:19093]
+
+    # Load rules once and periodically evaluate them according to the global evaluation_interval.
+    rule_files:
+      - instance-down-rules.yml
+    # - first_rules.yml
+    # - second_rules.yml
+
+    # A scrape configuration containing exactly one endpoint to scrape:
+    # Here it's Prometheus itself.
+    scrape_configs:
+      # The job name is added as a label "job=job_name" to any timeseries scraped from this config.
+      # Monitored information captured by prometheus
+
+      # prometheus fetches application services
+      - job_name: node_exporter
+        static_configs:
+          - targets: [ internal_ip:20500 ]
+      - job_name: openimserver-openim-api
+        static_configs:
+          - targets: [ internal_ip:12002 ]
+            labels:
+              namespace: default
+      - job_name: openimserver-openim-msggateway
+        static_configs:
+          - targets: [ internal_ip:12140 ]
+    #      - targets: [ internal_ip:12140, internal_ip:12141, internal_ip:12142, internal_ip:12143, internal_ip:12144, internal_ip:12145, internal_ip:12146, internal_ip:12147, internal_ip:12148, internal_ip:12149, internal_ip:12150, internal_ip:12151, internal_ip:12152, internal_ip:12153, internal_ip:12154, internal_ip:12155 ]
+            labels:
+              namespace: default
+      - job_name: openimserver-openim-msgtransfer
+        static_configs:
+          - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027 ]
+    #      - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027, internal_ip:12028, internal_ip:12029, internal_ip:12030, internal_ip:12031, internal_ip:12032, internal_ip:12033, internal_ip:12034, internal_ip:12035 ]
+            labels:
+              namespace: default
+      - job_name: openimserver-openim-push
+        static_configs:
+          - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177 ]
+    #      - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180,  internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185, internal_ip:12186 ]
+            labels:
+              namespace: default
+      - job_name: openimserver-openim-rpc-auth
+        static_configs:
+          - targets: [ internal_ip:12200 ]
+            labels:
+              namespace: default
+      - job_name: openimserver-openim-rpc-conversation
+        static_configs:
+          - targets: [ internal_ip:12220 ]
+            labels:
+              namespace: default
+      - job_name: openimserver-openim-rpc-friend
+        static_configs:
+          - targets: [ internal_ip:12240 ]
+            labels:
+              namespace: default
+      - job_name: openimserver-openim-rpc-group
+        static_configs:
+          - targets: [ internal_ip:12260 ]
+            labels:
+              namespace: default
+      - job_name: openimserver-openim-rpc-msg
+        static_configs:
+          - targets: [ internal_ip:12280 ]
+            labels:
+              namespace: default
+      - job_name: openimserver-openim-rpc-third
+        static_configs:
+          - targets: [ internal_ip:12300 ]
+            labels:
+              namespace: default
+      - job_name: openimserver-openim-rpc-user
+        static_configs:
+          - targets: [ internal_ip:12320 ]
+            labels:
+              namespace: default
+
 ---
 apiVersion: v1
 kind: Secret
--- a/deployments/deploy/openim-crontask-deployment.yml
+++ b/deployments/deploy/openim-crontask-deployment.yml
@ -18,7 +18,6 @@ spec:
          env:
            - name: CONFIG_PATH
              value: "/config"
-
          volumeMounts:
            - name: openim-config
              mountPath: "/config"
--- a/deployments/deploy/openim-msggateway-deployment.yml
+++ b/deployments/deploy/openim-msggateway-deployment.yml
@ -1,16 +1,16 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: openim-msggateway-server
+  name: messagegateway-rpc-server
 spec:
  replicas: 2
  selector:
    matchLabels:
-      app: openim-msggateway-server
+      app: messagegateway-rpc-server
  template:
    metadata:
      labels:
-        app: openim-msggateway-server
+        app: messagegateway-rpc-server
    spec:
      containers:
        - name: openim-msggateway-container
@ -28,7 +28,7 @@ spec:
              mountPath: "/config"
              readOnly: true
          ports:
-            - containerPort: 10001
+            - containerPort: 10140
            - containerPort: 12001
      volumes:
        - name: openim-config
--- a/deployments/deploy/openim-msggateway-service.yml
+++ b/deployments/deploy/openim-msggateway-service.yml
@ -1,19 +1,19 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: openim-msggateway-service
+  name: messagegateway-rpc-service
 spec:
  selector:
-    app: openim-msggateway-server
+    app: messagegateway-rpc-server
  ports:
    - name: longConnServer-10001
      protocol: TCP
      port: 10001
      targetPort: 10001
-    - name: http-12001
+    - name: grpc-10140
      protocol: TCP
-      port: 12001
-      targetPort: 12001
+      port: 10140
+      targetPort: 10140
    - name: prometheus-12001
      protocol: TCP
      port: 12001
--- a/deployments/deploy/openim-push-deployment.yml
+++ b/deployments/deploy/openim-push-deployment.yml
@ -1,19 +1,19 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: openim-push-server
+  name: push-rpc-server
 spec:
  replicas: 2
  selector:
    matchLabels:
-      app: openim-push-server
+      app: push-rpc-server
  template:
    metadata:
      labels:
-        app: openim-push-server
+        app: push-rpc-server
    spec:
      containers:
-        - name: openim-push-container
+        - name: push-rpc-server-container
          image: openim/openim-push:v3.8.3
          env:
            - name: CONFIG_PATH
--- a/deployments/deploy/openim-push-service.yml
+++ b/deployments/deploy/openim-push-service.yml
@ -1,10 +1,10 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: openim-push-service
+  name: push-rpc-service
 spec:
  selector:
-    app: openim-push-server
+    app: push-rpc-server
  ports:
    - name: http-10170
      protocol: TCP
--- a/deployments/deploy/openim-rpc-conversation-deployment.yml
+++ b/deployments/deploy/openim-rpc-conversation-deployment.yml
@ -15,7 +15,6 @@ spec:
      containers:
        - name: conversation-rpc-server-container
          image: openim/openim-rpc-conversation:v3.8.3
-          imagePullPolicy: Never
          env:
            - name: CONFIG_PATH
              value: "/config"
--- a/deployments/deploy/openim-rpc-third-deployment.yml
+++ b/deployments/deploy/openim-rpc-third-deployment.yml
@ -18,16 +18,6 @@ spec:
          env:
            - name: CONFIG_PATH
              value: "/config"
-            - name: IMENV_REDIS_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: redis-secret
-                  key: redis-password
-            - name: IMENV_MONGODB_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mongo-secret
-                  key: mongo_openim_password
            - name: IMENV_MINIO_ACCESSKEYID
              valueFrom:
                secretKeyRef:
@ -38,6 +28,16 @@ spec:
                secretKeyRef:
                  name: minio-secret
                  key: minio-root-password
+            - name: IMENV_REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: redis-secret
+                  key: redis-password
+            - name: IMENV_MONGODB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mongo-secret
+                  key: mongo_openim_password
          volumeMounts:
            - name: openim-config
              mountPath: "/config"
--- a/deployments/deploy/prometheus-config.yml
+++ b/deployments/deploy/prometheus-config.yml
@ -1,112 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: prometheus-config
-data:
-  prometheus.yml: |
-    # my global config
-    global:
-      scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
-      evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
-      # scrape_timeout is set to the global default (10s).
-
-    # Alertmanager configuration
-    alerting:
-      alertmanagers:
-        - static_configs:
-            - targets: [internal_ip:19093]
-
-    # Load rules once and periodically evaluate them according to the global evaluation_interval.
-    rule_files:
-      - instance-down-rules.yml
-    # - first_rules.yml
-    # - second_rules.yml
-
-    # A scrape configuration containing exactly one endpoint to scrape:
-    # Here it's Prometheus itself.
-    scrape_configs:
-      # The job name is added as a label "job=job_name" to any timeseries scraped from this config.
-      # Monitored information captured by prometheus
-
-      # prometheus fetches application services
-      - job_name: node_exporter
-        static_configs:
-          - targets: [internal_ip:20500]
-      - job_name: openimserver-openim-api
-        static_configs:
-          - targets: [internal_ip:12002]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-msggateway
-        static_configs:
-          - targets: [internal_ip:12140]
-            #      - targets: [ internal_ip:12140, internal_ip:12141, internal_ip:12142, internal_ip:12143, internal_ip:12144, internal_ip:12145, internal_ip:12146, internal_ip:12147, internal_ip:12148, internal_ip:12149, internal_ip:12150, internal_ip:12151, internal_ip:12152, internal_ip:12153, internal_ip:12154, internal_ip:12155 ]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-msgtransfer
-        static_configs:
-          - targets:
-              [
-                internal_ip:12020,
-                internal_ip:12021,
-                internal_ip:12022,
-                internal_ip:12023,
-                internal_ip:12024,
-                internal_ip:12025,
-                internal_ip:12026,
-                internal_ip:12027,
-              ]
-            #      - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027, internal_ip:12028, internal_ip:12029, internal_ip:12030, internal_ip:12031, internal_ip:12032, internal_ip:12033, internal_ip:12034, internal_ip:12035 ]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-push
-        static_configs:
-          - targets:
-              [
-                internal_ip:12170,
-                internal_ip:12171,
-                internal_ip:12172,
-                internal_ip:12173,
-                internal_ip:12174,
-                internal_ip:12175,
-                internal_ip:12176,
-                internal_ip:12177,
-              ]
-            #      - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180,  internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185, internal_ip:12186 ]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-auth
-        static_configs:
-          - targets: [internal_ip:12200]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-conversation
-        static_configs:
-          - targets: [internal_ip:12220]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-friend
-        static_configs:
-          - targets: [internal_ip:12240]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-group
-        static_configs:
-          - targets: [internal_ip:12260]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-msg
-        static_configs:
-          - targets: [internal_ip:12280]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-third
-        static_configs:
-          - targets: [internal_ip:12300]
-            labels:
-              namespace: default
-      - job_name: openimserver-openim-rpc-user
-        static_configs:
-          - targets: [internal_ip:12320]
-            labels:
-              namespace: default