cicd: add e2e in ci & before build docker image (#2346)

* cicd: add e2e in ci & before build docker image * cicd: fix env minio addr * cicd: fix build image timing
3 months ago · e9b3a1952f
parent 7c3f6db0e3
commit e9b3a1952f
4 changed files with 169 additions and 290 deletions
--- a/.github/workflows/build-docker-image.yml
+++ b/.github/workflows/build-docker-image.yml
@ -19,8 +19,27 @@ on:
    branches:
      - main
      - release-*
    paths-ignore:
      - "docs/**"
      - "README.md"
      - "README_zh-CN.md"
      - "**.md"
      - "docs/**"
      - "CONTRIBUTING.md"
    tags:
      - v*
  pull_request:
    types: [closed]
    branches:
      - main
      - release-*
    paths-ignore:
      - "docs/**"
      - "README.md"
      - "README_zh-CN.md"
      - "**.md"
      - "docs/**"
      - "CONTRIBUTING.md"
  workflow_dispatch:
 env:
@ -29,109 +48,112 @@ env:
 jobs:
  build-dockerhub:
    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true)
    runs-on: ubuntu-latest
    if: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.merged == false) }}
    steps:
-      - name: Checkout
+      - name: Checkout main repository
        uses: actions/checkout@v4
        with:
          path: main-repo
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
-# docker.io/openim/openim-server:latest
+      - name: Build and push Docker image
-      - name: Extract metadata (tags, labels) for Docker
+        uses: docker/build-push-action@v5
        id: meta
        uses: docker/metadata-action@v5.5.1
        with:
-          images: openim/openim-server
+          context: ./main-repo
-          # generate Docker tags based on the following events/attributes
+          load: true
-          tags: |
+          tags: "openim/openim-server:local"
            type=ref,event=tag
            type=schedule
            type=ref,event=branch
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern=v{{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{major}}
            type=sha
-      - name: Log in to Docker Hub
+      - name: Checkout compose repository
-        uses: docker/login-action@v3
+        uses: actions/checkout@v4
        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
+          repository: "openimsdk/openim-docker"
-          password: ${{ secrets.DOCKER_PASSWORD }}
+          path: "compose-repo"
-      - name: Build and push Docker image
+      - name: Get Internal IP Address
-        uses: docker/build-push-action@v5
+        id: get-ip
-        with:
+        run: |
-          context: .
+          IP=$(hostname -I | awk '{print $1}')
-          # linux/ppc64le,linux/s390x
+          echo "The IP Address is: $IP"
-          platforms: linux/amd64,linux/arm64
+          echo "::set-output name=ip::$IP"
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
-  build-aliyun:
+      - name: Update .env to use the local image
-    runs-on: ubuntu-latest
+        run: |
-    steps:
+          sed -i 's|OPENIM_SERVER_IMAGE=.*|OPENIM_SERVER_IMAGE=openim/openim-server:local|' ${{ github.workspace }}/compose-repo/.env
-      - name: Checkout
+          sed -i 's|MINIO_EXTERNAL_ADDRESS=.*|MINIO_EXTERNAL_ADDRESS=http://${{ steps.get-ip.outputs.ip }}:10005|' ${{ github.workspace }}/compose-repo/.env
      - name: Start services using Docker Compose
        run: |
          cd ${{ github.workspace }}/compose-repo
          docker compose up -d
          sleep 30
      - name: Check openim-server health
        run: |
          timeout=300
          interval=30
          elapsed=0
          while [[ $elapsed -le $timeout ]]; do
            if ! docker exec openim-server mage check; then
              echo "openim-server is not ready, waiting..."
              sleep $interval
              elapsed=$(($elapsed + $interval))
            else
              echo "Health check successful"
              exit 0
            fi
          done
          echo "Health check failed after 5 minutes"
          exit 1
      - name: Check openim-chat health
        if: success()
        run: |
          if ! docker exec openim-chat mage check; then
              echo "openim-chat check failed"
              exit 1
            else
              echo "Health check successful"
              exit 0
            fi
      - name: Checkout e2e
        if: success()
        uses: actions/checkout@v4
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
 # registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server:latest
      - name: Extract metadata (tags, labels) for Docker
        id: meta2
        uses: docker/metadata-action@v5.5.1
        with:
-          images: registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server
+          repository: "openimsdk/test-e2e"
-          # generate Docker tags based on the following events/attributes
+          path: e2e-repo
          tags: |
            type=ref,event=tag
            type=schedule
            type=ref,event=branch
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern=v{{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{major}}
            type=sha
-      - name: Log in to AliYun Docker Hub
+      - name: Set up Python 3.9
-        uses: docker/login-action@v3
+        uses: actions/setup-python@v4
        with:
-          registry: registry.cn-hangzhou.aliyuncs.com
+          python-version: '3.9'
          username: ${{ secrets.ALIREGISTRY_USERNAME }}
          password: ${{ secrets.ALIREGISTRY_TOKEN }}
-      - name: Build and push Docker image
+      - name: Install dependencies
-        uses: docker/build-push-action@v5
+        run: |
-        with:
+          sudo apt-get update
-          context: .
+          sudo apt-get install -y xvfb libxi6 libgconf-2-4
-          # linux/ppc64le,linux/s390x
+          cd ${{ github.workspace }}/e2e-repo
-          platforms: linux/amd64,linux/arm64
+          pip install -r requirements.txt
-          push: ${{ github.event_name != 'pull_request' }}
+      
-          tags: ${{ steps.meta2.outputs.tags }}
+      - name: Run tests
-          labels: ${{ steps.meta2.outputs.labels }}
+        run: |
          cd ${{ github.workspace }}/e2e-repo
          xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' pytest -v -s ./script
  build-ghcr:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
 # ghcr.io/openimsdk/openim-server:latest
      - name: Extract metadata (tags, labels) for Docker
-        id: meta3
+        if: success()
        id: meta
        uses: docker/metadata-action@v5.5.1
        with:
-          images: ghcr.io/openimsdk/openim-server
+          images: |
            openim/openim-server
            ghcr.io/openimsdk/openim-server
            registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server
          # generate Docker tags based on the following events/attributes
          tags: |
            type=ref,event=tag
@ -144,19 +166,33 @@ jobs:
            type=semver,pattern={{major}}
            type=sha
      - name: Log in to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Log in to Aliyun Container Registry
        uses: docker/login-action@v2
        with:
          registry: registry.cn-hangzhou.aliyuncs.com
          username: ${{ secrets.ALIREGISTRY_USERNAME }}
          password: ${{ secrets.ALIREGISTRY_TOKEN }}
      - name: Build and push Docker image
        uses: docker/build-push-action@v5
        with:
-          context: .
+          context: ./main-repo
          push: true
          # linux/ppc64le,linux/s390x
          platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
-          tags: ${{ steps.meta3.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
-          labels: ${{ steps.meta3.outputs.labels }}
+      
--- a/.github/workflows/link-pr.yml
+++ b/.github/workflows/link-pr.yml
@ -1,53 +0,0 @@
 # Copyright © 2023 OpenIM open source community. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 name: Github Rebot for Link check error
 # Every Monday at 12:30 p.m
 on:
  schedule:
    - cron: '30 12 * * 1'
 jobs:
  linkChecker:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Link Checker
        id: lychee
        uses: lycheeverse/lychee-action@v1.9.3
        with:
            # For parameter description, see https://github.com/lycheeverse/lychee#commandline-parameters
            # Actions Link address -> https://github.com/lycheeverse/lychee-action
            # -E, --exclude-all-private    Exclude all private IPs from checking.
            # -i, --insecure               Proceed for server connections considered insecure (invalid TLS)
            # -n, --no-progress            Do not show progress bar.
            # -t, --timeout <timeout>      Website timeout in seconds from connect to response finished [default:20]
            # --max-concurrency <max-concurrency>    Maximum number of concurrent network requests [default: 128]
            # -a --accept <accept>                      Comma-separated list of accepted status codes for valid links
            # docs/.vitepress/dist the site directory to check
            # ./*.md all markdown files in the root directory
          args: --verbose  -E -i --no-progress --exclude-path './CHANGELOG' './**/*.md'
        env:
          GITHUB_TOKEN: ${{secrets.BOT_GITHUB_TOKEN}}
      - name: Create Issue From File
        if: env.lychee_exit_code != 0
        uses: peter-evans/create-issue-from-file@v5
        with:
          title: Bug reports for links in OpenIM docs
          content-filepath: ./lychee/out.md
          labels: kind/documentation, triage/unresolved, report
          token: ${{ secrets.BOT_GITHUB_TOKEN }}
--- a/.github/workflows/openimci.yml
+++ b/.github/workflows/openimci.yml
@ -71,6 +71,18 @@ jobs:
      run: sudo bash bootstrap.sh
      timeout-minutes: 20
    - name: Get Internal IP Address
      id: get-ip
      run: |
        IP=$(hostname -I | awk '{print $1}')
        echo "The IP Address is: $IP"
        echo "::set-output name=ip::$IP"
    - name: Update .env
      run: |
        sed -i 's|externalAddress:.*|externalAddress: "http://${{ steps.get-ip.outputs.ip }}:10005"|' config/minio.yml
        cat config/minio.yml
    - name: Build, Start, Check Services and Print Logs for Linux
      run: |
        sudo mage
@ -84,108 +96,40 @@ jobs:
        sudo mage start
        sudo mage check
    - name: Checkout chat repository
      uses: actions/checkout@v4
      with:
        repository: 'openimsdk/chat'
        path: 'chat-repo'
-#  build-mac:
+    - name: Build and Start Chat Services
-#    name: Execute OpenIM Script On macOS
+      run: |
-#    runs-on: macos-latest
+        cd ${{ github.workspace }}/chat-repo
-#    permissions:
+        sudo mage
-#      contents: write
+        sudo mage start
-#      pull-requests: write
+        sudo mage check
-#    environment:
+    
-#      name: openim
+    - name: Checkout e2e repository
-#    strategy:
+      uses: actions/checkout@v4
-#      matrix:
+      with:
-#        arch: [arm64, armv7, amd64]
+          repository: "openimsdk/test-e2e"
-#    
+          path: e2e-repo
 #    steps:
 #    - uses: actions/checkout@v3
 #    - name: Set up Go
 #      uses: actions/setup-go@v4
 #      with:
 #        go-version: '1.21'
 #        while ! docker system info > /dev/null 2>&1; do
 #          echo "Waiting for Docker to start..."
 #          sleep 10  # Increased delay to ensure Docker starts properly
 #        done
 #    - name: Install Docker
 #      run: |
 #        brew install docker
 #        brew install docker-compose
 #        sleep 10
 #        docker-compose up -d
 #        sleep 30
 #      timeout-minutes: 20 
 #        
 #    - name: init
 #      run: sudo bash bootstrap.sh
 #      timeout-minutes: 20
 #    - name: Build, Start, Check Services and Print Logs for Linux
 #      run: |
 #        sudo mage
 #        sudo mage start
 #        sudo mage check
 #    - name: Restart Services and Print Logs
 #      run: |
 #        sudo mage stop
 #        sudo mage start
 #        sudo mage check
 #  build-windows:
 #    name: Execute OpenIM Script On Windows
 #    runs-on: windows-latest
 #    permissions:
 #      contents: write
 #      pull-requests: write
 #    environment:
 #      name: openim
 #    strategy:
 #      matrix:
 #        arch: [arm64, armv7, amd64]
 #    
 #    steps:
 #    - uses: actions/checkout@v3
 #    - name: Set up Go
 #      uses: actions/setup-go@v4
 #      with:
 #        go-version: '1.21'
 #    - name: Set up Docker for Windows
 #      run: |
 #        $images = @("zookeeper", "redis", "kafka")
 #        foreach ($image in $images) {
 #          $tag = "$image:latest"
 #          docker pull $tag | Out-Null
 #          if ($LASTEXITCODE -ne 0) {
 #            Write-Host "Skipping $image as it is not available for Windows"
 #          } else {
 #            Write-Host "Successfully pulled $image"
 #          }
 #        }
 #        docker compose up -d
 #        Start-Sleep -Seconds 30
 #      timeout-minutes: 20
 #      shell: pwsh
 #    - name: init
 #      run: bootstrap.bat
 #      timeout-minutes: 20
 #    - name: Build, Start, Check Services and Print Logs for Linux
 #      run: |
 #        mage
 #        mage start
 #        mage check        
-#    - name: Restart Services and Print Logs
+    - name: Set up Python 3.9
-#      run: |
+      uses: actions/setup-python@v4
-#        mage stop
+      with:
-#        mage start
+        python-version: '3.9'
-#        mage check
+    
    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y xvfb libxi6 libgconf-2-4
        cd ${{ github.workspace }}/e2e-repo
        pip install -r requirements.txt
    - name: Run tests
      run: |
        cd ${{ github.workspace }}/e2e-repo
        xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' pytest -v -s ./script
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@ -1,48 +0,0 @@
 # Copyright © 2023 OpenIM. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
 #
 # You can adjust the behavior by modifying this file.
 # For more information, see:
 # https://github.com/actions/stale
 name: Mark stale issues and pull requests
 on:
  schedule:
  - cron: '0 8 * * 1'
 jobs:
  stale:
    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write
    steps:
    - uses: actions/stale@v9
      with:
        repo-token: ${{ secrets.BOT_GITHUB_TOKEN }}
        days-before-stale: 60
        days-before-close: 305
        stale-issue-message: 'This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
        stale-pr-message: 'This issue is stale because it has been open 60 days with no activity.'
        close-issue-message: 'This issue was closed because it has been stalled for 7 days with no activity.'
        close-pr-message: 'This PR was closed because it has been stalled for 7 days with no activity. You can reopen it if you want.'
        stale-pr-label: lifecycle/stale
        stale-issue-label: lifecycle/stale
        exempt-issue-labels: 'openim'
        exempt-pr-labels: 'openim'
        exempt-draft-pr: true