build: create changelog tool and workflows. (#2869)

.github/workflows/changelog.yml

@@ -0,0 +1,78 @@
+name: Release Changelog
+
+on:
+  release:
+    types: [released]
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  update-changelog:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Run Go Changelog Generator
+      run: |
+        # Run the Go changelog generator, passing the release tag if available
+        if [ "${{ github.event.release.tag_name }}" = "latest" ]; then
+          go run tools/changelog/changelog.go > "${{ github.event.release.tag_name }}-changelog.md"
+        else
+          go run tools/changelog/changelog.go "${{ github.event.release.tag_name }}" > "${{ github.event.release.tag_name }}-changelog.md"
+        fi
+
+    - name: Handle changelog files
+      run: |
+        # Ensure that the CHANGELOG directory exists
+        mkdir -p CHANGELOG
+
+        # Extract Major.Minor version by removing the 'v' prefix from the tag name
+        TAG_NAME=${{ github.event.release.tag_name }}
+        CHANGELOG_VERSION_NUMBER=$(echo "$TAG_NAME" | sed 's/^v//' | grep -oP '^\d+\.\d+')
+
+        # Define the new changelog file path
+        CHANGELOG_FILENAME="CHANGELOG-$CHANGELOG_VERSION_NUMBER.md"
+        CHANGELOG_PATH="CHANGELOG/$CHANGELOG_FILENAME"
+
+        # Check if the changelog file for the current release already exists
+        if [ -f "$CHANGELOG_PATH" ]; then
+          # If the file exists, append the new changelog to the existing one
+          cat "$CHANGELOG_PATH" >> "${TAG_NAME}-changelog.md"
+          # Overwrite the existing changelog with the updated content
+          mv "${TAG_NAME}-changelog.md" "$CHANGELOG_PATH"
+        else
+          # If the changelog file doesn't exist, rename the temp changelog file to the new changelog file
+          mv "${TAG_NAME}-changelog.md" "$CHANGELOG_PATH"
+
+          # Ensure that README.md exists
+          if [ ! -f "CHANGELOG/README.md" ]; then
+            echo -e "# CHANGELOGs\n\n" > CHANGELOG/README.md
+          fi
+          
+            # Add the new changelog entry at the top of the README.md
+            if ! grep -q "\[$CHANGELOG_FILENAME\]" CHANGELOG/README.md; then
+            sed -i "3i- [$CHANGELOG_FILENAME](./$CHANGELOG_FILENAME)" CHANGELOG/README.md
+            # Remove the extra newline character added by sed
+            # sed -i '4d' CHANGELOG/README.md
+            fi
+          fi
+
+    - name: Clean up
+      run: |
+        # Remove any temporary files that were created during the process
+        rm -f "${{ github.event.release.tag_name }}-changelog.md"
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v7.0.5
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        commit-message: "Update CHANGELOG for release ${{ github.event.release.tag_name }}"
+        title: "Update CHANGELOG for release ${{ github.event.release.tag_name }}"
+        body: "This PR updates the CHANGELOG files for release ${{ github.event.release.tag_name }}"
+        branch: changelog-${{ github.event.release.tag_name }} 
+        base: main 
+        delete-branch: true
+        labels: changelog

tools/changelog/changelog.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"regexp"
+	"strings"
+)
+
+// You can specify a tag as a command line argument to generate the changelog for a specific version.
+// Example: go run tools/changelog/changelog.go v0.0.33
+// If no tag is provided, the latest release will be used.
+
+// Setting repo owner and repo name by generate changelog
+const (
+	repoOwner = "openimsdk"
+	repoName  = "open-im-server"
+)
+
+// GitHubRepo struct represents the repo details.
+type GitHubRepo struct {
+	Owner         string
+	Repo          string
+	FullChangelog string
+}
+
+// ReleaseData represents the JSON structure for release data.
+type ReleaseData struct {
+	TagName   string `json:"tag_name"`
+	Body      string `json:"body"`
+	HtmlUrl   string `json:"html_url"`
+	Published string `json:"published_at"`
+}
+
+// Method to classify and format release notes.
+func (g *GitHubRepo) classifyReleaseNotes(body string) map[string][]string {
+	result := map[string][]string{
+		"feat":     {},
+		"fix":      {},
+		"chore":    {},
+		"refactor": {},
+		"build":    {},
+		"other":    {},
+	}
+
+	// Regular expression to extract PR number and URL (case insensitive)
+	rePR := regexp.MustCompile(`(?i)in (https://github\.com/[^\s]+/pull/(\d+))`)
+
+	// Split the body into individual lines.
+	lines := strings.Split(body, "\n")
+
+	for _, line := range lines {
+		// Skip lines that contain "deps: Merge"
+		if strings.Contains(strings.ToLower(line), "deps: merge #") {
+			continue
+		}
+
+		// Use a regular expression to extract Full Changelog link and its title (case insensitive).
+		if strings.Contains(strings.ToLower(line), "**full changelog**") {
+			matches := regexp.MustCompile(`(?i)\*\*full changelog\*\*: (https://github\.com/[^\s]+/compare/([^\s]+))`).FindStringSubmatch(line)
+			if len(matches) > 2 {
+				// Format the Full Changelog link with title
+				g.FullChangelog = fmt.Sprintf("[%s](%s)", matches[2], matches[1])
+			}
+			continue // Skip further processing for this line.
+		}
+
+		if strings.HasPrefix(line, "*") {
+			var category string
+
+			// Use strings.ToLower to make the matching case insensitive
+			lowerLine := strings.ToLower(line)
+
+			// Determine the category based on the prefix (case insensitive).
+			if strings.HasPrefix(lowerLine, "* feat") {
+				category = "feat"
+			} else if strings.HasPrefix(lowerLine, "* fix") {
+				category = "fix"
+			} else if strings.HasPrefix(lowerLine, "* chore") {
+				category = "chore"
+			} else if strings.HasPrefix(lowerLine, "* refactor") {
+				category = "refactor"
+			} else if strings.HasPrefix(lowerLine, "* build") {
+				category = "build"
+			} else {
+				category = "other"
+			}
+
+			// Extract PR number and URL (case insensitive)
+			matches := rePR.FindStringSubmatch(line)
+			if len(matches) == 3 {
+				prURL := matches[1]
+				prNumber := matches[2]
+				// Format the line with the PR link and use original content for the final result
+				formattedLine := fmt.Sprintf("* %s [#%s](%s)", strings.Split(line, " by ")[0][2:], prNumber, prURL)
+				result[category] = append(result[category], formattedLine)
+			} else {
+				// If no PR link is found, just add the line as is
+				result[category] = append(result[category], line)
+			}
+		}
+	}
+
+	return result
+}
+
+// Method to generate the final changelog.
+func (g *GitHubRepo) generateChangelog(tag, date, htmlURL, body string) string {
+	sections := g.classifyReleaseNotes(body)
+
+	// Convert ISO 8601 date to simpler format (YYYY-MM-DD)
+	formattedDate := date[:10]
+
+	// Changelog header with tag, date, and links.
+	changelog := fmt.Sprintf("## [%s](%s) \t(%s)\n\n", tag, htmlURL, formattedDate)
+
+	if len(sections["feat"]) > 0 {
+		changelog += "### New Features\n" + strings.Join(sections["feat"], "\n") + "\n\n"
+	}
+	if len(sections["fix"]) > 0 {
+		changelog += "### Bug Fixes\n" + strings.Join(sections["fix"], "\n") + "\n\n"
+	}
+	if len(sections["chore"]) > 0 {
+		changelog += "### Chores\n" + strings.Join(sections["chore"], "\n") + "\n\n"
+	}
+	if len(sections["refactor"]) > 0 {
+		changelog += "### Refactors\n" + strings.Join(sections["refactor"], "\n") + "\n\n"
+	}
+	if len(sections["build"]) > 0 {
+		changelog += "### Builds\n" + strings.Join(sections["build"], "\n") + "\n\n"
+	}
+	if len(sections["other"]) > 0 {
+		changelog += "### Others\n" + strings.Join(sections["other"], "\n") + "\n\n"
+	}
+
+	if g.FullChangelog != "" {
+		changelog += fmt.Sprintf("**Full Changelog**: %s\n", g.FullChangelog)
+	}
+
+	return changelog
+}
+
+// Method to fetch release data from GitHub API.
+func (g *GitHubRepo) fetchReleaseData(version string) (*ReleaseData, error) {
+	var apiURL string
+
+	if version == "" {
+		// Fetch the latest release.
+		apiURL = fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", g.Owner, g.Repo)
+	} else {
+		// Fetch a specific version.
+		apiURL = fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/tags/%s", g.Owner, g.Repo, version)
+	}
+
+	resp, err := http.Get(apiURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var releaseData ReleaseData
+	err = json.Unmarshal(body, &releaseData)
+	if err != nil {
+		return nil, err
+	}
+
+	return &releaseData, nil
+}
+
+func main() {
+	repo := &GitHubRepo{Owner: repoOwner, Repo: repoName}
+
+	// Get the version from command line arguments, if provided
+	var version string // Default is use latest
+
+	if len(os.Args) > 1 {
+		version = os.Args[1] // Use the provided version
+	}
+
+	// Fetch release data (either for latest or specific version)
+	releaseData, err := repo.fetchReleaseData(version)
+	if err != nil {
+		fmt.Println("Error fetching release data:", err)
+		return
+	}
+
+	// Generate and print the formatted changelog
+	changelog := repo.generateChangelog(releaseData.TagName, releaseData.Published, releaseData.HtmlUrl, releaseData.Body)
+	fmt.Println(changelog)
+}

tools/changelog/main.go

@@ -1,308 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"os/exec"
-	"regexp"
-	"sort"
-	"strings"
-)
-
-var (
-	mergeRequest   = regexp.MustCompile(`Merge pull request #([\d]+)`)
-	webconsoleBump = regexp.MustCompile(regexp.QuoteMeta("bump(github.com/openshift/origin-web-console): ") + `([\w]+)`)
-	upstreamKube   = regexp.MustCompile(`^UPSTREAM: (\d+)+:(.+)`)
-	upstreamRepo   = regexp.MustCompile(`^UPSTREAM: ([\w/-]+): (\d+)+:(.+)`)
-	prefix         = regexp.MustCompile(`^[\w-]: `)
-
-	assignments = []prefixAssignment{
-		{"cluster up", "cluster"},
-		{" pv ", "storage"},
-		{"haproxy", "router"},
-		{"router", "router"},
-		{"route", "route"},
-		{"authoriz", "auth"},
-		{"rbac", "auth"},
-		{"authent", "auth"},
-		{"reconcil", "auth"},
-		{"auth", "auth"},
-		{"role", "auth"},
-		{" dc ", "deploy"},
-		{"deployment", "deploy"},
-		{"rolling", "deploy"},
-		{"security context constr", "security"},
-		{"scc", "security"},
-		{"pipeline", "build"},
-		{"build", "build"},
-		{"registry", "registry"},
-		{"registries", "image"},
-		{"image", "image"},
-		{" arp ", "network"},
-		{" cni ", "network"},
-		{"egress", "network"},
-		{"network", "network"},
-		{"oc ", "cli"},
-		{"template", "template"},
-		{"etcd", "server"},
-		{"pod", "node"},
-		{"scripts/", "hack"},
-		{"e2e", "test"},
-		{"integration", "test"},
-		{"cluster", "cluster"},
-		{"master", "server"},
-		{"packages", "hack"},
-		{"api", "server"},
-	}
-)
-
-type prefixAssignment struct {
-	term   string
-	prefix string
-}
-
-type commit struct {
-	short   string
-	parents []string
-	message string
-}
-
-func contains(arr []string, value string) bool {
-	for _, s := range arr {
-		if s == value {
-			return true
-		}
-	}
-	return false
-}
-
-func main() {
-	log.SetFlags(0)
-	if len(os.Args) != 3 {
-		log.Fatalf("Must specify two arguments, FROM and TO")
-	}
-	from := os.Args[1]
-	to := os.Args[2]
-
-	out, err := exec.Command("git", "log", "--topo-order", "--pretty=tformat:%h %p|%s", "--reverse", fmt.Sprintf("%s..%s", from, to)).CombinedOutput()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	hide := make(map[string]struct{})
-	var apiChanges []string
-	var webconsole []string
-	var commits []commit
-	var upstreams []commit
-	var bumps []commit
-	for _, line := range strings.Split(string(out), "\n") {
-		if len(strings.TrimSpace(line)) == 0 {
-			continue
-		}
-		parts := strings.SplitN(line, "|", 2)
-		hashes := strings.Split(parts[0], " ")
-		c := commit{short: hashes[0], parents: hashes[1:], message: parts[1]}
-
-		if strings.HasPrefix(c.message, "UPSTREAM: ") {
-			hide[c.short] = struct{}{}
-			upstreams = append(upstreams, c)
-		}
-		if strings.HasPrefix(c.message, "bump(") {
-			hide[c.short] = struct{}{}
-			bumps = append(bumps, c)
-		}
-
-		if len(c.parents) == 1 {
-			commits = append(commits, c)
-			continue
-		}
-
-		matches := mergeRequest.FindStringSubmatch(line)
-		if len(matches) == 0 {
-			// this may have been a human pressing the merge button, we'll just record this as a direct push
-			continue
-		}
-
-		// split the accumulated commits into any that are force merges (assumed to be the initial set due
-		// to --topo-order) from the PR commits as soon as we see any of our merge parents. Then print
-		// any of the force merges
-		var first int
-		for i := range commits {
-			first = i
-			if contains(c.parents, commits[i].short) {
-				first++
-				break
-			}
-		}
-		individual := commits[:first]
-		merged := commits[first:]
-		for _, commit := range individual {
-			if len(commit.parents) > 1 {
-				continue
-			}
-			if _, ok := hide[commit.short]; ok {
-				continue
-			}
-			fmt.Printf("force-merge: %s %s\n", commit.message, commit.short)
-		}
-
-		// try to find either the PR title or the first commit title from the merge commit
-		out, err := exec.Command("git", "show", "--pretty=tformat:%b", c.short).CombinedOutput()
-		if err != nil {
-			log.Fatal(err)
-		}
-		var message string
-		para := strings.Split(string(out), "\n\n")
-		if len(para) > 0 && strings.HasPrefix(para[0], "Automatic merge from submit-queue") {
-			para = para[1:]
-		}
-		// this is no longer necessary with the submit queue in place
-		if len(para) > 0 && strings.HasPrefix(para[0], "Merged by ") {
-			para = para[1:]
-		}
-		// post submit-queue, the merge bot will add the PR title, which is usually pretty good
-		if len(para) > 0 {
-			message = strings.Split(para[0], "\n")[0]
-		}
-		if len(message) == 0 && len(merged) > 0 {
-			message = merged[0].message
-		}
-		if len(message) > 0 && len(merged) == 1 && message == merged[0].message {
-			merged = nil
-		}
-
-		// try to calculate a prefix based on the diff
-		if len(message) > 0 && !prefix.MatchString(message) {
-			prefix, ok := findPrefixFor(message, merged)
-			if ok {
-				message = prefix + ": " + message
-			}
-		}
-
-		// github merge
-
-		// has api changes
-		display := fmt.Sprintf("%s [\\#%s](https://github.com/openimsdk/Open-IM-Server/pull/%s)", message, matches[1], matches[1])
-		if hasFileChanges(c.short, "pkg/apistruct/") {
-			apiChanges = append(apiChanges, display)
-		}
-
-		var filtered []commit
-		for _, commit := range merged {
-			if _, ok := hide[commit.short]; ok {
-				continue
-			}
-			filtered = append(filtered, commit)
-		}
-		if len(filtered) > 0 {
-			fmt.Printf("- %s\n", display)
-			for _, commit := range filtered {
-				fmt.Printf("  - %s (%s)\n", commit.message, commit.short)
-			}
-		}
-
-		// stick the merge commit in at the beginning of the next list so we can anchor the previous parent
-		commits = []commit{c}
-	}
-
-	// chunk the bumps
-	var lines []string
-	for _, commit := range bumps {
-		if m := webconsoleBump.FindStringSubmatch(commit.message); len(m) > 0 {
-			webconsole = append(webconsole, m[1])
-			continue
-		}
-		lines = append(lines, commit.message)
-	}
-	lines = sortAndUniq(lines)
-	for _, line := range lines {
-		fmt.Printf("- %s\n", line)
-	}
-
-	// chunk the upstreams
-	lines = nil
-	for _, commit := range upstreams {
-		lines = append(lines, commit.message)
-	}
-	lines = sortAndUniq(lines)
-	for _, line := range lines {
-		fmt.Printf("- %s\n", upstreamLinkify(line))
-	}
-
-	if len(webconsole) > 0 {
-		fmt.Printf("- web: from %s^..%s\n", webconsole[0], webconsole[len(webconsole)-1])
-	}
-
-	for _, apiChange := range apiChanges {
-		fmt.Printf("  - %s\n", apiChange)
-	}
-}
-
-func findPrefixFor(message string, commits []commit) (string, bool) {
-	message = strings.ToLower(message)
-	for _, m := range assignments {
-		if strings.Contains(message, m.term) {
-			return m.prefix, true
-		}
-	}
-	for _, c := range commits {
-		if prefix, ok := findPrefixFor(c.message, nil); ok {
-			return prefix, ok
-		}
-	}
-	return "", false
-}
-
-func hasFileChanges(commit string, prefixes ...string) bool {
-	out, err := exec.Command("git", "diff", "--name-only", fmt.Sprintf("%s^..%s", commit, commit)).CombinedOutput()
-	if err != nil {
-		log.Fatal(err)
-	}
-	for _, file := range strings.Split(string(out), "\n") {
-		for _, prefix := range prefixes {
-			if strings.HasPrefix(file, prefix) {
-				return true
-			}
-		}
-	}
-	return false
-}
-
-func sortAndUniq(lines []string) []string {
-	sort.Strings(lines)
-	out := make([]string, 0, len(lines))
-	last := ""
-	for _, s := range lines {
-		if last == s {
-			continue
-		}
-		last = s
-		out = append(out, s)
-	}
-	return out
-}
-
-func upstreamLinkify(line string) string {
-	if m := upstreamKube.FindStringSubmatch(line); len(m) > 0 {
-		return fmt.Sprintf("UPSTREAM: [#%s](https://github.com/openimsdk/open-im-server/pull/%s):%s", m[1], m[1], m[2])
-	}
-	if m := upstreamRepo.FindStringSubmatch(line); len(m) > 0 {
-		return fmt.Sprintf("UPSTREAM: [%s#%s](https://github.com/%s/pull/%s):%s", m[1], m[2], m[1], m[2], m[3])
-	}
-	return line
-}