From 31157313ebf5a6e1f9cd15236cd9828cb7a0189c Mon Sep 17 00:00:00 2001 From: Michael Li Date: Mon, 13 Feb 2023 16:20:30 +0800 Subject: [PATCH 1/5] update proposal and roadmap --- ROADMAP.md | 21 +++++++++++----- ...作为数据逻辑层ORM的构想 copy.md} | 0 ...作为用户密码加密算法的设计.md | 25 +++++++++++++++++++ 3 files changed, 40 insertions(+), 6 deletions(-) rename docs/proposal/{014-关于paopao-ce引入sqlx与sqlc作为数据逻辑层ORM的构想.md => 014-关于paopao-ce引入sqlx与sqlc作为数据逻辑层ORM的构想 copy.md} (100%) create mode 100644 docs/proposal/015-关于paopao-ce引入bcrypt作为用户密码加密算法的设计.md diff --git a/ROADMAP.md b/ROADMAP.md index 5f3e1f06..aa674e06 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,11 +1,14 @@ ## Roadmap for paopao-ce -[paopao-ce](https://github.com/rocboss/paopao-ce/tree/dev)/[paopao-plus](https://github.com/rocboss/paopao-ce/tree/r/paopao-plus)/[paopao-pro](https://github.com/rocboss/paopao-ce/tree/r/paopao-pro) features develop or optimize and bug fix roadmap. +[paopao-ce](https://github.com/rocboss/paopao-ce/tree/dev)/[paopao-ce-plus](https://github.com/rocboss/paopao-ce/tree/r/paopao-ce-plus)/[paopao-ce-pro](https://github.com/rocboss/paopao-ce/tree/r/paopao-ce-pro) features develop or optimize and bug fix roadmap. -### paopao-ce roadmap +## paopao-ce roadmap #### v0.3.0 * [ ] add `Followship` feature +* [ ] add `Auth:Bcrypt` feature +* [ ] add `Auth:MD5` feature (just for compatible) * [ ] optimize media tweet submit logic * [ ] optimize search logic service +* [ ] remove `Deprecated:OldWeb` feature #### v0.2.0 * [x] add `Friendship` feature @@ -37,10 +40,16 @@ * [ ] optimize topics service * [ ] optimize backend data logic service(optimize database CRUD operate) -### paopao-plus roadmap -#### v0.3.0 +## paopao-ce-plus roadmap +#### paopao-ce-plus/v0.2.0 +* [ ] adapt for paopao-ce v0.2.0 + +#### paopao-ce-plus/v0.3.0 * [ ] adapt for paopao-ce v0.3.0 -### paopao-pro roadmap -#### v0.3.0 +## paopao-ce-pro roadmap +#### paopao-ce-pro/v0.2.0 +* [ ] adapt for paopao-ce v0.2.0 + +#### paopao-ce-pro/v0.3.0 * [ ] adapt for paopao-ce v0.3.0 \ No newline at end of file diff --git a/docs/proposal/014-关于paopao-ce引入sqlx与sqlc作为数据逻辑层ORM的构想.md b/docs/proposal/014-关于paopao-ce引入sqlx与sqlc作为数据逻辑层ORM的构想 copy.md similarity index 100% rename from docs/proposal/014-关于paopao-ce引入sqlx与sqlc作为数据逻辑层ORM的构想.md rename to docs/proposal/014-关于paopao-ce引入sqlx与sqlc作为数据逻辑层ORM的构想 copy.md diff --git a/docs/proposal/015-关于paopao-ce引入bcrypt作为用户密码加密算法的设计.md b/docs/proposal/015-关于paopao-ce引入bcrypt作为用户密码加密算法的设计.md new file mode 100644 index 00000000..3bd02a48 --- /dev/null +++ b/docs/proposal/015-关于paopao-ce引入bcrypt作为用户密码加密算法的设计.md @@ -0,0 +1,25 @@ +| 编号 | 作者 | 发表时间 | 变更时间 | 版本 | 状态 | +| ----- | ----- | ----- | ----- | ----- | ----- | +| 015| 北野 | 2023-02-13 | 2023-02-13 | v0.0 | 提议 | + +### 概述 +TODO； + +### 需求 +TODO； + +### 方案 +TODO； + +### 疑问 + +1. 为什么要引入bcrypt？ +TODO； + +### 参考文档 +[bcrypt](https://github.com/golang/crypto/blob/master/bcrypt/bcrypt.go) +[bcrypt module](https://pkg.go.dev/golang.org/x/crypto@v0.6.0/bcrypt) + +### 更新记录 +#### v0.0(2023-02-13) - 北野 +* 初始文档， 先占个位置 From d583d1649cf7918f315d01860f9bafb1401383b2 Mon Sep 17 00:00:00 2001 From: Michael Li Date: Mon, 13 Feb 2023 22:49:33 +0800 Subject: [PATCH 2/5] sqlx: fixed select limit/offset sql clause error in MySQL/Sqlite3 --- internal/core/topics.go | 2 +- internal/dao/sakila/topics.go | 14 +++++++------- internal/servants/web/broker/post.go | 2 +- internal/servants/web/pub.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/core/topics.go b/internal/core/topics.go index cbcc046a..9484baeb 100644 --- a/internal/core/topics.go +++ b/internal/core/topics.go @@ -12,6 +12,6 @@ import ( type TopicService interface { UpsertTags(userId int64, tags []string) (cs.TagInfoList, error) DecrTagsById(ids []int64) error - ListTags(typ cs.TagType, offset int, limit int) (cs.TagList, error) + ListTags(typ cs.TagType, limit int, offset int) (cs.TagList, error) TagsByKeyword(keyword string) (cs.TagInfoList, error) } diff --git a/internal/dao/sakila/topics.go b/internal/dao/sakila/topics.go index 44bbebf0..a23ebf01 100644 --- a/internal/dao/sakila/topics.go +++ b/internal/dao/sakila/topics.go @@ -100,12 +100,12 @@ func (s *topicServant) DecrTagsById(ids []int64) error { }) } -func (s *topicServant) ListTags(typ cs.TagType, offset int, limit int) (res cs.TagList, err error) { +func (s *topicServant) ListTags(typ cs.TagType, limit int, offset int) (res cs.TagList, err error) { switch typ { case cs.TagTypeHot: - err = s.stmtHotTags.Select(&res, offset, limit) + err = s.stmtHotTags.Select(&res, limit, offset) case cs.TagTypeNew: - err = s.stmtNewestTags.Select(&res, offset, limit) + err = s.stmtNewestTags.Select(&res, limit, offset) } return } @@ -123,10 +123,10 @@ func (s *topicServant) TagsByKeyword(keyword string) (res cs.TagInfoList, err er func newTopicService(db *sqlx.DB) core.TopicService { return &topicServant{ sqlxServant: newSqlxServant(db), - stmtNewestTags: c(`SELECT t.id id, t.user_id user_id, t.tag tag, t.quote_num quote_num, u.id, u.nickname, u.username, u.status, u.avatar, u.is_admin FROM @tag t JOIN @user u ON t.user_id = u.id WHERE t.is_del = 0 AND t.quote_num > 0 ORDER BY t.id DESC OFFSET ? LIMIT ?`), - stmtHotTags: c(`SELECT t.id id, t.user_id user_id, t.tag tag, t.quote_num quote_num, u.id, u.nickname, u.username, u.status, u.avatar, u.is_admin FROM @tag t JOIN @user u ON t.user_id = u.id WHERE t.is_del = 0 AND t.quote_num > 0 ORDER BY t.quote_num DESC OFFSET ? LIMIT ?`), - stmtTagsByKeywordA: c(`SELECT id, user_id, tag, quote_num FROM @tag WHERE is_del = 0 ORDER BY quote_num DESC OFFSET 0 LIMIT 6`), - stmtTagsByKeywordB: c(`SELECT id, user_id, tag, quote_num FROM @tag WHERE is_del = 0 AND tag LIKE ? ORDER BY quote_num DESC OFFSET 0 LIMIT 6`), + stmtNewestTags: c(`SELECT t.id id, t.user_id user_id, t.tag tag, t.quote_num quote_num, u.id, u.nickname, u.username, u.status, u.avatar, u.is_admin FROM @tag t JOIN @user u ON t.user_id = u.id WHERE t.is_del = 0 AND t.quote_num > 0 ORDER BY t.id DESC LIMIT ? OFFSET ?`), + stmtHotTags: c(`SELECT t.id id, t.user_id user_id, t.tag tag, t.quote_num quote_num, u.id, u.nickname, u.username, u.status, u.avatar, u.is_admin FROM @tag t JOIN @user u ON t.user_id = u.id WHERE t.is_del = 0 AND t.quote_num > 0 ORDER BY t.quote_num DESC LIMIT ? OFFSET ?`), + stmtTagsByKeywordA: c(`SELECT id, user_id, tag, quote_num FROM @tag WHERE is_del = 0 ORDER BY quote_num DESC LIMIT 6`), + stmtTagsByKeywordB: c(`SELECT id, user_id, tag, quote_num FROM @tag WHERE is_del = 0 AND tag LIKE ? ORDER BY quote_num DESC LIMIT 6`), stmtInsertTag: c(`INSERT INTO @tag (user_id, tag, created_on, modified_on, quote_num) VALUES (?, ?, ?, ?, 1)`), sqlTagsByIdA: r(`SELECT id FROM @tag WHERE id IN (?) AND is_del = 0 AND quote_num > 0`), sqlTagsByIdB: r(`SELECT id, user_id, tag, quote_num FROM @tag WHERE id IN (?)`), diff --git a/internal/servants/web/broker/post.go b/internal/servants/web/broker/post.go index 12ae46de..e177dd27 100644 --- a/internal/servants/web/broker/post.go +++ b/internal/servants/web/broker/post.go @@ -535,7 +535,7 @@ func GetPostTags(param *PostTagsReq) (cs.TagList, error) { if num > conf.AppSetting.MaxPageSize { num = conf.AppSetting.MaxPageSize } - tags, err := ds.ListTags(param.Type, 0, num) + tags, err := ds.ListTags(param.Type, num, 0) if err != nil { return nil, err } diff --git a/internal/servants/web/pub.go b/internal/servants/web/pub.go index 2b10ad4a..abe82ad9 100644 --- a/internal/servants/web/pub.go +++ b/internal/servants/web/pub.go @@ -75,7 +75,7 @@ func (s *pubSrv) TopicList(req *web.TopicListReq) (*web.TopicListResp, mir.Error if num > conf.AppSetting.MaxPageSize { num = conf.AppSetting.MaxPageSize } - tags, err := s.Ds.ListTags(req.Type, 0, num) + tags, err := s.Ds.ListTags(req.Type, num, 0) if err != nil { return nil, _errGetPostTagsFailed } From f0eb47b0641bfedff8078e07de24b6bba4d3fe80 Mon Sep 17 00:00:00 2001 From: Michael Li Date: Wed, 15 Feb 2023 09:14:31 +0800 Subject: [PATCH 3/5] upgrade github.com/alimy/mir/v3 => v3.1.0 --- go.mod | 2 +- go.sum | 4 ++-- mirc/web/v1/alipay.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index d8d4272e..dc419341 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/afocus/captcha v0.0.0-20191010092841-4bd1f21c8868 github.com/alimy/cfg v0.3.0 - github.com/alimy/mir/v3 v3.0.1 + github.com/alimy/mir/v3 v3.1.0 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/allegro/bigcache/v3 v3.0.2 github.com/bytedance/sonic v1.5.0 diff --git a/go.sum b/go.sum index 62851846..dfe21b6a 100644 --- a/go.sum +++ b/go.sum @@ -147,8 +147,8 @@ github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:C github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/alimy/cfg v0.3.0 h1:9xgA0QWVCPSq9fFNRcYahVCAX22IL9ts2wrTQPfAStY= github.com/alimy/cfg v0.3.0/go.mod h1:rOxbasTH2srl6StAjNF5Vyi8bfrdkl3fLGmOYtSw81c= -github.com/alimy/mir/v3 v3.0.1 h1:70J/KMiEUDjpXfE2myqJ9aw408rshHdTKcN5yH74xqM= -github.com/alimy/mir/v3 v3.0.1/go.mod h1:ybhT2ijOiDn0lLwWzIY6vXdv+uzZrctS7VFfczcXBWU= +github.com/alimy/mir/v3 v3.1.0 h1:zk8Afsi1rcbRpL8MdGCkKqWqqLUupeSY9NcJgnXxic8= +github.com/alimy/mir/v3 v3.1.0/go.mod h1:ybhT2ijOiDn0lLwWzIY6vXdv+uzZrctS7VFfczcXBWU= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible h1:9gWa46nstkJ9miBReJcN8Gq34cBFbzSpQZVVT9N09TM= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= diff --git a/mirc/web/v1/alipay.go b/mirc/web/v1/alipay.go index 86ab6936..63b763a3 100644 --- a/mirc/web/v1/alipay.go +++ b/mirc/web/v1/alipay.go @@ -7,7 +7,7 @@ import ( ) func init() { - AddEntries(new(AlipayPub), new(AlipayPriv)) + AddEntry(new(AlipayPub), new(AlipayPriv)) } // AlipayPub 支付宝相关不用授权的服务 From a821f47dede4cc397dcac4fcaf55d30076701ed8 Mon Sep 17 00:00:00 2001 From: Michael Li Date: Wed, 15 Feb 2023 17:48:31 +0800 Subject: [PATCH 4/5] add Pyroscope support --- README.md | 27 +++++++++++ ROADMAP.md | 11 +++-- auto/api/m/v1/user.go | 2 +- auto/api/r/v1/user.go | 2 +- auto/api/s/v1/user.go | 2 +- auto/api/v1/admin.go | 2 +- auto/api/v1/alipay_priv.go | 2 +- auto/api/v1/alipay_pub.go | 2 +- auto/api/v1/core.go | 2 +- auto/api/v1/followship.go | 2 +- auto/api/v1/friendship.go | 2 +- auto/api/v1/loose.go | 2 +- auto/api/v1/priv.go | 2 +- auto/api/v1/pub.go | 2 +- auto/api/x/v1/user.go | 2 +- docker-compose.yaml | 10 ++++ ...作为用户密码加密算法的设计.md | 4 +- ...用pyroscope用于性能调试的设计.md | 43 +++++++++++++++++ features-status.md | 9 +++- go.mod | 4 +- go.sum | 8 +++- internal/conf/conf.go | 2 + internal/conf/config.yaml | 5 ++ internal/conf/settting.go | 18 +++++++ main.go | 7 ++- pkg/debug/pyroscope.go | 19 ++++++++ pkg/debug/pyroscope_embed.go | 48 +++++++++++++++++++ 27 files changed, 215 insertions(+), 26 deletions(-) create mode 100644 docs/proposal/016-关于使用pyroscope用于性能调试的设计.md create mode 100644 pkg/debug/pyroscope.go create mode 100644 pkg/debug/pyroscope_embed.go diff --git a/README.md b/README.md index 9264a7ab..16c5011c 100644 --- a/README.md +++ b/README.md @@ -370,6 +370,7 @@ release/paopao-ce --no-default-features --features sqlite3,localoss,loggerfile,r |`Alipay` | 支付 | 稳定 | 开启基于[支付宝开放平台](https://open.alipay.com/)的钱包功能 | |`Sms` | 短信验证 | 稳定 | 开启短信验证码功能，用于手机绑定验证手机是否注册者的；功能如果没有开启，手机绑定时任意短信验证码都可以绑定手机 | |`Docs:OpenAPI` | 开发文档 | 稳定 | 开启openapi文档功能，提供web api文档说明(visit http://127.0.0.1:8008/docs/openapi) | +|[`Pyroscope`](docs/proposal/016-关于使用pyroscope用于性能调试的设计.md)| 性能优化 | 内测 | 开启Pyroscope功能用于性能调试 | |`PhoneBind` | 其他 | 稳定 | 手机绑定功能 | > 功能项状态详情参考 [features-status](features-status.md). @@ -492,6 +493,32 @@ MinIO: # MinIO 存储配置 ... ``` +#### [Pyroscope](https://github.com/pyroscope-io/pyroscope) 性能剖析 +* Pyroscope运行 +```sh +mkdir -p data/minio/data + +# 使用Docker运行 +docker run -it -p 4040:4040 pyroscope/pyroscope:latest server +# 使用docker compose运行， 需要删除docker-compose.yaml中关于pyroscope的注释 +docker compose up -d pyroscope +# visit http://loclahost:4040 +``` + +* 修改Pyroscope配置 +```yaml +# features中加上 MinIO +Features: + Default: ["Meili", "LoggerMeili", "Base", "Sqlite3", "BigCacheIndex", "Pyroscope"] +... +Pyroscope: # Pyroscope配置 + AppName: "paopao-ce" + Endpoint: "http://localhost:4040" # Pyroscope server address + AuthToken: # Pyroscope authentication token + Logger: none # Pyroscope logger (standard | logrus | none) +... +``` + ### 源代码分支管理 **主代码库`github.com/rocboss/paopao-ce`** ```bash diff --git a/ROADMAP.md b/ROADMAP.md index aa674e06..47eb091c 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -13,6 +13,7 @@ #### v0.2.0 * [x] add `Friendship` feature * [x] add `Lightship` feature +* [x] add `Pyroscope` feature * [ ] add extend base ORM code for implement data logic base sqlx/sqlc * [x] add new `Web` service * [x] add `Frontend:Web` feature @@ -41,15 +42,15 @@ * [ ] optimize backend data logic service(optimize database CRUD operate) ## paopao-ce-plus roadmap +#### paopao-ce-plus/v0.3.0 +* [ ] adapt for paopao-ce v0.3.0 + #### paopao-ce-plus/v0.2.0 * [ ] adapt for paopao-ce v0.2.0 -#### paopao-ce-plus/v0.3.0 +## paopao-ce-pro roadmap +#### paopao-ce-pro/v0.3.0 * [ ] adapt for paopao-ce v0.3.0 -## paopao-ce-pro roadmap #### paopao-ce-pro/v0.2.0 * [ ] adapt for paopao-ce v0.2.0 - -#### paopao-ce-pro/v0.3.0 -* [ ] adapt for paopao-ce v0.3.0 \ No newline at end of file diff --git a/auto/api/m/v1/user.go b/auto/api/m/v1/user.go index 58c6fcae..394b86b7 100644 --- a/auto/api/m/v1/user.go +++ b/auto/api/m/v1/user.go @@ -1,6 +1,6 @@ // Code generated by go-mir. DO NOT EDIT. // versions: -// - mir v3.0.1 +// - mir v3.1.1 package v1 diff --git a/auto/api/r/v1/user.go b/auto/api/r/v1/user.go index 8f1a81a0..40782a83 100644 --- a/auto/api/r/v1/user.go +++ b/auto/api/r/v1/user.go @@ -1,6 +1,6 @@ // Code generated by go-mir. DO NOT EDIT. // versions: -// - mir v3.0.1 +// - mir v3.1.1 package v1 diff --git a/auto/api/s/v1/user.go b/auto/api/s/v1/user.go index 6b1de404..f4a7c1dd 100644 --- a/auto/api/s/v1/user.go +++ b/auto/api/s/v1/user.go @@ -1,6 +1,6 @@ // Code generated by go-mir. DO NOT EDIT. // versions: -// - mir v3.0.1 +// - mir v3.1.1 package v1 diff --git a/auto/api/v1/admin.go b/auto/api/v1/admin.go index 277993fa..27b7a999 100644 --- a/auto/api/v1/admin.go +++ b/auto/api/v1/admin.go @@ -1,6 +1,6 @@ // Code generated by go-mir. DO NOT EDIT. // versions: -// - mir v3.0.1 +// - mir v3.1.1 package v1 diff --git a/auto/api/v1/alipay_priv.go b/auto/api/v1/alipay_priv.go index 60347ef6..c9784968 100644 --- a/auto/api/v1/alipay_priv.go +++ b/auto/api/v1/alipay_priv.go @@ -1,6 +1,6 @@ // Code generated by go-mir. DO NOT EDIT. // versions: -// - mir v3.0.1 +// - mir v3.1.1 package v1 diff --git a/auto/api/v1/alipay_pub.go b/auto/api/v1/alipay_pub.go index ebfee808..7ac65188 100644 --- a/auto/api/v1/alipay_pub.go +++ b/auto/api/v1/alipay_pub.go @@ -1,6 +1,6 @@ // Code generated by go-mir. DO NOT EDIT. // versions: -// - mir v3.0.1 +// - mir v3.1.1 package v1 diff --git a/auto/api/v1/core.go b/auto/api/v1/core.go index c29c9140..4685ceba 100644 --- a/auto/api/v1/core.go +++ b/auto/api/v1/core.go @@ -1,6 +1,6 @@ // Code generated by go-mir. DO NOT EDIT. // versions: -// - mir v3.0.1 +// - mir v3.1.1 package v1 diff --git a/auto/api/v1/followship.go b/auto/api/v1/followship.go index cd87c7ad..971ce7d6 100644 --- a/auto/api/v1/followship.go +++ b/auto/api/v1/followship.go @@ -1,6 +1,6 @@ // Code generated by go-mir. DO NOT EDIT. // versions: -// - mir v3.0.1 +// - mir v3.1.1 package v1 diff --git a/auto/api/v1/friendship.go b/auto/api/v1/friendship.go index 47f50632..ef9e6fbe 100644 --- a/auto/api/v1/friendship.go +++ b/auto/api/v1/friendship.go @@ -1,6 +1,6 @@ // Code generated by go-mir. DO NOT EDIT. // versions: -// - mir v3.0.1 +// - mir v3.1.1 package v1 diff --git a/auto/api/v1/loose.go b/auto/api/v1/loose.go index da634068..832f6fc3 100644 --- a/auto/api/v1/loose.go +++ b/auto/api/v1/loose.go @@ -1,6 +1,6 @@ // Code generated by go-mir. DO NOT EDIT. // versions: -// - mir v3.0.1 +// - mir v3.1.1 package v1 diff --git a/auto/api/v1/priv.go b/auto/api/v1/priv.go index 9e6f0b04..a29d2858 100644 --- a/auto/api/v1/priv.go +++ b/auto/api/v1/priv.go @@ -1,6 +1,6 @@ // Code generated by go-mir. DO NOT EDIT. // versions: -// - mir v3.0.1 +// - mir v3.1.1 package v1 diff --git a/auto/api/v1/pub.go b/auto/api/v1/pub.go index 8b7b7845..3417cacd 100644 --- a/auto/api/v1/pub.go +++ b/auto/api/v1/pub.go @@ -1,6 +1,6 @@ // Code generated by go-mir. DO NOT EDIT. // versions: -// - mir v3.0.1 +// - mir v3.1.1 package v1 diff --git a/auto/api/x/v1/user.go b/auto/api/x/v1/user.go index fa8a47c5..a7b16a8c 100644 --- a/auto/api/x/v1/user.go +++ b/auto/api/x/v1/user.go @@ -1,6 +1,6 @@ // Code generated by go-mir. DO NOT EDIT. // versions: -// - mir v3.0.1 +// - mir v3.1.1 package v1 diff --git a/docker-compose.yaml b/docker-compose.yaml index d458f94e..6f40f0eb 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -76,6 +76,16 @@ services: # networks: # - paopao-network + pyroscope: + image: pyroscope/pyroscope:latest + restart: always + ports: + - 4040:4040 + command: + - 'server' + networks: + - paopao-network + phpmyadmin: image: phpmyadmin:5.2 depends_on: diff --git a/docs/proposal/015-关于paopao-ce引入bcrypt作为用户密码加密算法的设计.md b/docs/proposal/015-关于paopao-ce引入bcrypt作为用户密码加密算法的设计.md index 3bd02a48..6ae0367a 100644 --- a/docs/proposal/015-关于paopao-ce引入bcrypt作为用户密码加密算法的设计.md +++ b/docs/proposal/015-关于paopao-ce引入bcrypt作为用户密码加密算法的设计.md @@ -17,8 +17,8 @@ TODO； TODO； ### 参考文档 -[bcrypt](https://github.com/golang/crypto/blob/master/bcrypt/bcrypt.go) -[bcrypt module](https://pkg.go.dev/golang.org/x/crypto@v0.6.0/bcrypt) +* [bcrypt](https://github.com/golang/crypto/blob/master/bcrypt/bcrypt.go) +* [bcrypt module](https://pkg.go.dev/golang.org/x/crypto@v0.6.0/bcrypt) ### 更新记录 #### v0.0(2023-02-13) - 北野 diff --git a/docs/proposal/016-关于使用pyroscope用于性能调试的设计.md b/docs/proposal/016-关于使用pyroscope用于性能调试的设计.md new file mode 100644 index 00000000..d2313bc7 --- /dev/null +++ b/docs/proposal/016-关于使用pyroscope用于性能调试的设计.md @@ -0,0 +1,43 @@ +| 编号 | 作者 | 发表时间 | 变更时间 | 版本 | 状态 | +| ----- | ----- | ----- | ----- | ----- | ----- | +| 016| 北野 | 2023-02-15 | 2023-02-15 | v1.0 | 提议 | + +### 概述 +Pyroscope 是一个开源的持续性能剖析平台。它能够帮你： +* 找出源代码中的性能问题和瓶颈 +* 解决 CPU 利用率高的问题 +* 理解应用程序的调用树（call tree） +* 追踪随一段时间内变化的情况 + +### 需求 +* 开发环境下启用Pyroscope，但是部署环境下禁用。 + +### 方案 +* 使用//go:build pyroscope 可选编译app集成Pyroscope功能 +* config.yaml中添加`Pyroscope` 功能来启用Pyroscope + +### 疑问 + +1. 为什么要引入Pyroscope？ +用于开发环境下对paopao-ce进行性能优化。 + +2. 如何开启这个功能？ +在配置文件config.yaml中的`Features`中添加`Pyroscope`功能项开启该功能： + ```yaml + ... + # features中加上 Friendship + Features: + Default: ["Meili", "LoggerMeili", "Base", "Sqlite3", "BigCacheIndex", "MinIO", "Pyroscope"] + Base: ["Redis", "PhoneBind"] + ... + ``` + + + +### 参考文档 +* [pyroscope](https://github.com/pyroscope-io/pyroscope) +* [pyroscope client](https://github.com/pyroscope-io/client) + +### 更新记录 +#### v1.0(2023-02-15) - 北野 +* 初始文档 diff --git a/features-status.md b/features-status.md index a1432069..132121da 100644 --- a/features-status.md +++ b/features-status.md @@ -164,10 +164,15 @@ * [x] 业务逻辑实现 ### 开发文档: -* `Docs:OpenAPI` 开启openapi文档功能，提供web api文档说明(visit http://127.0.0.1:8008/docs/openapi)； +* `Docs:OpenAPI` 开启openapi文档功能，提供web api文档说明(visit http://127.0.0.1:8008/docs/openapi); * [ ] 提按文档 * [x] 接口定义 - * [x] 业务逻辑实现 + * [x] 业务逻辑实现 + +### 性能优化 +* [`Pyroscope`](docs/proposal/016-关于使用pyroscope用于性能调试的设计.md) 开启Pyroscope功能用于性能调试(目前状态: 内测); + * [x] 提按文档 + * [x] 业务逻辑实现 ### 其他: * `PhoneBind` 手机绑定功能； diff --git a/go.mod b/go.mod index dc419341..8a40225f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/afocus/captcha v0.0.0-20191010092841-4bd1f21c8868 github.com/alimy/cfg v0.3.0 - github.com/alimy/mir/v3 v3.1.0 + github.com/alimy/mir/v3 v3.1.1 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/allegro/bigcache/v3 v3.0.2 github.com/bytedance/sonic v1.5.0 @@ -26,6 +26,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/meilisearch/meilisearch-go v0.21.0 github.com/minio/minio-go/v7 v7.0.45 + github.com/pyroscope-io/client v0.6.0 github.com/sirupsen/logrus v1.9.0 github.com/smartwalle/alipay/v3 v3.1.7 github.com/spf13/viper v1.14.0 @@ -96,6 +97,7 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pyroscope-io/godeltaprof v0.1.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/rs/xid v1.4.0 // indirect github.com/smartwalle/crypto4go v1.0.2 // indirect diff --git a/go.sum b/go.sum index dfe21b6a..8a90f8f9 100644 --- a/go.sum +++ b/go.sum @@ -147,8 +147,8 @@ github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:C github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/alimy/cfg v0.3.0 h1:9xgA0QWVCPSq9fFNRcYahVCAX22IL9ts2wrTQPfAStY= github.com/alimy/cfg v0.3.0/go.mod h1:rOxbasTH2srl6StAjNF5Vyi8bfrdkl3fLGmOYtSw81c= -github.com/alimy/mir/v3 v3.1.0 h1:zk8Afsi1rcbRpL8MdGCkKqWqqLUupeSY9NcJgnXxic8= -github.com/alimy/mir/v3 v3.1.0/go.mod h1:ybhT2ijOiDn0lLwWzIY6vXdv+uzZrctS7VFfczcXBWU= +github.com/alimy/mir/v3 v3.1.1 h1:3tz7uGOwuA1IKU0BysyBvGbyqKtEVMuhPBD/APk1ANw= +github.com/alimy/mir/v3 v3.1.1/go.mod h1:ybhT2ijOiDn0lLwWzIY6vXdv+uzZrctS7VFfczcXBWU= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible h1:9gWa46nstkJ9miBReJcN8Gq34cBFbzSpQZVVT9N09TM= github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= @@ -1250,6 +1250,10 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/pyroscope-io/client v0.6.0 h1:rcUFgcnfmuyVYDYT+4d0zfqc8YedOyruHSsUb9ImaBw= +github.com/pyroscope-io/client v0.6.0/go.mod h1:4h21iOU4pUOq0prKyDlvYRL+SCKsBc5wKiEtV+rJGqU= +github.com/pyroscope-io/godeltaprof v0.1.0 h1:UBqtjt0yZi4jTxqZmLAs34XG6ycS3vUTlhEUSq4NHLE= +github.com/pyroscope-io/godeltaprof v0.1.0/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= diff --git a/internal/conf/conf.go b/internal/conf/conf.go index 950d2d8a..77e3d5b1 100644 --- a/internal/conf/conf.go +++ b/internal/conf/conf.go @@ -19,6 +19,7 @@ var ( loggerMeiliSetting *LoggerMeiliSettingS redisSetting *RedisSettingS + PyroscopeSetting *PyroscopeSettingS DatabaseSetting *DatabaseSetingS MysqlSetting *MySQLSettingS PostgresSetting *PostgresSettingS @@ -81,6 +82,7 @@ func setupSetting(suite []string, noDefault bool) error { "BigCacheIndex": &BigCacheIndexSetting, "Alipay": &AlipaySetting, "SmsJuhe": &SmsJuheSetting, + "Pyroscope": &PyroscopeSetting, "Logger": &loggerSetting, "LoggerFile": &loggerFileSetting, "LoggerZinc": &loggerZincSetting, diff --git a/internal/conf/config.yaml b/internal/conf/config.yaml index c86d168a..f0465e41 100644 --- a/internal/conf/config.yaml +++ b/internal/conf/config.yaml @@ -72,6 +72,11 @@ BigCacheIndex: # 使用BigCache缓存泡泡广场消息流 MaxIndexPage: 1024 # 最大缓存页数，必须是2^n, 代表最大同时缓存多少页数据 Verbose: False # 是否打印cache操作的log ExpireInSecond: 300 # 多少秒(>0)后强制过期缓存 +Pyroscope: # Pyroscope配置 + AppName: "paopao-ce" # application name + Endpoint: "http://localhost:4040" # Pyroscope server address + AuthToken: # Pyroscope authentication token + Logger: none # Pyroscope logger (standard | logrus | none) Logger: # 日志通用配置 Level: debug # 日志级别 panic|fatal|error|warn|info|debug|trace LoggerFile: # 使用File写日志 diff --git a/internal/conf/settting.go b/internal/conf/settting.go index 93443f64..6e08ce66 100644 --- a/internal/conf/settting.go +++ b/internal/conf/settting.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/pyroscope-io/client/pyroscope" "github.com/sirupsen/logrus" "github.com/spf13/viper" "gorm.io/gorm/logger" @@ -22,6 +23,13 @@ type Setting struct { vp *viper.Viper } +type PyroscopeSettingS struct { + AppName string + Endpoint string + AuthToken string + Logger string +} + type LoggerSettingS struct { Level string } @@ -381,6 +389,16 @@ func (s *MeiliSettingS) Endpoint() string { return endpoint(s.Host, s.Secure) } +func (s *PyroscopeSettingS) GetLogger() (logger pyroscope.Logger) { + switch strings.ToLower(s.Logger) { + case "standard": + logger = pyroscope.StandardLogger + case "logrus": + logger = logrus.StandardLogger() + } + return +} + func endpoint(host string, secure bool) string { schema := "http" if secure { diff --git a/main.go b/main.go index 48545822..9245cc24 100644 --- a/main.go +++ b/main.go @@ -59,10 +59,15 @@ func main() { fmt.Fprintln(color.Output, "no service need start so just exit") return } - wg := &sync.WaitGroup{} + + // start pyroscope if need + debug.StartPyroscope() + // start services + wg := &sync.WaitGroup{} fmt.Fprintf(color.Output, "\nstarting run service...\n\n") service.Start(wg) + // graceful stop services wg.Add(1) go func() { diff --git a/pkg/debug/pyroscope.go b/pkg/debug/pyroscope.go new file mode 100644 index 00000000..60a8ed74 --- /dev/null +++ b/pkg/debug/pyroscope.go @@ -0,0 +1,19 @@ +// Copyright 2023 ROC. All rights reserved. +// Use of this source code is governed by a MIT style +// license that can be found in the LICENSE file. + +//go:build !pyroscope +// +build !pyroscope + +package debug + +import ( + "github.com/alimy/cfg" + "github.com/sirupsen/logrus" +) + +func StartPyroscope() { + if cfg.If("Pyroscope") { + logrus.Infoln("want Pyroscope feature but not support in this compile version") + } +} diff --git a/pkg/debug/pyroscope_embed.go b/pkg/debug/pyroscope_embed.go new file mode 100644 index 00000000..71f7ad1d --- /dev/null +++ b/pkg/debug/pyroscope_embed.go @@ -0,0 +1,48 @@ +// Copyright 2023 ROC. All rights reserved. +// Use of this source code is governed by a MIT style +// license that can be found in the LICENSE file. + +//go:build pyroscope +// +build pyroscope + +package debug + +import ( + "os" + + "github.com/alimy/cfg" + "github.com/pyroscope-io/client/pyroscope" + "github.com/rocboss/paopao-ce/internal/conf" + "github.com/sirupsen/logrus" +) + +func StartPyroscope() { + if !cfg.If("Pyroscope") { + logrus.Infoln("skip Pyroscope because not add Pyroscope feature in config.yaml") + return + } + s := conf.PyroscopeSetting + c := pyroscope.Config{ + ApplicationName: s.AppName, + ServerAddress: s.Endpoint, + AuthToken: os.Getenv("PYROSCOPE_AUTH_TOKEN"), + Logger: s.GetLogger(), + ProfileTypes: []pyroscope.ProfileType{ + pyroscope.ProfileCPU, + pyroscope.ProfileGoroutines, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + pyroscope.ProfileBlockCount, + pyroscope.ProfileBlockDuration, + pyroscope.ProfileMutexCount, + pyroscope.ProfileMutexDuration, + }, + } + if len(c.AuthToken) == 0 { + c.AuthToken = s.AuthToken + } + pyroscope.Start(c) + +} From f0c4320a4549402d5aaad602707c2b2ade72dd4b Mon Sep 17 00:00:00 2001 From: Michael Li Date: Thu, 16 Feb 2023 10:07:02 +0800 Subject: [PATCH 5/5] optimize BigCacheIndex add HardMaxCacheSize to customer max cache size in runtime --- ...用pyroscope用于性能调试的设计.md | 20 ++++++++++++++----- internal/conf/config.yaml | 1 + internal/conf/settting.go | 7 ++++--- internal/dao/cache/cache.go | 14 ++++++------- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/docs/proposal/016-关于使用pyroscope用于性能调试的设计.md b/docs/proposal/016-关于使用pyroscope用于性能调试的设计.md index d2313bc7..b8926cd5 100644 --- a/docs/proposal/016-关于使用pyroscope用于性能调试的设计.md +++ b/docs/proposal/016-关于使用pyroscope用于性能调试的设计.md @@ -1,6 +1,6 @@ | 编号 | 作者 | 发表时间 | 变更时间 | 版本 | 状态 | | ----- | ----- | ----- | ----- | ----- | ----- | -| 016| 北野 | 2023-02-15 | 2023-02-15 | v1.0 | 提议 | +| 016| 北野 | 2023-02-15 | 2023-02-16 | v1.1 | 提议 | ### 概述 Pyroscope 是一个开源的持续性能剖析平台。它能够帮你： @@ -13,16 +13,25 @@ Pyroscope 是一个开源的持续性能剖析平台。它能够帮你： * 开发环境下启用Pyroscope，但是部署环境下禁用。 ### 方案 +#### 设计要点 * 使用//go:build pyroscope 可选编译app集成Pyroscope功能 * config.yaml中添加`Pyroscope` 功能来启用Pyroscope +#### 设计细节 +* 参考实现(PR): +[add Pyroscope support #199](https://github.com/rocboss/paopao-ce/pull/199) + ### 疑问 1. 为什么要引入Pyroscope？ 用于开发环境下对paopao-ce进行性能优化。 -2. 如何开启这个功能？ -在配置文件config.yaml中的`Features`中添加`Pyroscope`功能项开启该功能： +2. 如何开启这个功能？ +* 构建时将 `pyroscope` 添加到TAGS中: + ```sh + make run TAGS='pyroscope' + ``` +* 在配置文件config.yaml中的`Features`中添加`Pyroscope`功能项开启该功能： ```yaml ... # features中加上 Friendship @@ -32,8 +41,6 @@ Pyroscope 是一个开源的持续性能剖析平台。它能够帮你： ... ``` - - ### 参考文档 * [pyroscope](https://github.com/pyroscope-io/pyroscope) * [pyroscope client](https://github.com/pyroscope-io/client) @@ -41,3 +48,6 @@ Pyroscope 是一个开源的持续性能剖析平台。它能够帮你： ### 更新记录 #### v1.0(2023-02-15) - 北野 * 初始文档 + +#### v1.1(2023-02-16) - 北野 +* 添加参考实现 diff --git a/internal/conf/config.yaml b/internal/conf/config.yaml index f0465e41..6ea7d3c6 100644 --- a/internal/conf/config.yaml +++ b/internal/conf/config.yaml @@ -70,6 +70,7 @@ SimpleCacheIndex: # 缓存泡泡广场消息流 ExpireTickDuration: 300 # 每多少秒后强制过期缓存, 设置为0禁止强制使缓存过期 BigCacheIndex: # 使用BigCache缓存泡泡广场消息流 MaxIndexPage: 1024 # 最大缓存页数，必须是2^n, 代表最大同时缓存多少页数据 + HardMaxCacheSize: 256 # 最大缓存大小(MB)，0表示无限制 Verbose: False # 是否打印cache操作的log ExpireInSecond: 300 # 多少秒(>0)后强制过期缓存 Pyroscope: # Pyroscope配置 diff --git a/internal/conf/settting.go b/internal/conf/settting.go index 6e08ce66..a9b2643d 100644 --- a/internal/conf/settting.go +++ b/internal/conf/settting.go @@ -91,9 +91,10 @@ type SimpleCacheIndexSettingS struct { } type BigCacheIndexSettingS struct { - MaxIndexPage int - ExpireInSecond time.Duration - Verbose bool + MaxIndexPage int + HardMaxCacheSize int + ExpireInSecond time.Duration + Verbose bool } type AlipaySettingS struct { diff --git a/internal/dao/cache/cache.go b/internal/dao/cache/cache.go index 773c9a2f..e2be7a32 100644 --- a/internal/dao/cache/cache.go +++ b/internal/dao/cache/cache.go @@ -15,13 +15,13 @@ import ( func NewBigCacheIndexService(ips core.IndexPostsService, ams core.AuthorizationManageService) (core.CacheIndexService, core.VersionInfo) { s := conf.BigCacheIndexSetting - - config := bigcache.DefaultConfig(s.ExpireInSecond) - config.Shards = s.MaxIndexPage - config.Verbose = s.Verbose - config.MaxEntrySize = 10000 - config.Logger = logrus.StandardLogger() - cache, err := bigcache.NewBigCache(config) + c := bigcache.DefaultConfig(s.ExpireInSecond) + c.Shards = s.MaxIndexPage + c.HardMaxCacheSize = s.HardMaxCacheSize + c.Verbose = s.Verbose + c.MaxEntrySize = 10000 + c.Logger = logrus.StandardLogger() + cache, err := bigcache.NewBigCache(c) if err != nil { logrus.Fatalf("initial bigCahceIndex failure by err: %v", err) }