From 4c0cfdd2b0098eccb33ee2599a144c23051e6d46 Mon Sep 17 00:00:00 2001 From: Michael Li Date: Sun, 6 Nov 2022 01:01:10 +0800 Subject: [PATCH] optimize core search interface and upgrade support meilisearch to 0.29.0 --- README.md | 2 +- docker-compose.yaml | 2 +- go.mod | 7 ++-- go.sum | 20 +++++---- internal/core/search.go | 7 +++- internal/dao/search/bridge.go | 4 +- internal/dao/search/meili.go | 78 +++++++++++++++++++++++++++++++---- internal/dao/search/zinc.go | 43 +++++++++++++++++-- internal/routers/api/user.go | 2 +- internal/service/post.go | 70 ++++++++----------------------- web/build/info.json | 4 +- 11 files changed, 153 insertions(+), 86 deletions(-) diff --git a/README.md b/README.md index 15497298..6f06f7d0 100644 --- a/README.md +++ b/README.md @@ -380,7 +380,7 @@ Zinc: # Zinc搜索配置 mkdir -p data/meili/data # 使用Docker运行 -docker run -d --name meili -v ${PWD}/data/meili/data:/meili_data -p 7700:7700 -e MEILI_MASTER_KEY=paopao-meilisearch getmeili/meilisearch:v0.27.0 +docker run -d --name meili -v ${PWD}/data/meili/data:/meili_data -p 7700:7700 -e MEILI_MASTER_KEY=paopao-meilisearch getmeili/meilisearch:v0.29.0 # visit http://localhost:7700 打开自带的搜索前端ui # 使用docker compose运行，需要删除docker-compose.yaml中关于meili的注释 diff --git a/docker-compose.yaml b/docker-compose.yaml index 6e92b829..d458f94e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -56,7 +56,7 @@ services: - paopao-network # meili: - # image: getmeili/meilisearch:v0.27.0 + # image: getmeili/meilisearch:v0.29.0 # restart: always # ports: # - 7700:7700 diff --git a/go.mod b/go.mod index ea5d0cbe..8f4c9435 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/golang-migrate/migrate/v4 v4.15.2 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible github.com/json-iterator/go v1.1.12 - github.com/meilisearch/meilisearch-go v0.19.1 + github.com/meilisearch/meilisearch-go v0.21.0 github.com/minio/minio-go/v7 v7.0.27 github.com/sirupsen/logrus v1.8.1 github.com/smartwalle/alipay/v3 v3.1.7 @@ -56,7 +56,6 @@ require ( github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-playground/validator/v10 v10.10.1 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect - github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.7 // indirect @@ -77,7 +76,7 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.15.0 // indirect + github.com/klauspost/compress v1.15.6 // indirect github.com/klauspost/cpuid v1.3.1 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/leodido/go-urn v1.2.1 // indirect @@ -108,7 +107,7 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.7 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.36.0 // indirect + github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d // indirect go.uber.org/atomic v1.7.0 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 // indirect diff --git a/go.sum b/go.sum index 3c46467a..9221a97a 100644 --- a/go.sum +++ b/go.sum @@ -630,8 +630,6 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= @@ -947,8 +945,9 @@ github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdY github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY= +github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= @@ -1045,8 +1044,8 @@ github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4f github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= -github.com/meilisearch/meilisearch-go v0.19.1 h1:CDi7p5Ev18h0hMXaJZ/1GzSKu3lvPCsaJfrLco3bMEM= -github.com/meilisearch/meilisearch-go v0.19.1/go.mod h1:PnFFq9tELcH5mLVKCoTHRS58B3HEA8vKdBSoG6g/FCE= +github.com/meilisearch/meilisearch-go v0.21.0 h1:SwYMWJVi6vDdSDJdOmbkJ4T26PavjYc4MlZcJZF9+Qs= +github.com/meilisearch/meilisearch-go v0.21.0/go.mod h1:3dvPYZGUWu40qHoTK187fmqF2lrarboPa5m2Yu2Seh4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= @@ -1333,6 +1332,7 @@ github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -1341,8 +1341,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1378,8 +1379,8 @@ github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.36.0 h1:NhqfO/cB7Ajn1czkKnWkMHyPYr5nyND14ZGPk23g0/c= -github.com/valyala/fasthttp v1.36.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= +github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d h1:xS9QTPgKl9ewGsAOPc+xW7DeStJDqYPfisDmeSCcbco= +github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= @@ -2132,8 +2133,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.0.3/go.mod h1:twGxftLBlFgNVNakL7F+P/x9oYqoymG3YYT8cAfI9oI= gorm.io/driver/mysql v1.3.4 h1:/KoBMgsUHC3bExsekDcmNYaBnfH2WNeFuXqqrqMc98Q= gorm.io/driver/mysql v1.3.4/go.mod h1:s4Tq0KmD0yhPGHbZEwg1VPlH0vT/GBHJZorPzhcxBUE= diff --git a/internal/core/search.go b/internal/core/search.go index 0469a089..3ac90288 100644 --- a/internal/core/search.go +++ b/internal/core/search.go @@ -22,12 +22,15 @@ type QueryResp struct { Total int64 } -type DocItems []map[string]any +type TsDocItem struct { + Post *model.Post + Content string +} // TweetSearchService tweet search service interface type TweetSearchService interface { IndexName() string - AddDocuments(documents DocItems, primaryKey ...string) (bool, error) + AddDocuments(data []TsDocItem, primaryKey ...string) (bool, error) DeleteDocuments(identifiers []string) error Search(user *model.User, q *QueryReq, offset, limit int) (*QueryResp, error) } diff --git a/internal/dao/search/bridge.go b/internal/dao/search/bridge.go index 20b9a73d..84f469a5 100644 --- a/internal/dao/search/bridge.go +++ b/internal/dao/search/bridge.go @@ -14,7 +14,7 @@ var ( type documents struct { primaryKey []string - docItems core.DocItems + docItems []core.TsDocItem identifiers []string } @@ -28,7 +28,7 @@ func (s *bridgeTweetSearchServant) IndexName() string { return s.ts.IndexName() } -func (s *bridgeTweetSearchServant) AddDocuments(data core.DocItems, primaryKey ...string) (bool, error) { +func (s *bridgeTweetSearchServant) AddDocuments(data []core.TsDocItem, primaryKey ...string) (bool, error) { s.updateDocs(&documents{ primaryKey: primaryKey, docItems: data, diff --git a/internal/dao/search/meili.go b/internal/dao/search/meili.go index d86ec4c3..321953ee 100644 --- a/internal/dao/search/meili.go +++ b/internal/dao/search/meili.go @@ -2,6 +2,7 @@ package search import ( "fmt" + "strings" "github.com/Masterminds/semver/v3" "github.com/meilisearch/meilisearch-go" @@ -26,20 +27,41 @@ type meiliTweetSearchServant struct { friendFilter string } +type postInfo struct { + ID int64 `json:"id"` + UserID int64 `json:"user_id"` + CommentCount int64 `json:"comment_count"` + CollectionCount int64 `json:"collection_count"` + UpvoteCount int64 `json:"upvote_count"` + Visibility model.PostVisibleT `json:"visibility"` + IsTop int `json:"is_top"` + IsEssence int `json:"is_essence"` + IsLock int `json:"is_lock"` + LatestRepliedOn int64 `json:"latest_replied_on"` + CreatedOn int64 `json:"created_on"` + ModifiedOn int64 `json:"modified_on"` + AttachmentPrice int64 `json:"attachment_price"` + IPLoc string `json:"ip_loc"` +} + func (s *meiliTweetSearchServant) Name() string { return "Meili" } func (s *meiliTweetSearchServant) Version() *semver.Version { - return semver.MustParse("v0.2.0") + return semver.MustParse("v0.2.1") } func (s *meiliTweetSearchServant) IndexName() string { return s.index.UID } -func (s *meiliTweetSearchServant) AddDocuments(data core.DocItems, primaryKey ...string) (bool, error) { - if _, err := s.index.AddDocuments(data, primaryKey...); err != nil { +func (s *meiliTweetSearchServant) AddDocuments(data []core.TsDocItem, primaryKey ...string) (bool, error) { + docs := s.toDocs(data) + if len(docs) == 0 { + return true, nil + } + if _, err := s.index.AddDocuments(docs, primaryKey...); err != nil { logrus.Errorf("meiliTweetSearchServant.AddDocuments error: %v", err) return false, err } @@ -103,7 +125,7 @@ func (s *meiliTweetSearchServant) queryByTag(user *model.User, q *core.QueryReq, } filter := s.filterList(user) - tagFilter := []string{"tags." + q.Query + "=1"} + tagFilter := []string{"tags=" + q.Query} if len(filter) > 0 { request.Filter = [][]string{tagFilter, {filter}} } else { @@ -154,19 +176,57 @@ func (s *meiliTweetSearchServant) filterList(user *model.User) string { func (s *meiliTweetSearchServant) postsFrom(resp *meilisearch.SearchResponse) (*core.QueryResp, error) { posts := make([]*model.PostFormated, 0, len(resp.Hits)) for _, hit := range resp.Hits { - item := &model.PostFormated{} raw, err := json.Marshal(hit) if err != nil { return nil, err } - if err = json.Unmarshal(raw, item); err != nil { + p := &postInfo{} + if err = json.Unmarshal(raw, p); err != nil { return nil, err } - posts = append(posts, item) + posts = append(posts, &model.PostFormated{ + ID: p.ID, + UserID: p.UserID, + CommentCount: p.CommentCount, + CollectionCount: p.CollectionCount, + UpvoteCount: p.UpvoteCount, + Visibility: p.Visibility, + IsTop: p.IsTop, + IsEssence: p.IsEssence, + IsLock: p.IsLock, + LatestRepliedOn: p.LatestRepliedOn, + CreatedOn: p.CreatedOn, + ModifiedOn: p.ModifiedOn, + AttachmentPrice: p.AttachmentPrice, + IPLoc: p.IPLoc, + }) } - return &core.QueryResp{ Items: posts, - Total: resp.NbHits, + Total: resp.EstimatedTotalHits, }, nil } + +func (s *meiliTweetSearchServant) toDocs(data []core.TsDocItem) []map[string]any { + docs := make([]map[string]any, 0, len(data)) + for _, d := range data { + docs = append(docs, map[string]any{ + "id": d.Post.ID, + "user_id": d.Post.UserID, + "comment_count": d.Post.CommentCount, + "collection_count": d.Post.CollectionCount, + "upvote_count": d.Post.UpvoteCount, + "visibility": d.Post.Visibility, + "is_top": d.Post.IsTop, + "is_essence": d.Post.IsEssence, + "content": d.Content, + "tags": strings.Split(d.Post.Tags, ","), + "ip_loc": d.Post.IPLoc, + "latest_replied_on": d.Post.LatestRepliedOn, + "attachment_price": d.Post.AttachmentPrice, + "created_on": d.Post.CreatedOn, + "modified_on": d.Post.ModifiedOn, + }) + } + return docs +} diff --git a/internal/dao/search/zinc.go b/internal/dao/search/zinc.go index e9fa36e7..844db904 100644 --- a/internal/dao/search/zinc.go +++ b/internal/dao/search/zinc.go @@ -1,6 +1,8 @@ package search import ( + "strings" + "github.com/Masterminds/semver/v3" "github.com/rocboss/paopao-ce/internal/core" "github.com/rocboss/paopao-ce/internal/model" @@ -36,8 +38,12 @@ func (s *zincTweetSearchServant) IndexName() string { return s.indexName } -func (s *zincTweetSearchServant) AddDocuments(data core.DocItems, primaryKey ...string) (bool, error) { - buf := make(core.DocItems, 0, len(data)+1) +func (s *zincTweetSearchServant) AddDocuments(data []core.TsDocItem, primaryKey ...string) (bool, error) { + if len(data) == 0 { + return true, nil + } + + buf := make([]map[string]any, 0, len(data)+1) if len(primaryKey) > 0 { buf = append(buf, map[string]any{ "index": map[string]any{ @@ -52,7 +58,9 @@ func (s *zincTweetSearchServant) AddDocuments(data core.DocItems, primaryKey ... }, }) } - buf = append(buf, data...) + docs := s.toDocs(data) + buf = append(buf, docs...) + return s.client.BulkPushDoc(buf) } @@ -246,3 +254,32 @@ func (s *zincTweetSearchServant) createIndex() { }, }) } + +func (s *zincTweetSearchServant) toDocs(data []core.TsDocItem) []map[string]any { + docs := make([]map[string]any, len(data)) + for _, d := range data { + tagMaps := map[string]int8{} + for _, tag := range strings.Split(d.Post.Tags, ",") { + tagMaps[tag] = 1 + } + + docs = append(docs, map[string]any{ + "id": d.Post.ID, + "user_id": d.Post.UserID, + "comment_count": d.Post.CommentCount, + "collection_count": d.Post.CollectionCount, + "upvote_count": d.Post.UpvoteCount, + "visibility": d.Post.Visibility, + "is_top": d.Post.IsTop, + "is_essence": d.Post.IsEssence, + "content": d.Content, + "tags": tagMaps, + "ip_loc": d.Post.IPLoc, + "latest_replied_on": d.Post.LatestRepliedOn, + "attachment_price": d.Post.AttachmentPrice, + "created_on": d.Post.CreatedOn, + "modified_on": d.Post.ModifiedOn, + }) + } + return docs +} diff --git a/internal/routers/api/user.go b/internal/routers/api/user.go index 18267d81..5bced3c8 100644 --- a/internal/routers/api/user.go +++ b/internal/routers/api/user.go @@ -323,7 +323,7 @@ func GetUserPosts(c *gin.Context) { "ORDER": "latest_replied_on DESC", } - posts, err := service.GetPostList(&service.PostListReq{ + _, posts, err := service.GetPostList(&service.PostListReq{ Conditions: &conditions, Offset: (app.GetPage(c) - 1) * app.GetPageSize(c), Limit: app.GetPageSize(c), diff --git a/internal/service/post.go b/internal/service/post.go index 875b7fe2..3d76f1f4 100644 --- a/internal/service/post.go +++ b/internal/service/post.go @@ -439,14 +439,13 @@ func GetIndexPosts(user *model.User, offset int, limit int) (*rest.IndexTweetsRe return ds.IndexPosts(user, offset, limit) } -func GetPostList(req *PostListReq) ([]*model.PostFormated, error) { +func GetPostList(req *PostListReq) ([]*model.Post, []*model.PostFormated, error) { posts, err := ds.GetPosts(req.Conditions, req.Offset, req.Limit) - if err != nil { - return nil, err + return nil, nil, err } - - return ds.MergePosts(posts) + postFormated, err := ds.MergePosts(posts) + return posts, postFormated, err } func GetPostCount(conditions *model.ConditionsT) (int64, error) { @@ -484,37 +483,17 @@ func PushPostToSearch(post *model.Post) { } contentFormated := "" - for _, content := range postFormated.Contents { if content.Type == model.CONTENT_TYPE_TEXT || content.Type == model.CONTENT_TYPE_TITLE { contentFormated = contentFormated + content.Content + "\n" } } - tagMaps := map[string]int8{} - for _, tag := range strings.Split(post.Tags, ",") { - tagMaps[tag] = 1 - } - - data := core.DocItems{{ - "id": post.ID, - "user_id": post.UserID, - "comment_count": post.CommentCount, - "collection_count": post.CollectionCount, - "upvote_count": post.UpvoteCount, - "visibility": post.Visibility, - "is_top": post.IsTop, - "is_essence": post.IsEssence, - "content": contentFormated, - "tags": tagMaps, - "ip_loc": post.IPLoc, - "latest_replied_on": post.LatestRepliedOn, - "attachment_price": post.AttachmentPrice, - "created_on": post.CreatedOn, - "modified_on": post.ModifiedOn, + docs := []core.TsDocItem{{ + Post: post, + Content: contentFormated, }} - - ts.AddDocuments(data, fmt.Sprintf("%d", post.ID)) + ts.AddDocuments(docs, fmt.Sprintf("%d", post.ID)) } func DeleteSearchPost(post *model.Post) error { @@ -532,39 +511,26 @@ func PushPostsToSearch(c *gin.Context) { nums := int(pages) for i := 0; i < nums; i++ { - posts, _ := GetPostList(&PostListReq{ + posts, postsFormated, err := GetPostList(&PostListReq{ Conditions: &model.ConditionsT{}, Offset: i * splitNum, Limit: splitNum, }) - - for _, post := range posts { + if err != nil || len(posts) != len(postsFormated) { + continue + } + for i, pf := range postsFormated { contentFormated := "" - - for _, content := range post.Contents { + for _, content := range pf.Contents { if content.Type == model.CONTENT_TYPE_TEXT || content.Type == model.CONTENT_TYPE_TITLE { contentFormated = contentFormated + content.Content + "\n" } } - - docs := core.DocItems{{ - "id": post.ID, - "user_id": post.User.ID, - "comment_count": post.CommentCount, - "collection_count": post.CollectionCount, - "upvote_count": post.UpvoteCount, - "visibility": post.Visibility, - "is_top": post.IsTop, - "is_essence": post.IsEssence, - "content": contentFormated, - "tags": post.Tags, - "ip_loc": post.IPLoc, - "latest_replied_on": post.LatestRepliedOn, - "attachment_price": post.AttachmentPrice, - "created_on": post.CreatedOn, - "modified_on": post.ModifiedOn, + docs := []core.TsDocItem{{ + Post: posts[i], + Content: contentFormated, }} - ts.AddDocuments(docs, fmt.Sprintf("%d", post.ID)) + ts.AddDocuments(docs, fmt.Sprintf("%d", posts[i].ID)) } } diff --git a/web/build/info.json b/web/build/info.json index 290a7eb3..9a7454ac 100644 --- a/web/build/info.json +++ b/web/build/info.json @@ -1,4 +1,4 @@ { - "version": "8", - "buildTime": "2022-06-13 17:16:22" + "version": "9", + "buildTime": "2022-11-05 22:09:37" } \ No newline at end of file