mirror of https://github.com/rocboss/paopao-ce
Merge pull request #129 from rocboss/feature/meilisearch
support meilisearch as tweet search servicepull/130/head
Michael Li
3 years ago
committed by
GitHub
commit
432885d164
@ -0,0 +1,71 @@
|
||||
package conf
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/rocboss/paopao-ce/pkg/json"
|
||||
"github.com/sirupsen/logrus"
|
||||
"gopkg.in/resty.v1"
|
||||
)
|
||||
|
||||
type zincLogData struct {
|
||||
Time time.Time `json:"time"`
|
||||
Level logrus.Level `json:"level"`
|
||||
Message string `json:"message"`
|
||||
Data logrus.Fields `json:"data"`
|
||||
}
|
||||
|
||||
type zincLogIndex struct {
|
||||
Index map[string]string `json:"index"`
|
||||
}
|
||||
|
||||
type zincLogHook struct {
|
||||
host string
|
||||
index string
|
||||
user string
|
||||
password string
|
||||
}
|
||||
|
||||
func (h *zincLogHook) Fire(entry *logrus.Entry) error {
|
||||
index := &zincLogIndex{
|
||||
Index: map[string]string{
|
||||
"_index": h.index,
|
||||
},
|
||||
}
|
||||
indexBytes, _ := json.Marshal(index)
|
||||
|
||||
data := &zincLogData{
|
||||
Time: entry.Time,
|
||||
Level: entry.Level,
|
||||
Message: entry.Message,
|
||||
Data: entry.Data,
|
||||
}
|
||||
dataBytes, _ := json.Marshal(data)
|
||||
|
||||
logStr := string(indexBytes) + "\n" + string(dataBytes) + "\n"
|
||||
client := resty.New()
|
||||
|
||||
if _, err := client.SetDisableWarn(true).R().
|
||||
SetHeader("Content-Type", "application/json").
|
||||
SetBasicAuth(h.user, h.password).
|
||||
SetBody(logStr).
|
||||
Post(h.host); err != nil {
|
||||
fmt.Println(err.Error())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *zincLogHook) Levels() []logrus.Level {
|
||||
return logrus.AllLevels
|
||||
}
|
||||
|
||||
func newZincLogHook() *zincLogHook {
|
||||
return &zincLogHook{
|
||||
host: loggerZincSetting.Endpoint() + "/es/_bulk",
|
||||
index: loggerZincSetting.Index,
|
||||
user: loggerZincSetting.User,
|
||||
password: loggerZincSetting.Password,
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,139 @@
|
||||
package dao
|
||||
|
||||
import (
|
||||
"github.com/Masterminds/semver/v3"
|
||||
"github.com/meilisearch/meilisearch-go"
|
||||
"github.com/rocboss/paopao-ce/internal/conf"
|
||||
"github.com/rocboss/paopao-ce/internal/core"
|
||||
"github.com/rocboss/paopao-ce/internal/model"
|
||||
"github.com/rocboss/paopao-ce/pkg/json"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func newMeiliTweetSearchServant() *meiliTweetSearchServant {
|
||||
s := conf.MeiliSetting
|
||||
client := meilisearch.NewClient(meilisearch.ClientConfig{
|
||||
Host: s.Endpoint(),
|
||||
APIKey: s.ApiKey,
|
||||
})
|
||||
|
||||
if _, err := client.Index(s.Index).FetchInfo(); err != nil {
|
||||
logrus.Debugf("create index because fetch index info error: %v", err)
|
||||
client.CreateIndex(&meilisearch.IndexConfig{
|
||||
Uid: s.Index,
|
||||
PrimaryKey: "id",
|
||||
})
|
||||
searchableAttributes := []string{"content", "tags"}
|
||||
sortableAttributes := []string{"is_top", "latest_replied_on"}
|
||||
|
||||
index := client.Index(s.Index)
|
||||
index.UpdateSearchableAttributes(&searchableAttributes)
|
||||
index.UpdateSortableAttributes(&sortableAttributes)
|
||||
}
|
||||
|
||||
return &meiliTweetSearchServant{
|
||||
client: client,
|
||||
index: client.Index(s.Index),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *meiliTweetSearchServant) Name() string {
|
||||
return "Meili"
|
||||
}
|
||||
|
||||
func (s *meiliTweetSearchServant) Version() *semver.Version {
|
||||
return semver.MustParse("v0.1.0")
|
||||
}
|
||||
|
||||
func (s *meiliTweetSearchServant) IndexName() string {
|
||||
return s.index.UID
|
||||
}
|
||||
|
||||
func (s *meiliTweetSearchServant) AddDocuments(data core.DocItems, primaryKey ...string) (bool, error) {
|
||||
task, err := s.index.AddDocuments(data, primaryKey...)
|
||||
if err != nil {
|
||||
logrus.Errorf("meiliTweetSearchServant.AddDocuments error: %v", err)
|
||||
return false, err
|
||||
}
|
||||
logrus.Debugf("meiliTweetSearchServant.AddDocuments task: %+v", task.Details)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *meiliTweetSearchServant) DeleteDocuments(identifiers []string) error {
|
||||
task, err := s.index.DeleteDocuments(identifiers)
|
||||
if err != nil {
|
||||
logrus.Errorf("meiliTweetSearchServant.DeleteDocuments error: %v", err)
|
||||
return err
|
||||
}
|
||||
logrus.Debugf("meiliTweetSearchServant.DeleteDocuments task: %+v", task.Details)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *meiliTweetSearchServant) Search(q *core.QueryReq, offset, limit int) (*core.QueryResp, error) {
|
||||
if q.Type == core.SearchTypeDefault && q.Query != "" {
|
||||
return s.queryByContent(q, offset, limit)
|
||||
} else if q.Type == core.SearchTypeTag && q.Query != "" {
|
||||
return s.queryByTag(q, offset, limit)
|
||||
}
|
||||
return s.queryAny(offset, limit)
|
||||
}
|
||||
|
||||
func (s *meiliTweetSearchServant) queryByContent(q *core.QueryReq, offset, limit int) (*core.QueryResp, error) {
|
||||
resp, err := s.index.Search(q.Query, &meilisearch.SearchRequest{
|
||||
Offset: int64(offset),
|
||||
Limit: int64(limit),
|
||||
AttributesToRetrieve: []string{"*"},
|
||||
Sort: []string{"is_top:desc", "latest_replied_on:desc"},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.postsFrom(resp)
|
||||
}
|
||||
|
||||
func (s *meiliTweetSearchServant) queryByTag(q *core.QueryReq, offset, limit int) (*core.QueryResp, error) {
|
||||
resp, err := s.index.Search("#"+q.Query, &meilisearch.SearchRequest{
|
||||
Offset: int64(offset),
|
||||
Limit: int64(limit),
|
||||
AttributesToRetrieve: []string{"*"},
|
||||
Sort: []string{"is_top:desc", "latest_replied_on:desc"},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.postsFrom(resp)
|
||||
}
|
||||
|
||||
func (s *meiliTweetSearchServant) queryAny(offset, limit int) (*core.QueryResp, error) {
|
||||
resp, err := s.index.Search("", &meilisearch.SearchRequest{
|
||||
Offset: int64(offset),
|
||||
Limit: int64(limit),
|
||||
Matches: true,
|
||||
Sort: []string{"is_top:desc", "latest_replied_on:desc"},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.postsFrom(resp)
|
||||
}
|
||||
|
||||
func (s *meiliTweetSearchServant) postsFrom(resp *meilisearch.SearchResponse) (*core.QueryResp, error) {
|
||||
logrus.Debugf("resp Hits:%d NbHits:%d offset: %d limit:%d ", len(resp.Hits), resp.NbHits, resp.Offset, resp.Limit)
|
||||
posts := make([]*model.PostFormated, 0, len(resp.Hits))
|
||||
for _, hit := range resp.Hits {
|
||||
item := &model.PostFormated{}
|
||||
raw, err := json.Marshal(hit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = json.Unmarshal(raw, item); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
posts = append(posts, item)
|
||||
}
|
||||
|
||||
return &core.QueryResp{
|
||||
Items: posts,
|
||||
Total: resp.NbHits,
|
||||
}, nil
|
||||
}
|
||||
Loading…
Reference in new issue