mirror of https://github.com/rocboss/paopao-ce
alimy
3 years ago
parent
0b5dd8d15e
commit
edc52e71b1
@ -0,0 +1,40 @@
|
||||
package dao
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/allegro/bigcache/v3"
|
||||
"github.com/rocboss/paopao-ce/internal/core"
|
||||
"github.com/rocboss/paopao-ce/internal/model"
|
||||
)
|
||||
|
||||
var (
|
||||
_ core.CacheIndexService = (*simpleCacheIndexServant)(nil)
|
||||
_ core.CacheIndexService = (*bigCacheIndexServant)(nil)
|
||||
)
|
||||
|
||||
type postsEntry struct {
|
||||
key string
|
||||
posts []*model.PostFormated
|
||||
}
|
||||
|
||||
type indexPostsFunc func(int64, int, int) ([]*model.PostFormated, error)
|
||||
|
||||
type bigCacheIndexServant struct {
|
||||
getIndexPosts indexPostsFunc
|
||||
indexActionCh chan core.IndexActionT
|
||||
cachePostsCh chan *postsEntry
|
||||
cache *bigcache.BigCache
|
||||
lastCacheResetTime time.Time
|
||||
}
|
||||
|
||||
type simpleCacheIndexServant struct {
|
||||
getIndexPosts indexPostsFunc
|
||||
indexActionCh chan core.IndexActionT
|
||||
indexPosts []*model.PostFormated
|
||||
atomicIndex atomic.Value
|
||||
maxIndexSize int
|
||||
checkTick *time.Ticker
|
||||
expireIndexTick *time.Ticker
|
||||
}
|
||||
@ -0,0 +1,54 @@
|
||||
package dao
|
||||
|
||||
import (
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/rocboss/paopao-ce/internal/conf"
|
||||
"github.com/rocboss/paopao-ce/internal/core"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var (
|
||||
_ core.ObjectStorageService = (*aliossServant)(nil)
|
||||
_ core.ObjectStorageService = (*minioServant)(nil)
|
||||
_ core.ObjectStorageService = (*s3Servant)(nil)
|
||||
_ core.ObjectStorageService = (*localossServant)(nil)
|
||||
)
|
||||
|
||||
type localossServant struct {
|
||||
savePath string
|
||||
domain string
|
||||
}
|
||||
|
||||
type aliossServant struct {
|
||||
bucket *oss.Bucket
|
||||
domain string
|
||||
}
|
||||
|
||||
type minioServant struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
domain string
|
||||
}
|
||||
|
||||
type s3Servant = minioServant
|
||||
|
||||
func NewObjectStorageService() (oss core.ObjectStorageService) {
|
||||
if conf.CfgIf("AliOSS") {
|
||||
oss = newAliossServent()
|
||||
} else if conf.CfgIf("MinIO") {
|
||||
oss = newMinioServeant()
|
||||
} else if conf.CfgIf("S3") {
|
||||
oss = newS3Servent()
|
||||
logrus.Infof("use S3 as object storage by version %s", oss.Version())
|
||||
return
|
||||
} else if conf.CfgIf("LocalOSS") {
|
||||
oss = newLocalossServent()
|
||||
} else {
|
||||
// default use AliOSS as object storage service
|
||||
oss = newAliossServent()
|
||||
logrus.Infof("use default AliOSS as object storage by version %s", oss.Version())
|
||||
}
|
||||
logrus.Infof("use %s as object storage by version %s", oss.Name(), oss.Version())
|
||||
return
|
||||
}
|
||||
@ -1,165 +1,28 @@
|
||||
package dao
|
||||
|
||||
import (
|
||||
"github.com/rocboss/paopao-ce/internal/conf"
|
||||
"github.com/rocboss/paopao-ce/internal/core"
|
||||
"github.com/rocboss/paopao-ce/pkg/zinc"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func (d *dataServant) CreateSearchIndex(indexName string) {
|
||||
// 不存在则创建索引
|
||||
d.zinc.CreateIndex(indexName, &zinc.ZincIndexProperty{
|
||||
"id": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Store: true,
|
||||
Sortable: true,
|
||||
},
|
||||
"user_id": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Store: true,
|
||||
},
|
||||
"comment_count": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"collection_count": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"upvote_count": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"is_top": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"is_essence": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"content": &zinc.ZincIndexPropertyT{
|
||||
Type: "text",
|
||||
Index: true,
|
||||
Store: true,
|
||||
Aggregatable: true,
|
||||
Highlightable: true,
|
||||
Analyzer: "gse_search",
|
||||
SearchAnalyzer: "gse_standard",
|
||||
},
|
||||
"tags": &zinc.ZincIndexPropertyT{
|
||||
Type: "keyword",
|
||||
Index: true,
|
||||
Store: true,
|
||||
},
|
||||
"ip_loc": &zinc.ZincIndexPropertyT{
|
||||
Type: "keyword",
|
||||
Index: true,
|
||||
Store: true,
|
||||
},
|
||||
"latest_replied_on": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"attachment_price": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"created_on": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"modified_on": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func (d *dataServant) BulkPushDoc(data []map[string]interface{}) (bool, error) {
|
||||
return d.zinc.BulkPushDoc(data)
|
||||
}
|
||||
|
||||
func (d *dataServant) DelDoc(indexName, id string) error {
|
||||
return d.zinc.DelDoc(indexName, id)
|
||||
}
|
||||
|
||||
func (d *dataServant) QueryAll(q *core.QueryT, indexName string, offset, limit int) (*zinc.QueryResultT, error) {
|
||||
// 普通搜索
|
||||
if q.Type == core.SearchTypeDefault && q.Query != "" {
|
||||
return d.QuerySearch(indexName, q.Query, offset, limit)
|
||||
}
|
||||
// Tag分类
|
||||
if q.Type == core.SearchTypeTag && q.Query != "" {
|
||||
return d.QueryTagSearch(indexName, q.Query, offset, limit)
|
||||
}
|
||||
|
||||
queryMap := map[string]interface{}{
|
||||
"query": map[string]interface{}{
|
||||
"match_all": map[string]string{},
|
||||
},
|
||||
"sort": []string{"-is_top", "-latest_replied_on"},
|
||||
"from": offset,
|
||||
"size": limit,
|
||||
}
|
||||
rsp, err := d.zinc.EsQuery(indexName, queryMap)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return rsp, err
|
||||
}
|
||||
|
||||
func (d *dataServant) QuerySearch(indexName, query string, offset, limit int) (*zinc.QueryResultT, error) {
|
||||
rsp, err := d.zinc.EsQuery(indexName, map[string]interface{}{
|
||||
"query": map[string]interface{}{
|
||||
"match_phrase": map[string]interface{}{
|
||||
"content": query,
|
||||
},
|
||||
},
|
||||
"sort": []string{"-is_top", "-latest_replied_on"},
|
||||
"from": offset,
|
||||
"size": limit,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var (
|
||||
_ core.TweetSearchService = (*zincTweetSearchServant)(nil)
|
||||
)
|
||||
|
||||
return rsp, err
|
||||
type zincTweetSearchServant struct {
|
||||
indexName string
|
||||
client *zinc.ZincClient
|
||||
}
|
||||
|
||||
func (d *dataServant) QueryTagSearch(indexName, query string, offset, limit int) (*zinc.QueryResultT, error) {
|
||||
rsp, err := d.zinc.ApiQuery(indexName, map[string]interface{}{
|
||||
"search_type": "querystring",
|
||||
"query": map[string]interface{}{
|
||||
"term": "tags." + query + ":1",
|
||||
},
|
||||
"sort_fields": []string{"-is_top", "-latest_replied_on"},
|
||||
"from": offset,
|
||||
"max_results": limit,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
func NewTweetSearchService() (ts core.TweetSearchService) {
|
||||
if conf.CfgIf("Zinc") {
|
||||
ts = newZincTweetSearchServant()
|
||||
} else {
|
||||
// default use Zinc as tweet search service
|
||||
ts = newZincTweetSearchServant()
|
||||
}
|
||||
|
||||
return rsp, err
|
||||
logrus.Infof("use %s as tweet search serice by version %s", ts.Name(), ts.Version())
|
||||
return
|
||||
}
|
||||
|
||||
@ -0,0 +1,219 @@
|
||||
package dao
|
||||
|
||||
import (
|
||||
"github.com/Masterminds/semver/v3"
|
||||
"github.com/rocboss/paopao-ce/internal/conf"
|
||||
"github.com/rocboss/paopao-ce/internal/core"
|
||||
"github.com/rocboss/paopao-ce/internal/model"
|
||||
"github.com/rocboss/paopao-ce/pkg/json"
|
||||
"github.com/rocboss/paopao-ce/pkg/zinc"
|
||||
)
|
||||
|
||||
func newZincTweetSearchServant() *zincTweetSearchServant {
|
||||
s := conf.ZincSetting
|
||||
zts := &zincTweetSearchServant{
|
||||
indexName: s.Index,
|
||||
client: zinc.NewClient(s),
|
||||
}
|
||||
zts.createIndex()
|
||||
|
||||
return zts
|
||||
}
|
||||
|
||||
func (s *zincTweetSearchServant) Name() string {
|
||||
return "Zinc"
|
||||
}
|
||||
|
||||
func (s *zincTweetSearchServant) Version() *semver.Version {
|
||||
return semver.MustParse("v0.1.0")
|
||||
}
|
||||
|
||||
func (s *zincTweetSearchServant) IndexName() string {
|
||||
return s.indexName
|
||||
}
|
||||
|
||||
func (s *zincTweetSearchServant) AddDocuments(data []map[string]interface{}, primaryKey ...string) (bool, error) {
|
||||
return s.client.BulkPushDoc(data)
|
||||
}
|
||||
|
||||
func (s *zincTweetSearchServant) DeleteDocuments(identifiers []string) error {
|
||||
for _, id := range identifiers {
|
||||
if err := s.client.DelDoc(s.indexName, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *zincTweetSearchServant) Search(q *core.QueryReq, offset, limit int) (*core.QueryResp, error) {
|
||||
if q.Type == core.SearchTypeDefault && q.Query != "" {
|
||||
return s.queryByContent(q, offset, limit)
|
||||
} else if q.Type == core.SearchTypeTag && q.Query != "" {
|
||||
return s.queryByTag(q, offset, limit)
|
||||
}
|
||||
return s.queryAny(offset, limit)
|
||||
}
|
||||
|
||||
func (s *zincTweetSearchServant) queryByContent(q *core.QueryReq, offset, limit int) (*core.QueryResp, error) {
|
||||
resp, err := s.client.EsQuery(s.indexName, map[string]interface{}{
|
||||
"query": map[string]interface{}{
|
||||
"match_phrase": map[string]interface{}{
|
||||
"content": q.Query,
|
||||
},
|
||||
},
|
||||
"sort": []string{"-is_top", "-latest_replied_on"},
|
||||
"from": offset,
|
||||
"size": limit,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.postsFrom(resp)
|
||||
}
|
||||
|
||||
func (s *zincTweetSearchServant) queryByTag(q *core.QueryReq, offset, limit int) (*core.QueryResp, error) {
|
||||
resp, err := s.client.ApiQuery(s.indexName, map[string]interface{}{
|
||||
"search_type": "querystring",
|
||||
"query": map[string]interface{}{
|
||||
"term": "tags." + q.Query + ":1",
|
||||
},
|
||||
"sort_fields": []string{"-is_top", "-latest_replied_on"},
|
||||
"from": offset,
|
||||
"max_results": limit,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.postsFrom(resp)
|
||||
}
|
||||
|
||||
func (s *zincTweetSearchServant) queryAny(offset, limit int) (*core.QueryResp, error) {
|
||||
queryMap := map[string]interface{}{
|
||||
"query": map[string]interface{}{
|
||||
"match_all": map[string]string{},
|
||||
},
|
||||
"sort": []string{"-is_top", "-latest_replied_on"},
|
||||
"from": offset,
|
||||
"size": limit,
|
||||
}
|
||||
resp, err := s.client.EsQuery(s.indexName, queryMap)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.postsFrom(resp)
|
||||
}
|
||||
|
||||
func (s *zincTweetSearchServant) postsFrom(resp *zinc.QueryResultT) (*core.QueryResp, error) {
|
||||
posts := make([]*model.PostFormated, 0, len(resp.Hits.Hits))
|
||||
for _, hit := range resp.Hits.Hits {
|
||||
item := &model.PostFormated{}
|
||||
raw, err := json.Marshal(hit.Source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = json.Unmarshal(raw, item); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
posts = append(posts, item)
|
||||
}
|
||||
|
||||
return &core.QueryResp{
|
||||
Items: posts,
|
||||
Total: resp.Hits.Total.Value,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *zincTweetSearchServant) createIndex() {
|
||||
// 不存在则创建索引
|
||||
s.client.CreateIndex(s.indexName, &zinc.ZincIndexProperty{
|
||||
"id": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Store: true,
|
||||
Sortable: true,
|
||||
},
|
||||
"user_id": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Store: true,
|
||||
},
|
||||
"comment_count": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"collection_count": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"upvote_count": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"visibility": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"is_top": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"is_essence": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"content": &zinc.ZincIndexPropertyT{
|
||||
Type: "text",
|
||||
Index: true,
|
||||
Store: true,
|
||||
Aggregatable: true,
|
||||
Highlightable: true,
|
||||
Analyzer: "gse_search",
|
||||
SearchAnalyzer: "gse_standard",
|
||||
},
|
||||
"tags": &zinc.ZincIndexPropertyT{
|
||||
Type: "keyword",
|
||||
Index: true,
|
||||
Store: true,
|
||||
},
|
||||
"ip_loc": &zinc.ZincIndexPropertyT{
|
||||
Type: "keyword",
|
||||
Index: true,
|
||||
Store: true,
|
||||
},
|
||||
"latest_replied_on": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"attachment_price": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"created_on": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
"modified_on": &zinc.ZincIndexPropertyT{
|
||||
Type: "numeric",
|
||||
Index: true,
|
||||
Sortable: true,
|
||||
Store: true,
|
||||
},
|
||||
})
|
||||
}
|
||||
Loading…
Reference in new issue