support meilisearch as tweet search service

pull/129/head
alimy 3 years ago
parent fe3b694eb4
commit bed9bee806

@ -25,8 +25,8 @@ WORKDIR /paopao-ce
COPY . .
COPY --from=frontend /web/dist ./web/dist
ENV GOPROXY=https://goproxy.cn
RUN [ $EMBED_UI != yes ] || make build TAGS='embed'
RUN [ $EMBED_UI = yes ] || make build
RUN [ $EMBED_UI != yes ] || make build TAGS='embed jsoniter'
RUN [ $EMBED_UI = yes ] || make build TAGS='jsoniter'
FROM alpine:3.16
ARG API_HOST

@ -290,14 +290,19 @@ release/paopao-ce --no-default-features --features sqlite3,localoss,loggerfile,r
目前支持的功能集合:
* 数据库: MySQL/Sqlite3/PostgreSQL
* 对象存储: AliOSS/MinIO/LocalOSS
`AliOSS` 阿里云对象存储服务;
`MinIO` [MinIO](https://github.com/minio/minio)对象存储服务;
`LocalOSS` 提供使用本地目录文件作为对象存储的功能,仅用于开发调试环境;
* 缓存: Redis/SimpleCacheIndex/BigCacheIndex
`SimpleCacheIndex`提供简单的 广场推文列表 的缓存功能;
`SimpleCacheIndex` 提供简单的 广场推文列表 的缓存功能;
`BigCacheIndex` 使用[BigCache](https://github.com/allegro/bigcache)缓存 广场推文列表,缓存每个用户每一页,简单做到千人千面;
* 搜索: Zinc
* 日志: LoggerFile/LoggerZinc
`LoggerFile` 使用文件写日志;
`LoggerZinc` 使用Zinc写日志;
* 搜索: Zinc/Meili
`Zinc` 基于[Zinc](https://github.com/zinclabs/zinc)搜索引擎提供推文搜索服务(目前状态: 稳定,推荐使用)
`Meili` 基于[Meilisearch](https://github.com/meilisearch/meilisearch)搜索引擎提供推文搜索服务(目前状态: 内测阶段);
* 日志: LoggerFile/LoggerZinc/LoggerMeili
`LoggerFile` 使用文件写日志(目前状态: 稳定);
`LoggerZinc` 使用[Zinc](https://github.com/zinclabs/zinc)写日志(目前状态: 稳定,推荐使用);
`LoggerMeili` 使用[Meilisearch](https://github.com/meilisearch/meilisearch)写日志(目前状态: 调试阶段);
* 支付: Alipay
* 短信验证码: SmsJuhe(需要开启sms)
`Sms`功能如果没有开启,任意短信验证码都可以绑定手机;

@ -12,11 +12,11 @@ Server: # 服务设置
ReadTimeout: 60
WriteTimeout: 60
Features:
Default: ["Base", "MySQL", "Option", "LocalOSS", "LoggerFile"]
Develop: ["Base", "MySQL", "BigCacheIndex", "Sms", "AliOSS", "LoggerZinc"]
Demo: ["Base", "MySQL", "Option", "Sms", "MinIO", "LoggerZinc"]
Default: ["Base", "MySQL", "Option", "Zinc", "LocalOSS", "LoggerFile"]
Develop: ["Base", "MySQL", "BigCacheIndex", "Meili", "Sms", "AliOSS", "LoggerMeili"]
Demo: ["Base", "MySQL", "Option", "Zinc", "Sms", "MinIO", "LoggerZinc"]
Slim: ["Base", "Sqlite3", "LocalOSS", "LoggerFile"]
Base: ["Zinc", "Redis", "Alipay",]
Base: ["Redis", "Alipay"]
Option: ["SimpleCacheIndex"]
Sms: "SmsJuhe"
SmsJuhe:
@ -36,17 +36,23 @@ BigCacheIndex: # 使用BigCache缓存泡泡广场消息流
MaxIndexPage: 1024 # 最大缓存页数必须是2^n, 代表最大同时缓存多少页数据
Verbose: False # 是否打印cache操作的log
ExpireInSecond: 300 # 多少秒(>0)后强制过期缓存
Logger: # 日志通用配置
Level: debug # 日志级别 panic|fatal|error|warn|info|debug|trace
LoggerFile: # 使用File写日志
SavePath: data/paopao-ce/logs
FileName: app
FileExt: .log
Level: info
LoggerZinc: # 使用Zinc写日志
Host: http://zinc:4080/es/_bulk
Host: zinc:4080
Index: paopao-log
User: admin
Password: admin
Level: info
Secure: False
LoggerMeili: # 使用Meili写日志
Host: meili:7700
Index: paopao-log
ApiKey: paopao-meilisearch
Secure: False
JWT: # 鉴权加密
Secret: 18a6413dc4fe394c66345ebe501b2f26
Issuer: paopao-api
@ -55,10 +61,16 @@ TweetSearch: # 推文关键字搜索相关配置
MaxUpdateQPS: 100 # 最大添加/删除/更新Post的QPS设置范围[10, 10000], 默认100
MinWorker: 10 # 最小后台更新工作者, 设置范围[5, 1000], 默认10
Zinc: # Zinc搜索配置
Host: http://zinc:4080
Host: zinc:4080
Index: paopao-data
User: admin
Password: admin
Secure: False
Meili: # Meili搜索配置
Host: meili:7700
Index: paopao-data
ApiKey: paopao-meilisearch
Secure: False
AliOSS: # 阿里云OSS存储配置
Endpoint:
AccessKeyID:

@ -39,7 +39,19 @@ services:
DATA_PATH: /data
networks:
- paopao-network
# meili:
# image: getmeili/meilisearch:v0.27.0
# restart: always
# ports:
# - 7700:7700
# volumes:
# - ./data/meili/data:/meili_data
# environment:
# - MEILI_MASTER_KEY=paopao-meilisearch
# networks:
# - paopao-network
phpmyadmin:
image: phpmyadmin:5.2
depends_on:

@ -22,6 +22,7 @@ require (
github.com/gofrs/uuid v4.0.0+incompatible
github.com/google/go-cmp v0.5.7 // indirect
github.com/json-iterator/go v1.1.12
github.com/meilisearch/meilisearch-go v0.19.1
github.com/minio/minio-go/v7 v7.0.27
github.com/sirupsen/logrus v1.8.1
github.com/smartwalle/alipay/v3 v3.1.7
@ -30,7 +31,6 @@ require (
github.com/yinheli/mahonia v0.0.0-20131226213531-0eef680515cc
golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 // indirect
golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9 // indirect
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 // indirect
google.golang.org/protobuf v1.27.1
gopkg.in/natefinch/lumberjack.v2 v2.0.0

@ -87,6 +87,8 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax
github.com/allegro/bigcache/v3 v3.0.2 h1:AKZCw+5eAaVyNTBmI2fgyPVJhHkdWder3O9IrprcQfI=
github.com/allegro/bigcache/v3 v3.0.2/go.mod h1:aPyh7jEvrog9zAwx5N7+JUQX5dZTSGpxF1LAR4dr35I=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0=
github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI=
@ -283,6 +285,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
@ -493,6 +497,8 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@ -516,8 +522,9 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4=
github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U=
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
@ -557,6 +564,8 @@ github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaW
github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
@ -589,6 +598,8 @@ github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ
github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/meilisearch/meilisearch-go v0.19.1 h1:CDi7p5Ev18h0hMXaJZ/1GzSKu3lvPCsaJfrLco3bMEM=
github.com/meilisearch/meilisearch-go v0.19.1/go.mod h1:PnFFq9tELcH5mLVKCoTHRS58B3HEA8vKdBSoG6g/FCE=
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
@ -748,8 +759,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
@ -768,9 +780,13 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY
github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.36.0 h1:NhqfO/cB7Ajn1czkKnWkMHyPYr5nyND14ZGPk23g0/c=
github.com/valyala/fasthttp v1.36.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I=
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
@ -834,6 +850,7 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 h1:syTAU9FwmvzEoIYMqcPHOcVm4H3U5u90WsvuYgwpETU=
golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -931,8 +948,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -1045,6 +1062,7 @@ golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng=
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=

@ -7,14 +7,16 @@ import (
)
var (
loggerFileSetting *LoggerFileSettingS
loggerZincSetting *LoggerZincSettingS
databaseSetting *DatabaseSetingS
mysqlSetting *MySQLSettingS
postgresSetting *PostgresSettingS
sqlite3Setting *Sqlite3SettingS
redisSetting *RedisSettingS
features *FeaturesSettingS
LoggerSetting *LoggerSettingS
loggerFileSetting *LoggerFileSettingS
loggerZincSetting *LoggerZincSettingS
loggerMeiliSetting *LoggerMeiliSettingS
databaseSetting *DatabaseSetingS
mysqlSetting *MySQLSettingS
postgresSetting *PostgresSettingS
sqlite3Setting *Sqlite3SettingS
redisSetting *RedisSettingS
features *FeaturesSettingS
ServerSetting *ServerSettingS
AppSetting *AppSettingS
@ -25,6 +27,7 @@ var (
AlipaySetting *AlipaySettingS
TweetSearchSetting *TweetSearchS
ZincSetting *ZincSettingS
MeiliSetting *MeiliSettingS
AliOSSSetting *AliOSSSettingS
MinIOSetting *MinIOSettingS
S3Setting *S3SettingS
@ -54,14 +57,17 @@ func setupSetting(suite []string, noDefault bool) error {
"BigCacheIndex": &BigCacheIndexSetting,
"Alipay": &AlipaySetting,
"SmsJuhe": &SmsJuheSetting,
"Logger": &LoggerSetting,
"LoggerFile": &loggerFileSetting,
"LoggerZinc": &loggerZincSetting,
"LoggerMeili": &loggerMeiliSetting,
"Database": &databaseSetting,
"MySQL": &mysqlSetting,
"Postgres": &postgresSetting,
"Sqlite3": &sqlite3Setting,
"TweetSearch": &TweetSearchSetting,
"Zinc": &ZincSetting,
"Meili": &MeiliSetting,
"Redis": &redisSetting,
"JWT": &JWTSetting,
"AliOSS": &AliOSSSetting,

@ -5,23 +5,24 @@ import (
"io"
"time"
"github.com/meilisearch/meilisearch-go"
"github.com/rocboss/paopao-ce/pkg/json"
"github.com/sirupsen/logrus"
"gopkg.in/natefinch/lumberjack.v2"
"gopkg.in/resty.v1"
)
type zincLogIndex struct {
Index map[string]string `json:"index"`
}
type zincLogData struct {
type logData struct {
Time time.Time `json:"time"`
Level logrus.Level `json:"level"`
Message string `json:"message"`
Data logrus.Fields `json:"data"`
}
type zincLogIndex struct {
Index map[string]string `json:"index"`
}
type zincLogHook struct {
host string
index string
@ -29,6 +30,10 @@ type zincLogHook struct {
password string
}
type meiliLogHook struct {
index *meilisearch.Index
}
func (h *zincLogHook) Fire(entry *logrus.Entry) error {
index := &zincLogIndex{
Index: map[string]string{
@ -37,7 +42,7 @@ func (h *zincLogHook) Fire(entry *logrus.Entry) error {
}
indexBytes, _ := json.Marshal(index)
data := &zincLogData{
data := &logData{
Time: entry.Time,
Level: entry.Level,
Message: entry.Message,
@ -63,30 +68,73 @@ func (h *zincLogHook) Levels() []logrus.Level {
return logrus.AllLevels
}
func (h *meiliLogHook) Fire(entry *logrus.Entry) error {
data := &logData{
Time: entry.Time,
Level: entry.Level,
Message: entry.Message,
Data: entry.Data,
}
if _, err := h.index.AddDocuments(data); err != nil {
fmt.Println(err.Error())
}
return nil
}
func (h *meiliLogHook) Levels() []logrus.Level {
return logrus.AllLevels
}
func newZincLogHook() *zincLogHook {
return &zincLogHook{
host: loggerZincSetting.Endpoint() + "/es/_bulk",
index: loggerZincSetting.Index,
user: loggerZincSetting.User,
password: loggerZincSetting.Password,
}
}
func newMeiliLogHook() *meiliLogHook {
client := meilisearch.NewClient(meilisearch.ClientConfig{
Host: loggerMeiliSetting.Endpoint(),
APIKey: loggerMeiliSetting.ApiKey,
})
index := client.Index(loggerMeiliSetting.Index)
if _, err := index.FetchInfo(); err != nil {
logrus.Debugf("newMeiliLogHook create index because fetch index info error: %v", err)
client.CreateIndex(&meilisearch.IndexConfig{
Uid: loggerMeiliSetting.Index,
})
}
return &meiliLogHook{
index: index,
}
}
func newFileLogger() io.Writer {
return &lumberjack.Logger{
Filename: loggerFileSetting.SavePath + "/" + loggerFileSetting.FileName + loggerFileSetting.FileExt,
MaxSize: 600,
MaxAge: 10,
LocalTime: true,
}
}
func setupLogger() {
logrus.SetFormatter(&logrus.JSONFormatter{})
logrus.SetLevel(LoggerSetting.logLevel())
if CfgIf("LoggerFile") {
out := &lumberjack.Logger{
Filename: loggerFileSetting.SavePath + "/" + loggerFileSetting.FileName + loggerFileSetting.FileExt,
MaxSize: 600,
MaxAge: 10,
LocalTime: true,
}
out := newFileLogger()
logrus.SetOutput(out)
if level, err := logrus.ParseLevel(loggerFileSetting.Level); err == nil {
logrus.SetLevel(level)
}
} else if CfgIf("LoggerZinc") {
hook := &zincLogHook{
host: loggerZincSetting.Host,
index: loggerZincSetting.Index,
user: loggerZincSetting.User,
password: loggerZincSetting.Password,
}
if level, err := logrus.ParseLevel(loggerZincSetting.Level); err == nil {
logrus.SetLevel(level)
}
hook := newZincLogHook()
logrus.SetOutput(io.Discard)
logrus.AddHook(hook)
} else if CfgIf("LoggerMeili") {
hook := newMeiliLogHook()
logrus.SetOutput(io.Discard)
logrus.AddHook(hook)
}

@ -5,6 +5,7 @@ import (
"strings"
"time"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
"gorm.io/gorm/logger"
)
@ -13,11 +14,14 @@ type Setting struct {
vp *viper.Viper
}
type LoggerSettingS struct {
Level string
}
type LoggerFileSettingS struct {
SavePath string
FileName string
FileExt string
Level string
}
type LoggerZincSettingS struct {
@ -25,7 +29,14 @@ type LoggerZincSettingS struct {
Index string
User string
Password string
Level string
Secure bool
}
type LoggerMeiliSettingS struct {
Host string
Index string
ApiKey string
Secure bool
}
type ServerSettingS struct {
@ -91,6 +102,14 @@ type ZincSettingS struct {
Index string
User string
Password string
Secure bool
}
type MeiliSettingS struct {
Host string
Index string
ApiKey string
Secure bool
}
type DatabaseSetingS struct {
@ -312,3 +331,48 @@ func (s *DatabaseSetingS) logLevel() logger.LogLevel {
return logger.Error
}
}
func (s *LoggerSettingS) logLevel() logrus.Level {
switch strings.ToLower(s.Level) {
case "panic":
return logrus.PanicLevel
case "fatal":
return logrus.FatalLevel
case "error":
return logrus.ErrorLevel
case "warn", "warning":
return logrus.WarnLevel
case "info":
return logrus.InfoLevel
case "debug":
return logrus.DebugLevel
case "trace":
return logrus.TraceLevel
default:
return logrus.ErrorLevel
}
}
func (s *LoggerZincSettingS) Endpoint() string {
return endpoint(s.Host, s.Secure)
}
func (s *LoggerMeiliSettingS) Endpoint() string {
return endpoint(s.Host, s.Secure)
}
func (s *ZincSettingS) Endpoint() string {
return endpoint(s.Host, s.Secure)
}
func (s *MeiliSettingS) Endpoint() string {
return endpoint(s.Host, s.Secure)
}
func endpoint(host string, secure bool) string {
schema := "http"
if secure {
schema = "https"
}
return schema + "://" + host
}

@ -1,6 +1,7 @@
package dao
import (
"github.com/meilisearch/meilisearch-go"
"github.com/rocboss/paopao-ce/internal/conf"
"github.com/rocboss/paopao-ce/internal/core"
"github.com/rocboss/paopao-ce/pkg/zinc"
@ -28,6 +29,11 @@ type zincTweetSearchServant struct {
client *zinc.ZincClient
}
type meiliTweetSearchServant struct {
client *meilisearch.Client
index *meilisearch.Index
}
func NewTweetSearchService() core.TweetSearchService {
bts := &bridgeTweetSearchServant{}
@ -41,6 +47,8 @@ func NewTweetSearchService() core.TweetSearchService {
if conf.CfgIf("Zinc") {
bts.ts = newZincTweetSearchServant()
} else if conf.CfgIf("Meili") {
bts.ts = newMeiliTweetSearchServant()
} else {
// default use Zinc as tweet search service
bts.ts = newZincTweetSearchServant()

@ -0,0 +1,139 @@
package dao
import (
"github.com/Masterminds/semver/v3"
"github.com/meilisearch/meilisearch-go"
"github.com/rocboss/paopao-ce/internal/conf"
"github.com/rocboss/paopao-ce/internal/core"
"github.com/rocboss/paopao-ce/internal/model"
"github.com/rocboss/paopao-ce/pkg/json"
"github.com/sirupsen/logrus"
)
func newMeiliTweetSearchServant() *meiliTweetSearchServant {
s := conf.MeiliSetting
client := meilisearch.NewClient(meilisearch.ClientConfig{
Host: s.Endpoint(),
APIKey: s.ApiKey,
})
if _, err := client.Index(s.Index).FetchInfo(); err != nil {
logrus.Debugf("create index because fetch index info error: %v", err)
client.CreateIndex(&meilisearch.IndexConfig{
Uid: s.Index,
PrimaryKey: "id",
})
searchableAttributes := []string{"content", "tags"}
sortableAttributes := []string{"is_top", "latest_replied_on"}
index := client.Index(s.Index)
index.UpdateSearchableAttributes(&searchableAttributes)
index.UpdateSortableAttributes(&sortableAttributes)
}
return &meiliTweetSearchServant{
client: client,
index: client.Index(s.Index),
}
}
func (s *meiliTweetSearchServant) Name() string {
return "Meili"
}
func (s *meiliTweetSearchServant) Version() *semver.Version {
return semver.MustParse("v0.1.0")
}
func (s *meiliTweetSearchServant) IndexName() string {
return s.index.UID
}
func (s *meiliTweetSearchServant) AddDocuments(data core.DocItems, primaryKey ...string) (bool, error) {
task, err := s.index.AddDocuments(data, primaryKey...)
if err != nil {
logrus.Errorf("meiliTweetSearchServant.AddDocuments error: %v", err)
return false, err
}
logrus.Debugf("meiliTweetSearchServant.AddDocuments task: %+v", task.Details)
return true, nil
}
func (s *meiliTweetSearchServant) DeleteDocuments(identifiers []string) error {
task, err := s.index.DeleteDocuments(identifiers)
if err != nil {
logrus.Errorf("meiliTweetSearchServant.DeleteDocuments error: %v", err)
return err
}
logrus.Debugf("meiliTweetSearchServant.DeleteDocuments task: %+v", task.Details)
return nil
}
func (s *meiliTweetSearchServant) Search(q *core.QueryReq, offset, limit int) (*core.QueryResp, error) {
if q.Type == core.SearchTypeDefault && q.Query != "" {
return s.queryByContent(q, offset, limit)
} else if q.Type == core.SearchTypeTag && q.Query != "" {
return s.queryByTag(q, offset, limit)
}
return s.queryAny(offset, limit)
}
func (s *meiliTweetSearchServant) queryByContent(q *core.QueryReq, offset, limit int) (*core.QueryResp, error) {
resp, err := s.index.Search(q.Query, &meilisearch.SearchRequest{
Offset: int64(offset),
Limit: int64(limit),
AttributesToRetrieve: []string{"*"},
Sort: []string{"is_top:desc", "latest_replied_on:desc"},
})
if err != nil {
return nil, err
}
return s.postsFrom(resp)
}
func (s *meiliTweetSearchServant) queryByTag(q *core.QueryReq, offset, limit int) (*core.QueryResp, error) {
resp, err := s.index.Search("#"+q.Query, &meilisearch.SearchRequest{
Offset: int64(offset),
Limit: int64(limit),
AttributesToRetrieve: []string{"*"},
Sort: []string{"is_top:desc", "latest_replied_on:desc"},
})
if err != nil {
return nil, err
}
return s.postsFrom(resp)
}
func (s *meiliTweetSearchServant) queryAny(offset, limit int) (*core.QueryResp, error) {
resp, err := s.index.Search("", &meilisearch.SearchRequest{
Offset: int64(offset),
Limit: int64(limit),
Matches: true,
Sort: []string{"is_top:desc", "latest_replied_on:desc"},
})
if err != nil {
return nil, err
}
return s.postsFrom(resp)
}
func (s *meiliTweetSearchServant) postsFrom(resp *meilisearch.SearchResponse) (*core.QueryResp, error) {
logrus.Debugf("resp Hits:%d NbHits:%d offset: %d limit:%d ", len(resp.Hits), resp.NbHits, resp.Offset, resp.Limit)
posts := make([]*model.PostFormated, 0, len(resp.Hits))
for _, hit := range resp.Hits {
item := &model.PostFormated{}
raw, err := json.Marshal(hit)
if err != nil {
return nil, err
}
if err = json.Unmarshal(raw, item); err != nil {
return nil, err
}
posts = append(posts, item)
}
return &core.QueryResp{
Items: posts,
Total: resp.NbHits,
}, nil
}

@ -33,6 +33,22 @@ func (s *zincTweetSearchServant) IndexName() string {
}
func (s *zincTweetSearchServant) AddDocuments(data core.DocItems, primaryKey ...string) (bool, error) {
buf := make(core.DocItems, 0, len(data)+1)
if len(primaryKey) > 0 {
buf = append(buf, map[string]interface{}{
"index": map[string]interface{}{
"_index": s.indexName,
"_id": primaryKey[0],
},
})
} else {
buf = append(buf, map[string]interface{}{
"index": map[string]interface{}{
"_index": s.indexName,
},
})
}
buf = append(buf, data...)
return s.client.BulkPushDoc(data)
}

@ -525,13 +525,7 @@ func PushPostToSearch(post *model.Post) {
tagMaps[tag] = 1
}
data := core.DocItems{}
data = append(data, map[string]interface{}{
"index": map[string]interface{}{
"_index": ts.IndexName(),
"_id": fmt.Sprintf("%d", post.ID),
},
}, map[string]interface{}{
data := core.DocItems{{
"id": post.ID,
"user_id": post.UserID,
"comment_count": post.CommentCount,
@ -547,9 +541,9 @@ func PushPostToSearch(post *model.Post) {
"attachment_price": post.AttachmentPrice,
"created_on": post.CreatedOn,
"modified_on": post.ModifiedOn,
})
}}
ts.AddDocuments(data)
ts.AddDocuments(data, fmt.Sprintf("%d", post.ID))
}
func DeleteSearchPost(post *model.Post) error {
@ -567,8 +561,6 @@ func PushPostsToSearch(c *gin.Context) {
nums := int(pages)
for i := 0; i < nums; i++ {
data := []map[string]interface{}{}
posts, _ := GetPostList(&PostListReq{
Conditions: &model.ConditionsT{
"visibility IN ?": []model.PostVisibleT{model.PostVisitPublic, model.PostVisitFriend},
@ -586,12 +578,7 @@ func PushPostsToSearch(c *gin.Context) {
}
}
data = append(data, map[string]interface{}{
"index": map[string]interface{}{
"_index": ts.IndexName(),
"_id": fmt.Sprintf("%d", post.ID),
},
}, map[string]interface{}{
docs := core.DocItems{{
"id": post.ID,
"user_id": post.User.ID,
"comment_count": post.CommentCount,
@ -607,11 +594,8 @@ func PushPostsToSearch(c *gin.Context) {
"attachment_price": post.AttachmentPrice,
"created_on": post.CreatedOn,
"modified_on": post.ModifiedOn,
})
}
if len(data) > 0 {
ts.AddDocuments(data)
}}
ts.AddDocuments(docs, fmt.Sprintf("%d", post.ID))
}
}

@ -74,7 +74,7 @@ type HitItem struct {
func NewClient(conf *conf.ZincSettingS) *ZincClient {
return &ZincClient{
ZincClientConfig: &ZincClientConfig{
ZincHost: conf.Host,
ZincHost: conf.Endpoint(),
ZincUser: conf.User,
ZincPassword: conf.Password,
},
@ -90,7 +90,7 @@ func (c *ZincClient) CreateIndex(name string, p *ZincIndexProperty) bool {
Properties: p,
},
}
resp, err := c.request().SetBody(data).Put(c.ZincHost + "/api/index")
resp, err := c.request().SetBody(data).Put("/api/index")
if err != nil || resp.StatusCode() != http.StatusOK {
return false
@ -101,7 +101,7 @@ func (c *ZincClient) CreateIndex(name string, p *ZincIndexProperty) bool {
// 检查索引是否存在
func (c *ZincClient) ExistIndex(name string) bool {
resp, err := c.request().Get(c.ZincHost + "/api/index")
resp, err := c.request().Get("/api/index")
if err != nil || resp.StatusCode() != http.StatusOK {
return false
@ -122,7 +122,7 @@ func (c *ZincClient) ExistIndex(name string) bool {
// 新增/更新文档
func (c *ZincClient) PutDoc(name string, id int64, doc interface{}) (bool, error) {
resp, err := c.request().SetBody(doc).Put(fmt.Sprintf("%s/api/%s/_doc/%d", c.ZincHost, name, id))
resp, err := c.request().SetBody(doc).Put(fmt.Sprintf("/api/%s/_doc/%d", name, id))
if err != nil {
return false, err
@ -145,7 +145,7 @@ func (c *ZincClient) BulkPushDoc(docs []map[string]interface{}) (bool, error) {
}
}
resp, err := c.request().SetBody(dataStr).Post(fmt.Sprintf("%s/api/_bulk", c.ZincHost))
resp, err := c.request().SetBody(dataStr).Post("/api/_bulk")
if err != nil {
return false, err
}
@ -158,7 +158,7 @@ func (c *ZincClient) BulkPushDoc(docs []map[string]interface{}) (bool, error) {
}
func (c *ZincClient) EsQuery(indexName string, q interface{}) (*QueryResultT, error) {
resp, err := c.request().SetBody(q).Post(fmt.Sprintf("%s/es/%s/_search", c.ZincHost, indexName))
resp, err := c.request().SetBody(q).Post(fmt.Sprintf("/es/%s/_search", indexName))
if err != nil {
return nil, err
}
@ -177,7 +177,7 @@ func (c *ZincClient) EsQuery(indexName string, q interface{}) (*QueryResultT, er
}
func (c *ZincClient) ApiQuery(indexName string, q interface{}) (*QueryResultT, error) {
resp, err := c.request().SetBody(q).Post(fmt.Sprintf("%s/api/%s/_search", c.ZincHost, indexName))
resp, err := c.request().SetBody(q).Post(fmt.Sprintf("/api/%s/_search", indexName))
if err != nil {
return nil, err
}
@ -196,7 +196,7 @@ func (c *ZincClient) ApiQuery(indexName string, q interface{}) (*QueryResultT, e
}
func (c *ZincClient) DelDoc(indexName, id string) error {
resp, err := c.request().Delete(fmt.Sprintf("%s/api/%s/_doc/%s", c.ZincHost, indexName, id))
resp, err := c.request().Delete(fmt.Sprintf("/api/%s/_doc/%s", indexName, id))
if err != nil {
return err
}
@ -211,6 +211,7 @@ func (c *ZincClient) DelDoc(indexName, id string) error {
func (c *ZincClient) request() *resty.Request {
client := resty.New()
client.DisableWarn = true
client.SetBaseURL(c.ZincHost)
client.SetBasicAuth(c.ZincUser, c.ZincPassword)
return client.R()

Loading…
Cancel
Save