From bed9bee806bbe71133fb13e1339950932e88e0b5 Mon Sep 17 00:00:00 2001 From: alimy Date: Thu, 23 Jun 2022 18:44:16 +0800 Subject: [PATCH] support meilisearch as tweet search service --- Dockerfile | 4 +- README.md | 15 ++-- config.yaml.sample | 28 +++++-- docker-compose.yaml | 14 +++- go.mod | 2 +- go.sum | 26 ++++++- internal/conf/conf.go | 22 ++++-- internal/conf/logger.go | 96 ++++++++++++++++++------ internal/conf/settting.go | 68 ++++++++++++++++- internal/dao/search.go | 8 ++ internal/dao/search_meili.go | 139 +++++++++++++++++++++++++++++++++++ internal/dao/search_zinc.go | 16 ++++ internal/service/post.go | 28 ++----- pkg/zinc/zinc.go | 17 +++-- 14 files changed, 398 insertions(+), 85 deletions(-) create mode 100644 internal/dao/search_meili.go diff --git a/Dockerfile b/Dockerfile index d04b9afa..7378fd8b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,8 +25,8 @@ WORKDIR /paopao-ce COPY . . COPY --from=frontend /web/dist ./web/dist ENV GOPROXY=https://goproxy.cn -RUN [ $EMBED_UI != yes ] || make build TAGS='embed' -RUN [ $EMBED_UI = yes ] || make build +RUN [ $EMBED_UI != yes ] || make build TAGS='embed jsoniter' +RUN [ $EMBED_UI = yes ] || make build TAGS='jsoniter' FROM alpine:3.16 ARG API_HOST diff --git a/README.md b/README.md index fc80c9c5..09e8dba5 100644 --- a/README.md +++ b/README.md @@ -290,14 +290,19 @@ release/paopao-ce --no-default-features --features sqlite3,localoss,loggerfile,r 目前支持的功能集合: * 数据库: MySQL/Sqlite3/PostgreSQL * 对象存储: AliOSS/MinIO/LocalOSS + `AliOSS` 阿里云对象存储服务； + `MinIO` [MinIO](https://github.com/minio/minio)对象存储服务； `LocalOSS` 提供使用本地目录文件作为对象存储的功能，仅用于开发调试环境； * 缓存: Redis/SimpleCacheIndex/BigCacheIndex - `SimpleCacheIndex`提供简单的 广场推文列表 的缓存功能； + `SimpleCacheIndex` 提供简单的 广场推文列表 的缓存功能； `BigCacheIndex` 使用[BigCache](https://github.com/allegro/bigcache)缓存 广场推文列表，缓存每个用户每一页，简单做到千人千面； -* 搜索: Zinc -* 日志: LoggerFile/LoggerZinc - `LoggerFile` 使用文件写日志； - `LoggerZinc` 使用Zinc写日志; +* 搜索: Zinc/Meili + `Zinc` 基于[Zinc](https://github.com/zinclabs/zinc)搜索引擎提供推文搜索服务(目前状态: 稳定，推荐使用)； + `Meili` 基于[Meilisearch](https://github.com/meilisearch/meilisearch)搜索引擎提供推文搜索服务(目前状态: 内测阶段); +* 日志: LoggerFile/LoggerZinc/LoggerMeili + `LoggerFile` 使用文件写日志(目前状态: 稳定); + `LoggerZinc` 使用[Zinc](https://github.com/zinclabs/zinc)写日志(目前状态: 稳定，推荐使用); + `LoggerMeili` 使用[Meilisearch](https://github.com/meilisearch/meilisearch)写日志(目前状态: 调试阶段); * 支付: Alipay * 短信验证码: SmsJuhe(需要开启sms) `Sms`功能如果没有开启，任意短信验证码都可以绑定手机； diff --git a/config.yaml.sample b/config.yaml.sample index dc4a81ac..ff57d086 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -12,11 +12,11 @@ Server: # 服务设置 ReadTimeout: 60 WriteTimeout: 60 Features: - Default: ["Base", "MySQL", "Option", "LocalOSS", "LoggerFile"] - Develop: ["Base", "MySQL", "BigCacheIndex", "Sms", "AliOSS", "LoggerZinc"] - Demo: ["Base", "MySQL", "Option", "Sms", "MinIO", "LoggerZinc"] + Default: ["Base", "MySQL", "Option", "Zinc", "LocalOSS", "LoggerFile"] + Develop: ["Base", "MySQL", "BigCacheIndex", "Meili", "Sms", "AliOSS", "LoggerMeili"] + Demo: ["Base", "MySQL", "Option", "Zinc", "Sms", "MinIO", "LoggerZinc"] Slim: ["Base", "Sqlite3", "LocalOSS", "LoggerFile"] - Base: ["Zinc", "Redis", "Alipay",] + Base: ["Redis", "Alipay"] Option: ["SimpleCacheIndex"] Sms: "SmsJuhe" SmsJuhe: @@ -36,17 +36,23 @@ BigCacheIndex: # 使用BigCache缓存泡泡广场消息流 MaxIndexPage: 1024 # 最大缓存页数，必须是2^n, 代表最大同时缓存多少页数据 Verbose: False # 是否打印cache操作的log ExpireInSecond: 300 # 多少秒(>0)后强制过期缓存 +Logger: # 日志通用配置 + Level: debug # 日志级别 panic|fatal|error|warn|info|debug|trace LoggerFile: # 使用File写日志 SavePath: data/paopao-ce/logs FileName: app FileExt: .log - Level: info LoggerZinc: # 使用Zinc写日志 - Host: http://zinc:4080/es/_bulk + Host: zinc:4080 Index: paopao-log User: admin Password: admin - Level: info + Secure: False +LoggerMeili: # 使用Meili写日志 + Host: meili:7700 + Index: paopao-log + ApiKey: paopao-meilisearch + Secure: False JWT: # 鉴权加密 Secret: 18a6413dc4fe394c66345ebe501b2f26 Issuer: paopao-api @@ -55,10 +61,16 @@ TweetSearch: # 推文关键字搜索相关配置 MaxUpdateQPS: 100 # 最大添加/删除/更新Post的QPS，设置范围[10, 10000], 默认100 MinWorker: 10 # 最小后台更新工作者, 设置范围[5, 1000], 默认10 Zinc: # Zinc搜索配置 - Host: http://zinc:4080 + Host: zinc:4080 Index: paopao-data User: admin Password: admin + Secure: False +Meili: # Meili搜索配置 + Host: meili:7700 + Index: paopao-data + ApiKey: paopao-meilisearch + Secure: False AliOSS: # 阿里云OSS存储配置 Endpoint: AccessKeyID: diff --git a/docker-compose.yaml b/docker-compose.yaml index 3ae2b783..101ef67c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -39,7 +39,19 @@ services: DATA_PATH: /data networks: - paopao-network - + + # meili: + # image: getmeili/meilisearch:v0.27.0 + # restart: always + # ports: + # - 7700:7700 + # volumes: + # - ./data/meili/data:/meili_data + # environment: + # - MEILI_MASTER_KEY=paopao-meilisearch + # networks: + # - paopao-network + phpmyadmin: image: phpmyadmin:5.2 depends_on: diff --git a/go.mod b/go.mod index 331a99d5..a87f9d3c 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/gofrs/uuid v4.0.0+incompatible github.com/google/go-cmp v0.5.7 // indirect github.com/json-iterator/go v1.1.12 + github.com/meilisearch/meilisearch-go v0.19.1 github.com/minio/minio-go/v7 v7.0.27 github.com/sirupsen/logrus v1.8.1 github.com/smartwalle/alipay/v3 v3.1.7 @@ -30,7 +31,6 @@ require ( github.com/yinheli/mahonia v0.0.0-20131226213531-0eef680515cc golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 // indirect golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9 // indirect - golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 // indirect google.golang.org/protobuf v1.27.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 diff --git a/go.sum b/go.sum index fe11f231..2d899cbb 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,8 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax github.com/allegro/bigcache/v3 v3.0.2 h1:AKZCw+5eAaVyNTBmI2fgyPVJhHkdWder3O9IrprcQfI= github.com/allegro/bigcache/v3 v3.0.2/go.mod h1:aPyh7jEvrog9zAwx5N7+JUQX5dZTSGpxF1LAR4dr35I= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI= @@ -283,6 +285,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= @@ -493,6 +497,8 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -516,8 +522,9 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U= +github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= @@ -557,6 +564,8 @@ github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaW github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -589,6 +598,8 @@ github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/meilisearch/meilisearch-go v0.19.1 h1:CDi7p5Ev18h0hMXaJZ/1GzSKu3lvPCsaJfrLco3bMEM= +github.com/meilisearch/meilisearch-go v0.19.1/go.mod h1:PnFFq9tELcH5mLVKCoTHRS58B3HEA8vKdBSoG6g/FCE= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= @@ -748,8 +759,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= @@ -768,9 +780,13 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.36.0 h1:NhqfO/cB7Ajn1czkKnWkMHyPYr5nyND14ZGPk23g0/c= +github.com/valyala/fasthttp v1.36.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= @@ -834,6 +850,7 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 h1:syTAU9FwmvzEoIYMqcPHOcVm4H3U5u90WsvuYgwpETU= golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -931,8 +948,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1045,6 +1062,7 @@ golang.org/x/sys v0.0.0-20211102061401-a2f17f7b995c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/internal/conf/conf.go b/internal/conf/conf.go index 6f22f46f..818fdb0d 100644 --- a/internal/conf/conf.go +++ b/internal/conf/conf.go @@ -7,14 +7,16 @@ import ( ) var ( - loggerFileSetting *LoggerFileSettingS - loggerZincSetting *LoggerZincSettingS - databaseSetting *DatabaseSetingS - mysqlSetting *MySQLSettingS - postgresSetting *PostgresSettingS - sqlite3Setting *Sqlite3SettingS - redisSetting *RedisSettingS - features *FeaturesSettingS + LoggerSetting *LoggerSettingS + loggerFileSetting *LoggerFileSettingS + loggerZincSetting *LoggerZincSettingS + loggerMeiliSetting *LoggerMeiliSettingS + databaseSetting *DatabaseSetingS + mysqlSetting *MySQLSettingS + postgresSetting *PostgresSettingS + sqlite3Setting *Sqlite3SettingS + redisSetting *RedisSettingS + features *FeaturesSettingS ServerSetting *ServerSettingS AppSetting *AppSettingS @@ -25,6 +27,7 @@ var ( AlipaySetting *AlipaySettingS TweetSearchSetting *TweetSearchS ZincSetting *ZincSettingS + MeiliSetting *MeiliSettingS AliOSSSetting *AliOSSSettingS MinIOSetting *MinIOSettingS S3Setting *S3SettingS @@ -54,14 +57,17 @@ func setupSetting(suite []string, noDefault bool) error { "BigCacheIndex": &BigCacheIndexSetting, "Alipay": &AlipaySetting, "SmsJuhe": &SmsJuheSetting, + "Logger": &LoggerSetting, "LoggerFile": &loggerFileSetting, "LoggerZinc": &loggerZincSetting, + "LoggerMeili": &loggerMeiliSetting, "Database": &databaseSetting, "MySQL": &mysqlSetting, "Postgres": &postgresSetting, "Sqlite3": &sqlite3Setting, "TweetSearch": &TweetSearchSetting, "Zinc": &ZincSetting, + "Meili": &MeiliSetting, "Redis": &redisSetting, "JWT": &JWTSetting, "AliOSS": &AliOSSSetting, diff --git a/internal/conf/logger.go b/internal/conf/logger.go index 16a2b8f7..7e8f787b 100644 --- a/internal/conf/logger.go +++ b/internal/conf/logger.go @@ -5,23 +5,24 @@ import ( "io" "time" + "github.com/meilisearch/meilisearch-go" "github.com/rocboss/paopao-ce/pkg/json" "github.com/sirupsen/logrus" "gopkg.in/natefinch/lumberjack.v2" "gopkg.in/resty.v1" ) -type zincLogIndex struct { - Index map[string]string `json:"index"` -} - -type zincLogData struct { +type logData struct { Time time.Time `json:"time"` Level logrus.Level `json:"level"` Message string `json:"message"` Data logrus.Fields `json:"data"` } +type zincLogIndex struct { + Index map[string]string `json:"index"` +} + type zincLogHook struct { host string index string @@ -29,6 +30,10 @@ type zincLogHook struct { password string } +type meiliLogHook struct { + index *meilisearch.Index +} + func (h *zincLogHook) Fire(entry *logrus.Entry) error { index := &zincLogIndex{ Index: map[string]string{ @@ -37,7 +42,7 @@ func (h *zincLogHook) Fire(entry *logrus.Entry) error { } indexBytes, _ := json.Marshal(index) - data := &zincLogData{ + data := &logData{ Time: entry.Time, Level: entry.Level, Message: entry.Message, @@ -63,30 +68,73 @@ func (h *zincLogHook) Levels() []logrus.Level { return logrus.AllLevels } +func (h *meiliLogHook) Fire(entry *logrus.Entry) error { + data := &logData{ + Time: entry.Time, + Level: entry.Level, + Message: entry.Message, + Data: entry.Data, + } + if _, err := h.index.AddDocuments(data); err != nil { + fmt.Println(err.Error()) + } + return nil +} + +func (h *meiliLogHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +func newZincLogHook() *zincLogHook { + return &zincLogHook{ + host: loggerZincSetting.Endpoint() + "/es/_bulk", + index: loggerZincSetting.Index, + user: loggerZincSetting.User, + password: loggerZincSetting.Password, + } +} + +func newMeiliLogHook() *meiliLogHook { + client := meilisearch.NewClient(meilisearch.ClientConfig{ + Host: loggerMeiliSetting.Endpoint(), + APIKey: loggerMeiliSetting.ApiKey, + }) + + index := client.Index(loggerMeiliSetting.Index) + if _, err := index.FetchInfo(); err != nil { + logrus.Debugf("newMeiliLogHook create index because fetch index info error: %v", err) + client.CreateIndex(&meilisearch.IndexConfig{ + Uid: loggerMeiliSetting.Index, + }) + } + + return &meiliLogHook{ + index: index, + } +} + +func newFileLogger() io.Writer { + return &lumberjack.Logger{ + Filename: loggerFileSetting.SavePath + "/" + loggerFileSetting.FileName + loggerFileSetting.FileExt, + MaxSize: 600, + MaxAge: 10, + LocalTime: true, + } +} + func setupLogger() { logrus.SetFormatter(&logrus.JSONFormatter{}) + logrus.SetLevel(LoggerSetting.logLevel()) if CfgIf("LoggerFile") { - out := &lumberjack.Logger{ - Filename: loggerFileSetting.SavePath + "/" + loggerFileSetting.FileName + loggerFileSetting.FileExt, - MaxSize: 600, - MaxAge: 10, - LocalTime: true, - } + out := newFileLogger() logrus.SetOutput(out) - if level, err := logrus.ParseLevel(loggerFileSetting.Level); err == nil { - logrus.SetLevel(level) - } } else if CfgIf("LoggerZinc") { - hook := &zincLogHook{ - host: loggerZincSetting.Host, - index: loggerZincSetting.Index, - user: loggerZincSetting.User, - password: loggerZincSetting.Password, - } - if level, err := logrus.ParseLevel(loggerZincSetting.Level); err == nil { - logrus.SetLevel(level) - } + hook := newZincLogHook() + logrus.SetOutput(io.Discard) + logrus.AddHook(hook) + } else if CfgIf("LoggerMeili") { + hook := newMeiliLogHook() logrus.SetOutput(io.Discard) logrus.AddHook(hook) } diff --git a/internal/conf/settting.go b/internal/conf/settting.go index 9bceceb9..4b3f9ae3 100644 --- a/internal/conf/settting.go +++ b/internal/conf/settting.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "github.com/sirupsen/logrus" "github.com/spf13/viper" "gorm.io/gorm/logger" ) @@ -13,11 +14,14 @@ type Setting struct { vp *viper.Viper } +type LoggerSettingS struct { + Level string +} + type LoggerFileSettingS struct { SavePath string FileName string FileExt string - Level string } type LoggerZincSettingS struct { @@ -25,7 +29,14 @@ type LoggerZincSettingS struct { Index string User string Password string - Level string + Secure bool +} + +type LoggerMeiliSettingS struct { + Host string + Index string + ApiKey string + Secure bool } type ServerSettingS struct { @@ -91,6 +102,14 @@ type ZincSettingS struct { Index string User string Password string + Secure bool +} + +type MeiliSettingS struct { + Host string + Index string + ApiKey string + Secure bool } type DatabaseSetingS struct { @@ -312,3 +331,48 @@ func (s *DatabaseSetingS) logLevel() logger.LogLevel { return logger.Error } } + +func (s *LoggerSettingS) logLevel() logrus.Level { + switch strings.ToLower(s.Level) { + case "panic": + return logrus.PanicLevel + case "fatal": + return logrus.FatalLevel + case "error": + return logrus.ErrorLevel + case "warn", "warning": + return logrus.WarnLevel + case "info": + return logrus.InfoLevel + case "debug": + return logrus.DebugLevel + case "trace": + return logrus.TraceLevel + default: + return logrus.ErrorLevel + } +} + +func (s *LoggerZincSettingS) Endpoint() string { + return endpoint(s.Host, s.Secure) +} + +func (s *LoggerMeiliSettingS) Endpoint() string { + return endpoint(s.Host, s.Secure) +} + +func (s *ZincSettingS) Endpoint() string { + return endpoint(s.Host, s.Secure) +} + +func (s *MeiliSettingS) Endpoint() string { + return endpoint(s.Host, s.Secure) +} + +func endpoint(host string, secure bool) string { + schema := "http" + if secure { + schema = "https" + } + return schema + "://" + host +} diff --git a/internal/dao/search.go b/internal/dao/search.go index 0b25f4ca..04791618 100644 --- a/internal/dao/search.go +++ b/internal/dao/search.go @@ -1,6 +1,7 @@ package dao import ( + "github.com/meilisearch/meilisearch-go" "github.com/rocboss/paopao-ce/internal/conf" "github.com/rocboss/paopao-ce/internal/core" "github.com/rocboss/paopao-ce/pkg/zinc" @@ -28,6 +29,11 @@ type zincTweetSearchServant struct { client *zinc.ZincClient } +type meiliTweetSearchServant struct { + client *meilisearch.Client + index *meilisearch.Index +} + func NewTweetSearchService() core.TweetSearchService { bts := &bridgeTweetSearchServant{} @@ -41,6 +47,8 @@ func NewTweetSearchService() core.TweetSearchService { if conf.CfgIf("Zinc") { bts.ts = newZincTweetSearchServant() + } else if conf.CfgIf("Meili") { + bts.ts = newMeiliTweetSearchServant() } else { // default use Zinc as tweet search service bts.ts = newZincTweetSearchServant() diff --git a/internal/dao/search_meili.go b/internal/dao/search_meili.go new file mode 100644 index 00000000..12d7335c --- /dev/null +++ b/internal/dao/search_meili.go @@ -0,0 +1,139 @@ +package dao + +import ( + "github.com/Masterminds/semver/v3" + "github.com/meilisearch/meilisearch-go" + "github.com/rocboss/paopao-ce/internal/conf" + "github.com/rocboss/paopao-ce/internal/core" + "github.com/rocboss/paopao-ce/internal/model" + "github.com/rocboss/paopao-ce/pkg/json" + "github.com/sirupsen/logrus" +) + +func newMeiliTweetSearchServant() *meiliTweetSearchServant { + s := conf.MeiliSetting + client := meilisearch.NewClient(meilisearch.ClientConfig{ + Host: s.Endpoint(), + APIKey: s.ApiKey, + }) + + if _, err := client.Index(s.Index).FetchInfo(); err != nil { + logrus.Debugf("create index because fetch index info error: %v", err) + client.CreateIndex(&meilisearch.IndexConfig{ + Uid: s.Index, + PrimaryKey: "id", + }) + searchableAttributes := []string{"content", "tags"} + sortableAttributes := []string{"is_top", "latest_replied_on"} + + index := client.Index(s.Index) + index.UpdateSearchableAttributes(&searchableAttributes) + index.UpdateSortableAttributes(&sortableAttributes) + } + + return &meiliTweetSearchServant{ + client: client, + index: client.Index(s.Index), + } +} + +func (s *meiliTweetSearchServant) Name() string { + return "Meili" +} + +func (s *meiliTweetSearchServant) Version() *semver.Version { + return semver.MustParse("v0.1.0") +} + +func (s *meiliTweetSearchServant) IndexName() string { + return s.index.UID +} + +func (s *meiliTweetSearchServant) AddDocuments(data core.DocItems, primaryKey ...string) (bool, error) { + task, err := s.index.AddDocuments(data, primaryKey...) + if err != nil { + logrus.Errorf("meiliTweetSearchServant.AddDocuments error: %v", err) + return false, err + } + logrus.Debugf("meiliTweetSearchServant.AddDocuments task: %+v", task.Details) + return true, nil +} + +func (s *meiliTweetSearchServant) DeleteDocuments(identifiers []string) error { + task, err := s.index.DeleteDocuments(identifiers) + if err != nil { + logrus.Errorf("meiliTweetSearchServant.DeleteDocuments error: %v", err) + return err + } + logrus.Debugf("meiliTweetSearchServant.DeleteDocuments task: %+v", task.Details) + return nil +} + +func (s *meiliTweetSearchServant) Search(q *core.QueryReq, offset, limit int) (*core.QueryResp, error) { + if q.Type == core.SearchTypeDefault && q.Query != "" { + return s.queryByContent(q, offset, limit) + } else if q.Type == core.SearchTypeTag && q.Query != "" { + return s.queryByTag(q, offset, limit) + } + return s.queryAny(offset, limit) +} + +func (s *meiliTweetSearchServant) queryByContent(q *core.QueryReq, offset, limit int) (*core.QueryResp, error) { + resp, err := s.index.Search(q.Query, &meilisearch.SearchRequest{ + Offset: int64(offset), + Limit: int64(limit), + AttributesToRetrieve: []string{"*"}, + Sort: []string{"is_top:desc", "latest_replied_on:desc"}, + }) + if err != nil { + return nil, err + } + return s.postsFrom(resp) +} + +func (s *meiliTweetSearchServant) queryByTag(q *core.QueryReq, offset, limit int) (*core.QueryResp, error) { + resp, err := s.index.Search("#"+q.Query, &meilisearch.SearchRequest{ + Offset: int64(offset), + Limit: int64(limit), + AttributesToRetrieve: []string{"*"}, + Sort: []string{"is_top:desc", "latest_replied_on:desc"}, + }) + if err != nil { + return nil, err + } + return s.postsFrom(resp) +} + +func (s *meiliTweetSearchServant) queryAny(offset, limit int) (*core.QueryResp, error) { + resp, err := s.index.Search("", &meilisearch.SearchRequest{ + Offset: int64(offset), + Limit: int64(limit), + Matches: true, + Sort: []string{"is_top:desc", "latest_replied_on:desc"}, + }) + if err != nil { + return nil, err + } + return s.postsFrom(resp) +} + +func (s *meiliTweetSearchServant) postsFrom(resp *meilisearch.SearchResponse) (*core.QueryResp, error) { + logrus.Debugf("resp Hits:%d NbHits:%d offset: %d limit:%d ", len(resp.Hits), resp.NbHits, resp.Offset, resp.Limit) + posts := make([]*model.PostFormated, 0, len(resp.Hits)) + for _, hit := range resp.Hits { + item := &model.PostFormated{} + raw, err := json.Marshal(hit) + if err != nil { + return nil, err + } + if err = json.Unmarshal(raw, item); err != nil { + return nil, err + } + posts = append(posts, item) + } + + return &core.QueryResp{ + Items: posts, + Total: resp.NbHits, + }, nil +} diff --git a/internal/dao/search_zinc.go b/internal/dao/search_zinc.go index 6cda0503..debae521 100644 --- a/internal/dao/search_zinc.go +++ b/internal/dao/search_zinc.go @@ -33,6 +33,22 @@ func (s *zincTweetSearchServant) IndexName() string { } func (s *zincTweetSearchServant) AddDocuments(data core.DocItems, primaryKey ...string) (bool, error) { + buf := make(core.DocItems, 0, len(data)+1) + if len(primaryKey) > 0 { + buf = append(buf, map[string]interface{}{ + "index": map[string]interface{}{ + "_index": s.indexName, + "_id": primaryKey[0], + }, + }) + } else { + buf = append(buf, map[string]interface{}{ + "index": map[string]interface{}{ + "_index": s.indexName, + }, + }) + } + buf = append(buf, data...) return s.client.BulkPushDoc(data) } diff --git a/internal/service/post.go b/internal/service/post.go index 2401f0ef..35ac6764 100644 --- a/internal/service/post.go +++ b/internal/service/post.go @@ -525,13 +525,7 @@ func PushPostToSearch(post *model.Post) { tagMaps[tag] = 1 } - data := core.DocItems{} - data = append(data, map[string]interface{}{ - "index": map[string]interface{}{ - "_index": ts.IndexName(), - "_id": fmt.Sprintf("%d", post.ID), - }, - }, map[string]interface{}{ + data := core.DocItems{{ "id": post.ID, "user_id": post.UserID, "comment_count": post.CommentCount, @@ -547,9 +541,9 @@ func PushPostToSearch(post *model.Post) { "attachment_price": post.AttachmentPrice, "created_on": post.CreatedOn, "modified_on": post.ModifiedOn, - }) + }} - ts.AddDocuments(data) + ts.AddDocuments(data, fmt.Sprintf("%d", post.ID)) } func DeleteSearchPost(post *model.Post) error { @@ -567,8 +561,6 @@ func PushPostsToSearch(c *gin.Context) { nums := int(pages) for i := 0; i < nums; i++ { - data := []map[string]interface{}{} - posts, _ := GetPostList(&PostListReq{ Conditions: &model.ConditionsT{ "visibility IN ?": []model.PostVisibleT{model.PostVisitPublic, model.PostVisitFriend}, @@ -586,12 +578,7 @@ func PushPostsToSearch(c *gin.Context) { } } - data = append(data, map[string]interface{}{ - "index": map[string]interface{}{ - "_index": ts.IndexName(), - "_id": fmt.Sprintf("%d", post.ID), - }, - }, map[string]interface{}{ + docs := core.DocItems{{ "id": post.ID, "user_id": post.User.ID, "comment_count": post.CommentCount, @@ -607,11 +594,8 @@ func PushPostsToSearch(c *gin.Context) { "attachment_price": post.AttachmentPrice, "created_on": post.CreatedOn, "modified_on": post.ModifiedOn, - }) - } - - if len(data) > 0 { - ts.AddDocuments(data) + }} + ts.AddDocuments(docs, fmt.Sprintf("%d", post.ID)) } } diff --git a/pkg/zinc/zinc.go b/pkg/zinc/zinc.go index 63dfeeb1..8d280124 100644 --- a/pkg/zinc/zinc.go +++ b/pkg/zinc/zinc.go @@ -74,7 +74,7 @@ type HitItem struct { func NewClient(conf *conf.ZincSettingS) *ZincClient { return &ZincClient{ ZincClientConfig: &ZincClientConfig{ - ZincHost: conf.Host, + ZincHost: conf.Endpoint(), ZincUser: conf.User, ZincPassword: conf.Password, }, @@ -90,7 +90,7 @@ func (c *ZincClient) CreateIndex(name string, p *ZincIndexProperty) bool { Properties: p, }, } - resp, err := c.request().SetBody(data).Put(c.ZincHost + "/api/index") + resp, err := c.request().SetBody(data).Put("/api/index") if err != nil || resp.StatusCode() != http.StatusOK { return false @@ -101,7 +101,7 @@ func (c *ZincClient) CreateIndex(name string, p *ZincIndexProperty) bool { // 检查索引是否存在 func (c *ZincClient) ExistIndex(name string) bool { - resp, err := c.request().Get(c.ZincHost + "/api/index") + resp, err := c.request().Get("/api/index") if err != nil || resp.StatusCode() != http.StatusOK { return false @@ -122,7 +122,7 @@ func (c *ZincClient) ExistIndex(name string) bool { // 新增/更新文档 func (c *ZincClient) PutDoc(name string, id int64, doc interface{}) (bool, error) { - resp, err := c.request().SetBody(doc).Put(fmt.Sprintf("%s/api/%s/_doc/%d", c.ZincHost, name, id)) + resp, err := c.request().SetBody(doc).Put(fmt.Sprintf("/api/%s/_doc/%d", name, id)) if err != nil { return false, err @@ -145,7 +145,7 @@ func (c *ZincClient) BulkPushDoc(docs []map[string]interface{}) (bool, error) { } } - resp, err := c.request().SetBody(dataStr).Post(fmt.Sprintf("%s/api/_bulk", c.ZincHost)) + resp, err := c.request().SetBody(dataStr).Post("/api/_bulk") if err != nil { return false, err } @@ -158,7 +158,7 @@ func (c *ZincClient) BulkPushDoc(docs []map[string]interface{}) (bool, error) { } func (c *ZincClient) EsQuery(indexName string, q interface{}) (*QueryResultT, error) { - resp, err := c.request().SetBody(q).Post(fmt.Sprintf("%s/es/%s/_search", c.ZincHost, indexName)) + resp, err := c.request().SetBody(q).Post(fmt.Sprintf("/es/%s/_search", indexName)) if err != nil { return nil, err } @@ -177,7 +177,7 @@ func (c *ZincClient) EsQuery(indexName string, q interface{}) (*QueryResultT, er } func (c *ZincClient) ApiQuery(indexName string, q interface{}) (*QueryResultT, error) { - resp, err := c.request().SetBody(q).Post(fmt.Sprintf("%s/api/%s/_search", c.ZincHost, indexName)) + resp, err := c.request().SetBody(q).Post(fmt.Sprintf("/api/%s/_search", indexName)) if err != nil { return nil, err } @@ -196,7 +196,7 @@ func (c *ZincClient) ApiQuery(indexName string, q interface{}) (*QueryResultT, e } func (c *ZincClient) DelDoc(indexName, id string) error { - resp, err := c.request().Delete(fmt.Sprintf("%s/api/%s/_doc/%s", c.ZincHost, indexName, id)) + resp, err := c.request().Delete(fmt.Sprintf("/api/%s/_doc/%s", indexName, id)) if err != nil { return err } @@ -211,6 +211,7 @@ func (c *ZincClient) DelDoc(indexName, id string) error { func (c *ZincClient) request() *resty.Request { client := resty.New() client.DisableWarn = true + client.SetBaseURL(c.ZincHost) client.SetBasicAuth(c.ZincUser, c.ZincPassword) return client.R()