diff --git a/config.yaml.sample b/config.yaml.sample index ae78ddd5..4b66e9e8 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -12,9 +12,9 @@ Server: # 服务设置 ReadTimeout: 60 WriteTimeout: 60 Features: - Default: ["Alipay", "Zinc", "MySQL", "Redis", "AliOSS", "LoggerZinc"] - Develop: ["Sms", "Zinc", "MySQL", "AliOSS", "LoggerFile"] - Slim: ["Zinc", "MySQL", "Redis", "AliOSS", "LoggerFile"] + Default: ["Alipay", "Zinc", "MySQL", "Redis", "MinIO", "LoggerFile"] + Develop: ["Sms", "Zinc", "MySQL", "AliOSS", "LoggerZinc"] + Slim: ["Zinc", "MySQL", "Redis", "MinIO", "LoggerFile"] Sms: "SmsJuhe" SmsJuhe: Key: @@ -42,11 +42,25 @@ Zinc: # Zinc搜索配置 User: admin Password: admin AliOSS: # 阿里云OSS存储配置 + Endpoint: AccessKeyID: AccessKeySecret: - Endpoint: Bucket: Domain: +MinIO: # MinIO 存储配置 + AccessKey: Q3AM3UQ867SPQQA43P2F + SecretKey: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG + Secure: False + Endpoint: minio:9000 + Bucket: paopao + Domain: 127.0.0.1:9000 +S3: # Amazon S3 存储配置 + AccessKey: "YOUR-ACCESSKEYID" + SecretKey: "YOUR-SECRETACCESSKEY" + Secure: True + Endpoint: s3.amazonaws.com + Bucket: paopao + Domain: MySQL: # MySQL数据库 Username: paopao Password: paopao diff --git a/docker-compose.yaml b/docker-compose.yaml index 8751fc76..7a2387f9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -25,6 +25,21 @@ services: networks: - paopao-network + minio: + image: bitnami/minio:latest + restart: always + environment: + MINIO_ROOT_USER: minio-root-user + MINIO_ROOT_PASSWORD: minio-root-password + MINIO_DEFAULT_BUCKETS: paopao:public + ports: + - 127.0.0.1:9000:9000 + - 127.0.0.1:9001:9001 + volumes: + - ./data/minio/data:/data + networks: + - paopao-network + zinc: image: public.ecr.aws/prabhat/zinc:latest restart: always @@ -47,12 +62,13 @@ services: - db - redis - zinc + - minio # modify below to reflect your custom configure volumes: - ./config.yaml.sample:/app/paopao-ce/config.yaml - ./data/paopao-ce/data:/app/paopao-ce/data ports: - - 8008:8008 + - 127.0.0.1:8008:8008 networks: - paopao-network diff --git a/global/setting.go b/global/setting.go index f4d49a66..75889707 100644 --- a/global/setting.go +++ b/global/setting.go @@ -17,6 +17,8 @@ var ( AlipaySetting *setting.AlipaySettingS ZincSetting *setting.ZincSettingS AliOSSSetting *setting.AliOSSSettingS + MinIOSetting *setting.MinIOSettingS + S3Setting *setting.S3SettingS JWTSetting *setting.JWTSettingS LoggerFileSetting *setting.LoggerFileSettingS LoggerZincSetting *setting.LoggerZincSettingS diff --git a/go.mod b/go.mod index 2fc783ce..9a84bf6e 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/go-resty/resty/v2 v2.7.0 github.com/gofrs/uuid v3.3.0+incompatible github.com/google/go-cmp v0.5.7 // indirect + github.com/minio/minio-go/v7 v7.0.27 github.com/satori/go.uuid v1.2.0 // indirect github.com/sirupsen/logrus v1.8.1 github.com/smartwalle/alipay/v3 v3.1.7 diff --git a/go.sum b/go.sum index 9aaf5068..70483ca9 100644 --- a/go.sum +++ b/go.sum @@ -183,6 +183,8 @@ github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwu github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -346,7 +348,9 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.5 h1:kxhtnfFVi+rYdOALN0B3k9UT86zVJKfBimRaciULW4I= github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -448,7 +452,12 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= +github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -508,7 +517,14 @@ github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4f github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/minio-go/v7 v7.0.27 h1:yJCvm78B+2+ll1PqO9eSD1as6Ibw3IYnnD8PyBEB2zo= +github.com/minio/minio-go/v7 v7.0.27/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -597,6 +613,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM= @@ -720,6 +738,7 @@ golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -895,6 +914,7 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1183,6 +1203,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= diff --git a/init.go b/init.go index 6d05edb9..7cc5369b 100644 --- a/init.go +++ b/init.go @@ -10,6 +10,7 @@ import ( "github.com/go-redis/redis/v8" "github.com/rocboss/paopao-ce/global" "github.com/rocboss/paopao-ce/internal/model" + "github.com/rocboss/paopao-ce/internal/routers/api" "github.com/rocboss/paopao-ce/internal/service" "github.com/rocboss/paopao-ce/pkg/logger" "github.com/rocboss/paopao-ce/pkg/setting" @@ -54,6 +55,7 @@ func init() { client := zinc.NewClient(global.ZincSetting) service.Initialize(global.DBEngine, client) + api.Initialize() } func setupSetting() error { @@ -81,6 +83,8 @@ func setupSetting() error { "Redis": &global.RedisSetting, "JWT": &global.JWTSetting, "AliOSS": &global.AliOSSSetting, + "MinIO": &global.MinIOSetting, + "S3": &global.S3Setting, } if err = setting.Unmarshal(objects); err != nil { return err diff --git a/internal/core/assets.go b/internal/core/assets.go new file mode 100644 index 00000000..d2bf7228 --- /dev/null +++ b/internal/core/assets.go @@ -0,0 +1,5 @@ +package core + +type AttachmentCheckService interface { + Check(cUrl string) error +} diff --git a/internal/core/core.go b/internal/core/core.go index 15ba27f6..7a0b426f 100644 --- a/internal/core/core.go +++ b/internal/core/core.go @@ -8,7 +8,6 @@ import ( type DataService interface { WalletService SearchService - StorageService GetComments(conditions *model.ConditionsT, offset, limit int) ([]*model.Comment, error) GetCommentByID(id int64) (*model.Comment, error) @@ -29,6 +28,7 @@ type DataService interface { GetMessages(conditions *model.ConditionsT, offset, limit int) ([]*model.MessageFormated, error) GetMessageCount(conditions *model.ConditionsT) (int64, error) + CreateAttachment(attachment *model.Attachment) (*model.Attachment, error) CreatePost(post *model.Post) (*model.Post, error) DeletePost(post *model.Post) error LockPost(post *model.Post) error diff --git a/internal/core/storage.go b/internal/core/storage.go index 158c250b..f1aa7ce7 100644 --- a/internal/core/storage.go +++ b/internal/core/storage.go @@ -1,10 +1,13 @@ package core import ( - "github.com/rocboss/paopao-ce/internal/model" + "io" ) -// StorageService storage service interface that implement base AliOSS、MINIO or other -type StorageService interface { - CreateAttachment(attachment *model.Attachment) (*model.Attachment, error) +// ObjectStorageService storage service interface that implement base AliOSS、MINIO or other +type ObjectStorageService interface { + PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) + SignURL(objectKey string, expiredInSec int64) (string, error) + ObjectURL(objetKey string) string + ObjectKey(cUrl string) string } diff --git a/internal/dao/attachment.go b/internal/dao/attachment.go index 43500d54..3bb1c82c 100644 --- a/internal/dao/attachment.go +++ b/internal/dao/attachment.go @@ -1,7 +1,19 @@ package dao -import "github.com/rocboss/paopao-ce/internal/model" +import ( + "fmt" + "strings" + + "github.com/rocboss/paopao-ce/internal/model" +) func (d *dataServant) CreateAttachment(attachment *model.Attachment) (*model.Attachment, error) { return attachment.Create(d.engine) } + +func (s *attachmentCheckServant) Check(uri string) error { + if strings.Index(uri, s.domain) != 0 { + return fmt.Errorf("附件非本站资源") + } + return nil +} diff --git a/internal/dao/dao.go b/internal/dao/dao.go index 5bb900e8..9f6080a2 100644 --- a/internal/dao/dao.go +++ b/internal/dao/dao.go @@ -1,13 +1,20 @@ package dao import ( + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/minio/minio-go/v7" + "github.com/rocboss/paopao-ce/global" "github.com/rocboss/paopao-ce/internal/core" "github.com/rocboss/paopao-ce/pkg/zinc" "gorm.io/gorm" ) var ( - _ core.DataService = (*dataServant)(nil) + _ core.DataService = (*dataServant)(nil) + _ core.ObjectStorageService = (*aliossServant)(nil) + _ core.ObjectStorageService = (*minioServant)(nil) + _ core.ObjectStorageService = (*s3Servant)(nil) + _ core.AttachmentCheckService = (*attachmentCheckServant)(nil) ) type dataServant struct { @@ -15,9 +22,50 @@ type dataServant struct { zinc *zinc.ZincClient } +type aliossServant struct { + bucket *oss.Bucket + domain string +} + +type minioServant struct { + client *minio.Client + bucket string + domain string +} + +type s3Servant = minioServant + +type attachmentCheckServant struct { + domain string +} + func NewDataService(engine *gorm.DB, zinc *zinc.ZincClient) core.DataService { return &dataServant{ engine: engine, zinc: zinc, } } + +func NewObjectStorageService() (oss core.ObjectStorageService) { + if global.CfgIf("AliOSS") { + oss = newAliossServent() + global.Logger.Infoln("use AliOSS as object storage") + } else if global.CfgIf("MinIO") { + oss = newMinioServeant() + global.Logger.Infoln("use MinIO as object storage") + } else if global.CfgIf("S3") { + oss = newS3Servent() + global.Logger.Infoln("use S3 as object storage") + } else { + // default use AliOSS + oss = newAliossServent() + global.Logger.Infoln("use default AliOSS as object storage") + } + return +} + +func NewAttachmentCheckerService() core.AttachmentCheckService { + return &attachmentCheckServant{ + domain: getOssDomain(), + } +} diff --git a/internal/dao/oss_alioss.go b/internal/dao/oss_alioss.go new file mode 100644 index 00000000..b3669480 --- /dev/null +++ b/internal/dao/oss_alioss.go @@ -0,0 +1,66 @@ +package dao + +import ( + "io" + "net/url" + "strings" + + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/rocboss/paopao-ce/global" +) + +func newAliossServent() *aliossServant { + client, err := oss.New(global.AliOSSSetting.Endpoint, global.AliOSSSetting.AccessKeyID, global.AliOSSSetting.AccessKeySecret) + if err != nil { + global.Logger.Fatalf("alioss.New err: %v", err) + } + + bucket, err := client.Bucket(global.AliOSSSetting.Bucket) + if err != nil { + global.Logger.Fatalf("client.Bucket err: %v", err) + } + + return &aliossServant{ + bucket: bucket, + domain: getOssDomain(), + } +} + +func (s *aliossServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) { + err := s.bucket.PutObject(objectKey, reader, oss.ContentLength(objectSize), oss.ContentType(contentType)) + if err != nil { + return "", err + } + return s.domain + objectKey, nil +} + +func (s *aliossServant) SignURL(objectKey string, expiredInSec int64) (string, error) { + signedURL, err := s.bucket.SignURL(objectKey, oss.HTTPGet, expiredInSec) + if err != nil { + global.Logger.Errorf("client.SignURL err: %v", err) + return "", err + } + + ur, err := url.Parse(signedURL) + if err != nil { + global.Logger.Errorf("url.Parse err: %v", err) + return "", err + } + + epath, err := url.PathUnescape(ur.Path) + if err != nil { + global.Logger.Errorf("url.PathUnescape err: %v", err) + return "", err + } + + ur.Path, ur.RawPath = epath, epath + return ur.String(), nil +} + +func (s *aliossServant) ObjectURL(objetKey string) string { + return s.domain + objetKey +} + +func (s *aliossServant) ObjectKey(objectUrl string) string { + return strings.Replace(objectUrl, s.domain, "", -1) +} diff --git a/internal/dao/oss_minio.go b/internal/dao/oss_minio.go new file mode 100644 index 00000000..260ba20a --- /dev/null +++ b/internal/dao/oss_minio.go @@ -0,0 +1,55 @@ +package dao + +import ( + "context" + "io" + "net/url" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/rocboss/paopao-ce/global" +) + +func newMinioServeant() *minioServant { + // Initialize minio client object. + client, err := minio.New(global.MinIOSetting.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(global.MinIOSetting.AccessKey, global.MinIOSetting.SecretKey, ""), + Secure: global.MinIOSetting.Secure, + }) + if err != nil { + global.Logger.Fatalf("minio.New err: %v", err) + } + return &minioServant{ + client: client, + bucket: global.MinIOSetting.Bucket, + domain: getOssDomain(), + } +} + +func (s *minioServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) { + uploadInfo, err := s.client.PutObject(context.Background(), s.bucket, objectKey, reader, objectSize, minio.PutObjectOptions{ContentType: contentType}) + if err != nil { + return "", err + } + global.Logger.Infoln("Successfully uploaded bytes: ", uploadInfo) + return s.domain + objectKey, nil +} +func (s *minioServant) SignURL(objectKey string, expiredInSec int64) (string, error) { + // TODO: Set request parameters for content-disposition. + reqParams := make(url.Values) + signedURL, err := s.client.PresignedGetObject(context.Background(), s.bucket, objectKey, time.Duration(expiredInSec)*time.Second, reqParams) + if err != nil { + return "", err + } + return signedURL.String(), nil +} + +func (s *minioServant) ObjectURL(objetKey string) string { + return s.domain + objetKey +} + +func (s *minioServant) ObjectKey(objectUrl string) string { + return strings.Replace(objectUrl, s.domain, "", -1) +} diff --git a/internal/dao/oss_s3.go b/internal/dao/oss_s3.go new file mode 100644 index 00000000..23eb1859 --- /dev/null +++ b/internal/dao/oss_s3.go @@ -0,0 +1,23 @@ +package dao + +import ( + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/rocboss/paopao-ce/global" +) + +func newS3Servent() *s3Servant { + // Initialize s3 client object use minio-go. + client, err := minio.New(global.S3Setting.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(global.S3Setting.AccessKey, global.S3Setting.SecretKey, ""), + Secure: global.S3Setting.Secure, + }) + if err != nil { + global.Logger.Fatalf("s3.New err: %v", err) + } + return &s3Servant{ + client: client, + bucket: global.MinIOSetting.Bucket, + domain: getOssDomain(), + } +} diff --git a/internal/dao/utils.go b/internal/dao/utils.go new file mode 100644 index 00000000..e7d37f55 --- /dev/null +++ b/internal/dao/utils.go @@ -0,0 +1,24 @@ +package dao + +import ( + "github.com/rocboss/paopao-ce/global" +) + +func getOssDomain() string { + uri := "https://" + if global.CfgIf("AliOSS") { + return uri + global.AliOSSSetting.Domain + "/" + } else if global.CfgIf("MinIO") { + if !global.MinIOSetting.Secure { + uri = "http://" + } + return uri + global.MinIOSetting.Domain + "/" + global.MinIOSetting.Bucket + "/" + } else if global.CfgIf("S3") { + if !global.S3Setting.Secure { + uri = "http://" + } + // TODO: will not work well need test in real world + return uri + global.S3Setting.Domain + "/" + global.S3Setting.Bucket + "/" + } + return "" +} diff --git a/internal/routers/api/api.go b/internal/routers/api/api.go new file mode 100644 index 00000000..9049e110 --- /dev/null +++ b/internal/routers/api/api.go @@ -0,0 +1,16 @@ +package api + +import ( + "github.com/rocboss/paopao-ce/internal/core" + "github.com/rocboss/paopao-ce/internal/dao" +) + +var ( + objectStorage core.ObjectStorageService + attachmentChecker core.AttachmentCheckService +) + +func Initialize() { + objectStorage = dao.NewObjectStorageService() + attachmentChecker = dao.NewAttachmentCheckerService() +} diff --git a/internal/routers/api/attachment.go b/internal/routers/api/attachment.go index 9723cea8..6bceed32 100644 --- a/internal/routers/api/attachment.go +++ b/internal/routers/api/attachment.go @@ -2,10 +2,7 @@ package api import ( "image" - "net/url" - "strings" - "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/disintegration/imaging" "github.com/gin-gonic/gin" "github.com/gofrs/uuid" @@ -17,6 +14,13 @@ import ( "github.com/rocboss/paopao-ce/pkg/errcode" ) +var uploadAttachmentTypeMap = map[string]model.AttachmentType{ + "public/image": model.ATTACHMENT_TYPE_IMAGE, + "public/avatar": model.ATTACHMENT_TYPE_IMAGE, + "public/video": model.ATTACHMENT_TYPE_VIDEO, + "attachment": model.ATTACHMENT_TYPE_OTHER, +} + func GeneratePath(s string) string { n := len(s) if n <= 2 { @@ -86,6 +90,7 @@ func UploadAttachment(c *gin.Context) { return } + contentType := fileHeader.Header.Get("Content-Type") fileExt, err := GetFileExt(fileHeader.Header.Get("Content-Type")) if err != nil { global.Logger.Errorf("GetFileExt err: %v", err) @@ -97,23 +102,9 @@ func UploadAttachment(c *gin.Context) { randomPath := uuid.Must(uuid.NewV4()).String() ossSavePath := uploadType + "/" + GeneratePath(randomPath[:8]) + "/" + randomPath[9:] + fileExt - client, err := oss.New(global.AliOSSSetting.Endpoint, global.AliOSSSetting.AccessKeyID, global.AliOSSSetting.AccessKeySecret) + objectUrl, err := objectStorage.PutObject(ossSavePath, file, fileHeader.Size, contentType) if err != nil { - global.Logger.Errorf("oss.New err: %v", err) - response.ToErrorResponse(errcode.FileUploadFailed) - return - } - - bucket, err := client.Bucket(global.AliOSSSetting.Bucket) - if err != nil { - global.Logger.Errorf("client.Bucket err: %v", err) - response.ToErrorResponse(errcode.FileUploadFailed) - return - } - - err = bucket.PutObject(ossSavePath, file) - if err != nil { - global.Logger.Errorf("bucket.PutObject err: %v", err) + global.Logger.Errorf("putObject err: %v", err) response.ToErrorResponse(errcode.FileUploadFailed) return } @@ -121,20 +112,13 @@ func UploadAttachment(c *gin.Context) { // 构造附件Model attachment := &model.Attachment{ FileSize: fileHeader.Size, - Content: "https://" + global.AliOSSSetting.Domain + "/" + ossSavePath, + Content: objectUrl, } if userID, exists := c.Get("UID"); exists { attachment.UserID = userID.(int64) } - var uploadAttachmentTypeMap = map[string]model.AttachmentType{ - "public/image": model.ATTACHMENT_TYPE_IMAGE, - "public/avatar": model.ATTACHMENT_TYPE_IMAGE, - "public/video": model.ATTACHMENT_TYPE_VIDEO, - "attachment": model.ATTACHMENT_TYPE_OTHER, - } - attachment.Type = uploadAttachmentTypeMap[uploadType] if attachment.Type == model.ATTACHMENT_TYPE_IMAGE { var src image.Image @@ -255,43 +239,12 @@ func DownloadAttachment(c *gin.Context) { } } - // 开始构造下载地址 - client, err := oss.New(global.AliOSSSetting.Endpoint, global.AliOSSSetting.AccessKeyID, global.AliOSSSetting.AccessKeySecret) - if err != nil { - global.Logger.Errorf("oss.New err: %v", err) - response.ToErrorResponse(errcode.DownloadReqError) - return - } - - bucket, err := client.Bucket(global.AliOSSSetting.Bucket) - if err != nil { - global.Logger.Errorf("client.Bucket err: %v", err) - response.ToErrorResponse(errcode.DownloadReqError) - return - } - - // 签名 - objectKey := strings.Replace(content.Content, "https://"+global.AliOSSSetting.Domain+"/", "", -1) - signedURL, err := bucket.SignURL(objectKey, oss.HTTPGet, 60) + objectKey := objectStorage.ObjectKey(content.Content) + signedURL, err := objectStorage.SignURL(objectKey, 60) if err != nil { global.Logger.Errorf("client.SignURL err: %v", err) response.ToErrorResponse(errcode.DownloadReqError) return } - ur, err := url.Parse(signedURL) - if err != nil { - global.Logger.Errorf("url.Parse err: %v", err) - response.ToErrorResponse(errcode.DownloadReqError) - return - } - epath, err := url.PathUnescape(ur.Path) - if err != nil { - global.Logger.Errorf("url.PathUnescape err: %v", err) - response.ToErrorResponse(errcode.DownloadReqError) - return - } - - ur.Path = epath - ur.RawPath = epath - response.ToResponse(ur.String()) + response.ToResponse(signedURL) } diff --git a/internal/routers/api/user.go b/internal/routers/api/user.go index 30b1da97..30f53155 100644 --- a/internal/routers/api/user.go +++ b/internal/routers/api/user.go @@ -3,7 +3,6 @@ package api import ( "fmt" "net/http" - "strings" "unicode/utf8" "github.com/gin-gonic/gin" @@ -204,7 +203,7 @@ func ChangeAvatar(c *gin.Context) { user = u.(*model.User) } - if strings.Index(param.Avatar, "https://"+global.AliOSSSetting.Domain) != 0 { + if err := attachmentChecker.Check(param.Avatar); err != nil { response.ToErrorResponse(errcode.InvalidParams) return } diff --git a/internal/service/comment.go b/internal/service/comment.go index 0d745eb8..7a70dd51 100644 --- a/internal/service/comment.go +++ b/internal/service/comment.go @@ -1,7 +1,6 @@ package service import ( - "strings" "time" "github.com/gin-gonic/gin" @@ -116,7 +115,7 @@ func CreatePostComment(ctx *gin.Context, userID int64, param CommentCreationReq) for _, item := range param.Contents { // 检查附件是否是本站资源 if item.Type == model.CONTENT_TYPE_IMAGE || item.Type == model.CONTENT_TYPE_VIDEO || item.Type == model.CONTENT_TYPE_ATTACHMENT { - if strings.Index(item.Content, "https://"+global.AliOSSSetting.Domain) != 0 { + if err := attachmentChecker.Check(item.Content); err != nil { continue } } diff --git a/internal/service/post.go b/internal/service/post.go index a3711b73..4942196c 100644 --- a/internal/service/post.go +++ b/internal/service/post.go @@ -68,8 +68,8 @@ func (p *PostContentItem) Check() error { // 检查附件是否是本站资源 if p.Type == model.CONTENT_TYPE_IMAGE || p.Type == model.CONTENT_TYPE_VIDEO || p.Type == model. CONTENT_TYPE_ATTACHMENT { - if strings.Index(p.Content, "https://"+global.AliOSSSetting.Domain) != 0 { - return fmt.Errorf("附件非本站资源") + if err := attachmentChecker.Check(p.Content); err != nil { + return err } } // 检查链接是否合法 @@ -109,6 +109,7 @@ func CreatePost(c *gin.Context, userID int64, param PostCreationReq) (*model.Pos for _, item := range param.Contents { if err = item.Check(); err != nil { // 属性非法 + global.Logger.Infof("contents check err: %v", err) continue } diff --git a/internal/service/service.go b/internal/service/service.go index 5f50f546..6e8a1116 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -10,10 +10,12 @@ import ( var ( ds core.DataService + attachmentChecker core.AttachmentCheckService DisablePhoneVerify bool ) func Initialize(engine *gorm.DB, client *zinc.ZincClient) { ds = dao.NewDataService(engine, client) + attachmentChecker = dao.NewAttachmentCheckerService() DisablePhoneVerify = !global.CfgIf("Sms") } diff --git a/pkg/setting/settting.go b/pkg/setting/settting.go index 693c400b..f880247e 100644 --- a/pkg/setting/settting.go +++ b/pkg/setting/settting.go @@ -81,12 +81,22 @@ type MySQLSettingS struct { MaxOpenConns int } -type AliossSettingS struct { - AliossAccessKeyID string - AliossAccessKeySecret string - AliossEndpoint string - AliossBucket string - AliossDomain string +type MinIOSettingS struct { + AccessKey string + SecretKey string + Secure bool + Endpoint string + Bucket string + Domain string +} + +type S3SettingS struct { + AccessKey string + SecretKey string + Secure bool + Endpoint string + Bucket string + Domain string } type AliOSSSettingS struct {