From 9f9041ead9ec209b7f999554e0405969aeb5cb45 Mon Sep 17 00:00:00 2001 From: Michael Li Date: Tue, 4 Apr 2023 22:54:30 +0800 Subject: [PATCH 1/2] optimize Makefile --- Makefile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 561f9220..2d7f244c 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,10 @@ TARGET = paopao-ce ifeq ($(OS),Windows_NT) TARGET := $(TARGET).exe endif +TARGET_BIN = $(basename $(TARGET)) ifeq (n$(CGO_ENABLED),n) -CGO_ENABLED := 1 +CGO_ENABLED := 0 endif RELEASE_ROOT = release @@ -51,22 +52,22 @@ release: linux-amd64 darwin-amd64 darwin-arm64 windows-x64 .PHONY: linux-amd64 linux-amd64: @echo Build paopao-ce [linux-amd64] CGO_ENABLED=$(CGO_ENABLED) - @CGO_ENABLED=$(CGO_ENABLED) GOOS=linux GOARCH=amd64 go build -trimpath -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(RELEASE_LINUX_AMD64)/$(TARGET) + @CGO_ENABLED=$(CGO_ENABLED) GOOS=linux GOARCH=amd64 go build -trimpath -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(RELEASE_LINUX_AMD64)/$(TARGET_BIN) .PHONY: darwin-amd64 darwin-amd64: @echo Build paopao-ce [darwin-amd64] CGO_ENABLED=$(CGO_ENABLED) - @CGO_ENABLED=$(CGO_ENABLED) GOOS=darwin GOARCH=amd64 go build -trimpath -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(RELEASE_DARWIN_AMD64)/$(TARGET) + @CGO_ENABLED=$(CGO_ENABLED) GOOS=darwin GOARCH=amd64 go build -trimpath -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(RELEASE_DARWIN_AMD64)/$(TARGET_BIN) .PHONY: darwin-arm64 darwin-arm64: @echo Build paopao-ce [darwin-arm64] CGO_ENABLED=$(CGO_ENABLED) - @CGO_ENABLED=$(CGO_ENABLED) GOOS=darwin GOARCH=arm64 go build -trimpath -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(RELEASE_DARWIN_ARM64)/$(TARGET) + @CGO_ENABLED=$(CGO_ENABLED) GOOS=darwin GOARCH=arm64 go build -trimpath -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(RELEASE_DARWIN_ARM64)/$(TARGET_BIN) .PHONY: windows-x64 windows-x64: @echo Build paopao-ce [windows-x64] CGO_ENABLED=$(CGO_ENABLED) - @CGO_ENABLED=$(CGO_ENABLED) GOOS=windows GOARCH=amd64 go build -trimpath -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(RELEASE_WINDOWS_AMD64)/$(basename $(TARGET)).exe + @CGO_ENABLED=$(CGO_ENABLED) GOOS=windows GOARCH=amd64 go build -trimpath -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(RELEASE_WINDOWS_AMD64)/$(TARGET_BIN).exe .PHONY: generate generate: gen-mir gen-grpc From 78282539f3d9dec6dc2058aa945e02c30b63c4de Mon Sep 17 00:00:00 2001 From: Michael Li Date: Tue, 4 Apr 2023 23:26:55 +0800 Subject: [PATCH 2/2] optimize config process logic --- internal/conf/conf.go | 81 ++++++------- internal/conf/{settting.go => core.go} | 161 ++++++++----------------- internal/conf/setting.go | 70 +++++++++++ internal/dao/search/search.go | 2 +- internal/internal.go | 2 +- main.go | 4 +- pkg/zinc/zinc.go | 9 +- 7 files changed, 164 insertions(+), 165 deletions(-) rename internal/conf/{settting.go => core.go} (62%) create mode 100644 internal/conf/setting.go diff --git a/internal/conf/conf.go b/internal/conf/conf.go index 75d4bc4c..2d565e7d 100644 --- a/internal/conf/conf.go +++ b/internal/conf/conf.go @@ -6,56 +6,54 @@ package conf import ( "log" - "sync" "time" "github.com/alimy/cfg" ) var ( - loggerSetting *LoggerSettingS - loggerFileSetting *LoggerFileSettingS - loggerZincSetting *LoggerZincSettingS - loggerMeiliSetting *LoggerMeiliSettingS - sentrySetting *SentrySettingS - redisSetting *RedisSettingS + loggerSetting *loggerConf + loggerFileSetting *loggerFileConf + loggerZincSetting *loggerZincConf + loggerMeiliSetting *loggerMeiliConf + sentrySetting *sentryConf + redisSetting *redisConf - PyroscopeSetting *PyroscopeSettingS - DatabaseSetting *DatabaseSetingS - MysqlSetting *MySQLSettingS - PostgresSetting *PostgresSettingS - Sqlite3Setting *Sqlite3SettingS - WebServerSetting *HttpServerSettingS - AdminServerSetting *HttpServerSettingS - SpaceXServerSetting *HttpServerSettingS - BotServerSetting *HttpServerSettingS - LocalossServerSetting *HttpServerSettingS - FrontendWebSetting *HttpServerSettingS - DocsServerSetting *HttpServerSettingS - MobileServerSetting *GRPCServerSettingS - AppSetting *AppSettingS - CacheIndexSetting *CacheIndexSettingS - SimpleCacheIndexSetting *SimpleCacheIndexSettingS - BigCacheIndexSetting *BigCacheIndexSettingS - RedisCacheIndexSetting *RedisCacheIndexSettingS - SmsJuheSetting *SmsJuheSettings - AlipaySetting *AlipaySettingS - TweetSearchSetting *TweetSearchS - ZincSetting *ZincSettingS - MeiliSetting *MeiliSettingS - ObjectStorage *ObjectStorageS - AliOSSSetting *AliOSSSettingS - COSSetting *COSSettingS - HuaweiOBSSetting *HuaweiOBSSettingS - MinIOSetting *MinIOSettingS - S3Setting *S3SettingS - LocalOSSSetting *LocalOSSSettingS - JWTSetting *JWTSettingS - Mutex *sync.Mutex + PyroscopeSetting *pyroscopeConf + DatabaseSetting *databaseConf + MysqlSetting *mysqlConf + PostgresSetting *postgresConf + Sqlite3Setting *sqlite3Conf + WebServerSetting *httpServerConf + AdminServerSetting *httpServerConf + SpaceXServerSetting *httpServerConf + BotServerSetting *httpServerConf + LocalossServerSetting *httpServerConf + FrontendWebSetting *httpServerConf + DocsServerSetting *httpServerConf + MobileServerSetting *grpcServerConf + AppSetting *appConf + CacheIndexSetting *cacheIndexConf + SimpleCacheIndexSetting *simpleCacheIndexConf + BigCacheIndexSetting *bigCacheIndexConf + RedisCacheIndexSetting *redisCacheIndexConf + SmsJuheSetting *smsJuheConf + AlipaySetting *alipayConf + TweetSearchSetting *tweetSearchConf + ZincSetting *zincConf + MeiliSetting *meiliConf + ObjectStorage *objectStorageS + AliOSSSetting *aliOSSConf + COSSetting *cosConf + HuaweiOBSSetting *huaweiOBSConf + MinIOSetting *minioConf + S3Setting *s3Conf + LocalOSSSetting *localossConf + JWTSetting *jwtConf ) func setupSetting(suite []string, noDefault bool) error { - setting, err := NewSetting() + setting, err := newSetting() if err != nil { return err } @@ -117,11 +115,10 @@ func setupSetting(suite []string, noDefault bool) error { RedisCacheIndexSetting.ExpireInSecond *= time.Second redisSetting.ConnWriteTimeout *= time.Second - Mutex = &sync.Mutex{} return nil } -func Initialize(suite []string, noDefault bool) { +func Initial(suite []string, noDefault bool) { err := setupSetting(suite, noDefault) if err != nil { log.Fatalf("init.setupSetting err: %v", err) diff --git a/internal/conf/settting.go b/internal/conf/core.go similarity index 62% rename from internal/conf/settting.go rename to internal/conf/core.go index c2e40e4a..773f3d07 100644 --- a/internal/conf/settting.go +++ b/internal/conf/core.go @@ -1,36 +1,27 @@ -// Copyright 2022 ROC. All rights reserved. +// Copyright 2023 ROC. All rights reserved. // Use of this source code is governed by a MIT style // license that can be found in the LICENSE file. package conf import ( - "embed" "fmt" "strings" "time" "github.com/pyroscope-io/client/pyroscope" "github.com/sirupsen/logrus" - "github.com/spf13/viper" "gorm.io/gorm/logger" ) -//go:embed config.yaml -var files embed.FS - -type Setting struct { - vp *viper.Viper -} - -type PyroscopeSettingS struct { +type pyroscopeConf struct { AppName string Endpoint string AuthToken string Logger string } -type SentrySettingS struct { +type sentryConf struct { Dsn string Debug bool AttachStacktrace bool @@ -39,17 +30,17 @@ type SentrySettingS struct { AttachGin bool } -type LoggerSettingS struct { +type loggerConf struct { Level string } -type LoggerFileSettingS struct { +type loggerFileConf struct { SavePath string FileName string FileExt string } -type LoggerZincSettingS struct { +type loggerZincConf struct { Host string Index string User string @@ -57,7 +48,7 @@ type LoggerZincSettingS struct { Secure bool } -type LoggerMeiliSettingS struct { +type loggerMeiliConf struct { Host string Index string ApiKey string @@ -66,7 +57,7 @@ type LoggerMeiliSettingS struct { MinWorker int } -type HttpServerSettingS struct { +type httpServerConf struct { RunMode string HttpIp string HttpPort string @@ -74,12 +65,12 @@ type HttpServerSettingS struct { WriteTimeout time.Duration } -type GRPCServerSettingS struct { +type grpcServerConf struct { Host string Port string } -type AppSettingS struct { +type appConf struct { RunMode string MaxCommentCount int64 AttachmentIncomeRate float64 @@ -88,30 +79,30 @@ type AppSettingS struct { MaxPageSize int } -type CacheIndexSettingS struct { +type cacheIndexConf struct { MaxUpdateQPS int MinWorker int } -type SimpleCacheIndexSettingS struct { +type simpleCacheIndexConf struct { MaxIndexSize int CheckTickDuration time.Duration ExpireTickDuration time.Duration } -type BigCacheIndexSettingS struct { +type bigCacheIndexConf struct { MaxIndexPage int HardMaxCacheSize int ExpireInSecond time.Duration Verbose bool } -type RedisCacheIndexSettingS struct { +type redisCacheIndexConf struct { ExpireInSecond time.Duration Verbose bool } -type AlipaySettingS struct { +type alipayConf struct { AppID string PrivateKey string RootCertFile string @@ -120,19 +111,19 @@ type AlipaySettingS struct { InProduction bool } -type SmsJuheSettings struct { +type smsJuheConf struct { Gateway string Key string TplID string TplVal string } -type TweetSearchS struct { +type tweetSearchConf struct { MaxUpdateQPS int MinWorker int } -type ZincSettingS struct { +type zincConf struct { Host string Index string User string @@ -140,19 +131,19 @@ type ZincSettingS struct { Secure bool } -type MeiliSettingS struct { +type meiliConf struct { Host string Index string ApiKey string Secure bool } -type DatabaseSetingS struct { +type databaseConf struct { TablePrefix string LogLevel string } -type MySQLSettingS struct { +type mysqlConf struct { UserName string Password string Host string @@ -163,18 +154,18 @@ type MySQLSettingS struct { MaxOpenConns int } -type PostgresSettingS map[string]string +type postgresConf map[string]string -type Sqlite3SettingS struct { +type sqlite3Conf struct { Path string } -type ObjectStorageS struct { +type objectStorageS struct { RetainInDays int TempDir string } -type MinIOSettingS struct { +type minioConf struct { AccessKey string SecretKey string Secure bool @@ -183,7 +174,7 @@ type MinIOSettingS struct { Domain string } -type S3SettingS struct { +type s3Conf struct { AccessKey string SecretKey string Secure bool @@ -192,7 +183,7 @@ type S3SettingS struct { Domain string } -type AliOSSSettingS struct { +type aliOSSConf struct { AccessKeyID string AccessKeySecret string Endpoint string @@ -200,7 +191,7 @@ type AliOSSSettingS struct { Domain string } -type COSSettingS struct { +type cosConf struct { SecretID string SecretKey string Region string @@ -208,7 +199,7 @@ type COSSettingS struct { Domain string } -type HuaweiOBSSettingS struct { +type huaweiOBSConf struct { AccessKey string SecretKey string Endpoint string @@ -216,14 +207,14 @@ type HuaweiOBSSettingS struct { Domain string } -type LocalOSSSettingS struct { +type localossConf struct { SavePath string Secure bool Bucket string Domain string } -type RedisSettingS struct { +type redisConf struct { InitAddress []string Username string Password string @@ -231,79 +222,21 @@ type RedisSettingS struct { ConnWriteTimeout time.Duration } -type JWTSettingS struct { +type jwtConf struct { Secret string Issuer string Expire time.Duration } -func NewSetting() (*Setting, error) { - cfgFile, err := files.Open("config.yaml") - if err != nil { - return nil, err - } - defer cfgFile.Close() - - vp := viper.New() - vp.SetConfigName("config") - vp.AddConfigPath(".") - vp.AddConfigPath("custom/") - vp.SetConfigType("yaml") - if err = vp.ReadConfig(cfgFile); err != nil { - return nil, err - } - if err = vp.MergeInConfig(); err != nil { - return nil, err - } - - return &Setting{vp}, nil -} - -func (s *Setting) ReadSection(k string, v any) error { - err := s.vp.UnmarshalKey(k, v) - if err != nil { - return err - } - return nil -} - -func (s *Setting) Unmarshal(objects map[string]any) error { - for k, v := range objects { - err := s.vp.UnmarshalKey(k, v) - if err != nil { - return err - } - } - return nil -} - -func (s *Setting) featuresInfoFrom(k string) (map[string][]string, map[string]string) { - sub := s.vp.Sub(k) - keys := sub.AllKeys() - - suites := make(map[string][]string) - kv := make(map[string]string, len(keys)) - for _, key := range sub.AllKeys() { - val := sub.Get(key) - switch v := val.(type) { - case string: - kv[key] = v - case []any: - suites[key] = sub.GetStringSlice(key) - } - } - return suites, kv -} - -func (s *HttpServerSettingS) GetReadTimeout() time.Duration { +func (s *httpServerConf) GetReadTimeout() time.Duration { return s.ReadTimeout * time.Second } -func (s *HttpServerSettingS) GetWriteTimeout() time.Duration { +func (s *httpServerConf) GetWriteTimeout() time.Duration { return s.WriteTimeout * time.Second } -func (s *MySQLSettingS) Dsn() string { +func (s *mysqlConf) Dsn() string { return fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=%s&parseTime=%t&loc=Local", s.UserName, s.Password, @@ -314,7 +247,7 @@ func (s *MySQLSettingS) Dsn() string { ) } -func (s PostgresSettingS) Dsn() string { +func (s postgresConf) Dsn() string { var params []string for k, v := range s { if len(v) == 0 { @@ -334,7 +267,7 @@ func (s PostgresSettingS) Dsn() string { return strings.Join(params, " ") } -func (s *Sqlite3SettingS) Dsn(driverName string) string { +func (s *sqlite3Conf) Dsn(driverName string) string { pragmas := "_foreign_keys=1&_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=8000" if driverName == "sqlite" { pragmas = "_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)&_pragma=busy_timeout(8000)&_pragma=journal_size_limit(100000000)" @@ -342,7 +275,7 @@ func (s *Sqlite3SettingS) Dsn(driverName string) string { return fmt.Sprintf("file:%s?%s", s.Path, pragmas) } -func (s *DatabaseSetingS) logLevel() logger.LogLevel { +func (s *databaseConf) logLevel() logger.LogLevel { switch strings.ToLower(s.LogLevel) { case "silent": return logger.Silent @@ -357,7 +290,7 @@ func (s *DatabaseSetingS) logLevel() logger.LogLevel { } } -func (s *LoggerSettingS) logLevel() logrus.Level { +func (s *loggerConf) logLevel() logrus.Level { switch strings.ToLower(s.Level) { case "panic": return logrus.PanicLevel @@ -378,15 +311,15 @@ func (s *LoggerSettingS) logLevel() logrus.Level { } } -func (s *LoggerZincSettingS) Endpoint() string { +func (s *loggerZincConf) Endpoint() string { return endpoint(s.Host, s.Secure) } -func (s *LoggerMeiliSettingS) Endpoint() string { +func (s *loggerMeiliConf) Endpoint() string { return endpoint(s.Host, s.Secure) } -func (s *LoggerMeiliSettingS) minWork() int { +func (s *loggerMeiliConf) minWork() int { if s.MinWorker < 5 { return 5 } else if s.MinWorker > 100 { @@ -395,7 +328,7 @@ func (s *LoggerMeiliSettingS) minWork() int { return s.MinWorker } -func (s *LoggerMeiliSettingS) maxLogBuffer() int { +func (s *loggerMeiliConf) maxLogBuffer() int { if s.MaxLogBuffer < 10 { return 10 } else if s.MaxLogBuffer > 1000 { @@ -404,19 +337,19 @@ func (s *LoggerMeiliSettingS) maxLogBuffer() int { return s.MaxLogBuffer } -func (s *ObjectStorageS) TempDirSlash() string { +func (s *objectStorageS) TempDirSlash() string { return strings.Trim(s.TempDir, " /") + "/" } -func (s *ZincSettingS) Endpoint() string { +func (s *zincConf) Endpoint() string { return endpoint(s.Host, s.Secure) } -func (s *MeiliSettingS) Endpoint() string { +func (s *meiliConf) Endpoint() string { return endpoint(s.Host, s.Secure) } -func (s *PyroscopeSettingS) GetLogger() (logger pyroscope.Logger) { +func (s *pyroscopeConf) GetLogger() (logger pyroscope.Logger) { switch strings.ToLower(s.Logger) { case "standard": logger = pyroscope.StandardLogger diff --git a/internal/conf/setting.go b/internal/conf/setting.go new file mode 100644 index 00000000..27037e49 --- /dev/null +++ b/internal/conf/setting.go @@ -0,0 +1,70 @@ +// Copyright 2022 ROC. All rights reserved. +// Use of this source code is governed by a MIT style +// license that can be found in the LICENSE file. + +package conf + +import ( + "bytes" + _ "embed" + "github.com/spf13/viper" +) + +//go:embed config.yaml +var fileBytes []byte + +type setting struct { + vp *viper.Viper +} + +func newSetting() (*setting, error) { + vp := viper.New() + vp.SetConfigName("config") + vp.AddConfigPath(".") + vp.AddConfigPath("custom/") + vp.SetConfigType("yaml") + err := vp.ReadConfig(bytes.NewReader(fileBytes)) + if err != nil { + return nil, err + } + if err = vp.MergeInConfig(); err != nil { + return nil, err + } + return &setting{vp}, nil +} + +func (s *setting) ReadSection(k string, v any) error { + err := s.vp.UnmarshalKey(k, v) + if err != nil { + return err + } + return nil +} + +func (s *setting) Unmarshal(objects map[string]any) error { + for k, v := range objects { + err := s.vp.UnmarshalKey(k, v) + if err != nil { + return err + } + } + return nil +} + +func (s *setting) featuresInfoFrom(k string) (map[string][]string, map[string]string) { + sub := s.vp.Sub(k) + keys := sub.AllKeys() + + suites := make(map[string][]string) + kv := make(map[string]string, len(keys)) + for _, key := range sub.AllKeys() { + val := sub.Get(key) + switch v := val.(type) { + case string: + kv[key] = v + case []any: + suites[key] = sub.GetStringSlice(key) + } + } + return suites, kv +} diff --git a/internal/dao/search/search.go b/internal/dao/search/search.go index baf0923b..3a7e4f72 100644 --- a/internal/dao/search/search.go +++ b/internal/dao/search/search.go @@ -57,7 +57,7 @@ func NewZincTweetSearchService(ams core.AuthorizationManageService) (core.TweetS ams: ams, }, indexName: s.Index, - client: zinc.NewClient(s), + client: zinc.NewClient(s.Endpoint(), s.User, s.Password), publicFilter: fmt.Sprintf("visibility:%d", core.PostVisitPublic), privateFilter: fmt.Sprintf("visibility:%d AND user_id:%%d", core.PostVisitPrivate), friendFilter: fmt.Sprintf("visibility:%d", core.PostVisitFriend), diff --git a/internal/internal.go b/internal/internal.go index 038a33a5..926c9f75 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -8,7 +8,7 @@ import ( "github.com/rocboss/paopao-ce/internal/migration" ) -func Initialize() { +func Initial() { // migrate database if needed migration.Run() } diff --git a/main.go b/main.go index 12e929fb..c2514e19 100644 --- a/main.go +++ b/main.go @@ -47,8 +47,8 @@ func (s *suites) Set(value string) error { func init() { flagParse() - conf.Initialize(features, noDefaultFeatures) - internal.Initialize() + conf.Initial(features, noDefaultFeatures) + internal.Initial() } func deferFn() { diff --git a/pkg/zinc/zinc.go b/pkg/zinc/zinc.go index 0ad61ecb..4f44bb78 100644 --- a/pkg/zinc/zinc.go +++ b/pkg/zinc/zinc.go @@ -11,7 +11,6 @@ import ( "github.com/cockroachdb/errors" "github.com/go-resty/resty/v2" - "github.com/rocboss/paopao-ce/internal/conf" "github.com/rocboss/paopao-ce/pkg/json" ) @@ -75,12 +74,12 @@ type HitItem struct { } // NewClient 获取ZincClient新实例 -func NewClient(conf *conf.ZincSettingS) *ZincClient { +func NewClient(host, user, passwd string) *ZincClient { return &ZincClient{ ZincClientConfig: &ZincClientConfig{ - ZincHost: conf.Endpoint(), - ZincUser: conf.User, - ZincPassword: conf.Password, + ZincHost: host, + ZincUser: user, + ZincPassword: passwd, }, } }