chenqinghe
4 years ago
parent
e863299c1a
commit
cb0a1a20f1
@ -0,0 +1,8 @@
|
||||
# Default ignored files
|
||||
/shelf/
|
||||
/workspace.xml
|
||||
# Editor-based HTTP Client requests
|
||||
/httpRequests/
|
||||
# Datasource local storage ignored files
|
||||
/dataSources/
|
||||
/dataSources.local.xml
|
||||
@ -0,0 +1,9 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="WEB_MODULE" version="4">
|
||||
<component name="Go" enabled="true" />
|
||||
<component name="NewModuleRootManager">
|
||||
<content url="file://$MODULE_DIR$" />
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
</module>
|
||||
@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectModuleManager">
|
||||
<modules>
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/Open-IM-Server.iml" filepath="$PROJECT_DIR$/.idea/Open-IM-Server.iml" />
|
||||
</modules>
|
||||
</component>
|
||||
</project>
|
||||
@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="$PROJECT_DIR$" vcs="Git" />
|
||||
</component>
|
||||
</project>
|
||||
@ -1,53 +0,0 @@
|
||||
/*
|
||||
** description("").
|
||||
** copyright('tuoyun,www.tuoyun.net').
|
||||
** author("fg,Gordon@tuoyun.net").
|
||||
** time(2021/5/11 9:36).
|
||||
*/
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/Shopify/sarama"
|
||||
)
|
||||
|
||||
type MConsumerGroup struct {
|
||||
sarama.ConsumerGroup
|
||||
groupID string
|
||||
topics []string
|
||||
}
|
||||
|
||||
type MConsumerGroupConfig struct {
|
||||
KafkaVersion sarama.KafkaVersion
|
||||
OffsetsInitial int64
|
||||
IsReturnErr bool
|
||||
}
|
||||
|
||||
func NewMConsumerGroup(consumerConfig *MConsumerGroupConfig, topics, addr []string, groupID string) *MConsumerGroup {
|
||||
config := sarama.NewConfig()
|
||||
config.Version = consumerConfig.KafkaVersion
|
||||
config.Consumer.Offsets.Initial = consumerConfig.OffsetsInitial
|
||||
config.Consumer.Return.Errors = consumerConfig.IsReturnErr
|
||||
client, err := sarama.NewClient(addr, config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
consumerGroup, err := sarama.NewConsumerGroupFromClient(groupID, client)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &MConsumerGroup{
|
||||
consumerGroup,
|
||||
groupID,
|
||||
topics,
|
||||
}
|
||||
}
|
||||
func (mc *MConsumerGroup) RegisterHandleAndConsumer(handler sarama.ConsumerGroupHandler) {
|
||||
ctx := context.Background()
|
||||
for {
|
||||
err := mc.ConsumerGroup.Consume(ctx, mc.topics, handler)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
package mq
|
||||
|
||||
import "time"
|
||||
|
||||
type Consumer interface {
|
||||
// RegisterMessageHandler is used to register message handler
|
||||
// any received messages will be passed to handler to process
|
||||
// once the Consumer started, it is forbidden to register handlers.
|
||||
RegisterMessageHandler(topic string, handler MessageHandler)
|
||||
|
||||
// Start to consume messages
|
||||
Start() error
|
||||
}
|
||||
|
||||
type MessageHandler interface {
|
||||
// HandleMessage process received messages,
|
||||
// if returned error is nil, the message will be auto committed.
|
||||
HandleMessage(msg *Message) error
|
||||
}
|
||||
|
||||
type MessageHandleFunc func(msg *Message) error
|
||||
|
||||
func (fn MessageHandleFunc) HandleMessage(msg *Message) error {
|
||||
return fn(msg)
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
Key, Value []byte
|
||||
Topic string
|
||||
Partition int32
|
||||
Offset int64
|
||||
Timestamp time.Time // only set if kafka is version 0.10+, inner message timestamp
|
||||
BlockTimestamp time.Time // only set if kafka is version 0.10+, outer (compressed) block timestamp
|
||||
Headers []*RecordHeader // only set if kafka is version 0.11+
|
||||
}
|
||||
|
||||
type RecordHeader struct {
|
||||
Key []byte
|
||||
Value []byte
|
||||
}
|
||||
@ -0,0 +1,110 @@
|
||||
/*
|
||||
** description("").
|
||||
** copyright('tuoyun,www.tuoyun.net').
|
||||
** author("fg,Gordon@tuoyun.net").
|
||||
** time(2021/5/11 9:36).
|
||||
*/
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"Open_IM/pkg/common/mq"
|
||||
|
||||
"github.com/Shopify/sarama"
|
||||
)
|
||||
|
||||
type kafkaConsumerGroup struct {
|
||||
sarama.ConsumerGroup
|
||||
groupID string
|
||||
|
||||
mu *sync.RWMutex
|
||||
handlers map[string][]mq.MessageHandler
|
||||
}
|
||||
|
||||
var _ mq.Consumer = (*kafkaConsumerGroup)(nil)
|
||||
|
||||
type MConsumerGroupConfig struct {
|
||||
KafkaVersion sarama.KafkaVersion
|
||||
OffsetsInitial int64
|
||||
IsReturnErr bool
|
||||
}
|
||||
|
||||
func NewMConsumerGroup(consumerConfig *MConsumerGroupConfig, addr []string, groupID string) *kafkaConsumerGroup {
|
||||
config := sarama.NewConfig()
|
||||
config.Version = consumerConfig.KafkaVersion
|
||||
config.Consumer.Offsets.Initial = consumerConfig.OffsetsInitial
|
||||
config.Consumer.Return.Errors = consumerConfig.IsReturnErr
|
||||
client, err := sarama.NewClient(addr, config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
consumerGroup, err := sarama.NewConsumerGroupFromClient(groupID, client)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &kafkaConsumerGroup{
|
||||
ConsumerGroup: consumerGroup,
|
||||
groupID: groupID,
|
||||
handlers: make(map[string][]mq.MessageHandler),
|
||||
}
|
||||
}
|
||||
|
||||
func (mc *kafkaConsumerGroup) RegisterMessageHandler(topic string, handler mq.MessageHandler) {
|
||||
mc.mu.Lock()
|
||||
defer mc.mu.Unlock()
|
||||
|
||||
handlers := mc.handlers[topic]
|
||||
handlers = append(handlers, handler)
|
||||
mc.handlers[topic] = handlers
|
||||
}
|
||||
|
||||
func (mc *kafkaConsumerGroup) Start() error {
|
||||
topics := make([]string, 0, len(mc.handlers))
|
||||
for topic := range mc.handlers {
|
||||
topics = append(topics, topic)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
for {
|
||||
err := mc.ConsumerGroup.Consume(ctx, topics, mc)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (mc *kafkaConsumerGroup) Setup(sarama.ConsumerGroupSession) error { return nil }
|
||||
func (mc *kafkaConsumerGroup) Cleanup(sarama.ConsumerGroupSession) error { return nil }
|
||||
func (mc *kafkaConsumerGroup) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
|
||||
for msg := range claim.Messages() {
|
||||
|
||||
mc.mu.RLock()
|
||||
handlers, ok := mc.handlers[msg.Topic]
|
||||
mc.mu.RUnlock()
|
||||
if !ok {
|
||||
panic(fmt.Sprintf("no handlers for topic: %s", msg.Topic))
|
||||
}
|
||||
|
||||
message := &mq.Message{
|
||||
Key: msg.Key,
|
||||
Value: msg.Value,
|
||||
Topic: msg.Topic,
|
||||
Partition: msg.Partition,
|
||||
Offset: msg.Offset,
|
||||
Timestamp: msg.Timestamp,
|
||||
}
|
||||
for _, handler := range handlers {
|
||||
for {
|
||||
if err := handler.HandleMessage(message); err == nil { // error is nil, auto commit
|
||||
sess.MarkMessage(msg, "")
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
package mq
|
||||
|
||||
import "github.com/golang/protobuf/proto"
|
||||
|
||||
type Producer interface {
|
||||
SendMessage(m proto.Message, key ...string) (int32, int64, error)
|
||||
}
|
||||
Loading…
Reference in new issue