withchao
2 years ago
parent
53b64ae7c8
commit
519a9153e3
@ -1,78 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/IBM/sarama"
|
||||
"github.com/openimsdk/tools/log"
|
||||
kfk "github.com/openimsdk/tools/mq/kafka"
|
||||
)
|
||||
|
||||
type MConsumerGroup struct {
|
||||
sarama.ConsumerGroup
|
||||
groupID string
|
||||
topics []string
|
||||
}
|
||||
|
||||
type MConsumerGroupConfig struct {
|
||||
KafkaVersion sarama.KafkaVersion
|
||||
OffsetsInitial int64
|
||||
IsReturnErr bool
|
||||
UserName string
|
||||
Password string
|
||||
}
|
||||
|
||||
func NewMConsumerGroup(conf *kfk.Config, groupID string, topics []string) (*MConsumerGroup, error) {
|
||||
config, err := kfk.BuildConsumerGroupConfig(conf, sarama.OffsetNewest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
group, err := kfk.NewConsumerGroup(config, conf.Addr, groupID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &MConsumerGroup{
|
||||
ConsumerGroup: group,
|
||||
groupID: groupID,
|
||||
topics: topics,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (mc *MConsumerGroup) GetContextFromMsg(cMsg *sarama.ConsumerMessage) context.Context {
|
||||
return GetContextWithMQHeader(cMsg.Headers)
|
||||
}
|
||||
|
||||
func (mc *MConsumerGroup) RegisterHandleAndConsumer(ctx context.Context, handler sarama.ConsumerGroupHandler) {
|
||||
log.ZDebug(ctx, "register consumer group", "groupID", mc.groupID)
|
||||
for {
|
||||
err := mc.ConsumerGroup.Consume(ctx, mc.topics, handler)
|
||||
if errors.Is(err, sarama.ErrClosedConsumerGroup) {
|
||||
return
|
||||
}
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
log.ZWarn(ctx, "consume err", err, "topic", mc.topics, "groupID", mc.groupID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (mc *MConsumerGroup) Close() error {
|
||||
return mc.ConsumerGroup.Close()
|
||||
}
|
||||
@ -1 +0,0 @@
|
||||
package kafka // import "github.com/openimsdk/open-im-server/v3/pkg/common/kafka"
|
||||
@ -1,128 +0,0 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/IBM/sarama"
|
||||
"github.com/openimsdk/protocol/constant"
|
||||
"github.com/openimsdk/tools/errs"
|
||||
"github.com/openimsdk/tools/log"
|
||||
"github.com/openimsdk/tools/mcontext"
|
||||
kfk "github.com/openimsdk/tools/mq/kafka"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
var errEmptyMsg = errors.New("kafka binary msg is empty")
|
||||
|
||||
// Producer represents a Kafka producer.
|
||||
type Producer struct {
|
||||
addr []string
|
||||
topic string
|
||||
config *sarama.Config
|
||||
producer sarama.SyncProducer
|
||||
}
|
||||
|
||||
type ProducerConfig struct {
|
||||
ProducerAck string
|
||||
CompressType string
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
func BuildProducerConfig(conf kfk.Config) (*sarama.Config, error) {
|
||||
return kfk.BuildProducerConfig(conf)
|
||||
}
|
||||
|
||||
func NewKafkaProducer(config *sarama.Config, addr []string, topic string) (*Producer, error) {
|
||||
producer, err := kfk.NewProducer(config, addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Producer{
|
||||
addr: addr,
|
||||
topic: topic,
|
||||
config: config,
|
||||
producer: producer,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetMQHeaderWithContext extracts message queue headers from the context.
|
||||
func GetMQHeaderWithContext(ctx context.Context) ([]sarama.RecordHeader, error) {
|
||||
operationID, opUserID, platform, connID, err := mcontext.GetCtxInfos(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []sarama.RecordHeader{
|
||||
{Key: []byte(constant.OperationID), Value: []byte(operationID)},
|
||||
{Key: []byte(constant.OpUserID), Value: []byte(opUserID)},
|
||||
{Key: []byte(constant.OpUserPlatform), Value: []byte(platform)},
|
||||
{Key: []byte(constant.ConnID), Value: []byte(connID)},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetContextWithMQHeader creates a context from message queue headers.
|
||||
func GetContextWithMQHeader(header []*sarama.RecordHeader) context.Context {
|
||||
var values []string
|
||||
for _, recordHeader := range header {
|
||||
values = append(values, string(recordHeader.Value))
|
||||
}
|
||||
return mcontext.WithMustInfoCtx(values) // Attach extracted values to context
|
||||
}
|
||||
|
||||
// SendMessage sends a message to the Kafka topic configured in the Producer.
|
||||
func (p *Producer) SendMessage(ctx context.Context, key string, msg proto.Message) (int32, int64, error) {
|
||||
log.ZDebug(ctx, "SendMessage", "msg", msg, "topic", p.topic, "key", key)
|
||||
|
||||
// Marshal the protobuf message
|
||||
bMsg, err := proto.Marshal(msg)
|
||||
if err != nil {
|
||||
return 0, 0, errs.WrapMsg(err, "kafka proto Marshal err")
|
||||
}
|
||||
if len(bMsg) == 0 {
|
||||
return 0, 0, errs.WrapMsg(errEmptyMsg, "kafka proto Marshal err")
|
||||
}
|
||||
|
||||
// Prepare Kafka message
|
||||
kMsg := &sarama.ProducerMessage{
|
||||
Topic: p.topic,
|
||||
Key: sarama.StringEncoder(key),
|
||||
Value: sarama.ByteEncoder(bMsg),
|
||||
}
|
||||
|
||||
// Validate message key and value
|
||||
if kMsg.Key.Length() == 0 || kMsg.Value.Length() == 0 {
|
||||
return 0, 0, errs.Wrap(errEmptyMsg)
|
||||
}
|
||||
|
||||
// Attach context metadata as headers
|
||||
header, err := GetMQHeaderWithContext(ctx)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
kMsg.Headers = header
|
||||
|
||||
// Send the message
|
||||
partition, offset, err := p.producer.SendMessage(kMsg)
|
||||
if err != nil {
|
||||
log.ZWarn(ctx, "p.producer.SendMessage error", err)
|
||||
return 0, 0, errs.Wrap(err)
|
||||
}
|
||||
|
||||
log.ZDebug(ctx, "ByteEncoder SendMessage end", "key", kMsg.Key, "key length", kMsg.Value.Length())
|
||||
return partition, offset, nil
|
||||
}
|
||||
Loading…
Reference in new issue