withchao
2 years ago
parent
53b64ae7c8
commit
519a9153e3
@ -1,78 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package kafka
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
|
|
||||||
"github.com/IBM/sarama"
|
|
||||||
"github.com/openimsdk/tools/log"
|
|
||||||
kfk "github.com/openimsdk/tools/mq/kafka"
|
|
||||||
)
|
|
||||||
|
|
||||||
type MConsumerGroup struct {
|
|
||||||
sarama.ConsumerGroup
|
|
||||||
groupID string
|
|
||||||
topics []string
|
|
||||||
}
|
|
||||||
|
|
||||||
type MConsumerGroupConfig struct {
|
|
||||||
KafkaVersion sarama.KafkaVersion
|
|
||||||
OffsetsInitial int64
|
|
||||||
IsReturnErr bool
|
|
||||||
UserName string
|
|
||||||
Password string
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewMConsumerGroup(conf *kfk.Config, groupID string, topics []string) (*MConsumerGroup, error) {
|
|
||||||
config, err := kfk.BuildConsumerGroupConfig(conf, sarama.OffsetNewest)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
group, err := kfk.NewConsumerGroup(config, conf.Addr, groupID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &MConsumerGroup{
|
|
||||||
ConsumerGroup: group,
|
|
||||||
groupID: groupID,
|
|
||||||
topics: topics,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mc *MConsumerGroup) GetContextFromMsg(cMsg *sarama.ConsumerMessage) context.Context {
|
|
||||||
return GetContextWithMQHeader(cMsg.Headers)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mc *MConsumerGroup) RegisterHandleAndConsumer(ctx context.Context, handler sarama.ConsumerGroupHandler) {
|
|
||||||
log.ZDebug(ctx, "register consumer group", "groupID", mc.groupID)
|
|
||||||
for {
|
|
||||||
err := mc.ConsumerGroup.Consume(ctx, mc.topics, handler)
|
|
||||||
if errors.Is(err, sarama.ErrClosedConsumerGroup) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if errors.Is(err, context.Canceled) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
log.ZWarn(ctx, "consume err", err, "topic", mc.topics, "groupID", mc.groupID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mc *MConsumerGroup) Close() error {
|
|
||||||
return mc.ConsumerGroup.Close()
|
|
||||||
}
|
|
||||||
@ -1 +0,0 @@
|
|||||||
package kafka // import "github.com/openimsdk/open-im-server/v3/pkg/common/kafka"
|
|
||||||
@ -1,128 +0,0 @@
|
|||||||
// Copyright © 2023 OpenIM. All rights reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package kafka
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
|
|
||||||
"github.com/IBM/sarama"
|
|
||||||
"github.com/openimsdk/protocol/constant"
|
|
||||||
"github.com/openimsdk/tools/errs"
|
|
||||||
"github.com/openimsdk/tools/log"
|
|
||||||
"github.com/openimsdk/tools/mcontext"
|
|
||||||
kfk "github.com/openimsdk/tools/mq/kafka"
|
|
||||||
"google.golang.org/protobuf/proto"
|
|
||||||
)
|
|
||||||
|
|
||||||
var errEmptyMsg = errors.New("kafka binary msg is empty")
|
|
||||||
|
|
||||||
// Producer represents a Kafka producer.
|
|
||||||
type Producer struct {
|
|
||||||
addr []string
|
|
||||||
topic string
|
|
||||||
config *sarama.Config
|
|
||||||
producer sarama.SyncProducer
|
|
||||||
}
|
|
||||||
|
|
||||||
type ProducerConfig struct {
|
|
||||||
ProducerAck string
|
|
||||||
CompressType string
|
|
||||||
Username string
|
|
||||||
Password string
|
|
||||||
}
|
|
||||||
|
|
||||||
func BuildProducerConfig(conf kfk.Config) (*sarama.Config, error) {
|
|
||||||
return kfk.BuildProducerConfig(conf)
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewKafkaProducer(config *sarama.Config, addr []string, topic string) (*Producer, error) {
|
|
||||||
producer, err := kfk.NewProducer(config, addr)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &Producer{
|
|
||||||
addr: addr,
|
|
||||||
topic: topic,
|
|
||||||
config: config,
|
|
||||||
producer: producer,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetMQHeaderWithContext extracts message queue headers from the context.
|
|
||||||
func GetMQHeaderWithContext(ctx context.Context) ([]sarama.RecordHeader, error) {
|
|
||||||
operationID, opUserID, platform, connID, err := mcontext.GetCtxInfos(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return []sarama.RecordHeader{
|
|
||||||
{Key: []byte(constant.OperationID), Value: []byte(operationID)},
|
|
||||||
{Key: []byte(constant.OpUserID), Value: []byte(opUserID)},
|
|
||||||
{Key: []byte(constant.OpUserPlatform), Value: []byte(platform)},
|
|
||||||
{Key: []byte(constant.ConnID), Value: []byte(connID)},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetContextWithMQHeader creates a context from message queue headers.
|
|
||||||
func GetContextWithMQHeader(header []*sarama.RecordHeader) context.Context {
|
|
||||||
var values []string
|
|
||||||
for _, recordHeader := range header {
|
|
||||||
values = append(values, string(recordHeader.Value))
|
|
||||||
}
|
|
||||||
return mcontext.WithMustInfoCtx(values) // Attach extracted values to context
|
|
||||||
}
|
|
||||||
|
|
||||||
// SendMessage sends a message to the Kafka topic configured in the Producer.
|
|
||||||
func (p *Producer) SendMessage(ctx context.Context, key string, msg proto.Message) (int32, int64, error) {
|
|
||||||
log.ZDebug(ctx, "SendMessage", "msg", msg, "topic", p.topic, "key", key)
|
|
||||||
|
|
||||||
// Marshal the protobuf message
|
|
||||||
bMsg, err := proto.Marshal(msg)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, errs.WrapMsg(err, "kafka proto Marshal err")
|
|
||||||
}
|
|
||||||
if len(bMsg) == 0 {
|
|
||||||
return 0, 0, errs.WrapMsg(errEmptyMsg, "kafka proto Marshal err")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prepare Kafka message
|
|
||||||
kMsg := &sarama.ProducerMessage{
|
|
||||||
Topic: p.topic,
|
|
||||||
Key: sarama.StringEncoder(key),
|
|
||||||
Value: sarama.ByteEncoder(bMsg),
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate message key and value
|
|
||||||
if kMsg.Key.Length() == 0 || kMsg.Value.Length() == 0 {
|
|
||||||
return 0, 0, errs.Wrap(errEmptyMsg)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Attach context metadata as headers
|
|
||||||
header, err := GetMQHeaderWithContext(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, err
|
|
||||||
}
|
|
||||||
kMsg.Headers = header
|
|
||||||
|
|
||||||
// Send the message
|
|
||||||
partition, offset, err := p.producer.SendMessage(kMsg)
|
|
||||||
if err != nil {
|
|
||||||
log.ZWarn(ctx, "p.producer.SendMessage error", err)
|
|
||||||
return 0, 0, errs.Wrap(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.ZDebug(ctx, "ByteEncoder SendMessage end", "key", kMsg.Key, "key length", kMsg.Value.Length())
|
|
||||||
return partition, offset, nil
|
|
||||||
}
|
|
||||||
Loading…
Reference in new issue