goKafka/consume-partition.go

package main

import (
	"github.com/Shopify/sarama"
	"log"
	"os"
	"os/signal"
)

func main() {
	// 一，获取消费者
	consumer, err := sarama.NewConsumer([]string{"localhost:9092"}, nil)
	if err != nil {
		log.Fatalln(err)
	}
	defer func() {
		if err := consumer.Close(); err != nil {
			log.Fatalln(err)
		}
	}()

	// 二，获取topic下的分区列表
	topic := "topic_more_partition_1" // 3
	partitions, err := consumer.Partitions(topic)
	if err != nil {
		log.Fatalln(err)
	}
	log.Println(partitions)

	// 三，遍历每个分区
	var consumeCounter = 0
	for _, partition := range partitions {
		// 获取每个分区的消费者
		partitionConsumer, err := consumer.ConsumePartition(topic, partition, sarama.OffsetNewest)
		if err != nil {
			log.Fatalln(err)
		}

		// 四，利用 goroutine 完成消费
		go func(pc sarama.PartitionConsumer) {
			// 关闭
			defer func() {
				if err := pc.Close(); err != nil {
					log.Fatalln(err)
				}
			}()

			// 消费
			for msg := range pc.Messages() {
				log.Println(pc)
				log.Printf("Consumed message, partition: %d, offset: %d, topic: %s, value: %s\n", msg.Partition, msg.Offset, msg.Topic, msg.Value)
				consumeCounter++
			}

		}(partitionConsumer)
	}

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)
	select {
	case <-signals:
	}

	log.Printf("Consumed: %d\n", consumeCounter)
}