goKafka/consume.go

package main

import (
	"github.com/Shopify/sarama"
	"log"
	"os"
	"os/signal"
)

func main() {
	//consumeTopic()
	consumeTopicMulti()
}

func consumeTopicMulti() {
	// 一，获取消费者
	consumer, err := sarama.NewConsumer([]string{"localhost:9092"}, nil)
	if err != nil {
		log.Fatalln(err)
	}
	defer func() {
		if err := consumer.Close(); err != nil {
			log.Fatalln(err)
		}
	}()

	// 二，获取topic下的分区列表
	topic := "async_topic"
	partitions, err := consumer.Partitions(topic)
	if err != nil {
		log.Fatalln(err)
	}

	// 三，遍历每个分区
	var consumeCounter = 0
	for _, partition := range partitions {
		// 获取每个分区的消费者
		partitionConsumer, err := consumer.ConsumePartition(topic, partition, sarama.OffsetNewest)
		if err != nil {
			log.Fatalln(err)
		}

		// 四，利用 goroutine 完成消费
		go func(pc sarama.PartitionConsumer) {
			// 关闭
			defer func() {
				if err := pc.Close(); err != nil {
					log.Fatalln(err)
				}
			}()

			// 消费
			for msg := range pc.Messages() {
				log.Printf("Consumed message, partition: %d, offset: %d, topic: %s, value: %s\n", msg.Partition, msg.Offset, msg.Topic, msg.Value)
				consumeCounter++
			}

		}(partitionConsumer)
	}

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)
	select {
	case <-signals:
	}

	log.Printf("Consumed: %d\n", consumeCounter)
}

func consumeTopic() {
	// 一，获取消费者
	consumer, err := sarama.NewConsumer([]string{"localhost:9092"}, nil)
	if err != nil {
		log.Fatalln(err)
	}
	defer func() {
		if err := consumer.Close(); err != nil {
			log.Fatalln(err)
		}
	}()

	// 二，获取topic具体某个partition的的消费者
	// topic 1:N partition
	partitionConsumer, err := consumer.ConsumePartition("async_topic", 0, sarama.OffsetNewest)
	if err != nil {
		log.Fatalln(err)
	}
	defer func() {
		if err := partitionConsumer.Close(); err != nil {
			log.Fatalln(err)
		}
	}()

	// 三，消费
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)

	consumeCounter := 0
loop:
	for {
		select {
		case msg := <-partitionConsumer.Messages():
			log.Printf("Consumed message, partition: %d, offset: %d, topic: %s, value: %s\n", msg.Partition, msg.Offset, msg.Topic, msg.Value)
			consumeCounter++

		case <-signals:
			break loop
		}
	}

	log.Printf("Consumed: %d\n", consumeCounter)

}