persist consumer group offset

1. use one follower
2. read write consumer group offset
This commit is contained in:
chrislu
2024-05-19 00:46:12 -07:00
parent 8d5bb7420d
commit b1871427c3
18 changed files with 609 additions and 538 deletions

View File

@@ -145,7 +145,7 @@ func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) erro
Topic: p.config.Topic.ToPbTopic(),
Partition: job.Partition,
AckInterval: 128,
FollowerBrokers: job.FollowerBrokers,
FollowerBroker: job.FollowerBroker,
PublisherName: p.config.PublisherName,
},
},

View File

@@ -102,11 +102,10 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
Topic: sub.ContentConfig.Topic.ToPbTopic(),
PartitionOffset: &mq_pb.PartitionOffset{
Partition: assigned.Partition,
StartTsNs: sub.alreadyProcessedTsNs,
StartType: mq_pb.PartitionOffsetStartType_EARLIEST_IN_MEMORY,
},
Filter: sub.ContentConfig.Filter,
FollowerBrokers: assigned.FollowerBrokers,
FollowerBroker: assigned.FollowerBroker,
},
},
});err != nil {
@@ -154,7 +153,6 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
if processErr != nil {
return fmt.Errorf("process error: %v", processErr)
}
sub.alreadyProcessedTsNs = m.Data.TsNs
partitionOffsetChan <- m.Data.TsNs
if !shouldContinue {
return nil

View File

@@ -37,7 +37,6 @@ type TopicSubscriber struct {
OnCompletionFunc OnCompletionFunc
bootstrapBrokers []string
waitForMoreMessage bool
alreadyProcessedTsNs int64
activeProcessors map[topic.Partition]*ProcessorState
activeProcessorsLock sync.Mutex
}
@@ -50,7 +49,6 @@ func NewTopicSubscriber(bootstrapBrokers []string, subscriber *SubscriberConfigu
brokerPartitionAssignmentChan: make(chan *mq_pb.BrokerPartitionAssignment, 1024),
bootstrapBrokers: bootstrapBrokers,
waitForMoreMessage: true,
alreadyProcessedTsNs: content.StartTime.UnixNano(),
activeProcessors: make(map[topic.Partition]*ProcessorState),
}
}