mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2025-10-15 20:06:19 +08:00
persist consumer group offset
1. use one follower 2. read write consumer group offset
This commit is contained in:
@@ -145,7 +145,7 @@ func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) erro
|
||||
Topic: p.config.Topic.ToPbTopic(),
|
||||
Partition: job.Partition,
|
||||
AckInterval: 128,
|
||||
FollowerBrokers: job.FollowerBrokers,
|
||||
FollowerBroker: job.FollowerBroker,
|
||||
PublisherName: p.config.PublisherName,
|
||||
},
|
||||
},
|
||||
|
@@ -102,11 +102,10 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
|
||||
Topic: sub.ContentConfig.Topic.ToPbTopic(),
|
||||
PartitionOffset: &mq_pb.PartitionOffset{
|
||||
Partition: assigned.Partition,
|
||||
StartTsNs: sub.alreadyProcessedTsNs,
|
||||
StartType: mq_pb.PartitionOffsetStartType_EARLIEST_IN_MEMORY,
|
||||
},
|
||||
Filter: sub.ContentConfig.Filter,
|
||||
FollowerBrokers: assigned.FollowerBrokers,
|
||||
FollowerBroker: assigned.FollowerBroker,
|
||||
},
|
||||
},
|
||||
});err != nil {
|
||||
@@ -154,7 +153,6 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
|
||||
if processErr != nil {
|
||||
return fmt.Errorf("process error: %v", processErr)
|
||||
}
|
||||
sub.alreadyProcessedTsNs = m.Data.TsNs
|
||||
partitionOffsetChan <- m.Data.TsNs
|
||||
if !shouldContinue {
|
||||
return nil
|
||||
|
@@ -37,7 +37,6 @@ type TopicSubscriber struct {
|
||||
OnCompletionFunc OnCompletionFunc
|
||||
bootstrapBrokers []string
|
||||
waitForMoreMessage bool
|
||||
alreadyProcessedTsNs int64
|
||||
activeProcessors map[topic.Partition]*ProcessorState
|
||||
activeProcessorsLock sync.Mutex
|
||||
}
|
||||
@@ -50,7 +49,6 @@ func NewTopicSubscriber(bootstrapBrokers []string, subscriber *SubscriberConfigu
|
||||
brokerPartitionAssignmentChan: make(chan *mq_pb.BrokerPartitionAssignment, 1024),
|
||||
bootstrapBrokers: bootstrapBrokers,
|
||||
waitForMoreMessage: true,
|
||||
alreadyProcessedTsNs: content.StartTime.UnixNano(),
|
||||
activeProcessors: make(map[topic.Partition]*ProcessorState),
|
||||
}
|
||||
}
|
||||
|
Reference in New Issue
Block a user