lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-08 04:14:29 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

persist consumer group offset

Browse Source 
1. use one follower
2. read write consumer group offset
This commit is contained in:
chrislu
2024-05-19 00:46:12 -07:00
parent 8d5bb7420d
commit b1871427c3
18 changed files with 609 additions and 538 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
weed/mq/topic/local_manager.go
View File

@@ -99,6 +99,7 @@ func (manager *LocalTopicManager) CollectStats(duration time.Duration) *mq_pb.Br
Partition: localPartition.Partition.ToPbPartition(),
PublisherCount: int32(localPartition.Publishers.Size()),
SubscriberCount: int32(localPartition.Subscribers.Size()),
Follower: localPartition.Follower,
}
// fmt.Printf("collect topic %+v partition %+v\n", topicPartition, localPartition.Partition)
}

24
weed/mq/topic/local_partition.go
View File

@@ -30,7 +30,7 @@ type LocalPartition struct {
publishFolloweMeStream mq_pb.SeaweedMessaging_PublishFollowMeClient
followerGrpcConnection *grpc.ClientConn
follower string
Follower string
}
var TIME_FORMAT = "2006-01-02-15-04-05"
@@ -62,7 +62,7 @@ func (p *LocalPartition) Publish(message *mq_pb.DataMessage) error {
Data: message,
},
}); followErr != nil {
return fmt.Errorf("send to follower %s: %v", p.follower, followErr)
return fmt.Errorf("send to follower %s: %v", p.Follower, followErr)
}
} else {
atomic.StoreInt64(&p.AckTsNs, message.TsNs)
@@ -137,15 +137,15 @@ func (p *LocalPartition) MaybeConnectToFollowers(initMessage *mq_pb.PublishMessa
if p.publishFolloweMeStream != nil {
return nil
}
if len(initMessage.FollowerBrokers) == 0 {
if initMessage.FollowerBroker == "" {
return nil
}
p.follower = initMessage.FollowerBrokers[0]
p.Follower = initMessage.FollowerBroker
ctx := context.Background()
p.followerGrpcConnection, err = pb.GrpcDial(ctx, p.follower, true, grpcDialOption)
p.followerGrpcConnection, err = pb.GrpcDial(ctx, p.Follower, true, grpcDialOption)
if err != nil {
return fmt.Errorf("fail to dial %s: %v", p.follower, err)
return fmt.Errorf("fail to dial %s: %v", p.Follower, err)
}
followerClient := mq_pb.NewSeaweedMessagingClient(p.followerGrpcConnection)
p.publishFolloweMeStream, err = followerClient.PublishFollowMe(ctx)
@@ -174,10 +174,10 @@ func (p *LocalPartition) MaybeConnectToFollowers(initMessage *mq_pb.PublishMessa
if err != nil {
e, _ := status.FromError(err)
if e.Code() == codes.Canceled {
glog.V(0).Infof("local partition %v follower %v stopped", p.Partition, p.follower)
glog.V(0).Infof("local partition %v follower %v stopped", p.Partition, p.Follower)
return
}
glog.Errorf("Receiving local partition %v follower %s ack: %v", p.Partition, p.follower, err)
glog.Errorf("Receiving local partition %v follower %s ack: %v", p.Partition, p.Follower, err)
return
}
atomic.StoreInt64(&p.AckTsNs, ack.AckTsNs)
@@ -206,13 +206,13 @@ func (p *LocalPartition) MaybeShutdownLocalPartition() (hasShutdown bool) {
glog.V(4).Infof("closing grpcConnection to follower")
p.followerGrpcConnection.Close()
p.publishFolloweMeStream = nil
p.follower = ""
p.Follower = ""
}
hasShutdown = true
}
glog.V(0).Infof("local partition %v Publisher:%d Subscriber:%d follower:%s shutdown %v", p.Partition, p.Publishers.Size(), p.Subscribers.Size(), p.follower, hasShutdown)
glog.V(0).Infof("local partition %v Publisher:%d Subscriber:%d follower:%s shutdown %v", p.Partition, p.Publishers.Size(), p.Subscribers.Size(), p.Follower, hasShutdown)
return
}
@@ -232,8 +232,8 @@ func (p *LocalPartition) NotifyLogFlushed(flushTsNs int64) {
},
},
}); followErr != nil {
glog.Errorf("send follower %s flush message: %v", p.follower, followErr)
glog.Errorf("send follower %s flush message: %v", p.Follower, followErr)
}
// println("notifying", p.follower, "flushed at", flushTsNs)
// println("notifying", p.Follower, "flushed at", flushTsNs)
}
}