lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-10-15 20:06:19 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

publisher is able to fully send data and receive acks

Browse Source 
still needs to close the pipes cleanly
This commit is contained in:
chrislu
2024-03-20 12:25:40 -07:00
parent 96149cbf7f
commit 7003ce7425
7 changed files with 779 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
weed/mq/client/pub_client/publish.go
View File

@@ -5,6 +5,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"time"
)
func (p *TopicPublisher) Publish(key, value []byte) error {
@@ -20,5 +21,6 @@ func (p *TopicPublisher) Publish(key, value []byte) error {
return inputBuffer.Enqueue(&mq_pb.DataMessage{
Key: key,
Value: value,
TsNs: time.Now().UnixNano(),
})
}

18
weed/mq/client/pub_client/scheduler.go
View File

@@ -150,6 +150,7 @@ func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) erro
}); err != nil {
return fmt.Errorf("send init message: %v", err)
}
// process the hello message
resp, err := stream.Recv()
if err != nil {
return fmt.Errorf("recv init response: %v", err)
@@ -158,21 +159,25 @@ func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) erro
return fmt.Errorf("init response error: %v", resp.Error)
}
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
for {
ackResp, err := publishClient.Recv()
if err != nil {
e, _ := status.FromError(err)
if e.Code() == codes.Unknown && e.Message() == "EOF" {
log.Printf("publish to %s EOF", publishClient.Broker)
return
}
publishClient.Err = err
fmt.Printf("publish1 to %s error: %v\n", publishClient.Broker, err)
log.Printf("publish1 to %s error: %v\n", publishClient.Broker, err)
return
}
if ackResp.Error != "" {
publishClient.Err = fmt.Errorf("ack error: %v", ackResp.Error)
fmt.Printf("publish2 to %s error: %v\n", publishClient.Broker, ackResp.Error)
log.Printf("publish2 to %s error: %v\n", publishClient.Broker, ackResp.Error)
return
}
if ackResp.AckSequence > 0 {
@@ -193,11 +198,12 @@ func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) erro
publishCounter++
}
if err := publishClient.CloseSend(); err != nil {
return fmt.Errorf("close send: %v", err)
}
// CloseSend would cancel the context on the server side
//if err := publishClient.CloseSend(); err != nil {
// return fmt.Errorf("close send: %v", err)
//}
time.Sleep(3 * time.Second)
wg.Wait()
log.Printf("published %d messages to %v for topic partition %+v", publishCounter, job.LeaderBroker, job.Partition)