This commit is contained in:
chrislu
2024-05-20 11:05:18 -07:00
parent 5038577f7e
commit 3622196881
15 changed files with 58 additions and 56 deletions

View File

@@ -12,7 +12,6 @@ import (
"time"
)
func (b *MessageQueueBroker) SubscribeFollowMe(stream mq_pb.SeaweedMessaging_SubscribeFollowMeServer) (err error) {
var req *mq_pb.SubscribeFollowMeRequest
req, err = stream.Recv()

View File

@@ -130,7 +130,6 @@ func (b *MessageQueueBroker) GetGrpcDialOption() grpc.DialOption {
return b.grpcDialOption
}
func (b *MessageQueueBroker) GetFiler() pb.ServerAddress {
return b.currentFiler
}

View File

@@ -44,7 +44,7 @@ func main() {
subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, processorConfig)
counter := 0
subscriber.SetEachMessageFunc(func(key, value []byte) (error) {
subscriber.SetEachMessageFunc(func(key, value []byte) error {
counter++
println(string(key), "=>", string(value), counter)
return nil

View File

@@ -71,7 +71,7 @@ func main() {
subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, processorConfig)
counter := 0
subscriber.SetEachMessageFunc(func(key, value []byte) (error) {
subscriber.SetEachMessageFunc(func(key, value []byte) error {
counter++
record := &schema_pb.RecordValue{}
proto.Unmarshal(value, record)

View File

@@ -125,11 +125,11 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
close(partitionOffsetChan)
}()
concurrentPartitionLimit := int(sub.ProcessorConfig.MaxPartitionCount)
if concurrentPartitionLimit <= 0 {
concurrentPartitionLimit = 1
perPartitionConcurrency := int(sub.ProcessorConfig.PerPartitionConcurrency)
if perPartitionConcurrency <= 0 {
perPartitionConcurrency = 1
}
executors := util.NewLimitedConcurrentExecutor(concurrentPartitionLimit)
executors := util.NewLimitedConcurrentExecutor(perPartitionConcurrency)
go func() {
for ack := range partitionOffsetChan {

View File

@@ -9,7 +9,6 @@ import (
)
type ProcessorState struct {
}
// Subscribe subscribes to a topic's specified partitions.

View File

@@ -26,6 +26,7 @@ func (imt *InflightMessageTracker) InflightMessage(key []byte, tsNs int64) {
imt.messages[string(key)] = tsNs
imt.timestamps.Add(tsNs)
}
// IsMessageAcknowledged returns true if the message has been acknowledged.
// If the message is older than the oldest inflight messages, returns false.
// returns false if the message is inflight.
@@ -47,6 +48,7 @@ func (imt *InflightMessageTracker) IsMessageAcknowledged(key []byte, tsNs int64)
return true
}
// AcknowledgeMessage acknowledges the message with the key and timestamp.
func (imt *InflightMessageTracker) AcknowledgeMessage(key []byte, tsNs int64) bool {
imt.mu.Lock()
@@ -71,12 +73,14 @@ type RingBuffer struct {
head int
size int
}
// NewRingBuffer creates a new RingBuffer of the given capacity.
func NewRingBuffer(capacity int) *RingBuffer {
return &RingBuffer{
buffer: make([]int64, capacity),
}
}
// Add adds a new timestamp to the ring buffer.
func (rb *RingBuffer) Add(timestamp int64) {
rb.buffer[rb.head] = timestamp
@@ -85,6 +89,7 @@ func (rb *RingBuffer) Add(timestamp int64) {
rb.size++
}
}
// Remove removes the specified timestamp from the ring buffer.
func (rb *RingBuffer) Remove(timestamp int64) {
// Perform binary search