Merge accumulated changes related to message queue (#5098)

* balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * tracking topic=>broker * merge * comment
2025-12-17 17:51:20 +08:00 · 2023-12-11 12:05:54 -08:00
parent 8784553501
commit 580940bf82
58 changed files with 3853 additions and 2752 deletions
--- a/weed/mq/topic/topic.go
+++ b/weed/mq/topic/topic.go
@@ -2,9 +2,7 @@ package topic

 import (
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
-	"time"
 )

 type Topic struct {
@@ -25,47 +23,13 @@ func FromPbTopic(topic *mq_pb.Topic) Topic {
 	}
 }

+func (tp Topic) ToPbTopic() *mq_pb.Topic {
+	return &mq_pb.Topic{
+		Namespace: tp.Namespace,
+		Name:      tp.Name,
+	}
+}
+
 func (tp Topic) String() string {
 	return fmt.Sprintf("%s.%s", tp.Namespace, tp.Name)
 }
-
-type Segment struct {
-	Topic        Topic
-	Id           int32
-	Partition    Partition
-	LastModified time.Time
-}
-
-func FromPbSegment(segment *mq_pb.Segment) *Segment {
-	return &Segment{
-		Topic: Topic{
-			Namespace: segment.Namespace,
-			Name:      segment.Topic,
-		},
-		Id: segment.Id,
-		Partition: Partition{
-			RangeStart: segment.Partition.RangeStart,
-			RangeStop:  segment.Partition.RangeStop,
-			RingSize:   segment.Partition.RingSize,
-		},
-	}
-}
-
-func (segment *Segment) ToPbSegment() *mq_pb.Segment {
-	return &mq_pb.Segment{
-		Namespace: string(segment.Topic.Namespace),
-		Topic:     segment.Topic.Name,
-		Id:        segment.Id,
-		Partition: &mq_pb.Partition{
-			RingSize:   segment.Partition.RingSize,
-			RangeStart: segment.Partition.RangeStart,
-			RangeStop:  segment.Partition.RangeStop,
-		},
-	}
-}
-
-func (segment *Segment) DirAndName() (dir string, name string) {
-	dir = fmt.Sprintf("%s/%s/%s", filer.TopicsDir, segment.Topic.Namespace, segment.Topic.Name)
-	name = fmt.Sprintf("%4d.segment", segment.Id)
-	return
-}