create topic and report topic

weed/mq/balancer/allocate.go

@@ -6,10 +6,10 @@ import (
 	"math/rand"
 )
 
-func allocateTopicPartitions(brokers cmap.ConcurrentMap[string, *BrokerStats], partitionCount int) (assignments []*mq_pb.BrokerPartitionAssignment) {
+func allocateTopicPartitions(brokers cmap.ConcurrentMap[string, *BrokerStats], partitionCount int32) (assignments []*mq_pb.BrokerPartitionAssignment) {
 	// divide the ring into partitions
 	rangeSize := MaxPartitionCount / partitionCount
-	for i := 0; i < partitionCount; i++ {
+	for i := int32(0); i < partitionCount; i++ {
 		assignment := &mq_pb.BrokerPartitionAssignment{
 			Partition: &mq_pb.Partition{
 				RingSize:   MaxPartitionCount,
@@ -35,13 +35,13 @@ func allocateTopicPartitions(brokers cmap.ConcurrentMap[string, *BrokerStats], p
 
 // for now: randomly pick brokers
 // TODO pick brokers based on the broker stats
-func pickBrokers(brokers cmap.ConcurrentMap[string, *BrokerStats], count int) []string {
+func pickBrokers(brokers cmap.ConcurrentMap[string, *BrokerStats], count int32) []string {
 	candidates := make([]string, 0, brokers.Count())
 	for brokerStatsItem := range brokers.IterBuffered() {
 		candidates = append(candidates, brokerStatsItem.Key)
 	}
 	pickedBrokers := make([]string, 0, count)
-	for i := 0; i < count; i++ {
+	for i := int32(0); i < count; i++ {
 		p := rand.Int() % len(candidates)
 		if p < 0 {
 			p = -p

weed/mq/balancer/balancer.go

@@ -52,6 +52,11 @@ func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) {
 
 }
 
+func (bs *BrokerStats) String() string {
+	return fmt.Sprintf("BrokerStats{TopicPartitionCount:%d, ConsumerCount:%d, CpuUsagePercent:%d, Stats:%+v}",
+		bs.TopicPartitionCount, bs.ConsumerCount, bs.CpuUsagePercent, bs.Stats.Items())
+}
+
 type TopicPartition struct {
 	Namespace  string
 	Topic      string

weed/mq/balancer/lookup.go

@@ -47,5 +47,5 @@ func (b *Balancer) LookupOrAllocateTopicPartitions(topic *mq_pb.Topic, publish b
 	if b.Brokers.IsEmpty() {
 		return nil, ErrNoBroker
 	}
-	return allocateTopicPartitions(b.Brokers, 6), nil
+	return allocateTopicPartitions(b.Brokers, partitionCount), nil
 }

weed/mq/broker/broker_grpc_balancer.go

@@ -43,6 +43,7 @@ func (broker *MessageQueueBroker) ConnectToBalancer(stream mq_pb.SeaweedMessagin
 			brokerStats.UpdateStats(receivedStats)
 
 			glog.V(3).Infof("broker %s stats: %+v", initMessage.Broker, brokerStats)
+			glog.V(3).Infof("received stats: %+v", receivedStats)
 		}
 	}
 

weed/mq/broker/broker_grpc_create.go

@@ -3,6 +3,7 @@ package broker
 import (
 	"context"
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -27,6 +28,18 @@ func (broker *MessageQueueBroker) CreateTopic(ctx context.Context, request *mq_p
 	ret := &mq_pb.CreateTopicResponse{}
 	ret.BrokerPartitionAssignments, err = broker.Balancer.LookupOrAllocateTopicPartitions(request.Topic, true, request.PartitionCount)
 
+	for _, bpa := range ret.BrokerPartitionAssignments {
+		if doCreateErr := broker.withBrokerClient(false, pb.ServerAddress(bpa.LeaderBroker), func(client mq_pb.SeaweedMessagingClient) error {
+			_, doCreateErr := client.DoCreateTopic(ctx, &mq_pb.DoCreateTopicRequest{
+				Topic:     request.Topic,
+				Partition: bpa.Partition,
+			})
+			return doCreateErr
+		}); doCreateErr != nil {
+			return nil, doCreateErr
+		}
+	}
+
 	return ret, err
 }
 

weed/mq/broker/broker_grpc_lookup.go

@@ -45,3 +45,38 @@ func (broker *MessageQueueBroker) CheckTopicPartitionsStatus(c context.Context,
 	ret := &mq_pb.CheckTopicPartitionsStatusResponse{}
 	return ret, nil
 }
+
+func (broker *MessageQueueBroker) ListTopics(ctx context.Context, request *mq_pb.ListTopicsRequest) (resp *mq_pb.ListTopicsResponse, err error) {
+	if broker.currentBalancer == "" {
+		return nil, status.Errorf(codes.Unavailable, "no balancer")
+	}
+	if !broker.lockAsBalancer.IsLocked() {
+		proxyErr := broker.withBrokerClient(false, broker.currentBalancer, func(client mq_pb.SeaweedMessagingClient) error {
+			resp, err = client.ListTopics(ctx, request)
+			return nil
+		})
+		if proxyErr != nil {
+			return nil, proxyErr
+		}
+		return resp, err
+	}
+
+	ret := &mq_pb.ListTopicsResponse{}
+	knownTopics := make(map[*mq_pb.Topic]struct{})
+	for brokerStatsItem := range broker.Balancer.Brokers.IterBuffered() {
+		_, brokerStats := brokerStatsItem.Key, brokerStatsItem.Val
+		for topicPartitionStatsItem := range brokerStats.Stats.IterBuffered() {
+			topicPartitionStat := topicPartitionStatsItem.Val
+			topic := &mq_pb.Topic{
+				Namespace: topicPartitionStat.TopicPartition.Namespace,
+				Name:      topicPartitionStat.TopicPartition.Topic,
+			}
+			if _, found := knownTopics[topic]; found {
+				continue
+			}
+			ret.Topics = append(ret.Topics, topic)
+		}
+	}
+
+	return ret, nil
+}