address some comments

This commit is contained in:
chrislu
2025-09-02 16:54:34 -07:00
parent e901abffd3
commit 900bd94456
5 changed files with 161 additions and 25 deletions

View File

@@ -1,17 +1,25 @@
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"math/rand"
"os"
"strconv"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/mq/client/pub_client"
"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
type UserEvent struct {
@@ -232,17 +240,99 @@ func convertToRecordValue(data interface{}) (*schema_pb.RecordValue, error) {
return &schema_pb.RecordValue{Fields: fields}, nil
}
// convertHTTPToGRPC converts HTTP address to gRPC address
// Follows SeaweedFS convention: gRPC port = HTTP port + 10000
func convertHTTPToGRPC(httpAddress string) string {
if strings.Contains(httpAddress, ":") {
parts := strings.Split(httpAddress, ":")
if len(parts) == 2 {
if port, err := strconv.Atoi(parts[1]); err == nil {
return fmt.Sprintf("%s:%d", parts[0], port+10000)
}
}
}
// Fallback: return original address if conversion fails
return httpAddress
}
// discoverFiler finds a filer from the master server
func discoverFiler(masterHTTPAddress string) (string, error) {
masterGRPCAddress := convertHTTPToGRPC(masterHTTPAddress)
conn, err := grpc.Dial(masterGRPCAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return "", fmt.Errorf("failed to connect to master at %s: %v", masterGRPCAddress, err)
}
defer conn.Close()
client := master_pb.NewSeaweedClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
ClientType: cluster.FilerType,
})
if err != nil {
return "", fmt.Errorf("failed to list filers from master: %v", err)
}
if len(resp.ClusterNodes) == 0 {
return "", fmt.Errorf("no filers found in cluster")
}
// Use the first available filer and convert HTTP address to gRPC
filerHTTPAddress := resp.ClusterNodes[0].Address
return convertHTTPToGRPC(filerHTTPAddress), nil
}
// discoverBroker finds the broker balancer using filer lock mechanism
func discoverBroker(masterHTTPAddress string) (string, error) {
// First discover filer from master
filerAddress, err := discoverFiler(masterHTTPAddress)
if err != nil {
return "", fmt.Errorf("failed to discover filer: %v", err)
}
conn, err := grpc.Dial(filerAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return "", fmt.Errorf("failed to connect to filer at %s: %v", filerAddress, err)
}
defer conn.Close()
client := filer_pb.NewSeaweedFilerClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
resp, err := client.FindLockOwner(ctx, &filer_pb.FindLockOwnerRequest{
Name: pub_balancer.LockBrokerBalancer,
})
if err != nil {
return "", fmt.Errorf("failed to find broker balancer: %v", err)
}
return resp.Owner, nil
}
func createTopicData(masterAddr, filerAddr, namespace, topicName string,
generator func() interface{}, count int) error {
// Create schema based on topic type
recordType := createSchemaForTopic(topicName)
// Dynamically discover broker address instead of hardcoded port replacement
brokerAddress, err := discoverBroker(masterAddr)
if err != nil {
// Fallback to hardcoded port replacement if discovery fails
log.Printf("Warning: Failed to discover broker dynamically (%v), using hardcoded port replacement", err)
brokerAddress = strings.Replace(masterAddr, ":9333", ":17777", 1)
}
// Create publisher configuration
config := &pub_client.PublisherConfiguration{
Topic: topic.NewTopic(namespace, topicName),
PartitionCount: 1,
Brokers: []string{strings.Replace(masterAddr, ":9333", ":17777", 1)}, // Use broker port
Brokers: []string{brokerAddress}, // Use dynamically discovered broker address
PublisherName: fmt.Sprintf("test-producer-%s-%s", namespace, topicName),
RecordType: recordType, // Use structured schema
}