mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2025-09-22 16:13:34 +08:00
address some comments
This commit is contained in:
@@ -1,17 +1,25 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/client/pub_client"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
type UserEvent struct {
|
||||
@@ -232,17 +240,99 @@ func convertToRecordValue(data interface{}) (*schema_pb.RecordValue, error) {
|
||||
return &schema_pb.RecordValue{Fields: fields}, nil
|
||||
}
|
||||
|
||||
// convertHTTPToGRPC converts HTTP address to gRPC address
|
||||
// Follows SeaweedFS convention: gRPC port = HTTP port + 10000
|
||||
func convertHTTPToGRPC(httpAddress string) string {
|
||||
if strings.Contains(httpAddress, ":") {
|
||||
parts := strings.Split(httpAddress, ":")
|
||||
if len(parts) == 2 {
|
||||
if port, err := strconv.Atoi(parts[1]); err == nil {
|
||||
return fmt.Sprintf("%s:%d", parts[0], port+10000)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fallback: return original address if conversion fails
|
||||
return httpAddress
|
||||
}
|
||||
|
||||
// discoverFiler finds a filer from the master server
|
||||
func discoverFiler(masterHTTPAddress string) (string, error) {
|
||||
masterGRPCAddress := convertHTTPToGRPC(masterHTTPAddress)
|
||||
|
||||
conn, err := grpc.Dial(masterGRPCAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to connect to master at %s: %v", masterGRPCAddress, err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
client := master_pb.NewSeaweedClient(conn)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
|
||||
ClientType: cluster.FilerType,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to list filers from master: %v", err)
|
||||
}
|
||||
|
||||
if len(resp.ClusterNodes) == 0 {
|
||||
return "", fmt.Errorf("no filers found in cluster")
|
||||
}
|
||||
|
||||
// Use the first available filer and convert HTTP address to gRPC
|
||||
filerHTTPAddress := resp.ClusterNodes[0].Address
|
||||
return convertHTTPToGRPC(filerHTTPAddress), nil
|
||||
}
|
||||
|
||||
// discoverBroker finds the broker balancer using filer lock mechanism
|
||||
func discoverBroker(masterHTTPAddress string) (string, error) {
|
||||
// First discover filer from master
|
||||
filerAddress, err := discoverFiler(masterHTTPAddress)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to discover filer: %v", err)
|
||||
}
|
||||
|
||||
conn, err := grpc.Dial(filerAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to connect to filer at %s: %v", filerAddress, err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
client := filer_pb.NewSeaweedFilerClient(conn)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
resp, err := client.FindLockOwner(ctx, &filer_pb.FindLockOwnerRequest{
|
||||
Name: pub_balancer.LockBrokerBalancer,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to find broker balancer: %v", err)
|
||||
}
|
||||
|
||||
return resp.Owner, nil
|
||||
}
|
||||
|
||||
func createTopicData(masterAddr, filerAddr, namespace, topicName string,
|
||||
generator func() interface{}, count int) error {
|
||||
|
||||
// Create schema based on topic type
|
||||
recordType := createSchemaForTopic(topicName)
|
||||
|
||||
// Dynamically discover broker address instead of hardcoded port replacement
|
||||
brokerAddress, err := discoverBroker(masterAddr)
|
||||
if err != nil {
|
||||
// Fallback to hardcoded port replacement if discovery fails
|
||||
log.Printf("Warning: Failed to discover broker dynamically (%v), using hardcoded port replacement", err)
|
||||
brokerAddress = strings.Replace(masterAddr, ":9333", ":17777", 1)
|
||||
}
|
||||
|
||||
// Create publisher configuration
|
||||
config := &pub_client.PublisherConfiguration{
|
||||
Topic: topic.NewTopic(namespace, topicName),
|
||||
PartitionCount: 1,
|
||||
Brokers: []string{strings.Replace(masterAddr, ":9333", ":17777", 1)}, // Use broker port
|
||||
Brokers: []string{brokerAddress}, // Use dynamically discovered broker address
|
||||
PublisherName: fmt.Sprintf("test-producer-%s-%s", namespace, topicName),
|
||||
RecordType: recordType, // Use structured schema
|
||||
}
|
||||
|
Reference in New Issue
Block a user