merge current message queue code changes (#6201)

* listing files to convert to parquet

* write parquet files

* save logs into parquet files

* pass by value

* compact logs into parquet format

* can skip existing files

* refactor

* refactor

* fix compilation

* when no partition found

* refactor

* add untested parquet file read

* rename package

* refactor

* rename files

* remove unused

* add merged log read func

* parquet wants to know the file size

* rewind by time

* pass in stop ts

* add stop ts

* adjust log

* minor

* adjust log

* skip .parquet files when reading message logs

* skip non message files

* Update subscriber_record.go

* send messages

* skip message data with only ts

* skip non log files

* update parquet-go package

* ensure a valid record type

* add new field to a record type

* Update read_parquet_to_log.go

* fix parquet file name generation

* separating reading parquet and logs

* add key field

* add skipped logs

* use in memory cache

* refactor

* refactor

* refactor

* refactor, and change compact log

* refactor

* rename

* refactor

* fix format

* prefix v to version directory

weed/mq/topic/local_manager.go

@@ -88,7 +88,7 @@ func (manager *LocalTopicManager) CollectStats(duration time.Duration) *mq_pb.Br
 				Topic:     Topic{Namespace: localTopic.Namespace, Name: localTopic.Name},
 				Partition: localPartition.Partition,
 			}
-			stats.Stats[topicPartition.String()] = &mq_pb.TopicPartitionStats{
+			stats.Stats[topicPartition.TopicPartitionId()] = &mq_pb.TopicPartitionStats{
 				Topic: &mq_pb.Topic{
 					Namespace: string(localTopic.Namespace),
 					Name:      localTopic.Name,

weed/mq/topic/local_partition.go

@@ -34,6 +34,7 @@ type LocalPartition struct {
 }
 
 var TIME_FORMAT = "2006-01-02-15-04-05"
+var PartitionGenerationFormat = "v2006-01-02-15-04-05"
 
 func NewLocalPartition(partition Partition, logFlushFn log_buffer.LogFlushFuncType, readFromDiskFn log_buffer.LogReadFromDiskFuncType) *LocalPartition {
 	lp := &LocalPartition{

weed/mq/topic/partition.go

@@ -3,6 +3,7 @@ package topic
 import (
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"time"
 )
 
 const PartitionCount = 4096
@@ -89,6 +90,19 @@ func (partition Partition) String() string {
 	return fmt.Sprintf("%04d-%04d", partition.RangeStart, partition.RangeStop)
 }
 
-func ToString(partition *mq_pb.Partition) string {
-	return fmt.Sprintf("%04d-%04d", partition.RangeStart, partition.RangeStop)
+func ParseTopicVersion(name string) (t time.Time, err error) {
+	return time.Parse(PartitionGenerationFormat, name)
+}
+
+func ParsePartitionBoundary(name string) (start, stop int32) {
+	_, err := fmt.Sscanf(name, "%04d-%04d", &start, &stop)
+	if err != nil {
+		return 0, 0
+	}
+	return start, stop
+}
+
+func PartitionDir(t Topic, p Partition) string {
+	partitionGeneration := time.Unix(0, p.UnixTimeNs).UTC().Format(PartitionGenerationFormat)
+	return fmt.Sprintf("%s/%s/%04d-%04d", t.Dir(), partitionGeneration, p.RangeStart, p.RangeStop)
 }

weed/mq/topic/topic.go

@@ -1,8 +1,13 @@
 package topic
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	jsonpb "google.golang.org/protobuf/encoding/protojson"
 )
 
 type Topic struct {
@@ -23,13 +28,42 @@ func FromPbTopic(topic *mq_pb.Topic) Topic {
 	}
 }
 
-func (tp Topic) ToPbTopic() *mq_pb.Topic {
+func (t Topic) ToPbTopic() *mq_pb.Topic {
 	return &mq_pb.Topic{
-		Namespace: tp.Namespace,
-		Name:      tp.Name,
+		Namespace: t.Namespace,
+		Name:      t.Name,
 	}
 }
 
-func (tp Topic) String() string {
-	return fmt.Sprintf("%s.%s", tp.Namespace, tp.Name)
+func (t Topic) String() string {
+	return fmt.Sprintf("%s.%s", t.Namespace, t.Name)
+}
+
+func (t Topic) Dir() string {
+	return fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
+}
+
+func (t Topic) ReadConfFile(client filer_pb.SeaweedFilerClient) (*mq_pb.ConfigureTopicResponse, error) {
+	data, err := filer.ReadInsideFiler(client, t.Dir(), filer.TopicConfFile)
+	if errors.Is(err, filer_pb.ErrNotFound) {
+		return nil, err
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read topic.conf of %v: %v", t, err)
+	}
+	// parse into filer conf object
+	conf := &mq_pb.ConfigureTopicResponse{}
+	if err = jsonpb.Unmarshal(data, conf); err != nil {
+		return nil, fmt.Errorf("unmarshal topic %v conf: %v", t, err)
+	}
+	return conf, nil
+}
+
+func (t Topic) WriteConfFile(client filer_pb.SeaweedFilerClient, conf *mq_pb.ConfigureTopicResponse) error {
+	var buf bytes.Buffer
+	filer.ProtoToText(&buf, conf)
+	if err := filer.SaveInsideFiler(client, t.Dir(), filer.TopicConfFile, buf.Bytes()); err != nil {
+		return fmt.Errorf("save topic %v conf: %v", t, err)
+	}
+	return nil
 }

weed/mq/topic/topic_partition.go

@@ -7,6 +7,6 @@ type TopicPartition struct {
 	Partition
 }
 
-func (tp *TopicPartition) String() string {
+func (tp *TopicPartition) TopicPartitionId() string {
 	return fmt.Sprintf("%v.%v-%04d-%04d", tp.Namespace, tp.Topic, tp.RangeStart, tp.RangeStop)
 }