lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-08-24 02:57:54 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

working for in memory single log buffer

Browse Source
This commit is contained in:
Chris Lu 2020-04-19 23:37:04 -07:00
parent f373232227
commit ce3cb25cfb
11 changed files with 270 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
weed/filer2/filer_notify.go
View File

@ -1,6 +1,7 @@
package filer2 package filer2
import ( import (
"bytes"
"fmt" "fmt"
"strings" "strings"
"time" "time"
@ -81,8 +82,13 @@ func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {
func (f *Filer) ReadLogBuffer(lastReadTime time.Time, eachEventFn func(fullpath string, eventNotification *filer_pb.EventNotification) error) (newLastReadTime time.Time, err error) { func (f *Filer) ReadLogBuffer(lastReadTime time.Time, eachEventFn func(fullpath string, eventNotification *filer_pb.EventNotification) error) (newLastReadTime time.Time, err error) {
var buf []byte var bytesBuf *bytes.Buffer
newLastReadTime, buf = f.metaLogBuffer.ReadFromBuffer(lastReadTime) bytesBuf = f.metaLogBuffer.ReadFromBuffer(lastReadTime)
if bytesBuf == nil {
return
}
defer f.metaLogBuffer.ReleaseMeory(bytesBuf)
buf := bytesBuf.Bytes()
var processedTs int64 var processedTs int64
for pos := 0; pos+4 < len(buf); { for pos := 0; pos+4 < len(buf); {

4
weed/messaging/broker/broker_append.go
View File

@ -16,10 +16,6 @@ import (
func (broker *MessageBroker) appendToFile(targetFile string, topicConfig *messaging_pb.TopicConfiguration, data []byte) error { func (broker *MessageBroker) appendToFile(targetFile string, topicConfig *messaging_pb.TopicConfiguration, data []byte) error {
if topicConfig.IsTransient {
return nil
}
assignResult, uploadResult, err2 := broker.assignAndUpload(topicConfig, data) assignResult, uploadResult, err2 := broker.assignAndUpload(topicConfig, data)
if err2 != nil { if err2 != nil {
return err2 return err2

12
weed/messaging/broker/broker_grpc_server_publish.go
View File

@ -2,7 +2,6 @@ package broker
import ( import (
"io" "io"
"time"
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
@ -77,16 +76,9 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis
continue continue
} }
m := &messaging_pb.Message{ // fmt.Printf("received: %d : %s\n", len(in.Data.Value), string(in.Data.Value))
Timestamp: time.Now().UnixNano(),
Key: in.Data.Key,
Value: in.Data.Value,
Headers: in.Data.Headers,
}
// fmt.Printf("received: %d : %s\n", len(m.Value), string(m.Value)) data, err := proto.Marshal(in.Data)
data, err := proto.Marshal(m)
if err != nil { if err != nil {
glog.Errorf("marshall error: %v\n", err) glog.Errorf("marshall error: %v\n", err)
continue continue

65
weed/messaging/broker/broker_grpc_server_subscribe.go
View File

@ -1,6 +1,7 @@
package broker package broker
import ( import (
"fmt"
"io" "io"
"time" "time"
@ -9,7 +10,6 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
"github.com/chrislusf/seaweedfs/weed/util"
) )
func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_SubscribeServer) error { func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_SubscribeServer) error {
@ -23,12 +23,22 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs
return err return err
} }
var messageCount int64
subscriberId := in.Init.SubscriberId subscriberId := in.Init.SubscriberId
println("+ subscriber:", subscriberId) fmt.Printf("+ subscriber %s\n", subscriberId)
defer println("- subscriber:", subscriberId) defer func() {
fmt.Printf("- subscriber %s: %d messages\n", subscriberId, messageCount)
}()
// TODO look it up // TODO look it up
topicConfig := &messaging_pb.TopicConfiguration{ topicConfig := &messaging_pb.TopicConfiguration{
IsTransient: true,
}
if err = stream.Send(&messaging_pb.BrokerMessage{
Redirect: nil,
}); err != nil {
return err
} }
// get lock // get lock
@ -60,42 +70,25 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs
return err return err
} }
// loop through all messages messageCount, err = lock.logBuffer.LoopProcessLogData(lastReadTime, func() bool {
for {
_, buf := lock.logBuffer.ReadFromBuffer(lastReadTime)
for pos := 0; pos+4 < len(buf); {
size := util.BytesToUint32(buf[pos : pos+4])
entryData := buf[pos+4 : pos+4+int(size)]
logEntry := &filer_pb.LogEntry{}
if err = proto.Unmarshal(entryData, logEntry); err != nil {
glog.Errorf("unexpected unmarshal messaging_pb.Message: %v", err)
pos += 4 + int(size)
continue
}
m := &messaging_pb.Message{}
if err = proto.Unmarshal(logEntry.Data, m); err != nil {
glog.Errorf("unexpected unmarshal messaging_pb.Message: %v", err)
pos += 4 + int(size)
continue
}
// fmt.Printf("sending : %d : %s\n", len(m.Value), string(m.Value))
if err = eachMessageFn(m); err != nil {
return err
}
lastReadTime = time.Unix(0, m.Timestamp)
pos += 4 + int(size)
}
lock.Mutex.Lock() lock.Mutex.Lock()
lock.cond.Wait() lock.cond.Wait()
lock.Mutex.Unlock() lock.Mutex.Unlock()
return true
}, func(logEntry *filer_pb.LogEntry) error {
m := &messaging_pb.Message{}
if err = proto.Unmarshal(logEntry.Data, m); err != nil {
glog.Errorf("unexpected unmarshal messaging_pb.Message: %v", err)
return err
} }
// fmt.Printf("sending : %d bytes\n", len(m.Value))
if err = eachMessageFn(m); err != nil {
glog.Errorf("sending %d bytes to %s: %s", len(m.Value), subscriberId, err)
return err
}
return nil
})
return err
} }

4
weed/messaging/broker/topic_lock.go
View File

@ -41,6 +41,10 @@ func (locks *TopicLocks) buildLogBuffer(tl *TopicLock, tp TopicPartition, topicC
flushFn := func(startTime, stopTime time.Time, buf []byte) { flushFn := func(startTime, stopTime time.Time, buf []byte) {
if topicConfig.IsTransient {
return
}
targetFile := fmt.Sprintf( targetFile := fmt.Sprintf(
"%s/%s/%s/%04d-%02d-%02d/%02d-%02d.part%02d", "%s/%s/%s/%04d-%02d-%02d/%02d-%02d.part%02d",
filer2.TopicsDir, tp.Namespace, tp.Topic, filer2.TopicsDir, tp.Namespace, tp.Topic,

2
weed/messaging/client/publisher.go
View File

@ -61,7 +61,7 @@ func (mc *MessagingClient) NewPublisher(namespace, topic string) (*Publisher, er
}, nil }, nil
} }
func (p *Publisher) Publish(m *messaging_pb.RawData) error { func (p *Publisher) Publish(m *messaging_pb.Message) error {
return p.publishClient.Send(&messaging_pb.PublishRequest{ return p.publishClient.Send(&messaging_pb.PublishRequest{
Data: m, Data: m,

10
weed/pb/messaging.proto
View File

@ -46,14 +46,8 @@ message SubscriberMessage {
AckMessage ack = 2; AckMessage ack = 2;
} }
message RawData {
bytes key = 1; // Message key
bytes value = 2; // Message payload
map<string, bytes> headers = 3; // Message headers
}
message Message { message Message {
int64 timestamp = 1 [jstype = JS_STRING]; // When the message was received by the broker int64 event_time_ns = 1 [jstype = JS_STRING];
bytes key = 2; // Message key bytes key = 2; // Message key
bytes value = 3; // Message payload bytes value = 3; // Message payload
map<string, bytes> headers = 4; // Message headers map<string, bytes> headers = 4; // Message headers
@ -74,7 +68,7 @@ message PublishRequest {
int32 partition = 3; int32 partition = 3;
} }
InitMessage init = 1; InitMessage init = 1;
RawData data = 2; Message data = 2;
} }
message PublishResponse { message PublishResponse {

178
weed/pb/messaging_pb/messaging.pb.go
View File

@ -10,7 +10,6 @@ It is generated from these files:
It has these top-level messages: It has these top-level messages:
SubscriberMessage SubscriberMessage
RawData
Message Message
BrokerMessage BrokerMessage
PublishRequest PublishRequest
@ -167,40 +166,8 @@ func (m *SubscriberMessage_AckMessage) GetMessageId() int64 {
return 0 return 0
} }
type RawData struct {
Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
Headers map[string][]byte `protobuf:"bytes,3,rep,name=headers" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"`
}
func (m *RawData) Reset() { *m = RawData{} }
func (m *RawData) String() string { return proto.CompactTextString(m) }
func (*RawData) ProtoMessage() {}
func (*RawData) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
func (m *RawData) GetKey() []byte {
if m != nil {
return m.Key
}
return nil
}
func (m *RawData) GetValue() []byte {
if m != nil {
return m.Value
}
return nil
}
func (m *RawData) GetHeaders() map[string][]byte {
if m != nil {
return m.Headers
}
return nil
}
type Message struct { type Message struct {
Timestamp int64 `protobuf:"varint,1,opt,name=timestamp" json:"timestamp,omitempty"` EventTimeNs int64 `protobuf:"varint,1,opt,name=event_time_ns,json=eventTimeNs" json:"event_time_ns,omitempty"`
Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
Value []byte `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` Value []byte `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"`
Headers map[string][]byte `protobuf:"bytes,4,rep,name=headers" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"` Headers map[string][]byte `protobuf:"bytes,4,rep,name=headers" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"`
@ -209,11 +176,11 @@ type Message struct {
func (m *Message) Reset() { *m = Message{} } func (m *Message) Reset() { *m = Message{} }
func (m *Message) String() string { return proto.CompactTextString(m) } func (m *Message) String() string { return proto.CompactTextString(m) }
func (*Message) ProtoMessage() {} func (*Message) ProtoMessage() {}
func (*Message) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } func (*Message) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
func (m *Message) GetTimestamp() int64 { func (m *Message) GetEventTimeNs() int64 {
if m != nil { if m != nil {
return m.Timestamp return m.EventTimeNs
} }
return 0 return 0
} }
@ -247,7 +214,7 @@ type BrokerMessage struct {
func (m *BrokerMessage) Reset() { *m = BrokerMessage{} } func (m *BrokerMessage) Reset() { *m = BrokerMessage{} }
func (m *BrokerMessage) String() string { return proto.CompactTextString(m) } func (m *BrokerMessage) String() string { return proto.CompactTextString(m) }
func (*BrokerMessage) ProtoMessage() {} func (*BrokerMessage) ProtoMessage() {}
func (*BrokerMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } func (*BrokerMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
func (m *BrokerMessage) GetData() *Message { func (m *BrokerMessage) GetData() *Message {
if m != nil { if m != nil {
@ -271,7 +238,7 @@ func (m *BrokerMessage_RedirectMessage) Reset() { *m = BrokerMessage_Red
func (m *BrokerMessage_RedirectMessage) String() string { return proto.CompactTextString(m) } func (m *BrokerMessage_RedirectMessage) String() string { return proto.CompactTextString(m) }
func (*BrokerMessage_RedirectMessage) ProtoMessage() {} func (*BrokerMessage_RedirectMessage) ProtoMessage() {}
func (*BrokerMessage_RedirectMessage) Descriptor() ([]byte, []int) { func (*BrokerMessage_RedirectMessage) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{3, 0} return fileDescriptor0, []int{2, 0}
} }
func (m *BrokerMessage_RedirectMessage) GetNewBroker() string { func (m *BrokerMessage_RedirectMessage) GetNewBroker() string {
@ -283,13 +250,13 @@ func (m *BrokerMessage_RedirectMessage) GetNewBroker() string {
type PublishRequest struct { type PublishRequest struct {
Init *PublishRequest_InitMessage `protobuf:"bytes,1,opt,name=init" json:"init,omitempty"` Init *PublishRequest_InitMessage `protobuf:"bytes,1,opt,name=init" json:"init,omitempty"`
Data *RawData `protobuf:"bytes,2,opt,name=data" json:"data,omitempty"` Data *Message `protobuf:"bytes,2,opt,name=data" json:"data,omitempty"`
} }
func (m *PublishRequest) Reset() { *m = PublishRequest{} } func (m *PublishRequest) Reset() { *m = PublishRequest{} }
func (m *PublishRequest) String() string { return proto.CompactTextString(m) } func (m *PublishRequest) String() string { return proto.CompactTextString(m) }
func (*PublishRequest) ProtoMessage() {} func (*PublishRequest) ProtoMessage() {}
func (*PublishRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } func (*PublishRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
func (m *PublishRequest) GetInit() *PublishRequest_InitMessage { func (m *PublishRequest) GetInit() *PublishRequest_InitMessage {
if m != nil { if m != nil {
@ -298,7 +265,7 @@ func (m *PublishRequest) GetInit() *PublishRequest_InitMessage {
return nil return nil
} }
func (m *PublishRequest) GetData() *RawData { func (m *PublishRequest) GetData() *Message {
if m != nil { if m != nil {
return m.Data return m.Data
} }
@ -314,7 +281,7 @@ type PublishRequest_InitMessage struct {
func (m *PublishRequest_InitMessage) Reset() { *m = PublishRequest_InitMessage{} } func (m *PublishRequest_InitMessage) Reset() { *m = PublishRequest_InitMessage{} }
func (m *PublishRequest_InitMessage) String() string { return proto.CompactTextString(m) } func (m *PublishRequest_InitMessage) String() string { return proto.CompactTextString(m) }
func (*PublishRequest_InitMessage) ProtoMessage() {} func (*PublishRequest_InitMessage) ProtoMessage() {}
func (*PublishRequest_InitMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4, 0} } func (*PublishRequest_InitMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3, 0} }
func (m *PublishRequest_InitMessage) GetNamespace() string { func (m *PublishRequest_InitMessage) GetNamespace() string {
if m != nil { if m != nil {
@ -345,7 +312,7 @@ type PublishResponse struct {
func (m *PublishResponse) Reset() { *m = PublishResponse{} } func (m *PublishResponse) Reset() { *m = PublishResponse{} }
func (m *PublishResponse) String() string { return proto.CompactTextString(m) } func (m *PublishResponse) String() string { return proto.CompactTextString(m) }
func (*PublishResponse) ProtoMessage() {} func (*PublishResponse) ProtoMessage() {}
func (*PublishResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } func (*PublishResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
func (m *PublishResponse) GetConfig() *PublishResponse_ConfigMessage { func (m *PublishResponse) GetConfig() *PublishResponse_ConfigMessage {
if m != nil { if m != nil {
@ -369,7 +336,7 @@ func (m *PublishResponse_ConfigMessage) Reset() { *m = PublishResponse_C
func (m *PublishResponse_ConfigMessage) String() string { return proto.CompactTextString(m) } func (m *PublishResponse_ConfigMessage) String() string { return proto.CompactTextString(m) }
func (*PublishResponse_ConfigMessage) ProtoMessage() {} func (*PublishResponse_ConfigMessage) ProtoMessage() {}
func (*PublishResponse_ConfigMessage) Descriptor() ([]byte, []int) { func (*PublishResponse_ConfigMessage) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{5, 0} return fileDescriptor0, []int{4, 0}
} }
func (m *PublishResponse_ConfigMessage) GetPartitionCount() int32 { func (m *PublishResponse_ConfigMessage) GetPartitionCount() int32 {
@ -387,7 +354,7 @@ func (m *PublishResponse_RedirectMessage) Reset() { *m = PublishResponse
func (m *PublishResponse_RedirectMessage) String() string { return proto.CompactTextString(m) } func (m *PublishResponse_RedirectMessage) String() string { return proto.CompactTextString(m) }
func (*PublishResponse_RedirectMessage) ProtoMessage() {} func (*PublishResponse_RedirectMessage) ProtoMessage() {}
func (*PublishResponse_RedirectMessage) Descriptor() ([]byte, []int) { func (*PublishResponse_RedirectMessage) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{5, 1} return fileDescriptor0, []int{4, 1}
} }
func (m *PublishResponse_RedirectMessage) GetNewBroker() string { func (m *PublishResponse_RedirectMessage) GetNewBroker() string {
@ -406,7 +373,7 @@ type ConfigureTopicRequest struct {
func (m *ConfigureTopicRequest) Reset() { *m = ConfigureTopicRequest{} } func (m *ConfigureTopicRequest) Reset() { *m = ConfigureTopicRequest{} }
func (m *ConfigureTopicRequest) String() string { return proto.CompactTextString(m) } func (m *ConfigureTopicRequest) String() string { return proto.CompactTextString(m) }
func (*ConfigureTopicRequest) ProtoMessage() {} func (*ConfigureTopicRequest) ProtoMessage() {}
func (*ConfigureTopicRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } func (*ConfigureTopicRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
func (m *ConfigureTopicRequest) GetNamespace() string { func (m *ConfigureTopicRequest) GetNamespace() string {
if m != nil { if m != nil {
@ -435,7 +402,7 @@ type ConfigureTopicResponse struct {
func (m *ConfigureTopicResponse) Reset() { *m = ConfigureTopicResponse{} } func (m *ConfigureTopicResponse) Reset() { *m = ConfigureTopicResponse{} }
func (m *ConfigureTopicResponse) String() string { return proto.CompactTextString(m) } func (m *ConfigureTopicResponse) String() string { return proto.CompactTextString(m) }
func (*ConfigureTopicResponse) ProtoMessage() {} func (*ConfigureTopicResponse) ProtoMessage() {}
func (*ConfigureTopicResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } func (*ConfigureTopicResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
type GetTopicConfigurationRequest struct { type GetTopicConfigurationRequest struct {
Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"` Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"`
@ -445,7 +412,7 @@ type GetTopicConfigurationRequest struct {
func (m *GetTopicConfigurationRequest) Reset() { *m = GetTopicConfigurationRequest{} } func (m *GetTopicConfigurationRequest) Reset() { *m = GetTopicConfigurationRequest{} }
func (m *GetTopicConfigurationRequest) String() string { return proto.CompactTextString(m) } func (m *GetTopicConfigurationRequest) String() string { return proto.CompactTextString(m) }
func (*GetTopicConfigurationRequest) ProtoMessage() {} func (*GetTopicConfigurationRequest) ProtoMessage() {}
func (*GetTopicConfigurationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } func (*GetTopicConfigurationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
func (m *GetTopicConfigurationRequest) GetNamespace() string { func (m *GetTopicConfigurationRequest) GetNamespace() string {
if m != nil { if m != nil {
@ -468,7 +435,7 @@ type GetTopicConfigurationResponse struct {
func (m *GetTopicConfigurationResponse) Reset() { *m = GetTopicConfigurationResponse{} } func (m *GetTopicConfigurationResponse) Reset() { *m = GetTopicConfigurationResponse{} }
func (m *GetTopicConfigurationResponse) String() string { return proto.CompactTextString(m) } func (m *GetTopicConfigurationResponse) String() string { return proto.CompactTextString(m) }
func (*GetTopicConfigurationResponse) ProtoMessage() {} func (*GetTopicConfigurationResponse) ProtoMessage() {}
func (*GetTopicConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } func (*GetTopicConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
func (m *GetTopicConfigurationResponse) GetConfiguration() *TopicConfiguration { func (m *GetTopicConfigurationResponse) GetConfiguration() *TopicConfiguration {
if m != nil { if m != nil {
@ -487,7 +454,7 @@ type TopicConfiguration struct {
func (m *TopicConfiguration) Reset() { *m = TopicConfiguration{} } func (m *TopicConfiguration) Reset() { *m = TopicConfiguration{} }
func (m *TopicConfiguration) String() string { return proto.CompactTextString(m) } func (m *TopicConfiguration) String() string { return proto.CompactTextString(m) }
func (*TopicConfiguration) ProtoMessage() {} func (*TopicConfiguration) ProtoMessage() {}
func (*TopicConfiguration) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } func (*TopicConfiguration) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
func (m *TopicConfiguration) GetPartitionCount() int32 { func (m *TopicConfiguration) GetPartitionCount() int32 {
if m != nil { if m != nil {
@ -521,7 +488,6 @@ func init() {
proto.RegisterType((*SubscriberMessage)(nil), "messaging_pb.SubscriberMessage") proto.RegisterType((*SubscriberMessage)(nil), "messaging_pb.SubscriberMessage")
proto.RegisterType((*SubscriberMessage_InitMessage)(nil), "messaging_pb.SubscriberMessage.InitMessage") proto.RegisterType((*SubscriberMessage_InitMessage)(nil), "messaging_pb.SubscriberMessage.InitMessage")
proto.RegisterType((*SubscriberMessage_AckMessage)(nil), "messaging_pb.SubscriberMessage.AckMessage") proto.RegisterType((*SubscriberMessage_AckMessage)(nil), "messaging_pb.SubscriberMessage.AckMessage")
proto.RegisterType((*RawData)(nil), "messaging_pb.RawData")
proto.RegisterType((*Message)(nil), "messaging_pb.Message") proto.RegisterType((*Message)(nil), "messaging_pb.Message")
proto.RegisterType((*BrokerMessage)(nil), "messaging_pb.BrokerMessage") proto.RegisterType((*BrokerMessage)(nil), "messaging_pb.BrokerMessage")
proto.RegisterType((*BrokerMessage_RedirectMessage)(nil), "messaging_pb.BrokerMessage.RedirectMessage") proto.RegisterType((*BrokerMessage_RedirectMessage)(nil), "messaging_pb.BrokerMessage.RedirectMessage")
@ -777,59 +743,57 @@ var _SeaweedMessaging_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("messaging.proto", fileDescriptor0) } func init() { proto.RegisterFile("messaging.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{ var fileDescriptor0 = []byte{
// 849 bytes of a gzipped FileDescriptorProto // 832 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xdd, 0x6e, 0xe3, 0x44, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0x5d, 0x8f, 0xea, 0x44,
0x14, 0xde, 0xb1, 0xd3, 0x1f, 0x9f, 0xfc, 0x34, 0x8c, 0x28, 0x8a, 0x4c, 0x0b, 0xc6, 0x8b, 0x44, 0x18, 0x3e, 0x53, 0x3e, 0x76, 0x79, 0xa1, 0x80, 0x13, 0xd7, 0x90, 0xba, 0xab, 0xd8, 0x63, 0x14,
0xa0, 0xc2, 0xaa, 0xc2, 0x4d, 0x59, 0xad, 0x84, 0xda, 0x52, 0x96, 0x48, 0x0d, 0x44, 0x93, 0xdc, 0xdd, 0xd8, 0x6c, 0xf0, 0x66, 0x3d, 0x39, 0x89, 0x01, 0x82, 0x47, 0x92, 0xe5, 0x48, 0x06, 0x6e,
0xa2, 0x68, 0xe2, 0xcc, 0x66, 0x47, 0x49, 0x6c, 0xe3, 0x99, 0x10, 0xed, 0x35, 0xdc, 0x72, 0xc5, 0x4d, 0x53, 0xca, 0x1c, 0x76, 0x02, 0xb4, 0xb5, 0x33, 0xec, 0x66, 0xaf, 0xf5, 0xd6, 0x2b, 0xff,
0x1b, 0x70, 0xcb, 0x35, 0x0f, 0xc0, 0x03, 0xf0, 0x02, 0x3c, 0x0d, 0xf2, 0xf8, 0x27, 0x76, 0xe2, 0x81, 0xff, 0xc1, 0x1f, 0xe0, 0x6f, 0xf0, 0xce, 0x5f, 0x63, 0x3a, 0xfd, 0xa0, 0x05, 0x96, 0x5d,
0x66, 0x4b, 0xc4, 0xde, 0xd9, 0xc7, 0xdf, 0xf9, 0xce, 0xf9, 0xce, 0xcf, 0x8c, 0xe1, 0x64, 0xc1, 0x49, 0xce, 0x5d, 0xfb, 0xce, 0xf3, 0x3c, 0xef, 0xf7, 0xb4, 0x50, 0x5b, 0x51, 0xce, 0xad, 0x39,
0x84, 0xa0, 0x53, 0xee, 0x4d, 0x9d, 0x20, 0xf4, 0xa5, 0x8f, 0x6b, 0x99, 0x61, 0x14, 0x8c, 0xed, 0x73, 0xe6, 0x86, 0xe7, 0xbb, 0xc2, 0xc5, 0x95, 0xc4, 0x60, 0x7a, 0x53, 0xfd, 0xd7, 0x3c, 0x7c,
0x9f, 0x2b, 0xf0, 0xce, 0x60, 0x39, 0x16, 0x6e, 0xc8, 0xc7, 0x2c, 0xec, 0xa9, 0x4f, 0x0c, 0x7f, 0x30, 0x5e, 0x4f, 0xb9, 0xed, 0xb3, 0x29, 0xf5, 0x87, 0xf2, 0x88, 0xe2, 0xef, 0x21, 0xcf, 0x1c,
0x05, 0x15, 0xee, 0x71, 0xd9, 0x42, 0x16, 0x6a, 0x57, 0x3b, 0x17, 0x4e, 0xde, 0xc5, 0xd9, 0x82, 0x26, 0x1a, 0xa8, 0x89, 0x5a, 0xe5, 0xf6, 0xa5, 0x91, 0xa6, 0x18, 0x3b, 0x70, 0x63, 0xe0, 0x30,
0x3b, 0x5d, 0x8f, 0xcb, 0xe4, 0x99, 0x28, 0x47, 0xfc, 0x1c, 0x74, 0xea, 0xce, 0x5a, 0x9a, 0xf2, 0x11, 0x3d, 0x13, 0x49, 0xc4, 0xaf, 0x21, 0x67, 0xd9, 0x8b, 0x86, 0x22, 0xf9, 0x5f, 0x3f, 0xc5,
0xff, 0xec, 0x4d, 0xfe, 0xd7, 0xee, 0x2c, 0x75, 0x8f, 0xdc, 0xcc, 0xbf, 0x34, 0xa8, 0xe6, 0x38, 0xef, 0xd8, 0x8b, 0x98, 0x1e, 0xd0, 0xb4, 0xbf, 0x15, 0x28, 0xa7, 0x34, 0xf1, 0x39, 0x94, 0x1c,
0xf1, 0x19, 0x18, 0x1e, 0x5d, 0x30, 0x11, 0x50, 0x97, 0xa9, 0x9c, 0x0c, 0xb2, 0x36, 0xe0, 0x77, 0x6b, 0x45, 0xb9, 0x67, 0xd9, 0x54, 0xc6, 0x54, 0x22, 0x1b, 0x03, 0xfe, 0x10, 0x0a, 0xc2, 0xf5,
0xe1, 0x40, 0xfa, 0x01, 0x77, 0x55, 0x34, 0x83, 0xc4, 0x2f, 0x91, 0x4f, 0x40, 0x43, 0xc9, 0x25, 0x98, 0x2d, 0xbd, 0x95, 0x48, 0xf8, 0x12, 0x70, 0x3c, 0xcb, 0x17, 0x4c, 0x30, 0xd7, 0x69, 0xe4,
0xf7, 0xbd, 0x96, 0x6e, 0xa1, 0xf6, 0x01, 0x59, 0x1b, 0xf0, 0x08, 0xea, 0x42, 0xd2, 0x50, 0xf6, 0x9a, 0xa8, 0x55, 0x20, 0x1b, 0x03, 0x36, 0x41, 0xe5, 0xc2, 0xf2, 0xc5, 0xc8, 0xe5, 0x21, 0x22,
0x7d, 0x11, 0x23, 0x2a, 0x16, 0x6a, 0x37, 0x3a, 0x5f, 0xfe, 0x07, 0xa5, 0xce, 0x20, 0x4f, 0x40, 0xdf, 0x44, 0xad, 0x6a, 0xfb, 0xbb, 0xff, 0x91, 0xa9, 0x31, 0x4e, 0x0b, 0x90, 0xac, 0x1e, 0x6e,
0x8a, 0x7c, 0xd8, 0x82, 0xaa, 0xe4, 0x0b, 0x26, 0x24, 0x5d, 0x04, 0xdf, 0x89, 0xd6, 0x81, 0x85, 0x42, 0x59, 0xb0, 0x15, 0xe5, 0xc2, 0x5a, 0x79, 0x6f, 0x79, 0xa3, 0xd0, 0x44, 0xad, 0x1c, 0x49,
0xda, 0x3a, 0xc9, 0x9b, 0xf0, 0x53, 0xa8, 0x8b, 0x8c, 0x7f, 0xc4, 0x27, 0xad, 0x43, 0x95, 0x7e, 0x9b, 0xf0, 0x4b, 0x50, 0x79, 0xa2, 0x6f, 0xb2, 0x59, 0xa3, 0x28, 0xc3, 0xaf, 0x6c, 0x8c, 0x83,
0x6d, 0x6d, 0xec, 0x4e, 0xec, 0x2b, 0xa8, 0x17, 0xc2, 0x60, 0x80, 0xc3, 0xfb, 0xeb, 0xe1, 0xdd, 0x99, 0x7e, 0x0d, 0x6a, 0xc6, 0x0d, 0x06, 0x28, 0xde, 0x74, 0x26, 0xfd, 0xf1, 0xa4, 0xfe, 0x02,
0x60, 0xd8, 0x7c, 0x82, 0x6b, 0x70, 0x7c, 0x77, 0x4d, 0xee, 0xbb, 0xd1, 0x1b, 0xc2, 0x75, 0x30, 0x57, 0xe0, 0xb4, 0xdf, 0x21, 0x37, 0x83, 0xe0, 0x0d, 0x61, 0x15, 0x4a, 0x93, 0xc1, 0xb0, 0x3f,
0x86, 0xdd, 0xde, 0xdd, 0x60, 0x78, 0xdd, 0xeb, 0x37, 0x35, 0xf3, 0x02, 0x60, 0x5d, 0x56, 0x7c, 0x9e, 0x74, 0x86, 0xa3, 0xba, 0xa2, 0x5d, 0x02, 0x6c, 0xca, 0x8a, 0x2f, 0x00, 0xc2, 0xcc, 0x68,
0x0e, 0x10, 0x2b, 0x63, 0x51, 0x24, 0xa4, 0xb2, 0x31, 0x12, 0x4b, 0x77, 0x62, 0xff, 0x81, 0xe0, 0xe0, 0x09, 0xc9, 0x68, 0x4a, 0x91, 0x65, 0x30, 0xd3, 0xff, 0x41, 0x70, 0x12, 0x43, 0xbf, 0x00,
0x88, 0xd0, 0xd5, 0xd7, 0x54, 0x52, 0xdc, 0x04, 0x7d, 0xc6, 0x5e, 0x2b, 0x4c, 0x8d, 0x44, 0x8f, 0x95, 0xde, 0x51, 0x47, 0x98, 0x41, 0xb0, 0xa6, 0xc3, 0x43, 0x74, 0x57, 0xb9, 0x42, 0xa4, 0x2c,
0x51, 0x81, 0x7f, 0xa2, 0xf3, 0x25, 0x53, 0x05, 0xae, 0x91, 0xf8, 0x05, 0x3f, 0x87, 0xa3, 0x57, 0x0f, 0x26, 0x6c, 0x45, 0xdf, 0x72, 0x5c, 0x87, 0xdc, 0x82, 0x3e, 0xc8, 0xa2, 0x57, 0x48, 0xf0,
0x8c, 0x4e, 0x58, 0x28, 0x5a, 0xba, 0xa5, 0xb7, 0xab, 0x1d, 0xbb, 0x58, 0xbc, 0x84, 0xcf, 0xf9, 0x18, 0x34, 0xe2, 0xce, 0x5a, 0xae, 0xa9, 0x2c, 0x77, 0x85, 0x84, 0x2f, 0xf8, 0x35, 0x9c, 0xdc,
0x36, 0x06, 0xdd, 0x79, 0x32, 0x7c, 0x4d, 0x52, 0x17, 0xf3, 0x19, 0xd4, 0xf2, 0x1f, 0xf2, 0x51, 0x52, 0x6b, 0x46, 0x7d, 0xde, 0xc8, 0x37, 0x73, 0xad, 0x72, 0x5b, 0xcf, 0x16, 0x39, 0x2e, 0xe7,
0x8d, 0x1d, 0x51, 0x9f, 0x69, 0x57, 0xc8, 0xfe, 0x1b, 0xc1, 0x51, 0x2a, 0xcc, 0x02, 0x23, 0x2b, 0x8f, 0x21, 0xa8, 0xef, 0x08, 0xff, 0x81, 0xc4, 0x14, 0xed, 0x15, 0x54, 0xd2, 0x07, 0xb1, 0xd7,
0x6a, 0xac, 0xeb, 0x46, 0xbb, 0x44, 0x64, 0x6d, 0x4c, 0x99, 0xb5, 0x12, 0x3d, 0xfa, 0x03, 0x7a, 0x70, 0x08, 0xb2, 0x5e, 0x95, 0x94, 0xd7, 0x57, 0xca, 0x35, 0xd2, 0xff, 0x42, 0xa0, 0x76, 0x7d,
0x2a, 0x65, 0x7a, 0xd2, 0xb6, 0xff, 0xff, 0x7a, 0xfe, 0x44, 0x50, 0xbf, 0x09, 0xfd, 0xd9, 0x7a, 0x77, 0xb1, 0x99, 0xeb, 0xaf, 0x20, 0x3f, 0xb3, 0x84, 0x15, 0xcd, 0xf5, 0xd9, 0xde, 0x40, 0x88,
0xff, 0x3e, 0x85, 0xca, 0x84, 0x4a, 0x9a, 0xec, 0xdf, 0x69, 0x69, 0x22, 0x44, 0x41, 0xf0, 0x0b, 0x84, 0xe0, 0x37, 0x70, 0xea, 0xd3, 0x19, 0xf3, 0xa9, 0x2d, 0xa2, 0x31, 0xde, 0x5a, 0x83, 0x8c,
0x38, 0x0e, 0xd9, 0x84, 0x87, 0xcc, 0x95, 0xc9, 0xba, 0x6d, 0xac, 0x6b, 0x81, 0xd9, 0x21, 0x09, 0xb2, 0x41, 0x22, 0x6c, 0x2c, 0x92, 0x90, 0xb5, 0x2b, 0xa8, 0x6d, 0x1d, 0x06, 0xdd, 0x70, 0xe8,
0x36, 0x25, 0xc9, 0x9c, 0xcd, 0x4b, 0x38, 0xd9, 0xf8, 0x18, 0x4d, 0x8d, 0xc7, 0x56, 0xa3, 0xb1, 0xbd, 0x39, 0x95, 0x0a, 0xc9, 0x40, 0xd3, 0xfb, 0x50, 0x52, 0xff, 0x17, 0x41, 0x75, 0xb4, 0x9e,
0x62, 0xc8, 0x16, 0x8f, 0xad, 0x62, 0x4a, 0xfb, 0x1f, 0x04, 0x8d, 0xfe, 0x72, 0x3c, 0xe7, 0xe2, 0x2e, 0x19, 0xbf, 0x25, 0xf4, 0x97, 0x35, 0xe5, 0xc1, 0x3e, 0xa5, 0x17, 0xb2, 0x95, 0x8d, 0x24,
0x15, 0x61, 0x3f, 0x2e, 0x99, 0x88, 0xf6, 0x3e, 0x7f, 0x70, 0xb4, 0x8b, 0x99, 0x14, 0xb1, 0x25, 0x8b, 0xdd, 0xb3, 0x8d, 0x71, 0xda, 0xca, 0x93, 0x69, 0x6b, 0xe6, 0x7b, 0xde, 0x3c, 0xfd, 0x77,
0xa7, 0x46, 0x2a, 0x5b, 0x2b, 0x93, 0x9d, 0xcc, 0x53, 0x2c, 0xdb, 0x1c, 0xbd, 0xe5, 0x13, 0xc2, 0x05, 0x6a, 0x49, 0xc0, 0xdc, 0x73, 0x1d, 0x4e, 0x71, 0x0f, 0x8a, 0xb6, 0xeb, 0xbc, 0x63, 0xf3,
0xfe, 0x55, 0x83, 0x93, 0x2c, 0x61, 0x11, 0xf8, 0x9e, 0x60, 0xf8, 0x16, 0x0e, 0x5d, 0xdf, 0x7b, 0xfd, 0x17, 0xce, 0x16, 0xdc, 0xe8, 0x49, 0x6c, 0x1c, 0x77, 0x44, 0xc5, 0x83, 0x9d, 0x86, 0x7d,
0xc9, 0xa7, 0xe5, 0x07, 0xe3, 0x06, 0xdc, 0xb9, 0x55, 0xd8, 0x54, 0x62, 0xe2, 0x8a, 0xbb, 0x5b, 0x73, 0x58, 0xe6, 0xf1, 0x96, 0x5d, 0x83, 0x9a, 0xf1, 0x81, 0xbf, 0x84, 0x5a, 0x92, 0x81, 0x69,
0x0d, 0xfb, 0x7c, 0x37, 0xcd, 0xc3, 0x2d, 0xbb, 0x82, 0x7a, 0x21, 0x06, 0xfe, 0x04, 0x4e, 0x32, 0xbb, 0x6b, 0x27, 0xec, 0x44, 0x81, 0x54, 0x13, 0x73, 0x2f, 0xb0, 0x1e, 0xd1, 0xec, 0x3f, 0x10,
0x05, 0x23, 0xd7, 0x5f, 0x7a, 0x71, 0x27, 0x0e, 0x48, 0x23, 0x33, 0xdf, 0x46, 0xd6, 0x3d, 0x9a, 0x9c, 0x85, 0xce, 0xd6, 0x3e, 0x9d, 0x04, 0x05, 0x8c, 0x7b, 0x7e, 0x4c, 0xed, 0x7f, 0x00, 0xd5,
0xfd, 0x1b, 0x82, 0xd3, 0x38, 0xd8, 0x32, 0x64, 0xc3, 0xa8, 0x80, 0x69, 0xcf, 0xf7, 0xa9, 0xfd, 0x8e, 0xc4, 0xac, 0xa4, 0xfe, 0xe5, 0x76, 0x33, 0x5b, 0x09, 0xe9, 0xa6, 0x97, 0xc6, 0x91, 0x2c,
0x37, 0x50, 0x77, 0x13, 0x32, 0x9a, 0xd5, 0xbf, 0xda, 0xb1, 0x8a, 0x95, 0x50, 0x61, 0x6e, 0xf3, 0x4d, 0x6f, 0xc0, 0x47, 0xdb, 0x41, 0x85, 0x55, 0xd3, 0x09, 0x9c, 0xbf, 0xa1, 0x62, 0x8f, 0xc2,
0x38, 0x52, 0x74, 0xb3, 0x5b, 0xf0, 0xde, 0x66, 0x52, 0x71, 0xd5, 0x6c, 0x02, 0x67, 0x2f, 0x98, 0xf1, 0x51, 0xeb, 0x73, 0xb8, 0x78, 0x44, 0x33, 0x1a, 0x90, 0x9d, 0xb4, 0xd0, 0x71, 0x69, 0xfd,
0x2c, 0x61, 0xd8, 0x3f, 0x6b, 0x7b, 0x0a, 0xe7, 0x0f, 0x70, 0x26, 0x03, 0xb2, 0x25, 0x0b, 0xed, 0x89, 0x00, 0xef, 0xa2, 0x9e, 0xdd, 0x5e, 0xfc, 0x09, 0x80, 0xed, 0x2e, 0x97, 0xd4, 0x96, 0x41,
0x27, 0xeb, 0x77, 0x04, 0x78, 0x1b, 0xf5, 0xe8, 0xf6, 0xe2, 0x0f, 0x00, 0x5c, 0x7f, 0x3e, 0x67, 0x84, 0x39, 0xa4, 0x2c, 0xc1, 0xad, 0xef, 0x53, 0x6f, 0xc9, 0xec, 0x4d, 0xf1, 0x4b, 0x24, 0x6d,
0xae, 0x4a, 0x22, 0xd6, 0x90, 0xb3, 0x44, 0xb7, 0x53, 0xc8, 0x82, 0x39, 0x77, 0xd7, 0xc5, 0x37, 0xc2, 0x9f, 0x41, 0x85, 0x71, 0x53, 0xf8, 0x96, 0xc3, 0x19, 0x75, 0x84, 0xfc, 0xee, 0x9c, 0x92,
0x48, 0xde, 0x84, 0x3f, 0x82, 0x1a, 0x17, 0x23, 0x19, 0x52, 0x4f, 0x70, 0xe6, 0x49, 0x75, 0x3f, 0x32, 0xe3, 0x93, 0xd8, 0xd4, 0xfe, 0x2d, 0x07, 0xf5, 0x31, 0xb5, 0xee, 0x29, 0x9d, 0x0d, 0xe3,
0x1e, 0x93, 0x2a, 0x17, 0xc3, 0xd4, 0xd4, 0xf9, 0x45, 0x87, 0xe6, 0x80, 0xd1, 0x15, 0x63, 0x93, 0xf4, 0xf0, 0x4f, 0x50, 0x4a, 0xbe, 0x46, 0xf8, 0xd3, 0x27, 0x3e, 0x53, 0xda, 0xc7, 0x07, 0xae,
0x5e, 0x2a, 0x0f, 0x7f, 0x0f, 0x46, 0x76, 0x6b, 0xe2, 0x0f, 0xdf, 0x70, 0x9d, 0x9a, 0xef, 0xef, 0x2a, 0xfd, 0x45, 0x0b, 0x5d, 0x21, 0x7c, 0x03, 0x27, 0xd1, 0x42, 0xe0, 0xf3, 0x43, 0xd7, 0x89,
0x38, 0xaa, 0xec, 0x27, 0x6d, 0x74, 0x89, 0xf0, 0x3d, 0x1c, 0x25, 0x0b, 0x81, 0xcf, 0x76, 0x1d, 0x76, 0x71, 0x70, 0x8b, 0x22, 0xb5, 0x9f, 0xa1, 0x9a, 0x9d, 0x17, 0xfc, 0x32, 0x4b, 0xdb, 0x3b,
0x27, 0xe6, 0xf9, 0xce, 0x2d, 0x4a, 0xd8, 0x7e, 0x80, 0x46, 0x71, 0x5e, 0xf0, 0xd3, 0xa2, 0x5b, 0xe2, 0xda, 0xe7, 0x87, 0x41, 0xb1, 0x0b, 0xec, 0xc3, 0xd9, 0xde, 0x01, 0xc1, 0x5b, 0xbf, 0x16,
0xe9, 0x88, 0x9b, 0x1f, 0xef, 0x06, 0xa5, 0x21, 0x70, 0x08, 0xa7, 0xa5, 0x03, 0x82, 0x37, 0x7e, 0x87, 0x26, 0x53, 0xbb, 0x7c, 0x16, 0x36, 0xf6, 0xd9, 0xd5, 0xa1, 0xce, 0xc3, 0x2e, 0xbc, 0xe3,
0x81, 0x76, 0x4d, 0xa6, 0x79, 0xf1, 0x28, 0x6c, 0x1a, 0xf3, 0xc6, 0x86, 0xa6, 0x88, 0xbb, 0xf0, 0x86, 0xbd, 0x0c, 0x5a, 0xd3, 0xad, 0x26, 0x0d, 0x19, 0x05, 0xff, 0x52, 0xd3, 0xa2, 0xfc, 0xa5,
0x52, 0x38, 0xee, 0x3c, 0x6a, 0xcd, 0x4d, 0x23, 0x6b, 0x48, 0x3f, 0xfa, 0xe7, 0x1b, 0x1f, 0xaa, 0xfa, 0xf6, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa8, 0x6c, 0x82, 0x62, 0x65, 0x09, 0x00, 0x00,
0x5f, 0xbf, 0x2f, 0xfe, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x62, 0x10, 0x0f, 0xed, 0x0d, 0x0a, 0x00,
0x00,
} }

49
weed/util/log_buffer/log_buffer.go
View File

@ -1,6 +1,7 @@
package log_buffer package log_buffer
import ( import (
"bytes"
"sync" "sync"
"time" "time"
@ -17,7 +18,7 @@ const PreviousBufferCount = 3
type dataToFlush struct { type dataToFlush struct {
startTime time.Time startTime time.Time
stopTime time.Time stopTime time.Time
data []byte data *bytes.Buffer
} }
type LogBuffer struct { type LogBuffer struct {
@ -108,7 +109,8 @@ func (m *LogBuffer) Shutdown() {
func (m *LogBuffer) loopFlush() { func (m *LogBuffer) loopFlush() {
for d := range m.flushChan { for d := range m.flushChan {
if d != nil { if d != nil {
m.flushFn(d.startTime, d.stopTime, d.data) m.flushFn(d.startTime, d.stopTime, d.data.Bytes())
d.releaseMemory()
} }
} }
} }
@ -140,21 +142,26 @@ func (m *LogBuffer) copyToFlush() *dataToFlush {
return nil return nil
} }
func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (ts time.Time, bufferCopy []byte) { func (d *dataToFlush) releaseMemory() {
d.data.Reset()
bufferPool.Put(d.data)
}
func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (bufferCopy *bytes.Buffer) {
m.RLock() m.RLock()
defer m.RUnlock() defer m.RUnlock()
// fmt.Printf("read from buffer: %v\n", lastReadTime) // fmt.Printf("read from buffer: %v last stop time: %v\n", lastReadTime.UnixNano(), m.stopTime.UnixNano())
if lastReadTime.Equal(m.stopTime) { if lastReadTime.Equal(m.stopTime) {
return lastReadTime, nil return nil
} }
if lastReadTime.After(m.stopTime) { if lastReadTime.After(m.stopTime) {
// glog.Fatalf("unexpected last read time %v, older than latest %v", lastReadTime, m.stopTime) // glog.Fatalf("unexpected last read time %v, older than latest %v", lastReadTime, m.stopTime)
return lastReadTime, nil return nil
} }
if lastReadTime.Before(m.startTime) { if lastReadTime.Before(m.startTime) {
return m.stopTime, copiedBytes(m.buf[:m.pos]) return copiedBytes(m.buf[:m.pos])
} }
lastTs := lastReadTime.UnixNano() lastTs := lastReadTime.UnixNano()
@ -177,7 +184,7 @@ func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (ts time.Time, buffer
for l <= h { for l <= h {
mid := (l + h) / 2 mid := (l + h) / 2
pos := m.idx[mid] pos := m.idx[mid]
_, t := readTs(m.buf, m.idx[mid]) _, t := readTs(m.buf, pos)
if t <= lastTs { if t <= lastTs {
l = mid + 1 l = mid + 1
} else if lastTs < t { } else if lastTs < t {
@ -186,22 +193,32 @@ func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (ts time.Time, buffer
_, prevT = readTs(m.buf, m.idx[mid-1]) _, prevT = readTs(m.buf, m.idx[mid-1])
} }
if prevT <= lastTs { if prevT <= lastTs {
// println("found mid = ", mid) // fmt.Printf("found l=%d, m-1=%d(ts=%d), m=%d(ts=%d), h=%d [%d, %d) \n", l, mid-1, prevT, mid, t, h, pos, m.pos)
return time.Unix(0, t), copiedBytes(m.buf[pos:m.pos]) return copiedBytes(m.buf[pos:m.pos])
} }
h = mid - 1 h = mid
} }
// fmt.Printf("l=%d, h=%d\n", l, h) // fmt.Printf("l=%d, h=%d\n", l, h)
} }
// FIXME: this could be that the buffer has been flushed already // FIXME: this could be that the buffer has been flushed already
// println("not found") return nil
return lastReadTime, nil
} }
func copiedBytes(buf []byte) (copied []byte) { func (m *LogBuffer) ReleaseMeory(b *bytes.Buffer) {
copied = make([]byte, len(buf)) b.Reset()
copy(copied, buf) bufferPool.Put(b)
}
var bufferPool = sync.Pool{
New: func() interface{} {
return new(bytes.Buffer)
},
}
func copiedBytes(buf []byte) (copied *bytes.Buffer) {
copied = bufferPool.Get().(*bytes.Buffer)
copied.Write(buf)
return return
} }

41
weed/util/log_buffer/log_buffer_test.go Normal file
View File

@ -0,0 +1,41 @@
package log_buffer
import (
"math/rand"
"testing"
"time"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)
func TestNewLogBuffer(t *testing.T) {
lb := NewLogBuffer(time.Second, func(startTime, stopTime time.Time, buf []byte) {
}, func() {
})
startTime := time.Now()
messageSize := 1024
messageCount := 994
var buf = make([]byte, messageSize)
for i := 0; i < messageCount; i++ {
rand.Read(buf)
lb.AddToBuffer(nil, buf)
}
receivedmessageCount := 0
lb.LoopProcessLogData(startTime, func() bool {
// stop if no more messages
return false
}, func(logEntry *filer_pb.LogEntry) error {
receivedmessageCount++
return nil
})
if receivedmessageCount != messageCount {
t.Errorf("sent %d received %d", messageCount, receivedmessageCount)
}
}

74
weed/util/log_buffer/log_read.go Normal file
View File

@ -0,0 +1,74 @@
package log_buffer
import (
"bytes"
"time"
"github.com/golang/protobuf/proto"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
)
func (logBuffer *LogBuffer) LoopProcessLogData(
startTreadTime time.Time, waitForDataFn func() bool, eachLogDataFn func(logEntry *filer_pb.LogEntry) error) (processed int64, err error) {
// loop through all messages
var bytesBuf *bytes.Buffer
lastReadTime := startTreadTime
defer func() {
if bytesBuf != nil {
logBuffer.ReleaseMeory(bytesBuf)
}
}()
for {
if bytesBuf != nil {
logBuffer.ReleaseMeory(bytesBuf)
}
bytesBuf = logBuffer.ReadFromBuffer(lastReadTime)
if bytesBuf == nil {
if waitForDataFn() {
continue
} else {
return
}
}
buf := bytesBuf.Bytes()
batchSize := 0
var startReadTime time.Time
for pos := 0; pos+4 < len(buf); {
size := util.BytesToUint32(buf[pos : pos+4])
entryData := buf[pos+4 : pos+4+int(size)]
// fmt.Printf("read buffer read %d [%d,%d) from [0,%d)\n", batchSize, pos, pos+int(size)+4, len(buf))
logEntry := &filer_pb.LogEntry{}
if err = proto.Unmarshal(entryData, logEntry); err != nil {
glog.Errorf("unexpected unmarshal messaging_pb.Message: %v", err)
pos += 4 + int(size)
continue
}
lastReadTime = time.Unix(0, logEntry.TsNs)
if startReadTime.IsZero() {
startReadTime = lastReadTime
}
if err = eachLogDataFn(logEntry); err != nil {
return
}
pos += 4 + int(size)
batchSize++
processed++
}
// fmt.Printf("sent message ts[%d,%d] size %d\n", startReadTime.UnixNano(), lastReadTime.UnixNano(), batchSize)
}
}