can read/write files in format version 2

weed-fs/src/cmd/weed/version.go

@@ -6,7 +6,7 @@ import (
 )
 
 const (
-	VERSION = "0.25"
+	VERSION = "0.26"
 )
 
 var cmdVersion = &Command{

weed-fs/src/pkg/storage/needle.go

@@ -20,13 +20,15 @@ type Needle struct {
 	Cookie   uint32 "random number to mitigate brute force lookups"
 	Id       uint64 "needle id"
 	Size     uint32 "sum of DataSize,Data,NameSize,Name,MimeSize,Mime"
-	Flags    byte   "boolean flags" //version2
+
-	DataSize uint32 "Data size"     //version2
+	DataSize uint32 "Data size" //version2
 	Data     []byte "The actual file data"
+	Flags    byte   "boolean flags" //version2
 	NameSize uint8  //version2
 	Name     []byte "maximum 256 characters" //version2
 	MimeSize uint8  //version2
 	Mime     []byte "maximum 256 characters" //version2
+
 	Checksum CRC    "CRC32 to check integrity"
 	Padding  []byte "Aligned to 8 bytes"
 }

weed-fs/src/pkg/storage/needle_read_write.go

@@ -19,7 +19,44 @@ func (n *Needle) Append(w io.Writer, version Version) uint32 {
 		rest := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + 4) % NeedlePaddingSize)
 		util.Uint32toBytes(header[0:4], n.Checksum.Value())
 		w.Write(header[0 : 4+rest])
+        return n.Size
 	} else if version == Version2 {
+		header := make([]byte, NeedleHeaderSize)
+		util.Uint32toBytes(header[0:4], n.Cookie)
+		util.Uint64toBytes(header[4:12], n.Id)
+		n.DataSize, n.NameSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Name)), uint8(len(n.Mime))
+		if n.DataSize > 0 {
+			n.Size = 4 + n.DataSize + 1
+			if n.NameSize > 0 {
+				n.Size = n.Size + 1 + uint32(n.NameSize)
+			}
+			if n.NameSize > 0 {
+				n.Size = n.Size + 1 + uint32(n.MimeSize)
+			}
+		}
+		util.Uint32toBytes(header[12:16], n.Size)
+		w.Write(header)
+		if n.DataSize > 0 {
+			util.Uint32toBytes(header[0:4], n.DataSize)
+			w.Write(header[0:4])
+			w.Write(n.Data)
+			util.Uint8toBytes(header[0:1], n.Flags)
+			w.Write(header[0:1])
+		}
+		if n.NameSize > 0 {
+			util.Uint8toBytes(header[0:1], n.NameSize)
+			w.Write(header[0:1])
+			w.Write(n.Name)
+		}
+		if n.MimeSize > 0 {
+			util.Uint8toBytes(header[0:1], n.MimeSize)
+			w.Write(header[0:1])
+			w.Write(n.Mime)
+		}
+		rest := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + 4) % NeedlePaddingSize)
+		util.Uint32toBytes(header[0:4], n.Checksum.Value())
+		w.Write(header[0 : 4+rest])
+        return n.DataSize
 	}
 	return n.Size
 }
@@ -27,33 +64,62 @@ func (n *Needle) Read(r io.Reader, size uint32, version Version) (int, error) {
 	if version == Version1 {
 		bytes := make([]byte, NeedleHeaderSize+size+4)
 		ret, e := r.Read(bytes)
-		n.Cookie = util.BytesToUint32(bytes[0:4])
+		n.readNeedleHeader(bytes)
-		n.Id = util.BytesToUint64(bytes[4:12])
+		n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size]
-		n.Size = util.BytesToUint32(bytes[12:16])
+		checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+4])
-		n.Data = bytes[16 : 16+size]
-		checksum := util.BytesToUint32(bytes[16+size : 16+size+4])
 		if checksum != NewCRC(n.Data).Value() {
 			return 0, errors.New("CRC error! Data On Disk Corrupted!")
 		}
 		return ret, e
 	} else if version == Version2 {
+		bytes := make([]byte, NeedleHeaderSize+size+4)
+		ret, e := r.Read(bytes)
+		n.readNeedleHeader(bytes)
+		n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)])
+		checksum := util.BytesToUint32(bytes[NeedleHeaderSize+n.Size : NeedleHeaderSize+n.Size+4])
+		if checksum != NewCRC(n.Data).Value() {
+			return 0, errors.New("CRC error! Data On Disk Corrupted!")
+		}
+		return ret, e
 	}
 	return 0, errors.New("Unsupported Version!")
 }
+func (n *Needle) readNeedleHeader(bytes []byte) {
+	n.Cookie = util.BytesToUint32(bytes[0:4])
+	n.Id = util.BytesToUint64(bytes[4:12])
+	n.Size = util.BytesToUint32(bytes[12:NeedleHeaderSize])
+}
+func (n *Needle) readNeedleDataVersion2(bytes []byte) {
+	index := 0
+	n.DataSize = util.BytesToUint32(bytes[index : index+4])
+	index = index + 4
+	n.Data = bytes[index : index+int(n.DataSize)]
+	index = index + int(n.DataSize)
+	n.Flags = bytes[index]
+	index = index + 1
+	if index < len(bytes) {
+		n.NameSize = uint8(bytes[index])
+		index = index + 1
+		n.Name = bytes[index : index+int(n.NameSize)]
+		index = index + int(n.NameSize)
+	}
+	if index < len(bytes) {
+		n.MimeSize = uint8(bytes[index])
+		index = index + 1
+		n.Mime = bytes[index : index+int(n.MimeSize)]
+	}
+}
 func ReadNeedleHeader(r *os.File, version Version) (n *Needle, bodyLength uint32) {
 	n = new(Needle)
-	if version == Version1 {
+	if version == Version1 || version == Version2 {
 		bytes := make([]byte, NeedleHeaderSize)
 		count, e := r.Read(bytes)
 		if count <= 0 || e != nil {
 			return nil, 0
 		}
-		n.Cookie = util.BytesToUint32(bytes[0:4])
+		n.readNeedleHeader(bytes)
-		n.Id = util.BytesToUint64(bytes[4:12])
+		rest := NeedlePaddingSize - ((n.Size + NeedleHeaderSize + 4) % NeedlePaddingSize)
-		n.Size = util.BytesToUint32(bytes[12:16])
-		rest := NeedlePaddingSize - ((n.Size + 16 + 4) % NeedlePaddingSize)
 		bodyLength = n.Size + 4 + rest
-	} else if version == Version2 {
 	}
 	return
 }
@@ -67,6 +133,10 @@ func (n *Needle) ReadNeedleBody(r *os.File, version Version, bodyLength uint32)
 		n.Data = bytes[:n.Size]
 		n.Checksum = NewCRC(n.Data)
 	} else if version == Version2 {
+		bytes := make([]byte, bodyLength)
+		r.Read(bytes)
+		n.readNeedleDataVersion2(bytes[0:n.Size])
+		n.Checksum = NewCRC(n.Data)
 	}
 	return
 }

weed-fs/src/pkg/storage/volume.go

@@ -51,7 +51,7 @@ func (v *Volume) load() error {
 	return nil
 }
 func (v *Volume) Version() Version {
-	return CurrentVersion
+	return v.version
 }
 func (v *Volume) Size() int64 {
 	v.accessLock.Lock()

weed-fs/src/pkg/storage/volume_version.go

@@ -6,7 +6,7 @@ import (
 type Version uint8
 
 const (
-	CurrentVersion = Version(1)
 	Version1 = Version(1)
     Version2 = Version(2)
+    CurrentVersion = Version2
 )

weed-fs/src/pkg/util/bytes.go

@@ -28,4 +28,7 @@ func Uint32toBytes(b []byte, v uint32){
     b[3-i] = byte(v>>(i*8))
   }
 }
+func Uint8toBytes(b []byte, v uint8){
+  b[0] = byte(v)
+}