add compaction test

weed/storage/volume.go

@@ -29,6 +29,7 @@ type Volume struct {
 }
 
 func NewVolume(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL, preallocate int64) (v *Volume, e error) {
+	// if replicaPlacement is nil, the superblock will be loaded from disk
 	v = &Volume{dir: dirname, Collection: collection, Id: id}
 	v.SuperBlock = SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}
 	v.needleMapKind = needleMapKind

weed/storage/volume_vacuum_test.go

@@ -2,6 +2,9 @@ package storage
 
 import (
 	"testing"
+	"os"
+	"io/ioutil"
+	"math/rand"
 )
 
 /*
@@ -53,3 +56,97 @@ func TestMakeDiff(t *testing.T) {
 		}
 	*/
 }
+
+func TestCompaction(t *testing.T) {
+	dir, err := ioutil.TempDir("", "example")
+	if err != nil {
+		t.Fatalf("temp dir creation: %v", err)
+	}
+	defer os.RemoveAll(dir) // clean up
+
+	v, err := NewVolume(dir, "", 1, NeedleMapInMemory, &ReplicaPlacement{}, &TTL{}, 0)
+	if err != nil {
+		t.Fatalf("volume creation: %v", err)
+	}
+
+	FILE_COUNT := 234
+
+	infos := make([]*needleInfo, FILE_COUNT)
+
+	for i := 1; i <= FILE_COUNT; i++ {
+		n := newRandomNeedle(uint64(i))
+		size, err := v.writeNeedle(n)
+		if err != nil {
+			t.Fatalf("write file %d: %v", i, err)
+		}
+		infos[i-1] = &needleInfo{
+			size: size,
+			crc:  n.Checksum,
+		}
+
+		println("written file", i, "checksum", n.Checksum.Value(), "size", size)
+
+		if rand.Float64() < 0.5 {
+			toBeDeleted := rand.Intn(i) + 1
+			oldNeedle := newEmptyNeedle(uint64(toBeDeleted))
+			v.deleteNeedle(oldNeedle)
+			println("deleted file", toBeDeleted)
+			infos[toBeDeleted-1] = &needleInfo{
+				size: 0,
+				crc:  n.Checksum,
+			}
+		}
+	}
+
+	v.Compact(0)
+	v.commitCompact()
+
+	v.Close()
+
+	v, err = NewVolume(dir, "", 1, NeedleMapInMemory, nil, nil, 0)
+	if err != nil {
+		t.Fatalf("volume reloading: %v", err)
+	}
+
+	for i := 1; i <= FILE_COUNT; i++ {
+
+		if infos[i-1].size == 0 {
+			continue
+		}
+
+		n := newEmptyNeedle(uint64(i))
+		size, err := v.readNeedle(n)
+		if err != nil {
+			t.Fatalf("read file %d: %v", i, err)
+		}
+		if infos[i-1].size != uint32(size) {
+			t.Fatalf("read file %d size mismatch expected %d found %d", i, infos[i-1].size, size)
+		}
+		if infos[i-1].crc != n.Checksum {
+			t.Fatalf("read file %d checksum mismatch expected %d found %d", i, infos[i-1].crc, n.Checksum)
+		}
+
+	}
+
+}
+
+type needleInfo struct {
+	size uint32
+	crc  CRC
+}
+
+func newRandomNeedle(id uint64) *Needle {
+	n := new(Needle)
+	n.Data = make([]byte, rand.Intn(1024))
+	rand.Read(n.Data)
+
+	n.Checksum = NewCRC(n.Data)
+	n.Id = id
+	return n
+}
+
+func newEmptyNeedle(id uint64) *Needle {
+	n := new(Needle)
+	n.Id = id
+	return n
+}