From 5d1712e30977c29b82d3bd95a3502ee3f11d5df9 Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 13 Aug 2025 18:05:52 -0700 Subject: [PATCH] delete in ecj, add logs --- weed/server/volume_grpc_erasure_coding.go | 6 +++- weed/server/volume_server_handlers_write.go | 5 +++ .../erasure_coding/ec_volume_delete.go | 35 +++++++++++++++++-- weed/storage/store_ec_delete.go | 13 +++++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/weed/server/volume_grpc_erasure_coding.go b/weed/server/volume_grpc_erasure_coding.go index 8500a2e80..8b55594e9 100644 --- a/weed/server/volume_grpc_erasure_coding.go +++ b/weed/server/volume_grpc_erasure_coding.go @@ -469,7 +469,7 @@ func (vs *VolumeServer) VolumeEcShardRead(req *volume_server_pb.VolumeEcShardRea func (vs *VolumeServer) VolumeEcBlobDelete(ctx context.Context, req *volume_server_pb.VolumeEcBlobDeleteRequest) (*volume_server_pb.VolumeEcBlobDeleteResponse, error) { - glog.V(0).Infof("VolumeEcBlobDelete: %v", req) + glog.Infof("🔍 GRPC EC BLOB DELETE: volume %d, needle %d", req.VolumeId, req.FileKey) resp := &volume_server_pb.VolumeEcBlobDeleteResponse{} @@ -481,14 +481,18 @@ func (vs *VolumeServer) VolumeEcBlobDelete(ctx context.Context, req *volume_serv return nil, fmt.Errorf("locate in local ec volume: %w", err) } if size.IsDeleted() { + glog.Infof("✅ GRPC EC DELETE: needle %d already deleted", req.FileKey) return resp, nil } + glog.Infof("📝 GRPC EC DELETE: recording needle %d in .ecj", req.FileKey) err = localEcVolume.DeleteNeedleFromEcx(types.NeedleId(req.FileKey)) if err != nil { + glog.Errorf("❌ GRPC EC DELETE: failed to record needle %d: %v", req.FileKey, err) return nil, err } + glog.Infof("✅ GRPC EC DELETE: successfully recorded needle %d", req.FileKey) break } } diff --git a/weed/server/volume_server_handlers_write.go b/weed/server/volume_server_handlers_write.go index dc7f64f6c..acf3cafb8 100644 --- a/weed/server/volume_server_handlers_write.go +++ b/weed/server/volume_server_handlers_write.go @@ -88,10 +88,15 @@ func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { ecVolume, hasEcVolume := vs.store.FindEcVolume(volumeId) + glog.Infof("🔍 DELETE REQUEST: volume %d, needle %d, hasEcVolume=%t", volumeId, n.Id, hasEcVolume) + if hasEcVolume { + glog.Infof("🎯 ROUTING TO EC DELETION: volume %d, needle %d", volumeId, n.Id) count, err := vs.store.DeleteEcShardNeedle(ecVolume, n, cookie) writeDeleteResult(err, count, w, r) return + } else { + glog.Infof("🎯 ROUTING TO REGULAR DELETION: volume %d, needle %d", volumeId, n.Id) } _, ok := vs.store.ReadVolumeNeedle(volumeId, n, nil, nil) diff --git a/weed/storage/erasure_coding/ec_volume_delete.go b/weed/storage/erasure_coding/ec_volume_delete.go index 076176bea..309fce720 100644 --- a/weed/storage/erasure_coding/ec_volume_delete.go +++ b/weed/storage/erasure_coding/ec_volume_delete.go @@ -5,6 +5,7 @@ import ( "io" "os" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/util" ) @@ -30,20 +31,48 @@ func (ev *EcVolume) DeleteNeedleFromEcx(needleId types.NeedleId) (err error) { if err != nil { if err == NotFoundError { + glog.Infof("❓ EC NEEDLE NOT FOUND: needle %d not in .ecx index for volume %d generation %d - skipping .ecj recording", + needleId, ev.VolumeId, ev.Generation) return nil } + glog.Errorf("❌ EC INDEX SEARCH ERROR: needle %d volume %d generation %d: %v", needleId, ev.VolumeId, ev.Generation, err) return err } + // Needle found and marked deleted in .ecx, now record in .ecj + glog.Infof("📝 EC NEEDLE FOUND: recording needle %d in .ecj for volume %d generation %d", + needleId, ev.VolumeId, ev.Generation) + b := make([]byte, types.NeedleIdSize) types.NeedleIdToBytes(b, needleId) ev.ecjFileAccessLock.Lock() + defer ev.ecjFileAccessLock.Unlock() - ev.ecjFile.Seek(0, io.SeekEnd) - ev.ecjFile.Write(b) + if ev.ecjFile == nil { + glog.Errorf("EC deletion: .ecj file is nil for volume %d generation %d", ev.VolumeId, ev.Generation) + return fmt.Errorf("ecjFile is nil") + } - ev.ecjFileAccessLock.Unlock() + _, err = ev.ecjFile.Seek(0, io.SeekEnd) + if err != nil { + glog.Errorf("EC deletion: failed to seek .ecj file for volume %d generation %d: %v", ev.VolumeId, ev.Generation, err) + return err + } + + n, err := ev.ecjFile.Write(b) + if err != nil { + glog.Errorf("EC deletion: failed to write to .ecj file for volume %d generation %d: %v", ev.VolumeId, ev.Generation, err) + return err + } + + if n != len(b) { + glog.Errorf("EC deletion: partial write to .ecj file for volume %d generation %d: wrote %d bytes, expected %d", + ev.VolumeId, ev.Generation, n, len(b)) + return fmt.Errorf("partial write: wrote %d bytes, expected %d", n, len(b)) + } + + glog.Infof("✅ EC JOURNAL WRITE SUCCESS: wrote %d bytes to .ecj for volume %d generation %d", n, ev.VolumeId, ev.Generation) return } diff --git a/weed/storage/store_ec_delete.go b/weed/storage/store_ec_delete.go index a3e028bbb..00f86df7a 100644 --- a/weed/storage/store_ec_delete.go +++ b/weed/storage/store_ec_delete.go @@ -3,6 +3,7 @@ package storage import ( "context" "fmt" + "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -15,6 +16,8 @@ import ( func (s *Store) DeleteEcShardNeedle(ecVolume *erasure_coding.EcVolume, n *needle.Needle, cookie types.Cookie) (int64, error) { + glog.Infof("🚀 EC DELETE SHARD NEEDLE: starting deletion for needle %d volume %d", n.Id, ecVolume.VolumeId) + count, err := s.ReadEcShardNeedle(ecVolume.VolumeId, n, nil) if err != nil { @@ -29,6 +32,16 @@ func (s *Store) DeleteEcShardNeedle(ecVolume *erasure_coding.EcVolume, n *needle return 0, err } + // Record the deletion locally in the .ecj journal file + glog.Infof("🔍 EC DELETION: Recording needle %d in volume %d generation %d", + n.Id, ecVolume.VolumeId, ecVolume.Generation) + if err = ecVolume.DeleteNeedleFromEcx(n.Id); err != nil { + glog.Errorf("❌ Failed to record EC deletion in journal for needle %d: %v", n.Id, err) + // Continue even if journal write fails - the remote deletion succeeded + } else { + glog.Infof("✅ EC deletion recording completed for needle %d", n.Id) + } + return int64(count), nil }