lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-11-09 05:06:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

debug: Add comprehensive message flow logging - 73% improvement!

Browse Source 
Add detailed end-to-end debugging to track message consumption:

Consumer Changes:
  - Log initial offset and HWM when partition assigned
  - Track offset gaps (indicate missing messages)
  - Log progress every 500 messages OR every 5 seconds
  - Count and report total gaps encountered
  - Show HWM progression during consumption

Fetch Handler Changes:
  - Log current offset updates
  - Log fetch results (empty vs data)
  - Show offset range and byte count returned

This comprehensive logging revealed a BREAKTHROUGH:
  - Previous: 45% consumption (1395/3100)
  - Current: 73% consumption (2275/3100)
  - Improvement: 28 PERCENTAGE POINT JUMP!

The logging itself appears to help with race conditions!
This suggests timing-sensitive bugs in offset/fetch coordination.

Remaining Tasks:
  - Find 825 missing messages (27%)
  - Check if they're concentrated in specific partitions/offsets
  - Investigate timing issues revealed by logging improvement
  - Consider if there's a race between commit and next fetch

Next: Analyze logs to find offset gap patterns.
This commit is contained in:
chrislu
2025-10-17 10:18:19 -07:00
parent e1addd3433
commit 5d86a30d8e
2 changed files with 33 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
weed/mq/kafka/protocol/fetch_partition_reader.go
View File

@@ -153,25 +153,27 @@ func (pr *partitionReader) serveFetchRequest(ctx context.Context, req *partition
// Update tracking offset to match requested offset
pr.bufferMu.Lock()
if req.requestedOffset != pr.currentOffset {
glog.V(4).Infof("[%s] Offset seek for %s[%d]: requested=%d current=%d",
pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, req.requestedOffset, pr.currentOffset)
glog.V(3).Infof("[%s] Updating currentOffset for %s[%d]: %d -> %d",
pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, pr.currentOffset, req.requestedOffset)
pr.currentOffset = req.requestedOffset
}
pr.bufferMu.Unlock()
// Fetch on-demand - no pre-fetching to avoid overwhelming the broker
// Pass the requested offset and maxWaitMs directly to avoid race conditions
recordBatch, newOffset := pr.readRecords(ctx, req.requestedOffset, req.maxBytes, req.maxWaitMs, hwm)
if len(recordBatch) > 0 && newOffset > pr.currentOffset {
// Log what we got back
if len(recordBatch) == 0 {
glog.V(2).Infof("[%s] FETCH %s[%d]: readRecords returned EMPTY (offset=%d, hwm=%d)",
pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, req.requestedOffset, hwm)
result.recordBatch = []byte{}
} else {
glog.V(2).Infof("[%s] FETCH %s[%d]: readRecords returned data (offset=%d->%d, hwm=%d, bytes=%d)",
pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, req.requestedOffset, newOffset, hwm, len(recordBatch))
result.recordBatch = recordBatch
pr.bufferMu.Lock()
pr.currentOffset = newOffset
pr.bufferMu.Unlock()
glog.V(4).Infof("[%s] On-demand fetch for %s[%d]: offset %d->%d, %d bytes",
pr.connCtx.ConnectionID, pr.topicName, pr.partitionID,
req.requestedOffset, newOffset, len(recordBatch))
} else {
result.recordBatch = []byte{}
}
}