lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-09-23 04:13:35 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: Add logical type support to SQL query engine

Browse Source 
Extended SQL engine to handle new Parquet logical types:
- Added TimestampValue comparison support (microsecond precision)
- Added DateValue comparison support (days since epoch)
- Added DecimalValue comparison support with string conversion
- Added TimeValue comparison support (microseconds since midnight)
- Enhanced valuesEqual(), valueLessThan(), valueGreaterThan() functions
- Added decimalToString() helper for precise decimal-to-string conversion
- Imported math/big for arbitrary precision decimal handling

The SQL engine can now:
- ✅ Compare TIMESTAMP values for filtering (e.g., WHERE timestamp > 1672531200000000000)
- ✅ Compare DATE values for date-based queries (e.g., WHERE birth_date >= 12345)
- ✅ Compare DECIMAL values for precise financial calculations
- ✅ Compare TIME values for time-of-day filtering

Next: Add YEAR(), MONTH(), DAY() extraction functions for date analytics.
This commit is contained in:
chrislu
2025-09-03 07:29:03 -07:00
parent 3570027656
commit 699e2f4413
2 changed files with 112 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
test/postgres/producer.go
View File

@@ -31,7 +31,7 @@ type UserEvent struct {
Status string `json:"status"`
Amount float64 `json:"amount,omitempty"`
PreciseAmount string `json:"precise_amount,omitempty"` // Will be converted to DECIMAL
BirthDate time.Time `json:"birth_date"` // Will be converted to DATE
BirthDate time.Time `json:"birth_date"` // Will be converted to DATE
Timestamp time.Time `json:"timestamp"`
Metadata string `json:"metadata,omitempty"`
}
@@ -199,16 +199,16 @@ func convertToDecimal(value string) ([]byte, int32, int32) {
if _, success := rat.SetString(value); !success {
return nil, 0, 0
}
// Convert to a fixed scale (e.g., 4 decimal places)
scale := int32(4)
precision := int32(18) // Total digits
// Scale the rational number to integer representation
multiplier := new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(scale)), nil)
scaled := new(big.Int).Mul(rat.Num(), multiplier)
scaled.Div(scaled, rat.Denom())
return scaled.Bytes(), precision, scale
}
@@ -224,7 +224,7 @@ func convertToRecordValue(data interface{}) (*schema_pb.RecordValue, error) {
fields["action"] = &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: v.Action}}
fields["status"] = &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: v.Status}}
fields["amount"] = &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: v.Amount}}
// Convert precise amount to DECIMAL logical type
if v.PreciseAmount != "" {
if decimal, precision, scale := convertToDecimal(v.PreciseAmount); decimal != nil {
@@ -235,15 +235,15 @@ func convertToRecordValue(data interface{}) (*schema_pb.RecordValue, error) {
}}}
}
}
// Convert birth date to DATE logical type
fields["birth_date"] = &schema_pb.Value{Kind: &schema_pb.Value_DateValue{DateValue: &schema_pb.DateValue{
DaysSinceEpoch: int32(v.BirthDate.Unix() / 86400), // Convert to days since epoch
}}}
fields["timestamp"] = &schema_pb.Value{Kind: &schema_pb.Value_TimestampValue{TimestampValue: &schema_pb.TimestampValue{
TimestampMicros: v.Timestamp.UnixMicro(),
IsUtc: true,
IsUtc: true,
}}}
fields["metadata"] = &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: v.Metadata}}
@@ -255,7 +255,7 @@ func convertToRecordValue(data interface{}) (*schema_pb.RecordValue, error) {
fields["error_code"] = &schema_pb.Value{Kind: &schema_pb.Value_Int32Value{Int32Value: int32(v.ErrorCode)}}
fields["timestamp"] = &schema_pb.Value{Kind: &schema_pb.Value_TimestampValue{TimestampValue: &schema_pb.TimestampValue{
TimestampMicros: v.Timestamp.UnixMicro(),
IsUtc: true,
IsUtc: true,
}}}
case MetricEntry:
@@ -265,7 +265,7 @@ func convertToRecordValue(data interface{}) (*schema_pb.RecordValue, error) {
fields["tags"] = &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: v.Tags}}
fields["timestamp"] = &schema_pb.Value{Kind: &schema_pb.Value_TimestampValue{TimestampValue: &schema_pb.TimestampValue{
TimestampMicros: v.Timestamp.UnixMicro(),
IsUtc: true,
IsUtc: true,
}}}
case ProductView:
@@ -277,7 +277,7 @@ func convertToRecordValue(data interface{}) (*schema_pb.RecordValue, error) {
fields["view_count"] = &schema_pb.Value{Kind: &schema_pb.Value_Int32Value{Int32Value: int32(v.ViewCount)}}
fields["timestamp"] = &schema_pb.Value{Kind: &schema_pb.Value_TimestampValue{TimestampValue: &schema_pb.TimestampValue{
TimestampMicros: v.Timestamp.UnixMicro(),
IsUtc: true,
IsUtc: true,
}}}
default:
@@ -441,7 +441,7 @@ func generateUserEvent() interface{} {
birthMonth := 1 + rand.Intn(12)
birthDay := 1 + rand.Intn(28) // Keep it simple, avoid month-specific day issues
birthDate := time.Date(birthYear, time.Month(birthMonth), birthDay, 0, 0, 0, 0, time.UTC)
// Generate a precise amount as a string with 4 decimal places
preciseAmount := fmt.Sprintf("%.4f", rand.Float64()*10000)