lsm/SqlSugar
1
0
Fork 0
mirror of https://gitee.com/dotnetchina/SqlSugar.git synced 2025-07-15 05:13:27 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

优化ToPivotTable和ToPivotList

Browse Source 
1. 进一步的提升转换效率  一百万行转换一千列不到一秒完成
2. .ToPivotTable(it => it.day, it =>new{ it.name}, it => it.Sum(x => x.count))
    it.day 存在空值会造成列名与列数据对不齐
3. 优化代码减小内存消耗，减少循环次数
4. 将ToPivotList的返回值 List<dynamic> 改为 IEnumerable<dynamic> 减少内存读写次数。
This commit is contained in:
YHS 2024-01-05 22:48:10 +08:00
parent 64133f6f28
commit bd5808de71
6 changed files with 104 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Src/Asp.Net/SqlSugar/Abstract/QueryableProvider/QueryableExecuteSql.cs
View File

@ -228,7 +228,7 @@ namespace SqlSugar
{ {
return this.ToList().ToPivotTable(columnSelector, rowSelector, dataSelector); return this.ToList().ToPivotTable(columnSelector, rowSelector, dataSelector);
} }
public virtual List<dynamic> ToPivotList<TColumn, TRow, TData>(Func<T, TColumn> columnSelector, Expression<Func<T, TRow>> rowSelector, Func<IEnumerable<T>, TData> dataSelector) public virtual IEnumerable<dynamic> ToPivotList<TColumn, TRow, TData>(Func<T, TColumn> columnSelector, Expression<Func<T, TRow>> rowSelector, Func<IEnumerable<T>, TData> dataSelector)
{ {
return this.ToList().ToPivotList(columnSelector, rowSelector, dataSelector); return this.ToList().ToPivotList(columnSelector, rowSelector, dataSelector);
} }

2
Src/Asp.Net/SqlSugar/Interface/IQueryable.cs
View File

@ -286,7 +286,7 @@ namespace SqlSugar
ISugarQueryable<T> IgnoreColumns(Expression<Func<T, object>> columns); ISugarQueryable<T> IgnoreColumns(Expression<Func<T, object>> columns);
ISugarQueryable<T> IgnoreColumns(params string[] columns); ISugarQueryable<T> IgnoreColumns(params string[] columns);
DataTable ToPivotTable<TColumn, TRow, TData>(Func<T, TColumn> columnSelector,Expression<Func<T, TRow>> rowSelector,Func<IEnumerable<T>, TData> dataSelector); DataTable ToPivotTable<TColumn, TRow, TData>(Func<T, TColumn> columnSelector,Expression<Func<T, TRow>> rowSelector,Func<IEnumerable<T>, TData> dataSelector);
List<dynamic> ToPivotList<TColumn, TRow, TData>(Func<T, TColumn> columnSelector, Expression<Func<T, TRow>> rowSelector, Func<IEnumerable<T>, TData> dataSelector); IEnumerable<dynamic> ToPivotList<TColumn, TRow, TData>(Func<T, TColumn> columnSelector, Expression<Func<T, TRow>> rowSelector, Func<IEnumerable<T>, TData> dataSelector);
string ToPivotJson<TColumn, TRow, TData>(Func<T, TColumn> columnSelector, Expression<Func<T, TRow>> rowSelector, Func<IEnumerable<T>, TData> dataSelector); string ToPivotJson<TColumn, TRow, TData>(Func<T, TColumn> columnSelector, Expression<Func<T, TRow>> rowSelector, Func<IEnumerable<T>, TData> dataSelector);
ISugarQueryable<T> SplitTable(Func<List<SplitTableInfo>,IEnumerable<SplitTableInfo>> getTableNamesFunc); ISugarQueryable<T> SplitTable(Func<List<SplitTableInfo>,IEnumerable<SplitTableInfo>> getTableNamesFunc);
ISugarQueryable<T> SplitTable(DateTime beginTime,DateTime endTime); ISugarQueryable<T> SplitTable(DateTime beginTime,DateTime endTime);

124
Src/Asp.Net/SqlSugar/Utilities/DataTableExtensions.cs
View File

@ -19,59 +19,51 @@ namespace SqlSugar
Func<IEnumerable<T>, TData> dataSelector) Func<IEnumerable<T>, TData> dataSelector)
{ {
DataTable table = new DataTable(); DataTable table = new DataTable();
var rowName = new List<string>(); var rowName = new List<string>();
if (rowSelector.Body is MemberExpression) if (rowSelector.Body is MemberExpression)
rowName.Add(((MemberExpression)rowSelector.Body).Member.Name); rowName.Add(((MemberExpression)rowSelector.Body).Member.Name);
else else
rowName.AddRange(((NewExpression)rowSelector.Body).Arguments.Select(it => it as MemberExpression).Select(it => it.Member.Name)); rowName.AddRange(((NewExpression)rowSelector.Body).Arguments.Select(it => it as MemberExpression).Select(it => it.Member.Name));
table.Columns.AddRange(rowName.Select(x => new DataColumn(x)).ToArray()); table.Columns.AddRange(rowName.Select(x => new DataColumn(x)).ToArray());
var columns = source.Select(columnSelector).Distinct(); var columns = source.Select(columnSelector).Distinct();
table.Columns.AddRange(columns.Select(x => new DataColumn(x?.ToString())).ToArray());
foreach (var column in columns)
table.Columns.Add(new DataColumn(column?.ToString()));
var rows = source.GroupBy(rowSelector.Compile()) var rows = source.GroupBy(rowSelector.Compile())
.Select(rowGroup => new .Select(rowGroup =>
{ {
Key = rowGroup.Key, var anonymousType = rowGroup.Key.GetType();
Values = columns.GroupJoin( var properties = anonymousType.GetProperties();
rowGroup, var row = table.NewRow();
c => c, columns.GroupJoin(rowGroup, c => c, r => columnSelector(r),
r => columnSelector(r), (c, columnGroup) =>
(c, columnGroup) => dataSelector(columnGroup)) {
});
foreach (var row in rows)
{
var dataRow = table.NewRow();
var items = row.Values.Cast<object>().ToList();
// 获取匿名对象的动态类型
var anonymousType = row.Key.GetType();
// 获取匿名对象的所有属性
var properties = anonymousType.GetProperties();
for (var i = 0; i < rowName.Count; i++)
{
items.Insert(i, properties[i].GetValue(row.Key, null));
}
dataRow.ItemArray = items.ToArray();
table.Rows.Add(dataRow);
}
var dic = new Dictionary<string, object>();
if (c != null)
dic[c.ToString()] = dataSelector(columnGroup);
return dic;
})
.SelectMany(x => x)
.Select(x => row[x.Key] = x.Value)
.SelectMany(x => properties, (x, y) => row[y.Name] = y.GetValue(rowGroup.Key, null))
.ToArray();
table.Rows.Add(row);
return row;
}
).ToList();
return table; return table;
} }
public static List<dynamic> ToPivotList<T, TColumn, TRow, TData>( public static IEnumerable<dynamic> ToPivotList<T, TColumn, TRow, TData>(
this IEnumerable<T> source, this IEnumerable<T> source,
Func<T, TColumn> columnSelector, Func<T, TColumn> columnSelector,
Expression<Func<T, TRow>> rowSelector, Expression<Func<T, TRow>> rowSelector,
Func<IEnumerable<T>, TData> dataSelector) Func<IEnumerable<T>, TData> dataSelector)
{ {
var arr = new List<dynamic>();
var cols = new List<string>();
var rowName = new List<string>(); var rowName = new List<string>();
if (rowSelector.Body is MemberExpression) if (rowSelector.Body is MemberExpression)
rowName.Add(((MemberExpression)rowSelector.Body).Member.Name); rowName.Add(((MemberExpression)rowSelector.Body).Member.Name);
@ -80,47 +72,27 @@ namespace SqlSugar
var columns = source.Select(columnSelector).Distinct(); var columns = source.Select(columnSelector).Distinct();
cols = rowName.Concat(columns.Select(x => x?.ToString())).ToList();
var rows = source.GroupBy(rowSelector.Compile()) var rows = source.GroupBy(rowSelector.Compile())
.Select(rowGroup => new .Select(rowGroup =>
{
Key = rowGroup.Key,
Values = columns.GroupJoin(
rowGroup,
c => c,
r => columnSelector(r),
(c, columnGroup) => dataSelector(columnGroup))
});
foreach (var row in rows)
{
var items = row.Values.Cast<object>().ToList();
// 获取匿名对象的动态类型
var anonymousType = row.Key.GetType();
// 获取匿名对象的所有属性
var properties = anonymousType.GetProperties();
for (var i = 0; i < rowName.Count; i++)
{ {
items.Insert(i, properties[i].GetValue(row.Key, null)); var anonymousType = rowGroup.Key.GetType();
} var properties = anonymousType.GetProperties();
var obj = GetAnonymousObject(cols, items); IDictionary<string, object> row = new ExpandoObject();
arr.Add(obj); columns.GroupJoin(rowGroup, c => c, r => columnSelector(r),
} (c, columnGroup) =>
return arr; {
} IDictionary<string, object> dic = new ExpandoObject();
private static dynamic GetAnonymousObject(IEnumerable<string> columns, IEnumerable<object> values) if (c != null)
{ dic[c.ToString()] = dataSelector(columnGroup);
IDictionary<string, object> eo = new ExpandoObject() as IDictionary<string, object>; return dic;
int i; })
for (i = 0; i < columns.Count(); i++) .SelectMany(x => x)
{ .Select(x => row[x.Key] = x.Value)
eo.Add(columns.ElementAt<string>(i), values.ElementAt<object>(i)); .SelectMany(x => properties, (x, y) => row[y.Name] = y.GetValue(rowGroup.Key, null))
} .ToList();
return eo; return row;
});
return rows;
} }
} }
} }

2
Src/Asp.NetCore2/SqlSugar/Abstract/QueryableProvider/QueryableExecuteSql.cs
View File

@ -228,7 +228,7 @@ namespace SqlSugar
{ {
return this.ToList().ToPivotTable(columnSelector, rowSelector, dataSelector); return this.ToList().ToPivotTable(columnSelector, rowSelector, dataSelector);
} }
public virtual List<dynamic> ToPivotList<TColumn, TRow, TData>(Func<T, TColumn> columnSelector, Expression<Func<T, TRow>> rowSelector, Func<IEnumerable<T>, TData> dataSelector) public virtual IEnumerable<dynamic> ToPivotList<TColumn, TRow, TData>(Func<T, TColumn> columnSelector, Expression<Func<T, TRow>> rowSelector, Func<IEnumerable<T>, TData> dataSelector)
{ {
return this.ToList().ToPivotList(columnSelector, rowSelector, dataSelector); return this.ToList().ToPivotList(columnSelector, rowSelector, dataSelector);
} }

2
Src/Asp.NetCore2/SqlSugar/Interface/IQueryable.cs
View File

@ -286,7 +286,7 @@ namespace SqlSugar
ISugarQueryable<T> IgnoreColumns(Expression<Func<T, object>> columns); ISugarQueryable<T> IgnoreColumns(Expression<Func<T, object>> columns);
ISugarQueryable<T> IgnoreColumns(params string[] columns); ISugarQueryable<T> IgnoreColumns(params string[] columns);
DataTable ToPivotTable<TColumn, TRow, TData>(Func<T, TColumn> columnSelector,Expression<Func<T, TRow>> rowSelector,Func<IEnumerable<T>, TData> dataSelector); DataTable ToPivotTable<TColumn, TRow, TData>(Func<T, TColumn> columnSelector,Expression<Func<T, TRow>> rowSelector,Func<IEnumerable<T>, TData> dataSelector);
List<dynamic> ToPivotList<TColumn, TRow, TData>(Func<T, TColumn> columnSelector, Expression<Func<T, TRow>> rowSelector, Func<IEnumerable<T>, TData> dataSelector); IEnumerable<dynamic> ToPivotList<TColumn, TRow, TData>(Func<T, TColumn> columnSelector, Expression<Func<T, TRow>> rowSelector, Func<IEnumerable<T>, TData> dataSelector);
string ToPivotJson<TColumn, TRow, TData>(Func<T, TColumn> columnSelector, Expression<Func<T, TRow>> rowSelector, Func<IEnumerable<T>, TData> dataSelector); string ToPivotJson<TColumn, TRow, TData>(Func<T, TColumn> columnSelector, Expression<Func<T, TRow>> rowSelector, Func<IEnumerable<T>, TData> dataSelector);
ISugarQueryable<T> SplitTable(Func<List<SplitTableInfo>,IEnumerable<SplitTableInfo>> getTableNamesFunc); ISugarQueryable<T> SplitTable(Func<List<SplitTableInfo>,IEnumerable<SplitTableInfo>> getTableNamesFunc);
ISugarQueryable<T> SplitTable(DateTime beginTime,DateTime endTime); ISugarQueryable<T> SplitTable(DateTime beginTime,DateTime endTime);

134
Src/Asp.NetCore2/SqlSugar/Utilities/DataTableExtensions.cs
View File

@ -2,10 +2,8 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Data; using System.Data;
using System.Dynamic; using System.Dynamic;
using System.IO;
using System.Linq; using System.Linq;
using System.Linq.Expressions; using System.Linq.Expressions;
using System.Reflection;
using System.Text; using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -15,65 +13,57 @@ namespace SqlSugar
internal static class DataTableExtensions internal static class DataTableExtensions
{ {
public static DataTable ToPivotTable<T, TColumn, TRow, TData>( public static DataTable ToPivotTable<T, TColumn, TRow, TData>(
this IEnumerable<T> source, this IEnumerable<T> source,
Func<T, TColumn> columnSelector, Func<T, TColumn> columnSelector,
Expression<Func<T, TRow>> rowSelector, Expression<Func<T, TRow>> rowSelector,
Func<IEnumerable<T>, TData> dataSelector) Func<IEnumerable<T>, TData> dataSelector)
{ {
DataTable table = new DataTable(); DataTable table = new DataTable();
var rowName = new List<string>(); var rowName = new List<string>();
if (rowSelector.Body is MemberExpression) if (rowSelector.Body is MemberExpression)
rowName.Add(((MemberExpression)rowSelector.Body).Member.Name); rowName.Add(((MemberExpression)rowSelector.Body).Member.Name);
else else
rowName.AddRange(((NewExpression)rowSelector.Body).Arguments.Select(it => it as MemberExpression).Select(it => it.Member.Name)); rowName.AddRange(((NewExpression)rowSelector.Body).Arguments.Select(it => it as MemberExpression).Select(it => it.Member.Name));
table.Columns.AddRange(rowName.Select(x => new DataColumn(x)).ToArray()); table.Columns.AddRange(rowName.Select(x => new DataColumn(x)).ToArray());
var columns = source.Select(columnSelector).Distinct(); var columns = source.Select(columnSelector).Distinct();
table.Columns.AddRange(columns.Select(x => new DataColumn(x?.ToString())).ToArray());
foreach (var column in columns)
table.Columns.Add(new DataColumn(column?.ToString()));
var rows = source.GroupBy(rowSelector.Compile()) var rows = source.GroupBy(rowSelector.Compile())
.Select(rowGroup => new .Select(rowGroup =>
{ {
Key = rowGroup.Key, var anonymousType = rowGroup.Key.GetType();
Values = columns.GroupJoin( var properties = anonymousType.GetProperties();
rowGroup, var row = table.NewRow();
c => c, columns.GroupJoin(rowGroup, c => c, r => columnSelector(r),
r => columnSelector(r), (c, columnGroup) =>
(c, columnGroup) => dataSelector(columnGroup)) {
});
foreach (var row in rows)
{
var dataRow = table.NewRow();
var items = row.Values.Cast<object>().ToList();
// 获取匿名对象的动态类型
var anonymousType = row.Key.GetType();
// 获取匿名对象的所有属性
var properties = anonymousType.GetProperties();
for (var i = 0; i < rowName.Count; i++)
{
items.Insert(i, properties[i].GetValue(row.Key, null));
}
dataRow.ItemArray = items.ToArray();
table.Rows.Add(dataRow);
}
var dic = new Dictionary<string, object>();
if (c != null)
dic[c.ToString()] = dataSelector(columnGroup);
return dic;
})
.SelectMany(x => x)
.Select(x => row[x.Key] = x.Value)
.SelectMany(x => properties, (x, y) => row[y.Name] = y.GetValue(rowGroup.Key, null))
.ToArray();
table.Rows.Add(row);
return row;
}
).ToList();
return table; return table;
} }
public static List<dynamic> ToPivotList<T, TColumn, TRow, TData>( public static IEnumerable<dynamic> ToPivotList<T, TColumn, TRow, TData>(
this IEnumerable<T> source, this IEnumerable<T> source,
Func<T, TColumn> columnSelector, Func<T, TColumn> columnSelector,
Expression<Func<T, TRow>> rowSelector, Expression<Func<T, TRow>> rowSelector,
Func<IEnumerable<T>, TData> dataSelector) Func<IEnumerable<T>, TData> dataSelector)
{ {
var arr = new List<dynamic>();
var cols = new List<string>();
var rowName = new List<string>(); var rowName = new List<string>();
if (rowSelector.Body is MemberExpression) if (rowSelector.Body is MemberExpression)
rowName.Add(((MemberExpression)rowSelector.Body).Member.Name); rowName.Add(((MemberExpression)rowSelector.Body).Member.Name);
@ -82,47 +72,27 @@ namespace SqlSugar
var columns = source.Select(columnSelector).Distinct(); var columns = source.Select(columnSelector).Distinct();
cols = rowName.Concat(columns.Select(x => x?.ToString())).ToList();
var rows = source.GroupBy(rowSelector.Compile()) var rows = source.GroupBy(rowSelector.Compile())
.Select(rowGroup => new .Select(rowGroup =>
{
Key = rowGroup.Key,
Values = columns.GroupJoin(
rowGroup,
c => c,
r => columnSelector(r),
(c, columnGroup) => dataSelector(columnGroup))
});
foreach (var row in rows)
{
var items = row.Values.Cast<object>().ToList();
// 获取匿名对象的动态类型
var anonymousType = row.Key.GetType();
// 获取匿名对象的所有属性
var properties = anonymousType.GetProperties();
for (var i = 0; i < rowName.Count; i++)
{ {
items.Insert(i, properties[i].GetValue(row.Key, null)); var anonymousType = rowGroup.Key.GetType();
} var properties = anonymousType.GetProperties();
var obj = GetAnonymousObject(cols, items); IDictionary<string, object> row = new ExpandoObject();
arr.Add(obj); columns.GroupJoin(rowGroup, c => c, r => columnSelector(r),
} (c, columnGroup) =>
return arr; {
} IDictionary<string, object> dic = new ExpandoObject();
private static dynamic GetAnonymousObject(IEnumerable<string> columns, IEnumerable<object> values) if (c != null)
{ dic[c.ToString()] = dataSelector(columnGroup);
IDictionary<string, object> eo = new ExpandoObject() as IDictionary<string, object>; return dic;
int i; })
for (i = 0; i < columns.Count(); i++) .SelectMany(x => x)
{ .Select(x => row[x.Key] = x.Value)
eo.Add(columns.ElementAt<string>(i), values.ElementAt<object>(i)); .SelectMany(x => properties, (x, y) => row[y.Name] = y.GetValue(rowGroup.Key, null))
} .ToList();
return eo; return row;
});
return rows;
} }
} }
} }