Skip to content

Commit 72f86e5

Browse files
CinchooCinchoo
committed
Upgraded to .NET 4.8
1 parent fa64986 commit 72f86e5

File tree

75 files changed

+820
-188
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+820
-188
lines changed

‎src/Assets/Db/Northwind.MDF‎

0 Bytes
Binary file not shown.

‎src/Assets/Db/Northwind_log.ldf‎

0 Bytes
Binary file not shown.

‎src/ChoETL.Avro/ChoAvroRecordReader.cs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -305,12 +305,16 @@ private bool LoadNode(Tuple<long, object> pair, ref object rec)
305305

306306
try
307307
{
308-
if (!RaiseBeforeRecordLoad(rec, ref pair))
308+
bool isHandled = false;
309+
if (!RaiseBeforeRecordLoad(rec, ref pair, ref isHandled))
309310
{
310311
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
311312
rec = null;
312313
return true;
313314
}
315+
if (isHandled)
316+
return true;
317+
314318
//if (Configuration.CustomNodeSelecter != null)
315319
//{
316320
// pair = new Tuple<long, object>(pair.Item1, Configuration.CustomNodeSelecter(pair.Item2));
@@ -466,13 +470,15 @@ private void RaiseEndLoad(object state)
466470
return null;
467471
}
468472

469-
private bool RaiseBeforeRecordLoad(object target, ref Tuple<long, object> pair)
473+
private bool RaiseBeforeRecordLoad(object target, ref Tuple<long, object> pair, ref bool isHandled)
470474
{
471475
if (Reader != null && Reader.HasBeforeRecordLoadSubscribed)
472476
{
473477
long index = pair.Item1;
474478
object state = pair.Item2;
475-
bool retValue = ChoFuncEx.RunWithIgnoreError(() => Reader.RaiseBeforeRecordLoad(target, index, ref state), true);
479+
bool isHandled1 = false;
480+
bool retValue = ChoFuncEx.RunWithIgnoreError(() => Reader.RaiseBeforeRecordLoad(target, index, ref state, ref isHandled1), true);
481+
isHandled = isHandled1;
476482

477483
if (retValue)
478484
pair = new Tuple<long, object>(index, state as IDictionary<string, object>);

‎src/ChoETL.Benchmark/ChoETL.Benchmark.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net5.0</TargetFramework>
5+
<TargetFrameworks>netstandard2.0;net9.0;net8.0;net7.0;net5.0;net6.0</TargetFrameworks>
66
</PropertyGroup>
77

88
<ItemGroup>

‎src/ChoETL.Benchmark/Program.cs‎

Lines changed: 86 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,87 @@ internal class Program
1111
static void Main(string[] args)
1212
{
1313
ChoETLFrxBootstrap.TraceLevel = TraceLevel.Off;
14-
CSV2JSON();
14+
QuoteFieldValuesTest();
15+
return;
16+
17+
CustomerLoadTest();
18+
//CSV2JSON();
19+
}
20+
21+
static void QuoteFieldValuesTest()
22+
{
23+
string csv = @"V BELT,30-1/32""|19.44||12.69";
24+
25+
using (var r = ChoCSVReader.LoadText(csv)
26+
.WithDelimiter("|")
27+
.Configure(c => c.QuoteChar = '#')
28+
)
29+
{
30+
r.ToArray().Print();
31+
}
32+
}
33+
public class Customer
34+
{
35+
[ChoCSVRecordField()]
36+
public int Index { get; set; }
37+
[ChoCSVRecordField(FieldName = "Customer Id")]
38+
public string CustomerId { get; set; }
39+
[ChoCSVRecordField(FieldName = "First Name")]
40+
public string FirstName { get; set; }
41+
[ChoCSVRecordField(FieldName = "Last Name")]
42+
public string LastName { get; set; }
43+
[ChoCSVRecordField]
44+
public string Company { get; set; }
45+
[ChoCSVRecordField]
46+
public string City { get; set; }
47+
[ChoCSVRecordField]
48+
public string Country { get; set; }
1549
}
50+
public static void CustomerLoadTest()
51+
{
52+
//string csv = @"C:\Users\nraj39\Downloads\customers-10000.csv";
53+
string csv = @"C:\Users\nraj39\Downloads\customers-2000000.csv";
54+
55+
for (var i = 0; i < 10; i++)
56+
{
57+
using (var r = new ChoCSVReader<Customer>(csv)
58+
.WithFirstLineHeader()
59+
.NotifyAfter(1000)
60+
.Setup(s => s.RowsLoaded += (o, e) =>
61+
{
62+
//$"Rows read: {e.RowsLoaded}.".Print();
63+
if (e.RowsLoaded == 1000000)
64+
e.Abort = true;
65+
})
66+
.Setup(s => s.BeforeRecordLoad += (o, e) =>
67+
{
68+
var customer = e.Record as Customer;
69+
var payload = e.Payload as string[];
70+
71+
if (payload != null)
72+
{
73+
customer.Index = int.Parse(payload[0]); //.CastTo<int>();
74+
customer.CustomerId = payload[1];
75+
customer.FirstName = payload[2];
76+
customer.LastName = payload[3];
77+
customer.Company = payload[4];
78+
customer.City = payload[5];
79+
customer.Country = payload[6];
80+
}
81+
e.Handled = true;
82+
})
83+
)
84+
{
85+
Stopwatch w = Stopwatch.StartNew();
86+
var customers = r.ToList();
87+
88+
customers.LastOrDefault()?.Print();
89+
w.Stop();
90+
$"Total rows read: {customers.Count}, Elapsed: {w.ElapsedMilliseconds} ms, {w.Elapsed}".Print();
91+
}
92+
}
93+
}
94+
1695
public class Trade
1796
{
1897
public string Id { get; set; }
@@ -27,7 +106,12 @@ static void CSV2JSON()
27106

28107
using (var w = new ChoJSONWriter<Trade>(@"C:\Projects\GitHub\ChoETL\data\XBTUSD.json")
29108
.NotifyAfter(100000)
30-
.Setup(s => s.RowsWritten += (o, e) => $"Rows written: {e.RowsWritten}.".Print())
109+
.Setup(s => s.RowsWritten += (o, e) =>
110+
{
111+
$"Rows written: {e.RowsWritten}.".Print();
112+
if (e.RowsWritten == 1000000)
113+
e.Abort = true;
114+
})
31115
)
32116
{
33117
w.Write(parser.ReadFile<Trade>(filePath, mapper: (lineNo, cols, rec) =>

‎src/ChoETL.JSON.NETStandard/ChoETL.JSON.NETStandard.csproj‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFramework>netstandard2.0</TargetFramework>
5-
<AssemblyName>ChoETL.JSON.Core</AssemblyName>
4+
<TargetFrameworks>netstandard2.0;net9.0;net8.0;net7.0;net5.0;net6.0</TargetFrameworks>
5+
<AssemblyName>ChoETL.JSON.Core</AssemblyName>
66
<RootNamespace>ChoETL.JSON</RootNamespace>
77
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
88
<PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>

‎src/ChoETL.JSON/ChoJSONRecordReader.cs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -849,12 +849,16 @@ private bool LoadNode(Tuple<long, JObject> pair, ref object rec)
849849

850850
try
851851
{
852-
if (!RaiseBeforeRecordLoad(rec, ref pair))
852+
bool isHandled = false;
853+
if (!RaiseBeforeRecordLoad(rec, ref pair, ref isHandled))
853854
{
854855
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
855856
rec = null;
856857
return true;
857858
}
859+
if (isHandled)
860+
return true;
861+
858862
if (Configuration.CustomNodeSelecter != null)
859863
{
860864
pair = new Tuple<long, JObject>(pair.Item1, Configuration.CustomNodeSelecter(pair.Item2));
@@ -2554,13 +2558,15 @@ private void RaiseEndLoad(object state)
25542558
return null;
25552559
}
25562560

2557-
private bool RaiseBeforeRecordLoad(object target, ref Tuple<long, JObject> pair)
2561+
private bool RaiseBeforeRecordLoad(object target, ref Tuple<long, JObject> pair, ref bool isHandled)
25582562
{
25592563
if (Reader != null && Reader.HasBeforeRecordLoadSubscribed)
25602564
{
25612565
long index = pair.Item1;
25622566
object state = pair.Item2;
2563-
bool retValue = ChoFuncEx.RunWithIgnoreError(() => Reader.RaiseBeforeRecordLoad(target, index, ref state), true);
2567+
bool isHandled1 = false;
2568+
bool retValue = ChoFuncEx.RunWithIgnoreError(() => Reader.RaiseBeforeRecordLoad(target, index, ref state, ref isHandled1), true);
2569+
isHandled = isHandled1;
25642570

25652571
if (retValue)
25662572
pair = new Tuple<long, JObject>(index, state as JObject);

‎src/ChoETL.NETStandard/ChoETL.NETStandard.csproj‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFramework>netstandard2.0</TargetFramework>
4+
<TargetFrameworks>netstandard2.0;net9.0;net8.0;net7.0;net5.0;net6.0</TargetFrameworks>
55
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
66
<PackageId>ChoETL.NETStandard</PackageId>
77
<Authors>Cinchoo</Authors>
@@ -39,6 +39,7 @@
3939
<PackageReference Include="System.ComponentModel.Annotations" Version="4.4.1" />
4040
<PackageReference Include="System.Configuration.ConfigurationManager" Version="4.4.1" />
4141
<PackageReference Include="System.Data.SqlClient" Version="4.9.0" />
42+
<PackageReference Include="System.Memory" Version="4.5.4" />
4243
<PackageReference Include="System.Reflection.Emit" Version="4.3.0" />
4344
<PackageReference Include="System.Reflection.Emit.Lightweight" Version="4.3.0" />
4445
</ItemGroup>
@@ -268,7 +269,8 @@
268269
<Compile Include="..\ChoETL\File\TSV\ChoTSVReader.cs" Link="File\TSV\ChoTSVReader.cs" />
269270
<Compile Include="..\ChoETL\File\CSV\ChoCSVWriter.cs" Link="File\CSV\ChoCSVWriter.cs" />
270271
<Compile Include="..\ChoETL\File\CSV\ChoCSVReader.cs" Link="File\CSV\ChoCSVReader.cs" />
271-
<Compile Include="..\ChoETL\File\CSV\ChoCSVLiteReader.cs" Link="File\CSV\ChoCSVLiteReader.cs" />
272+
<Compile Include="..\ChoETL\File\CSV\RowBy.cs" Link="File\CSV\RowBy.cs" />
273+
<Compile Include="..\ChoETL\File\CSV\ChoCSVLiteReader.cs" Link="File\CSV\ChoCSVLiteReader.cs" />
272274
<Compile Include="..\ChoETL\File\CSV\ChoCSVLiteWriter.cs" Link="File\CSV\ChoCSVLiteWriter.cs" />
273275
<Compile Include="..\ChoETL\File\CSV\ChoCSVRecordReader.cs" Link="File\CSV\ChoCSVRecordReader.cs" />
274276
<Compile Include="..\ChoETL\File\ChoFileHeaderConfiguration.cs" Link="File\ChoFileHeaderConfiguration.cs" />

‎src/ChoETL.Parquet/ChoETL.Parquet.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
<SignAssembly>true</SignAssembly>
1919
<AssemblyOriginatorKeyFile>ChoETL.Parquet.snk</AssemblyOriginatorKeyFile>
2020
<PackageIcon>cinchoo.png</PackageIcon>
21-
<TargetFrameworks>net6.0;net5.0;</TargetFrameworks>
21+
<TargetFrameworks>netstandard2.0;net9.0;net8.0;net7.0;net5.0;net6.0</TargetFrameworks>
2222
</PropertyGroup>
2323

2424
<ItemGroup>

‎src/ChoETL.Parquet/ChoParquetRecordReader.cs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -365,12 +365,16 @@ private bool LoadNode(Tuple<long, IDictionary<string, object>> pair, ref object
365365

366366
try
367367
{
368-
if (!RaiseBeforeRecordLoad(rec, ref pair))
368+
bool isHandled = false;
369+
if (!RaiseBeforeRecordLoad(rec, ref pair, ref isHandled))
369370
{
370371
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
371372
rec = null;
372373
return true;
373374
}
375+
if (isHandled)
376+
return true;
377+
374378
//if (Configuration.CustomNodeSelecter != null)
375379
//{
376380
// pair = new Tuple<long, IDictionary<string, object>>(pair.Item1, Configuration.CustomNodeSelecter(pair.Item2));
@@ -561,7 +565,8 @@ private bool FillRecord(ref object rec, Tuple<long, IDictionary<string, object>>
561565

562566
object v1 = node;
563567

564-
if (Configuration.IsDynamicObjectInternal && fieldConfig.FieldType != null && IsInNeedOfCustomFormatter(fieldConfig.FieldType))
568+
if (Configuration.IsDynamicObjectInternal && fieldConfig.FieldType != null && IsInNeedOfCustomFormatter(fieldConfig.FieldType)
569+
&& fieldConfig.ValueConverter == null)
565570
{
566571
fieldValue = CustomDeserialize(fieldValue.ToNString(), fieldConfig.FieldType);
567572
}
@@ -850,13 +855,15 @@ private void RaiseEndLoad(object state)
850855
return null;
851856
}
852857

853-
private bool RaiseBeforeRecordLoad(object target, ref Tuple<long, IDictionary<string, object>> pair)
858+
private bool RaiseBeforeRecordLoad(object target, ref Tuple<long, IDictionary<string, object>> pair, ref bool isHandled)
854859
{
855860
if (Reader != null && Reader.HasBeforeRecordLoadSubscribed)
856861
{
857862
long index = pair.Item1;
858863
object state = pair.Item2;
859-
bool retValue = ChoFuncEx.RunWithIgnoreError(() => Reader.RaiseBeforeRecordLoad(target, index, ref state), true);
864+
bool isHandled1 = false;
865+
bool retValue = ChoFuncEx.RunWithIgnoreError(() => Reader.RaiseBeforeRecordLoad(target, index, ref state, ref isHandled1), true);
866+
isHandled = isHandled1;
860867

861868
if (retValue)
862869
pair = new Tuple<long, IDictionary<string, object>>(index, state as IDictionary<string, object>);

0 commit comments

Comments
 (0)