Skip to content

Commit 4bb7cce

Browse files
zapovzapov
committed
Improvements and new examples
Make Android work without Xalan (use stax instead) Remove usage of thread locals in AlternativeProperty (use references instead) Show how to get streaming behavior with iterator and how to stream XML. Pass xml instruction via XML instead of tag metadata.
1 parent 2b65f3b commit 4bb7cce

File tree

35 files changed

+529
-137
lines changed

35 files changed

+529
-137
lines changed

‎Advanced/CsvStreaming/Readme.md‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎Advanced/README.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,12 @@ Consuming embedded CSV or Excel table via Power Query (Requires Excel 2010+)
4444

4545
[template](PowerQuery/template/PowerQuery.xlsx?raw=true) - [result](PowerQuery/result.xlsx?raw=true)
4646

47-
### [CSV streaming](CsvStreaming/Readme.md)
47+
### [CSV streaming](Streaming/Readme.md)
4848

49-
Stream CSV while processing to support huge exports
49+
Stream CSV/XML while processing to support huge exports
5050

51-
[template](CsvStreaming/template/input.csv) - [result](CsvStreaming/result.csv)
51+
[csv template](Streaming/template/input.csv) - [result](Streaming/result.csv)
52+
[xml template](Streaming/template/input.xml) - [result](Streaming/result.xml)
5253

5354
### [Various JSON examples](TemplaterServer/Readme.md)
5455

‎Advanced/Streaming/Readme.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
## Streaming of large documents
2+
3+
Streaming in Templater is supported out of the box if streaming type is used (ResultSet/Iterator/Enumerator).
4+
Alternatively streaming can be simulated manually by multiple calls to process API.
5+
6+
Both methods allows Templater to flush the content of populated stream and reuse memory in next call to process API.
7+
8+
Streaming can be done only up to row without tags. This means that first non-streaming tags should be processed (if there are any)
9+
and then streaming tags can be processed which will perform flushing.

‎Advanced/CsvStreaming/CsvStreaming.csproj‎ renamed to ‎Advanced/Streaming/Streaming.csproj‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88
<ProjectGuid>{5BB2AABB-A28F-404F-8C37-DBE122E893F5}</ProjectGuid>
99
<OutputType>Exe</OutputType>
1010
<AppDesignerFolder>Properties</AppDesignerFolder>
11-
<RootNamespace>CsvStreaming</RootNamespace>
12-
<AssemblyName>CsvStreaming</AssemblyName>
11+
<RootNamespace>Streaming</RootNamespace>
12+
<AssemblyName>Streaming</AssemblyName>
1313
<TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
1414
<TargetFrameworkProfile>Client</TargetFrameworkProfile>
1515
<FileAlignment>512</FileAlignment>
@@ -39,7 +39,7 @@
3939
<HintPath>..\..\packages\DotNetZip.1.13.0\lib\net40\DotNetZip.dll</HintPath>
4040
<Private>True</Private>
4141
</Reference>
42-
<Reference Include="NGS.Templater">
42+
<Reference Include="NGS.Templater">
4343
<HintPath>..\..\packages\Templater.7.0.0\lib\Net40\NGS.Templater.dll</HintPath>
4444
<SpecificVersion>False</SpecificVersion>
4545
</Reference>
@@ -65,6 +65,11 @@
6565
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
6666
</None>
6767
</ItemGroup>
68+
<ItemGroup>
69+
<None Include="template\input.xml">
70+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
71+
</None>
72+
</ItemGroup>
6873
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
6974
<Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
7075
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">

‎Advanced/CsvStreaming/packages.config‎ renamed to ‎Advanced/Streaming/packages.config‎

File renamed without changes.

‎Advanced/CsvStreaming/pom.xml‎ renamed to ‎Advanced/Streaming/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@
22
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>hr.ngs.templater.example</groupId>
5-
<artifactId>csv-streaming-example</artifactId>
5+
<artifactId>streaming-example</artifactId>
66
<packaging>jar</packaging>
77
<version>7.0.0</version>
8-
<name>CSV streaming</name>
8+
<name>Streaming</name>
99
<url>https://github.com/ngs-doo/TemplaterExamples</url>
1010

1111
<properties>

‎Advanced/CsvStreaming/result.csv‎ renamed to ‎Advanced/Streaming/result.csv‎

File renamed without changes.

‎Advanced/Streaming/result.xml‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
<?xml version="1.0" encoding="utf-8"?><root>
2+
<filter date="All" user="All"/>
3+
<items>
4+
<data id="1000000" status="">
5+
<amount>260</amount>
6+
<date>27. 07. 2019.</date>
7+
<created on="2019-07-27 13:22:55.117"/>
8+
<reference>reference0</reference>
9+
<branch>branch0</branch>
10+
<verified by="suzane" on="2019-07-27 13:22:55.117"/>
11+
12+
</data><data id="1000001" status="APPROVED">
13+
<amount>260</amount>
14+
<date>27. 07. 2019.</date>
15+
<created by="" on="2019-07-27 13:22:55.117"/>
16+
<reference>reference1</reference>
17+
<branch>branch1</branch>
18+
<verified by="eric" on="2019-07-27 13:22:55.117"/>
19+
20+
</data><data id="1000002" status="">
21+
<amount>260</amount>
22+
<date>27. 07. 2019.</date>
23+
<created by="rick" on="2019-07-27 13:22:55.117"/>
24+
<reference>reference2</reference>
25+
<branch>branch2</branch>
26+
<verified by="mick" on="2019-07-27 13:22:55.117"/>
27+
-
28+
</data><data id="1000003" status="APPROVED">
29+
<amount>260</amount>
30+
<date>27. 07. 2019.</date>
31+
<created by="marty" on="2019-07-27 13:22:55.117"/>
32+
<reference>reference3</reference>
33+
<branch>branch3</branch>
34+
<verified by="admin" on="2019-07-27 13:22:55.117"/>
35+
...
36+
</data><data id="1000004" status="">
37+
<amount>261</amount>
38+
<date>27. 07. 2019.</date>
39+
<created by="suzane" on="2019-07-27 13:22:55.117"/>
40+
<reference>reference4</reference>
41+
<branch>branch4</branch>
42+
<verified on="2019-07-27 13:22:55.117"/>
43+
IMPORTANT
44+
</data><data id="1000005" status="APPROVED">
45+
<amount>261</amount>
46+
<date>27. 07. 2019.</date>
47+
<created by="eric" on="2019-07-27 13:22:55.117"/>
48+
<reference>reference5</reference>
49+
<branch>branch5</branch>
50+
<verified by="" on="2019-07-27 13:22:55.117"/>
51+
REMINDER
52+
</data><data id="1000006" status="VERIFIED">
53+
<amount>261</amount>
54+
<date>27. 07. 2019.</date>
55+
<created by="mick" on="2019-07-27 13:22:55.117"/>
56+
<reference>reference6</reference>
57+
<branch>branch6</branch>
58+
<verified by="rick" on="2019-07-27 13:22:55.117"/>
59+
something to look "into later
60+
</data><data id="1000007" status="CANCELED">
61+
<amount>261</amount>
62+
<date>27. 07. 2019.</date>
63+
<created by="admin" on="2019-07-27 13:22:55.117"/>
64+
<reference>reference7</reference>
65+
<branch>branch7</branch>
66+
<verified by="marty" on="2019-07-27 13:22:55.117"/>
67+
special" char,
68+
</data><data id="1000008" status="">
69+
<amount>261</amount>
70+
<date>27. 07. 2019.</date>
71+
<created on="2019-07-27 13:22:55.117"/>
72+
<reference>reference8</reference>
73+
<branch>branch8</branch>
74+
<verified by="suzane" on="2019-07-27 13:22:55.117"/>
75+
76+
</data><data id="1000009" status="APPROVED">
77+
<amount>261</amount>
78+
<date>27. 07. 2019.</date>
79+
<created by="" on="2019-07-27 13:22:55.117"/>
80+
<reference>reference9</reference>
81+
<branch>branch9</branch>
82+
<verified by="eric" on="2019-07-27 13:22:55.117"/>
83+
84+
</data><data id="1000010" status="">
85+
<amount>261</amount>
86+
<date>27. 07. 2019.</date>
87+
<created by="rick" on="2019-07-27 13:22:55.117"/>
88+
<reference>reference10</reference>
89+
<branch>branch10</branch>
90+
<verified by="mick" on="2019-07-27 13:22:55.117"/>
91+
-
92+
</data><data id="1000011" status="APPROVED">
93+
<amount>261</amount>
94+
<date>27. 07. 2019.</date>
95+
<created by="marty" on="2019-07-27 13:22:55.117"/>
96+
<reference>reference11</reference>
97+
<branch>branch11</branch>
98+
<verified by="admin" on="2019-07-27 13:22:55.117"/>
99+
...
100+
</data><data id="1000012" status="">
101+
<amount>262</amount>
102+
<date>27. 07. 2019.</date>
103+
<created by="suzane" on="2019-07-27 13:22:55.117"/>
104+
<reference>reference12</reference>
105+
<branch>branch12</branch>
106+
<verified on="2019-07-27 13:22:55.117"/>
107+
IMPORTANT
108+
</data><data id="1000013" status="APPROVED">
109+
<amount>262</amount>
110+
<date>27. 07. 2019.</date>
111+
<created by="eric" on="2019-07-27 13:22:55.117"/>
112+
<reference>reference13</reference>
113+
<branch>branch13</branch>
114+
<verified by="" on="2019-07-27 13:22:55.117"/>
115+
REMINDER
116+
</data><data id="1000014" status="VERIFIED">
117+
<amount>262</amount>
118+
<date>27. 07. 2019.</date>
119+
<created by="mick" on="2019-07-27 13:22:55.117"/>
120+
<reference>reference14</reference>
121+
<branch>branch14</branch>
122+
<verified by="rick" on="2019-07-27 13:22:55.117"/>
123+
something to look "into later
124+
</data><data id="1000015" status="CANCELED">
125+
<amount>262</amount>
126+
<date>27. 07. 2019.</date>
127+
<created by="admin" on="2019-07-27 13:22:55.117"/>
128+
<reference>reference15</reference>
129+
<branch>branch15</branch>
130+
<verified by="marty" on="2019-07-27 13:22:55.117"/>
131+
special" char,
132+
</data><data id="1000016" status="">
133+
<amount>262</amount>
134+
<date>27. 07. 2019.</date>
135+
<created on="2019-07-27 13:22:55.117"/>
136+
<reference>reference16</reference>
137+
<branch>branch16</branch>
138+
<verified by="suzane" on="2019-07-27 13:22:55.117"/>
139+
140+
</data><data id="1000017" status="APPROVED">
141+
<amount>262</amount>
142+
<date>27. 07. 2019.</date>
143+
<created by="" on="2019-07-27 13:22:55.117"/>
144+
<reference>reference17</reference>
145+
<branch>branch17</branch>
146+
<verified by="eric" on="2019-07-27 13:22:55.117"/>
147+
148+
</data><data id="1000018" status="">
149+
<amount>262</amount>
150+
<date>27. 07. 2019.</date>
151+
<created by="rick" on="2019-07-27 13:22:55.117"/>
152+
<reference>reference18</reference>
153+
<branch>branch18</branch>
154+
<verified by="mick" on="2019-07-27 13:22:55.117"/>
155+
-
156+
</data><data id="1000019" status="APPROVED">
157+
<amount>262</amount>
158+
<date>27. 07. 2019.</date>
159+
<created by="marty" on="2019-07-27 13:22:55.117"/>
160+
<reference>reference19</reference>
161+
<branch>branch19</branch>
162+
<verified by="admin" on="2019-07-27 13:22:55.117"/>
163+
...
164+
</data>
165+
</items></root>

‎Advanced/CsvStreaming/src/Program.cs‎ renamed to ‎Advanced/Streaming/src/Program.cs‎

Lines changed: 80 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Collections;
23
using System.Collections.Generic;
34
using System.Data;
45
using System.Diagnostics;
@@ -8,7 +9,7 @@
89
using Ionic.Zip;
910
using NGS.Templater;
1011

11-
namespace CsvStreaming
12+
namespace Streaming
1213
{
1314
public class Program
1415
{
@@ -45,7 +46,7 @@ struct StreamingRow
4546
public string verifiedBy;
4647
public DateTime verifiedOn;
4748

48-
public StreamingRow(DataTableReader reader)
49+
public StreamingRow(IDataReader reader)
4950
{
5051
id = reader.GetInt32(0);
5152
amount = reader.GetDecimal(1);
@@ -59,6 +60,22 @@ public StreamingRow(DataTableReader reader)
5960
verifiedBy = reader.IsDBNull(9) ? null : reader.GetString(9);
6061
verifiedOn = reader.GetDateTime(10);
6162
}
63+
64+
public class ReaderIterator : IEnumerator<StreamingRow>
65+
{
66+
private readonly IDataReader Reader;
67+
68+
public ReaderIterator(IDataReader reader)
69+
{
70+
this.Reader = reader;
71+
}
72+
73+
public StreamingRow Current { get { return new StreamingRow(Reader); } }
74+
object IEnumerator.Current { get { return Current; } }
75+
public bool MoveNext() { return Reader.Read(); }
76+
public void Reset() { }
77+
public void Dispose() { }
78+
}
6279
}
6380

6481
public static void Main(string[] args)
@@ -97,42 +114,77 @@ public static void Main(string[] args)
97114
startTimestamp.AddMinutes(i)
98115
);
99116
}
100-
var reader = table.CreateDataReader();
101-
var config = Configuration.Builder.Include(Quoter);
117+
var reader1 = table.CreateDataReader();
118+
var reader2 = table.CreateDataReader();
119+
var reader3 = table.CreateDataReader();
120+
var csvConfig = Configuration.Builder.Include(Quoter); //we need quoting as we are simulating CSV
121+
var xmlConfig = Configuration.Builder; //we don't need quoting as XML is natively supported
102122
//if we are using a culture which has comma as decimal separator, change the output to dot
103123
//we could apply this always, but it adds a bit of overhead, so let's apply it conditionally
104124
if (Thread.CurrentThread.CurrentCulture.NumberFormat.NumberDecimalSeparator.Contains(","))
105-
config.Include(NumberAsDot);
125+
{
126+
csvConfig.Include(NumberAsDot);
127+
xmlConfig.Include(NumberAsDot);
128+
}
129+
csvConfig.Streaming(50000);//by default streaming is 16k, lets leave the default for xml
130+
var csvFactory = csvConfig.Build();
131+
var xmlFactory = xmlConfig.Build();
106132
//for example purposes we will stream it a zip file
107133
using (var zip = new ZipOutputStream("output.zip"))
108134
{
109-
zip.PutNextEntry("output.csv");
110-
using (var doc = config.Build().Open(File.OpenRead("template/input.csv"), "csv", zip))
135+
zip.PutNextEntry("manual.csv");
136+
var sw = Stopwatch.StartNew();
137+
ManualStreaming(reader1, csvFactory, zip);
138+
Console.WriteLine("manual csv took: " + sw.ElapsedMilliseconds);
139+
zip.PutNextEntry("automatic.csv");
140+
sw = Stopwatch.StartNew();
141+
AutomaticStreaming(reader2, csvFactory, "csv", zip);
142+
Console.WriteLine("automatic csv took: " + sw.ElapsedMilliseconds);
143+
zip.PutNextEntry("data.xml");
144+
sw = Stopwatch.StartNew();
145+
AutomaticStreaming(reader3, xmlFactory, "xml", zip);
146+
Console.WriteLine("automatic xml took: " + sw.ElapsedMilliseconds);
147+
}
148+
Process.Start(new ProcessStartInfo("output.zip") { UseShellExecute = true });
149+
}
150+
151+
private static void ManualStreaming(IDataReader reader, IDocumentFactory factory, ZipOutputStream zip)
152+
{
153+
using (var doc = factory.Open(File.OpenRead("template/input.csv"), "csv", zip))
154+
{
155+
//streaming processing assumes we have only a single collection, which means we first need to process all other tags
156+
doc.Process(new { filter = new { date = "All", user = "All" } });
157+
//to do a streaming processing we need to process collection in chunks
158+
var chunk = new List<StreamingRow>(50000);
159+
var hasData = reader.Read();
160+
while (hasData)
111161
{
112-
//streaming processing assumes we have only a single collection, which means we first need to process all other tags
113-
doc.Process(new { filter = new { date = "All", user = "All" } });
114-
//to do a streaming processing we need to process collection in chunks
115-
var chunk = new List<StreamingRow>(50000);
116-
var hasData = reader.Read();
117-
while (hasData)
162+
//one way of doing streaming is first duplicating the template row (context)
163+
doc.Templater.Resize(doc.Templater.Tags, 2);
164+
//and then process that row with all known data
165+
//this way we will have additional row to process (or remove) later
166+
do
118167
{
119-
//one way of doing streaming is first duplicating the template row (context)
120-
doc.Templater.Resize(doc.Templater.Tags, 2);
121-
//and then process that row with all known data
122-
//this way we will have additional row to process (or remove) later
123-
do
124-
{
125-
chunk.Add(new StreamingRow(reader));
126-
hasData = reader.Read();
127-
} while (chunk.Count < 50000 && hasData);
128-
doc.Process(new { data = chunk });
129-
chunk.Clear();
130-
}
131-
//remove remaining rows
132-
doc.Templater.Resize(doc.Templater.Tags, 0);
168+
chunk.Add(new StreamingRow(reader));
169+
hasData = reader.Read();
170+
} while (chunk.Count < 50000 && hasData);
171+
doc.Process(new { data = chunk });
172+
chunk.Clear();
133173
}
174+
//remove remaining rows
175+
doc.Templater.Resize(doc.Templater.Tags, 0);
176+
}
177+
}
178+
179+
private static void AutomaticStreaming(IDataReader reader, IDocumentFactory factory, string extension, ZipOutputStream zip)
180+
{
181+
using (var doc = factory.Open(File.OpenRead("template/input." + extension), extension, zip))
182+
{
183+
//we still want to make sure all non collection tags are processed first (or they are at the end of document)
184+
doc.Process(new { filter = new { date = "All", user = "All" } });
185+
//for streaming lets just pass enumerator for processing
186+
doc.Process(new { data = new StreamingRow.ReaderIterator(reader) });
134187
}
135-
Process.Start(new ProcessStartInfo("output.zip") { UseShellExecute = true });
136188
}
137189
}
138190
}

0 commit comments

Comments
 (0)