Overview of cleanup - suggested indexes and export tool

Altinn.Correspondence.sln

@@ -21,53 +21,138 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Altinn.Correspondence.Commo
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Altinn.Correspondence.LoadTests.DatabasePopulater", "Test\Altinn.Correspondence.LoadTests.DatabasePopulater\Altinn.Correspondence.LoadTests.DatabasePopulater.csproj", "{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{07C2787E-EAC7-C090-1BA3-A61EC2A24D84}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Altinn.Correspondence.DialogActivityExporter", "tools\Altinn.Correspondence.DialogActivityExporter\Altinn.Correspondence.DialogActivityExporter.csproj", "{2228175C-EBE9-4085-B083-F78BCD021BAE}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
 		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{28B9820A-981B-4AED-BA75-B43D9BEFCFF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{28B9820A-981B-4AED-BA75-B43D9BEFCFF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{28B9820A-981B-4AED-BA75-B43D9BEFCFF9}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{28B9820A-981B-4AED-BA75-B43D9BEFCFF9}.Debug|x64.Build.0 = Debug|Any CPU
+		{28B9820A-981B-4AED-BA75-B43D9BEFCFF9}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{28B9820A-981B-4AED-BA75-B43D9BEFCFF9}.Debug|x86.Build.0 = Debug|Any CPU
 		{28B9820A-981B-4AED-BA75-B43D9BEFCFF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{28B9820A-981B-4AED-BA75-B43D9BEFCFF9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{28B9820A-981B-4AED-BA75-B43D9BEFCFF9}.Release|x64.ActiveCfg = Release|Any CPU
+		{28B9820A-981B-4AED-BA75-B43D9BEFCFF9}.Release|x64.Build.0 = Release|Any CPU
+		{28B9820A-981B-4AED-BA75-B43D9BEFCFF9}.Release|x86.ActiveCfg = Release|Any CPU
+		{28B9820A-981B-4AED-BA75-B43D9BEFCFF9}.Release|x86.Build.0 = Release|Any CPU
 		{F7FC66C9-8BA5-4E3F-913B-0624CDE9C65A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{F7FC66C9-8BA5-4E3F-913B-0624CDE9C65A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F7FC66C9-8BA5-4E3F-913B-0624CDE9C65A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F7FC66C9-8BA5-4E3F-913B-0624CDE9C65A}.Debug|x64.Build.0 = Debug|Any CPU
+		{F7FC66C9-8BA5-4E3F-913B-0624CDE9C65A}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F7FC66C9-8BA5-4E3F-913B-0624CDE9C65A}.Debug|x86.Build.0 = Debug|Any CPU
 		{F7FC66C9-8BA5-4E3F-913B-0624CDE9C65A}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{F7FC66C9-8BA5-4E3F-913B-0624CDE9C65A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F7FC66C9-8BA5-4E3F-913B-0624CDE9C65A}.Release|x64.ActiveCfg = Release|Any CPU
+		{F7FC66C9-8BA5-4E3F-913B-0624CDE9C65A}.Release|x64.Build.0 = Release|Any CPU
+		{F7FC66C9-8BA5-4E3F-913B-0624CDE9C65A}.Release|x86.ActiveCfg = Release|Any CPU
+		{F7FC66C9-8BA5-4E3F-913B-0624CDE9C65A}.Release|x86.Build.0 = Release|Any CPU
 		{D139FEEB-DAC3-4874-9608-CF8801F1C0C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{D139FEEB-DAC3-4874-9608-CF8801F1C0C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D139FEEB-DAC3-4874-9608-CF8801F1C0C2}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{D139FEEB-DAC3-4874-9608-CF8801F1C0C2}.Debug|x64.Build.0 = Debug|Any CPU
+		{D139FEEB-DAC3-4874-9608-CF8801F1C0C2}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{D139FEEB-DAC3-4874-9608-CF8801F1C0C2}.Debug|x86.Build.0 = Debug|Any CPU
 		{D139FEEB-DAC3-4874-9608-CF8801F1C0C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D139FEEB-DAC3-4874-9608-CF8801F1C0C2}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D139FEEB-DAC3-4874-9608-CF8801F1C0C2}.Release|x64.ActiveCfg = Release|Any CPU
+		{D139FEEB-DAC3-4874-9608-CF8801F1C0C2}.Release|x64.Build.0 = Release|Any CPU
+		{D139FEEB-DAC3-4874-9608-CF8801F1C0C2}.Release|x86.ActiveCfg = Release|Any CPU
+		{D139FEEB-DAC3-4874-9608-CF8801F1C0C2}.Release|x86.Build.0 = Release|Any CPU
 		{820A2274-CE87-40D5-B173-CCBC3E805D9C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{820A2274-CE87-40D5-B173-CCBC3E805D9C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{820A2274-CE87-40D5-B173-CCBC3E805D9C}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{820A2274-CE87-40D5-B173-CCBC3E805D9C}.Debug|x64.Build.0 = Debug|Any CPU
+		{820A2274-CE87-40D5-B173-CCBC3E805D9C}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{820A2274-CE87-40D5-B173-CCBC3E805D9C}.Debug|x86.Build.0 = Debug|Any CPU
 		{820A2274-CE87-40D5-B173-CCBC3E805D9C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{820A2274-CE87-40D5-B173-CCBC3E805D9C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{820A2274-CE87-40D5-B173-CCBC3E805D9C}.Release|x64.ActiveCfg = Release|Any CPU
+		{820A2274-CE87-40D5-B173-CCBC3E805D9C}.Release|x64.Build.0 = Release|Any CPU
+		{820A2274-CE87-40D5-B173-CCBC3E805D9C}.Release|x86.ActiveCfg = Release|Any CPU
+		{820A2274-CE87-40D5-B173-CCBC3E805D9C}.Release|x86.Build.0 = Release|Any CPU
 		{D78A36B1-D457-4FF7-B01E-82A0C3C64EDC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{D78A36B1-D457-4FF7-B01E-82A0C3C64EDC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D78A36B1-D457-4FF7-B01E-82A0C3C64EDC}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{D78A36B1-D457-4FF7-B01E-82A0C3C64EDC}.Debug|x64.Build.0 = Debug|Any CPU
+		{D78A36B1-D457-4FF7-B01E-82A0C3C64EDC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{D78A36B1-D457-4FF7-B01E-82A0C3C64EDC}.Debug|x86.Build.0 = Debug|Any CPU
 		{D78A36B1-D457-4FF7-B01E-82A0C3C64EDC}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D78A36B1-D457-4FF7-B01E-82A0C3C64EDC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D78A36B1-D457-4FF7-B01E-82A0C3C64EDC}.Release|x64.ActiveCfg = Release|Any CPU
+		{D78A36B1-D457-4FF7-B01E-82A0C3C64EDC}.Release|x64.Build.0 = Release|Any CPU
+		{D78A36B1-D457-4FF7-B01E-82A0C3C64EDC}.Release|x86.ActiveCfg = Release|Any CPU
+		{D78A36B1-D457-4FF7-B01E-82A0C3C64EDC}.Release|x86.Build.0 = Release|Any CPU
 		{5C88BAAD-CF82-4D7A-8FB6-879605EF4E03}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{5C88BAAD-CF82-4D7A-8FB6-879605EF4E03}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5C88BAAD-CF82-4D7A-8FB6-879605EF4E03}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{5C88BAAD-CF82-4D7A-8FB6-879605EF4E03}.Debug|x64.Build.0 = Debug|Any CPU
+		{5C88BAAD-CF82-4D7A-8FB6-879605EF4E03}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{5C88BAAD-CF82-4D7A-8FB6-879605EF4E03}.Debug|x86.Build.0 = Debug|Any CPU
 		{5C88BAAD-CF82-4D7A-8FB6-879605EF4E03}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{5C88BAAD-CF82-4D7A-8FB6-879605EF4E03}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5C88BAAD-CF82-4D7A-8FB6-879605EF4E03}.Release|x64.ActiveCfg = Release|Any CPU
+		{5C88BAAD-CF82-4D7A-8FB6-879605EF4E03}.Release|x64.Build.0 = Release|Any CPU
+		{5C88BAAD-CF82-4D7A-8FB6-879605EF4E03}.Release|x86.ActiveCfg = Release|Any CPU
+		{5C88BAAD-CF82-4D7A-8FB6-879605EF4E03}.Release|x86.Build.0 = Release|Any CPU
 		{AC3CF570-F9E1-4616-A56B-FF94C89CBC0B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{AC3CF570-F9E1-4616-A56B-FF94C89CBC0B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AC3CF570-F9E1-4616-A56B-FF94C89CBC0B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{AC3CF570-F9E1-4616-A56B-FF94C89CBC0B}.Debug|x64.Build.0 = Debug|Any CPU
+		{AC3CF570-F9E1-4616-A56B-FF94C89CBC0B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{AC3CF570-F9E1-4616-A56B-FF94C89CBC0B}.Debug|x86.Build.0 = Debug|Any CPU
 		{AC3CF570-F9E1-4616-A56B-FF94C89CBC0B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{AC3CF570-F9E1-4616-A56B-FF94C89CBC0B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AC3CF570-F9E1-4616-A56B-FF94C89CBC0B}.Release|x64.ActiveCfg = Release|Any CPU
+		{AC3CF570-F9E1-4616-A56B-FF94C89CBC0B}.Release|x64.Build.0 = Release|Any CPU
+		{AC3CF570-F9E1-4616-A56B-FF94C89CBC0B}.Release|x86.ActiveCfg = Release|Any CPU
+		{AC3CF570-F9E1-4616-A56B-FF94C89CBC0B}.Release|x86.Build.0 = Release|Any CPU
 		{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD}.Debug|x64.Build.0 = Debug|Any CPU
+		{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD}.Debug|x86.Build.0 = Debug|Any CPU
 		{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD}.Release|x64.ActiveCfg = Release|Any CPU
+		{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD}.Release|x64.Build.0 = Release|Any CPU
+		{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD}.Release|x86.ActiveCfg = Release|Any CPU
+		{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD}.Release|x86.Build.0 = Release|Any CPU
+		{2228175C-EBE9-4085-B083-F78BCD021BAE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2228175C-EBE9-4085-B083-F78BCD021BAE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2228175C-EBE9-4085-B083-F78BCD021BAE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2228175C-EBE9-4085-B083-F78BCD021BAE}.Debug|x64.Build.0 = Debug|Any CPU
+		{2228175C-EBE9-4085-B083-F78BCD021BAE}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2228175C-EBE9-4085-B083-F78BCD021BAE}.Debug|x86.Build.0 = Debug|Any CPU
+		{2228175C-EBE9-4085-B083-F78BCD021BAE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2228175C-EBE9-4085-B083-F78BCD021BAE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2228175C-EBE9-4085-B083-F78BCD021BAE}.Release|x64.ActiveCfg = Release|Any CPU
+		{2228175C-EBE9-4085-B083-F78BCD021BAE}.Release|x64.Build.0 = Release|Any CPU
+		{2228175C-EBE9-4085-B083-F78BCD021BAE}.Release|x86.ActiveCfg = Release|Any CPU
+		{2228175C-EBE9-4085-B083-F78BCD021BAE}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{D78A36B1-D457-4FF7-B01E-82A0C3C64EDC} = {DC791FEF-108F-4773-8B84-56986CB90767}
 		{3E5EEBF3-366E-48F8-9D57-1975F95D4FAD} = {DC791FEF-108F-4773-8B84-56986CB90767}
+		{2228175C-EBE9-4085-B083-F78BCD021BAE} = {07C2787E-EAC7-C090-1BA3-A61EC2A24D84}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {3508DF12-D3DE-4956-ABAB-31D7321B72C7}
 	EndGlobalSection
-EndGlobal
+EndGlobal

docs/DialogActivityExport_CombinedMode.md

@@ -0,0 +1,178 @@
+# Combined Export Feature
+## Export Both Issues to Single CSV File
+
+## Overview
+
+The DialogActivityExporter now supports exporting **both Issue #1716 and #1951** to a single CSV file using `--issue all`.
+
+## Why Combine?
+
+✅ **Single unified dataset** - All dialog activities in one file  
+✅ **Simpler processing** - No need to merge CSVs manually  
+✅ **Better progress tracking** - See combined progress across both issues  
+✅ **Consistent ordering** - Results ordered by CorrespondenceId across both issues  
+
+## Usage
+
+### Export Both Issues
+
+```powershell
+dotnet run --project tools/Altinn.Correspondence.DialogActivityExporter -- `
+  --issue all `
+  --output "C:\temp\all_dialog_activities.csv" `
+  --connection "Host=prod-db;Database=correspondence;..." `
+  --cutoff "2026-05-19 11:35:59" `
+  --oldest "2019-03-23"
+```
+
+### How It Works
+
+1. **Phase 1**: Export Issue #1716 (Synced events)
+   - Processes ~7-9M records first (faster)
+   - Uses `SyncedFromAltinn2 IS NOT NULL` filter
+
+2. **Phase 2**: Export Issue #1951 (Migrated events)
+   - Processes ~150M records second
+   - Uses `SyncedFromAltinn2 IS NULL` filter
+
+3. **Result**: Single CSV file with all records combined
+
+## Output
+
+The CSV will contain records from both issues:
+
+```csv
+DialogId,DialogActivityId,Timestamp,ActorId,ActorName,ActivityType
+"uuid1",...  -- From Issue #1716
+"uuid2",...  -- From Issue #1716
+...
+"uuid3",...  -- From Issue #1951
+"uuid4",...  -- From Issue #1951
+...
+```
+
+## Progress Reporting
+
+The progress bar shows combined progress:
+
+```text
+[████████████░░░░░░░░] 47.23%
+```
+
+- **TotalCount**: Combined count from both issues
+- **TotalProcessed**: Running total across both issues
+- **ETA**: Estimated time for complete export
+
+## Performance
+
+| Mode | Records | Estimated Time |
+|------|---------|----------------|
+| `--issue 1716` | 7-9M | 15-30 minutes |
+| `--issue 1951` | 150M | 30-60 minutes |
+| `--issue all` | 157-159M | **45-90 minutes** |
+
+**Note:** With proper indexes. Without indexes, export will take hours.
+
+## Alternative: Separate Files
+
+If you prefer separate files, export individually:
+
+```powershell
+# Export Issue #1716
+dotnet run -- --issue 1716 --output "C:\temp\issue1716.csv" ...
+
+# Export Issue #1951
+dotnet run -- --issue 1951 --output "C:\temp\issue1951.csv" ...
+
+# Later combine manually if needed
+# Keep first file's header, skip headers from subsequent files
+Get-Content C:\temp\issue1716.csv | Out-File C:\temp\combined.csv
+Get-Content C:\temp\issue1951.csv | Select-Object -Skip 1 | Out-File -Append C:\temp\combined.csv
+```
+
+## Implementation Details
+
+### Key Changes
+
+1. **Program.cs**
+   - Added `--issue all` option
+   - Added `ExportBoth` flag to `ExportOptions`
+   - Updated help text with new example
+
+2. **DialogActivityExportService.cs**
+   - Added `ExportBothToCSVAsync()` method
+   - Added `ExportIssueToWriter()` helper method
+   - Progress tracking spans both issues
+
+### Benefits of Implementation
+
+✅ **Sequential processing** - One issue completes before next starts  
+✅ **Single file handle** - No need to merge files  
+✅ **Consistent error handling** - Single try/catch for entire operation  
+✅ **Combined progress** - Real-time tracking across both exports  
+
+## When to Use Each Mode
+
+| Use Case | Recommended Mode |
+|----------|------------------|
+| **Production bulk export** | `--issue all` |
+| **Testing/verification** | Individual issues |
+| **Partial re-export** | Individual issues |
+| **Different timestamps per issue** | Individual issues |
+| **Maximum simplicity** | `--issue all` |
+
+## Example Workflows
+
+### Workflow 1: Single Combined Export
+```powershell
+# One command, one file, done
+dotnet run -- `
+  --issue all `
+  --output "C:\exports\dialog_activities_$(Get-Date -Format 'yyyyMMdd').csv" `
+  --connection $connStr `
+  --cutoff "2026-05-19 11:35:59" `
+  --oldest "2019-03-23" `
+  -y
+```
+
+### Workflow 2: Separate with Different Cutoffs
+```powershell
+# Issue #1716 with earlier cutoff
+dotnet run -- --issue 1716 --output "C:\temp\1716.csv" --cutoff "2026-02-15"
+
+# Issue #1951 with later cutoff
+dotnet run -- --issue 1951 --output "C:\temp\1951.csv" --cutoff "2026-05-19" --oldest "2019-03-23"
+
+# Manually combine if needed
+```
+
+## FAQ
+
+**Q: Will the combined file be properly sorted?**  
+A: Each issue is sorted by CorrespondenceId internally. The file will have #1716 records first, then #1951 records, each section sorted.
+
+**Q: Can I filter by issue after export?**  
+A: The CSV only contains ActivityType field. Issue #1716 and #1951 both produce the same ActivityType values ("CorrespondenceOpened" or "CorrespondenceConfirmed"). To distinguish between issues, you would need to track the row numbers (Issue #1716 is exported first).
+
+**Q: What if one issue fails?**  
+A: The export will fail completely. Use individual issue modes if you need fault tolerance.
+
+**Q: Does this use more memory?**  
+A: No, it uses the same streaming approach. Memory usage is constant regardless of total row count.
+
+**Q: Is it slower than separate exports?**  
+A: Slightly faster actually, since it only opens one database connection and one file handle.
+
+## Troubleshooting
+
+### Issue: Progress seems stuck
+- First issue (#1716) processes first - may appear slow initially
+- After ~15-30 min, progress will accelerate for #1951
+
+### Issue: Want to resume interrupted export
+- Not currently supported
+- Use separate issue exports if you need resume capability
+
+### Issue: Need different parameters per issue
+- Use individual exports with `--issue 1716` and `--issue 1951`
+- The `--oldest` parameter only applies to Issue #1951 anyway