-+
--
-\ No newline at end of file
-+
-Index: etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.html
-IDEA additional info:
-Subsystem: com.intellij.openapi.diff.impl.patch.BaseRevisionTextPatchEP
-<+>
\r\n\r\n
\r\n test name
\r\n \r\n \r\n
-Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
-<+>UTF-8
-===================================================================
-diff --git a/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.html b/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.html
---- a/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.html (revision 2f917e8acb8c246f18bf1ef823e5c171eac97bf8)
-+++ b/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.html (date 1755689993909)
-@@ -3,9 +3,7 @@
-
- test name
-
-
-Index: etl_frontend/src/app/features/dashboard/components/profile/profile.module.ts
-IDEA additional info:
-Subsystem: com.intellij.openapi.diff.impl.patch.BaseRevisionTextPatchEP
-<+>import { NgModule } from '@angular/core';\r\nimport { CommonModule } from '@angular/common';\r\nimport { RouterModule } from '@angular/router';\r\nimport { profileRoutes } from './profile.module.routing';\r\nimport { HeaderComponent } from '../../../../shared/components/header/header.component';\r\nimport { ProfileComponent } from './profile.component';\r\nimport { ButtonModule } from 'primeng/button';\r\n\r\n\r\n\r\n@NgModule({\r\n declarations: [ProfileComponent],\r\n imports: [\r\n CommonModule,\r\n HeaderComponent,\r\n RouterModule.forChild(profileRoutes),\r\n ButtonModule\r\n ]\r\n})\r\nexport class ProfileModule { }\r\n
-Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
-<+>UTF-8
-===================================================================
-diff --git a/etl_frontend/src/app/features/dashboard/components/profile/profile.module.ts b/etl_frontend/src/app/features/dashboard/components/profile/profile.module.ts
---- a/etl_frontend/src/app/features/dashboard/components/profile/profile.module.ts (revision 2f917e8acb8c246f18bf1ef823e5c171eac97bf8)
-+++ b/etl_frontend/src/app/features/dashboard/components/profile/profile.module.ts (date 1755691313346)
-@@ -5,8 +5,7 @@
- import { HeaderComponent } from '../../../../shared/components/header/header.component';
- import { ProfileComponent } from './profile.component';
- import { ButtonModule } from 'primeng/button';
--
--
-+import { TabsModule } from 'primeng/tabs'
-
- @NgModule({
- declarations: [ProfileComponent],
-@@ -14,7 +13,8 @@
- CommonModule,
- HeaderComponent,
- RouterModule.forChild(profileRoutes),
-- ButtonModule
-+ ButtonModule,
-+ TabsModule,
- ]
- })
- export class ProfileModule { }
-Index: etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.ts
-IDEA additional info:
-Subsystem: com.intellij.openapi.diff.impl.patch.BaseRevisionTextPatchEP
-<+>import { Component, ViewChild } from '@angular/core';\r\nimport { Button } from \"primeng/button\";\r\nimport { Popover } from 'primeng/popover';\r\nimport { SignOutDirective } from '../../../../shared/directives/sign-out/sign-out.directive';\r\nimport { RouterLink } from \"../../../../../../node_modules/@angular/router\";\r\n\r\n@Component({\r\n selector: 'app-profile-popover',\r\n imports: [\r\n Button,\r\n Popover,\r\n SignOutDirective,\r\n RouterLink\r\n ],\r\n templateUrl: './profile-popover.component.html',\r\n styleUrl: './profile-popover.component.scss'\r\n})\r\nexport class ProfilePopoverComponent {\r\n @ViewChild('op') op!: Popover;\r\n\r\n public readonly options = [\r\n { label: 'Account', link: '/dashboard/profile', icon: 'pi pi-user' },\r\n { label: 'Sign out', link: '', icon: 'pi pi-sign-out' },\r\n ];\r\n\r\n public toggle(event: Event) {\r\n this.op.toggle(event);\r\n }\r\n\r\n public selectOption(link: string) {\r\n this.op.hide();\r\n }\r\n}\r\n
-Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
-<+>UTF-8
-===================================================================
-diff --git a/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.ts b/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.ts
---- a/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.ts (revision 2f917e8acb8c246f18bf1ef823e5c171eac97bf8)
-+++ b/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.ts (date 1755691098798)
-@@ -1,8 +1,8 @@
--import { Component, ViewChild } from '@angular/core';
--import { Button } from "primeng/button";
--import { Popover } from 'primeng/popover';
--import { SignOutDirective } from '../../../../shared/directives/sign-out/sign-out.directive';
--import { RouterLink } from "../../../../../../node_modules/@angular/router";
-+import {Component, ViewChild} from '@angular/core';
-+import {Button} from "primeng/button";
-+import {Popover} from 'primeng/popover';
-+import {SignOutDirective} from '../../../../shared/directives/sign-out/sign-out.directive';
-+import {RouterModule} from "../../../../../../node_modules/@angular/router";
-
- @Component({
- selector: 'app-profile-popover',
-@@ -10,7 +10,7 @@
- Button,
- Popover,
- SignOutDirective,
-- RouterLink
-+ RouterModule
- ],
- templateUrl: './profile-popover.component.html',
- styleUrl: './profile-popover.component.scss'
-@@ -19,8 +19,8 @@
- @ViewChild('op') op!: Popover;
-
- public readonly options = [
-- { label: 'Account', link: '/dashboard/profile', icon: 'pi pi-user' },
-- { label: 'Sign out', link: '', icon: 'pi pi-sign-out' },
-+ {label: 'Account', link: '/dashboard/profile', icon: 'pi pi-user'},
-+ {label: 'Sign out', link: '', icon: 'pi pi-sign-out'},
- ];
-
- public toggle(event: Event) {
-Index: .idea/workspace.xml
-IDEA additional info:
-Subsystem: com.intellij.openapi.diff.impl.patch.BaseRevisionTextPatchEP
-<+>\r\n
\r\n \r\n \r\n \r\n \r\n
\r\n \r\n \r\n \r\n \r\n
\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n {\r\n "lastFilter": {\r\n "state": "OPEN",\r\n "assignee": "MohammadAminDadkhah"\r\n }\r\n} \r\n {\r\n "selectedUrlAndAccountId": {\r\n "url": "git@github.com:Star-Academy/Summer1404-Project-Team03.git",\r\n "accountId": "fc338c9a-1671-4db6-9bac-ecf9ac67d7e8"\r\n },\r\n "recentNewPullRequestHead": {\r\n "server": {\r\n "useHttp": false,\r\n "host": "github.com",\r\n "port": null,\r\n "suffix": null\r\n },\r\n "owner": "Star-Academy",\r\n "repository": "Summer1404-Project-Team03"\r\n }\r\n} \r\n \r\n \r\n {\r\n "customColor": "",\r\n "associatedIndex": 2\r\n} \r\n \r\n \r\n \r\n \r\n {\r\n "keyToString": {\r\n "DefaultHtmlFileTemplate": "HTML File",\r\n "RunOnceActivity.ShowReadmeOnStart": "true",\r\n "RunOnceActivity.git.unshallow": "true",\r\n "git-widget-placeholder": "frontend/profile",\r\n "last_opened_file_path": "C:/Users/Mohap/Desktop/ETL-Project/Summer1404-Project-Team03/etl_frontend/public/landing",\r\n "node.js.detected.package.eslint": "true",\r\n "node.js.detected.package.tslint": "true",\r\n "node.js.selected.package.eslint": "(autodetect)",\r\n "node.js.selected.package.tslint": "(autodetect)",\r\n "nodejs_package_manager_path": "npm",\r\n "ts.external.directory.path": "C:\\\\Program Files\\\\JetBrains\\\\WebStorm 2024.3.5\\\\plugins\\\\javascript-plugin\\\\jsLanguageServicesImpl\\\\external",\r\n "vue.rearranger.settings.migration": "true"\r\n }\r\n} \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n 1755439563544 \r\n 1755439563544 \r\n \r\n \r\n 1755506055288 \r\n 1755506055288 \r\n \r\n \r\n 1755520009383 \r\n 1755520009383 \r\n \r\n \r\n 1755520161593 \r\n 1755520161593 \r\n \r\n \r\n 1755520273732 \r\n 1755520273732 \r\n \r\n \r\n 1755524106455 \r\n 1755524106455 \r\n \r\n \r\n 1755525743246 \r\n 1755525743246 \r\n \r\n \r\n 1755525887515 \r\n 1755525887515 \r\n \r\n \r\n 1755590667118 \r\n 1755590667118 \r\n \r\n \r\n 1755592428818 \r\n 1755592428818 \r\n \r\n \r\n 1755681370145 \r\n 1755681370145 \r\n \r\n \r\n 1755684869155 \r\n 1755684869155 \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n
-Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
-<+>UTF-8
-===================================================================
-diff --git a/.idea/workspace.xml b/.idea/workspace.xml
---- a/.idea/workspace.xml (revision 2f917e8acb8c246f18bf1ef823e5c171eac97bf8)
-+++ b/.idea/workspace.xml (date 1755692010280)
-@@ -4,10 +4,12 @@
-
-
-
--
-+
-
-
- 1755684869155
-
--
-+ 1755685917519
-+ 1755685917519
-+
-+
-
-@@ -239,6 +249,7 @@
-
-
-\ No newline at end of file
diff --git a/.idea/shelf/Uncommitted_changes_before_Update_at_8_20_2025_3_44_PM__Changes_.xml b/.idea/shelf/Uncommitted_changes_before_Update_at_8_20_2025_3_44_PM__Changes_.xml
deleted file mode 100644
index 5f23c9e7..00000000
--- a/.idea/shelf/Uncommitted_changes_before_Update_at_8_20_2025_3_44_PM__Changes_.xml
+++ /dev/null
@@ -1,4 +0,0 @@
-
-
\ No newline at end of file
diff --git a/.idea/shelf/Uncommitted_changes_before_Update_at_8_20_2025_4_14_PM_[Changes]/shelved.patch b/.idea/shelf/Uncommitted_changes_before_Update_at_8_20_2025_4_14_PM_[Changes]/shelved.patch
deleted file mode 100644
index cfb5daa4..00000000
--- a/.idea/shelf/Uncommitted_changes_before_Update_at_8_20_2025_4_14_PM_[Changes]/shelved.patch
+++ /dev/null
@@ -1,120 +0,0 @@
-Index: etl_frontend/src/app/features/dashboard/components/profile/profile.component.html
-IDEA additional info:
-Subsystem: com.intellij.openapi.diff.impl.patch.BaseRevisionTextPatchEP
-<+>
\r\n \r\n
\r\n \r\n Your Profile \r\n Manage User \r\n \r\n \r\n
\r\n
-Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
-<+>UTF-8
-===================================================================
-diff --git a/etl_frontend/src/app/features/dashboard/components/profile/profile.component.html b/etl_frontend/src/app/features/dashboard/components/profile/profile.component.html
---- a/etl_frontend/src/app/features/dashboard/components/profile/profile.component.html (revision d18da9558b89bec6f0590dfdf2ae1acd13ae54a2)
-+++ b/etl_frontend/src/app/features/dashboard/components/profile/profile.component.html (date 1755692367592)
-@@ -1,6 +1,6 @@
-
-
--
-+
-
- Your Profile
- Manage User
-Index: etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.ts
-IDEA additional info:
-Subsystem: com.intellij.openapi.diff.impl.patch.BaseRevisionTextPatchEP
-<+>import { Component, ViewChild } from '@angular/core';\r\nimport { Button } from \"primeng/button\";\r\nimport { Popover } from 'primeng/popover';\r\nimport { SignOutDirective } from '../../../../shared/directives/sign-out/sign-out.directive';\r\nimport { RouterLink, RouterModule } from \"../../../../../../node_modules/@angular/router\";\r\n\r\n@Component({\r\n selector: 'app-profile-popover',\r\n imports: [\r\n Button,\r\n Popover,\r\n SignOutDirective,\r\n RouterModule\r\n ],\r\n templateUrl: './profile-popover.component.html',\r\n styleUrl: './profile-popover.component.scss'\r\n})\r\nexport class ProfilePopoverComponent {\r\n @ViewChild('op') op!: Popover;\r\n\r\n public readonly options = [\r\n { label: 'Account', link: '/dashboard/profile', icon: 'pi pi-user' },\r\n { label: 'Sign out', link: '', icon: 'pi pi-sign-out' },\r\n ];\r\n\r\n public toggle(event: Event) {\r\n this.op.toggle(event);\r\n }\r\n\r\n public selectOption(link: string) {\r\n this.op.hide();\r\n }\r\n}\r\n
-Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
-<+>UTF-8
-===================================================================
-diff --git a/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.ts b/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.ts
---- a/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.ts (revision d18da9558b89bec6f0590dfdf2ae1acd13ae54a2)
-+++ b/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.ts (date 1755692148182)
-@@ -2,7 +2,7 @@
- import { Button } from "primeng/button";
- import { Popover } from 'primeng/popover';
- import { SignOutDirective } from '../../../../shared/directives/sign-out/sign-out.directive';
--import { RouterLink, RouterModule } from "../../../../../../node_modules/@angular/router";
-+import { RouterModule } from "../../../../../../node_modules/@angular/router";
-
- @Component({
- selector: 'app-profile-popover',
-Index: etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.html
-IDEA additional info:
-Subsystem: com.intellij.openapi.diff.impl.patch.BaseRevisionTextPatchEP
-<+>\r\n test name
\r\n \r\n \r\n
-Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
-<+>UTF-8
-===================================================================
-diff --git a/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.html b/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.html
---- a/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.html (revision d18da9558b89bec6f0590dfdf2ae1acd13ae54a2)
-+++ b/etl_frontend/src/app/features/dashboard/components/profile-popover/profile-popover.component.html (date 1755692109060)
-@@ -3,9 +3,7 @@
-
- test name
-
-
-Index: .idea/workspace.xml
-IDEA additional info:
-Subsystem: com.intellij.openapi.diff.impl.patch.BaseRevisionTextPatchEP
-<+>\r\n\r\n \r\n \r\n \r\n \r\n
\r\n \r\n \r\n \r\n \r\n
\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n {\r\n "lastFilter": {\r\n "state": "OPEN",\r\n "assignee": "MohammadAminDadkhah"\r\n }\r\n} \r\n {\r\n "selectedUrlAndAccountId": {\r\n "url": "git@github.com:Star-Academy/Summer1404-Project-Team03.git",\r\n "accountId": "fc338c9a-1671-4db6-9bac-ecf9ac67d7e8"\r\n },\r\n "recentNewPullRequestHead": {\r\n "server": {\r\n "useHttp": false,\r\n "host": "github.com",\r\n "port": null,\r\n "suffix": null\r\n },\r\n "owner": "Star-Academy",\r\n "repository": "Summer1404-Project-Team03"\r\n }\r\n} \r\n \r\n \r\n {\r\n "customColor": "",\r\n "associatedIndex": 2\r\n} \r\n \r\n \r\n \r\n \r\n {\r\n "keyToString": {\r\n "DefaultHtmlFileTemplate": "HTML File",\r\n "RunOnceActivity.ShowReadmeOnStart": "true",\r\n "RunOnceActivity.git.unshallow": "true",\r\n "git-widget-placeholder": "frontend/profile",\r\n "last_opened_file_path": "C:/Users/Mohap/Desktop/ETL-Project/Summer1404-Project-Team03/etl_frontend/public/landing",\r\n "node.js.detected.package.eslint": "true",\r\n "node.js.detected.package.tslint": "true",\r\n "node.js.selected.package.eslint": "(autodetect)",\r\n "node.js.selected.package.tslint": "(autodetect)",\r\n "nodejs_package_manager_path": "npm",\r\n "ts.external.directory.path": "C:\\\\Program Files\\\\JetBrains\\\\WebStorm 2024.3.5\\\\plugins\\\\javascript-plugin\\\\jsLanguageServicesImpl\\\\external",\r\n "vue.rearranger.settings.migration": "true"\r\n }\r\n} \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n 1755439563544 \r\n 1755439563544 \r\n \r\n \r\n 1755506055288 \r\n 1755506055288 \r\n \r\n \r\n 1755520009383 \r\n 1755520009383 \r\n \r\n \r\n 1755520161593 \r\n 1755520161593 \r\n \r\n \r\n 1755520273732 \r\n 1755520273732 \r\n \r\n \r\n 1755524106455 \r\n 1755524106455 \r\n \r\n \r\n 1755525743246 \r\n 1755525743246 \r\n \r\n \r\n 1755525887515 \r\n 1755525887515 \r\n \r\n \r\n 1755590667118 \r\n 1755590667118 \r\n \r\n \r\n 1755592428818 \r\n 1755592428818 \r\n \r\n \r\n 1755681370145 \r\n 1755681370145 \r\n \r\n \r\n 1755684869155 \r\n 1755684869155 \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n
-Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
-<+>UTF-8
-===================================================================
-diff --git a/.idea/workspace.xml b/.idea/workspace.xml
---- a/.idea/workspace.xml (revision d18da9558b89bec6f0590dfdf2ae1acd13ae54a2)
-+++ b/.idea/workspace.xml (date 1755693696067)
-@@ -4,10 +4,11 @@
-
--
-+
-
-
- 1755684869155
-
--
-+ 1755685917519
-+ 1755685917519
-+
-+
-
-@@ -239,6 +248,7 @@
-
-
-\ No newline at end of file
diff --git a/.idea/shelf/Uncommitted_changes_before_Update_at_8_20_2025_4_14_PM__Changes_.xml b/.idea/shelf/Uncommitted_changes_before_Update_at_8_20_2025_4_14_PM__Changes_.xml
deleted file mode 100644
index 01915ef1..00000000
--- a/.idea/shelf/Uncommitted_changes_before_Update_at_8_20_2025_4_14_PM__Changes_.xml
+++ /dev/null
@@ -1,4 +0,0 @@
-
-
\ No newline at end of file
diff --git a/.idea/shelf/Uncommitted_changes_before_Update_at_8_23_2025_12_43_PM_[Changes]/shelved.patch b/.idea/shelf/Uncommitted_changes_before_Update_at_8_23_2025_12_43_PM_[Changes]/shelved.patch
deleted file mode 100644
index 52ff8c3c..00000000
--- a/.idea/shelf/Uncommitted_changes_before_Update_at_8_23_2025_12_43_PM_[Changes]/shelved.patch
+++ /dev/null
@@ -1,60 +0,0 @@
-Index: .idea/workspace.xml
-IDEA additional info:
-Subsystem: com.intellij.openapi.diff.impl.patch.BaseRevisionTextPatchEP
-<+>\r\n\r\n \r\n \r\n \r\n \r\n
\r\n \r\n \r\n \r\n \r\n
\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n {\r\n "lastFilter": {\r\n "state": "OPEN",\r\n "assignee": "MohammadAminDadkhah"\r\n }\r\n} \r\n {\r\n "selectedUrlAndAccountId": {\r\n "url": "git@github.com:Star-Academy/Summer1404-Project-Team03.git",\r\n "accountId": "fc338c9a-1671-4db6-9bac-ecf9ac67d7e8"\r\n },\r\n "recentNewPullRequestHead": {\r\n "server": {\r\n "useHttp": false,\r\n "host": "github.com",\r\n "port": null,\r\n "suffix": null\r\n },\r\n "owner": "Star-Academy",\r\n "repository": "Summer1404-Project-Team03"\r\n }\r\n} \r\n \r\n \r\n {\r\n "customColor": "",\r\n "associatedIndex": 2\r\n} \r\n \r\n \r\n \r\n \r\n {\r\n "keyToString": {\r\n "DefaultHtmlFileTemplate": "HTML File",\r\n "RunOnceActivity.ShowReadmeOnStart": "true",\r\n "RunOnceActivity.git.unshallow": "true",\r\n "git-widget-placeholder": "frontend/profile",\r\n "last_opened_file_path": "C:/Users/Mohap/Desktop/ETL-Project/Summer1404-Project-Team03/etl_frontend/src/app/features/dashboard/components/profile/modals/shared",\r\n "node.js.detected.package.eslint": "true",\r\n "node.js.detected.package.tslint": "true",\r\n "node.js.selected.package.eslint": "(autodetect)",\r\n "node.js.selected.package.tslint": "(autodetect)",\r\n "nodejs_package_manager_path": "npm",\r\n "settings.editor.selected.configurable": "preferences.pluginManager",\r\n "ts.external.directory.path": "C:\\\\Program Files\\\\JetBrains\\\\WebStorm 2024.3.5\\\\plugins\\\\javascript-plugin\\\\jsLanguageServicesImpl\\\\external",\r\n "vue.rearranger.settings.migration": "true"\r\n }\r\n} \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n 1755439563544 \r\n 1755439563544 \r\n \r\n \r\n 1755506055288 \r\n 1755506055288 \r\n \r\n \r\n 1755520009383 \r\n 1755520009383 \r\n \r\n \r\n 1755520161593 \r\n 1755520161593 \r\n \r\n \r\n 1755520273732 \r\n 1755520273732 \r\n \r\n \r\n 1755524106455 \r\n 1755524106455 \r\n \r\n \r\n 1755525743246 \r\n 1755525743246 \r\n \r\n \r\n 1755525887515 \r\n 1755525887515 \r\n \r\n \r\n 1755590667118 \r\n 1755590667118 \r\n \r\n \r\n 1755592428818 \r\n 1755592428818 \r\n \r\n \r\n 1755681370145 \r\n 1755681370145 \r\n \r\n \r\n 1755684869155 \r\n 1755684869155 \r\n \r\n \r\n 1755685917519 \r\n 1755685917519 \r\n \r\n \r\n 1755933599261 \r\n 1755933599261 \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n
-Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
-<+>UTF-8
-===================================================================
-diff --git a/.idea/workspace.xml b/.idea/workspace.xml
---- a/.idea/workspace.xml (revision 9be7c900e20d7c82c5b1ae1b8b6cdf21e8419b0f)
-+++ b/.idea/workspace.xml (date 1755940225843)
-@@ -4,12 +4,7 @@
-
--
--
-+
-
- 1755933599261
-
--
-+ 1755940190110
-+ 1755940190110
-+
-+
-
-@@ -264,6 +267,7 @@
-
-
-\ No newline at end of file
diff --git a/.idea/shelf/Uncommitted_changes_before_Update_at_8_23_2025_12_43_PM__Changes_.xml b/.idea/shelf/Uncommitted_changes_before_Update_at_8_23_2025_12_43_PM__Changes_.xml
deleted file mode 100644
index ee484b66..00000000
--- a/.idea/shelf/Uncommitted_changes_before_Update_at_8_23_2025_12_43_PM__Changes_.xml
+++ /dev/null
@@ -1,4 +0,0 @@
-
-
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
deleted file mode 100644
index 35eb1ddf..00000000
--- a/.idea/vcs.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
\ No newline at end of file
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
deleted file mode 100644
index f77af0a9..00000000
--- a/.idea/workspace.xml
+++ /dev/null
@@ -1,326 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- {
- "lastFilter": {
- "state": "OPEN",
- "assignee": "MohammadAminDadkhah"
- }
-}
- {
- "selectedUrlAndAccountId": {
- "url": "git@github.com:Star-Academy/Summer1404-Project-Team03.git",
- "accountId": "fc338c9a-1671-4db6-9bac-ecf9ac67d7e8"
- },
- "recentNewPullRequestHead": {
- "server": {
- "useHttp": false,
- "host": "github.com",
- "port": null,
- "suffix": null
- },
- "owner": "Star-Academy",
- "repository": "Summer1404-Project-Team03"
- }
-}
-
-
- {
- "customColor": "",
- "associatedIndex": 2
-}
-
-
-
-
- {
- "keyToString": {
- "DefaultHtmlFileTemplate": "HTML File",
- "RunOnceActivity.ShowReadmeOnStart": "true",
- "RunOnceActivity.git.unshallow": "true",
- "git-widget-placeholder": "#10 on frontend/profile",
- "last_opened_file_path": "C:/Users/Mohap/Desktop/ETL-Project/Summer1404-Project-Team03/etl_frontend/src/app/features/dashboard/components/profile/modals/shared",
- "node.js.detected.package.eslint": "true",
- "node.js.detected.package.tslint": "true",
- "node.js.selected.package.eslint": "(autodetect)",
- "node.js.selected.package.tslint": "(autodetect)",
- "nodejs_package_manager_path": "npm",
- "settings.editor.selected.configurable": "preferences.pluginManager",
- "ts.external.directory.path": "C:\\Program Files\\JetBrains\\WebStorm 2024.3.5\\plugins\\javascript-plugin\\jsLanguageServicesImpl\\external",
- "vue.rearranger.settings.migration": "true"
- }
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 1755439563544
- 1755439563544
-
-
- 1755506055288
- 1755506055288
-
-
- 1755520009383
- 1755520009383
-
-
- 1755520161593
- 1755520161593
-
-
- 1755520273732
- 1755520273732
-
-
- 1755524106455
- 1755524106455
-
-
- 1755525743246
- 1755525743246
-
-
- 1755525887515
- 1755525887515
-
-
- 1755590667118
- 1755590667118
-
-
- 1755592428818
- 1755592428818
-
-
- 1755681370145
- 1755681370145
-
-
- 1755684869155
- 1755684869155
-
-
- 1755685917519
- 1755685917519
-
-
- 1755933599261
- 1755933599261
-
-
- 1755940190110
- 1755940190110
-
-
- 1755941105342
- 1755941105342
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/etl_backend/.dockerignore b/etl_backend/.dockerignore
new file mode 100644
index 00000000..38bece4e
--- /dev/null
+++ b/etl_backend/.dockerignore
@@ -0,0 +1,25 @@
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/.idea
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md
\ No newline at end of file
diff --git a/etl_backend/Application/Application.csproj b/etl_backend/Application/Application.csproj
new file mode 100644
index 00000000..47198614
--- /dev/null
+++ b/etl_backend/Application/Application.csproj
@@ -0,0 +1,22 @@
+
+
+
+ net8.0
+ enable
+ enable
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/etl_backend/Application/Common/Authorization/AppPolicies.cs b/etl_backend/Application/Common/Authorization/AppPolicies.cs
new file mode 100644
index 00000000..3f361725
--- /dev/null
+++ b/etl_backend/Application/Common/Authorization/AppPolicies.cs
@@ -0,0 +1,11 @@
+namespace Application.Common.Authorization;
+
+public static class AppPolicies
+{
+ public const string RequireSysAdmin = nameof(RequireSysAdmin);
+ public const string RequireDataAdmin = nameof(RequireDataAdmin);
+ public const string RequireAnalyst = nameof(RequireAnalyst);
+
+ public const string RequireAtLeastAnalyst = nameof(RequireAtLeastAnalyst);
+ public const string RequireAtLeastDataAdmin = nameof(RequireAtLeastDataAdmin);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Authorization/AppRoles.cs b/etl_backend/Application/Common/Authorization/AppRoles.cs
new file mode 100644
index 00000000..b3c9859f
--- /dev/null
+++ b/etl_backend/Application/Common/Authorization/AppRoles.cs
@@ -0,0 +1,8 @@
+namespace Application.Common.Authorization;
+
+public static class AppRoles
+{
+ public const string SysAdmin = "sys_admin";
+ public const string DataAdmin = "data_admin";
+ public const string Analyst = "data_analyst";
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Authorization/RequireRoleAttribute.cs b/etl_backend/Application/Common/Authorization/RequireRoleAttribute.cs
new file mode 100644
index 00000000..d09ac829
--- /dev/null
+++ b/etl_backend/Application/Common/Authorization/RequireRoleAttribute.cs
@@ -0,0 +1,14 @@
+namespace Application.Common.Authorization;
+
+using System;
+
+[AttributeUsage(AttributeTargets.Class)]
+public class RequireRoleAttribute : Attribute
+{
+ public string[] Roles { get; }
+
+ public RequireRoleAttribute(params string[] roles)
+ {
+ Roles = roles ?? throw new ArgumentNullException(nameof(roles));
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Behaviors/AuthorizationBehavior.cs b/etl_backend/Application/Common/Behaviors/AuthorizationBehavior.cs
new file mode 100644
index 00000000..a4acc62c
--- /dev/null
+++ b/etl_backend/Application/Common/Behaviors/AuthorizationBehavior.cs
@@ -0,0 +1,43 @@
+using System.Reflection;
+using Application.Common.Authorization;
+using Application.Common.Exceptions;
+using Application.Services.Abstractions;
+using MediatR;
+
+namespace Application.Common.Behaviors;
+
+
+public class AuthorizationBehavior : IPipelineBehavior
+ where TRequest : notnull
+{
+ private readonly ICurrentUserService _currentUser;
+
+ public AuthorizationBehavior(ICurrentUserService currentUser)
+ {
+ _currentUser = currentUser;
+ }
+
+ public async Task Handle(TRequest request, RequestHandlerDelegate next, CancellationToken ct)
+ {
+ var authorizeAttribute = request.GetType().GetCustomAttribute();
+ if (authorizeAttribute == null)
+ {
+ return await next();
+ }
+
+ if (!_currentUser.IsAuthenticated)
+ {
+ throw new ForbiddenException("User is not authenticated.");
+ }
+
+ var userHasRequiredRole = authorizeAttribute.Roles
+ .Any(requiredRole => _currentUser.Roles.Contains(requiredRole));
+
+ if (!userHasRequiredRole)
+ {
+ throw new ForbiddenException($"User is not authorized. Required roles: {string.Join(", ", authorizeAttribute.Roles)}");
+ }
+
+ return await next();
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Configurations/ColumnTypeConfiguration.cs b/etl_backend/Application/Common/Configurations/ColumnTypeConfiguration.cs
new file mode 100644
index 00000000..511c1059
--- /dev/null
+++ b/etl_backend/Application/Common/Configurations/ColumnTypeConfiguration.cs
@@ -0,0 +1,7 @@
+namespace Application.Common.Configurations;
+
+public class ColumnTypeConfiguration
+{
+ public List CanonicalTypes { get; set; } = new();
+ public Dictionary TypeAliases { get; set; } = new();
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Dtos/BaseUserDto.cs b/etl_backend/Application/Common/Dtos/BaseUserDto.cs
new file mode 100644
index 00000000..1774fb14
--- /dev/null
+++ b/etl_backend/Application/Common/Dtos/BaseUserDto.cs
@@ -0,0 +1,9 @@
+
+namespace Application.Dtos;
+
+public class BaseUserDto
+{
+ public string Email { get; set; } = string.Empty;
+ public string? FirstName { get; set; }
+ public string? LastName { get; set; }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Dtos/EditUserRequestDto.cs b/etl_backend/Application/Common/Dtos/EditUserRequestDto.cs
new file mode 100644
index 00000000..899d0949
--- /dev/null
+++ b/etl_backend/Application/Common/Dtos/EditUserRequestDto.cs
@@ -0,0 +1,6 @@
+namespace Application.Dtos;
+
+public class EditUserRequestDto: BaseUserDto
+{
+
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Dtos/LoadResult.cs b/etl_backend/Application/Common/Dtos/LoadResult.cs
new file mode 100644
index 00000000..a5861b74
--- /dev/null
+++ b/etl_backend/Application/Common/Dtos/LoadResult.cs
@@ -0,0 +1,6 @@
+namespace Application.Dtos;
+
+public record LoadResult(
+ long RowsInserted,
+ double ElapsedMs
+);
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Dtos/RegisterAndLoadReslut.cs b/etl_backend/Application/Common/Dtos/RegisterAndLoadReslut.cs
new file mode 100644
index 00000000..7e12b835
--- /dev/null
+++ b/etl_backend/Application/Common/Dtos/RegisterAndLoadReslut.cs
@@ -0,0 +1,11 @@
+using Application.Files.Commands;
+
+namespace Application.Dtos;
+
+public record RegisterAndLoadResult(
+ int SchemaId,
+ string TableName,
+ List Columns,
+ StagedFileStatusDto Staged,
+ LoadResult Load
+);
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Dtos/RoleDto.cs b/etl_backend/Application/Common/Dtos/RoleDto.cs
new file mode 100644
index 00000000..cc1513e2
--- /dev/null
+++ b/etl_backend/Application/Common/Dtos/RoleDto.cs
@@ -0,0 +1,7 @@
+namespace Application.Dtos;
+
+public class RoleDto
+{
+ public string Id { get; set; } = string.Empty;
+ public string Name { get; set; } = string.Empty;
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Dtos/RowPreviewDto.cs b/etl_backend/Application/Common/Dtos/RowPreviewDto.cs
new file mode 100644
index 00000000..950e2542
--- /dev/null
+++ b/etl_backend/Application/Common/Dtos/RowPreviewDto.cs
@@ -0,0 +1,12 @@
+namespace Application.Dtos;
+
+public sealed class RowPreviewDto
+{
+ public List> Rows { get; set; } = new();
+ public int NextOffset { get; set; }
+}
+public sealed class RowCountDto
+{
+ public bool Exact { get; set; }
+ public long Count { get; set; }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Dtos/TableDetailsDto.cs b/etl_backend/Application/Common/Dtos/TableDetailsDto.cs
new file mode 100644
index 00000000..764adbb9
--- /dev/null
+++ b/etl_backend/Application/Common/Dtos/TableDetailsDto.cs
@@ -0,0 +1,18 @@
+namespace Application.Dtos;
+
+public sealed class TableDetailsDto
+{
+ public int SchemaId { get; set; }
+ public string TableName { get; set; } = default!;
+ public bool PhysicalExists { get; set; }
+ public long RowCountApprox { get; set; }
+ public long SizeBytes { get; set; }
+ public List Columns { get; set; } = new();
+}
+
+public sealed class ColumnDetailsDto
+{
+ public int Ordinal { get; set; }
+ public string Name { get; set; } = default!;
+ public string Type { get; set; } = default!;
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Dtos/UserCreateDto.cs b/etl_backend/Application/Common/Dtos/UserCreateDto.cs
new file mode 100644
index 00000000..0d3b41f0
--- /dev/null
+++ b/etl_backend/Application/Common/Dtos/UserCreateDto.cs
@@ -0,0 +1,11 @@
+
+namespace Application.Dtos;
+
+public class UserCreateDto: BaseUserDto
+{
+
+ public string Username { get; set; } = string.Empty;
+
+ public string? Password { get; set; }
+
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Dtos/UserDto.cs b/etl_backend/Application/Common/Dtos/UserDto.cs
new file mode 100644
index 00000000..45364063
--- /dev/null
+++ b/etl_backend/Application/Common/Dtos/UserDto.cs
@@ -0,0 +1,18 @@
+
+namespace Application.Dtos;
+
+public class UserDto: BaseUserDto
+{
+ public string Id { get; set; }
+ public string Username { get; set; } = string.Empty;
+
+ public UserDto() { }
+
+ public UserDto(string username, string email, string? firstName = null, string? lastName = null)
+ {
+ Username = username;
+ Email = email;
+ FirstName = firstName;
+ LastName = lastName;
+ }
+}
diff --git a/etl_backend/Application/Common/Dtos/UserWithRolesDto.cs b/etl_backend/Application/Common/Dtos/UserWithRolesDto.cs
new file mode 100644
index 00000000..d52fed5d
--- /dev/null
+++ b/etl_backend/Application/Common/Dtos/UserWithRolesDto.cs
@@ -0,0 +1,20 @@
+
+namespace Application.Dtos;
+
+public class UserWithRolesDto: UserDto
+{
+ public IEnumerable Roles { get; set; } = Enumerable.Empty();
+
+ public UserWithRolesDto() { }
+
+ public UserWithRolesDto(UserDto user, IEnumerable roles)
+ {
+ Id = user.Id;
+ Username = user.Username;
+ Email = user.Email;
+ FirstName = user.FirstName;
+ LastName = user.LastName;
+ Roles = roles;
+ }
+ public UserDto ToUserDto() => this;
+}
diff --git a/etl_backend/Application/Common/Enums/LoadMode.cs b/etl_backend/Application/Common/Enums/LoadMode.cs
new file mode 100644
index 00000000..20bca1cd
--- /dev/null
+++ b/etl_backend/Application/Common/Enums/LoadMode.cs
@@ -0,0 +1,8 @@
+namespace Application.Enums;
+
+public enum LoadMode
+{
+ FailIfExists,
+ Replace,
+ Append
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Exceptions/ConflictException.cs b/etl_backend/Application/Common/Exceptions/ConflictException.cs
new file mode 100644
index 00000000..e97f23d5
--- /dev/null
+++ b/etl_backend/Application/Common/Exceptions/ConflictException.cs
@@ -0,0 +1,6 @@
+namespace Application.Common.Exceptions;
+
+public class ConflictException : Exception
+{
+ public ConflictException(string message) : base(message) { }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Exceptions/ForbiddenException.cs b/etl_backend/Application/Common/Exceptions/ForbiddenException.cs
new file mode 100644
index 00000000..7f53e084
--- /dev/null
+++ b/etl_backend/Application/Common/Exceptions/ForbiddenException.cs
@@ -0,0 +1,6 @@
+namespace Application.Common.Exceptions;
+
+public class ForbiddenException : Exception
+{
+ public ForbiddenException(string message) : base(message) { }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Exceptions/NotFoundException.cs b/etl_backend/Application/Common/Exceptions/NotFoundException.cs
new file mode 100644
index 00000000..aba1adee
--- /dev/null
+++ b/etl_backend/Application/Common/Exceptions/NotFoundException.cs
@@ -0,0 +1,7 @@
+namespace Application.Common.Exceptions;
+
+public class NotFoundException : Exception
+{
+ public NotFoundException(string message) : base(message) { }
+ public NotFoundException(string name, object key) : base($"{name} with ID {key} not found.") { }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Exceptions/UnprocessableEntityException.cs b/etl_backend/Application/Common/Exceptions/UnprocessableEntityException.cs
new file mode 100644
index 00000000..a2c6fce3
--- /dev/null
+++ b/etl_backend/Application/Common/Exceptions/UnprocessableEntityException.cs
@@ -0,0 +1,6 @@
+namespace Application.Common.Exceptions;
+
+public class UnprocessableEntityException : Exception
+{
+ public UnprocessableEntityException(string message) : base(message) { }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Mappers/StageFileMapper.cs b/etl_backend/Application/Common/Mappers/StageFileMapper.cs
new file mode 100644
index 00000000..632fb8b0
--- /dev/null
+++ b/etl_backend/Application/Common/Mappers/StageFileMapper.cs
@@ -0,0 +1,20 @@
+using Application.Files.Commands;
+using Domain.Entities;
+using Domain.Enums;
+
+namespace Application.Common.Mappers;
+
+internal static class StageFileMapper
+{
+ public static StageFileResponse Map(StagedFile s) => new(
+ Id: s.Id,
+ OriginalFileName: s.OriginalFileName,
+ StoredFilePath: s.StoredFilePath,
+ FileSize: s.FileSize,
+ UploadedAt: s.UploadedAt,
+ Stage: s.Stage.ToString(),
+ Status: s.Status.ToString(),
+ ErrorCode: s.ErrorCode == ProcessingErrorCode.None ? null : s.ErrorCode.ToString(),
+ ErrorMessage: s.ErrorMessage
+ );
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Services/Abstractions/IColumnDefenitionBuilder.cs b/etl_backend/Application/Common/Services/Abstractions/IColumnDefenitionBuilder.cs
new file mode 100644
index 00000000..0a4d875b
--- /dev/null
+++ b/etl_backend/Application/Common/Services/Abstractions/IColumnDefenitionBuilder.cs
@@ -0,0 +1,8 @@
+using Domain.Entities;
+
+namespace Application.Abstractions;
+
+public interface IColumnDefinitionBuilder
+{
+ List Build(IReadOnlyList headers);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Services/Abstractions/IColumnNameSanitizer.cs b/etl_backend/Application/Common/Services/Abstractions/IColumnNameSanitizer.cs
new file mode 100644
index 00000000..bb6e56a0
--- /dev/null
+++ b/etl_backend/Application/Common/Services/Abstractions/IColumnNameSanitizer.cs
@@ -0,0 +1,6 @@
+namespace Application.Abstractions;
+
+public interface IColumnNameSanitizer
+{
+ string Sanitize(string? raw, int index);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Services/Abstractions/ICurrentUserService.cs b/etl_backend/Application/Common/Services/Abstractions/ICurrentUserService.cs
new file mode 100644
index 00000000..1dd00b29
--- /dev/null
+++ b/etl_backend/Application/Common/Services/Abstractions/ICurrentUserService.cs
@@ -0,0 +1,9 @@
+namespace Application.Services.Abstractions;
+
+public interface ICurrentUserService
+{
+ string? UserId { get; }
+ string? UserName { get; }
+ string[] Roles { get; }
+ bool IsAuthenticated { get; }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Services/Abstractions/IHeaderProvider.cs b/etl_backend/Application/Common/Services/Abstractions/IHeaderProvider.cs
new file mode 100644
index 00000000..a7838d28
--- /dev/null
+++ b/etl_backend/Application/Common/Services/Abstractions/IHeaderProvider.cs
@@ -0,0 +1,8 @@
+using Domain.Entities;
+
+namespace Application.Abstractions;
+
+public interface IHeaderProvider
+{
+ Task> GetAsync(StagedFile staged, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Services/Abstractions/IRegisterAndLoadService.cs b/etl_backend/Application/Common/Services/Abstractions/IRegisterAndLoadService.cs
new file mode 100644
index 00000000..edfb9653
--- /dev/null
+++ b/etl_backend/Application/Common/Services/Abstractions/IRegisterAndLoadService.cs
@@ -0,0 +1,15 @@
+using Application.Dtos;
+using Application.Enums;
+using Application.Files.Commands;
+
+namespace Application.Abstractions;
+public interface IRegisterAndLoadService
+{
+ Task ExecuteAsync(
+ int stagedFileId,
+ Dictionary columnTypeMap,
+ Dictionary columnNameMap,
+ LoadMode mode = LoadMode.Append,
+ bool dropOnFailure = false,
+ CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Services/Abstractions/ITableCatalogService.cs b/etl_backend/Application/Common/Services/Abstractions/ITableCatalogService.cs
new file mode 100644
index 00000000..7f9503b1
--- /dev/null
+++ b/etl_backend/Application/Common/Services/Abstractions/ITableCatalogService.cs
@@ -0,0 +1,10 @@
+namespace Application.Abstractions;
+
+using System.Collections.Generic;
+
+public interface ITypeCatalogService
+{
+ IReadOnlyList GetAllowedTypes();
+ IReadOnlyDictionary GetTypeAliases();
+ bool TryNormalize(string input, out string normalized);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Services/Abstractions/IUserRoleManagementService.cs b/etl_backend/Application/Common/Services/Abstractions/IUserRoleManagementService.cs
new file mode 100644
index 00000000..75d4d9a6
--- /dev/null
+++ b/etl_backend/Application/Common/Services/Abstractions/IUserRoleManagementService.cs
@@ -0,0 +1,10 @@
+using Application.Dtos;
+
+namespace Application.Services.Abstractions;
+
+public interface IUserRoleManagementService
+{
+ Task AddRolesToUserAsync(string userId, RoleDto[] roles, CancellationToken cancellationToken);
+ Task RemoveRolesFromUserAsync(string userId, IEnumerable roles, CancellationToken cancellationToken);
+
+}
diff --git a/etl_backend/Application/Common/Services/Repositories/Abstractions/IColumnRepository.cs b/etl_backend/Application/Common/Services/Repositories/Abstractions/IColumnRepository.cs
new file mode 100644
index 00000000..f9b0d536
--- /dev/null
+++ b/etl_backend/Application/Common/Services/Repositories/Abstractions/IColumnRepository.cs
@@ -0,0 +1,8 @@
+namespace Application.Services.Repositories.Abstractions;
+
+public interface IColumnRepository
+{
+ Task RenameColumnAsync(string schemaName, string tableName, string oldName, string newName, CancellationToken ct = default);
+ Task DropColumnsAsync(string schemaName, string tableName, List columnNames, CancellationToken ct = default);
+ Task TableExistsAsync(string schemaName, string tableName, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Services/Repositories/Abstractions/IDataTableColumnRepository.cs b/etl_backend/Application/Common/Services/Repositories/Abstractions/IDataTableColumnRepository.cs
new file mode 100644
index 00000000..a1fa4bbc
--- /dev/null
+++ b/etl_backend/Application/Common/Services/Repositories/Abstractions/IDataTableColumnRepository.cs
@@ -0,0 +1,9 @@
+using Domain.Entities;
+
+namespace Application.Services.Repositories.Abstractions;
+
+public interface IDataTableColumnRepository
+{
+ Task UpdateNameAsync(int id, string newName, CancellationToken ct = default);
+ Task DeleteByIdsAsync(IEnumerable ids, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Services/Repositories/Abstractions/IDataTableSchemaRepository.cs b/etl_backend/Application/Common/Services/Repositories/Abstractions/IDataTableSchemaRepository.cs
new file mode 100644
index 00000000..7316949d
--- /dev/null
+++ b/etl_backend/Application/Common/Services/Repositories/Abstractions/IDataTableSchemaRepository.cs
@@ -0,0 +1,18 @@
+using Domain.Entities;
+
+namespace Application.Services.Repositories.Abstractions;
+
+public interface IDataTableSchemaRepository
+{
+ Task> ListAsync(CancellationToken ct = default);
+
+ Task GetByIdWithColumnsAsync(int id, CancellationToken ct = default);
+
+ Task AddAsync(DataTableSchema schema, CancellationToken ct = default);
+
+ Task UpdateAsync(DataTableSchema schema, CancellationToken ct = default);
+
+ Task UpdateTableNameAsync(int id, string newTableName, CancellationToken ct = default);
+
+ Task DeleteAsync(int id, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Services/Repositories/Abstractions/IStagedFileRepository.cs b/etl_backend/Application/Common/Services/Repositories/Abstractions/IStagedFileRepository.cs
new file mode 100644
index 00000000..b9a8ad10
--- /dev/null
+++ b/etl_backend/Application/Common/Services/Repositories/Abstractions/IStagedFileRepository.cs
@@ -0,0 +1,12 @@
+using Domain.Entities;
+
+namespace Application.Services.Repositories.Abstractions;
+
+public interface IStagedFileRepository
+{
+ Task AddAsync(StagedFile entity, CancellationToken ct = default);
+ Task GetByIdAsync(int id, CancellationToken ct = default);
+ Task UpdateAsync(StagedFile entity, CancellationToken ct = default);
+ Task> ListAsync(CancellationToken ct = default);
+ Task DeleteAsync(int id, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Common/Services/Repositories/Abstractions/ITableRepository.cs b/etl_backend/Application/Common/Services/Repositories/Abstractions/ITableRepository.cs
new file mode 100644
index 00000000..d3989947
--- /dev/null
+++ b/etl_backend/Application/Common/Services/Repositories/Abstractions/ITableRepository.cs
@@ -0,0 +1,21 @@
+using Application.Dtos;
+using Domain.Entities;
+
+namespace Application.Services.Repositories.Abstractions;
+
+public interface ITableRepository
+{
+ Task TableExistsAsync(string schemaName, string tableName, CancellationToken ct = default);
+ Task GetApproximateRowCountAsync(string schemaName, string tableName, CancellationToken ct = default);
+ Task GetTotalSizeAsync(string schemaName, string tableName, CancellationToken ct = default);
+ Task PreviewRowsAsync(
+ string schemaName,
+ string tableName,
+ List columns,
+ int offset,
+ int limit,
+ string? orderBy = null,
+ string? direction = null,
+ CancellationToken ct = default);
+ Task GetExactRowCountAsync(string schemaName, string tableName, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/DependencyInjection.cs b/etl_backend/Application/DependencyInjection.cs
new file mode 100644
index 00000000..746dd0a8
--- /dev/null
+++ b/etl_backend/Application/DependencyInjection.cs
@@ -0,0 +1,16 @@
+using Microsoft.Extensions.DependencyInjection;
+using MediatR;
+using System.Reflection;
+using Application.Common.Behaviors;
+
+namespace Application;
+
+public static class DependencyInjection
+{
+ public static IServiceCollection AddApplicationServices(this IServiceCollection services)
+ {
+ services.AddMediatR(cfg => cfg.RegisterServicesFromAssembly(typeof(DependencyInjection).Assembly));
+ services.AddTransient(typeof(IPipelineBehavior<,>), typeof(AuthorizationBehavior<,>));
+ return services;
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/Common/ServiceAbstractions/ILoadPolicy.cs b/etl_backend/Application/Files/Common/ServiceAbstractions/ILoadPolicy.cs
new file mode 100644
index 00000000..09109bb9
--- /dev/null
+++ b/etl_backend/Application/Files/Common/ServiceAbstractions/ILoadPolicy.cs
@@ -0,0 +1,9 @@
+using Application.Enums;
+
+namespace Application.Common.Services.Abstractions;
+
+public interface ILoadPolicy
+{
+ LoadMode Mode { get; }
+ bool DropOnFailure { get; } // if Replace and load fails, should we drop?
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/DeleteStagedFile/DeleteStagedFileCommand.cs b/etl_backend/Application/Files/DeleteStagedFile/DeleteStagedFileCommand.cs
new file mode 100644
index 00000000..99ab119a
--- /dev/null
+++ b/etl_backend/Application/Files/DeleteStagedFile/DeleteStagedFileCommand.cs
@@ -0,0 +1,6 @@
+using Application.Common.Authorization;
+using MediatR;
+
+namespace Application.Files.Commands;
+[RequireRole(AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record DeleteStagedFileCommand(int StagedFileId) : IRequest;
\ No newline at end of file
diff --git a/etl_backend/Application/Files/DeleteStagedFile/DeleteStagedFileCommandHandler.cs b/etl_backend/Application/Files/DeleteStagedFile/DeleteStagedFileCommandHandler.cs
new file mode 100644
index 00000000..e2820df8
--- /dev/null
+++ b/etl_backend/Application/Files/DeleteStagedFile/DeleteStagedFileCommandHandler.cs
@@ -0,0 +1,35 @@
+using Application.Abstractions;
+using Application.Common.Exceptions;
+using Application.Files.Commands;
+using Application.Files.DeleteStagedFile.ServiceAbstractions;
+using Application.Services.Repositories.Abstractions;
+using Domain.Enums;
+using MediatR;
+
+namespace Application.Files.Handlers;
+
+public class DeleteStagedFileCommandHandler : IRequestHandler
+{
+ private readonly IStagedFileRepository _stagedRepo;
+ private readonly IDeleteStagedFile _fileStagingService;
+
+ public DeleteStagedFileCommandHandler(
+ IStagedFileRepository stagedRepo,
+ IDeleteStagedFile fileStagingService)
+ {
+ _stagedRepo = stagedRepo;
+ _fileStagingService = fileStagingService;
+ }
+
+ public async Task Handle(DeleteStagedFileCommand request, CancellationToken ct)
+ {
+ var staged = await _stagedRepo.GetByIdAsync(request.StagedFileId, ct);
+ if (staged is null)
+ throw new NotFoundException("StagedFile", request.StagedFileId);
+ if (staged.Stage == ProcessingStage.Loaded)
+ throw new ConflictException("Cannot delete file after it has been loaded.");
+
+ await _fileStagingService.DeleteAsync(staged.StoredFilePath, ct);
+ await _stagedRepo.DeleteAsync(request.StagedFileId, ct);
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/DeleteStagedFile/ServiceAbstractions/IDeleteStagedFile.cs b/etl_backend/Application/Files/DeleteStagedFile/ServiceAbstractions/IDeleteStagedFile.cs
new file mode 100644
index 00000000..b4c598b5
--- /dev/null
+++ b/etl_backend/Application/Files/DeleteStagedFile/ServiceAbstractions/IDeleteStagedFile.cs
@@ -0,0 +1,6 @@
+namespace Application.Files.DeleteStagedFile.ServiceAbstractions;
+
+public interface IDeleteStagedFile
+{
+ Task DeleteAsync(string storedFilePath, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/ListStagedFiles/ListStagedFilesQuery.cs b/etl_backend/Application/Files/ListStagedFiles/ListStagedFilesQuery.cs
new file mode 100644
index 00000000..aef58849
--- /dev/null
+++ b/etl_backend/Application/Files/ListStagedFiles/ListStagedFilesQuery.cs
@@ -0,0 +1,16 @@
+using Application.Common.Authorization;
+using MediatR;
+
+namespace Application.Files.Queries;
+[RequireRole(AppRoles.Analyst, AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record ListStagedFilesQuery : IRequest>;
+
+public record ListFilesItem(
+ int Id,
+ string OriginalFileName,
+ string Stage,
+ string Status,
+ int? SchemaId,
+ long FileSize,
+ DateTime UploadedAt
+);
\ No newline at end of file
diff --git a/etl_backend/Application/Files/ListStagedFiles/ListStagedFilesQueryHandler.cs b/etl_backend/Application/Files/ListStagedFiles/ListStagedFilesQueryHandler.cs
new file mode 100644
index 00000000..b18126d5
--- /dev/null
+++ b/etl_backend/Application/Files/ListStagedFiles/ListStagedFilesQueryHandler.cs
@@ -0,0 +1,30 @@
+using Application.Files.Queries;
+using Application.Services.Repositories.Abstractions;
+using MediatR;
+
+namespace Application.Files.Handlers;
+
+public class ListStagedFilesQueryHandler : IRequestHandler>
+{
+ private readonly IStagedFileRepository _stagedRepo;
+
+ public ListStagedFilesQueryHandler(IStagedFileRepository stagedRepo)
+ {
+ _stagedRepo = stagedRepo;
+ }
+
+ public async Task> Handle(ListStagedFilesQuery request, CancellationToken ct)
+ {
+ var items = await _stagedRepo.ListAsync(ct);
+
+ return items.Select(s => new ListFilesItem(
+ Id: s.Id,
+ OriginalFileName: s.OriginalFileName,
+ Stage: s.Stage.ToString(),
+ Status: s.Status.ToString(),
+ SchemaId: s.SchemaId,
+ FileSize: s.FileSize,
+ UploadedAt: s.UploadedAt
+ )).ToList();
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/LoadFileIntoTable/LoadFileIntoTableCommand.cs b/etl_backend/Application/Files/LoadFileIntoTable/LoadFileIntoTableCommand.cs
new file mode 100644
index 00000000..866ec40f
--- /dev/null
+++ b/etl_backend/Application/Files/LoadFileIntoTable/LoadFileIntoTableCommand.cs
@@ -0,0 +1,14 @@
+using Application.Common.Authorization;
+using Application.Dtos;
+using Application.Enums;
+using MediatR;
+
+namespace Application.Files.Commands;
+[RequireRole(AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record LoadFileIntoTableCommand(
+ int StagedFileId,
+ LoadMode Mode = LoadMode.Append,
+ bool DropOnFailure = false
+) : IRequest;
+
+
diff --git a/etl_backend/Application/Files/LoadFileIntoTable/LoadFileIntoTableCommandHandler.cs b/etl_backend/Application/Files/LoadFileIntoTable/LoadFileIntoTableCommandHandler.cs
new file mode 100644
index 00000000..3469a10c
--- /dev/null
+++ b/etl_backend/Application/Files/LoadFileIntoTable/LoadFileIntoTableCommandHandler.cs
@@ -0,0 +1,48 @@
+using Application.Abstractions;
+using Application.Common.Exceptions;
+using Application.Dtos;
+using Application.Files.Commands;
+using MediatR;
+
+namespace Application.Files.Handlers;
+
+public class LoadFileIntoTableCommandHandler : IRequestHandler
+{
+ private readonly ITableLoadService _tableLoad;
+ private readonly ILoadPolicyFactory _loadPolicyFactory;
+
+ public LoadFileIntoTableCommandHandler(
+ ITableLoadService tableLoad,
+ ILoadPolicyFactory loadPolicyFactory)
+ {
+ _tableLoad = tableLoad;
+ _loadPolicyFactory = loadPolicyFactory;
+ }
+
+ public async Task Handle(LoadFileIntoTableCommand request, CancellationToken ct)
+ {
+ var policy = _loadPolicyFactory.Create(request.Mode, request.DropOnFailure);
+
+ try
+ {
+ var result = await _tableLoad.LoadAsync(request.StagedFileId, policy, ct);
+
+ return new LoadResult(
+ RowsInserted: result.RowsInserted,
+ ElapsedMs: result.ElapsedMs
+ );
+ }
+ catch (ArgumentException ex)
+ {
+ throw new UnprocessableEntityException(ex.Message);
+ }
+ catch (InvalidOperationException ex)
+ {
+ throw new ConflictException(ex.Message);
+ }
+ catch (Exception ex)
+ {
+ throw new ApplicationException("Load failed.", ex);
+ }
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/LoadFileIntoTable/ServiceAbstractions/ILoadPolicyFactory.cs b/etl_backend/Application/Files/LoadFileIntoTable/ServiceAbstractions/ILoadPolicyFactory.cs
new file mode 100644
index 00000000..e2bff842
--- /dev/null
+++ b/etl_backend/Application/Files/LoadFileIntoTable/ServiceAbstractions/ILoadPolicyFactory.cs
@@ -0,0 +1,7 @@
+using Application.Common.Services.Abstractions;
+using Application.Enums;
+using Application.Services.Abstractions;
+
+namespace Application.Abstractions;
+
+public interface ILoadPolicyFactory { ILoadPolicy Create(LoadMode mode, bool dropOnFailure); }
diff --git a/etl_backend/Application/Files/LoadFileIntoTable/ServiceAbstractions/ITableLoadService.cs b/etl_backend/Application/Files/LoadFileIntoTable/ServiceAbstractions/ITableLoadService.cs
new file mode 100644
index 00000000..8eaeb139
--- /dev/null
+++ b/etl_backend/Application/Files/LoadFileIntoTable/ServiceAbstractions/ITableLoadService.cs
@@ -0,0 +1,11 @@
+using Application.Common.Services.Abstractions;
+using Application.Dtos;
+using Application.Files.Commands;
+using Application.Services.Abstractions;
+
+namespace Application.Abstractions;
+
+public interface ITableLoadService
+{
+ Task LoadAsync(int stagedFileId, ILoadPolicy policy, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/PreviewSchema/PreviewSchemaQuery.cs b/etl_backend/Application/Files/PreviewSchema/PreviewSchemaQuery.cs
new file mode 100644
index 00000000..14f4dfa2
--- /dev/null
+++ b/etl_backend/Application/Files/PreviewSchema/PreviewSchemaQuery.cs
@@ -0,0 +1,18 @@
+using Application.Common.Authorization;
+using MediatR;
+
+namespace Application.Files.Queries;
+[RequireRole(AppRoles.Analyst, AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record PreviewSchemaQuery(int StagedFileId) : IRequest;
+
+public record ColumnPreviewResponse(
+ int StagedFileId,
+ List Columns
+);
+
+public record ColumnPreviewItem(
+ int OrdinalPosition,
+ string ColumnName,
+ string OriginalColumnName,
+ string ColumnType
+);
\ No newline at end of file
diff --git a/etl_backend/Application/Files/PreviewSchema/PreviewSchemaQueryHandler.cs b/etl_backend/Application/Files/PreviewSchema/PreviewSchemaQueryHandler.cs
new file mode 100644
index 00000000..b972eac6
--- /dev/null
+++ b/etl_backend/Application/Files/PreviewSchema/PreviewSchemaQueryHandler.cs
@@ -0,0 +1,80 @@
+using Application.Abstractions;
+using Application.Common.Exceptions;
+using Application.Files.Queries;
+using Application.Services.Repositories.Abstractions;
+using Domain.Entities;
+using Domain.Enums;
+using MediatR;
+
+namespace Application.Files.Handlers;
+
+public class PreviewSchemaQueryHandler : IRequestHandler
+{
+ private readonly IStagedFileRepository _stagedRepo;
+ private readonly IHeaderProvider _headerProvider;
+ private readonly IColumnDefinitionBuilder _columnDefinitionBuilder;
+ private readonly IDataTableSchemaRepository _schemaRepo;
+
+ public PreviewSchemaQueryHandler(
+ IStagedFileRepository stagedRepo,
+ IHeaderProvider headerProvider,
+ IColumnDefinitionBuilder columnDefinitionBuilder,
+ IDataTableSchemaRepository schemaRepo)
+ {
+ _stagedRepo = stagedRepo;
+ _headerProvider = headerProvider;
+ _columnDefinitionBuilder = columnDefinitionBuilder;
+ _schemaRepo = schemaRepo;
+ }
+
+ public async Task Handle(PreviewSchemaQuery request, CancellationToken ct)
+ {
+ var staged = await _stagedRepo.GetByIdAsync(request.StagedFileId, ct);
+ if (staged is null)
+ throw new NotFoundException($"Staged file with ID {request.StagedFileId} not found.");
+
+ if (staged.Status == ProcessingStatus.Failed)
+ throw new ConflictException("Staged file is in failed state.");
+
+ DataTableSchema? existingSchema = null;
+ if (staged.SchemaId.HasValue)
+ {
+ existingSchema = await _schemaRepo.GetByIdWithColumnsAsync(staged.SchemaId.Value, ct);
+ }
+
+ List columns;
+
+ if (existingSchema != null && existingSchema.Columns.Any())
+ {
+ columns = existingSchema.Columns
+ .OrderBy(c => c.OrdinalPosition)
+ .Select(c => new ColumnPreviewItem(
+ c.OrdinalPosition,
+ c.ColumnName,
+ c.OriginalColumnName ?? c.ColumnName,
+ c.ColumnType.ToString()
+ ))
+ .ToList();
+ }
+ else
+ {
+ var headerNames = await _headerProvider.GetAsync(staged, ct);
+ if (headerNames.Count == 0)
+ throw new UnprocessableEntityException("Header row not found or empty.");
+
+ var columnEntities = _columnDefinitionBuilder.Build(headerNames);
+
+ columns = columnEntities
+ .OrderBy(c => c.OrdinalPosition)
+ .Select(c => new ColumnPreviewItem(
+ c.OrdinalPosition,
+ c.ColumnName,
+ c.OriginalColumnName,
+ c.ColumnType.ToString()
+ ))
+ .ToList();
+ }
+
+ return new ColumnPreviewResponse(request.StagedFileId, columns);
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/RegisterSchema/RegisterSchemaCommand.cs b/etl_backend/Application/Files/RegisterSchema/RegisterSchemaCommand.cs
new file mode 100644
index 00000000..b09e2061
--- /dev/null
+++ b/etl_backend/Application/Files/RegisterSchema/RegisterSchemaCommand.cs
@@ -0,0 +1,30 @@
+using Application.Common.Authorization;
+using MediatR;
+
+namespace Application.Files.Commands;
+[RequireRole(AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record RegisterSchemaCommand(
+ int StagedFileId,
+ Dictionary ColumnTypeMap,
+ Dictionary ColumnNameMap
+) : IRequest;
+
+public record RegisterSchemaResult(
+ int SchemaId,
+ string TableName,
+ List Columns,
+ StagedFileStatusDto Staged
+);
+
+public record SchemaColumnDto(
+ int OrdinalPosition,
+ string ColumnName,
+ string ColumnType
+);
+
+public record StagedFileStatusDto(
+ int Id,
+ string Stage,
+ string Status,
+ string? ErrorCode
+);
\ No newline at end of file
diff --git a/etl_backend/Application/Files/RegisterSchema/RegisterSchemaCommandHandler.cs b/etl_backend/Application/Files/RegisterSchema/RegisterSchemaCommandHandler.cs
new file mode 100644
index 00000000..5c566067
--- /dev/null
+++ b/etl_backend/Application/Files/RegisterSchema/RegisterSchemaCommandHandler.cs
@@ -0,0 +1,48 @@
+using Application.Abstractions;
+using Application.Common.Exceptions;
+using Application.Files.Commands;
+using Application.Services.Repositories.Abstractions;
+using Domain.Enums;
+using MediatR;
+
+namespace Application.Files.Handlers;
+
+public class RegisterSchemaCommandHandler : IRequestHandler
+{
+ private readonly ISchemaRegistrationService _schemaReg;
+
+ public RegisterSchemaCommandHandler(
+ ISchemaRegistrationService schemaReg)
+ {
+ _schemaReg = schemaReg;
+ }
+
+ public async Task Handle(RegisterSchemaCommand request, CancellationToken ct)
+ {
+
+ var (schema, updatedStaged) = await _schemaReg.RegisterAsync(request.StagedFileId, request.ColumnTypeMap,request.ColumnNameMap, ct);
+
+ var columns = schema.Columns
+ .OrderBy(c => c.OrdinalPosition)
+ .Select(c => new SchemaColumnDto(
+ c.OrdinalPosition,
+ c.ColumnName,
+ c.ColumnType
+ ))
+ .ToList();
+
+ var stagedStatus = new StagedFileStatusDto(
+ updatedStaged.Id,
+ updatedStaged.Stage.ToString(),
+ updatedStaged.Status.ToString(),
+ updatedStaged.ErrorCode == ProcessingErrorCode.None ? null : updatedStaged.ErrorCode.ToString()
+ );
+
+ return new RegisterSchemaResult(
+ schema.Id,
+ schema.TableName,
+ columns,
+ stagedStatus
+ );
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/RegisterSchema/SchemaRegistrationService.cs b/etl_backend/Application/Files/RegisterSchema/SchemaRegistrationService.cs
new file mode 100644
index 00000000..3e420335
--- /dev/null
+++ b/etl_backend/Application/Files/RegisterSchema/SchemaRegistrationService.cs
@@ -0,0 +1,132 @@
+using Application.Abstractions;
+using Application.Services.Repositories.Abstractions;
+using Domain.Entities;
+using Domain.Enums;
+using Infrastructure.Files.Abstractions;
+using IColumnDefinitionBuilder = Application.Abstractions.IColumnDefinitionBuilder;
+
+namespace Infrastructure.Files;
+
+public sealed class SchemaRegistrationService : ISchemaRegistrationService
+{
+ private readonly IStagedFileRepository _stagedRepo;
+ private readonly IDataTableSchemaRepository _schemaRepo;
+ private readonly IHeaderProvider _headers;
+ private readonly IColumnDefinitionBuilder _cols;
+ private readonly ITableNameGenerator _names;
+ private readonly IStagedFileStateService _state;
+ private readonly IColumnTypeValidator _typeValidator;
+ private readonly IColumnNameSanitizer _nameSanitizer;
+
+ public SchemaRegistrationService(
+ IStagedFileRepository stagedRepo,
+ IDataTableSchemaRepository schemaRepo,
+ IHeaderProvider headers,
+ IColumnDefinitionBuilder cols,
+ ITableNameGenerator names,
+ IStagedFileStateService state,
+ IColumnTypeValidator typeValidator,
+ IColumnNameSanitizer? nameSanitizer = null)
+ {
+ _stagedRepo = stagedRepo;
+ _schemaRepo = schemaRepo;
+ _headers = headers;
+ _cols = cols;
+ _names = names;
+ _state = state;
+ _typeValidator = typeValidator;
+ _nameSanitizer = nameSanitizer;
+ }
+
+ public async Task<(DataTableSchema Schema, StagedFile Staged)> RegisterAsync(
+ int stagedFileId,
+ IReadOnlyDictionary columnTypesByOrdinal,
+ IReadOnlyDictionary columnNamesByOrdinal,
+ CancellationToken ct = default)
+ {
+ var staged = await _stagedRepo.GetByIdAsync(stagedFileId, ct)
+ ?? throw new InvalidOperationException($"Staged file {stagedFileId} not found.");
+
+ if (staged.Stage == ProcessingStage.TableCreated || staged.Stage == ProcessingStage.Loaded)
+ throw new InvalidOperationException("Columns cannot be modified after the table has been created.");
+
+ IReadOnlyList headers;
+ try
+ {
+ headers = await _headers.GetAsync(staged, ct);
+ if (headers.Count == 0) throw new InvalidOperationException("CSV header row not found or empty.");
+ }
+ catch (Exception ex)
+ {
+ await _state.FailSchemaRegistrationAsync(staged, ex.Message, ct);
+ throw;
+ }
+
+ try
+ {
+ var columns = _cols.Build(headers);
+
+ _typeValidator.ValidateOrThrow(
+ columns.Select(c =>
+ {
+ var t = "string";
+ if (columnTypesByOrdinal != null)
+ columnTypesByOrdinal.TryGetValue(c.OrdinalPosition, out t);
+ return (c.OrdinalPosition, t ?? "string");
+ }));
+
+ foreach (var col in columns)
+ {
+ var rawType = columnTypesByOrdinal != null && columnTypesByOrdinal.TryGetValue(col.OrdinalPosition, out var t)
+ ? t
+ : "string";
+
+ if (!_typeValidator.TryNormalize(rawType, out var norm))
+ throw new ArgumentException($"Invalid column type at ordinal {col.OrdinalPosition}: '{rawType}'");
+
+ col.ColumnType = norm;
+ if (columnNamesByOrdinal != null && columnNamesByOrdinal.TryGetValue(col.OrdinalPosition, out var newName))
+ {
+ var finalName = _nameSanitizer?.Sanitize(newName, col.OrdinalPosition) ?? newName.Trim();
+ if (string.IsNullOrWhiteSpace(finalName))
+ throw new ArgumentException($"Invalid column name at ordinal {col.OrdinalPosition}: '{newName}'");
+
+ col.ColumnName = finalName;
+ }
+ }
+
+ DataTableSchema schema;
+ if (staged.SchemaId is null)
+ {
+ schema = new DataTableSchema
+ {
+ TableName = _names.Generate(staged.Id, staged.OriginalFileName),
+ OriginalFileName = staged.OriginalFileName,
+ Columns = columns
+ };
+
+ await _schemaRepo.AddAsync(schema, ct);
+ await _state.MarkSchemaRegisteredAsync(staged, schema.Id, ct);
+ }
+ else
+ {
+ schema = await _schemaRepo.GetByIdWithColumnsAsync(staged.SchemaId.Value, ct)
+ ?? throw new InvalidOperationException($"Schema {staged.SchemaId} not found.");
+
+ schema.Columns.Clear();
+ foreach (var c in columns.OrderBy(c => c.OrdinalPosition))
+ schema.Columns.Add(c);
+
+ await _schemaRepo.UpdateAsync(schema, ct);
+ await _state.MarkSchemaRegisteredAsync(staged, schema.Id, ct);
+ }
+
+ return (schema, staged);
+ }
+ catch (Exception ex)
+ {
+ await _state.FailSchemaDbWriteAsync(staged, ex.Message, ct);
+ throw;
+ }
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/RegisterSchema/ServiceAbstractions/IColumnTypeValidator.cs b/etl_backend/Application/Files/RegisterSchema/ServiceAbstractions/IColumnTypeValidator.cs
new file mode 100644
index 00000000..1049848a
--- /dev/null
+++ b/etl_backend/Application/Files/RegisterSchema/ServiceAbstractions/IColumnTypeValidator.cs
@@ -0,0 +1,8 @@
+namespace Infrastructure.Files.Abstractions;
+
+public interface IColumnTypeValidator
+{
+ bool TryNormalize(string? input, out string normalized);
+ void ValidateOrThrow(IEnumerable<(int OrdinalPosition, string? Type)> items);
+ IReadOnlyCollection AllowedTypes { get; }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/RegisterSchema/ServiceAbstractions/ISchemaRegistrationService.cs b/etl_backend/Application/Files/RegisterSchema/ServiceAbstractions/ISchemaRegistrationService.cs
new file mode 100644
index 00000000..9e769eb9
--- /dev/null
+++ b/etl_backend/Application/Files/RegisterSchema/ServiceAbstractions/ISchemaRegistrationService.cs
@@ -0,0 +1,12 @@
+using Domain.Entities;
+
+namespace Application.Abstractions;
+
+public interface ISchemaRegistrationService
+{
+ Task<(DataTableSchema Schema, StagedFile Staged)> RegisterAsync(
+ int stagedFileId,
+ IReadOnlyDictionary columnTypesByOrdinal,
+ IReadOnlyDictionary columnNamesByOrdinal,
+ CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/RegisterSchema/ServiceAbstractions/IStagedFileStateService.cs b/etl_backend/Application/Files/RegisterSchema/ServiceAbstractions/IStagedFileStateService.cs
new file mode 100644
index 00000000..9421ca84
--- /dev/null
+++ b/etl_backend/Application/Files/RegisterSchema/ServiceAbstractions/IStagedFileStateService.cs
@@ -0,0 +1,15 @@
+using Domain.Entities;
+
+namespace Infrastructure.Files.Abstractions;
+
+public interface IStagedFileStateService
+{
+ Task MarkSchemaRegisteredAsync(StagedFile staged, int schemaId, CancellationToken ct = default);
+ Task MarkTableCreatedAsync(StagedFile staged, CancellationToken ct = default);
+ Task MarkLoadedSucceededAsync(StagedFile staged, CancellationToken ct = default);
+
+ Task FailSchemaRegistrationAsync(StagedFile staged, string error, CancellationToken ct = default);
+ Task FailSchemaDbWriteAsync(StagedFile staged, string error, CancellationToken ct = default);
+ Task FailCreateTableAsync(StagedFile staged, string error, CancellationToken ct = default);
+ Task FailLoadAsync(StagedFile staged, string error, CancellationToken ct = default);
+}
diff --git a/etl_backend/Application/Files/RegisterSchema/ServiceAbstractions/ITableNameGenerator.cs b/etl_backend/Application/Files/RegisterSchema/ServiceAbstractions/ITableNameGenerator.cs
new file mode 100644
index 00000000..5e828038
--- /dev/null
+++ b/etl_backend/Application/Files/RegisterSchema/ServiceAbstractions/ITableNameGenerator.cs
@@ -0,0 +1,6 @@
+namespace Infrastructure.Files.Abstractions;
+
+public interface ITableNameGenerator
+{
+ string Generate(int stagedFileId, string originalFileName);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/StageManyFiles/ServiceAbstractions/IAddStageFileService.cs b/etl_backend/Application/Files/StageManyFiles/ServiceAbstractions/IAddStageFileService.cs
new file mode 100644
index 00000000..37477148
--- /dev/null
+++ b/etl_backend/Application/Files/StageManyFiles/ServiceAbstractions/IAddStageFileService.cs
@@ -0,0 +1,12 @@
+using Domain.Entities;
+
+namespace Application.Files.StageManyFiles.ServiceAbstractions;
+
+public interface IAddStageFileService
+{
+ Task StageAsync(
+ Stream fileStream,
+ string originalFileName,
+ string? subdirectory = "uploads",
+ CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Files/StageManyFiles/StageManyFilesCommand.cs b/etl_backend/Application/Files/StageManyFiles/StageManyFilesCommand.cs
new file mode 100644
index 00000000..c1b40f87
--- /dev/null
+++ b/etl_backend/Application/Files/StageManyFiles/StageManyFilesCommand.cs
@@ -0,0 +1,33 @@
+using Application.Common.Authorization;
+using MediatR;
+
+namespace Application.Files.Commands;
+[RequireRole(AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record StageManyFilesCommand(
+ List Files,
+ string Subdirectory = "uploads"
+) : IRequest>;
+
+public record FileUploadItem(
+ string FileName,
+ Stream Content
+);
+
+public record StageFileBatchItem(
+ string FileName,
+ bool Success,
+ string? Error = null,
+ StageFileResponse? Data = null
+);
+
+public record StageFileResponse(
+ int Id,
+ string OriginalFileName,
+ string StoredFilePath,
+ long FileSize,
+ DateTime UploadedAt,
+ string Stage,
+ string Status,
+ string? ErrorCode,
+ string? ErrorMessage
+);
\ No newline at end of file
diff --git a/etl_backend/Application/Files/StageManyFiles/StageManyFilesCommandHandler.cs b/etl_backend/Application/Files/StageManyFiles/StageManyFilesCommandHandler.cs
new file mode 100644
index 00000000..d881954c
--- /dev/null
+++ b/etl_backend/Application/Files/StageManyFiles/StageManyFilesCommandHandler.cs
@@ -0,0 +1,56 @@
+using Application.Common.Mappers;
+using Application.Files.Commands;
+using Application.Files.StageManyFiles.ServiceAbstractions;
+using MediatR;
+
+namespace Application.Files.StageManyFiles;
+public class StageManyFilesCommandHandler : IRequestHandler>
+{
+ private readonly IAddStageFileService _staging;
+ public StageManyFilesCommandHandler(IAddStageFileService staging)
+ {
+ _staging = staging;
+ }
+
+ public async Task> Handle(StageManyFilesCommand request, CancellationToken ct)
+ {
+ var results = new List(request.Files.Count);
+
+ foreach (var file in request.Files)
+ {
+ if (file.Content is null || file.Content.Length == 0)
+ {
+ results.Add(new StageFileBatchItem(
+ FileName: file.FileName,
+ Success: false,
+ Error: "Empty file."
+ ));
+ continue;
+ }
+
+ try
+ {
+ var staged = await _staging.StageAsync(file.Content, file.FileName, request.Subdirectory, ct);
+
+ results.Add(new StageFileBatchItem(
+ FileName: file.FileName,
+ Success: true,
+ Data: StageFileMapper.Map(staged)
+ ));
+ }
+ catch (OperationCanceledException)
+ {
+ throw;
+ }
+ catch (Exception ex)
+ {
+ results.Add(new StageFileBatchItem(
+ FileName: file.FileName,
+ Success: false,
+ Error: ex.Message
+ ));
+ }
+ }
+ return results;
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/DeleteTable/DeleteTableCommand.cs b/etl_backend/Application/Tables/DeleteTable/DeleteTableCommand.cs
new file mode 100644
index 00000000..a4482b22
--- /dev/null
+++ b/etl_backend/Application/Tables/DeleteTable/DeleteTableCommand.cs
@@ -0,0 +1,6 @@
+using Application.Common.Authorization;
+using MediatR;
+
+namespace Application.Tables.Commands;
+[RequireRole(AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record DeleteTableCommand(int SchemaId) : IRequest;
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/DeleteTable/DeleteTableCommandHandler.cs b/etl_backend/Application/Tables/DeleteTable/DeleteTableCommandHandler.cs
new file mode 100644
index 00000000..3f9b6953
--- /dev/null
+++ b/etl_backend/Application/Tables/DeleteTable/DeleteTableCommandHandler.cs
@@ -0,0 +1,28 @@
+using Application.Common.Exceptions;
+using Application.Tables.Commands;
+using Application.Tables.DeleteTable.ServiceAbstractions;
+using MediatR;
+
+namespace Application.Tables.DeleteTable;
+
+public class DeleteTableCommandHandler : IRequestHandler
+{
+ private readonly ITableDeleteService _svc;
+
+ public DeleteTableCommandHandler(ITableDeleteService svc)
+ {
+ _svc = svc;
+ }
+
+ public async Task Handle(DeleteTableCommand request, CancellationToken ct)
+ {
+ try
+ {
+ await _svc.DeleteAsync(request.SchemaId, ct);
+ }
+ catch (InvalidOperationException ex)
+ {
+ throw new NotFoundException("Table", request.SchemaId);
+ }
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/DeleteTable/ServiceAbstractions/ITableDeleteService.cs b/etl_backend/Application/Tables/DeleteTable/ServiceAbstractions/ITableDeleteService.cs
new file mode 100644
index 00000000..b45080cd
--- /dev/null
+++ b/etl_backend/Application/Tables/DeleteTable/ServiceAbstractions/ITableDeleteService.cs
@@ -0,0 +1,6 @@
+namespace Application.Tables.DeleteTable.ServiceAbstractions;
+
+public interface ITableDeleteService
+{
+ Task DeleteAsync(int schemaId, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/DropColumns/DropColumnCommandHandler.cs b/etl_backend/Application/Tables/DropColumns/DropColumnCommandHandler.cs
new file mode 100644
index 00000000..406f7a53
--- /dev/null
+++ b/etl_backend/Application/Tables/DropColumns/DropColumnCommandHandler.cs
@@ -0,0 +1,69 @@
+using Application.Common.Exceptions;
+using Application.Services.Repositories.Abstractions;
+using Application.Tables.Commands;
+using MediatR;
+
+namespace Application.Tables.Handlers;
+
+public class DropColumnsCommandHandler : IRequestHandler
+{
+ private readonly IDataTableSchemaRepository _schemaRepo;
+ private readonly IDataTableColumnRepository _columnRepo;
+ private readonly IColumnRepository _physicalColumnRepo;
+
+ public DropColumnsCommandHandler(
+ IDataTableSchemaRepository schemaRepo,
+ IDataTableColumnRepository columnRepo,
+ IColumnRepository physicalColumnRepo)
+ {
+ _schemaRepo = schemaRepo;
+ _columnRepo = columnRepo;
+ _physicalColumnRepo = physicalColumnRepo;
+ }
+
+ public async Task Handle(DropColumnsCommand request, CancellationToken ct)
+ {
+ if (request.ColumnIds == null || request.ColumnIds.Count == 0)
+ return;
+
+ var schema = await _schemaRepo.GetByIdWithColumnsAsync(request.SchemaId, ct)
+ ?? throw new NotFoundException("Schema", request.SchemaId);
+
+ var existingIds = schema.Columns.Select(c => c.Id).ToHashSet();
+ var notFound = request.ColumnIds.Where(id => !existingIds.Contains(id)).ToList();
+ if (notFound.Count > 0)
+ throw new UnprocessableEntityException("Some columns were not found: " + string.Join(", ", notFound));
+
+ var remaining = schema.Columns.Count - request.ColumnIds.Count;
+ if (remaining < 1)
+ throw new ConflictException("Cannot drop all columns. A table must have at least one column.");
+
+ var namesToDrop = schema.Columns
+ .Where(c => request.ColumnIds.Contains(c.Id))
+ .Select(c => c.ColumnName)
+ .ToList();
+
+ if (!await _physicalColumnRepo.TableExistsAsync("public", schema.TableName, ct))
+ throw new ConflictException("Physical table does not exist.");
+
+ try
+ {
+ await _physicalColumnRepo.DropColumnsAsync("public", schema.TableName, namesToDrop, ct);
+ }
+ catch (Exception ex)
+ {
+ throw new ApplicationException("Failed to drop columns in database.", ex);
+ }
+ try
+ {
+ await _columnRepo.DeleteByIdsAsync(request.ColumnIds, ct);
+ }
+ catch (Exception ex)
+ {
+ throw new ApplicationException(
+ $"Columns were dropped physically but metadata update failed. Schema ID: {request.SchemaId}. " +
+ $"Manual intervention required to sync metadata.",
+ ex);
+ }
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/DropColumns/DropColumnsCommand.cs b/etl_backend/Application/Tables/DropColumns/DropColumnsCommand.cs
new file mode 100644
index 00000000..0019eeae
--- /dev/null
+++ b/etl_backend/Application/Tables/DropColumns/DropColumnsCommand.cs
@@ -0,0 +1,9 @@
+using Application.Common.Authorization;
+using MediatR;
+
+namespace Application.Tables.Commands;
+[RequireRole(AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record DropColumnsCommand(
+ int SchemaId,
+ List ColumnIds
+) : IRequest;
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/GetTableDetails/GetTableDetailsQuery.cs b/etl_backend/Application/Tables/GetTableDetails/GetTableDetailsQuery.cs
new file mode 100644
index 00000000..5f6f41fe
--- /dev/null
+++ b/etl_backend/Application/Tables/GetTableDetails/GetTableDetailsQuery.cs
@@ -0,0 +1,7 @@
+using Application.Common.Authorization;
+using Application.Dtos;
+using MediatR;
+
+namespace Application.Tables.Queries;
+[RequireRole(AppRoles.Analyst, AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record GetTableDetailsQuery(int SchemaId) : IRequest;
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/GetTableDetails/GetTableDetailsQueryHandler.cs b/etl_backend/Application/Tables/GetTableDetails/GetTableDetailsQueryHandler.cs
new file mode 100644
index 00000000..9b58a56e
--- /dev/null
+++ b/etl_backend/Application/Tables/GetTableDetails/GetTableDetailsQueryHandler.cs
@@ -0,0 +1,49 @@
+using Application.Common.Exceptions;
+using Application.Dtos;
+using Application.Services.Repositories.Abstractions;
+using Application.Tables.Queries;
+using MediatR;
+
+namespace Application.Tables.Handlers;
+
+public class GetTableDetailsQueryHandler : IRequestHandler
+{
+ private readonly IDataTableSchemaRepository _schemaRepo;
+ private readonly ITableRepository _tableRepo;
+
+ public GetTableDetailsQueryHandler(
+ IDataTableSchemaRepository schemaRepo,
+ ITableRepository tableRepo)
+ {
+ _schemaRepo = schemaRepo;
+ _tableRepo = tableRepo;
+ }
+
+ public async Task Handle(GetTableDetailsQuery request, CancellationToken ct)
+ {
+ var schema = await _schemaRepo.GetByIdWithColumnsAsync(request.SchemaId, ct)
+ ?? throw new NotFoundException("Schema", request.SchemaId);
+
+ var exists = await _tableRepo.TableExistsAsync("public", schema.TableName, ct);
+ var approxRowCount = exists ? await _tableRepo.GetApproximateRowCountAsync("public", schema.TableName, ct) : 0;
+ var sizeBytes = exists ? await _tableRepo.GetTotalSizeAsync("public", schema.TableName, ct) : 0;
+
+ return new TableDetailsDto
+ {
+ SchemaId = schema.Id,
+ TableName = schema.TableName,
+ PhysicalExists = exists,
+ RowCountApprox = approxRowCount,
+ SizeBytes = sizeBytes,
+ Columns = schema.Columns
+ .OrderBy(c => c.OrdinalPosition)
+ .Select(c => new ColumnDetailsDto
+ {
+ Ordinal = c.OrdinalPosition,
+ Name = c.ColumnName,
+ Type = c.ColumnType
+ })
+ .ToList()
+ };
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/GetTableRowCount/GetTableRowCountHandler.cs b/etl_backend/Application/Tables/GetTableRowCount/GetTableRowCountHandler.cs
new file mode 100644
index 00000000..42e67868
--- /dev/null
+++ b/etl_backend/Application/Tables/GetTableRowCount/GetTableRowCountHandler.cs
@@ -0,0 +1,40 @@
+using Application.Common.Exceptions;
+using Application.Dtos;
+using Application.Services.Repositories.Abstractions;
+using Application.Tables.Queries;
+using MediatR;
+
+namespace Application.Tables.Handlers;
+
+public class GetTableRowCountQueryHandler : IRequestHandler
+{
+ private readonly IDataTableSchemaRepository _schemaRepo;
+ private readonly ITableRepository _tableRepo;
+
+ public GetTableRowCountQueryHandler(
+ IDataTableSchemaRepository schemaRepo,
+ ITableRepository tableRepo)
+ {
+ _schemaRepo = schemaRepo;
+ _tableRepo = tableRepo;
+ }
+
+ public async Task Handle(GetTableRowCountQuery request, CancellationToken ct)
+ {
+ var schema = await _schemaRepo.GetByIdWithColumnsAsync(request.SchemaId, ct)
+ ?? throw new NotFoundException("Schema", request.SchemaId);
+
+ var exists = await _tableRepo.TableExistsAsync("public", schema.TableName, ct);
+ if (!exists)
+ throw new ConflictException("Physical table does not exist.");
+
+ if (!request.Exact)
+ {
+ var approx = await _tableRepo.GetApproximateRowCountAsync("public", schema.TableName, ct);
+ return new RowCountDto { Exact = false, Count = approx };
+ }
+
+ var exact = await _tableRepo.GetExactRowCountAsync("public", schema.TableName, ct);
+ return new RowCountDto { Exact = true, Count = exact };
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/GetTableRowCount/GetTableRowCountQuery.cs b/etl_backend/Application/Tables/GetTableRowCount/GetTableRowCountQuery.cs
new file mode 100644
index 00000000..218e5d5b
--- /dev/null
+++ b/etl_backend/Application/Tables/GetTableRowCount/GetTableRowCountQuery.cs
@@ -0,0 +1,7 @@
+using Application.Common.Authorization;
+using Application.Dtos;
+using MediatR;
+
+namespace Application.Tables.Queries;
+[RequireRole(AppRoles.Analyst, AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record GetTableRowCountQuery(int SchemaId, bool Exact = false) : IRequest;
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/ListColumns/ListColumnsQuery.cs b/etl_backend/Application/Tables/ListColumns/ListColumnsQuery.cs
new file mode 100644
index 00000000..c386f3c4
--- /dev/null
+++ b/etl_backend/Application/Tables/ListColumns/ListColumnsQuery.cs
@@ -0,0 +1,14 @@
+using Application.Common.Authorization;
+using MediatR;
+
+namespace Application.Tables.Queries;
+[RequireRole(AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record ListColumnsQuery(int SchemaId) : IRequest>;
+
+public record ColumnListItem(
+ int Id,
+ int OrdinalPosition,
+ string Name,
+ string Type,
+ string? OriginalName
+);
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/ListColumns/ListColumnsQueryHandler.cs b/etl_backend/Application/Tables/ListColumns/ListColumnsQueryHandler.cs
new file mode 100644
index 00000000..5f00bfbc
--- /dev/null
+++ b/etl_backend/Application/Tables/ListColumns/ListColumnsQueryHandler.cs
@@ -0,0 +1,33 @@
+using Application.Common.Exceptions;
+using Application.Services.Repositories.Abstractions;
+using Application.Tables.Queries;
+using MediatR;
+
+namespace Application.Tables.Handlers;
+
+public class ListColumnsQueryHandler : IRequestHandler>
+{
+ private readonly IDataTableSchemaRepository _schemaRepo;
+
+ public ListColumnsQueryHandler(IDataTableSchemaRepository schemaRepo)
+ {
+ _schemaRepo = schemaRepo;
+ }
+
+ public async Task> Handle(ListColumnsQuery request, CancellationToken ct)
+ {
+ var schema = await _schemaRepo.GetByIdWithColumnsAsync(request.SchemaId, ct)
+ ?? throw new NotFoundException("Schema", request.SchemaId);
+
+ return schema.Columns
+ .OrderBy(c => c.OrdinalPosition)
+ .Select(c => new ColumnListItem(
+ c.Id,
+ c.OrdinalPosition,
+ c.ColumnName,
+ c.ColumnType,
+ c.OriginalColumnName
+ ))
+ .ToList();
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/ListTables/ListTablesQuery.cs b/etl_backend/Application/Tables/ListTables/ListTablesQuery.cs
new file mode 100644
index 00000000..b0b1eba4
--- /dev/null
+++ b/etl_backend/Application/Tables/ListTables/ListTablesQuery.cs
@@ -0,0 +1,13 @@
+using Application.Common.Authorization;
+using MediatR;
+
+namespace Application.Tables.Queries;
+[RequireRole(AppRoles.Analyst, AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record ListTablesQuery : IRequest>;
+
+public record TableListItem(
+ int SchemaId,
+ string TableName,
+ string OriginalFileName,
+ int ColumnCount
+);
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/ListTables/ListTablesQueryHandler.cs b/etl_backend/Application/Tables/ListTables/ListTablesQueryHandler.cs
new file mode 100644
index 00000000..a4cf46bc
--- /dev/null
+++ b/etl_backend/Application/Tables/ListTables/ListTablesQueryHandler.cs
@@ -0,0 +1,31 @@
+using Application.Abstractions;
+using Application.Tables.ListTables.ServiceAbstractions;
+using Application.Tables.Queries;
+using MediatR;
+
+namespace Application.Tables.Handlers;
+
+public class ListTablesQueryHandler : IRequestHandler>
+{
+ private readonly ITablesListService _svc;
+
+ public ListTablesQueryHandler(ITablesListService svc)
+ {
+ _svc = svc;
+ }
+
+ public async Task> Handle(ListTablesQuery request, CancellationToken ct)
+ {
+ var data = await _svc.ListAsync(ct);
+
+ return data
+ .OrderByDescending(s => s.Id)
+ .Select(s => new TableListItem(
+ s.Id,
+ s.TableName,
+ s.OriginalFileName ?? "",
+ s.Columns?.Count ?? 0
+ ))
+ .ToList();
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/ListTables/ServiceAbstractions/ITablesListService.cs b/etl_backend/Application/Tables/ListTables/ServiceAbstractions/ITablesListService.cs
new file mode 100644
index 00000000..2b8f3601
--- /dev/null
+++ b/etl_backend/Application/Tables/ListTables/ServiceAbstractions/ITablesListService.cs
@@ -0,0 +1,8 @@
+using Domain.Entities;
+
+namespace Application.Tables.ListTables.ServiceAbstractions;
+
+public interface ITablesListService
+{
+ Task> ListAsync(CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/PreviewTableRows/PreviewTableRowsQuery.cs b/etl_backend/Application/Tables/PreviewTableRows/PreviewTableRowsQuery.cs
new file mode 100644
index 00000000..0489476b
--- /dev/null
+++ b/etl_backend/Application/Tables/PreviewTableRows/PreviewTableRowsQuery.cs
@@ -0,0 +1,13 @@
+using Application.Common.Authorization;
+using Application.Dtos;
+using MediatR;
+
+namespace Application.Tables.Queries;
+[RequireRole(AppRoles.Analyst, AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record PreviewTableRowsQuery(
+ int SchemaId,
+ int Offset = 0,
+ int Limit = 50,
+ string? OrderBy = null,
+ string? Direction = null
+) : IRequest;
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/PreviewTableRows/PreviewTableRowsQueryHandler.cs b/etl_backend/Application/Tables/PreviewTableRows/PreviewTableRowsQueryHandler.cs
new file mode 100644
index 00000000..697b8c7f
--- /dev/null
+++ b/etl_backend/Application/Tables/PreviewTableRows/PreviewTableRowsQueryHandler.cs
@@ -0,0 +1,45 @@
+using Application.Common.Exceptions;
+using Application.Dtos;
+using Application.Services.Repositories.Abstractions;
+using Application.Tables.Queries;
+using MediatR;
+
+namespace Application.Tables.Handlers;
+
+public class PreviewTableRowsQueryHandler : IRequestHandler
+{
+ private readonly IDataTableSchemaRepository _schemaRepo;
+ private readonly ITableRepository _tableRepo;
+
+ public PreviewTableRowsQueryHandler(
+ IDataTableSchemaRepository schemaRepo,
+ ITableRepository tableRepo)
+ {
+ _schemaRepo = schemaRepo;
+ _tableRepo = tableRepo;
+ }
+
+ public async Task Handle(PreviewTableRowsQuery request, CancellationToken ct)
+ {
+ var schema = await _schemaRepo.GetByIdWithColumnsAsync(request.SchemaId, ct)
+ ?? throw new NotFoundException("Schema", request.SchemaId);
+
+ var exists = await _tableRepo.TableExistsAsync("public", schema.TableName, ct);
+ if (!exists)
+ throw new ConflictException("Physical table does not exist.");
+
+ var limit = Math.Clamp(request.Limit, 1, 200);
+ var offset = Math.Max(request.Offset, 0);
+
+ return await _tableRepo.PreviewRowsAsync(
+ "public",
+ schema.TableName,
+ schema.Columns.OrderBy(c => c.OrdinalPosition).ToList(),
+ offset,
+ limit,
+ request.OrderBy,
+ request.Direction,
+ ct
+ );
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/RenameColumn/RenameColumnCommadHandler.cs b/etl_backend/Application/Tables/RenameColumn/RenameColumnCommadHandler.cs
new file mode 100644
index 00000000..9005859c
--- /dev/null
+++ b/etl_backend/Application/Tables/RenameColumn/RenameColumnCommadHandler.cs
@@ -0,0 +1,65 @@
+using Application.Abstractions;
+using Application.Common.Exceptions;
+using Application.Services.Repositories.Abstractions;
+using Application.Tables.Commands;
+using MediatR;
+
+namespace Application.Tables.Handlers;
+
+public class RenameColumnCommandHandler : IRequestHandler
+{
+ private readonly IDataTableSchemaRepository _schemaRepo;
+ private readonly IDataTableColumnRepository _columnRepo;
+ private readonly IColumnRepository _physicalColumnRepo;
+ private readonly IColumnNameSanitizer _sanitizer;
+
+ public RenameColumnCommandHandler(
+ IDataTableSchemaRepository schemaRepo,
+ IDataTableColumnRepository columnRepo,
+ IColumnRepository physicalColumnRepo,
+ IColumnNameSanitizer sanitizer)
+ {
+ _schemaRepo = schemaRepo;
+ _columnRepo = columnRepo;
+ _physicalColumnRepo = physicalColumnRepo;
+ _sanitizer = sanitizer;
+ }
+
+ public async Task Handle(RenameColumnCommand request, CancellationToken ct)
+ {
+ if (string.IsNullOrWhiteSpace(request.NewName))
+ throw new UnprocessableEntityException("New column name is required.");
+
+ var schema = await _schemaRepo.GetByIdWithColumnsAsync(request.SchemaId, ct)
+ ?? throw new NotFoundException("Schema", request.SchemaId);
+
+ var column = schema.Columns.FirstOrDefault(c => c.Id == request.ColumnId)
+ ?? throw new NotFoundException("Column", request.ColumnId);
+ var maxLen = 63;
+ var sanitized = _sanitizer.Sanitize(request.NewName.Trim(), maxLen);
+ if (string.IsNullOrWhiteSpace(sanitized))
+ throw new UnprocessableEntityException("Sanitized name is empty.");
+ if (schema.Columns.Any(c => c.Id != request.ColumnId &&
+ string.Equals(c.ColumnName, sanitized, StringComparison.OrdinalIgnoreCase)))
+ throw new UnprocessableEntityException($"A column with name '{sanitized}' already exists.");
+ if (!await _physicalColumnRepo.TableExistsAsync("public", schema.TableName, ct))
+ throw new ConflictException("Physical table does not exist.");
+
+ var oldName = column.ColumnName;
+
+ try
+ {
+ await _physicalColumnRepo.RenameColumnAsync("public", schema.TableName, oldName, sanitized, ct);
+ await _columnRepo.UpdateNameAsync(request.ColumnId, sanitized, ct);
+ }
+ catch
+ {
+ try
+ {
+ await _physicalColumnRepo.RenameColumnAsync("public", schema.TableName, sanitized, oldName, ct);
+ }
+ catch { }
+ throw;
+ }
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/RenameColumn/RenameColumnCommand.cs b/etl_backend/Application/Tables/RenameColumn/RenameColumnCommand.cs
new file mode 100644
index 00000000..db2f4fa3
--- /dev/null
+++ b/etl_backend/Application/Tables/RenameColumn/RenameColumnCommand.cs
@@ -0,0 +1,10 @@
+using Application.Common.Authorization;
+using MediatR;
+
+namespace Application.Tables.Commands;
+[RequireRole(AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record RenameColumnCommand(
+ int SchemaId,
+ int ColumnId,
+ string NewName
+) : IRequest;
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/RenameTable/RenameTableCommand.cs b/etl_backend/Application/Tables/RenameTable/RenameTableCommand.cs
new file mode 100644
index 00000000..ed74e089
--- /dev/null
+++ b/etl_backend/Application/Tables/RenameTable/RenameTableCommand.cs
@@ -0,0 +1,9 @@
+using Application.Common.Authorization;
+using MediatR;
+
+namespace Application.Tables.Commands;
+[RequireRole(AppRoles.DataAdmin, AppRoles.SysAdmin)]
+public record RenameTableCommand(
+ int SchemaId,
+ string NewTableName
+) : IRequest;
diff --git a/etl_backend/Application/Tables/RenameTable/RenameTableCommandHandler.cs b/etl_backend/Application/Tables/RenameTable/RenameTableCommandHandler.cs
new file mode 100644
index 00000000..5815686c
--- /dev/null
+++ b/etl_backend/Application/Tables/RenameTable/RenameTableCommandHandler.cs
@@ -0,0 +1,33 @@
+using Application.Abstractions;
+using Application.Common.Exceptions;
+using Application.Tables.Commands;
+using Application.Tables.RenameTable.ServiceAbstractions;
+using MediatR;
+
+namespace Application.Tables.Handlers;
+
+public class RenameTableCommandHandler : IRequestHandler
+{
+ private readonly ITableRenameService _svc;
+
+ public RenameTableCommandHandler(ITableRenameService svc)
+ {
+ _svc = svc;
+ }
+
+ public async Task Handle(RenameTableCommand request, CancellationToken ct)
+ {
+ try
+ {
+ await _svc.RenameAsync(request.SchemaId, request.NewTableName, ct);
+ }
+ catch (ArgumentException ex)
+ {
+ throw new UnprocessableEntityException(ex.Message);
+ }
+ catch (InvalidOperationException ex)
+ {
+ throw new NotFoundException("Table", request.SchemaId);
+ }
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Tables/RenameTable/ServiceAbstractions/ITableRenameService.cs b/etl_backend/Application/Tables/RenameTable/ServiceAbstractions/ITableRenameService.cs
new file mode 100644
index 00000000..8a61f39c
--- /dev/null
+++ b/etl_backend/Application/Tables/RenameTable/ServiceAbstractions/ITableRenameService.cs
@@ -0,0 +1,6 @@
+namespace Application.Tables.RenameTable.ServiceAbstractions;
+
+public interface ITableRenameService
+{
+ Task RenameAsync(int schemaId, string newTableName, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/CreateUser/CreateUserCommand.cs b/etl_backend/Application/Users/CreateUser/CreateUserCommand.cs
new file mode 100644
index 00000000..21b84860
--- /dev/null
+++ b/etl_backend/Application/Users/CreateUser/CreateUserCommand.cs
@@ -0,0 +1,14 @@
+using Application.Common.Authorization;
+using Application.Dtos;
+using MediatR;
+
+namespace Application.Users.Commands;
+
+[RequireRole(AppRoles.SysAdmin)]
+public record CreateUserCommand(
+ string Username,
+ string? Email = null,
+ string? FirstName = null,
+ string? LastName = null,
+ string? Password = null
+) : IRequest;
\ No newline at end of file
diff --git a/etl_backend/Application/Users/CreateUser/CreateUserCommandHandler.cs b/etl_backend/Application/Users/CreateUser/CreateUserCommandHandler.cs
new file mode 100644
index 00000000..50abe6d5
--- /dev/null
+++ b/etl_backend/Application/Users/CreateUser/CreateUserCommandHandler.cs
@@ -0,0 +1,36 @@
+using Application.Common.Exceptions;
+using Application.Dtos;
+using Application.Services.Abstractions;
+using Application.Users.Commands;
+using Application.Users.CreateUser.ServiceAbstractions;
+using MediatR;
+
+namespace Application.Users.Handlers;
+
+public class CreateUserCommandHandler : IRequestHandler
+{
+ private readonly ICreateUser _createUser;
+
+ public CreateUserCommandHandler(ICreateUser createUser)
+ {
+ _createUser = createUser;
+ }
+
+ public async Task Handle(CreateUserCommand request, CancellationToken ct)
+ {
+ if (string.IsNullOrWhiteSpace(request.Username))
+ throw new UnprocessableEntityException("Username is required.");
+
+ var newUser = new UserCreateDto
+ {
+ Username = request.Username,
+ Email = request.Email,
+ FirstName = request.FirstName,
+ LastName = request.LastName,
+ Password = request.Password
+ };
+
+ var createdUser = await _createUser.CreateUserAsync(newUser, ct);
+ return createdUser;
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/CreateUser/ServiceAbstractions/ICreateUser.cs b/etl_backend/Application/Users/CreateUser/ServiceAbstractions/ICreateUser.cs
new file mode 100644
index 00000000..4e004672
--- /dev/null
+++ b/etl_backend/Application/Users/CreateUser/ServiceAbstractions/ICreateUser.cs
@@ -0,0 +1,8 @@
+using Application.Dtos;
+
+namespace Application.Users.CreateUser.ServiceAbstractions;
+
+public interface ICreateUser
+{
+ Task CreateUserAsync(UserCreateDto newUser, CancellationToken cancellationToken);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/DeleteUser/DeleteUserCommand.cs b/etl_backend/Application/Users/DeleteUser/DeleteUserCommand.cs
new file mode 100644
index 00000000..b0132641
--- /dev/null
+++ b/etl_backend/Application/Users/DeleteUser/DeleteUserCommand.cs
@@ -0,0 +1,7 @@
+using Application.Common.Authorization;
+using MediatR;
+
+namespace Application.Users.Commands;
+
+[RequireRole(AppRoles.SysAdmin)]
+public record DeleteUserCommand(string UserId) : IRequest;
\ No newline at end of file
diff --git a/etl_backend/Application/Users/DeleteUser/DeleteUserCommandHandler.cs b/etl_backend/Application/Users/DeleteUser/DeleteUserCommandHandler.cs
new file mode 100644
index 00000000..b66bca93
--- /dev/null
+++ b/etl_backend/Application/Users/DeleteUser/DeleteUserCommandHandler.cs
@@ -0,0 +1,32 @@
+using Application.Common.Exceptions;
+using Application.Services.Abstractions;
+using Application.Users.Commands;
+using Application.Users.DeleteUser.ServiceAbstractions;
+using Application.Users.GetUserById.ServiceAbstractions;
+using MediatR;
+
+namespace Application.Users.Handlers;
+
+public class DeleteUserCommandHandler : IRequestHandler
+{
+ private readonly IDeleteUserService _deleteUserService;
+ private readonly IGetUserByIdService _getUserByIdService;
+
+ public DeleteUserCommandHandler(IDeleteUserService deleteUserService, IGetUserByIdService getUserByIdService)
+ {
+ _deleteUserService = deleteUserService;
+ _getUserByIdService = getUserByIdService;
+ }
+
+ public async Task Handle(DeleteUserCommand request, CancellationToken ct)
+ {
+ if (string.IsNullOrWhiteSpace(request.UserId))
+ throw new UnprocessableEntityException("UserId is required.");
+
+ var existingUser = await _getUserByIdService.GetUserByIdAsync(request.UserId, ct);
+ if (existingUser == null)
+ throw new NotFoundException("User", request.UserId);
+
+ await _deleteUserService.DeleteUserAsync(request.UserId, ct);
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/DeleteUser/ServiceAbstractions/IDeleteUserService.cs b/etl_backend/Application/Users/DeleteUser/ServiceAbstractions/IDeleteUserService.cs
new file mode 100644
index 00000000..619e7c64
--- /dev/null
+++ b/etl_backend/Application/Users/DeleteUser/ServiceAbstractions/IDeleteUserService.cs
@@ -0,0 +1,6 @@
+namespace Application.Users.DeleteUser.ServiceAbstractions;
+
+public interface IDeleteUserService
+{
+ Task DeleteUserAsync(string userId, CancellationToken cancellationToken);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/EditUser/EditUserCommand.cs b/etl_backend/Application/Users/EditUser/EditUserCommand.cs
new file mode 100644
index 00000000..586236c2
--- /dev/null
+++ b/etl_backend/Application/Users/EditUser/EditUserCommand.cs
@@ -0,0 +1,14 @@
+using Application.Common.Authorization;
+using Application.Dtos;
+using MediatR;
+
+namespace Application.Users.Commands;
+
+[RequireRole(AppRoles.SysAdmin, AppRoles.Analyst, AppRoles.DataAdmin)]
+public record EditUserCommand(
+ string UserId,
+ string? Username = null,
+ string? Email = null,
+ string? FirstName = null,
+ string? LastName = null
+) : IRequest;
\ No newline at end of file
diff --git a/etl_backend/Application/Users/EditUser/EditUserCommandHandler.cs b/etl_backend/Application/Users/EditUser/EditUserCommandHandler.cs
new file mode 100644
index 00000000..ec65b4d6
--- /dev/null
+++ b/etl_backend/Application/Users/EditUser/EditUserCommandHandler.cs
@@ -0,0 +1,42 @@
+using Application.Common.Exceptions;
+using Application.Dtos;
+using Application.Services.Abstractions;
+using Application.Users.Commands;
+using Application.Users.EditUser.ServiceAbstractions;
+using Application.Users.GetUserById.ServiceAbstractions;
+using MediatR;
+
+namespace Application.Users.Handlers;
+
+public class EditUserCommandHandler : IRequestHandler
+{
+ private readonly IAdminEditUserService _adminEditUserService;
+ private readonly IGetUserByIdService _getUserByIdService;
+
+ public EditUserCommandHandler(IAdminEditUserService adminEditUserService, IGetUserByIdService getUserByIdService)
+ {
+ _adminEditUserService = adminEditUserService;
+ _getUserByIdService = getUserByIdService;
+ }
+
+ public async Task Handle(EditUserCommand request, CancellationToken ct)
+ {
+ if (string.IsNullOrWhiteSpace(request.UserId))
+ throw new UnprocessableEntityException("UserId is required.");
+
+ // Optional: Validate user exists
+ var existingUser = await _getUserByIdService.GetUserByIdAsync(request.UserId, ct);
+ if (existingUser == null)
+ throw new NotFoundException("User", request.UserId);
+
+ var editRequest = new EditUserRequestDto
+ {
+ Email = request.Email,
+ FirstName = request.FirstName,
+ LastName = request.LastName
+ };
+
+ var updatedUser = await _adminEditUserService.EditUserAsync(request.UserId, editRequest, ct);
+ return updatedUser;
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/EditUser/ServiceAbstractions/IAdminEditUserService.cs b/etl_backend/Application/Users/EditUser/ServiceAbstractions/IAdminEditUserService.cs
new file mode 100644
index 00000000..2ee28454
--- /dev/null
+++ b/etl_backend/Application/Users/EditUser/ServiceAbstractions/IAdminEditUserService.cs
@@ -0,0 +1,8 @@
+using Application.Dtos;
+
+namespace Application.Users.EditUser.ServiceAbstractions;
+
+public interface IAdminEditUserService
+{
+ Task EditUserAsync(string userId, EditUserRequestDto userToUpdate, CancellationToken cancellationToken);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/EditUserRoles/EditUserRolesCommand.cs b/etl_backend/Application/Users/EditUserRoles/EditUserRolesCommand.cs
new file mode 100644
index 00000000..c3f0bc75
--- /dev/null
+++ b/etl_backend/Application/Users/EditUserRoles/EditUserRolesCommand.cs
@@ -0,0 +1,12 @@
+using Application.Common.Authorization;
+using Application.Dtos;
+using MediatR;
+
+namespace Application.Users.Commands;
+
+[RequireRole(AppRoles.SysAdmin)]
+public record EditUserRolesCommand(
+ string UserId,
+ List RolesToAdd,
+ List RolesToRemove
+) : IRequest;
\ No newline at end of file
diff --git a/etl_backend/Application/Users/EditUserRoles/EditUserRolesCommandHandler.cs b/etl_backend/Application/Users/EditUserRoles/EditUserRolesCommandHandler.cs
new file mode 100644
index 00000000..fbb93caa
--- /dev/null
+++ b/etl_backend/Application/Users/EditUserRoles/EditUserRolesCommandHandler.cs
@@ -0,0 +1,56 @@
+using Application.Common.Exceptions;
+using Application.Services.Abstractions;
+using Application.Users.Commands;
+using Application.Users.GetAllRoles.ServiceAbstractions;
+using Application.Users.GetUserById.ServiceAbstractions;
+using MediatR;
+
+namespace Application.Users.Handlers;
+
+public class EditUserRolesCommandHandler : IRequestHandler
+{
+ private readonly IUserRoleManagementService _userRoleManagementService;
+ private readonly IGetUserByIdService _getUserByIdService;
+ private readonly IGetAllRoles _getAllRolesService;
+
+ public EditUserRolesCommandHandler(IUserRoleManagementService userRoleManagementService, IGetUserByIdService getUserByIdService, IGetAllRoles getAllRolesService)
+ {
+ _userRoleManagementService = userRoleManagementService;
+ _getUserByIdService = getUserByIdService;
+ _getAllRolesService = getAllRolesService;
+ }
+
+ public async Task Handle(EditUserRolesCommand request, CancellationToken ct)
+ {
+ if (string.IsNullOrWhiteSpace(request.UserId))
+ throw new UnprocessableEntityException("UserId is required.");
+
+ var currentUser = await _getUserByIdService.GetUserByIdAsync(request.UserId, ct);
+ if (currentUser == null)
+ throw new NotFoundException("User", request.UserId);
+
+ if (request.RolesToAdd.Any())
+ {
+ var allRoles = await _getAllRolesService.GetAllRolesAsync(ct);
+ var allRoleNames = allRoles.Select(r => r.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
+
+ var invalidRoles = request.RolesToAdd
+ .Where(r => !allRoleNames.Contains(r.Name))
+ .Select(r => r.Name)
+ .ToList();
+
+ if (invalidRoles.Count > 0)
+ throw new UnprocessableEntityException($"Invalid roles to add: {string.Join(", ", invalidRoles)}");
+ }
+
+ if (request.RolesToAdd.Any())
+ {
+ await _userRoleManagementService.AddRolesToUserAsync(request.UserId, request.RolesToAdd.ToArray(), ct);
+ }
+
+ if (request.RolesToRemove.Any())
+ {
+ await _userRoleManagementService.RemoveRolesFromUserAsync(request.UserId, request.RolesToRemove, ct);
+ }
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/GetAllRoles/GetAllRolesQuery.cs b/etl_backend/Application/Users/GetAllRoles/GetAllRolesQuery.cs
new file mode 100644
index 00000000..284bf1e6
--- /dev/null
+++ b/etl_backend/Application/Users/GetAllRoles/GetAllRolesQuery.cs
@@ -0,0 +1,8 @@
+using Application.Common.Authorization;
+using Application.Dtos;
+using MediatR;
+
+namespace Application.Users.Queries;
+
+[RequireRole(AppRoles.SysAdmin, AppRoles.DataAdmin, AppRoles.Analyst)]
+public record GetAllRolesQuery : IRequest>;
\ No newline at end of file
diff --git a/etl_backend/Application/Users/GetAllRoles/GetAllRolesQueryHandler.cs b/etl_backend/Application/Users/GetAllRoles/GetAllRolesQueryHandler.cs
new file mode 100644
index 00000000..ea4d6f17
--- /dev/null
+++ b/etl_backend/Application/Users/GetAllRoles/GetAllRolesQueryHandler.cs
@@ -0,0 +1,23 @@
+using Application.Dtos;
+using Application.Services.Abstractions;
+using Application.Users.GetAllRoles.ServiceAbstractions;
+using Application.Users.Queries;
+using MediatR;
+
+namespace Application.Users.Handlers;
+
+public class GetAllRolesQueryHandler : IRequestHandler>
+{
+ private readonly IGetAllRoles _allRolesManager;
+
+ public GetAllRolesQueryHandler(IGetAllRoles userRoleManagementService)
+ {
+ _allRolesManager = userRoleManagementService;
+ }
+
+ public async Task> Handle(GetAllRolesQuery request, CancellationToken ct)
+ {
+ var roles = await _allRolesManager.GetAllRolesAsync(ct);
+ return roles.ToList();
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/GetAllRoles/ServiceAbstractions/IGetAllRoles.cs b/etl_backend/Application/Users/GetAllRoles/ServiceAbstractions/IGetAllRoles.cs
new file mode 100644
index 00000000..6f267d51
--- /dev/null
+++ b/etl_backend/Application/Users/GetAllRoles/ServiceAbstractions/IGetAllRoles.cs
@@ -0,0 +1,8 @@
+using Application.Dtos;
+
+namespace Application.Users.GetAllRoles.ServiceAbstractions;
+
+public interface IGetAllRoles
+{
+ Task> GetAllRolesAsync(CancellationToken cancellationToken);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/GetUserById/GetUserByIdQuery.cs b/etl_backend/Application/Users/GetUserById/GetUserByIdQuery.cs
new file mode 100644
index 00000000..30e86b5e
--- /dev/null
+++ b/etl_backend/Application/Users/GetUserById/GetUserByIdQuery.cs
@@ -0,0 +1,8 @@
+using Application.Common.Authorization;
+using Application.Dtos;
+using MediatR;
+
+namespace Application.Users.Queries;
+
+[RequireRole(AppRoles.SysAdmin)]
+public record GetUserByIdQuery(string UserId) : IRequest;
\ No newline at end of file
diff --git a/etl_backend/Application/Users/GetUserById/GetUserByIdQueryHandler.cs b/etl_backend/Application/Users/GetUserById/GetUserByIdQueryHandler.cs
new file mode 100644
index 00000000..79a31d6c
--- /dev/null
+++ b/etl_backend/Application/Users/GetUserById/GetUserByIdQueryHandler.cs
@@ -0,0 +1,31 @@
+using Application.Common.Authorization;
+using Application.Common.Exceptions;
+using Application.Dtos;
+using Application.Services.Abstractions;
+using Application.Users.GetUserById.ServiceAbstractions;
+using Application.Users.Queries;
+using MediatR;
+
+namespace Application.Users.Handlers;
+
+public class GetUserByIdQueryHandler : IRequestHandler
+{
+ private readonly IGetUserByIdService _userManagementService;
+
+ public GetUserByIdQueryHandler(IGetUserByIdService userManagementService)
+ {
+ _userManagementService = userManagementService;
+ }
+
+ public async Task Handle(GetUserByIdQuery request, CancellationToken ct)
+ {
+ if (string.IsNullOrWhiteSpace(request.UserId))
+ throw new UnprocessableEntityException("UserId is required.");
+
+ var user = await _userManagementService.GetUserByIdAsync(request.UserId, ct);
+ if (user == null)
+ throw new NotFoundException("User", request.UserId);
+
+ return user;
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/GetUserById/ServiceAbstractions/IGetUserByIdService.cs b/etl_backend/Application/Users/GetUserById/ServiceAbstractions/IGetUserByIdService.cs
new file mode 100644
index 00000000..2571b821
--- /dev/null
+++ b/etl_backend/Application/Users/GetUserById/ServiceAbstractions/IGetUserByIdService.cs
@@ -0,0 +1,9 @@
+using Application.Dtos;
+
+namespace Application.Users.GetUserById.ServiceAbstractions;
+
+public interface IGetUserByIdService
+{
+ Task GetUserByIdAsync(string userId, CancellationToken cancellationToken);
+
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/ListUsers/IListUsersService.cs b/etl_backend/Application/Users/ListUsers/IListUsersService.cs
new file mode 100644
index 00000000..e272f09d
--- /dev/null
+++ b/etl_backend/Application/Users/ListUsers/IListUsersService.cs
@@ -0,0 +1,8 @@
+using Application.Dtos;
+
+namespace Application.Users.ListUsers;
+
+public interface IListUsersService
+{
+ Task> GetAllUsersAsync(CancellationToken cancellationToken);
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/ListUsers/ListUserQueryHandler.cs b/etl_backend/Application/Users/ListUsers/ListUserQueryHandler.cs
new file mode 100644
index 00000000..29ff0bbf
--- /dev/null
+++ b/etl_backend/Application/Users/ListUsers/ListUserQueryHandler.cs
@@ -0,0 +1,23 @@
+using Application.Dtos;
+using Application.Services.Abstractions;
+using Application.Users.ListUsers;
+using Application.Users.Queries;
+using MediatR;
+
+namespace Application.Users.Handlers;
+
+public class ListUsersQueryHandler : IRequestHandler>
+{
+ private readonly IListUsersService _userManagementService;
+
+ public ListUsersQueryHandler(IListUsersService userManagementService)
+ {
+ _userManagementService = userManagementService;
+ }
+
+ public async Task> Handle(ListUsersQuery request, CancellationToken ct)
+ {
+ var users = await _userManagementService.GetAllUsersAsync(ct);
+ return users.ToList();
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Application/Users/ListUsers/ListUsersQuery.cs b/etl_backend/Application/Users/ListUsers/ListUsersQuery.cs
new file mode 100644
index 00000000..9edee5df
--- /dev/null
+++ b/etl_backend/Application/Users/ListUsers/ListUsersQuery.cs
@@ -0,0 +1,8 @@
+using Application.Common.Authorization;
+using Application.Dtos;
+using MediatR;
+
+namespace Application.Users.Queries;
+
+[RequireRole(AppRoles.SysAdmin)]
+public record ListUsersQuery : IRequest>;
diff --git a/etl_backend/Domain/AccessControl/Events/UserAuthenticated.cs b/etl_backend/Domain/AccessControl/Events/UserAuthenticated.cs
new file mode 100644
index 00000000..e69de29b
diff --git a/etl_backend/Domain/AccessControl/Events/UserLoggedOut.cs b/etl_backend/Domain/AccessControl/Events/UserLoggedOut.cs
new file mode 100644
index 00000000..e69de29b
diff --git a/etl_backend/etl_backend/etl_backend.csproj b/etl_backend/Domain/Domain.csproj
similarity index 52%
rename from etl_backend/etl_backend/etl_backend.csproj
rename to etl_backend/Domain/Domain.csproj
index ba1446fb..b3c0ed0b 100644
--- a/etl_backend/etl_backend/etl_backend.csproj
+++ b/etl_backend/Domain/Domain.csproj
@@ -1,14 +1,13 @@
-
+
net8.0
- enable
enable
+ enable
-
diff --git a/etl_backend/Domain/Entities/DataTableColumn.cs b/etl_backend/Domain/Entities/DataTableColumn.cs
new file mode 100644
index 00000000..f24b6398
--- /dev/null
+++ b/etl_backend/Domain/Entities/DataTableColumn.cs
@@ -0,0 +1,15 @@
+namespace Domain.Entities;
+
+public class DataTableColumn
+{
+ public int Id { get; set; }
+
+ public required string ColumnName { get; set; }
+ public string? OriginalColumnName { get; set; }
+ public int OrdinalPosition { get; set; }
+
+ public string ColumnType { get; set; } = "string"; // default all strings
+
+ public int DataTableSchemaId { get; set; }
+ public DataTableSchema? DataTable { get; set; }
+}
\ No newline at end of file
diff --git a/etl_backend/Domain/Entities/DataTableSchema.cs b/etl_backend/Domain/Entities/DataTableSchema.cs
new file mode 100644
index 00000000..536c1c0e
--- /dev/null
+++ b/etl_backend/Domain/Entities/DataTableSchema.cs
@@ -0,0 +1,12 @@
+namespace Domain.Entities;
+
+
+public class DataTableSchema
+{
+ public int Id { get; set; }
+ public string TableName { get; set; } = string.Empty;
+
+ public string? OriginalFileName { get; set; }
+
+ public List Columns { get; set; } = new();
+}
\ No newline at end of file
diff --git a/etl_backend/Domain/Entities/IMyClock.cs b/etl_backend/Domain/Entities/IMyClock.cs
new file mode 100644
index 00000000..6cac8c93
--- /dev/null
+++ b/etl_backend/Domain/Entities/IMyClock.cs
@@ -0,0 +1,6 @@
+namespace Domain.Entities;
+
+public interface IClock
+{
+ DateTime UtcNow { get; }
+}
\ No newline at end of file
diff --git a/etl_backend/Domain/Entities/StagedFile.cs b/etl_backend/Domain/Entities/StagedFile.cs
new file mode 100644
index 00000000..4c7927ed
--- /dev/null
+++ b/etl_backend/Domain/Entities/StagedFile.cs
@@ -0,0 +1,28 @@
+using Domain.Enums;
+
+namespace Domain.Entities;
+
+
+public class StagedFile
+{
+ public int Id { get; set; }
+
+
+ public required string OriginalFileName { get; set; }
+
+ public required string StoredFilePath { get; set; }
+ public long FileSize { get; set; }
+
+ public DateTime UploadedAt { get; set; } = DateTime.UtcNow;
+
+ public ProcessingStage Stage { get; set; } = ProcessingStage.None;
+ public ProcessingStatus Status { get; set; } = ProcessingStatus.InProgress;
+ public ProcessingErrorCode ErrorCode { get; set; } = ProcessingErrorCode.None;
+
+
+ public string? ErrorMessage { get; set; } // in case staging/validation fails
+
+ public int? SchemaId { get; set; }
+
+ public DataTableSchema? Schema { get; set; }
+}
\ No newline at end of file
diff --git a/etl_backend/Domain/Enums/ProcessingErrorCode.cs b/etl_backend/Domain/Enums/ProcessingErrorCode.cs
new file mode 100644
index 00000000..aec324a5
--- /dev/null
+++ b/etl_backend/Domain/Enums/ProcessingErrorCode.cs
@@ -0,0 +1,13 @@
+namespace Domain.Enums;
+
+public enum ProcessingErrorCode
+{
+ None = 0,
+ StorageSaveFailed,
+ StagingDbWriteFailed,
+ SchemaRegistrationFailed,
+ SchemaDbWriteFailed,
+ CreateTableFailed,
+ LoadFailed,
+ ValidationFailed
+}
\ No newline at end of file
diff --git a/etl_backend/Domain/Enums/ProcessingStage.cs b/etl_backend/Domain/Enums/ProcessingStage.cs
new file mode 100644
index 00000000..b50f6ab9
--- /dev/null
+++ b/etl_backend/Domain/Enums/ProcessingStage.cs
@@ -0,0 +1,10 @@
+namespace Domain.Enums;
+
+public enum ProcessingStage
+{
+ None = 0,
+ Uploaded, // file stored on disk
+ SchemaRegistered, // schema+columns persisted, linked
+ TableCreated,
+ Loaded // rows inserted
+}
\ No newline at end of file
diff --git a/etl_backend/Domain/Enums/ProcessingStatus.cs b/etl_backend/Domain/Enums/ProcessingStatus.cs
new file mode 100644
index 00000000..212778f3
--- /dev/null
+++ b/etl_backend/Domain/Enums/ProcessingStatus.cs
@@ -0,0 +1,8 @@
+namespace Domain.Enums;
+
+public enum ProcessingStatus
+{
+ InProgress = 0,
+ Succeeded,
+ Failed
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/AppEnviroment.cs b/etl_backend/Infrastructure/AppEnviroment.cs
new file mode 100644
index 00000000..e7a7539b
--- /dev/null
+++ b/etl_backend/Infrastructure/AppEnviroment.cs
@@ -0,0 +1,22 @@
+namespace Infrastructure;
+
+public enum AppEnvironment
+{
+ Development,
+ Test,
+ Production,
+ Unknown
+}
+
+public static class AppEnvironmentExtensions
+{
+ public static AppEnvironment ToAppEnvironment(this string? environmentName) =>
+ environmentName?.ToLowerInvariant() switch
+ {
+ "development" => AppEnvironment.Development,
+ "test" => AppEnvironment.Test,
+ "production" => AppEnvironment.Production,
+ _ => AppEnvironment.Unknown
+ };
+
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Configurations/CsvEncodings.cs b/etl_backend/Infrastructure/Configurations/CsvEncodings.cs
new file mode 100644
index 00000000..ee6fa977
--- /dev/null
+++ b/etl_backend/Infrastructure/Configurations/CsvEncodings.cs
@@ -0,0 +1,22 @@
+using System.Text;
+
+namespace Infrastructure.Configurations;
+
+public enum CsvEncoding
+{
+ Utf8,
+ Ascii,
+ Latin1
+}
+
+public static class CsvEncodingExtensions
+{
+ public static Encoding ToSystemEncoding(this CsvEncoding encoding) =>
+ encoding switch
+ {
+ CsvEncoding.Utf8 => Encoding.UTF8,
+ CsvEncoding.Ascii => Encoding.ASCII,
+ CsvEncoding.Latin1 => Encoding.Latin1,
+ _ => throw new ArgumentOutOfRangeException(nameof(encoding))
+ };
+}
diff --git a/etl_backend/Infrastructure/Configurations/CsvStagingOptions.cs b/etl_backend/Infrastructure/Configurations/CsvStagingOptions.cs
new file mode 100644
index 00000000..b6c14701
--- /dev/null
+++ b/etl_backend/Infrastructure/Configurations/CsvStagingOptions.cs
@@ -0,0 +1,31 @@
+namespace Infrastructure.Configurations;
+
+public class CsvStagingOptions
+{
+ ///
+ /// Character used to separate values (e.g. ',', ';', '\t').
+ ///
+ public char Delimiter { get; set; } = ',';
+
+ ///
+ /// Character used to quote values that contain delimiters.
+ ///
+ public char QuoteChar { get; set; } = '"';
+
+ ///
+ /// Whether the first row contains column headers.
+ ///
+ public bool HasHeader { get; set; } = true;
+
+ ///
+ /// Whether to trim whitespace around values.
+ ///
+ public bool TrimWhitespace { get; set; } = true;
+
+ ///
+ /// Encoding of the CSV file (default UTF-8).
+ ///
+ public CsvEncoding Encoding { get; set; } = CsvEncoding.Utf8;
+
+}
+
diff --git a/etl_backend/Infrastructure/Configurations/PostgresStoreOptions.cs b/etl_backend/Infrastructure/Configurations/PostgresStoreOptions.cs
new file mode 100644
index 00000000..67ef5d01
--- /dev/null
+++ b/etl_backend/Infrastructure/Configurations/PostgresStoreOptions.cs
@@ -0,0 +1,6 @@
+namespace Infrastructure.Configurations;
+
+public sealed class PostgresStoreOptions
+{
+ public string DefaultSchema { get; set; } = "public";
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Configurations/StorageSettings.cs b/etl_backend/Infrastructure/Configurations/StorageSettings.cs
new file mode 100644
index 00000000..4df1d777
--- /dev/null
+++ b/etl_backend/Infrastructure/Configurations/StorageSettings.cs
@@ -0,0 +1,7 @@
+namespace Infrastructure.Configurations;
+
+public class StorageSettings
+{
+ public required string Root { get; set; }
+ public required string BaseUrl { get; set; }
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Configurations/SystemClockAdaptor.cs b/etl_backend/Infrastructure/Configurations/SystemClockAdaptor.cs
new file mode 100644
index 00000000..96dd0e72
--- /dev/null
+++ b/etl_backend/Infrastructure/Configurations/SystemClockAdaptor.cs
@@ -0,0 +1,5 @@
+using Domain.Entities;
+
+namespace Infrastructure.Configurations;
+
+public sealed class SystemClock : IClock { public DateTime UtcNow => DateTime.UtcNow; }
diff --git a/etl_backend/Infrastructure/DbConfig/Abstraction/ICommonDbContext.cs b/etl_backend/Infrastructure/DbConfig/Abstraction/ICommonDbContext.cs
new file mode 100644
index 00000000..5a0ec6d6
--- /dev/null
+++ b/etl_backend/Infrastructure/DbConfig/Abstraction/ICommonDbContext.cs
@@ -0,0 +1,8 @@
+namespace Infrastructure.DbConfig.Abstraction;
+
+public interface ICommonDbContext : IAsyncDisposable, IDisposable
+{
+ public int SaveChanges();
+ Task SaveChangesAsync(CancellationToken cancellationToken = default);
+
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/DbConfig/Abstraction/IEtlDbContextFactory.cs b/etl_backend/Infrastructure/DbConfig/Abstraction/IEtlDbContextFactory.cs
new file mode 100644
index 00000000..1e8f8c50
--- /dev/null
+++ b/etl_backend/Infrastructure/DbConfig/Abstraction/IEtlDbContextFactory.cs
@@ -0,0 +1,7 @@
+namespace Infrastructure.DbConfig.Abstraction;
+
+public interface IEtlDbContextFactory
+{
+ IStagingDbContext CreateStagingDbContext();
+ ISchemaDbContext CreateSchemaDbContext();
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/DbConfig/Abstraction/ISchemaDbContext.cs b/etl_backend/Infrastructure/DbConfig/Abstraction/ISchemaDbContext.cs
new file mode 100644
index 00000000..c649ac75
--- /dev/null
+++ b/etl_backend/Infrastructure/DbConfig/Abstraction/ISchemaDbContext.cs
@@ -0,0 +1,10 @@
+using Domain.Entities;
+using Microsoft.EntityFrameworkCore;
+
+namespace Infrastructure.DbConfig.Abstraction;
+
+public interface ISchemaDbContext : ICommonDbContext
+{
+ DbSet DataTableSchemas { get; set; }
+ DbSet DataTableColumns { get; set; }
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/DbConfig/Abstraction/IStagingDbContext.cs b/etl_backend/Infrastructure/DbConfig/Abstraction/IStagingDbContext.cs
new file mode 100644
index 00000000..e18cdb95
--- /dev/null
+++ b/etl_backend/Infrastructure/DbConfig/Abstraction/IStagingDbContext.cs
@@ -0,0 +1,9 @@
+using Domain.Entities;
+using Microsoft.EntityFrameworkCore;
+
+namespace Infrastructure.DbConfig.Abstraction;
+
+public interface IStagingDbContext : ICommonDbContext
+{
+ DbSet StagedFiles { get; set; }
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/DbConfig/Configurations/DataTableColumnConfig.cs b/etl_backend/Infrastructure/DbConfig/Configurations/DataTableColumnConfig.cs
new file mode 100644
index 00000000..6bbe5dc0
--- /dev/null
+++ b/etl_backend/Infrastructure/DbConfig/Configurations/DataTableColumnConfig.cs
@@ -0,0 +1,27 @@
+using Domain.Entities;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace Infrastructure.DbConfig.Configurations;
+
+public class DataTableColumnConfig : IEntityTypeConfiguration
+{
+ public void Configure(EntityTypeBuilder builder)
+ {
+
+ builder.HasKey(c => c.Id);
+
+ builder.Property(c => c.ColumnName)
+ .IsRequired()
+ .HasMaxLength(255);
+
+ builder.Property(c => c.ColumnType)
+ .HasMaxLength(100)
+ .HasDefaultValue("string");
+
+ builder.Property(c => c.OrdinalPosition)
+ .IsRequired();
+
+ builder.HasIndex(c => new { c.DataTableSchemaId, c.ColumnName }).IsUnique();
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/DbConfig/Configurations/DataTableSchemaConfig.cs b/etl_backend/Infrastructure/DbConfig/Configurations/DataTableSchemaConfig.cs
new file mode 100644
index 00000000..120d22f8
--- /dev/null
+++ b/etl_backend/Infrastructure/DbConfig/Configurations/DataTableSchemaConfig.cs
@@ -0,0 +1,30 @@
+using Domain.Entities;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace Infrastructure.DbConfig.Configurations;
+
+public class DataTableSchemaConfig : IEntityTypeConfiguration
+{
+ public void Configure(EntityTypeBuilder builder)
+ {
+
+ builder.HasKey(t => t.Id);
+
+ builder.Property(t => t.TableName)
+ .IsRequired()
+ .HasMaxLength(100);
+
+ builder.Property(t => t.OriginalFileName)
+ .IsRequired()
+ .HasMaxLength(100);
+
+ builder.HasIndex(t => t.TableName)
+ .IsUnique();
+
+ builder.HasMany(t => t.Columns)
+ .WithOne(c => c.DataTable)
+ .HasForeignKey(c => c.DataTableSchemaId)
+ .OnDelete(DeleteBehavior.Cascade);
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/DbConfig/Configurations/StagedFileConfig.cs b/etl_backend/Infrastructure/DbConfig/Configurations/StagedFileConfig.cs
new file mode 100644
index 00000000..3f1f7f66
--- /dev/null
+++ b/etl_backend/Infrastructure/DbConfig/Configurations/StagedFileConfig.cs
@@ -0,0 +1,54 @@
+using Domain.Entities;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace Infrastructure.DbConfig.Configurations;
+
+public class StagedFileConfig : IEntityTypeConfiguration
+{
+ public void Configure(EntityTypeBuilder builder)
+ {
+ builder.ToTable("staged_files");
+
+ builder.HasKey(f => f.Id);
+
+ builder.Property(f => f.OriginalFileName)
+ .IsRequired()
+ .HasMaxLength(255);
+
+ builder.Property(f => f.StoredFilePath)
+ .IsRequired()
+ .HasMaxLength(1024);
+
+ builder.Property(f => f.ErrorMessage)
+ .HasMaxLength(2000);
+
+ builder.Property(f => f.Stage)
+ .IsRequired()
+ .HasConversion()
+ .HasMaxLength(64);
+
+ builder.Property(f => f.Status)
+ .IsRequired()
+ .HasConversion()
+ .HasMaxLength(32);
+
+ builder.Property(f => f.ErrorCode)
+ .IsRequired()
+ .HasConversion()
+ .HasMaxLength(64);
+
+ // Timestamptz for UTC times in Postgres
+ builder.Property(f => f.UploadedAt)
+ .HasColumnType("timestamptz");
+
+ builder.HasIndex(f => f.StoredFilePath).IsUnique(); // each saved file path is unique
+ builder.HasIndex(f => new { f.Stage, f.Status });
+ builder.HasIndex(f => f.UploadedAt);
+
+ builder.HasOne(f => f.Schema)
+ .WithOne()
+ .HasForeignKey(f => f.SchemaId)
+ .OnDelete(DeleteBehavior.SetNull);
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/DbConfig/EtlDbContext.cs b/etl_backend/Infrastructure/DbConfig/EtlDbContext.cs
new file mode 100644
index 00000000..016764a7
--- /dev/null
+++ b/etl_backend/Infrastructure/DbConfig/EtlDbContext.cs
@@ -0,0 +1,33 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Reflection;
+using Domain.Entities;
+using Infrastructure.DbConfig.Abstraction;
+using Infrastructure.DbConfig.Configurations;
+using Microsoft.EntityFrameworkCore;
+
+namespace Infrastructure.DbConfig;
+
+[ExcludeFromCodeCoverage]
+public class EtlDbContext : DbContext, IStagingDbContext, ISchemaDbContext
+{
+ public EtlDbContext(DbContextOptions options)
+ : base(options)
+ {
+ }
+
+ public DbSet DataTableSchemas { get; set; } = null!;
+ public DbSet DataTableColumns { get; set; } = null!;
+ public DbSet StagedFiles { get; set; } = null!;
+
+ protected override void OnModelCreating(ModelBuilder modelBuilder)
+ {
+ modelBuilder.ApplyConfigurationsFromAssembly(Assembly.GetExecutingAssembly());
+ base.OnModelCreating(modelBuilder);
+ modelBuilder.ApplyConfiguration(new DataTableSchemaConfig());
+ modelBuilder.ApplyConfiguration(new DataTableColumnConfig());
+ modelBuilder.ApplyConfiguration(new StagedFileConfig());
+
+
+ }
+
+}
diff --git a/etl_backend/Infrastructure/DbConfig/EtlDbContextDesignFactory.cs b/etl_backend/Infrastructure/DbConfig/EtlDbContextDesignFactory.cs
new file mode 100644
index 00000000..bbe02c7f
--- /dev/null
+++ b/etl_backend/Infrastructure/DbConfig/EtlDbContextDesignFactory.cs
@@ -0,0 +1,27 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Design;
+using Microsoft.Extensions.Configuration;
+
+namespace Infrastructure.DbConfig;
+
+public sealed class EtlDbContextDesignFactory : IDesignTimeDbContextFactory
+{
+ public EtlDbContext CreateDbContext(string[] args)
+ {
+ var cfg = new ConfigurationBuilder()
+ .SetBasePath(Directory.GetCurrentDirectory())
+ .AddJsonFile("appsettings.json", optional: true)
+ .AddJsonFile("appsettings.Development.json", optional: true)
+ .AddEnvironmentVariables()
+ .Build();
+
+ var cs = cfg.GetConnectionString("DefaultConnection")
+ ?? "Host=localhost;Port=5432;Database=etl_dev;Username=etl_user;Password=etl_password";
+
+ var opts = new DbContextOptionsBuilder()
+ .UseNpgsql(cs)
+ .Options;
+
+ return new EtlDbContext(opts);
+ }
+}
diff --git a/etl_backend/Infrastructure/DbConfig/EtlDbContextFactory.cs b/etl_backend/Infrastructure/DbConfig/EtlDbContextFactory.cs
new file mode 100644
index 00000000..90ad5078
--- /dev/null
+++ b/etl_backend/Infrastructure/DbConfig/EtlDbContextFactory.cs
@@ -0,0 +1,14 @@
+using Infrastructure.DbConfig.Abstraction;
+using Microsoft.EntityFrameworkCore;
+
+namespace Infrastructure.DbConfig;
+
+public sealed class EtlDbContextFactory : IEtlDbContextFactory
+{
+ private readonly IDbContextFactory _dbFactory;
+ public EtlDbContextFactory(IDbContextFactory dbFactory)
+ => _dbFactory = dbFactory ?? throw new ArgumentNullException(nameof(dbFactory));
+
+ public IStagingDbContext CreateStagingDbContext() => _dbFactory.CreateDbContext();
+ public ISchemaDbContext CreateSchemaDbContext() => _dbFactory.CreateDbContext();
+}
diff --git a/etl_backend/Infrastructure/DependencyInjection.cs b/etl_backend/Infrastructure/DependencyInjection.cs
new file mode 100644
index 00000000..e4894f1e
--- /dev/null
+++ b/etl_backend/Infrastructure/DependencyInjection.cs
@@ -0,0 +1,171 @@
+using Application.Abstractions;
+using Application.Common.Configurations;
+using Application.Common.Services.Abstractions;
+using Application.Enums;
+using Application.Files.DeleteStagedFile.ServiceAbstractions;
+using Application.Files.StageManyFiles.ServiceAbstractions;
+using Application.Services.Abstractions;
+using Application.Services.Repositories.Abstractions;
+using Application.Tables.DeleteTable.ServiceAbstractions;
+using Application.Tables.ListTables.ServiceAbstractions;
+using Application.Tables.RenameTable.ServiceAbstractions;
+using Application.Users.CreateUser.ServiceAbstractions;
+using Application.Users.DeleteUser.ServiceAbstractions;
+using Application.Users.EditUser.ServiceAbstractions;
+using Application.Users.GetAllRoles.ServiceAbstractions;
+using Application.Users.GetUserById.ServiceAbstractions;
+using Application.Users.ListUsers;
+using Domain.Entities;
+using etl_backend.Application.DataFile.Abstraction;
+using etl_backend.Application.DataFile.Services;
+using Infrastructure.Configurations;
+using Infrastructure.DbConfig;
+using Infrastructure.DbConfig.Abstraction;
+using Infrastructure.Dtos;
+using Infrastructure.Files;
+using Infrastructure.Files.Abstractions;
+using Infrastructure.Files.PostgresTableServices;
+using Infrastructure.Files.PostgresTableServices.HelperServices;
+using Infrastructure.Identity;
+using Infrastructure.Identity.Abstractions;
+using Infrastructure.Repositories;
+using Infrastructure.SsoServices.Admin;
+using Infrastructure.SsoServices.Admin.Abstractions;
+using Infrastructure.SsoServices.User;
+using Infrastructure.SsoServices.User.Abstractions;
+using Infrastructure.Tables;
+using Infrastructure.Tables.Abstractions;
+using Microsoft.AspNetCore.Http;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
+using Npgsql;
+using IColumnDefinitionBuilder = Application.Abstractions.IColumnDefinitionBuilder;
+using ILoadPolicyFactory = Application.Abstractions.ILoadPolicyFactory;
+using SystemClock = Infrastructure.Configurations.SystemClock;
+
+namespace Infrastructure;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+
+public static class DependencyInjection
+{
+ public static IServiceCollection AddInfrastructureServices(this IServiceCollection services, IConfiguration configuration)
+ {
+ services.AddDbContextFactory(options =>
+ options.UseNpgsql(configuration.GetConnectionString("DefaultConnection")));
+ services.AddSingleton();
+ services.Configure(configuration.GetSection("Storage"));
+ services.Configure(configuration.GetSection("CsvStaging"));
+ services.Configure(configuration.GetSection("PostgresStore"));
+ var cs = configuration.GetConnectionString("DefaultConnection");
+ services.AddSingleton(sp => new NpgsqlDataSourceBuilder(cs).Build());
+
+ // --- Repositories ---
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ // --- Cross-cutting ---
+ // services.AddSingleton();
+
+ services.AddSingleton();
+
+ // --- Storage + CSV header/rows ---
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ // services.AddScoped();
+
+
+ // --- Components / helpers ---
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton(sp =>
+ {
+ var opts = sp.GetRequiredService>().Value;
+ return new CsvRowFormatter(opts.Delimiter, opts.QuoteChar);
+ });
+
+ // --- Provider adapters (Postgres) ---
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+
+ // --- Use-cases / orchestrators ---
+ services.AddScoped();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+
+ services.AddSingleton(_ => new LoadPolicy(
+ mode: LoadMode.Append,
+ dropOnFailure: false
+ ));
+
+ services.AddDbContext(options =>
+ {
+ var cs = configuration.GetConnectionString("DefaultConnection");
+ options.UseNpgsql(cs);
+ // options.UseNpgsql(cs, o => o.MigrationsAssembly("etl_backend.Infrastructure"));
+ });
+ services.Configure(configuration.GetSection("ColumnTypeConfiguration"));
+ services.AddSingleton();
+ services.AddSingleton();
+
+ // --- Auth ---
+ services.AddScoped();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.Configure(
+ configuration.GetSection("Keycloak"));
+
+
+ // --- Admin ---
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+
+ return services;
+ }
+}
+
+
diff --git a/etl_backend/Infrastructure/Dtos/EditUserRolesRequestDto.cs b/etl_backend/Infrastructure/Dtos/EditUserRolesRequestDto.cs
new file mode 100644
index 00000000..249f1736
--- /dev/null
+++ b/etl_backend/Infrastructure/Dtos/EditUserRolesRequestDto.cs
@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+using Application.Dtos;
+
+namespace Infrastructure.Dtos;
+
+public class EditUserRolesRequestDto
+{
+ [JsonPropertyName("rolesToAdd")]
+ public IEnumerable RolesToAdd { get; set; } = Enumerable.Empty();
+ [JsonPropertyName("rolesToRemove")]
+ public IEnumerable RolesToRemove { get; set; } = Enumerable.Empty();
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Dtos/KeycloakOptions.cs b/etl_backend/Infrastructure/Dtos/KeycloakOptions.cs
new file mode 100644
index 00000000..4edd9996
--- /dev/null
+++ b/etl_backend/Infrastructure/Dtos/KeycloakOptions.cs
@@ -0,0 +1,24 @@
+namespace Infrastructure.Dtos;
+
+public class KeycloakOptions
+{
+ public string AuthServerUrl { get; set; } = string.Empty;
+ public string AuthServerUrlPublic { get; set; } = string.Empty;
+ public string Realm { get; set; } = string.Empty;
+ public string ClientId { get; set; } = string.Empty;
+ public string ClientSecret { get; set; } = string.Empty;
+ public string Audience { get; set; } = string.Empty;
+
+ public string AccessCookieName { get; set; } = "access_token";
+ public string RefreshCookieName { get; set; } = "refresh_token";
+
+ public string ExpClaimType { get; set; } = "exp";
+ public int ClockSkewSeconds { get; set; } = 30;
+
+ public string RoleScope { get; set; } = "realm_access";
+ public string RolesKey { get; set; } = "roles";
+
+ public string Authority => $"{AuthServerUrl}/realms/{Realm}";
+ public string ValidIssuer => $"{AuthServerUrlPublic}/realms/{Realm}";
+ public TimeSpan ClockSkew => TimeSpan.FromSeconds(ClockSkewSeconds);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Dtos/KeycloakUserCreateDto.cs b/etl_backend/Infrastructure/Dtos/KeycloakUserCreateDto.cs
new file mode 100644
index 00000000..582646da
--- /dev/null
+++ b/etl_backend/Infrastructure/Dtos/KeycloakUserCreateDto.cs
@@ -0,0 +1,35 @@
+using System.Text.Json.Serialization;
+
+namespace Infrastructure.Dtos;
+
+public class KeycloakUserCreateDto
+{
+ [JsonPropertyName("username")] public string Username { get; set; } = null!;
+
+ [JsonPropertyName("firstName")] public string? FirstName { get; set; }
+
+ [JsonPropertyName("lastName")] public string? LastName { get; set; }
+
+ [JsonPropertyName("email")] public string? Email { get; set; }
+
+ [JsonPropertyName("enabled")] public bool Enabled { get; set; } = true;
+
+
+ // Optional: initial password
+ [JsonPropertyName("credentials")] public List? Credentials { get; set; }
+
+ // Optional: custom attributes
+ [JsonPropertyName("attributes")] public Dictionary? Attributes { get; set; }
+}
+
+public class KeycloakCredentialDto
+{
+ [JsonPropertyName("type")]
+ public string Type { get; set; } = "password";
+
+ [JsonPropertyName("value")]
+ public string Value { get; set; } = null!;
+
+ [JsonPropertyName("temporary")]
+ public bool Temporary { get; set; } = true;
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Dtos/TokenResponseDto.cs b/etl_backend/Infrastructure/Dtos/TokenResponseDto.cs
new file mode 100644
index 00000000..cd7bdaea
--- /dev/null
+++ b/etl_backend/Infrastructure/Dtos/TokenResponseDto.cs
@@ -0,0 +1,9 @@
+namespace Infrastructure.Dtos;
+
+public class TokenResponseDto
+{
+ public string AccessToken { get; set; } = default!;
+ public string RefreshToken { get; set; } = default!;
+ public int ExpiresIn { get; set; }
+ public int RefreshExpiresIn { get; set; }
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Exceptions/ApiException.cs b/etl_backend/Infrastructure/Exceptions/ApiException.cs
new file mode 100644
index 00000000..268a8c6e
--- /dev/null
+++ b/etl_backend/Infrastructure/Exceptions/ApiException.cs
@@ -0,0 +1,14 @@
+namespace Infrastructure.Exceptions;
+
+public class ApiException : Exception
+{
+ public int StatusCode { get; }
+ public string? ResponseContent { get; }
+
+ public ApiException(string message, int statusCode, string? responseContent = null)
+ : base(message)
+ {
+ StatusCode = statusCode;
+ ResponseContent = responseContent;
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/IColumnAdmin.cs b/etl_backend/Infrastructure/Files/Abstractions/IColumnAdmin.cs
new file mode 100644
index 00000000..94b75bd0
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/IColumnAdmin.cs
@@ -0,0 +1,7 @@
+namespace Infrastructure.Files.Abstractions;
+
+public interface IColumnAdmin
+{
+ Task RenameAsync(Npgsql.NpgsqlConnection conn, string schema, string table, string oldName, string newName, CancellationToken ct = default);
+ Task DropAsync(Npgsql.NpgsqlConnection conn, string schema, string table, IReadOnlyCollection columnNames, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/IColumnDefinitionBuilder.cs b/etl_backend/Infrastructure/Files/Abstractions/IColumnDefinitionBuilder.cs
new file mode 100644
index 00000000..b171af77
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/IColumnDefinitionBuilder.cs
@@ -0,0 +1,8 @@
+using Domain.Entities;
+
+namespace Infrastructure.Files.Abstractions;
+
+public interface IColumnDefinitionBuilder
+{
+ List Build(IReadOnlyList headers);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/ICsvHeaderReader.cs b/etl_backend/Infrastructure/Files/Abstractions/ICsvHeaderReader.cs
new file mode 100644
index 00000000..b9851b60
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/ICsvHeaderReader.cs
@@ -0,0 +1,7 @@
+namespace Infrastructure.Files.Abstractions;
+
+public interface ICsvHeaderReader
+{
+ Task> ReadHeadersAsync(Stream stream, CancellationToken ct = default);
+}
+
diff --git a/etl_backend/Infrastructure/Files/Abstractions/ICsvRowFormatter.cs b/etl_backend/Infrastructure/Files/Abstractions/ICsvRowFormatter.cs
new file mode 100644
index 00000000..3b7ab934
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/ICsvRowFormatter.cs
@@ -0,0 +1,6 @@
+namespace etl_backend.Application.DataFile.Abstraction;
+
+public interface ICsvRowFormatter
+{
+ void WriteRow(TextWriter writer, string[] fields);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/IDataWrite.cs b/etl_backend/Infrastructure/Files/Abstractions/IDataWrite.cs
new file mode 100644
index 00000000..aa26e9a8
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/IDataWrite.cs
@@ -0,0 +1,9 @@
+using Application.Dtos;
+using Infrastructure.Files.Dtos;
+
+namespace Infrastructure.Files.Abstractions;
+
+public interface IDataWrite
+{
+ Task AppendRowsAsync(TableRef table, IRowSource rows, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/IDdlBuilder.cs b/etl_backend/Infrastructure/Files/Abstractions/IDdlBuilder.cs
new file mode 100644
index 00000000..4840c0be
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/IDdlBuilder.cs
@@ -0,0 +1,8 @@
+using Infrastructure.Files.Dtos;
+
+namespace Infrastructure.Files.Abstractions;
+
+public interface IDdlBuilder
+{
+ string BuildCreateTableSql(string qualifiedTableName, IReadOnlyList columns, bool ifNotExists = false);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/IFileStorage.cs b/etl_backend/Infrastructure/Files/Abstractions/IFileStorage.cs
new file mode 100644
index 00000000..0853c3ef
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/IFileStorage.cs
@@ -0,0 +1,9 @@
+namespace Infrastructure.Files.Abstractions;
+
+public interface IFileStorage
+{
+ Task SaveFileAsync(Stream fileStream, string fileName, string dirPath = "");
+ Task DeleteFileAsync(string filePath);
+ Task GetFileSizeAsync(string relativePath);
+ Task OpenReadAsync(string relativePath);
+}
diff --git a/etl_backend/Infrastructure/Files/Abstractions/IIdentifierPolicy.cs b/etl_backend/Infrastructure/Files/Abstractions/IIdentifierPolicy.cs
new file mode 100644
index 00000000..b08abf38
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/IIdentifierPolicy.cs
@@ -0,0 +1,8 @@
+namespace Infrastructure.Files.Abstractions;
+
+public interface IIdentifierPolicy
+{
+ string DefaultSchema { get; }
+ string QuoteIdentifier(string identifier);
+ string QualifyTable(string? schema, string table); // returns fully quoted
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/ILoadPolicyFactory.cs b/etl_backend/Infrastructure/Files/Abstractions/ILoadPolicyFactory.cs
new file mode 100644
index 00000000..38fc098f
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/ILoadPolicyFactory.cs
@@ -0,0 +1,7 @@
+using Application.Common.Services.Abstractions;
+using Application.Enums;
+using Application.Services.Abstractions;
+
+namespace Infrastructure.Files.Abstractions;
+
+public interface ILoadPolicyFactory { ILoadPolicy Create(LoadMode mode, bool dropOnFailure); }
diff --git a/etl_backend/Infrastructure/Files/Abstractions/ILoadPreconditionsService.cs b/etl_backend/Infrastructure/Files/Abstractions/ILoadPreconditionsService.cs
new file mode 100644
index 00000000..6cb2292a
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/ILoadPreconditionsService.cs
@@ -0,0 +1,8 @@
+using Domain.Entities;
+
+namespace Infrastructure.Files.Abstractions;
+
+public interface ILoadPreconditionsService
+{
+ Task<(StagedFile staged, DataTableSchema schema)> EnsureLoadableAsync(int stagedFileId, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/IRowSource.cs b/etl_backend/Infrastructure/Files/Abstractions/IRowSource.cs
new file mode 100644
index 00000000..b4b269fd
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/IRowSource.cs
@@ -0,0 +1,6 @@
+namespace Infrastructure.Files.Abstractions;
+
+public interface IRowSource : IAsyncDisposable
+{
+ IAsyncEnumerable ReadRowsAsync(CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/IRowSourceFactory.cs b/etl_backend/Infrastructure/Files/Abstractions/IRowSourceFactory.cs
new file mode 100644
index 00000000..3636299b
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/IRowSourceFactory.cs
@@ -0,0 +1,8 @@
+using Domain.Entities;
+
+namespace Infrastructure.Files.Abstractions;
+
+public interface IRowSourceFactory
+{
+ Task CreateForStagedFileAsync(StagedFile staged, int expectedColumns, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/ISqlExecuter.cs b/etl_backend/Infrastructure/Files/Abstractions/ISqlExecuter.cs
new file mode 100644
index 00000000..9a5d8e6d
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/ISqlExecuter.cs
@@ -0,0 +1,8 @@
+using Npgsql;
+
+namespace Infrastructure.Files.Abstractions;
+
+public interface ISqlExecutor
+{
+ Task ExecuteAsync(NpgsqlConnection conn, string sql, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/IStorageAppEnviroment.cs b/etl_backend/Infrastructure/Files/Abstractions/IStorageAppEnviroment.cs
new file mode 100644
index 00000000..181959ff
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/IStorageAppEnviroment.cs
@@ -0,0 +1,7 @@
+namespace Infrastructure.Files.Abstractions;
+
+public interface IStorageAppEnvironment
+{
+ bool IsDevelopment { get; }
+ string ContentRootPath { get; }
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/ITableAdmin.cs b/etl_backend/Infrastructure/Files/Abstractions/ITableAdmin.cs
new file mode 100644
index 00000000..b03e4ce0
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/ITableAdmin.cs
@@ -0,0 +1,13 @@
+using Application.Enums;
+using Infrastructure.Files.Dtos;
+
+namespace Infrastructure.Files.Abstractions;
+
+public interface ITableAdmin
+{
+ Task EnsureTableAsync(TableSpec spec, LoadMode mode, CancellationToken ct = default);
+ Task DropTableAsync(TableRef table, CancellationToken ct = default);
+ Task TruncateTableAsync(TableRef table, CancellationToken ct = default);
+ Task RenameTableAsync(TableRef table, string newName, CancellationToken ct = default);
+}
+
diff --git a/etl_backend/Infrastructure/Files/Abstractions/ITableCatalog.cs b/etl_backend/Infrastructure/Files/Abstractions/ITableCatalog.cs
new file mode 100644
index 00000000..0e1f2c8c
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/ITableCatalog.cs
@@ -0,0 +1,8 @@
+using Npgsql;
+
+namespace etl_backend.Application.DataFile.Abstraction;
+
+public interface ITableCatalog
+{
+ Task TableExistsAsync(NpgsqlConnection conn, string schema, string table, CancellationToken ct = default);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/ITableNameParser.cs b/etl_backend/Infrastructure/Files/Abstractions/ITableNameParser.cs
new file mode 100644
index 00000000..2ab9fedc
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/ITableNameParser.cs
@@ -0,0 +1,6 @@
+namespace etl_backend.Application.DataFile.Abstraction;
+
+public interface ITableNameParser
+{
+ (string Schema, string Table) Parse(string name, string defaultSchema);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/ITableSpecFactory.cs b/etl_backend/Infrastructure/Files/Abstractions/ITableSpecFactory.cs
new file mode 100644
index 00000000..8f16495a
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/ITableSpecFactory.cs
@@ -0,0 +1,9 @@
+using Domain.Entities;
+using Infrastructure.Files.Dtos;
+
+namespace Infrastructure.Files.Abstractions;
+
+public interface ITableSpecFactory
+{
+ TableSpec From(DataTableSchema schema);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/Abstractions/ITypeMapper.cs b/etl_backend/Infrastructure/Files/Abstractions/ITypeMapper.cs
new file mode 100644
index 00000000..fe8bf01a
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/Abstractions/ITypeMapper.cs
@@ -0,0 +1,6 @@
+namespace etl_backend.Application.DataFile.Abstraction;
+
+public interface ITypeMapper
+{
+ string ToProviderType(string logicalType);
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/AddStageFile.cs b/etl_backend/Infrastructure/Files/AddStageFile.cs
new file mode 100644
index 00000000..d69f9946
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/AddStageFile.cs
@@ -0,0 +1,52 @@
+using Application.Files.StageManyFiles.ServiceAbstractions;
+using Application.Services.Repositories.Abstractions;
+using Domain.Entities;
+using Domain.Enums;
+using Infrastructure.Files.Abstractions;
+
+namespace Infrastructure.Files;
+
+public class AddStageFile : IAddStageFileService
+{
+ private readonly IFileStorage _storage;
+ private readonly IStagedFileRepository _repo;
+ private readonly IClock _clock;
+
+ public AddStageFile(IFileStorage storage, IStagedFileRepository repo, IClock clock)
+ => (_storage, _repo, _clock) = (storage, repo, clock);
+
+ public async Task StageAsync(Stream fileStream, string originalFileName, string? subdir = "uploads", CancellationToken ct = default)
+ {
+ string? savedPath = null;
+ try
+ {
+ savedPath = await _storage.SaveFileAsync(fileStream, originalFileName, subdir ?? "");
+ var size = await _storage.GetFileSizeAsync(savedPath);
+ var staged = new StagedFile
+ {
+ OriginalFileName = originalFileName,
+ StoredFilePath = savedPath,
+ FileSize = size,
+ UploadedAt = _clock.UtcNow,
+
+ Stage = ProcessingStage.Uploaded,
+ Status = ProcessingStatus.InProgress,
+ ErrorCode = ProcessingErrorCode.None,
+ ErrorMessage = null,
+
+ SchemaId = null
+ };
+
+ return await _repo.AddAsync(staged, ct);
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine(e);
+ if (savedPath is not null)
+ {
+ try { await _storage.DeleteFileAsync(savedPath); } catch { /* best-effort cleanup */ }
+ }
+ throw;
+ }
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/CsvHeaderReader.cs b/etl_backend/Infrastructure/Files/CsvHeaderReader.cs
new file mode 100644
index 00000000..d36180e0
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/CsvHeaderReader.cs
@@ -0,0 +1,59 @@
+using System.Globalization;
+using CsvHelper;
+using CsvHelper.Configuration;
+using Infrastructure.Configurations;
+using Infrastructure.Files.Abstractions;
+using Microsoft.Extensions.Options;
+
+namespace Infrastructure.Files;
+
+public sealed class CsvHeaderReader : ICsvHeaderReader
+{
+ private readonly CsvStagingOptions _opts;
+
+ public CsvHeaderReader(IOptions options)
+ => _opts = options.Value;
+
+ public async Task> ReadHeadersAsync(Stream stream, CancellationToken ct = default)
+ {
+ // Leave the underlying stream open; caller owns it.
+ using var reader = new StreamReader(
+ stream,
+ _opts.Encoding.ToSystemEncoding(),
+ detectEncodingFromByteOrderMarks: true,
+ bufferSize: 1024,
+ leaveOpen: true);
+
+ var cfg = new CsvConfiguration(CultureInfo.InvariantCulture)
+ {
+ HasHeaderRecord = _opts.HasHeader,
+ Delimiter = _opts.Delimiter.ToString(),
+ Quote = _opts.QuoteChar,
+ TrimOptions = _opts.TrimWhitespace ? TrimOptions.Trim : TrimOptions.None,
+ BadDataFound = null,
+ MissingFieldFound = null,
+ DetectDelimiter = false
+ };
+
+ using var csv = new CsvReader(reader, cfg);
+
+ // If file is empty
+ if (!await csv.ReadAsync()) return Array.Empty();
+
+ if (_opts.HasHeader)
+ {
+ csv.ReadHeader();
+ var headers = csv.HeaderRecord ?? Array.Empty();
+ return headers.ToList();
+ }
+
+ // No header: read first record to determine column count, then synthesize names.
+ var record = csv.Parser.Record;
+ var count = record?.Length ?? 0;
+ if (count == 0) return Array.Empty();
+
+ var synthetic = new List(count);
+ for (int i = 1; i <= count; i++) synthetic.Add($"col_{i}");
+ return synthetic;
+ }
+}
diff --git a/etl_backend/Infrastructure/Files/CsvRowFormatter.cs b/etl_backend/Infrastructure/Files/CsvRowFormatter.cs
new file mode 100644
index 00000000..e8cdda20
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/CsvRowFormatter.cs
@@ -0,0 +1,25 @@
+using etl_backend.Application.DataFile.Abstraction;
+
+namespace etl_backend.Application.DataFile.Services;
+
+public sealed class CsvRowFormatter : ICsvRowFormatter
+{
+ private readonly char _delimiter;
+ private readonly char _quote;
+
+ public CsvRowFormatter(char delimiter = ',', char quote = '"')
+ => (_delimiter, _quote) = (delimiter, quote);
+
+ public void WriteRow(TextWriter writer, string[] fields)
+ {
+ for (int i = 0; i < fields.Length; i++)
+ {
+ if (i > 0) writer.Write(_delimiter);
+ var s = fields[i] ?? string.Empty;
+ writer.Write(_quote);
+ writer.Write(s.Replace(_quote.ToString(), new string(_quote, 2)));
+ writer.Write(_quote);
+ }
+ writer.Write('\n');
+ }
+}
\ No newline at end of file
diff --git a/etl_backend/Infrastructure/Files/CsvRowSource.cs b/etl_backend/Infrastructure/Files/CsvRowSource.cs
new file mode 100644
index 00000000..3cca7225
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/CsvRowSource.cs
@@ -0,0 +1,78 @@
+using System.Globalization;
+using CsvHelper;
+using CsvHelper.Configuration;
+using Infrastructure.Configurations;
+using Infrastructure.Files.Abstractions;
+
+namespace Infrastructure.Files;
+
+public sealed class CsvRowSource : IRowSource
+{
+ private readonly Stream _stream;
+ private readonly StreamReader _reader;
+ private readonly CsvReader _csv;
+ private readonly int _expectedCols;
+ private bool _initialized;
+
+ public CsvRowSource(Stream stream, CsvStagingOptions opts, int expectedColumns)
+ {
+ _stream = stream;
+ _reader = new StreamReader(
+ _stream,
+ opts.Encoding.ToSystemEncoding(),
+ detectEncodingFromByteOrderMarks: true,
+ bufferSize: 1 << 12,
+ leaveOpen: false);
+
+ var cfg = new CsvConfiguration(CultureInfo.InvariantCulture)
+ {
+ HasHeaderRecord = opts.HasHeader,
+ Delimiter = opts.Delimiter.ToString(),
+ Quote = opts.QuoteChar,
+ TrimOptions = opts.TrimWhitespace ? TrimOptions.Trim : TrimOptions.None,
+ BadDataFound = null,
+ MissingFieldFound = null,
+ DetectDelimiter = false
+ };
+
+ _csv = new CsvReader(_reader, cfg);
+ _expectedCols = expectedColumns;
+ }
+
+ public async IAsyncEnumerable ReadRowsAsync([System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct = default)
+ {
+ if (!_initialized)
+ {
+ if (_csv.Configuration.HasHeaderRecord)
+ {
+ if (!await _csv.ReadAsync()) yield break; // empty file
+ _csv.ReadHeader(); // skip header row
+ }
+ _initialized = true;
+ }
+
+ while (await _csv.ReadAsync())
+ {
+ ct.ThrowIfCancellationRequested();
+
+ // Build row with a fixed number of columns (pad/truncate)
+ var row = new string[_expectedCols];
+ for (int i = 0; i < _expectedCols; i++)
+ {
+ string? v;
+ try { v = _csv.GetField(i); }
+ catch { v = null; } // missing field -> null
+ row[i] = v ?? string.Empty; // keep empty for now; provider may map empty->NULL
+ }
+ yield return row;
+ }
+ }
+
+ public ValueTask DisposeAsync()
+ {
+ _csv.Dispose();
+ _reader.Dispose();
+ _stream.Dispose();
+ return ValueTask.CompletedTask;
+ }
+}
diff --git a/etl_backend/Infrastructure/Files/DefaultColumnDefinitionBuilder.cs b/etl_backend/Infrastructure/Files/DefaultColumnDefinitionBuilder.cs
new file mode 100644
index 00000000..ea934f09
--- /dev/null
+++ b/etl_backend/Infrastructure/Files/DefaultColumnDefinitionBuilder.cs
@@ -0,0 +1,38 @@
+using Application.Abstractions;
+using Domain.Entities;
+using Infrastructure.Files.Abstractions;
+using IColumnDefinitionBuilder = Application.Abstractions.IColumnDefinitionBuilder;
+
+namespace Infrastructure.Files;
+
+public sealed class DefaultColumnDefinitionBuilder : IColumnDefinitionBuilder
+{
+ private readonly IColumnNameSanitizer _sanitizer;
+
+ public DefaultColumnDefinitionBuilder(IColumnNameSanitizer sanitizer) => _sanitizer = sanitizer;
+
+ public List Build(IReadOnlyList