Revert "Bulk Copy" functionality

This caused significant issues with some SQL data types and resulted in imported failures.

Author: samgibsonmoj

Directory.Packages.props

@@ -48,11 +48,11 @@
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
     <PackageVersion Include="NSwag.AspNetCore" Version="14.6.3" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.4" />
-    <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.15.3" />
-    <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.15.3" />
-    <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.15.2" />
-    <PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.15.1" />
-    <PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.15.1" />
+    <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.14.0" />
+    <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.14.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.14.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.14.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.14.0" />
     <PackageVersion Include="Phonix" Version="1.0.2" />
     <PackageVersion Include="RabbitMQ.Client" Version="7.2.0" />
     <PackageVersion Include="Rebus" Version="8.9.0" />

src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql

@@ -1,24 +1,51 @@
 CREATE PROCEDURE [DeliusStaging].[StageDelius]
-	@processedFile VARCHAR(50)
+	@basePath VARCHAR(100), -- Should be in the format /app/delius/{fileName}.
+	@processedFile VARCHAR(50),
+	@inContainer VARCHAR(1)
 AS
-BEGIN
-	SET NOCOUNT ON;
-	SET DATEFORMAT DMY;
+	IF @inContainer = 'N'
+	BEGIN 
+		SET NOCOUNT ON; 
+		SET DATEFORMAT DMY;
+	END;
 
-	BEGIN TRANSACTION;
+	CREATE TABLE #TableNames(Id int Primary Key Identity, fileName VARCHAR(50));
+	INSERT INTO #TableNames(fileName) VALUES
+		('AdditionalIdentifier'), ('AliasDetails'), ('Disability'), ('Disposal'), ('EventDetails'), ('Header'), ('MainOffence'), 
+		('OAS'), ('OffenderAddress'), ('OffenderManager'), ('OffenderManagerBuildings'), ('OffenderManagerTeam'), 
+		('OffenderToOffenderManagerMappings'), ('Offenders'), ('OffenderTransfer'), ('PersonalCircumstances'), 
+		('Provision'), ('RegistrationDetails'), ('Requirement');
+	
+	DECLARE @i int;
+	DECLARE @sql NVARCHAR(250);
+	DECLARE @fileName VARCHAR(150);
+
+	SET @i = 1;
+
+	BEGIN TRANSACTION;	
 	BEGIN TRY
-		DECLARE @retMessage VARCHAR(500);
-		EXEC [DeliusStaging].[StandardiseData] @retMessage OUTPUT;
+		WHILE @i <= (Select COUNT(*) FROM #TableNames)
+		BEGIN
+			SET @fileName = (SELECT fileName FROM #TableNames WHERE Id = @i);
+			SET @sql = 'BULK INSERT DeliusStaging.' + @fileName + ' FROM ''' + @basePath + @fileName + '.txt''' +
+			' WITH (FieldTerminator=''|'', RowTerminator = ''0x0d0a'', MAXERRORS = 1000)';
+			
+			EXEC (@sql);
+			SET @i = @i+1;
+		END
+
+		--Standardise Delius Data
+        Declare @retMessage varchar(500);
+		EXEC [DeliusStaging].[StandardiseData] @retMessage;
 
 		UPDATE [DeliusRunningPictureDb].[DeliusRunningPicture].[ProcessedFiles]
 		SET [Status] = 'Imported'
 		WHERE FileName = @processedFile;
-
+		
 		COMMIT TRANSACTION;
 	END TRY
 	BEGIN CATCH
 		ROLLBACK TRANSACTION;
 		THROW;
 	END CATCH
-RETURN 0
-END
+RETURN 0

src/Database/OfflocStagingDb/Stored Procedures/Import.sql

@@ -2,23 +2,75 @@
 -- Author:		Daniel Fenelon
 -- Create date: 18/03/2024
 -- Description:	Import of the OffLoc files
--- Note:        File loading removed - data is now staged via SqlBulkCopy
---              in the DbInteractions service. This procedure handles
---              post-load standardisation and status update only.
 -- =============================================
 
 CREATE PROCEDURE [OfflocStaging].[Import]
+    @basePath VARCHAR(100),
     @processedFile VARCHAR(50)
 AS
 BEGIN
 
     SET NOCOUNT ON;
     SET DATEFORMAT DMY;
+    
+    CREATE TABLE #TableNames
+    (
+        Id INT PRIMARY KEY IDENTITY,
+        fileName VARCHAR(50) NOT NULL
+    );
+    INSERT INTO #TableNames
+    (
+        fileName
+    )
+    VALUES
+    ('Activities'),
+    ('Addresses'),
+    ('Agencies'),
+    ('Assessments'),
+    ('Bookings'),
+    ('Employment'),
+    ('Flags'),
+    ('Identifiers'),
+    ('IncentiveLevel'),
+    ('Locations'),
+    ('MainOffence'),
+    ('Movements'),
+    ('OffenderAgencies'),
+    ('OffenderStatus'),
+    ('OtherOffences'),
+    ('PersonalDetails'),
+    ('PNC'),
+    ('PreviousPrisonNumbers'),
+    ('SentenceInformation'),
+    ('SexOffenders');
+
+    DECLARE @i INT;
+    DECLARE @sql VARCHAR(250);
+    DECLARE @fileName VARCHAR(150);
+
+    SET @i = 1;
 
     BEGIN TRANSACTION;
     BEGIN TRY
-        DECLARE @retMessage VARCHAR(500);
-        EXEC [OfflocStaging].[StandardiseData] @retMessage OUTPUT;
+        WHILE @i <= (SELECT COUNT(*)FROM #TableNames)
+        BEGIN
+            SET @fileName =
+            (
+                SELECT fileName FROM #TableNames WHERE Id = @i
+            );
+            SET @sql
+                = N'BULK INSERT OfflocStaging.' + @fileName + N' FROM ''' + @basePath + @fileName + '.txt'''
+                  + N' WITH (FieldTerminator = ''|'', RowTerminator = ''0x0d0a'', MAXERRORS = 1000)';
+            EXEC (@sql);
+
+            SET @i = @i + 1;
+        END;
+        DECLARE @stringDate CHAR(8);
+        SET @stringDate = SUBSTRING(@processedFile, 18, 8);
+
+        --Standardise Offloc Data
+        Declare @retMessage varchar(500);
+        EXEC [OfflocStaging].[StandardiseData] @retMessage;
 
         UPDATE [OfflocRunningPictureDb].[OfflocRunningPicture].[ProcessedFiles]
         SET [Status] = 'Imported'