Features/http parameters

.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+.history/

src/Paillave.Etl.Http/HttpConnectionInfoEx.cs

@@ -1,18 +1,22 @@
 using System;
 using System.Collections.Generic;
 using System.Net.Http;
+using Fluid;
+using Paillave.Etl.Core;
 
 namespace Paillave.Etl.Http;
 
 public static class IHttpConnectionInfoEx
 {
     public static HttpClient CreateHttpClient(
         IHttpConnectionInfo httpConnectionInfo,
-        IHttpAdapterParameters adapterParameters
+        IHttpAdapterParameters adapterParameters,
+        IFileValueMetadata? additionalMetadata=null
     )
     {
         HttpClient client = new HttpClient();
         client = ManageHeaders(client, httpConnectionInfo.HttpHeaders);
+        client = ManageAdditionalParameters(client, httpConnectionInfo, adapterParameters);
         client = ManageAuthentication(client, httpConnectionInfo, adapterParameters);
         return client;
     }
@@ -28,17 +32,46 @@ private static HttpClient ManageHeaders(HttpClient client, Dictionary<string, st
                         $"Header '{header.Key}' already exists in the request."
                     );
 
-                client.DefaultRequestHeaders.Add(header.Key, header.Value);
+                client.DefaultRequestHeaders.TryAddWithoutValidation(header.Key, header.Value);
             }
         }
 
         return client;
     }
 
+    private static HttpClient ManageAdditionalParameters(
+        HttpClient client,
+        IHttpConnectionInfo connectionParameters,
+        object? additionalMetadata=null)
+    {
+        connectionParameters.Url = ApplyFluidParameters(connectionParameters.Url, additionalMetadata);
+        return client;
+    }
+
+    private static string ApplyFluidParameters(string text, object? additionalMetadata=null)
+    {
+        if (additionalMetadata == null) return text;
+        var parser = new FluidParser();
+        if (parser.TryParse(text, out var fTemplate, out var error))
+        {
+            var context = new TemplateContext(additionalMetadata ?? new(), new TemplateOptions
+            {
+                ModelNamesComparer = StringComparer.InvariantCultureIgnoreCase,
+                MemberAccessStrategy = new UnsafeMemberAccessStrategy()
+            }, true);
+            return fTemplate.Render(context);
+        }
+        else
+        {
+            throw new Exception($"Error: {error}");
+        }
+    }
+
     private static HttpClient ManageAuthentication(
         HttpClient client,
         IHttpConnectionInfo connectionParameters,
-        IHttpAdapterParameters adapterParameters
+        IHttpAdapterParameters adapterParameters,
+        object? additionalMetadata=null
     )
     {
         if (connectionParameters.Authentication?.Bearer != null)
@@ -51,13 +84,19 @@ IHttpAdapterParameters adapterParameters
             connectionParameters.Authentication.Digest.AddAuthenticationHeaders(client);
         else if (connectionParameters.Authentication?.Xcbaccess != null)
         {
+            string? body = HttpHelpers.GetRequestBodyAsString(
+                    adapterParameters.Body,
+                    adapterParameters.RequestFormat
+                );
+            if (body != null && additionalMetadata != null)
+            {
+                body=ApplyFluidParameters(body,additionalMetadata);
+            }
+
             connectionParameters.Authentication.Xcbaccess.SetMethodPathBody(
                 adapterParameters.Method,
                 connectionParameters.Url,
-                HttpHelpers.GetRequestBodyAsString(
-                    adapterParameters.Body,
-                    adapterParameters.RequestFormat
-                )
+                body
             );
             connectionParameters.Authentication.Xcbaccess.AddAuthenticationHeaders(client);
         }

src/Paillave.Etl.Http/HttpFileValue.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using Paillave.Etl.Core;
+using Fluid;
 
 namespace Paillave.Etl.Http;
 
@@ -11,14 +12,17 @@ public class HttpFileValue : FileValueBase<HttpFileValueMetadata>
     private readonly IHttpConnectionInfo _connectionInfo;
     private readonly IHttpAdapterParameters _parameters;
 
+    private readonly IFileValueMetadata? _additionalMetadata = null;
+
     public HttpFileValue(
         string name,
         string url,
         string connectorCode,
         string connectionName,
         string connectorName,
         IHttpConnectionInfo? connectionInfo = null,
-        IHttpAdapterParameters? parameters = null
+        IHttpAdapterParameters? parameters = null,
+        IFileValueMetadata? additionalMetadata = null
     )
         : base(
             new HttpFileValueMetadata
@@ -42,7 +46,7 @@ public override Stream GetContent() =>
 
     private Stream GetContentSingleTime()
     {
-        var httpClient = IHttpConnectionInfoEx.CreateHttpClient(_connectionInfo, _parameters);
+        var httpClient = IHttpConnectionInfoEx.CreateHttpClient(_connectionInfo, _parameters, _additionalMetadata);
 
         var response = HttpHelpers.GetResponse(_connectionInfo, _parameters, httpClient).Result;
 
@@ -60,7 +64,7 @@ private Stream GetContentSingleTime()
 
     protected override void DeleteFile()
     {
-        var httpClient = IHttpConnectionInfoEx.CreateHttpClient(_connectionInfo, _parameters);
+        var httpClient = IHttpConnectionInfoEx.CreateHttpClient(_connectionInfo, _parameters, _additionalMetadata);
 
         var parameters = new HttpAdapterProviderParameters(_parameters)
         {

src/Paillave.Etl.Http/HttpFileValueProcessor.cs

@@ -34,7 +34,8 @@ IExecutionContext context
 
         var httpClient = IHttpConnectionInfoEx.CreateHttpClient(
             connectionParameters,
-            processorParameters
+            processorParameters,
+            fileValue.Metadata
         );
 
         var response = HttpHelpers
@@ -67,7 +68,8 @@ IExecutionContext context
 
             push(
                 new HttpPostFileValue(
-                    content,
+                    //content,
+                    connectionParameters.Url.ToStream(),
                     fileName,
                     new HttpPostFileValueMetadata
                     {

src/Paillave.Etl.Http/Paillave.Etl.Http.csproj

@@ -14,7 +14,7 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" allowedVersions="13.0.1" />
-
+    <PackageReference Include="Fluid.Core" Version="2.24.0" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Paillave.Etl\Paillave.Etl.csproj" />

src/Paillave.Etl.Mail/MailFileValueProcessor.cs

@@ -109,6 +109,12 @@ private void SendSingleFile(MailAdapterConnectionParameters connectionParameters
                 smtpClient.Send(mailMessage);
             }
         }
+        /// <summary>
+        /// TODO : replace with liquid syntax
+        /// </summary>
+        /// <param name="text"></param>
+        /// <param name="metadata"></param>
+        /// <returns></returns>
         private static string FormatText(string text, JToken metadata)
         {
             var regex = new Regex("\\{(?<name>.+?)(:(.+?))?}", RegexOptions.Singleline);

src/Tutorials/BlogTutorial/Program copy.cs

@@ -9,7 +9,7 @@ namespace BlogTutorial
 {
     class Program
     {
-        static async Task Main(string[] args)
+        static async Task Main1(string[] args)
         {
             var processRunner = StreamProcessRunner.Create<string>(DefineProcess102);
             var executionOptions = new ExecutionOptions<string>

src/Tutorials/BlogTutorial/Program.cs

@@ -12,7 +12,7 @@ namespace BlogTutorial
 {
     class Program9
     {
-        static async Task Main9(string[] args)
+        static async Task Main(string[] args)
         {
             var processRunner = StreamProcessRunner.Create<string>(DefineProcess2);
             processRunner.DebugNodeStream += (sender, e)

src/Tutorials/Paillave.Etl.Samples/Program10.cs

@@ -0,0 +1,91 @@
+
+using Paillave.Etl.Core;
+using Paillave.Etl.Http;
+
+namespace Paillave.Etl.Samples
+{
+
+    class Program10
+    {
+        static async Task Main(string[] args)
+        {
+            var httpAdapterProviderParameters = new HttpAdapterProviderParameters
+            {
+                Method = HttpMethodCustomEnum.Post,
+                Body = new Dictionary<string, object>
+                {
+                    {"name", "{{AdditionalParameters.bodyValueName}}" },
+                    {"age", 30},
+                    {"city", "New York"}
+                }
+            };
+            var httpAdapterConnectionParameters = new HttpAdapterConnectionParameters
+            {
+                //Url = "https://www.google.com/search?q={{AdditionalParameters.query}}",
+                //Url = "https://www.w3schools.com/{{AdditionalParameters.format}}/{{AdditionalParameters.file}}"
+
+                Url = "http://echo.free.beeceptor.com/sample-request?author={{AdditionalParameters.author}}",
+            };
+
+            var connectors = new FileValueConnectors();
+            connectors.Register(
+                new HttpFileValueProvider(
+                    "MyHttpSourceForAdditionalParameters",
+                    "Input",
+                    "HttpConnection",
+                    httpAdapterConnectionParameters,
+                    httpAdapterProviderParameters
+                )
+            );
+
+            var executionOptions = new ExecutionOptions<string[]> { Connectors = connectors };
+            var processRunner = StreamProcessRunner.Create<string[]>(Import);
+            var res = await processRunner.ExecuteAsync(args, executionOptions);
+        }
+
+        public static void Import(ISingleStream<string[]> contextStream)
+        {
+            contextStream
+                .FromConnector("get from http", "MyHttpSourceForAdditionalParameters")
+                .Select(
+                    "select",
+                    i =>
+                    {
+                        if (((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters == null)
+                        {
+                            ((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters = new Dictionary<string, string>();
+                        }
+                        ((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters.Add("file", "xmlhttp_info.txt");
+                        ((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters.Add("format", "xml");
+                        ((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters.Add("query", "Voiture");
+
+
+
+                        ((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters.Add("author", "beeceptor");
+                        ((HttpFileValueMetadata)i.Metadata).Parameters.AdditionalParameters.Add("bodyValueName", "Mickael");
+
+
+
+
+                        return i;
+                    }
+                    )
+                 .Do(
+                    "print to console",
+                    i =>
+                    {
+                        using (var memoryStream = new MemoryStream())
+                        {
+                            i.GetContent().CopyTo(memoryStream);
+                            byte[] contentBytes = memoryStream.ToArray();
+                            string contentString = System.Text.Encoding.UTF8.GetString(
+                                contentBytes
+                            );
+                            Console.WriteLine($"result ({contentString}");
+                            Console.WriteLine($"result ({i.Name})");
+                        }
+                    }
+                );
+        }
+    }
+}

src/Tutorials/Paillave.Etl.Samples/Program9.cs

@@ -23,7 +23,7 @@ public class TestHeaders
 
     class Program9
     {
-        static async Task Main(string[] args)
+        static async Task Main9(string[] args)
         {
             var httpAdapterConnectionParameters = new HttpAdapterConnectionParameters
             {