Integration testing

src/Microsoft.TryDotNet.FileIntegration.Tests/CSharpFileCompilationTests.cs

@@ -0,0 +1,253 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Net.Http.Json;
+using System.Reflection;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using AwesomeAssertions;
+using AwesomeAssertions.Execution;
+using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.DotNet.Interactive.Connection;
+using Microsoft.DotNet.Interactive.CSharpProject.Events;
+using Microsoft.DotNet.Interactive.Events;
+using Xunit;
+
+namespace Microsoft.TryDotNet.FileIntegration.Tests;
+
+/// <summary>
+/// Integration tests that compile C# source files through the Try .NET API and verify
+/// compilation outcomes against optional chapter-listings.json metadata.
+/// </summary>
+/// <remarks>
+/// <para>Test data comes from two sources:</para>
+/// <list type="bullet">
+///   <item><strong>Embedded samples</strong> — <c>.cs</c> files in the <c>Samples/</c> directory
+///   deployed alongside the test assembly. These have no chapter metadata and are always
+///   expected to compile successfully.</item>
+///   <item><strong>External listings</strong> — files under the path specified by the
+///   <c>TRYDOTNET_LISTINGS_PATH</c> environment variable, following a
+///   <c>Chapter##/##.##.cs</c> naming convention. Compilation expectations are driven by
+///   the <see cref="ChapterListingsFixture"/> metadata.</item>
+/// </list>
+/// </remarks>
+public class CSharpFileCompilationTests : IClassFixture<WebApplicationFactory<Program>>, IClassFixture<ChapterListingsFixture>
+{
+    private readonly WebApplicationFactory<Program> _factory;
+    private readonly ChapterListingsFixture _listingsFixture;
+
+    public CSharpFileCompilationTests(WebApplicationFactory<Program> factory, ChapterListingsFixture listingsFixture)
+    {
+        _factory = factory;
+        _listingsFixture = listingsFixture;
+    }
+
+    /// <summary>
+    /// Provides test data by enumerating embedded sample files and, when configured,
+    /// external chapter listing files. Each row contains the full path, a display name,
+    /// and an optional chapter name.
+    /// </summary>
+    public static IEnumerable<object[]> SampleFiles()
+    {
+        // Always include embedded Samples/ directory
+        var samplesDir = Path.Combine(
+            Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!,
+            "Samples");
+
+        if (Directory.Exists(samplesDir))
+        {
+            foreach (var file in Directory.EnumerateFiles(samplesDir, "*.cs"))
+            {
+                // Embedded samples have no chapter
+                yield return new object[] { file, Path.GetFileName(file), null! };
+            }
+        }
+
+        // Additionally include external listings if configured
+        var externalPath = Environment.GetEnvironmentVariable(ChapterListingsFixture.ListingsPathEnvVar);
+
+        if (!string.IsNullOrEmpty(externalPath) && Directory.Exists(externalPath))
+        {
+            foreach (var file in Directory.EnumerateFiles(externalPath, "*.cs", SearchOption.AllDirectories))
+            {
+                var relativePath = Path.GetRelativePath(externalPath, file);
+                var segments = relativePath.Split(Path.DirectorySeparatorChar);
+
+                // Only include Chapter##/##.##.cs — exactly two segments, matching the numbering pattern
+                if (segments.Length != 2)
+                    continue;
+                if (!Regex.IsMatch(segments[0], @"^Chapter\d+$"))
+                    continue;
+                if (!Regex.IsMatch(segments[1], @"^\d+\.\d+\.cs$"))
+                    continue;
+
+                yield return new object[] { file, relativePath, segments[0] };
+            }
+        }
+    }
+
+    /// <summary>
+    /// Compiles a single C# source file through the Try .NET API and asserts the outcome
+    /// matches the expectation from chapter-listings.json (if available), or that embedded
+    /// samples compile successfully.
+    /// </summary>
+    [Theory]
+    [MemberData(nameof(SampleFiles))]
+    public async Task File_compilation_matches_expectation(string fullPath, string displayName, string? chapterName)
+    {
+        var fileName = Path.GetFileName(fullPath);
+        var fileContent = await File.ReadAllTextAsync(fullPath);
+
+        var client = _factory.CreateDefaultClient();
+
+        var requestJson = BuildCommandPayload(fileName, fileContent);
+        var requestBody = JsonContent.Create(JsonDocument.Parse(requestJson).RootElement);
+
+        var response = await client.PostAsync("commands", requestBody);
+
+        var responseJson = JsonDocument.Parse(
+            await response.Content.ReadAsStringAsync(CancellationToken.None)).RootElement;
+
+        var events = responseJson
+            .GetProperty("events")
+            .EnumerateArray()
+            .Select(KernelEventEnvelope.Deserialize)
+            .Select(ee => ee.Event)
+            .ToList();
+
+        response.EnsureSuccessStatusCode();
+
+        var assemblyProduced = events.OfType<AssemblyProduced>().SingleOrDefault();
+        bool actualCompiled = assemblyProduced is not null;
+
+        // Record result for fixture update logic
+        if (chapterName is not null)
+        {
+            _listingsFixture.ActualResults[(chapterName, fileName)] = actualCompiled;
+        }
+
+        var expectedCompile = _listingsFixture.GetExpectedCanCompile(chapterName, fileName);
+
+        // Chapter files require the metadata JSON to be present. Fail early with a clear
+        // message rather than silently asserting "all chapter files compile", which is wrong
+        // for listings that intentionally reference types defined in other listingfiles.
+        if (chapterName is not null && _listingsFixture.Model is null)
+        {
+            var resolvedPath = _listingsFixture.JsonFilePath ?? "(unresolved — set TRYDOTNET_LISTINGS_PATH or TRYDOTNET_LISTINGS_METADATA)";
+            false.Should().BeTrue(
+                $"chapter-listings.json could not be loaded, so compilation expectations for " +
+                $"chapter files are unavailable. Resolved path: {resolvedPath}");
+            return;
+        }
+
+        if (expectedCompile is null)
+        {
+            // No metadata entry (embedded samples, unknown files): assert compilation succeeds
+            AssertCompilationSucceeded(assemblyProduced, events, displayName);
+        }
+        else if (_listingsFixture.UpdateEnabled && expectedCompile.Value != actualCompiled)
+        {
+            // Update mode: mismatch is noted but test passes — fixture writes the JSON on dispose
+            Console.WriteLine(
+                $"[UPDATE] '{displayName}': expected can_compile={expectedCompile.Value}, actual={actualCompiled}. Will update JSON on dispose.");
+        }
+        else if (expectedCompile.Value)
+        {
+            // Expected to compile successfully per chapter-listings.json
+            AssertCompilationSucceeded(assemblyProduced, events, displayName,
+                "per chapter-listings.json");
+        }
+        else
+        {
+            // Expected to fail compilation per chapter-listings.json
+            actualCompiled.Should().BeFalse(
+                $"Expected '{displayName}' to fail compilation per chapter-listings.json, but it succeeded.");
+        }
+    }
+
+    /// <summary>
+    /// Asserts that compilation produced a non-empty assembly, collecting diagnostic details
+    /// on failure for a descriptive assertion message.
+    /// </summary>
+    private static void AssertCompilationSucceeded(
+        AssemblyProduced? assemblyProduced,
+        List<KernelEvent> events,
+        string displayName,
+        string? context = null)
+    {
+        using var scope = new AssertionScope();
+
+        if (assemblyProduced is null)
+        {
+            var details = FormatCompilationFailureDetails(events);
+            var contextSuffix = context is not null ? $" {context}" : "";
+
+            assemblyProduced.Should().NotBeNull(
+                $"Expected compilation of '{displayName}' to produce an assembly{contextSuffix}, but it failed:{Environment.NewLine}{details}");
+        }
+        else
+        {
+            assemblyProduced.Assembly.Value.Should().NotBeNullOrWhiteSpace(
+                $"AssemblyProduced for '{displayName}' had an empty assembly value.");
+        }
+    }
+
+    /// <summary>
+    /// Extracts <see cref="CommandFailed"/> messages and <see cref="DiagnosticsProduced"/>
+    /// entries from the event stream and formats them into a human-readable string.
+    /// </summary>
+    private static string FormatCompilationFailureDetails(List<KernelEvent> events)
+    {
+        var failures = events.OfType<CommandFailed>()
+            .Select(f => f.Message);
+
+        var diagnostics = events.OfType<DiagnosticsProduced>()
+            .SelectMany(d => d.Diagnostics)
+            .Select(d => $"{d.Severity} {d.Code}: {d.Message} ({d.LinePositionSpan})");
+
+        return string.Join(Environment.NewLine,
+            failures.Concat(diagnostics).DefaultIfEmpty("No AssemblyProduced event and no diagnostics found."));
+    }
+
+    /// <summary>
+    /// Builds the JSON command payload that opens a project with a single file,
+    /// opens that file as the active document, and compiles the project.
+    /// </summary>
+    private static string BuildCommandPayload(string fileName, string fileContent)
+    {
+        var escapedContent = JsonEncodedText.Encode(fileContent).ToString();
+
+        return $$"""
+            {
+                "commands": [
+                    {
+                        "commandType": "OpenProject",
+                        "command": {
+                            "project": {
+                                "files": [
+                                    {
+                                        "relativeFilePath": "{{fileName}}",
+                                        "content": "{{escapedContent}}"
+                                    }
+                                ]
+                            }
+                        },
+                        "token": "file-test::1"
+                    },
+                    {
+                        "commandType": "OpenDocument",
+                        "command": {
+                            "relativeFilePath": "{{fileName}}"
+                        },
+                        "token": "file-test::2"
+                    },
+                    {
+                        "commandType": "CompileProject",
+                        "command": {},
+                        "token": "file-test::3"
+                    }
+                ]
+            }
+            """;
+    }
+}

src/Microsoft.TryDotNet.FileIntegration.Tests/ChapterListingsFixture.cs

@@ -0,0 +1,144 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Collections.Concurrent;
+using System.Text.Json;
+
+namespace Microsoft.TryDotNet.FileIntegration.Tests;
+
+/// <summary>
+/// xUnit class fixture that loads chapter-listings.json metadata, tracks actual compilation
+/// results from test runs, and optionally updates the JSON file when results diverge from
+/// expectations. Shared across all tests in <see cref="CSharpFileCompilationTests"/>.
+/// </summary>
+/// <remarks>
+/// <para>
+/// Configuration is driven by three environment variables:
+/// </para>
+/// <list type="bullet">
+///   <item><c>TRYDOTNET_LISTINGS_METADATA</c> — explicit path to the chapter-listings.json file.</item>
+///   <item><c>TRYDOTNET_LISTINGS_PATH</c> — path to the external listings directory; the JSON
+///   file is resolved relative to this path at <c>../../Properties/chapter-listings.json</c>.</item>
+///   <item><c>TRYDOTNET_UPDATE_LISTINGS</c> — when set to <c>"true"</c>, mismatches between
+///   expected and actual compilation results are written back to the JSON file on dispose.</item>
+/// </list>
+/// </remarks>
+public class ChapterListingsFixture : IDisposable
+{
+    internal const string ListingsMetadataEnvVar = "TRYDOTNET_LISTINGS_METADATA";
+    internal const string ListingsPathEnvVar = "TRYDOTNET_LISTINGS_PATH";
+    internal const string UpdateListingsEnvVar = "TRYDOTNET_UPDATE_LISTINGS";
+
+    /// <summary>Deserialized chapter-listings metadata, or <c>null</c> if no file was found.</summary>
+    public ChapterListingsRoot? Model { get; }
+
+    /// <summary>Absolute path to the chapter-listings.json file, or <c>null</c> if unresolved.</summary>
+    public string? JsonFilePath { get; }
+
+    /// <summary>Whether the update-on-mismatch mode is enabled via environment variable.</summary>
+    public bool UpdateEnabled { get; }
+
+    /// <summary>
+    /// Thread-safe dictionary populated by test runs with actual compilation outcomes.
+    /// Keyed by (chapter name, filename).
+    /// </summary>
+    public ConcurrentDictionary<(string chapter, string filename), bool> ActualResults { get; } = new();
+
+    public ChapterListingsFixture()
+    {
+        UpdateEnabled = string.Equals(
+            Environment.GetEnvironmentVariable(UpdateListingsEnvVar),
+            "true",
+            StringComparison.OrdinalIgnoreCase);
+
+        JsonFilePath = ResolveJsonPath();
+
+        if (JsonFilePath is not null && File.Exists(JsonFilePath))
+        {
+            var json = File.ReadAllText(JsonFilePath);
+            Model = JsonSerializer.Deserialize<ChapterListingsRoot>(json);
+        }
+    }
+
+    private static string? ResolveJsonPath()
+    {
+        var explicitPath = Environment.GetEnvironmentVariable(ListingsMetadataEnvVar);
+        if (!string.IsNullOrEmpty(explicitPath))
+        {
+            return explicitPath;
+        }
+
+        var listingsPath = Environment.GetEnvironmentVariable(ListingsPathEnvVar);
+        if (!string.IsNullOrEmpty(listingsPath))
+        {
+            return Path.GetFullPath(Path.Combine(listingsPath, "../../Properties/chapter-listings.json"));
+        }
+
+        return null;
+    }
+
+    /// <summary>
+    /// Looks up the expected <c>can_compile</c> value for a given chapter and filename.
+    /// Returns <c>null</c> when metadata is unavailable or no matching entry exists.
+    /// </summary>
+    public bool? GetExpectedCanCompile(string? chapterName, string filename)
+    {
+        if (chapterName is null || Model is null)
+        {
+            return null;
+        }
+
+        if (Model.Chapters.TryGetValue(chapterName, out var entries))
+        {
+            var entry = entries.FirstOrDefault(e =>
+                string.Equals(e.Filename, filename, StringComparison.OrdinalIgnoreCase));
+            return entry?.CanCompile;
+        }
+
+        return null;
+    }
+
+    /// <summary>
+    /// When update mode is enabled, writes any mismatched compilation results back to the
+    /// chapter-listings.json file so the metadata stays in sync with actual compiler behavior.
+    /// </summary>
+    public void Dispose()
+    {
+        if (!UpdateEnabled || Model is null || JsonFilePath is null)
+        {
+            return;
+        }
+
+        var mismatches = new List<string>();
+
+        foreach (var ((chapter, filename), actual) in ActualResults)
+        {
+            if (!Model.Chapters.TryGetValue(chapter, out var entries))
+            {
+                continue;
+            }
+
+            var entry = entries.FirstOrDefault(e =>
+                string.Equals(e.Filename, filename, StringComparison.OrdinalIgnoreCase));
+
+            if (entry is not null && entry.CanCompile != actual)
+            {
+                mismatches.Add($"  {chapter}/{filename}: can_compile {entry.CanCompile} → {actual}");
+                entry.CanCompile = actual;
+            }
+        }
+
+        if (mismatches.Count > 0)
+        {
+            var options = new JsonSerializerOptions { WriteIndented = true };
+            var updatedJson = JsonSerializer.Serialize(Model, options);
+            File.WriteAllText(JsonFilePath, updatedJson + Environment.NewLine);
+
+            Console.WriteLine($"[ChapterListingsFixture] Updated {JsonFilePath} with {mismatches.Count} change(s):");
+            foreach (var m in mismatches)
+            {
+                Console.WriteLine(m);
+            }
+        }
+    }
+}