using FluentAssertions; using Microsoft.Agents.AI; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Moq; using OpenClawNet.Agent; using OpenClawNet.Models.Abstractions; using OpenClawNet.Models.AzureOpenAI; using OpenClawNet.Models.Ollama; using OpenClawNet.Storage; using OpenClawNet.Tools.Abstractions; #pragma warning disable MAAI001 namespace OpenClawNet.UnitTests.Integration; /// /// Live integration tests that hit REAL LLM providers — no fakes. /// Ollama tests require a running instance at localhost:11434 with gemma4:e2b. /// Azure OpenAI tests require user secrets configured on the Gateway project. /// All tests skip gracefully when their provider is unavailable (safe for CI). /// Run with: dotnet test --filter "Category=Live" /// [Trait("Category", "Live")] public sealed class LiveLlmTests : IDisposable { private const string OllamaEndpoint = "http://localhost:11434"; private const string OllamaModel = "gemma4:e2b"; private const string GatewayUserSecretsId = "c15754a6-dc90-4a2a-aecb-1233d1a54fe1"; private readonly IDbContextFactory _dbFactory; public LiveLlmTests() { var options = new DbContextOptionsBuilder() .UseInMemoryDatabase(databaseName: Guid.NewGuid().ToString()) .Options; _dbFactory = new TestDbContextFactory(options); } // ── Ollama: Direct Client ───────────────────────────────────────────── [SkippableFact] public async Task Ollama_CompleteAsync_ReturnsResponse() { var client = BuildOllamaClient(); Skip.IfNot(await client.IsAvailableAsync(), "Ollama is not running at localhost:11434"); var response = await client.CompleteAsync(new ChatRequest { Messages = [ new ChatMessage { Role = ChatMessageRole.System, Content = "You are a helpful assistant. Be brief." }, new ChatMessage { Role = ChatMessageRole.User, Content = "Say hello in exactly one sentence." } ] }); response.Content.Should().NotBeNullOrWhiteSpace("Ollama should return a non-empty greeting"); response.Role.Should().Be(ChatMessageRole.Assistant); } [SkippableFact] public async Task Ollama_StreamAsync_YieldsTokens() { var client = BuildOllamaClient(); Skip.IfNot(await client.IsAvailableAsync(), "Ollama is not running at localhost:11434"); var chunks = new List(); await foreach (var chunk in client.StreamAsync(new ChatRequest { Messages = [ new ChatMessage { Role = ChatMessageRole.System, Content = "You are a helpful assistant. Be brief." }, new ChatMessage { Role = ChatMessageRole.User, Content = "Count from 1 to 3." } ] })) { chunks.Add(chunk); } chunks.Should().HaveCountGreaterThan(1, "streaming should yield multiple incremental tokens"); var fullContent = string.Concat(chunks.Select(c => c.Content ?? "")); fullContent.Should().NotBeNullOrWhiteSpace("concatenated tokens should form a non-empty response"); } [SkippableFact] public async Task Ollama_Pipeline_SendMessage_ReturnsStreamedContent() { var client = BuildOllamaClient(); Skip.IfNot(await client.IsAvailableAsync(), "Ollama is not running at localhost:11434"); var runtime = BuildRealRuntime(client); var context = new AgentContext { SessionId = Guid.NewGuid(), UserMessage = "Say hello in one sentence." }; var events = new List(); await foreach (var evt in runtime.ExecuteStreamAsync(context)) events.Add(evt); events.Should().NotBeEmpty("the pipeline should yield stream events from a real Ollama response"); var contentEvents = events.Where(e => e.Type == AgentStreamEventType.ContentDelta).ToList(); contentEvents.Should().NotBeEmpty("at least one content token should stream through the pipeline"); var allContent = string.Join("", contentEvents.Select(e => e.Content)); allContent.Should().NotBeNullOrWhiteSpace( "Ollama's real response should flow through DefaultAgentRuntime as content deltas"); } // ── Azure OpenAI: Direct Client ─────────────────────────────────────── [SkippableFact] public async Task AzureOpenAI_CompleteAsync_ReturnsResponse() { var (client, configured) = BuildAzureOpenAIClient(); Skip.IfNot(configured, "Azure OpenAI credentials not configured — set user secrets to run live tests."); var response = await client.CompleteAsync(new ChatRequest { Messages = [ new ChatMessage { Role = ChatMessageRole.System, Content = "You are a helpful assistant. Be brief." }, new ChatMessage { Role = ChatMessageRole.User, Content = "Say hello in exactly one sentence." } ] }); response.Content.Should().NotBeNullOrWhiteSpace("Azure OpenAI should return a non-empty greeting"); response.Role.Should().Be(ChatMessageRole.Assistant); response.Usage.Should().NotBeNull(); response.Usage!.TotalTokens.Should().BeGreaterThan(0); } [SkippableFact] public async Task AzureOpenAI_StreamAsync_YieldsTokens() { var (client, configured) = BuildAzureOpenAIClient(); Skip.IfNot(configured, "Azure OpenAI credentials not configured — set user secrets to run live tests."); var chunks = new List(); await foreach (var chunk in client.StreamAsync(new ChatRequest { Messages = [ new ChatMessage { Role = ChatMessageRole.System, Content = "You are a helpful assistant. Be brief." }, new ChatMessage { Role = ChatMessageRole.User, Content = "Count from 1 to 3." } ] })) { chunks.Add(chunk); } chunks.Should().HaveCountGreaterThan(1, "streaming should yield multiple incremental tokens"); var fullContent = string.Concat(chunks.Select(c => c.Content ?? "")); fullContent.Should().NotBeNullOrWhiteSpace("concatenated tokens should form a non-empty response"); } [SkippableFact] public async Task AzureOpenAI_Pipeline_SendMessage_ReturnsStreamedContent() { var (client, configured) = BuildAzureOpenAIClient(); Skip.IfNot(configured, "Azure OpenAI credentials not configured — set user secrets to run live tests."); var runtime = BuildRealRuntime(client); var context = new AgentContext { SessionId = Guid.NewGuid(), UserMessage = "Say hello in one sentence." }; var events = new List(); await foreach (var evt in runtime.ExecuteStreamAsync(context)) events.Add(evt); events.Should().NotBeEmpty("the pipeline should yield stream events from a real Azure OpenAI response"); var contentEvents = events.Where(e => e.Type == AgentStreamEventType.ContentDelta).ToList(); contentEvents.Should().NotBeEmpty("at least one content token should stream through the pipeline"); var allContent = string.Join("", contentEvents.Select(e => e.Content)); allContent.Should().NotBeNullOrWhiteSpace( "Azure OpenAI's real response should flow through DefaultAgentRuntime as content deltas"); } // ── Builders ────────────────────────────────────────────────────────── private static OllamaModelClient BuildOllamaClient() { var options = Options.Create(new OllamaOptions { Endpoint = OllamaEndpoint, Model = OllamaModel, Temperature = 0.0, MaxTokens = 256 }); var httpClient = new HttpClient { BaseAddress = new Uri(OllamaEndpoint) }; return new OllamaModelClient(httpClient, options, NullLogger.Instance); } private static (AzureOpenAIModelClient Client, bool IsConfigured) BuildAzureOpenAIClient() { var config = new ConfigurationBuilder() .AddUserSecrets(GatewayUserSecretsId, reloadOnChange: false) .Build(); var opts = new AzureOpenAIOptions(); if (config["Model:Endpoint"] is { Length: > 0 } ep) opts.Endpoint = ep; if (config["Model:ApiKey"] is { Length: > 0 } key) opts.ApiKey = key; if (config["Model:DeploymentName"] is { Length: > 0 } dep) opts.DeploymentName = dep; if (config["Model:AuthMode"] is { Length: > 0 } mode) opts.AuthMode = mode; var isConfigured = !string.IsNullOrEmpty(opts.Endpoint) && (opts.AuthMode.Equals("integrated", StringComparison.OrdinalIgnoreCase) || !string.IsNullOrEmpty(opts.ApiKey)); var client = new AzureOpenAIModelClient( Options.Create(opts), NullLogger.Instance); return (client, isConfigured); } /// /// Builds a REAL DefaultAgentRuntime pipeline with a live IModelClient. /// No fakes for the model — only mock the peripherals (tools, storage, summary). /// private DefaultAgentRuntime BuildRealRuntime(IModelClient realModelClient) { var store = new ConversationStore(_dbFactory); var promptComposer = BuildDefaultPromptComposer(); var toolExecutor = new Mock().Object; var toolRegistry = BuildEmptyRegistry(); var summaryService = BuildNoOpSummary(); var loggerFactory = NullLoggerFactory.Instance; return new DefaultAgentRuntime( realModelClient, promptComposer, toolExecutor, toolRegistry, store, summaryService, new OpenClawNet.Memory.StubAgentMemoryStore(), new OpenClawNet.Agent.ToolApproval.ToolApprovalCoordinator( NullLogger.Instance), loggerFactory, NullLogger.Instance); } private static IPromptComposer BuildDefaultPromptComposer() { var workspaceLoader = new Mock(); workspaceLoader.Setup(w => w.LoadAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(new BootstrapContext(null, null, null)); var skillService = new Mock(); skillService.Setup(s => s.FindRelevantSkillsAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(Array.Empty()); return new DefaultPromptComposer( workspaceLoader.Object, skillService.Object, NullLogger.Instance, Options.Create(new WorkspaceOptions())); } private static IToolRegistry BuildEmptyRegistry() { var registry = new Mock(); registry.Setup(r => r.GetToolManifest()).Returns([]); registry.Setup(r => r.GetAllTools()).Returns([]); return registry.Object; } private static ISummaryService BuildNoOpSummary() { var summary = new Mock(); summary.Setup(s => s.SummarizeIfNeededAsync( It.IsAny(), It.IsAny>(), It.IsAny())) .ReturnsAsync((string?)null); return summary.Object; } public void Dispose() { } private sealed class TestDbContextFactory(DbContextOptions options) : IDbContextFactory { public OpenClawDbContext CreateDbContext() => new(options); } }