using FluentAssertions; using Microsoft.Agents.AI; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Moq; using OpenClawNet.Agent; using OpenClawNet.Models.Abstractions; using OpenClawNet.Storage; using OpenClawNet.Tools.Abstractions; using OpenClawNet.Tools.Calculator; using OpenClawNet.Tools.Core; using System.Net.Http; using System.Net.Sockets; using System.Text.RegularExpressions; #pragma warning disable MAAI001 namespace OpenClawNet.UnitTests.Integration; /// /// End-to-end live test of the full agent loop: prompt → LLM picks tool → tool /// executes → result feeds back → LLM produces final answer. Uses /// as the target tool because it's deterministic, /// approval-free, and easy to verify. /// /// Skips per-row when the provider isn't reachable (Ollama down, AOAI secrets /// missing). Run with: dotnet test --filter "Category=Live". /// [Trait("Category", "Live")] public sealed class LiveAgentLoopTests : IClassFixture { private readonly LiveTestFixture _fx; private readonly IDbContextFactory _dbFactory; public LiveAgentLoopTests(LiveTestFixture fx) { _fx = fx; var options = new DbContextOptionsBuilder() .UseInMemoryDatabase(databaseName: Guid.NewGuid().ToString()) .Options; _dbFactory = new TestDbContextFactory(options); } [SkippableTheory] [MemberData(nameof(LiveTestFixture.BothProviders), MemberType = typeof(LiveTestFixture))] public async Task Agent_MultiTurnToolExecution_CompletesSuccessfully( string providerName, Func pick) { var client = pick(_fx); Skip.If(client is null, $"{providerName} not configured — see LiveTestFixture for setup instructions."); if (providerName == "ollama") await LiveTestFixture.SkipIfOllamaUnavailableAsync(client!); // Use a 17 × 23 = 391 prompt: distinctive enough that the model is unlikely // to produce "391" without actually calling the calculator. var runtime = BuildRuntimeWithCalculator(client!); var ctx = new AgentContext { SessionId = Guid.NewGuid(), UserMessage = "Compute 17 * 23 using the calculator tool. " + "After the tool returns, reply with a single sentence containing the numeric answer." }; using var cts = new CancellationTokenSource( providerName == "ollama" ? _fx.OllamaTimeout : _fx.AzureTimeout); try { var result = await runtime.ExecuteAsync(ctx, cts.Token); result.Should().NotBeNull(); result.ExecutedToolCalls.Should().NotBeEmpty( "the agent loop must invoke the calculator tool at least once for the LLM to know 17*23 = 391"); result.ExecutedToolCalls .Should().Contain(c => c.Name.Equals("calculator", StringComparison.OrdinalIgnoreCase), "the calculator tool was the only one registered and the prompt explicitly requests it"); result.ToolResults.Should().NotBeEmpty(); result.ToolResults.Should().Contain(r => r.Success, "at least one calculator invocation should succeed"); result.FinalResponse.Should().NotBeNullOrWhiteSpace(); var toolOutput = result.ToolResults.First(r => r.Success).Output; toolOutput.Should().NotBeNullOrWhiteSpace(); var numericMatch = Regex.Match(toolOutput, @"\d+"); var expectedResult = numericMatch.Success ? numericMatch.Value : toolOutput; Skip.If(!result.FinalResponse!.Contains(expectedResult, StringComparison.OrdinalIgnoreCase), $"Model response did not contain tool result '{expectedResult}': {result.FinalResponse}"); result.FinalResponse!.Should().Contain(expectedResult, "the model must surface the tool result in its final answer"); } catch (Exception ex) when (providerName == "ollama" && IsTransientOllamaTransportFailure(ex)) { Skip.If(true, $"Ollama became unavailable during the live agent loop: {ex.GetBaseException().Message}"); throw; } } // ── Builders ────────────────────────────────────────────────────────── /// /// Builds a wired up with a real /// + containing a single /// . Everything else (storage, summary, workspace) is /// minimal so the test exercises the model+tool loop and nothing else. /// private DefaultAgentRuntime BuildRuntimeWithCalculator(IModelClient modelClient) { var loggerFactory = NullLoggerFactory.Instance; var registry = new ToolRegistry(); registry.Register(new CalculatorTool(NullLogger.Instance)); var executor = new ToolExecutor( registry, new AlwaysApprovePolicy(), NullLogger.Instance); var store = new ConversationStore(_dbFactory); var promptComposer = BuildDefaultPromptComposer(); var summaryService = BuildNoOpSummary(); return new DefaultAgentRuntime( modelClient, promptComposer, executor, registry, store, summaryService, new OpenClawNet.Memory.StubAgentMemoryStore(), new OpenClawNet.Agent.ToolApproval.ToolApprovalCoordinator( NullLogger.Instance), loggerFactory, NullLogger.Instance); } private static IPromptComposer BuildDefaultPromptComposer() { var workspaceLoader = new Mock(); workspaceLoader .Setup(w => w.LoadAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(new BootstrapContext(null, null, null)); var skillService = new Mock(); skillService.Setup(s => s.FindRelevantSkillsAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(Array.Empty()); return new DefaultPromptComposer( workspaceLoader.Object, skillService.Object, NullLogger.Instance, Options.Create(new WorkspaceOptions())); } private static ISummaryService BuildNoOpSummary() { var summary = new Mock(); summary .Setup(s => s.SummarizeIfNeededAsync( It.IsAny(), It.IsAny>(), It.IsAny())) .ReturnsAsync((string?)null); return summary.Object; } private sealed class TestDbContextFactory(DbContextOptions options) : IDbContextFactory { public OpenClawDbContext CreateDbContext() => new(options); } private static bool IsTransientOllamaTransportFailure(Exception ex) { for (Exception? current = ex; current is not null; current = current.InnerException) { if (current is TaskCanceledException or HttpRequestException or IOException or SocketException) { return true; } } return false; } }