using FluentAssertions;
using Microsoft.Agents.AI;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Moq;
using OpenClawNet.Agent;
using OpenClawNet.Models.Abstractions;
using OpenClawNet.Storage;
using OpenClawNet.Tools.Abstractions;
using OpenClawNet.Tools.Calculator;
using OpenClawNet.Tools.Core;
using System.Net.Http;
using System.Net.Sockets;
using System.Text.RegularExpressions;
#pragma warning disable MAAI001
namespace OpenClawNet.UnitTests.Integration;
///
/// End-to-end live test of the full agent loop: prompt → LLM picks tool → tool
/// executes → result feeds back → LLM produces final answer. Uses
/// as the target tool because it's deterministic,
/// approval-free, and easy to verify.
///
/// Skips per-row when the provider isn't reachable (Ollama down, AOAI secrets
/// missing). Run with: dotnet test --filter "Category=Live".
///
[Trait("Category", "Live")]
public sealed class LiveAgentLoopTests : IClassFixture
{
private readonly LiveTestFixture _fx;
private readonly IDbContextFactory _dbFactory;
public LiveAgentLoopTests(LiveTestFixture fx)
{
_fx = fx;
var options = new DbContextOptionsBuilder()
.UseInMemoryDatabase(databaseName: Guid.NewGuid().ToString())
.Options;
_dbFactory = new TestDbContextFactory(options);
}
[SkippableTheory]
[MemberData(nameof(LiveTestFixture.BothProviders), MemberType = typeof(LiveTestFixture))]
public async Task Agent_MultiTurnToolExecution_CompletesSuccessfully(
string providerName,
Func pick)
{
var client = pick(_fx);
Skip.If(client is null,
$"{providerName} not configured — see LiveTestFixture for setup instructions.");
if (providerName == "ollama")
await LiveTestFixture.SkipIfOllamaUnavailableAsync(client!);
// Use a 17 × 23 = 391 prompt: distinctive enough that the model is unlikely
// to produce "391" without actually calling the calculator.
var runtime = BuildRuntimeWithCalculator(client!);
var ctx = new AgentContext
{
SessionId = Guid.NewGuid(),
UserMessage =
"Compute 17 * 23 using the calculator tool. " +
"After the tool returns, reply with a single sentence containing the numeric answer."
};
using var cts = new CancellationTokenSource(
providerName == "ollama" ? _fx.OllamaTimeout : _fx.AzureTimeout);
try
{
var result = await runtime.ExecuteAsync(ctx, cts.Token);
result.Should().NotBeNull();
result.ExecutedToolCalls.Should().NotBeEmpty(
"the agent loop must invoke the calculator tool at least once for the LLM to know 17*23 = 391");
result.ExecutedToolCalls
.Should().Contain(c => c.Name.Equals("calculator", StringComparison.OrdinalIgnoreCase),
"the calculator tool was the only one registered and the prompt explicitly requests it");
result.ToolResults.Should().NotBeEmpty();
result.ToolResults.Should().Contain(r => r.Success,
"at least one calculator invocation should succeed");
result.FinalResponse.Should().NotBeNullOrWhiteSpace();
var toolOutput = result.ToolResults.First(r => r.Success).Output;
toolOutput.Should().NotBeNullOrWhiteSpace();
var numericMatch = Regex.Match(toolOutput, @"\d+");
var expectedResult = numericMatch.Success ? numericMatch.Value : toolOutput;
Skip.If(!result.FinalResponse!.Contains(expectedResult, StringComparison.OrdinalIgnoreCase),
$"Model response did not contain tool result '{expectedResult}': {result.FinalResponse}");
result.FinalResponse!.Should().Contain(expectedResult,
"the model must surface the tool result in its final answer");
}
catch (Exception ex) when (providerName == "ollama" && IsTransientOllamaTransportFailure(ex))
{
Skip.If(true, $"Ollama became unavailable during the live agent loop: {ex.GetBaseException().Message}");
throw;
}
}
// ── Builders ──────────────────────────────────────────────────────────
///
/// Builds a wired up with a real
/// + containing a single
/// . Everything else (storage, summary, workspace) is
/// minimal so the test exercises the model+tool loop and nothing else.
///
private DefaultAgentRuntime BuildRuntimeWithCalculator(IModelClient modelClient)
{
var loggerFactory = NullLoggerFactory.Instance;
var registry = new ToolRegistry();
registry.Register(new CalculatorTool(NullLogger.Instance));
var executor = new ToolExecutor(
registry,
new AlwaysApprovePolicy(),
NullLogger.Instance);
var store = new ConversationStore(_dbFactory);
var promptComposer = BuildDefaultPromptComposer();
var summaryService = BuildNoOpSummary();
return new DefaultAgentRuntime(
modelClient,
promptComposer,
executor,
registry,
store,
summaryService,
new OpenClawNet.Memory.StubAgentMemoryStore(),
new OpenClawNet.Agent.ToolApproval.ToolApprovalCoordinator(
NullLogger.Instance),
loggerFactory,
NullLogger.Instance);
}
private static IPromptComposer BuildDefaultPromptComposer()
{
var workspaceLoader = new Mock();
workspaceLoader
.Setup(w => w.LoadAsync(It.IsAny(), It.IsAny()))
.ReturnsAsync(new BootstrapContext(null, null, null));
var skillService = new Mock();
skillService.Setup(s => s.FindRelevantSkillsAsync(It.IsAny(), It.IsAny(), It.IsAny()))
.ReturnsAsync(Array.Empty());
return new DefaultPromptComposer(
workspaceLoader.Object,
skillService.Object,
NullLogger.Instance,
Options.Create(new WorkspaceOptions()));
}
private static ISummaryService BuildNoOpSummary()
{
var summary = new Mock();
summary
.Setup(s => s.SummarizeIfNeededAsync(
It.IsAny(),
It.IsAny>(),
It.IsAny()))
.ReturnsAsync((string?)null);
return summary.Object;
}
private sealed class TestDbContextFactory(DbContextOptions options)
: IDbContextFactory
{
public OpenClawDbContext CreateDbContext() => new(options);
}
private static bool IsTransientOllamaTransportFailure(Exception ex)
{
for (Exception? current = ex; current is not null; current = current.InnerException)
{
if (current is TaskCanceledException or HttpRequestException or IOException or SocketException)
{
return true;
}
}
return false;
}
}