feat(cli): add response guardrails and grounded followup re-anchoring
This commit is contained in:
@@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
from tai.ai_client import DEFAULT_AI_HOST, DEFAULT_MODEL, AIClient, AIConfig
|
||||
from tai.collectors import CollectedItem, CollectionReport
|
||||
from tai.prompt_builder import build_system_prompt, build_user_message
|
||||
from tai.prompt_builder import build_followup_message, build_system_prompt, build_user_message
|
||||
from tai.ssh_client import SSHCommandResult
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -218,3 +218,16 @@ def test_build_user_message_handles_no_output() -> None:
|
||||
report = _make_report([("empty", "cat /nonexistent", 1, "", "")])
|
||||
msg = build_user_message("test", report)
|
||||
assert "no output" in msg
|
||||
|
||||
|
||||
def test_build_followup_message_includes_question_context() -> None:
|
||||
report = _make_report([("kernel", "uname -a", 0, "Linux web01", "")])
|
||||
msg = build_followup_message(
|
||||
"nginx is failing",
|
||||
report,
|
||||
"what should I check next?",
|
||||
["is nginx running?", "show me logs"],
|
||||
)
|
||||
assert "Current follow-up question" in msg
|
||||
assert "what should I check next?" in msg
|
||||
assert "Recent user follow-up questions" in msg
|
||||
|
||||
24
tests/test_ai_guardrails.py
Normal file
24
tests/test_ai_guardrails.py
Normal file
@@ -0,0 +1,24 @@
|
||||
"""Tests for AI response guardrails."""
|
||||
|
||||
from tai.ai_guardrails import validate_ai_response
|
||||
|
||||
|
||||
def test_validate_ai_response_flags_missing_evidence_and_quotes() -> None:
|
||||
warnings = validate_ai_response("Root cause only, no structure.")
|
||||
assert any("Evidence section" in item for item in warnings)
|
||||
assert any("quoted evidence" in item for item in warnings)
|
||||
|
||||
|
||||
def test_validate_ai_response_flags_risky_actions() -> None:
|
||||
text = "Evidence: `PasswordAuthentication no`\nRun systemctl restart sshd now."
|
||||
warnings = validate_ai_response(text)
|
||||
assert any("modifying actions" in item for item in warnings)
|
||||
|
||||
|
||||
def test_validate_ai_response_allows_grounded_read_only_answer() -> None:
|
||||
text = (
|
||||
"Evidence: `PasswordAuthentication no`\n"
|
||||
"Recommended Actions: run `journalctl -u sshd -n 200 --no-pager`"
|
||||
)
|
||||
warnings = validate_ai_response(text)
|
||||
assert not warnings
|
||||
@@ -207,7 +207,7 @@ def test_interactive_unknown_command_prints_hint(monkeypatch) -> None: # type:
|
||||
commands = iter(["what should I check next?", "/quit"])
|
||||
monkeypatch.setattr("tai.cli.collect_from_plan", fake_collect_from_plan)
|
||||
monkeypatch.setattr(
|
||||
"tai.cli.AIClient.stream_messages",
|
||||
"tai.cli.AIClient.stream",
|
||||
lambda *_args, **_kwargs: iter(["Check logs."]),
|
||||
)
|
||||
monkeypatch.setattr("builtins.input", lambda _prompt: next(commands))
|
||||
|
||||
Reference in New Issue
Block a user