diff --git a/Shayan1/Prod/tests/test_llm_player.py b/Shayan1/Prod/tests/test_llm_player.py new file mode 100644 index 0000000..2940ef4 --- /dev/null +++ b/Shayan1/Prod/tests/test_llm_player.py @@ -0,0 +1,209 @@ +import pytest +from unittest.mock import MagicMock, patch +from nash.game import Game +from nash.players import LLMPlayer, make_player, PLAYER_KINDS + + +@pytest.fixture +def mock_anthropic(): + with patch("anthropic.Anthropic") as mock: + yield mock + + +def test_llm_player_initialization(mock_anthropic): + game = Game( + players=["P1", "P2"], + strategies=[["Cooperate", "Defect"], ["Cooperate", "Defect"]], + payoffs=[ + [[2.0, 0.0], [3.0, 1.0]], + [[2.0, 3.0], [0.0, 1.0]] + ] + ) + # Test PLAYER_KINDS + assert "llm" in PLAYER_KINDS + + # Test make_player + player = make_player("llm", game, player_index=0, model="my-test-model") + assert isinstance(player, LLMPlayer) + assert player.game == game + assert player.player_index == 0 + assert player.model == "my-test-model" + mock_anthropic.assert_called_once() + + +def test_llm_player_choose_clean_json(mock_anthropic): + game = Game( + players=["P1", "P2"], + strategies=[["Cooperate", "Defect"], ["Cooperate", "Defect"]], + payoffs=[ + [[2.0, 0.0], [3.0, 1.0]], + [[2.0, 3.0], [0.0, 1.0]] + ] + ) + logs = [] + + def debug_logger(player_index, message): + logs.append((player_index, message)) + + player = make_player("llm", game, player_index=0, debug_logger=debug_logger) + + mock_client = mock_anthropic.return_value + mock_response = MagicMock() + mock_content = MagicMock() + mock_content.text = '{"reasoning": "I want to cooperate to maximize payoff.", "strategy_index": 0}' + mock_response.content = [mock_content] + mock_client.messages.create.return_value = mock_response + + # Choose action with history + history = [(0, 1), (1, 0)] + action = player.choose(history) + + # Assert return value is strategy_index + assert action == 0 + + # Assert logs captured reasoning + assert len(logs) == 1 + assert logs[0] == (0, "I want to cooperate to maximize payoff.") + + # Check that client.messages.create was called + mock_client.messages.create.assert_called_once() + + +@pytest.mark.parametrize("model_output,expected_idx,expected_reasoning", [ + ( + '```json\n{"reasoning": "Markdown fence with json key", "strategy_index": 1}\n```', + 1, + "Markdown fence with json key" + ), + ( + '```\n{"reasoning": "Markdown fence without json key", "strategy_index": 0}\n```', + 0, + "Markdown fence without json key" + ), + ( + 'Here is the response:\n{"reasoning": "Surrounding text", "strategy_index": 1}\nHope this helps!', + 1, + "Surrounding text" + ), + ( + 'Some intro...\n```json\n{"reasoning": "Intro and fence and outro", "strategy_index": 0}\n```\nSome outro...', + 0, + "Intro and fence and outro" + ), +]) +def test_llm_player_resilient_json_parsing(mock_anthropic, model_output, expected_idx, expected_reasoning): + game = Game( + players=["P1", "P2"], + strategies=[["Cooperate", "Defect"], ["Cooperate", "Defect"]], + payoffs=[ + [[2.0, 0.0], [3.0, 1.0]], + [[2.0, 3.0], [0.0, 1.0]] + ] + ) + player = LLMPlayer(game, player_index=0) + + mock_client = mock_anthropic.return_value + mock_response = MagicMock() + mock_content = MagicMock() + mock_content.text = model_output + mock_response.content = [mock_content] + mock_client.messages.create.return_value = mock_response + + action = player.choose([]) + assert action == expected_idx + + +def test_llm_player_invalid_json_raises(mock_anthropic): + game = Game( + players=["P1", "P2"], + strategies=[["Cooperate", "Defect"], ["Cooperate", "Defect"]], + payoffs=[ + [[2.0, 0.0], [3.0, 1.0]], + [[2.0, 3.0], [0.0, 1.0]] + ] + ) + player = LLMPlayer(game, player_index=0) + + mock_client = mock_anthropic.return_value + mock_response = MagicMock() + mock_content = MagicMock() + mock_content.text = "This is just text with no json braces at all." + mock_response.content = [mock_content] + mock_client.messages.create.return_value = mock_response + + with pytest.raises(ValueError, match="No JSON object found in model response"): + player.choose([]) + + +def test_llm_player_prompt_contents(mock_anthropic): + game = Game( + players=["P1", "P2"], + strategies=[["Cooperate", "Defect"], ["Cooperate", "Defect"]], + payoffs=[ + [[2.0, 0.0], [3.0, 1.0]], + [[2.0, 3.0], [0.0, 1.0]] + ] + ) + player = LLMPlayer(game, player_index=0) + + mock_client = mock_anthropic.return_value + mock_response = MagicMock() + mock_content = MagicMock() + mock_content.text = '{"reasoning": "strategic choice", "strategy_index": 1}' + mock_response.content = [mock_content] + mock_client.messages.create.return_value = mock_response + + history = [(0, 1), (1, 0)] + player.choose(history) + + # Retrieve the arguments passed to create + mock_client.messages.create.assert_called_once() + kwargs = mock_client.messages.create.call_args[1] + + # Check messages structure + assert "messages" in kwargs + messages = kwargs["messages"] + assert len(messages) == 1 + prompt = messages[0]["content"] + + # 1. Player name + assert "Player 'P1'" in prompt + + # 2. Strategy options + assert "Cooperate" in prompt + assert "Defect" in prompt + + # 3. Payoff matrix formatting + # Profile: (Cooperate | Cooperate) -> Payoffs: (2.00, 2.00) + assert "Profile: (Cooperate | Cooperate) -> Payoffs: (2.00, 2.00)" in prompt + assert "Profile: (Defect | Cooperate) -> Payoffs: (3.00, 0.00)" in prompt + + # 4. History of recent rounds + # Round 1: P1: Cooperate, P2: Defect + assert "Round 1: P1: Cooperate, P2: Defect" in prompt + assert "Round 2: P1: Defect, P2: Cooperate" in prompt + + +def test_llm_player_empty_history(mock_anthropic): + game = Game( + players=["P1", "P2"], + strategies=[["Cooperate", "Defect"], ["Cooperate", "Defect"]], + payoffs=[ + [[2.0, 0.0], [3.0, 1.0]], + [[2.0, 3.0], [0.0, 1.0]] + ] + ) + player = LLMPlayer(game, player_index=0) + + mock_client = mock_anthropic.return_value + mock_response = MagicMock() + mock_content = MagicMock() + mock_content.text = '{"reasoning": "no history", "strategy_index": 0}' + mock_response.content = [mock_content] + mock_client.messages.create.return_value = mock_response + + player.choose([]) + + kwargs = mock_client.messages.create.call_args[1] + prompt = kwargs["messages"][0]["content"] + assert "No rounds played yet." in prompt