From 21f81f3569e71bd570bf2f1564c09980de39c061 Mon Sep 17 00:00:00 2001 From: juk0de Date: Thu, 21 Sep 2023 07:48:24 +0200 Subject: [PATCH 01/11] question_cmd: implemented repetition of multiple messages --- chatmastermind/commands/question.py | 46 +++++++++++++++++++---------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/chatmastermind/commands/question.py b/chatmastermind/commands/question.py index 37b62c4..e35bfe5 100644 --- a/chatmastermind/commands/question.py +++ b/chatmastermind/commands/question.py @@ -105,6 +105,25 @@ def make_request(ai: AI, chat: ChatDB, message: Message, args: argparse.Namespac print(response.tokens) +def repeat_messages(messages: list[Message], ai: AI, chat: ChatDB, args: argparse.Namespace) -> None: + """ + Repeat the given messages using the given arguments. + """ + for msg in messages: + print(f"--------- Repeating message '{msg.msg_id()}': ---------") + # overwrite the latest message if requested or empty + # -> but not if it's in the DB! + if ((msg.answer is None or args.overwrite is True) + and (not chat.msg_in_db(msg))): # noqa: W503 + msg.clear_answer() + make_request(ai, chat, msg, args) + # otherwise create a new one + else: + args.ask = [msg.question] + message = create_message(chat, args) + make_request(ai, chat, message, args) + + def question_cmd(args: argparse.Namespace, config: Config) -> None: """ Handler for the 'question' command. @@ -120,7 +139,6 @@ def question_cmd(args: argparse.Namespace, config: Config) -> None: message = create_message(chat, args) if args.create: return - # create the correct AI instance ai: AI = create_ai(args, config) @@ -129,22 +147,18 @@ def question_cmd(args: argparse.Namespace, config: Config) -> None: make_request(ai, chat, message, args) # === REPEAT === elif args.repeat is not None: - lmessage = chat.msg_latest(loc='cache') - if lmessage is None: - print("No message found to repeat!") - sys.exit(1) + repeat_msgs: list[Message] = [] + # repeat latest message + if len(args.repeat) == 0: + lmessage = chat.msg_latest(loc='cache') + if lmessage is None: + print("No message found to repeat!") + sys.exit(1) + repeat_msgs.append(lmessage) + # repeat given message(s) else: - print(f"Repeating message '{lmessage.msg_id()}':") - # overwrite the latest message if requested or empty - if lmessage.answer is None or args.overwrite is True: - lmessage.clear_answer() - make_request(ai, chat, lmessage, args) - # otherwise create a new one - else: - args.ask = [lmessage.question] - message = create_message(chat, args) - make_request(ai, chat, message, args) - + repeat_msgs = chat.msg_find(args.repeat, loc='disk') + repeat_messages(repeat_msgs, ai, chat, args) # === PROCESS === elif args.process is not None: # TODO: process either all questions without an -- 2.36.6 From e9175afaceda2fea8e33b15f8cc5040b7308cb15 Mon Sep 17 00:00:00 2001 From: juk0de Date: Thu, 21 Sep 2023 18:05:35 +0200 Subject: [PATCH 02/11] test_question_cmd: added testcase for --repeat with multiple messages --- tests/test_question_cmd.py | 47 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_question_cmd.py b/tests/test_question_cmd.py index 89c72c7..c51d5fd 100644 --- a/tests/test_question_cmd.py +++ b/tests/test_question_cmd.py @@ -282,6 +282,9 @@ class TestQuestionCmd(TestQuestionCmdBase): # exclude '.next' return sorted([f for f in Path(tmp_dir.name).glob('*.[ty]*')]) + +class TestQuestionCmdAsk(TestQuestionCmd): + @mock.patch('chatmastermind.commands.question.create_ai') def test_ask_single_answer(self, mock_create_ai: MagicMock) -> None: """ @@ -370,6 +373,9 @@ class TestQuestionCmd(TestQuestionCmdBase): self.assertEqual(len(self.message_list(self.cache_dir)), 1) self.assert_messages_equal(cached_msg, [expected_question]) + +class TestQuestionCmdRepeat(TestQuestionCmd): + @mock.patch('chatmastermind.commands.question.create_ai') def test_repeat_single_question(self, mock_create_ai: MagicMock) -> None: """ @@ -511,3 +517,44 @@ class TestQuestionCmd(TestQuestionCmdBase): cached_msg = chat.msg_gather(loc='cache') self.assertEqual(len(self.message_list(self.cache_dir)), 2) self.assert_messages_equal(cached_msg, expected_responses) + + @mock.patch('chatmastermind.commands.question.create_ai') + def test_repeat_multiple_questions(self, mock_create_ai: MagicMock) -> None: + """ + Repeat multiple questions. + """ + # 1. create some questions / messages + # cached message without an answer + message1 = Message(Question('Question 1'), + ai='foo', + model='bla', + file_path=Path(self.cache_dir.name) / '0001.txt') + # cached message with an answer + message2 = Message(Question('Question 2'), + Answer('Answer 2'), + ai='openai', + model='gpt-3.5-turbo', + file_path=Path(self.cache_dir.name) / '0002.txt') + # DB message without an answer + message3 = Message(Question('Question 3'), + ai='openai', + model='gpt-3.5-turbo', + file_path=Path(self.db_dir.name) / '0003.txt') + message1.to_file() + message2.to_file() + message3.to_file() + # chat = ChatDB.from_dir(Path(self.cache_dir.name), + # Path(self.db_dir.name)) + + # 2. repeat all three questions (without overwriting) + self.args.ask = None + self.args.repeat = ['0001', '0002', '0003'] + self.args.overwrite = False + question_cmd(self.args, self.config) + # two new files should be in the cache directory + # * the repeated cached message with answer + # * the repeated DB message + # -> the cached message wihtout answer should be overwritten + self.assertEqual(len(self.message_list(self.cache_dir)), 4) + self.assertEqual(len(self.message_list(self.db_dir)), 1) + # FIXME: also compare actual content! -- 2.36.6 From 0657a1bab8c7953860a632c8f27be42abbf8688a Mon Sep 17 00:00:00 2001 From: juk0de Date: Thu, 21 Sep 2023 18:21:43 +0200 Subject: [PATCH 03/11] question_cmd: fixed AI and model arguments when repeating messages --- chatmastermind/commands/question.py | 34 +++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/chatmastermind/commands/question.py b/chatmastermind/commands/question.py index e35bfe5..bc4a8c4 100644 --- a/chatmastermind/commands/question.py +++ b/chatmastermind/commands/question.py @@ -2,6 +2,7 @@ import sys import argparse from pathlib import Path from itertools import zip_longest +from copy import deepcopy from ..configuration import Config from ..chat import ChatDB from ..message import Message, MessageFilter, MessageError, Question, source_code @@ -105,11 +106,37 @@ def make_request(ai: AI, chat: ChatDB, message: Message, args: argparse.Namespac print(response.tokens) -def repeat_messages(messages: list[Message], ai: AI, chat: ChatDB, args: argparse.Namespace) -> None: +def make_msg_args(msg: Message, args: argparse.Namespace) -> argparse.Namespace: + """ + Takes an existing message and CLI arguments, and returns modified args based + on the members of the given message. Used e.g. when repeating messages, where + it's necessary to determine the correct AI, module and output tags to use + (either from the existing message or the given args). + """ + msg_args = args + # if AI, model or output tags have not been specified, + # use those from the original message + if (args.AI is None + or args.model is None # noqa: W503 + or args.output_tags is None # noqa: W503 + or len(args.output_tags) == 0): # noqa: W503 + msg_args = deepcopy(args) + if args.AI is None and msg.ai is not None: + msg_args.AI = msg.ai + if args.model is None and msg.model is not None: + msg_args.model = msg.model + if (args.output_tags is None or len(args.output_tags) == 0) and msg.tags is not None: + msg_args.output_tags = msg.tags + return msg_args + + +def repeat_messages(messages: list[Message], chat: ChatDB, args: argparse.Namespace, config: Config) -> None: """ Repeat the given messages using the given arguments. """ + ai: AI for msg in messages: + ai = create_ai(make_msg_args(msg, args), config) print(f"--------- Repeating message '{msg.msg_id()}': ---------") # overwrite the latest message if requested or empty # -> but not if it's in the DB! @@ -139,11 +166,10 @@ def question_cmd(args: argparse.Namespace, config: Config) -> None: message = create_message(chat, args) if args.create: return - # create the correct AI instance - ai: AI = create_ai(args, config) # === ASK === if args.ask: + ai: AI = create_ai(args, config) make_request(ai, chat, message, args) # === REPEAT === elif args.repeat is not None: @@ -158,7 +184,7 @@ def question_cmd(args: argparse.Namespace, config: Config) -> None: # repeat given message(s) else: repeat_msgs = chat.msg_find(args.repeat, loc='disk') - repeat_messages(repeat_msgs, ai, chat, args) + repeat_messages(repeat_msgs, chat, args, config) # === PROCESS === elif args.process is not None: # TODO: process either all questions without an -- 2.36.6 From 33df84beaac0fa90f85b3fb6044e487f99832286 Mon Sep 17 00:00:00 2001 From: juk0de Date: Fri, 22 Sep 2023 07:41:15 +0200 Subject: [PATCH 04/11] ai_factory: added optional 'def_ai' and 'def_model' arguments to 'create_ai' --- chatmastermind/ai_factory.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/chatmastermind/ai_factory.py b/chatmastermind/ai_factory.py index 36a987b..42b27c1 100644 --- a/chatmastermind/ai_factory.py +++ b/chatmastermind/ai_factory.py @@ -3,18 +3,20 @@ Creates different AI instances, based on the given configuration. """ import argparse -from typing import cast +from typing import cast, Optional from .configuration import Config, AIConfig, OpenAIConfig from .ai import AI, AIError from .ais.openai import OpenAI -def create_ai(args: argparse.Namespace, config: Config) -> AI: # noqa: 11 +def create_ai(args: argparse.Namespace, config: Config, # noqa: 11 + def_ai: Optional[str] = None, + def_model: Optional[str] = None) -> AI: """ - Creates an AI subclass instance from the given arguments - and configuration file. If AI has not been set in the - arguments, it searches for the ID 'default'. If that - is not found, it uses the first AI in the list. + Creates an AI subclass instance from the given arguments and configuration file. + If AI has not been set in the arguments, it searches for the ID 'default'. If + that is not found, it uses the first AI in the list. It's also possible to + specify a default AI and model using 'def_ai' and 'def_model'. """ ai_conf: AIConfig if hasattr(args, 'AI') and args.AI: @@ -22,6 +24,8 @@ def create_ai(args: argparse.Namespace, config: Config) -> AI: # noqa: 11 ai_conf = config.ais[args.AI] except KeyError: raise AIError(f"AI ID '{args.AI}' does not exist in this configuration") + elif def_ai: + ai_conf = config.ais[def_ai] elif 'default' in config.ais: ai_conf = config.ais['default'] else: @@ -34,6 +38,8 @@ def create_ai(args: argparse.Namespace, config: Config) -> AI: # noqa: 11 ai = OpenAI(cast(OpenAIConfig, ai_conf)) if hasattr(args, 'model') and args.model: ai.config.model = args.model + elif def_model: + ai.config.model = def_model if hasattr(args, 'max_tokens') and args.max_tokens: ai.config.max_tokens = args.max_tokens if hasattr(args, 'temperature') and args.temperature: -- 2.36.6 From 80c5dcc8010ce08e0652690c40ee0d2ae060bb95 Mon Sep 17 00:00:00 2001 From: juk0de Date: Fri, 22 Sep 2023 07:51:05 +0200 Subject: [PATCH 05/11] question_cmd: input tag options without a tag (e. g. '-t') now select ALL tags --- chatmastermind/commands/question.py | 28 ++++++++++++++++++++++------ chatmastermind/main.py | 6 +++--- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/chatmastermind/commands/question.py b/chatmastermind/commands/question.py index bc4a8c4..4c8b81c 100644 --- a/chatmastermind/commands/question.py +++ b/chatmastermind/commands/question.py @@ -118,14 +118,13 @@ def make_msg_args(msg: Message, args: argparse.Namespace) -> argparse.Namespace: # use those from the original message if (args.AI is None or args.model is None # noqa: W503 - or args.output_tags is None # noqa: W503 - or len(args.output_tags) == 0): # noqa: W503 + or args.output_tags is None): # noqa: W503 msg_args = deepcopy(args) if args.AI is None and msg.ai is not None: msg_args.AI = msg.ai if args.model is None and msg.model is not None: msg_args.model = msg.model - if (args.output_tags is None or len(args.output_tags) == 0) and msg.tags is not None: + if args.output_tags is None and msg.tags is not None: msg_args.output_tags = msg.tags return msg_args @@ -151,13 +150,30 @@ def repeat_messages(messages: list[Message], chat: ChatDB, args: argparse.Namesp make_request(ai, chat, message, args) +def invert_input_tag_args(args: argparse.Namespace) -> None: + """ + Changes the semantics of the INPUT tags for this command: + * not tags specified on the CLI -> no tags are selected + * empty tags specified on the CLI -> all tags are selected + """ + if args.or_tags is None: + args.or_tags = set() + elif len(args.or_tags) == 0: + args.or_tags = None + if args.and_tags is None: + args.and_tags = set() + elif len(args.and_tags) == 0: + args.and_tags = None + + def question_cmd(args: argparse.Namespace, config: Config) -> None: """ Handler for the 'question' command. """ - mfilter = MessageFilter(tags_or=args.or_tags if args.or_tags is not None else set(), - tags_and=args.and_tags if args.and_tags is not None else set(), - tags_not=args.exclude_tags if args.exclude_tags is not None else set()) + invert_input_tag_args(args) + mfilter = MessageFilter(tags_or=args.or_tags, + tags_and=args.and_tags, + tags_not=args.exclude_tags) chat = ChatDB.from_dir(cache_path=Path(config.cache), db_path=Path(config.db), mfilter=mfilter) diff --git a/chatmastermind/main.py b/chatmastermind/main.py index 62c4539..ac4f7cc 100755 --- a/chatmastermind/main.py +++ b/chatmastermind/main.py @@ -34,13 +34,13 @@ def create_parser() -> argparse.ArgumentParser: # a parent parser for all commands that support tag selection tag_parser = argparse.ArgumentParser(add_help=False) - tag_arg = tag_parser.add_argument('-t', '--or-tags', nargs='+', + tag_arg = tag_parser.add_argument('-t', '--or-tags', nargs='*', help='List of tags (one must match)', metavar='OTAGS') tag_arg.completer = tags_completer # type: ignore - atag_arg = tag_parser.add_argument('-k', '--and-tags', nargs='+', + atag_arg = tag_parser.add_argument('-k', '--and-tags', nargs='*', help='List of tags (all must match)', metavar='ATAGS') atag_arg.completer = tags_completer # type: ignore - etag_arg = tag_parser.add_argument('-x', '--exclude-tags', nargs='+', + etag_arg = tag_parser.add_argument('-x', '--exclude-tags', nargs='*', help='List of tags to exclude', metavar='XTAGS') etag_arg.completer = tags_completer # type: ignore otag_arg = tag_parser.add_argument('-o', '--output-tags', nargs='+', -- 2.36.6 From b50caa345c1d78a90ef8cca916e15f3978a4de0d Mon Sep 17 00:00:00 2001 From: juk0de Date: Fri, 22 Sep 2023 13:38:24 +0200 Subject: [PATCH 06/11] test_question_cmd: introduced 'FakeAI' class --- tests/test_question_cmd.py | 299 ++++++++++++++++++++----------------- 1 file changed, 163 insertions(+), 136 deletions(-) diff --git a/tests/test_question_cmd.py b/tests/test_question_cmd.py index c51d5fd..df62023 100644 --- a/tests/test_question_cmd.py +++ b/tests/test_question_cmd.py @@ -4,9 +4,9 @@ import argparse import tempfile from pathlib import Path from unittest import mock -from unittest.mock import MagicMock, call, ANY -from typing import Optional -from chatmastermind.configuration import Config +from unittest.mock import MagicMock, call +from typing import Optional, Union +from chatmastermind.configuration import Config, AIConfig from chatmastermind.commands.question import create_message, question_cmd from chatmastermind.tags import Tag from chatmastermind.message import Message, Question, Answer @@ -14,6 +14,56 @@ from chatmastermind.chat import Chat, ChatDB from chatmastermind.ai import AI, AIResponse, Tokens, AIError +class FakeAI(AI): + """ + A mocked version of the 'AI' class. + """ + ID: str + name: str + config: AIConfig + + def models(self) -> list[str]: + raise NotImplementedError + + def tokens(self, data: Union[Message, Chat]) -> int: + return 123 + + def print(self) -> None: + pass + + def print_models(self) -> None: + pass + + def __init__(self, ID: str, model: str, error: bool = False): + self.ID = ID + self.model = model + self.error = error + + def request(self, + question: Message, + chat: Chat, + num_answers: int = 1, + otags: Optional[set[Tag]] = None) -> AIResponse: + """ + Mock the 'ai.request()' function by either returning fake + answers or raising an exception. + """ + if self.error: + raise AIError + question.answer = Answer("Answer 0") + question.tags = set(otags) if otags is not None else None + question.ai = self.ID + question.model = self.model + answers: list[Message] = [question] + for n in range(1, num_answers): + answers.append(Message(question=question.question, + answer=Answer(f"Answer {n}"), + tags=otags, + ai=self.ID, + model=self.model)) + return AIResponse(answers, Tokens(10, 10, 20)) + + class TestQuestionCmdBase(unittest.TestCase): def assert_messages_equal(self, msg1: list[Message], msg2: list[Message]) -> None: """ @@ -24,6 +74,18 @@ class TestQuestionCmdBase(unittest.TestCase): # exclude the file_path, compare only Q, A and metadata self.assertTrue(m1.equals(m2, file_path=False, verbose=True)) + def mock_create_ai(self, args: argparse.Namespace, config: Config) -> AI: + """ + Mocked 'create_ai' that returns a 'FakeAI' instance. + """ + return FakeAI(args.AI, args.model) + + def mock_create_ai_with_error(self, args: argparse.Namespace, config: Config) -> AI: + """ + Mocked 'create_ai' that returns a 'FakeAI' instance. + """ + return FakeAI(args.AI, args.model, error=True) + class TestMessageCreate(TestQuestionCmdBase): """ @@ -227,8 +289,8 @@ class TestQuestionCmd(TestQuestionCmdBase): ask=['What is the meaning of life?'], num_answers=1, output_tags=['science'], - AI='openai', - model='gpt-3.5-turbo', + AI='FakeAI', + model='FakeModel', or_tags=None, and_tags=None, exclude_tags=None, @@ -239,9 +301,39 @@ class TestQuestionCmd(TestQuestionCmdBase): process=None, overwrite=None ) - # create a mock AI instance - self.ai = MagicMock(spec=AI) - self.ai.request.side_effect = self.mock_request + + def create_single_message(self, args: argparse.Namespace, with_answer: bool = True) -> Message: + message = Message(Question(args.ask[0]), + tags=set(args.output_tags) if args.output_tags is not None else None, + ai=args.AI, + model=args.model, + file_path=Path(self.cache_dir.name) / '0001.txt') + if with_answer: + message.answer = Answer('Answer 0') + message.to_file() + return message + + def create_multiple_messages(self) -> list[Message]: + # cached message without an answer + message1 = Message(Question('Question 1'), + ai='foo', + model='bla', + file_path=Path(self.cache_dir.name) / '0001.txt') + # cached message with an answer + message2 = Message(Question('Question 2'), + Answer('Answer 0'), + ai='openai', + model='gpt-3.5-turbo', + file_path=Path(self.cache_dir.name) / '0002.txt') + # DB message without an answer + message3 = Message(Question('Question 3'), + ai='openai', + model='gpt-3.5-turbo', + file_path=Path(self.db_dir.name) / '0003.txt') + message1.to_file() + message2.to_file() + message3.to_file() + return [message1, message2, message3] def input_message(self, args: argparse.Namespace) -> Message: """ @@ -257,27 +349,6 @@ class TestQuestionCmd(TestQuestionCmdBase): ai=args.AI, model=args.model) - def mock_request(self, - question: Message, - chat: Chat, - num_answers: int = 1, - otags: Optional[set[Tag]] = None) -> AIResponse: - """ - Mock the 'ai.request()' function - """ - question.answer = Answer("Answer 0") - question.tags = set(otags) if otags else None - question.ai = 'FakeAI' - question.model = 'FakeModel' - answers: list[Message] = [question] - for n in range(1, num_answers): - answers.append(Message(question=question.question, - answer=Answer(f"Answer {n}"), - tags=otags, - ai='FakeAI', - model='FakeModel')) - return AIResponse(answers, Tokens(10, 10, 20)) - def message_list(self, tmp_dir: tempfile.TemporaryDirectory) -> list[Path]: # exclude '.next' return sorted([f for f in Path(tmp_dir.name).glob('*.[ty]*')]) @@ -290,21 +361,17 @@ class TestQuestionCmdAsk(TestQuestionCmd): """ Test single answer with no errors. """ - mock_create_ai.return_value = self.ai + mock_create_ai.side_effect = self.mock_create_ai expected_question = self.input_message(self.args) - expected_responses = self.mock_request(expected_question, - Chat([]), - self.args.num_answers, - self.args.output_tags).messages + fake_ai = self.mock_create_ai(self.args, self.config) + expected_responses = fake_ai.request(expected_question, + Chat([]), + self.args.num_answers, + self.args.output_tags).messages # execute the command question_cmd(self.args, self.config) - # check for correct request call - self.ai.request.assert_called_once_with(expected_question, - ANY, - self.args.num_answers, - self.args.output_tags) # check for the expected message files chat = ChatDB.from_dir(Path(self.cache_dir.name), Path(self.db_dir.name)) @@ -321,22 +388,17 @@ class TestQuestionCmdAsk(TestQuestionCmd): chat = MagicMock(spec=ChatDB) mock_from_dir.return_value = chat - mock_create_ai.return_value = self.ai + mock_create_ai.side_effect = self.mock_create_ai expected_question = self.input_message(self.args) - expected_responses = self.mock_request(expected_question, - Chat([]), - self.args.num_answers, - self.args.output_tags).messages + fake_ai = self.mock_create_ai(self.args, self.config) + expected_responses = fake_ai.request(expected_question, + Chat([]), + self.args.num_answers, + self.args.output_tags).messages # execute the command question_cmd(self.args, self.config) - # check for correct request call - self.ai.request.assert_called_once_with(expected_question, - chat, - self.args.num_answers, - self.args.output_tags) - # check for the correct ChatDB calls: # - initial question has been written (prior to the actual request) # - responses have been written (after the request) @@ -353,19 +415,13 @@ class TestQuestionCmdAsk(TestQuestionCmd): Provoke an error during the AI request and verify that the question has been correctly stored in the cache. """ - mock_create_ai.return_value = self.ai + mock_create_ai.side_effect = self.mock_create_ai_with_error expected_question = self.input_message(self.args) - self.ai.request.side_effect = AIError # execute the command with self.assertRaises(AIError): question_cmd(self.args, self.config) - # check for correct request call - self.ai.request.assert_called_once_with(expected_question, - ANY, - self.args.num_answers, - self.args.output_tags) # check for the expected message files chat = ChatDB.from_dir(Path(self.cache_dir.name), Path(self.db_dir.name)) @@ -381,28 +437,27 @@ class TestQuestionCmdRepeat(TestQuestionCmd): """ Repeat a single question. """ - # 1. ask a question - mock_create_ai.return_value = self.ai - expected_question = self.input_message(self.args) - expected_responses = self.mock_request(expected_question, - Chat([]), - self.args.num_answers, - self.args.output_tags).messages + mock_create_ai.side_effect = self.mock_create_ai + # create a message + message = self.create_single_message(self.args) + + # repeat the last question (without overwriting) + # -> expect two identical messages (except for the file_path) + self.args.ask = None + self.args.repeat = [] + self.args.output_tags = [] + self.args.overwrite = False + fake_ai = self.mock_create_ai(self.args, self.config) + expected_response = fake_ai.request(message, + Chat([]), + self.args.num_answers, + set(self.args.output_tags)).messages + expected_responses = expected_response + expected_response question_cmd(self.args, self.config) chat = ChatDB.from_dir(Path(self.cache_dir.name), Path(self.db_dir.name)) cached_msg = chat.msg_gather(loc='cache') - self.assertEqual(len(self.message_list(self.cache_dir)), 1) - self.assert_messages_equal(cached_msg, expected_responses) - - # 2. repeat the last question (without overwriting) - # -> expect two identical messages (except for the file_path) - self.args.ask = None - self.args.repeat = [] - self.args.overwrite = False - expected_responses += expected_responses - question_cmd(self.args, self.config) - cached_msg = chat.msg_gather(loc='cache') + print(self.message_list(self.cache_dir)) self.assertEqual(len(self.message_list(self.cache_dir)), 2) self.assert_messages_equal(cached_msg, expected_responses) @@ -411,31 +466,29 @@ class TestQuestionCmdRepeat(TestQuestionCmd): """ Repeat a single question and overwrite the old one. """ - # 1. ask a question - mock_create_ai.return_value = self.ai - expected_question = self.input_message(self.args) - expected_responses = self.mock_request(expected_question, - Chat([]), - self.args.num_answers, - self.args.output_tags).messages - question_cmd(self.args, self.config) + mock_create_ai.side_effect = self.mock_create_ai + # create a message + message = self.create_single_message(self.args) chat = ChatDB.from_dir(Path(self.cache_dir.name), Path(self.db_dir.name)) cached_msg = chat.msg_gather(loc='cache') assert cached_msg[0].file_path cached_msg_file_id = cached_msg[0].file_path.stem - self.assertEqual(len(self.message_list(self.cache_dir)), 1) - self.assert_messages_equal(cached_msg, expected_responses) - # 2. repeat the last question (WITH overwriting) + # repeat the last question (WITH overwriting) # -> expect a single message afterwards self.args.ask = None self.args.repeat = [] self.args.overwrite = True + fake_ai = self.mock_create_ai(self.args, self.config) + expected_response = fake_ai.request(message, + Chat([]), + self.args.num_answers, + set(self.args.output_tags)).messages question_cmd(self.args, self.config) cached_msg = chat.msg_gather(loc='cache') self.assertEqual(len(self.message_list(self.cache_dir)), 1) - self.assert_messages_equal(cached_msg, expected_responses) + self.assert_messages_equal(cached_msg, expected_response) # also check that the file ID has not been changed assert cached_msg[0].file_path self.assertEqual(cached_msg_file_id, cached_msg[0].file_path.stem) @@ -445,35 +498,31 @@ class TestQuestionCmdRepeat(TestQuestionCmd): """ Repeat a single question after an error. """ - # 1. ask a question and provoke an error - mock_create_ai.return_value = self.ai - expected_question = self.input_message(self.args) - self.ai.request.side_effect = AIError - with self.assertRaises(AIError): - question_cmd(self.args, self.config) + mock_create_ai.side_effect = self.mock_create_ai + # create a question WITHOUT an answer + # -> just like after an error, which is tested above + question = self.create_single_message(self.args, with_answer=False) chat = ChatDB.from_dir(Path(self.cache_dir.name), Path(self.db_dir.name)) cached_msg = chat.msg_gather(loc='cache') assert cached_msg[0].file_path cached_msg_file_id = cached_msg[0].file_path.stem - self.assertEqual(len(self.message_list(self.cache_dir)), 1) - self.assert_messages_equal(cached_msg, [expected_question]) - # 2. repeat the last question (without overwriting) + # repeat the last question (without overwriting) # -> expect a single message because if the original has # no answer, it should be overwritten by default self.args.ask = None self.args.repeat = [] self.args.overwrite = False - self.ai.request.side_effect = self.mock_request - expected_responses = self.mock_request(expected_question, - Chat([]), - self.args.num_answers, - self.args.output_tags).messages + fake_ai = self.mock_create_ai(self.args, self.config) + expected_response = fake_ai.request(question, + Chat([]), + self.args.num_answers, + self.args.output_tags).messages question_cmd(self.args, self.config) cached_msg = chat.msg_gather(loc='cache') self.assertEqual(len(self.message_list(self.cache_dir)), 1) - self.assert_messages_equal(cached_msg, expected_responses) + self.assert_messages_equal(cached_msg, expected_response) # also check that the file ID has not been changed assert cached_msg[0].file_path self.assertEqual(cached_msg_file_id, cached_msg[0].file_path.stem) @@ -483,21 +532,15 @@ class TestQuestionCmdRepeat(TestQuestionCmd): """ Repeat a single question with new arguments. """ - # 1. ask a question - mock_create_ai.return_value = self.ai - expected_question = self.input_message(self.args) - expected_responses = self.mock_request(expected_question, - Chat([]), - self.args.num_answers, - self.args.output_tags).messages - question_cmd(self.args, self.config) + mock_create_ai.side_effect = self.mock_create_ai + # create a message + message = self.create_single_message(self.args) chat = ChatDB.from_dir(Path(self.cache_dir.name), Path(self.db_dir.name)) cached_msg = chat.msg_gather(loc='cache') - self.assertEqual(len(self.message_list(self.cache_dir)), 1) - self.assert_messages_equal(cached_msg, expected_responses) + assert cached_msg[0].file_path - # 2. repeat the last question with new arguments (without overwriting) + # repeat the last question with new arguments (without overwriting) # -> expect two messages with identical question and answer, but different metadata self.args.ask = None self.args.repeat = [] @@ -505,44 +548,28 @@ class TestQuestionCmdRepeat(TestQuestionCmd): self.args.output_tags = ['newtag'] self.args.AI = 'newai' self.args.model = 'newmodel' - new_expected_question = Message(question=Question(expected_question.question), + new_expected_question = Message(question=Question(message.question), tags=set(self.args.output_tags), ai=self.args.AI, model=self.args.model) - expected_responses += self.mock_request(new_expected_question, + fake_ai = self.mock_create_ai(self.args, self.config) + new_expected_response = fake_ai.request(new_expected_question, Chat([]), self.args.num_answers, set(self.args.output_tags)).messages question_cmd(self.args, self.config) cached_msg = chat.msg_gather(loc='cache') self.assertEqual(len(self.message_list(self.cache_dir)), 2) - self.assert_messages_equal(cached_msg, expected_responses) + self.assert_messages_equal(cached_msg, [message] + new_expected_response) + print(cached_msg) + print(message) + print(new_expected_question) @mock.patch('chatmastermind.commands.question.create_ai') def test_repeat_multiple_questions(self, mock_create_ai: MagicMock) -> None: """ Repeat multiple questions. """ - # 1. create some questions / messages - # cached message without an answer - message1 = Message(Question('Question 1'), - ai='foo', - model='bla', - file_path=Path(self.cache_dir.name) / '0001.txt') - # cached message with an answer - message2 = Message(Question('Question 2'), - Answer('Answer 2'), - ai='openai', - model='gpt-3.5-turbo', - file_path=Path(self.cache_dir.name) / '0002.txt') - # DB message without an answer - message3 = Message(Question('Question 3'), - ai='openai', - model='gpt-3.5-turbo', - file_path=Path(self.db_dir.name) / '0003.txt') - message1.to_file() - message2.to_file() - message3.to_file() # chat = ChatDB.from_dir(Path(self.cache_dir.name), # Path(self.db_dir.name)) -- 2.36.6 From 3c932aa88e742dcaeb7139571b6b8f9b951e4888 Mon Sep 17 00:00:00 2001 From: juk0de Date: Fri, 22 Sep 2023 22:04:12 +0200 Subject: [PATCH 07/11] openai: fixed assignment of output tags --- chatmastermind/ais/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatmastermind/ais/openai.py b/chatmastermind/ais/openai.py index 0e7ad41..d7bb12f 100644 --- a/chatmastermind/ais/openai.py +++ b/chatmastermind/ais/openai.py @@ -44,7 +44,7 @@ class OpenAI(AI): frequency_penalty=self.config.frequency_penalty, presence_penalty=self.config.presence_penalty) question.answer = Answer(response['choices'][0]['message']['content']) - question.tags = otags + question.tags = set(otags) if otags is not None else None question.ai = self.ID question.model = self.config.model answers: list[Message] = [question] -- 2.36.6 From b83b396c7b853f566e5f393ec1ae0b37adde465d Mon Sep 17 00:00:00 2001 From: juk0de Date: Fri, 22 Sep 2023 22:05:05 +0200 Subject: [PATCH 08/11] question_cmd: fixed msg specific argument creation --- chatmastermind/commands/question.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/chatmastermind/commands/question.py b/chatmastermind/commands/question.py index 4c8b81c..da77e1a 100644 --- a/chatmastermind/commands/question.py +++ b/chatmastermind/commands/question.py @@ -106,7 +106,7 @@ def make_request(ai: AI, chat: ChatDB, message: Message, args: argparse.Namespac print(response.tokens) -def make_msg_args(msg: Message, args: argparse.Namespace) -> argparse.Namespace: +def create_msg_args(msg: Message, args: argparse.Namespace) -> argparse.Namespace: """ Takes an existing message and CLI arguments, and returns modified args based on the members of the given message. Used e.g. when repeating messages, where @@ -135,19 +135,20 @@ def repeat_messages(messages: list[Message], chat: ChatDB, args: argparse.Namesp """ ai: AI for msg in messages: - ai = create_ai(make_msg_args(msg, args), config) + msg_args = create_msg_args(msg, args) + ai = create_ai(msg_args, config) print(f"--------- Repeating message '{msg.msg_id()}': ---------") # overwrite the latest message if requested or empty # -> but not if it's in the DB! - if ((msg.answer is None or args.overwrite is True) + if ((msg.answer is None or msg_args.overwrite is True) and (not chat.msg_in_db(msg))): # noqa: W503 msg.clear_answer() - make_request(ai, chat, msg, args) + make_request(ai, chat, msg, msg_args) # otherwise create a new one else: - args.ask = [msg.question] - message = create_message(chat, args) - make_request(ai, chat, message, args) + msg_args.ask = [msg.question] + message = create_message(chat, msg_args) + make_request(ai, chat, message, msg_args) def invert_input_tag_args(args: argparse.Namespace) -> None: -- 2.36.6 From a478408449ba409fd9c9e199bcfb18e7cebccd44 Mon Sep 17 00:00:00 2001 From: juk0de Date: Sat, 23 Sep 2023 08:10:35 +0200 Subject: [PATCH 09/11] test_question_cmd: test fixes and cleanup --- tests/test_question_cmd.py | 156 +++++++++++++++++++++---------------- 1 file changed, 90 insertions(+), 66 deletions(-) diff --git a/tests/test_question_cmd.py b/tests/test_question_cmd.py index df62023..8e55b8f 100644 --- a/tests/test_question_cmd.py +++ b/tests/test_question_cmd.py @@ -2,6 +2,7 @@ import os import unittest import argparse import tempfile +from copy import copy from pathlib import Path from unittest import mock from unittest.mock import MagicMock, call @@ -302,53 +303,6 @@ class TestQuestionCmd(TestQuestionCmdBase): overwrite=None ) - def create_single_message(self, args: argparse.Namespace, with_answer: bool = True) -> Message: - message = Message(Question(args.ask[0]), - tags=set(args.output_tags) if args.output_tags is not None else None, - ai=args.AI, - model=args.model, - file_path=Path(self.cache_dir.name) / '0001.txt') - if with_answer: - message.answer = Answer('Answer 0') - message.to_file() - return message - - def create_multiple_messages(self) -> list[Message]: - # cached message without an answer - message1 = Message(Question('Question 1'), - ai='foo', - model='bla', - file_path=Path(self.cache_dir.name) / '0001.txt') - # cached message with an answer - message2 = Message(Question('Question 2'), - Answer('Answer 0'), - ai='openai', - model='gpt-3.5-turbo', - file_path=Path(self.cache_dir.name) / '0002.txt') - # DB message without an answer - message3 = Message(Question('Question 3'), - ai='openai', - model='gpt-3.5-turbo', - file_path=Path(self.db_dir.name) / '0003.txt') - message1.to_file() - message2.to_file() - message3.to_file() - return [message1, message2, message3] - - def input_message(self, args: argparse.Namespace) -> Message: - """ - Create the expected input message for a question using the - given arguments. - """ - # NOTE: we only use the first question from the "ask" list - # -> message creation using "question.create_message()" is - # tested above - # the answer is always empty for the input message - return Message(Question(args.ask[0]), - tags=args.output_tags, - ai=args.AI, - model=args.model) - def message_list(self, tmp_dir: tempfile.TemporaryDirectory) -> list[Path]: # exclude '.next' return sorted([f for f in Path(tmp_dir.name).glob('*.[ty]*')]) @@ -362,7 +316,11 @@ class TestQuestionCmdAsk(TestQuestionCmd): Test single answer with no errors. """ mock_create_ai.side_effect = self.mock_create_ai - expected_question = self.input_message(self.args) + expected_question = Message(Question(self.args.ask[0]), + tags=set(self.args.output_tags), + ai=self.args.AI, + model=self.args.model, + file_path=Path('')) fake_ai = self.mock_create_ai(self.args, self.config) expected_responses = fake_ai.request(expected_question, Chat([]), @@ -389,7 +347,11 @@ class TestQuestionCmdAsk(TestQuestionCmd): mock_from_dir.return_value = chat mock_create_ai.side_effect = self.mock_create_ai - expected_question = self.input_message(self.args) + expected_question = Message(Question(self.args.ask[0]), + tags=set(self.args.output_tags), + ai=self.args.AI, + model=self.args.model, + file_path=Path('')) fake_ai = self.mock_create_ai(self.args, self.config) expected_responses = fake_ai.request(expected_question, Chat([]), @@ -416,7 +378,11 @@ class TestQuestionCmdAsk(TestQuestionCmd): has been correctly stored in the cache. """ mock_create_ai.side_effect = self.mock_create_ai_with_error - expected_question = self.input_message(self.args) + expected_question = Message(Question(self.args.ask[0]), + tags=set(self.args.output_tags), + ai=self.args.AI, + model=self.args.model, + file_path=Path('')) # execute the command with self.assertRaises(AIError): @@ -439,20 +405,28 @@ class TestQuestionCmdRepeat(TestQuestionCmd): """ mock_create_ai.side_effect = self.mock_create_ai # create a message - message = self.create_single_message(self.args) + message = Message(Question(self.args.ask[0]), + Answer('Old Answer'), + tags=set(self.args.output_tags), + ai=self.args.AI, + model=self.args.model, + file_path=Path(self.cache_dir.name) / '0001.txt') + message.to_file() # repeat the last question (without overwriting) # -> expect two identical messages (except for the file_path) self.args.ask = None self.args.repeat = [] - self.args.output_tags = [] self.args.overwrite = False fake_ai = self.mock_create_ai(self.args, self.config) - expected_response = fake_ai.request(message, + # since the message's answer is modified, we use a copy here + # -> the original is used for comparison below + expected_response = fake_ai.request(copy(message), Chat([]), self.args.num_answers, set(self.args.output_tags)).messages - expected_responses = expected_response + expected_response + # we expect the original message + the one with the new response + expected_responses = [message] + expected_response question_cmd(self.args, self.config) chat = ChatDB.from_dir(Path(self.cache_dir.name), Path(self.db_dir.name)) @@ -468,7 +442,13 @@ class TestQuestionCmdRepeat(TestQuestionCmd): """ mock_create_ai.side_effect = self.mock_create_ai # create a message - message = self.create_single_message(self.args) + message = Message(Question(self.args.ask[0]), + Answer('Old Answer'), + tags=set(self.args.output_tags), + ai=self.args.AI, + model=self.args.model, + file_path=Path(self.cache_dir.name) / '0001.txt') + message.to_file() chat = ChatDB.from_dir(Path(self.cache_dir.name), Path(self.db_dir.name)) cached_msg = chat.msg_gather(loc='cache') @@ -501,7 +481,12 @@ class TestQuestionCmdRepeat(TestQuestionCmd): mock_create_ai.side_effect = self.mock_create_ai # create a question WITHOUT an answer # -> just like after an error, which is tested above - question = self.create_single_message(self.args, with_answer=False) + message = Message(Question(self.args.ask[0]), + tags=set(self.args.output_tags), + ai=self.args.AI, + model=self.args.model, + file_path=Path(self.cache_dir.name) / '0001.txt') + message.to_file() chat = ChatDB.from_dir(Path(self.cache_dir.name), Path(self.db_dir.name)) cached_msg = chat.msg_gather(loc='cache') @@ -515,7 +500,7 @@ class TestQuestionCmdRepeat(TestQuestionCmd): self.args.repeat = [] self.args.overwrite = False fake_ai = self.mock_create_ai(self.args, self.config) - expected_response = fake_ai.request(question, + expected_response = fake_ai.request(message, Chat([]), self.args.num_answers, self.args.output_tags).messages @@ -534,7 +519,13 @@ class TestQuestionCmdRepeat(TestQuestionCmd): """ mock_create_ai.side_effect = self.mock_create_ai # create a message - message = self.create_single_message(self.args) + message = Message(Question(self.args.ask[0]), + Answer('Old Answer'), + tags=set(self.args.output_tags), + ai=self.args.AI, + model=self.args.model, + file_path=Path(self.cache_dir.name) / '0001.txt') + message.to_file() chat = ChatDB.from_dir(Path(self.cache_dir.name), Path(self.db_dir.name)) cached_msg = chat.msg_gather(loc='cache') @@ -561,19 +552,48 @@ class TestQuestionCmdRepeat(TestQuestionCmd): cached_msg = chat.msg_gather(loc='cache') self.assertEqual(len(self.message_list(self.cache_dir)), 2) self.assert_messages_equal(cached_msg, [message] + new_expected_response) - print(cached_msg) - print(message) - print(new_expected_question) @mock.patch('chatmastermind.commands.question.create_ai') def test_repeat_multiple_questions(self, mock_create_ai: MagicMock) -> None: """ Repeat multiple questions. """ - # chat = ChatDB.from_dir(Path(self.cache_dir.name), - # Path(self.db_dir.name)) + mock_create_ai.side_effect = self.mock_create_ai + # 1. === create three questions === + # cached message without an answer + message1 = Message(Question(self.args.ask[0]), + tags=self.args.output_tags, + ai=self.args.AI, + model=self.args.model, + file_path=Path(self.cache_dir.name) / '0001.txt') + # cached message with an answer + message2 = Message(Question(self.args.ask[0]), + Answer('Old Answer'), + tags=self.args.output_tags, + ai=self.args.AI, + model=self.args.model, + file_path=Path(self.cache_dir.name) / '0002.txt') + # DB message without an answer + message3 = Message(Question(self.args.ask[0]), + tags=self.args.output_tags, + ai=self.args.AI, + model=self.args.model, + file_path=Path(self.db_dir.name) / '0003.txt') + message1.to_file() + message2.to_file() + message3.to_file() + questions = [message1, message2, message3] + expected_responses: list[Message] = [] + fake_ai = self.mock_create_ai(self.args, self.config) + for question in questions: + # since the message's answer is modified, we use a copy + # -> the original is used for comparison below + expected_responses += fake_ai.request(copy(question), + Chat([]), + self.args.num_answers, + set(self.args.output_tags)).messages - # 2. repeat all three questions (without overwriting) + # 2. === repeat all three questions (without overwriting) === self.args.ask = None self.args.repeat = ['0001', '0002', '0003'] self.args.overwrite = False @@ -581,7 +601,11 @@ class TestQuestionCmdRepeat(TestQuestionCmd): # two new files should be in the cache directory # * the repeated cached message with answer # * the repeated DB message - # -> the cached message wihtout answer should be overwritten + # -> the cached message without answer should be overwritten self.assertEqual(len(self.message_list(self.cache_dir)), 4) self.assertEqual(len(self.message_list(self.db_dir)), 1) - # FIXME: also compare actual content! + expected_cache_messages = [expected_responses[0], message2, expected_responses[1], expected_responses[2]] + chat = ChatDB.from_dir(Path(self.cache_dir.name), + Path(self.db_dir.name)) + cached_msg = chat.msg_gather(loc='cache') + self.assert_messages_equal(cached_msg, expected_cache_messages) -- 2.36.6 From 87b25993be3610a960eac51b9607a2e40bcbb268 Mon Sep 17 00:00:00 2001 From: juk0de Date: Sat, 23 Sep 2023 09:03:20 +0200 Subject: [PATCH 10/11] tests: moved 'FakeAI' and common functions to 'test_common.py' --- tests/test_common.py | 100 ++++++++++++++++++++++++++++++++++++ tests/test_question_cmd.py | 102 ++++++------------------------------- 2 files changed, 115 insertions(+), 87 deletions(-) create mode 100644 tests/test_common.py diff --git a/tests/test_common.py b/tests/test_common.py new file mode 100644 index 0000000..eff7c00 --- /dev/null +++ b/tests/test_common.py @@ -0,0 +1,100 @@ +import unittest +import argparse +from typing import Union, Optional +from chatmastermind.configuration import Config, AIConfig +from chatmastermind.tags import Tag +from chatmastermind.message import Message, Answer +from chatmastermind.chat import Chat +from chatmastermind.ai import AI, AIResponse, Tokens, AIError + + +class FakeAI(AI): + """ + A mocked version of the 'AI' class. + """ + ID: str + name: str + config: AIConfig + + def models(self) -> list[str]: + raise NotImplementedError + + def tokens(self, data: Union[Message, Chat]) -> int: + return 123 + + def print(self) -> None: + pass + + def print_models(self) -> None: + pass + + def __init__(self, ID: str, model: str, error: bool = False): + self.ID = ID + self.model = model + self.error = error + + def request(self, + question: Message, + chat: Chat, + num_answers: int = 1, + otags: Optional[set[Tag]] = None) -> AIResponse: + """ + Mock the 'ai.request()' function by either returning fake + answers or raising an exception. + """ + if self.error: + raise AIError + question.answer = Answer("Answer 0") + question.tags = set(otags) if otags is not None else None + question.ai = self.ID + question.model = self.model + answers: list[Message] = [question] + for n in range(1, num_answers): + answers.append(Message(question=question.question, + answer=Answer(f"Answer {n}"), + tags=otags, + ai=self.ID, + model=self.model)) + return AIResponse(answers, Tokens(10, 10, 20)) + + +class TestWithFakeAI(unittest.TestCase): + """ + Base class for all tests that need to use the FakeAI. + """ + def assert_msgs_equal_except_file_path(self, msg1: list[Message], msg2: list[Message]) -> None: + """ + Compare messages using Question, Answer and all metadata excecot for the file_path. + """ + self.assertEqual(len(msg1), len(msg2)) + for m1, m2 in zip(msg1, msg2): + # exclude the file_path, compare only Q, A and metadata + self.assertTrue(m1.equals(m2, file_path=False, verbose=True)) + + def assert_msgs_all_equal(self, msg1: list[Message], msg2: list[Message]) -> None: + """ + Compare messages using Question, Answer and ALL metadata. + """ + self.assertEqual(len(msg1), len(msg2)) + for m1, m2 in zip(msg1, msg2): + self.assertTrue(m1.equals(m2, verbose=True)) + + def assert_msgs_content_equal(self, msg1: list[Message], msg2: list[Message]) -> None: + """ + Compare messages using only Question and Answer. + """ + self.assertEqual(len(msg1), len(msg2)) + for m1, m2 in zip(msg1, msg2): + self.assertEqual(m1, m2) + + def mock_create_ai(self, args: argparse.Namespace, config: Config) -> AI: + """ + Mocked 'create_ai' that returns a 'FakeAI' instance. + """ + return FakeAI(args.AI, args.model) + + def mock_create_ai_with_error(self, args: argparse.Namespace, config: Config) -> AI: + """ + Mocked 'create_ai' that returns a 'FakeAI' instance. + """ + return FakeAI(args.AI, args.model, error=True) diff --git a/tests/test_question_cmd.py b/tests/test_question_cmd.py index 8e55b8f..f216164 100644 --- a/tests/test_question_cmd.py +++ b/tests/test_question_cmd.py @@ -1,94 +1,19 @@ import os -import unittest import argparse import tempfile from copy import copy from pathlib import Path from unittest import mock from unittest.mock import MagicMock, call -from typing import Optional, Union -from chatmastermind.configuration import Config, AIConfig +from chatmastermind.configuration import Config from chatmastermind.commands.question import create_message, question_cmd -from chatmastermind.tags import Tag from chatmastermind.message import Message, Question, Answer from chatmastermind.chat import Chat, ChatDB -from chatmastermind.ai import AI, AIResponse, Tokens, AIError +from chatmastermind.ai import AIError +from .test_common import TestWithFakeAI -class FakeAI(AI): - """ - A mocked version of the 'AI' class. - """ - ID: str - name: str - config: AIConfig - - def models(self) -> list[str]: - raise NotImplementedError - - def tokens(self, data: Union[Message, Chat]) -> int: - return 123 - - def print(self) -> None: - pass - - def print_models(self) -> None: - pass - - def __init__(self, ID: str, model: str, error: bool = False): - self.ID = ID - self.model = model - self.error = error - - def request(self, - question: Message, - chat: Chat, - num_answers: int = 1, - otags: Optional[set[Tag]] = None) -> AIResponse: - """ - Mock the 'ai.request()' function by either returning fake - answers or raising an exception. - """ - if self.error: - raise AIError - question.answer = Answer("Answer 0") - question.tags = set(otags) if otags is not None else None - question.ai = self.ID - question.model = self.model - answers: list[Message] = [question] - for n in range(1, num_answers): - answers.append(Message(question=question.question, - answer=Answer(f"Answer {n}"), - tags=otags, - ai=self.ID, - model=self.model)) - return AIResponse(answers, Tokens(10, 10, 20)) - - -class TestQuestionCmdBase(unittest.TestCase): - def assert_messages_equal(self, msg1: list[Message], msg2: list[Message]) -> None: - """ - Compare messages using more than just Question and Answer. - """ - self.assertEqual(len(msg1), len(msg2)) - for m1, m2 in zip(msg1, msg2): - # exclude the file_path, compare only Q, A and metadata - self.assertTrue(m1.equals(m2, file_path=False, verbose=True)) - - def mock_create_ai(self, args: argparse.Namespace, config: Config) -> AI: - """ - Mocked 'create_ai' that returns a 'FakeAI' instance. - """ - return FakeAI(args.AI, args.model) - - def mock_create_ai_with_error(self, args: argparse.Namespace, config: Config) -> AI: - """ - Mocked 'create_ai' that returns a 'FakeAI' instance. - """ - return FakeAI(args.AI, args.model, error=True) - - -class TestMessageCreate(TestQuestionCmdBase): +class TestMessageCreate(TestWithFakeAI): """ Test if messages created by the 'question' command have the correct format. @@ -275,7 +200,7 @@ It is embedded code """)) -class TestQuestionCmd(TestQuestionCmdBase): +class TestQuestionCmd(TestWithFakeAI): def setUp(self) -> None: # create DB and cache @@ -335,7 +260,7 @@ class TestQuestionCmdAsk(TestQuestionCmd): Path(self.db_dir.name)) cached_msg = chat.msg_gather(loc='cache') self.assertEqual(len(self.message_list(self.cache_dir)), 1) - self.assert_messages_equal(cached_msg, expected_responses) + self.assert_msgs_equal_except_file_path(cached_msg, expected_responses) @mock.patch('chatmastermind.commands.question.ChatDB.from_dir') @mock.patch('chatmastermind.commands.question.create_ai') @@ -393,7 +318,7 @@ class TestQuestionCmdAsk(TestQuestionCmd): Path(self.db_dir.name)) cached_msg = chat.msg_gather(loc='cache') self.assertEqual(len(self.message_list(self.cache_dir)), 1) - self.assert_messages_equal(cached_msg, [expected_question]) + self.assert_msgs_equal_except_file_path(cached_msg, [expected_question]) class TestQuestionCmdRepeat(TestQuestionCmd): @@ -433,7 +358,7 @@ class TestQuestionCmdRepeat(TestQuestionCmd): cached_msg = chat.msg_gather(loc='cache') print(self.message_list(self.cache_dir)) self.assertEqual(len(self.message_list(self.cache_dir)), 2) - self.assert_messages_equal(cached_msg, expected_responses) + self.assert_msgs_equal_except_file_path(cached_msg, expected_responses) @mock.patch('chatmastermind.commands.question.create_ai') def test_repeat_single_question_overwrite(self, mock_create_ai: MagicMock) -> None: @@ -468,7 +393,7 @@ class TestQuestionCmdRepeat(TestQuestionCmd): question_cmd(self.args, self.config) cached_msg = chat.msg_gather(loc='cache') self.assertEqual(len(self.message_list(self.cache_dir)), 1) - self.assert_messages_equal(cached_msg, expected_response) + self.assert_msgs_equal_except_file_path(cached_msg, expected_response) # also check that the file ID has not been changed assert cached_msg[0].file_path self.assertEqual(cached_msg_file_id, cached_msg[0].file_path.stem) @@ -507,7 +432,7 @@ class TestQuestionCmdRepeat(TestQuestionCmd): question_cmd(self.args, self.config) cached_msg = chat.msg_gather(loc='cache') self.assertEqual(len(self.message_list(self.cache_dir)), 1) - self.assert_messages_equal(cached_msg, expected_response) + self.assert_msgs_equal_except_file_path(cached_msg, expected_response) # also check that the file ID has not been changed assert cached_msg[0].file_path self.assertEqual(cached_msg_file_id, cached_msg[0].file_path.stem) @@ -551,7 +476,7 @@ class TestQuestionCmdRepeat(TestQuestionCmd): question_cmd(self.args, self.config) cached_msg = chat.msg_gather(loc='cache') self.assertEqual(len(self.message_list(self.cache_dir)), 2) - self.assert_messages_equal(cached_msg, [message] + new_expected_response) + self.assert_msgs_equal_except_file_path(cached_msg, [message] + new_expected_response) @mock.patch('chatmastermind.commands.question.create_ai') def test_repeat_multiple_questions(self, mock_create_ai: MagicMock) -> None: @@ -608,4 +533,7 @@ class TestQuestionCmdRepeat(TestQuestionCmd): chat = ChatDB.from_dir(Path(self.cache_dir.name), Path(self.db_dir.name)) cached_msg = chat.msg_gather(loc='cache') - self.assert_messages_equal(cached_msg, expected_cache_messages) + self.assert_msgs_equal_except_file_path(cached_msg, expected_cache_messages) + # check that the DB message has not been modified at all + db_msg = chat.msg_gather(loc='db') + self.assert_msgs_all_equal(db_msg, [message3]) -- 2.36.6 From 601ebe731a2f56a00d84f65ce8816df3885fb83e Mon Sep 17 00:00:00 2001 From: juk0de Date: Sun, 24 Sep 2023 08:53:37 +0200 Subject: [PATCH 11/11] test_question_cmd: added a new testcase and made the old cases more explicit (easier to read) --- tests/test_question_cmd.py | 100 +++++++++++++++++++++++++------------ 1 file changed, 68 insertions(+), 32 deletions(-) diff --git a/tests/test_question_cmd.py b/tests/test_question_cmd.py index f216164..77d679c 100644 --- a/tests/test_question_cmd.py +++ b/tests/test_question_cmd.py @@ -7,6 +7,7 @@ from unittest import mock from unittest.mock import MagicMock, call from chatmastermind.configuration import Config from chatmastermind.commands.question import create_message, question_cmd +from chatmastermind.tags import Tag from chatmastermind.message import Message, Question, Answer from chatmastermind.chat import Chat, ChatDB from chatmastermind.ai import AIError @@ -343,15 +344,14 @@ class TestQuestionCmdRepeat(TestQuestionCmd): self.args.ask = None self.args.repeat = [] self.args.overwrite = False - fake_ai = self.mock_create_ai(self.args, self.config) - # since the message's answer is modified, we use a copy here - # -> the original is used for comparison below - expected_response = fake_ai.request(copy(message), - Chat([]), - self.args.num_answers, - set(self.args.output_tags)).messages + expected_response = Message(Question(message.question), + Answer('Answer 0'), + ai=message.ai, + model=message.model, + tags=message.tags, + file_path=Path('')) # we expect the original message + the one with the new response - expected_responses = [message] + expected_response + expected_responses = [message] + [expected_response] question_cmd(self.args, self.config) chat = ChatDB.from_dir(Path(self.cache_dir.name), Path(self.db_dir.name)) @@ -381,19 +381,20 @@ class TestQuestionCmdRepeat(TestQuestionCmd): cached_msg_file_id = cached_msg[0].file_path.stem # repeat the last question (WITH overwriting) - # -> expect a single message afterwards + # -> expect a single message afterwards (with a new answer) self.args.ask = None self.args.repeat = [] self.args.overwrite = True - fake_ai = self.mock_create_ai(self.args, self.config) - expected_response = fake_ai.request(message, - Chat([]), - self.args.num_answers, - set(self.args.output_tags)).messages + expected_response = Message(Question(message.question), + Answer('Answer 0'), + ai=message.ai, + model=message.model, + tags=message.tags, + file_path=Path('')) question_cmd(self.args, self.config) cached_msg = chat.msg_gather(loc='cache') self.assertEqual(len(self.message_list(self.cache_dir)), 1) - self.assert_msgs_equal_except_file_path(cached_msg, expected_response) + self.assert_msgs_equal_except_file_path(cached_msg, [expected_response]) # also check that the file ID has not been changed assert cached_msg[0].file_path self.assertEqual(cached_msg_file_id, cached_msg[0].file_path.stem) @@ -424,15 +425,16 @@ class TestQuestionCmdRepeat(TestQuestionCmd): self.args.ask = None self.args.repeat = [] self.args.overwrite = False - fake_ai = self.mock_create_ai(self.args, self.config) - expected_response = fake_ai.request(message, - Chat([]), - self.args.num_answers, - self.args.output_tags).messages + expected_response = Message(Question(message.question), + Answer('Answer 0'), + ai=message.ai, + model=message.model, + tags=message.tags, + file_path=Path('')) question_cmd(self.args, self.config) cached_msg = chat.msg_gather(loc='cache') self.assertEqual(len(self.message_list(self.cache_dir)), 1) - self.assert_msgs_equal_except_file_path(cached_msg, expected_response) + self.assert_msgs_equal_except_file_path(cached_msg, [expected_response]) # also check that the file ID has not been changed assert cached_msg[0].file_path self.assertEqual(cached_msg_file_id, cached_msg[0].file_path.stem) @@ -457,26 +459,60 @@ class TestQuestionCmdRepeat(TestQuestionCmd): assert cached_msg[0].file_path # repeat the last question with new arguments (without overwriting) - # -> expect two messages with identical question and answer, but different metadata + # -> expect two messages with identical question but different metadata and new answer self.args.ask = None self.args.repeat = [] self.args.overwrite = False self.args.output_tags = ['newtag'] self.args.AI = 'newai' self.args.model = 'newmodel' - new_expected_question = Message(question=Question(message.question), - tags=set(self.args.output_tags), - ai=self.args.AI, - model=self.args.model) - fake_ai = self.mock_create_ai(self.args, self.config) - new_expected_response = fake_ai.request(new_expected_question, - Chat([]), - self.args.num_answers, - set(self.args.output_tags)).messages + new_expected_response = Message(Question(message.question), + Answer('Answer 0'), + ai='newai', + model='newmodel', + tags={Tag('newtag')}, + file_path=Path('')) question_cmd(self.args, self.config) cached_msg = chat.msg_gather(loc='cache') self.assertEqual(len(self.message_list(self.cache_dir)), 2) - self.assert_msgs_equal_except_file_path(cached_msg, [message] + new_expected_response) + self.assert_msgs_equal_except_file_path(cached_msg, [message] + [new_expected_response]) + + @mock.patch('chatmastermind.commands.question.create_ai') + def test_repeat_single_question_new_args_overwrite(self, mock_create_ai: MagicMock) -> None: + """ + Repeat a single question with new arguments, overwriting the old one. + """ + mock_create_ai.side_effect = self.mock_create_ai + # create a message + message = Message(Question(self.args.ask[0]), + Answer('Old Answer'), + tags=set(self.args.output_tags), + ai=self.args.AI, + model=self.args.model, + file_path=Path(self.cache_dir.name) / '0001.txt') + message.to_file() + chat = ChatDB.from_dir(Path(self.cache_dir.name), + Path(self.db_dir.name)) + cached_msg = chat.msg_gather(loc='cache') + assert cached_msg[0].file_path + + # repeat the last question with new arguments + self.args.ask = None + self.args.repeat = [] + self.args.overwrite = True + self.args.output_tags = ['newtag'] + self.args.AI = 'newai' + self.args.model = 'newmodel' + new_expected_response = Message(Question(message.question), + Answer('Answer 0'), + ai='newai', + model='newmodel', + tags={Tag('newtag')}, + file_path=Path('')) + question_cmd(self.args, self.config) + cached_msg = chat.msg_gather(loc='cache') + self.assertEqual(len(self.message_list(self.cache_dir)), 1) + self.assert_msgs_equal_except_file_path(cached_msg, [new_expected_response]) @mock.patch('chatmastermind.commands.question.create_ai') def test_repeat_multiple_questions(self, mock_create_ai: MagicMock) -> None: -- 2.36.6