chat: added new functions: msg_unique_id(), msg_unique_content() and tests

chatmastermind/chat.py

@@ -146,6 +146,25 @@ class Chat:
         except MessageError:
             pass
 
+    def msg_unique_id(self) -> None:
+        """
+        Remove duplicates from the internal messages, based on the msg_id (i. e. file_path).
+        Messages without a file_path are kept.
+        """
+        old_msgs = self.messages.copy()
+        self.messages = []
+        for m in old_msgs:
+            if not message_in(m, self.messages):
+                self.messages.append(m)
+        self.msg_sort()
+
+    def msg_unique_content(self) -> None:
+        """
+        Remove duplicates from the internal messages, based on the content (i. e. question + answer).
+        """
+        self.messages = list(set(self.messages))
+        self.msg_sort()
+
     def msg_clear(self) -> None:
         """
         Delete all messages.
@@ -356,7 +375,13 @@ class ChatDB(Chat):
             source_messages += read_dir(self.cache_path, mfilter=mfilter)
         if source in ['db', 'disk', 'all']:
             source_messages += read_dir(self.db_path, mfilter=mfilter)
-        return source_messages
+        # remove_duplicates and sort the list
+        unique_messages: list[Message] = []
+        for m in source_messages:
+            if not message_in(m, unique_messages):
+                unique_messages.append(m)
+        unique_messages.sort(key=lambda m: m.msg_id())
+        return unique_messages
 
     def msg_find(self,
                  msg_names: list[str],
@@ -430,6 +455,7 @@ class ChatDB(Chat):
         Write messages to the cache directory. If a message has no file_path, a new one
         will be created. If message.file_path exists, it will be modified to point to
         the cache directory.
+        Does NOT add the messages to the internal list (use 'cache_add()' for that)!
         """
         write_dir(self.cache_path,
                   messages if messages else self.messages,
@@ -480,6 +506,7 @@ class ChatDB(Chat):
         Write messages to the DB directory. If a message has no file_path, a new one
         will be created. If message.file_path exists, it will be modified to point
         to the DB directory.
+        Does NOT add the messages to the internal list (use 'db_add()' for that)!
         """
         write_dir(self.db_path,
                   messages if messages else self.messages,