33 Commits

Author SHA1 Message Date
juk0de f17e76203a glossary cmd / main: added --print option 2024-02-26 16:34:30 +01:00
juk0de 6a77ec1d3b glossary test: added testcase for to_str() without description 2024-02-26 16:27:00 +01:00
juk0de f298a68140 glossary cmd test: added test for listing glossaries 2024-02-26 16:27:00 +01:00
juk0de 9bbf67af67 glossary: fixed printing of empty description 2024-02-26 16:27:00 +01:00
juk0de 284dd13201 glossary cmd: fixed globbing in 'list_glossaries' 2024-02-26 16:27:00 +01:00
juk0de 1932f8f6e9 configuration: improved error message when config file is missing 2024-02-26 16:27:00 +01:00
juk0de 15e8f8fd6b main: resolved conflicting short parameters 2024-02-26 16:27:00 +01:00
juk0de 9c683be994 glossary test: fixed cleanup of temporary files 2024-02-26 16:27:00 +01:00
juk0de 92fb2bbe15 glossary: added '__post_init__' 2024-02-26 16:27:00 +01:00
juk0de 2e0da31150 added test module for the 'glossary' command 2024-02-26 16:27:00 +01:00
juk0de ff6d4ded33 added 'glossary' command 2024-02-26 16:27:00 +01:00
juk0de 5377dc0784 main: missing directories are now created if user agrees 2024-02-26 16:27:00 +01:00
juk0de 3def4cb668 configuration: added 'glossaries' directory 2024-02-26 16:27:00 +01:00
juk0de 580c506483 glossary: now supports quoted and unquoted entries (incl. tests) 2024-02-26 16:27:00 +01:00
juk0de a1a090bcae glossary test: added testcases for 'to_str()' 2024-02-26 16:27:00 +01:00
juk0de 3cca32a40b glossary: added 'to_str()' function 2024-02-26 16:27:00 +01:00
juk0de 1b39fb1ac5 glossary test: added description test 2024-02-26 16:27:00 +01:00
juk0de b4ef2e43ca glossary: added description and removed useless input stripping 2024-02-26 16:27:00 +01:00
juk0de ff1e405991 glossary test: added suffix testcases 2024-02-26 16:27:00 +01:00
juk0de 4afd6d4e94 glossary: added suffix check 2024-02-26 16:27:00 +01:00
juk0de 94b812c31e glossary: added test module for glossaries 2024-02-26 16:27:00 +01:00
juk0de be873867ea added module 'glossary.py' 2024-02-26 16:27:00 +01:00
juk0de 82ad697b68 translation: added check for valid document format when using OpenAI 2024-02-26 16:27:00 +01:00
juk0de a185c0db7b translation: speficied / implemented the question format for OpenAI based translations 2024-02-26 16:27:00 +01:00
juk0de c1dc152f48 translation: some small required refactoring 2024-02-26 16:27:00 +01:00
juk0de f0129f7060 added new command 'translation' 2024-02-26 16:27:00 +01:00
Oleksandr Kozachuk 5d1bb1f9e4 Fix some of the commands. 2023-11-10 10:42:46 +01:00
Oleksandr Kozachuk 75a123eb72 Fix usage of the dynamic answer is some cases. 2023-10-24 12:59:13 +02:00
juk0de 7c1c67f8ff Merge pull request 'Dynamic Answer class and OpenAI streaming API' (#19) from dynamic_answer into main
Introduces several changes with the main objective of enabling OpenAI's streaming API in the chatmastermind application. This allows for the retrieval of AI responses gradually as a stream, which can significantly improve the user experience in interactions that involve large result sets.

* Added tiktoken import in 'openai.py' and modifications to the OpenAI class to support streaming. This includes the addition of a new class OpenAIAnswer to handle streaming API responses.
* Modified request function in the OpenAI class: the stream=True flag is added to the openai.ChatCompletion.create method to enable streaming API.
* Modified 'question.py' to print the answer parts as they are streamed.
* Replaced the Answer class's string data type with a generator which supports str and Generator[str, None, None] data types. Modifications are made to the Answer class methods to handle both data types accordingly.
* Updated the tests in 'test_ais_openai.py' and 'test_message.py' to reflect and validate these changes.
2023-10-21 15:50:45 +02:00
Oleksandr Kozachuk dbe72ff11c Activate and use OpenAI streaming API. 2023-10-21 14:21:48 +02:00
Oleksandr Kozachuk bbc1ab5a0a Fix source_code function with the dynamic answer class. 2023-10-20 14:02:09 +02:00
Oleksandr Kozachuk 2aee018708 Refactor message.Answer class in a way, that it can be constructed dynamically step by step, in preparation of using streaming API. 2023-10-20 13:43:31 +02:00
ok 17c6fa2453 Merge pull request 'Configurable glob and location on question and hist commands' (#18) from cust_loc_glob into main
Reviewed-on: #18
2023-10-20 09:47:03 +02:00
7 changed files with 530 additions and 61 deletions
+95
View File
@@ -0,0 +1,95 @@
import sys
import argparse
from pathlib import Path
from pydoc import pager
from ..configuration import Config
from ..glossary import Glossary
class GlossaryCmdError(Exception):
pass
def print_paged(text: str) -> None:
pager(text)
def get_glossary_file_path(name: str, config: Config) -> Path:
"""
Get the complete filename for a glossary with the given path.
"""
if not config.glossaries:
raise GlossaryCmdError("Can't create glossary name without a glossary directory")
return Path(config.glossaries, name).with_suffix(Glossary.file_suffix).absolute()
def list_glossaries(args: argparse.Namespace, config: Config) -> None:
"""
List existing glossaries in the 'glossaries' directory.
"""
if not config.glossaries:
raise GlossaryCmdError("Glossaries directory missing in the configuration file")
glossaries = Path(config.glossaries).glob(f'*{Glossary.file_suffix}')
for glo in sorted(glossaries):
print(Glossary.from_file(glo).to_str())
def print_glossary(args: argparse.Namespace, config: Config) -> None:
"""
Print an existing glossary.
"""
# sanity checks
if args.name is None:
raise GlossaryCmdError("Missing glossary name")
if config.glossaries is None and args.file is None:
raise GlossaryCmdError("Glossaries directory missing in the configuration file")
# create file path or use the given one
glo_file = Path(args.file) if args.file else get_glossary_file_path(args.name, config)
if not glo_file.exists():
raise GlossaryCmdError(f"Glossary '{glo_file}' does not exist")
# read glossary
glo = Glossary.from_file(glo_file)
print_paged(glo.to_str(with_entries=True))
def create_glossary(args: argparse.Namespace, config: Config) -> None:
"""
Create a new glossary and write it either to the glossaries directory
or the given file.
"""
# sanity checks
if args.name is None:
raise GlossaryCmdError("Missing glossary name")
if args.source_lang is None:
raise GlossaryCmdError("Missing source language")
if args.target_lang is None:
raise GlossaryCmdError("Missing target language")
if config.glossaries is None and args.file is None:
raise GlossaryCmdError("Glossaries directory missing in the configuration file")
# create file path or use the given one
glo_file = Path(args.file) if args.file else get_glossary_file_path(args.name, config)
if glo_file.exists():
raise GlossaryCmdError(f"Glossary '{glo_file}' already exists")
glo = Glossary(name=args.name,
source_lang=args.source_lang,
target_lang=args.target_lang,
desc=args.description,
file_path=glo_file)
glo.to_file()
print(f"Successfully created new glossary '{glo_file}'.")
def glossary_cmd(args: argparse.Namespace, config: Config) -> None:
"""
Handler for the 'glossary' command.
"""
try:
if args.create:
create_glossary(args, config)
elif args.list:
list_glossaries(args, config)
elif args.print:
print_glossary(args, config)
except GlossaryCmdError as err:
print(f"Error: {err}")
sys.exit(1)
+5 -1
View File
@@ -118,6 +118,7 @@ class Config:
# a default configuration # a default configuration
cache: str = '.' cache: str = '.'
db: str = './db/' db: str = './db/'
glossaries: str | None = './glossaries/'
ais: dict[str, AIConfig] = field(default_factory=create_default_ai_configs) ais: dict[str, AIConfig] = field(default_factory=create_default_ai_configs)
@classmethod @classmethod
@@ -135,7 +136,8 @@ class Config:
return cls( return cls(
cache=str(source['cache']) if 'cache' in source else '.', cache=str(source['cache']) if 'cache' in source else '.',
db=str(source['db']), db=str(source['db']),
ais=ais ais=ais,
glossaries=str(source['glossaries']) if 'glossaries' in source else None
) )
@classmethod @classmethod
@@ -148,6 +150,8 @@ class Config:
@classmethod @classmethod
def from_file(cls: Type[ConfigInst], path: str) -> ConfigInst: def from_file(cls: Type[ConfigInst], path: str) -> ConfigInst:
if not Path(path).exists():
raise ConfigError(f"Configuration file '{path}' not found. Use 'cmm config --create' to create one.")
with open(path, 'r') as f: with open(path, 'r') as f:
source = yaml.load(f, Loader=yaml.FullLoader) source = yaml.load(f, Loader=yaml.FullLoader)
return cls.from_dict(source) return cls.from_dict(source)
+45 -8
View File
@@ -7,7 +7,7 @@ import shutil
import csv import csv
from pathlib import Path from pathlib import Path
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import Type, TypeVar from typing import Type, TypeVar, ClassVar
GlossaryInst = TypeVar('GlossaryInst', bound='Glossary') GlossaryInst = TypeVar('GlossaryInst', bound='Glossary')
@@ -30,9 +30,10 @@ class Glossary:
""" """
A glossary consists of the following parameters: A glossary consists of the following parameters:
- Name (freely selectable) - Name (freely selectable)
- Path (full file path) - Path (full file path, suffix is automatically generated)
- Source language - Source language
- Target language - Target language
- Description (optional)
- Entries (pairs of source lang and target lang terms) - Entries (pairs of source lang and target lang terms)
- ID (automatically generated / modified, required by DeepL) - ID (automatically generated / modified, required by DeepL)
""" """
@@ -40,26 +41,39 @@ class Glossary:
name: str name: str
source_lang: str source_lang: str
target_lang: str target_lang: str
entries: dict[str, str] = field(default_factory=lambda: dict())
file_path: Path | None = None file_path: Path | None = None
desc: str | None = None
entries: dict[str, str] = field(default_factory=lambda: dict())
ID: str | None = None ID: str | None = None
file_suffix: ClassVar[str] = '.glo'
def __post_init__(self) -> None:
# FIXME: check for valid languages
pass
@classmethod @classmethod
def from_file(cls: Type[GlossaryInst], file_path: Path) -> GlossaryInst: def from_file(cls: Type[GlossaryInst], file_path: Path) -> GlossaryInst:
""" """
Create a glossary from the given file. Create a glossary from the given file.
""" """
if not file_path.exists():
raise GlossaryError(f"Glossary file '{file_path}' does not exist")
if file_path.suffix != cls.file_suffix:
raise GlossaryError(f"File type '{file_path.suffix}' is not supported")
with open(file_path, "r") as fd: with open(file_path, "r") as fd:
try: try:
data = yaml.load(fd, Loader=yaml.FullLoader) # use BaseLoader so every entry is read as a string
# remove any quotes from the entries that YAML may have added while dumping # - disables automatic conversions
# (e. g. for special keywords like 'yes') # - makes it possible to omit quoting for YAML keywords in entries (e. g. 'yes')
clean_entries = {key.strip('\"\' '): value for key, value in data['Entries'].items()} # - also correctly reads quoted entries
data = yaml.load(fd, Loader=yaml.BaseLoader)
clean_entries = data['Entries']
return cls(name=data['Name'], return cls(name=data['Name'],
source_lang=data['SourceLang'], source_lang=data['SourceLang'],
target_lang=data['TargetLang'], target_lang=data['TargetLang'],
entries=clean_entries,
file_path=file_path, file_path=file_path,
desc=data['Description'],
entries=clean_entries,
ID=data['ID'] if data['ID'] != 'None' else None) ID=data['ID'] if data['ID'] != 'None' else None)
except Exception: except Exception:
raise GlossaryError(f"'{file_path}' does not contain a valid glossary") raise GlossaryError(f"'{file_path}' does not contain a valid glossary")
@@ -72,10 +86,16 @@ class Glossary:
self.file_path = file_path self.file_path = file_path
if not self.file_path: if not self.file_path:
raise GlossaryError("Got no valid path to write glossary") raise GlossaryError("Got no valid path to write glossary")
# check / add valid suffix
if not self.file_path.suffix:
self.file_path = self.file_path.with_suffix(self.file_suffix)
elif self.file_path.suffix != self.file_suffix:
raise GlossaryError(f"File suffix '{self.file_path.suffix}' is not supported")
# write YAML # write YAML
with tempfile.NamedTemporaryFile(dir=self.file_path.parent, prefix=self.file_path.name, mode="w", delete=False) as temp_fd: with tempfile.NamedTemporaryFile(dir=self.file_path.parent, prefix=self.file_path.name, mode="w", delete=False) as temp_fd:
temp_file_path = Path(temp_fd.name) temp_file_path = Path(temp_fd.name)
data = {'Name': self.name, data = {'Name': self.name,
'Description': str(self.desc),
'ID': str(self.ID), 'ID': str(self.ID),
'SourceLang': self.source_lang, 'SourceLang': self.source_lang,
'TargetLang': self.target_lang, 'TargetLang': self.target_lang,
@@ -126,3 +146,20 @@ class Glossary:
self.entries[parts[0]] = parts[1] self.entries[parts[0]] = parts[1]
except Exception as e: except Exception as e:
raise GlossaryError(f"Error importing TSV: {e}") raise GlossaryError(f"Error importing TSV: {e}")
def to_str(self, with_entries: bool = False) -> str:
"""
Return the current glossary as a string.
"""
output: list[str] = []
output.append(f'{self.name} (ID: {self.ID}):')
if self.desc and self.desc != 'None':
output.append('- ' + self.desc)
output.append(f'- Languages: {self.source_lang} -> {self.target_lang}')
if with_entries:
output.append('- Entries:')
for source, target in self.entries.items():
output.append(f' {source} : {target}')
else:
output.append(f'- Entries: {len(self.entries)}')
return '\n'.join(output)
+68 -6
View File
@@ -3,11 +3,12 @@
# vim: set fileencoding=utf-8 : # vim: set fileencoding=utf-8 :
import sys import sys
import os
import argcomplete import argcomplete
import argparse import argparse
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from .configuration import Config, default_config_file from .configuration import Config, default_config_file, ConfigError
from .message import Message from .message import Message
from .commands.question import question_cmd from .commands.question import question_cmd
from .commands.tags import tags_cmd from .commands.tags import tags_cmd
@@ -15,6 +16,7 @@ from .commands.config import config_cmd
from .commands.hist import hist_cmd from .commands.hist import hist_cmd
from .commands.print import print_cmd from .commands.print import print_cmd
from .commands.translation import translation_cmd from .commands.translation import translation_cmd
from .commands.glossary import glossary_cmd
from .chat import msg_location from .chat import msg_location
@@ -53,7 +55,7 @@ def create_parser() -> argparse.ArgumentParser:
ai_parser = argparse.ArgumentParser(add_help=False) ai_parser = argparse.ArgumentParser(add_help=False)
ai_parser.add_argument('-A', '--AI', help='AI ID to use', metavar='AI_ID') ai_parser.add_argument('-A', '--AI', help='AI ID to use', metavar='AI_ID')
ai_parser.add_argument('-M', '--model', help='Model to use', metavar='MODEL') ai_parser.add_argument('-M', '--model', help='Model to use', metavar='MODEL')
ai_parser.add_argument('-n', '--num-answers', help='Number of answers to request', type=int, default=1) ai_parser.add_argument('-N', '--num-answers', help='Number of answers to request', type=int, default=1)
ai_parser.add_argument('-m', '--max-tokens', help='Max. nr. of tokens', type=int) ai_parser.add_argument('-m', '--max-tokens', help='Max. nr. of tokens', type=int)
ai_parser.add_argument('-T', '--temperature', help='Temperature value', type=float) ai_parser.add_argument('-T', '--temperature', help='Temperature value', type=float)
@@ -139,23 +141,78 @@ def create_parser() -> argparse.ArgumentParser:
# 'translation' command parser # 'translation' command parser
translation_cmd_parser = cmdparser.add_parser('translation', parents=[ai_parser, tag_parser], translation_cmd_parser = cmdparser.add_parser('translation', parents=[ai_parser, tag_parser],
help="ask, create and repeat translations.", help="Ask, create and repeat translations.",
aliases=['t']) aliases=['t'])
translation_cmd_parser.set_defaults(func=translation_cmd) translation_cmd_parser.set_defaults(func=translation_cmd)
translation_group = translation_cmd_parser.add_mutually_exclusive_group(required=True) translation_group = translation_cmd_parser.add_mutually_exclusive_group(required=True)
translation_group.add_argument('-a', '--ask', nargs='+', help='Ask to translate the given text', metavar='TEXT') translation_group.add_argument('-a', '--ask', nargs='+', help='Ask to translate the given text', metavar='TEXT')
translation_group.add_argument('-c', '--create', nargs='+', help='Create a translation', metavar='TEXT') translation_group.add_argument('-c', '--create', nargs='+', help='Create a translation', metavar='TEXT')
translation_group.add_argument('-r', '--repeat', nargs='*', help='Repeat a translation', metavar='MESSAGE') translation_group.add_argument('-r', '--repeat', nargs='*', help='Repeat a translation', metavar='MESSAGE')
translation_cmd_parser.add_argument('-S', '--source-lang', help="Source language", metavar="LANGUAGE", required=True) translation_cmd_parser.add_argument('-l', '--source-lang', help="Source language", metavar="LANGUAGE", required=True)
translation_cmd_parser.add_argument('-T', '--target-lang', help="Target language", metavar="LANGUAGE", required=True) translation_cmd_parser.add_argument('-L', '--target-lang', help="Target language", metavar="LANGUAGE", required=True)
translation_cmd_parser.add_argument('-G', '--glossaries', nargs='+', help="List of glossaries", metavar="GLOSSARY") translation_cmd_parser.add_argument('-G', '--glossaries', nargs='+', help="List of glossary names", metavar="GLOSSARY")
translation_cmd_parser.add_argument('-d', '--input-document', help="Document to translate", metavar="FILE") translation_cmd_parser.add_argument('-d', '--input-document', help="Document to translate", metavar="FILE")
translation_cmd_parser.add_argument('-D', '--output-document', help="Path for the translated document", metavar="FILE") translation_cmd_parser.add_argument('-D', '--output-document', help="Path for the translated document", metavar="FILE")
# 'glossary' command parser
glossary_cmd_parser = cmdparser.add_parser('glossary', parents=[ai_parser],
help="Manage glossaries.",
aliases=['g'])
glossary_cmd_parser.set_defaults(func=glossary_cmd)
glossary_group = glossary_cmd_parser.add_mutually_exclusive_group(required=True)
glossary_group.add_argument('-c', '--create', help='Create a glossary', action='store_true')
glossary_cmd_parser.add_argument('-n', '--name', help="Glossary name (not ID)", metavar="NAME")
glossary_cmd_parser.add_argument('-l', '--source-lang', help="Source language", metavar="LANGUAGE")
glossary_cmd_parser.add_argument('-L', '--target-lang', help="Target language", metavar="LANGUAGE")
glossary_cmd_parser.add_argument('-f', '--file', help='File path of the goven glossary', metavar='GLOSSARY_FILE')
glossary_cmd_parser.add_argument('-D', '--description', help="Glossary description", metavar="DESCRIPTION")
glossary_group.add_argument('-i', '--list', help='List existing glossaries', action='store_true')
glossary_group.add_argument('-p', '--print', help='Print an existing glossary', action='store_true')
argcomplete.autocomplete(parser) argcomplete.autocomplete(parser)
return parser return parser
def create_directories(config: Config) -> None: # noqa: 11
"""
Create the directories in the given configuration if they don't exist.
"""
def make_dir(path: Path) -> None:
try:
os.makedirs(path.absolute())
except Exception as e:
print(f"Creating directory '{path.absolute()}' failed with: {e}")
sys.exit(1)
# Cache
cache_path = Path(config.cache)
if not cache_path.exists():
answer = input(f"Cache directory '{cache_path}' does not exist. Create it? [y/n]")
if answer.lower() in ['y', 'yes']:
make_dir(cache_path.absolute())
else:
print("Can't continue without a valid cache directory!")
sys.exit(1)
# DB
db_path = Path(config.db)
if not db_path.exists():
answer = input(f"DB directory '{db_path}' does not exist. Create it? [y/n]")
if answer.lower() in ['y', 'yes']:
make_dir(db_path.absolute())
else:
print("Can't continue without a valid DB directory!")
sys.exit(1)
# Glossaries
if config.glossaries:
glossaries_path = Path(config.glossaries)
if not glossaries_path.exists():
answer = input(f"Glossaries directory '{glossaries_path}' does not exist. Create it? [y/n]")
if answer.lower() in ['y', 'yes']:
make_dir(glossaries_path.absolute())
else:
print("Can't continue without a valid glossaries directory. Create it or remove it from the configuration.")
sys.exit(1)
def main() -> int: def main() -> int:
parser = create_parser() parser = create_parser()
args = parser.parse_args() args = parser.parse_args()
@@ -164,7 +221,12 @@ def main() -> int:
if command.func == config_cmd: if command.func == config_cmd:
command.func(command) command.func(command)
else: else:
try:
config = Config.from_file(args.config) config = Config.from_file(args.config)
except ConfigError as err:
print(f"{err}")
return 1
create_directories(config)
command.func(command, config) command.func(command, config)
return 0 return 0
+6 -1
View File
@@ -71,11 +71,13 @@ class TestConfig(unittest.TestCase):
'frequency_penalty': 0.7, 'frequency_penalty': 0.7,
'presence_penalty': 0.2 'presence_penalty': 0.2
} }
} },
'glossaries': './glossaries/'
} }
config = Config.from_dict(source_dict) config = Config.from_dict(source_dict)
self.assertEqual(config.cache, '.') self.assertEqual(config.cache, '.')
self.assertEqual(config.db, './test_db/') self.assertEqual(config.db, './test_db/')
self.assertEqual(config.glossaries, './glossaries/')
self.assertEqual(len(config.ais), 1) self.assertEqual(len(config.ais), 1)
self.assertEqual(config.ais['myopenai'].name, 'openai') self.assertEqual(config.ais['myopenai'].name, 'openai')
self.assertEqual(cast(OpenAIConfig, config.ais['myopenai']).system, 'Custom system') self.assertEqual(cast(OpenAIConfig, config.ais['myopenai']).system, 'Custom system')
@@ -105,6 +107,7 @@ class TestConfig(unittest.TestCase):
'frequency_penalty': 0.7, 'frequency_penalty': 0.7,
'presence_penalty': 0.2 'presence_penalty': 0.2
} }
# omit glossaries, since it's optional
} }
} }
with open(self.test_file.name, 'w') as f: with open(self.test_file.name, 'w') as f:
@@ -113,6 +116,8 @@ class TestConfig(unittest.TestCase):
self.assertIsInstance(config, Config) self.assertIsInstance(config, Config)
self.assertEqual(config.cache, './test_cache/') self.assertEqual(config.cache, './test_cache/')
self.assertEqual(config.db, './test_db/') self.assertEqual(config.db, './test_db/')
# missing 'glossaries' should result in 'None'
self.assertEqual(config.glossaries, None)
self.assertEqual(len(config.ais), 1) self.assertEqual(len(config.ais), 1)
self.assertIsInstance(config.ais['default'], AIConfig) self.assertIsInstance(config.ais['default'], AIConfig)
self.assertEqual(cast(OpenAIConfig, config.ais['default']).system, 'Custom system') self.assertEqual(cast(OpenAIConfig, config.ais['default']).system, 'Custom system')
+137 -20
View File
@@ -1,59 +1,105 @@
import unittest import unittest
import tempfile import tempfile
from pathlib import Path from pathlib import Path
from chatmastermind.glossary import Glossary from chatmastermind.glossary import Glossary, GlossaryError
glossary_suffix: str = Glossary.file_suffix
class TestGlossary(unittest.TestCase): class TestGlossary(unittest.TestCase):
def test_from_file_valid_yaml(self) -> None: def test_from_file_yaml_unquoted(self) -> None:
# Prepare a temporary YAML file with valid content """
with tempfile.NamedTemporaryFile('w', delete=False) as yaml_file: Test glossary creatiom from YAML with unquoted entries.
"""
with tempfile.NamedTemporaryFile('w', delete=False, suffix=glossary_suffix) as yaml_file:
yaml_file.write("Name: Sample\n" yaml_file.write("Name: Sample\n"
"Description: A brief description\n"
"ID: '123'\n" "ID: '123'\n"
"SourceLang: en\n" "SourceLang: en\n"
"TargetLang: es\n" "TargetLang: es\n"
"Entries:\n" "Entries:\n"
" hello: hola\n" " hello: hola\n"
" goodbye: adiós\n" " goodbye: adiós\n"
" 'yes': sí\n") # 'yes' is a YAML keyword and therefore quoted # 'yes' is a YAML keyword and would normally be quoted
" yes: sí\n"
" I'm going home: me voy a casa\n")
yaml_file_path = Path(yaml_file.name) yaml_file_path = Path(yaml_file.name)
# create and check valid glossary
glossary = Glossary.from_file(yaml_file_path) glossary = Glossary.from_file(yaml_file_path)
self.assertEqual(glossary.name, "Sample") self.assertEqual(glossary.name, "Sample")
self.assertEqual(glossary.desc, "A brief description")
self.assertEqual(glossary.ID, "123")
self.assertEqual(glossary.source_lang, "en") self.assertEqual(glossary.source_lang, "en")
self.assertEqual(glossary.target_lang, "es") self.assertEqual(glossary.target_lang, "es")
self.assertEqual(glossary.entries, {"hello": "hola", "goodbye": "adiós", "yes": ""}) self.assertEqual(glossary.entries, {"hello": "hola",
"goodbye": "adiós",
"yes": "",
"I'm going home": "me voy a casa"})
yaml_file_path.unlink() # Remove the temporary file
def test_from_file_yaml_quoted(self) -> None:
"""
Test glossary creatiom from YAML with quoted entries.
"""
with tempfile.NamedTemporaryFile('w', delete=False, suffix=glossary_suffix) as yaml_file:
yaml_file.write("Name: Sample\n"
"Description: A brief description\n"
"ID: '123'\n"
"SourceLang: en\n"
"TargetLang: es\n"
"Entries:\n"
" 'hello': 'hola'\n"
" 'goodbye': 'adiós'\n"
" 'yes': ''\n"
" \"I'm going home\": 'me voy a casa'\n")
yaml_file_path = Path(yaml_file.name)
# create and check valid glossary
glossary = Glossary.from_file(yaml_file_path)
self.assertEqual(glossary.name, "Sample")
self.assertEqual(glossary.desc, "A brief description")
self.assertEqual(glossary.ID, "123")
self.assertEqual(glossary.source_lang, "en")
self.assertEqual(glossary.target_lang, "es")
self.assertEqual(glossary.entries, {"hello": "hola",
"goodbye": "adiós",
"yes": "",
"I'm going home": "me voy a casa"})
yaml_file_path.unlink() # Remove the temporary file yaml_file_path.unlink() # Remove the temporary file
def test_to_file_writes_yaml(self) -> None: def test_to_file_writes_yaml(self) -> None:
# Create glossary instance # Create glossary instance
glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={"yes": "oui"}) glossary = Glossary(name="Test",
desc="Test description",
ID="666",
source_lang="en",
target_lang="fr",
entries={"yes": "oui"})
with tempfile.NamedTemporaryFile('w', delete=False) as tmp_file: with tempfile.NamedTemporaryFile('w', suffix=glossary_suffix) as tmp_file:
file_path = Path(tmp_file.name) file_path = Path(tmp_file.name)
glossary.to_file(file_path) glossary.to_file(file_path)
# read and check valid YAML
with open(file_path, 'r') as file: with open(file_path, 'r') as file:
content = file.read() content = file.read()
self.assertIn("Name: Test", content) self.assertIn("Name: Test", content)
self.assertIn("Description: Test description", content)
self.assertIn("ID: '666'", content)
self.assertIn("SourceLang: en", content) self.assertIn("SourceLang: en", content)
self.assertIn("TargetLang: fr", content) self.assertIn("TargetLang: fr", content)
self.assertIn("Entries", content) self.assertIn("Entries", content)
# 'yes' is a YAML keyword and therefore quoted # 'yes' is a YAML keyword and therefore quoted
self.assertIn("'yes': oui", content) self.assertIn("'yes': oui", content)
file_path.unlink() # Remove the temporary file
def test_write_read_glossary(self) -> None: def test_write_read_glossary(self) -> None:
# Create glossary instance # Create glossary instance
# -> use 'yes' in order to test if the YAML quoting is correctly removed when reading the file # -> use 'yes' in order to test if the YAML quoting is correctly removed when reading the file
glossary_write = Glossary(name="Test", source_lang="en", target_lang="fr", entries={"yes": "oui"}) glossary_write = Glossary(name="Test", source_lang="en", target_lang="fr", entries={"yes": "oui"})
with tempfile.NamedTemporaryFile('w', delete=False) as tmp_file: with tempfile.NamedTemporaryFile('w', suffix=glossary_suffix) as tmp_file:
file_path = Path(tmp_file.name) file_path = Path(tmp_file.name)
glossary_write.to_file(file_path) glossary_write.to_file(file_path)
# create new instance from glossary file # create new instance from glossary file
glossary_read = Glossary.from_file(file_path) glossary_read = Glossary.from_file(file_path)
self.assertEqual(glossary_write.name, glossary_read.name) self.assertEqual(glossary_write.name, glossary_read.name)
@@ -61,13 +107,11 @@ class TestGlossary(unittest.TestCase):
self.assertEqual(glossary_write.target_lang, glossary_read.target_lang) self.assertEqual(glossary_write.target_lang, glossary_read.target_lang)
self.assertDictEqual(glossary_write.entries, glossary_read.entries) self.assertDictEqual(glossary_write.entries, glossary_read.entries)
file_path.unlink() # Remove the temporary file
def test_import_export_csv(self) -> None: def test_import_export_csv(self) -> None:
glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={}) glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={})
# First export to CSV # First export to CSV
with tempfile.NamedTemporaryFile('w', delete=False) as csvfile: with tempfile.NamedTemporaryFile('w', suffix=glossary_suffix) as csvfile:
csv_file_path = Path(csvfile.name) csv_file_path = Path(csvfile.name)
glossary.entries = {"hello": "salut", "goodbye": "au revoir"} glossary.entries = {"hello": "salut", "goodbye": "au revoir"}
glossary.export_csv(glossary.entries, csv_file_path) glossary.export_csv(glossary.entries, csv_file_path)
@@ -75,13 +119,12 @@ class TestGlossary(unittest.TestCase):
# Now import CSV # Now import CSV
glossary.import_csv(csv_file_path) glossary.import_csv(csv_file_path)
self.assertEqual(glossary.entries, {"hello": "salut", "goodbye": "au revoir"}) self.assertEqual(glossary.entries, {"hello": "salut", "goodbye": "au revoir"})
csv_file_path.unlink() # Remove the temporary file
def test_import_export_tsv(self) -> None: def test_import_export_tsv(self) -> None:
glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={}) glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={})
# First export to TSV # First export to TSV
with tempfile.NamedTemporaryFile('w', delete=False) as tsvfile: with tempfile.NamedTemporaryFile('w', suffix=glossary_suffix) as tsvfile:
tsv_file_path = Path(tsvfile.name) tsv_file_path = Path(tsvfile.name)
glossary.entries = {"hello": "salut", "goodbye": "au revoir"} glossary.entries = {"hello": "salut", "goodbye": "au revoir"}
glossary.export_tsv(glossary.entries, tsv_file_path) glossary.export_tsv(glossary.entries, tsv_file_path)
@@ -89,4 +132,78 @@ class TestGlossary(unittest.TestCase):
# Now import TSV # Now import TSV
glossary.import_tsv(tsv_file_path) glossary.import_tsv(tsv_file_path)
self.assertEqual(glossary.entries, {"hello": "salut", "goodbye": "au revoir"}) self.assertEqual(glossary.entries, {"hello": "salut", "goodbye": "au revoir"})
tsv_file_path.unlink() # Remove the temporary file
def test_to_file_wrong_suffix(self) -> None:
"""
Test for exception if suffix is wrong.
"""
glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={"yes": "oui"})
with tempfile.NamedTemporaryFile('w', suffix='.wrong') as tmp_file:
file_path = Path(tmp_file.name)
with self.assertRaises(GlossaryError) as err:
glossary.to_file(file_path)
self.assertEqual(str(err.exception), "File suffix '.wrong' is not supported")
def test_to_file_auto_suffix(self) -> None:
"""
Test if suffix is auto-generated if omitted.
"""
glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={"yes": "oui"})
with tempfile.NamedTemporaryFile('w', suffix='') as tmp_file:
file_path = Path(tmp_file.name)
glossary.to_file(file_path)
assert glossary.file_path is not None
self.assertEqual(glossary.file_path.suffix, glossary_suffix)
# remove glossary file (differs from 'tmp_file' because of the added suffix
glossary.file_path.unlink()
def test_to_str_with_id(self) -> None:
# Create a Glossary instance with an ID
glossary_with_id = Glossary(name="TestGlossary", source_lang="en", target_lang="fr",
desc="A simple test glossary", ID="1001", entries={"one": "un"})
glossary_str = glossary_with_id.to_str()
self.assertIn("TestGlossary (ID: 1001):", glossary_str)
self.assertIn("- A simple test glossary", glossary_str)
self.assertIn("- Languages: en -> fr", glossary_str)
self.assertIn("- Entries: 1", glossary_str)
def test_to_str_with_id_and_entries(self) -> None:
# Create a Glossary instance with an ID and include entries
glossary_with_entries = Glossary(name="TestGlossaryWithEntries", source_lang="en", target_lang="fr",
desc="Another test glossary", ID="2002",
entries={"hello": "salut", "goodbye": "au revoir"})
glossary_str_with_entries = glossary_with_entries.to_str(with_entries=True)
self.assertIn("TestGlossaryWithEntries (ID: 2002):", glossary_str_with_entries)
self.assertIn("- Entries:", glossary_str_with_entries)
self.assertIn(" hello : salut", glossary_str_with_entries)
self.assertIn(" goodbye : au revoir", glossary_str_with_entries)
def test_to_str_without_id(self) -> None:
# Create a Glossary instance without an ID
glossary_without_id = Glossary(name="TestGlossaryNoID", source_lang="en", target_lang="fr",
desc="A test glossary without an ID", ID=None, entries={"yes": "oui"})
glossary_str_no_id = glossary_without_id.to_str()
self.assertIn("TestGlossaryNoID (ID: None):", glossary_str_no_id)
self.assertIn("- A test glossary without an ID", glossary_str_no_id)
self.assertIn("- Languages: en -> fr", glossary_str_no_id)
self.assertIn("- Entries: 1", glossary_str_no_id)
def test_to_str_without_id_and_no_entries(self) -> None:
# Create a Glossary instance without an ID and no entries
glossary_no_id_no_entries = Glossary(name="EmptyGlossary", source_lang="en", target_lang="fr",
desc="An empty test glossary", ID=None, entries={})
glossary_str_no_id_no_entries = glossary_no_id_no_entries.to_str()
self.assertIn("EmptyGlossary (ID: None):", glossary_str_no_id_no_entries)
self.assertIn("- An empty test glossary", glossary_str_no_id_no_entries)
self.assertIn("- Languages: en -> fr", glossary_str_no_id_no_entries)
self.assertIn("- Entries: 0", glossary_str_no_id_no_entries)
def test_to_str_no_description(self) -> None:
# Create a Glossary instance with an ID
glossary_with_id = Glossary(name="TestGlossary", source_lang="en", target_lang="fr",
ID="1001", entries={"one": "un"})
glossary_str = glossary_with_id.to_str()
expected_str = """TestGlossary (ID: 1001):
- Languages: en -> fr
- Entries: 1"""
self.assertEqual(expected_str, glossary_str)
+149
View File
@@ -0,0 +1,149 @@
import unittest
import argparse
import tempfile
import io
from contextlib import redirect_stdout
from chatmastermind.configuration import Config
from chatmastermind.commands.glossary import (
Glossary,
GlossaryCmdError,
glossary_cmd,
get_glossary_file_path,
create_glossary,
print_glossary,
list_glossaries
)
class TestGlossaryCmdNoGlossaries(unittest.TestCase):
def setUp(self) -> None:
# create DB and cache
self.db_dir = tempfile.TemporaryDirectory()
self.cache_dir = tempfile.TemporaryDirectory()
self.glossaries_dir = tempfile.TemporaryDirectory()
# create configuration
self.config = Config()
self.config.cache = self.cache_dir.name
self.config.db = self.db_dir.name
self.config.glossaries = self.glossaries_dir.name
# create a mock argparse.Namespace
self.args = argparse.Namespace(
create=True,
list=False,
print=False,
name='new_glossary',
file=None,
source_lang='en',
target_lang='de',
description=False,
)
def test_glossary_create_no_glossaries_err(self) -> None:
self.config.glossaries = None
with self.assertRaises(GlossaryCmdError) as err:
create_glossary(self.args, self.config)
self.assertIn(str(err.exception).lower(), "glossaries directory missing")
def test_glossary_create_no_name_err(self) -> None:
self.args.name = None
with self.assertRaises(GlossaryCmdError) as err:
create_glossary(self.args, self.config)
self.assertIn(str(err.exception).lower(), "missing glossary name")
def test_glossary_create_no_source_lang_err(self) -> None:
self.args.source_lang = None
with self.assertRaises(GlossaryCmdError) as err:
create_glossary(self.args, self.config)
self.assertIn(str(err.exception).lower(), "missing source language")
def test_glossary_create_no_target_lang_err(self) -> None:
self.args.target_lang = None
with self.assertRaises(GlossaryCmdError) as err:
create_glossary(self.args, self.config)
self.assertIn(str(err.exception).lower(), "missing target language")
def test_glossary_print_no_name_err(self) -> None:
self.args.name = None
with self.assertRaises(GlossaryCmdError) as err:
print_glossary(self.args, self.config)
self.assertIn(str(err.exception).lower(), "missing glossary name")
def test_glossary_list_no_glossaries_err(self) -> None:
self.config.glossaries = None
with self.assertRaises(GlossaryCmdError) as err:
list_glossaries(self.args, self.config)
self.assertIn(str(err.exception).lower(), "glossaries directory missing")
def test_glossary_create(self) -> None:
self.args.create = True
self.args.list = False
self.args.print = False
glossary_cmd(self.args, self.config)
expected_path = get_glossary_file_path(self.args.name, self.config)
glo = Glossary.from_file(expected_path)
self.assertEqual(glo.name, self.args.name)
expected_path.unlink()
def test_glossary_create_twice_err(self) -> None:
self.args.create = True
self.args.list = False
self.args.print = False
glossary_cmd(self.args, self.config)
expected_path = get_glossary_file_path(self.args.name, self.config)
glo = Glossary.from_file(expected_path)
self.assertEqual(glo.name, self.args.name)
# create glossary with the same name again
with self.assertRaises(GlossaryCmdError) as err:
create_glossary(self.args, self.config)
self.assertIn(str(err.exception).lower(), "already exists")
expected_path.unlink()
class TestGlossaryCmdWithGlossaries(unittest.TestCase):
def setUp(self) -> None:
# create DB and cache
self.db_dir = tempfile.TemporaryDirectory()
self.cache_dir = tempfile.TemporaryDirectory()
self.glossaries_dir = tempfile.TemporaryDirectory()
# create configuration
self.config = Config()
self.config.cache = self.cache_dir.name
self.config.db = self.db_dir.name
self.config.glossaries = self.glossaries_dir.name
# create a mock argparse.Namespace
self.args = argparse.Namespace(
create=True,
list=False,
print=False,
name='Glossary1',
file=None,
source_lang='en',
target_lang='de',
description=False,
)
# create Glossary1
glossary_cmd(self.args, self.config)
self.Glossary1_path = get_glossary_file_path('Glossary1', self.config)
# create Glossary2
self.args.name = 'Glossary2'
glossary_cmd(self.args, self.config)
self.Glossary2_path = get_glossary_file_path('Glossary2', self.config)
def test_glossaries_exist(self) -> None:
"""
Test if the default glossaries created in setUp exist.
"""
glo = Glossary.from_file(self.Glossary1_path)
self.assertEqual(glo.name, 'Glossary1')
glo = Glossary.from_file(self.Glossary2_path)
self.assertEqual(glo.name, 'Glossary2')
def test_glossaries_list(self) -> None:
self.args.create = False
self.args.list = True
with redirect_stdout(io.StringIO()) as list_output:
glossary_cmd(self.args, self.config)
self.assertIn('Glossary1', list_output.getvalue())
self.assertIn('Glossary2', list_output.getvalue())