Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0ebf359647 | |||
| c0b49c96b7 | |||
| 2d94f8a108 | |||
| 86f0e6d99c | |||
| 15f6e819e5 | |||
| 86295c6492 | |||
| 91a7541581 | |||
| a2ae52014b | |||
| 6e89e014bf | |||
| 85315d9c1c | |||
| 2de0faabdb | |||
| 0c83ef6ce7 | |||
| 8046f1a424 | |||
| 2f95654c52 | |||
| 0449f70c63 | |||
| 0e1267139f | |||
| 02dab2807c | |||
| 70164a1d45 | |||
| 8f95a362d2 | |||
| ee363d9894 |
@@ -68,6 +68,7 @@ class OpenAI(AI):
|
|||||||
self.ID = config.ID
|
self.ID = config.ID
|
||||||
self.name = config.name
|
self.name = config.name
|
||||||
self.config = config
|
self.config = config
|
||||||
|
openai.api_key = self.config.api_key
|
||||||
|
|
||||||
def request(self,
|
def request(self,
|
||||||
question: Message,
|
question: Message,
|
||||||
@@ -80,7 +81,6 @@ class OpenAI(AI):
|
|||||||
nr. of messages in the 'AIResponse'.
|
nr. of messages in the 'AIResponse'.
|
||||||
"""
|
"""
|
||||||
self.encoding = tiktoken.encoding_for_model(self.config.model)
|
self.encoding = tiktoken.encoding_for_model(self.config.model)
|
||||||
openai.api_key = self.config.api_key
|
|
||||||
oai_chat, prompt_tokens = self.openai_chat(chat, self.config.system, question)
|
oai_chat, prompt_tokens = self.openai_chat(chat, self.config.system, question)
|
||||||
tokens: Tokens = Tokens(prompt_tokens, 0, prompt_tokens)
|
tokens: Tokens = Tokens(prompt_tokens, 0, prompt_tokens)
|
||||||
response = openai.ChatCompletion.create(
|
response = openai.ChatCompletion.create(
|
||||||
@@ -150,7 +150,7 @@ class OpenAI(AI):
|
|||||||
for message in chat.messages:
|
for message in chat.messages:
|
||||||
if message.answer:
|
if message.answer:
|
||||||
prompt_tokens += append('user', message.question)
|
prompt_tokens += append('user', message.question)
|
||||||
prompt_tokens += append('assistant', message.answer)
|
prompt_tokens += append('assistant', str(message.answer))
|
||||||
if question:
|
if question:
|
||||||
prompt_tokens += append('user', question.question)
|
prompt_tokens += append('user', question.question)
|
||||||
return oai_chat, prompt_tokens
|
return oai_chat, prompt_tokens
|
||||||
|
|||||||
@@ -0,0 +1,69 @@
|
|||||||
|
"""
|
||||||
|
Contains shared functions for the various CMM subcommands.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from pathlib import Path
|
||||||
|
from ..message import Message, MessageError, source_code
|
||||||
|
|
||||||
|
|
||||||
|
def read_text_file(file: Path) -> str:
|
||||||
|
with open(file) as r:
|
||||||
|
content = r.read().strip()
|
||||||
|
return content
|
||||||
|
|
||||||
|
|
||||||
|
def add_file_as_text(question_parts: list[str], file: str) -> None:
|
||||||
|
"""
|
||||||
|
Add the given file as plain text to the question part list.
|
||||||
|
If the file is a Message, add the answer.
|
||||||
|
"""
|
||||||
|
file_path = Path(file)
|
||||||
|
content: str
|
||||||
|
try:
|
||||||
|
message = Message.from_file(file_path)
|
||||||
|
if message and message.answer:
|
||||||
|
content = message.answer
|
||||||
|
except MessageError:
|
||||||
|
content = read_text_file(Path(file))
|
||||||
|
if len(content) > 0:
|
||||||
|
question_parts.append(content)
|
||||||
|
|
||||||
|
|
||||||
|
def add_file_as_code(question_parts: list[str], file: str) -> None:
|
||||||
|
"""
|
||||||
|
Add all source code from the given file. If no code segments can be extracted,
|
||||||
|
the whole content is added as source code segment. If the file is a Message,
|
||||||
|
extract the source code from the answer.
|
||||||
|
"""
|
||||||
|
file_path = Path(file)
|
||||||
|
content: str
|
||||||
|
try:
|
||||||
|
message = Message.from_file(file_path)
|
||||||
|
if message and message.answer:
|
||||||
|
content = message.answer
|
||||||
|
except MessageError:
|
||||||
|
with open(file) as r:
|
||||||
|
content = r.read().strip()
|
||||||
|
# extract and add source code
|
||||||
|
code_parts = source_code(content, include_delims=True)
|
||||||
|
if len(code_parts) > 0:
|
||||||
|
question_parts += code_parts
|
||||||
|
else:
|
||||||
|
question_parts.append(f"```\n{content}\n```")
|
||||||
|
|
||||||
|
|
||||||
|
def invert_input_tag_args(args: argparse.Namespace) -> None:
|
||||||
|
"""
|
||||||
|
Changes the semantics of the INPUT tags for this command:
|
||||||
|
* not tags specified on the CLI -> no tags are selected
|
||||||
|
* empty tags specified on the CLI -> all tags are selected
|
||||||
|
"""
|
||||||
|
if args.or_tags is None:
|
||||||
|
args.or_tags = set()
|
||||||
|
elif len(args.or_tags) == 0:
|
||||||
|
args.or_tags = None
|
||||||
|
if args.and_tags is None:
|
||||||
|
args.and_tags = set()
|
||||||
|
elif len(args.and_tags) == 0:
|
||||||
|
args.and_tags = None
|
||||||
@@ -3,9 +3,10 @@ import argparse
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from itertools import zip_longest
|
from itertools import zip_longest
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
|
from .common import invert_input_tag_args, add_file_as_code, add_file_as_text
|
||||||
from ..configuration import Config
|
from ..configuration import Config
|
||||||
from ..chat import ChatDB, msg_location
|
from ..chat import ChatDB, msg_location
|
||||||
from ..message import Message, MessageFilter, MessageError, Question, source_code
|
from ..message import Message, MessageFilter, Question
|
||||||
from ..ai_factory import create_ai
|
from ..ai_factory import create_ai
|
||||||
from ..ai import AI, AIResponse
|
from ..ai import AI, AIResponse
|
||||||
|
|
||||||
@@ -14,47 +15,6 @@ class QuestionCmdError(Exception):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def add_file_as_text(question_parts: list[str], file: str) -> None:
|
|
||||||
"""
|
|
||||||
Add the given file as plain text to the question part list.
|
|
||||||
If the file is a Message, add the answer.
|
|
||||||
"""
|
|
||||||
file_path = Path(file)
|
|
||||||
content: str
|
|
||||||
try:
|
|
||||||
message = Message.from_file(file_path)
|
|
||||||
if message and message.answer:
|
|
||||||
content = message.answer
|
|
||||||
except MessageError:
|
|
||||||
with open(file) as r:
|
|
||||||
content = r.read().strip()
|
|
||||||
if len(content) > 0:
|
|
||||||
question_parts.append(content)
|
|
||||||
|
|
||||||
|
|
||||||
def add_file_as_code(question_parts: list[str], file: str) -> None:
|
|
||||||
"""
|
|
||||||
Add all source code from the given file. If no code segments can be extracted,
|
|
||||||
the whole content is added as source code segment. If the file is a Message,
|
|
||||||
extract the source code from the answer.
|
|
||||||
"""
|
|
||||||
file_path = Path(file)
|
|
||||||
content: str
|
|
||||||
try:
|
|
||||||
message = Message.from_file(file_path)
|
|
||||||
if message and message.answer:
|
|
||||||
content = message.answer
|
|
||||||
except MessageError:
|
|
||||||
with open(file) as r:
|
|
||||||
content = r.read().strip()
|
|
||||||
# extract and add source code
|
|
||||||
code_parts = source_code(content, include_delims=True)
|
|
||||||
if len(code_parts) > 0:
|
|
||||||
question_parts += code_parts
|
|
||||||
else:
|
|
||||||
question_parts.append(f"```\n{content}\n```")
|
|
||||||
|
|
||||||
|
|
||||||
def create_msg_args(msg: Message, args: argparse.Namespace) -> argparse.Namespace:
|
def create_msg_args(msg: Message, args: argparse.Namespace) -> argparse.Namespace:
|
||||||
"""
|
"""
|
||||||
Takes an existing message and CLI arguments, and returns modified args based
|
Takes an existing message and CLI arguments, and returns modified args based
|
||||||
@@ -163,22 +123,6 @@ def repeat_messages(messages: list[Message], chat: ChatDB, args: argparse.Namesp
|
|||||||
make_request(ai, chat, message, msg_args)
|
make_request(ai, chat, message, msg_args)
|
||||||
|
|
||||||
|
|
||||||
def invert_input_tag_args(args: argparse.Namespace) -> None:
|
|
||||||
"""
|
|
||||||
Changes the semantics of the INPUT tags for this command:
|
|
||||||
* not tags specified on the CLI -> no tags are selected
|
|
||||||
* empty tags specified on the CLI -> all tags are selected
|
|
||||||
"""
|
|
||||||
if args.or_tags is None:
|
|
||||||
args.or_tags = set()
|
|
||||||
elif len(args.or_tags) == 0:
|
|
||||||
args.or_tags = None
|
|
||||||
if args.and_tags is None:
|
|
||||||
args.and_tags = set()
|
|
||||||
elif len(args.and_tags) == 0:
|
|
||||||
args.and_tags = None
|
|
||||||
|
|
||||||
|
|
||||||
def question_cmd(args: argparse.Namespace, config: Config) -> None:
|
def question_cmd(args: argparse.Namespace, config: Config) -> None:
|
||||||
"""
|
"""
|
||||||
Handler for the 'question' command.
|
Handler for the 'question' command.
|
||||||
|
|||||||
@@ -0,0 +1,105 @@
|
|||||||
|
import argparse
|
||||||
|
import mimetypes
|
||||||
|
from pathlib import Path
|
||||||
|
from .common import invert_input_tag_args, read_text_file
|
||||||
|
from ..configuration import Config
|
||||||
|
from ..message import MessageFilter, Message, Question
|
||||||
|
from ..chat import ChatDB, msg_location
|
||||||
|
|
||||||
|
|
||||||
|
class TranslationCmdError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
text_separator: str = 'TEXT:'
|
||||||
|
|
||||||
|
|
||||||
|
def assert_document_type_supported_openai(document_file: Path) -> None:
|
||||||
|
doctype = mimetypes.guess_type(document_file)
|
||||||
|
if doctype != 'text/plain':
|
||||||
|
raise TranslationCmdError("AI 'OpenAI' only supports document type 'text/plain''")
|
||||||
|
|
||||||
|
|
||||||
|
def translation_prompt_openai(source_lang: str, target_lang: str) -> str:
|
||||||
|
"""
|
||||||
|
Return the prompt for GPT that tells it to do the translation.
|
||||||
|
"""
|
||||||
|
return f"Translate the text below the line {text_separator} from {source_lang} to {target_lang}."
|
||||||
|
|
||||||
|
|
||||||
|
def create_message_openai(chat: ChatDB, args: argparse.Namespace) -> Message:
|
||||||
|
"""
|
||||||
|
Create a new message from the given arguments and write it to the cache directory.
|
||||||
|
|
||||||
|
Message format
|
||||||
|
1. Translation prompt (tells GPT to do a translation)
|
||||||
|
2. Glossary (if specified as an argument)
|
||||||
|
3. User provided prompt enhancements
|
||||||
|
4. Translation separator
|
||||||
|
5. User provided text to be translated
|
||||||
|
|
||||||
|
The text to be translated is determined as a follows:
|
||||||
|
- if a document is provided in the arguments, translate its content
|
||||||
|
- if no document is provided, translate the last text argument
|
||||||
|
|
||||||
|
The other text arguments will be put into the "header" and can be used
|
||||||
|
to improve the translation prompt.
|
||||||
|
"""
|
||||||
|
text_args: list[str] = []
|
||||||
|
if args.create is not None:
|
||||||
|
text_args = args.create
|
||||||
|
elif args.ask is not None:
|
||||||
|
text_args = args.ask
|
||||||
|
else:
|
||||||
|
raise TranslationCmdError("No input text found")
|
||||||
|
|
||||||
|
# extract user prompt and user text to be translated
|
||||||
|
user_text: str
|
||||||
|
user_prompt: str
|
||||||
|
if args.input_document is not None:
|
||||||
|
assert_document_type_supported_openai(Path(args.input_document))
|
||||||
|
user_text = read_text_file(Path(args.input_document))
|
||||||
|
user_prompt = '\n\n'.join([str(s) for s in text_args])
|
||||||
|
else:
|
||||||
|
user_text = text_args[-1]
|
||||||
|
user_prompt = '\n\n'.join([str(s) for s in text_args[:-1]])
|
||||||
|
|
||||||
|
# build full question string
|
||||||
|
# FIXME: add glossaries if given
|
||||||
|
question_text: str = '\n\n'.join([translation_prompt_openai(args.source_lang, args.target_lang),
|
||||||
|
user_prompt,
|
||||||
|
text_separator,
|
||||||
|
user_text])
|
||||||
|
# create and write the message
|
||||||
|
message = Message(question=Question(question_text),
|
||||||
|
tags=args.output_tags,
|
||||||
|
ai=args.AI,
|
||||||
|
model=args.model)
|
||||||
|
# only write the new message to the cache,
|
||||||
|
# don't add it to the internal list
|
||||||
|
chat.cache_write([message])
|
||||||
|
return message
|
||||||
|
|
||||||
|
|
||||||
|
def translation_cmd(args: argparse.Namespace, config: Config) -> None:
|
||||||
|
"""
|
||||||
|
Handler for the 'translation' command. Creates and executes translation
|
||||||
|
requests based on the input and selected AI. Depending on the AI, the
|
||||||
|
whole process may be significantly different (e.g. DeepL vs OpenAI).
|
||||||
|
"""
|
||||||
|
invert_input_tag_args(args)
|
||||||
|
mfilter = MessageFilter(tags_or=args.or_tags,
|
||||||
|
tags_and=args.and_tags,
|
||||||
|
tags_not=args.exclude_tags)
|
||||||
|
chat = ChatDB.from_dir(cache_path=Path(config.cache),
|
||||||
|
db_path=Path(config.db),
|
||||||
|
mfilter=mfilter,
|
||||||
|
glob=args.glob,
|
||||||
|
loc=msg_location(args.location))
|
||||||
|
# if it's a new translation, create and store it immediately
|
||||||
|
# FIXME: check AI type
|
||||||
|
if args.ask or args.create:
|
||||||
|
# message = create_message(chat, args)
|
||||||
|
create_message_openai(chat, args)
|
||||||
|
if args.create:
|
||||||
|
return
|
||||||
@@ -118,6 +118,7 @@ class Config:
|
|||||||
# a default configuration
|
# a default configuration
|
||||||
cache: str = '.'
|
cache: str = '.'
|
||||||
db: str = './db/'
|
db: str = './db/'
|
||||||
|
glossaries: str | None = './glossaries/'
|
||||||
ais: dict[str, AIConfig] = field(default_factory=create_default_ai_configs)
|
ais: dict[str, AIConfig] = field(default_factory=create_default_ai_configs)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -135,7 +136,8 @@ class Config:
|
|||||||
return cls(
|
return cls(
|
||||||
cache=str(source['cache']) if 'cache' in source else '.',
|
cache=str(source['cache']) if 'cache' in source else '.',
|
||||||
db=str(source['db']),
|
db=str(source['db']),
|
||||||
ais=ais
|
ais=ais,
|
||||||
|
glossaries=str(source['glossaries']) if 'glossaries' in source else None
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|||||||
@@ -0,0 +1,161 @@
|
|||||||
|
"""
|
||||||
|
Module implementing glossaries for translations.
|
||||||
|
"""
|
||||||
|
import yaml
|
||||||
|
import tempfile
|
||||||
|
import shutil
|
||||||
|
import csv
|
||||||
|
from pathlib import Path
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Type, TypeVar, ClassVar
|
||||||
|
|
||||||
|
GlossaryInst = TypeVar('GlossaryInst', bound='Glossary')
|
||||||
|
|
||||||
|
|
||||||
|
class GlossaryError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def str_presenter(dumper: yaml.Dumper, data: str) -> yaml.ScalarNode:
|
||||||
|
"""
|
||||||
|
Changes the YAML dump style to multiline syntax for multiline strings.
|
||||||
|
"""
|
||||||
|
if len(data.splitlines()) > 1:
|
||||||
|
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
|
||||||
|
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Glossary:
|
||||||
|
"""
|
||||||
|
A glossary consists of the following parameters:
|
||||||
|
- Name (freely selectable)
|
||||||
|
- Path (full file path, suffix is automatically generated)
|
||||||
|
- Source language
|
||||||
|
- Target language
|
||||||
|
- Description (optional)
|
||||||
|
- Entries (pairs of source lang and target lang terms)
|
||||||
|
- ID (automatically generated / modified, required by DeepL)
|
||||||
|
"""
|
||||||
|
|
||||||
|
name: str
|
||||||
|
source_lang: str
|
||||||
|
target_lang: str
|
||||||
|
file_path: Path | None = None
|
||||||
|
desc: str | None = None
|
||||||
|
entries: dict[str, str] = field(default_factory=lambda: dict())
|
||||||
|
ID: str | None = None
|
||||||
|
file_suffix: ClassVar[str] = '.glo'
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_file(cls: Type[GlossaryInst], file_path: Path) -> GlossaryInst:
|
||||||
|
"""
|
||||||
|
Create a glossary from the given file.
|
||||||
|
"""
|
||||||
|
if not file_path.exists():
|
||||||
|
raise GlossaryError(f"Glossary file '{file_path}' does not exist")
|
||||||
|
if file_path.suffix != cls.file_suffix:
|
||||||
|
raise GlossaryError(f"File type '{file_path.suffix}' is not supported")
|
||||||
|
with open(file_path, "r") as fd:
|
||||||
|
try:
|
||||||
|
# use BaseLoader so every entry is read as a string
|
||||||
|
# - disables automatic conversions
|
||||||
|
# - makes it possible to omit quoting for YAML keywords in entries (e. g. 'yes')
|
||||||
|
# - also correctly reads quoted entries
|
||||||
|
data = yaml.load(fd, Loader=yaml.BaseLoader)
|
||||||
|
clean_entries = data['Entries']
|
||||||
|
return cls(name=data['Name'],
|
||||||
|
source_lang=data['SourceLang'],
|
||||||
|
target_lang=data['TargetLang'],
|
||||||
|
file_path=file_path,
|
||||||
|
desc=data['Description'],
|
||||||
|
entries=clean_entries,
|
||||||
|
ID=data['ID'] if data['ID'] != 'None' else None)
|
||||||
|
except Exception:
|
||||||
|
raise GlossaryError(f"'{file_path}' does not contain a valid glossary")
|
||||||
|
|
||||||
|
def to_file(self, file_path: Path | None = None) -> None:
|
||||||
|
"""
|
||||||
|
Write glossary to given file.
|
||||||
|
"""
|
||||||
|
if file_path:
|
||||||
|
self.file_path = file_path
|
||||||
|
if not self.file_path:
|
||||||
|
raise GlossaryError("Got no valid path to write glossary")
|
||||||
|
# check / add valid suffix
|
||||||
|
if not self.file_path.suffix:
|
||||||
|
self.file_path = self.file_path.with_suffix(self.file_suffix)
|
||||||
|
elif self.file_path.suffix != self.file_suffix:
|
||||||
|
raise GlossaryError(f"File suffix '{self.file_path.suffix}' is not supported")
|
||||||
|
# write YAML
|
||||||
|
with tempfile.NamedTemporaryFile(dir=self.file_path.parent, prefix=self.file_path.name, mode="w", delete=False) as temp_fd:
|
||||||
|
temp_file_path = Path(temp_fd.name)
|
||||||
|
data = {'Name': self.name,
|
||||||
|
'Description': self.desc,
|
||||||
|
'ID': str(self.ID),
|
||||||
|
'SourceLang': self.source_lang,
|
||||||
|
'TargetLang': self.target_lang,
|
||||||
|
'Entries': self.entries}
|
||||||
|
yaml.dump(data, temp_fd, sort_keys=False)
|
||||||
|
shutil.move(temp_file_path, self.file_path)
|
||||||
|
|
||||||
|
def export_csv(self, dictionary: dict[str, str], file_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Export the 'entries' of this glossary to a file in CSV format (compatible with DeepL).
|
||||||
|
"""
|
||||||
|
with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
|
||||||
|
writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
|
||||||
|
for source_entry, target_entry in self.entries.items():
|
||||||
|
writer.writerow([source_entry, target_entry])
|
||||||
|
|
||||||
|
def export_tsv(self, entries: dict[str, str], file_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Export the 'entries' of this glossary to a file in TSV format (compatible with DeepL).
|
||||||
|
"""
|
||||||
|
with open(file_path, 'w', encoding='utf-8') as file:
|
||||||
|
for source_entry, target_entry in self.entries.items():
|
||||||
|
file.write(f"{source_entry}\t{target_entry}\n")
|
||||||
|
|
||||||
|
def import_csv(self, file_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Import the entries from the given CSV file to those of the current glossary.
|
||||||
|
Existing entries are overwritten.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
with open(file_path, mode='r', encoding='utf-8') as csvfile:
|
||||||
|
reader = csv.reader(csvfile, delimiter=',', quotechar='"')
|
||||||
|
self.entries = {rows[0]: rows[1] for rows in reader if len(rows) >= 2}
|
||||||
|
except Exception as e:
|
||||||
|
raise GlossaryError(f"Error importing CSV: {e}")
|
||||||
|
|
||||||
|
def import_tsv(self, file_path: Path) -> None:
|
||||||
|
"""
|
||||||
|
Import the entries from the given CSV file to those of the current glossary.
|
||||||
|
Existing entries are overwritten.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
with open(file_path, mode='r', encoding='utf-8') as tsvfile:
|
||||||
|
self.entries = {}
|
||||||
|
for line in tsvfile:
|
||||||
|
parts = line.strip().split('\t')
|
||||||
|
if len(parts) == 2:
|
||||||
|
self.entries[parts[0]] = parts[1]
|
||||||
|
except Exception as e:
|
||||||
|
raise GlossaryError(f"Error importing TSV: {e}")
|
||||||
|
|
||||||
|
def to_str(self, with_entries: bool = False) -> str:
|
||||||
|
"""
|
||||||
|
Return the current glossary as a string.
|
||||||
|
"""
|
||||||
|
output: list[str] = []
|
||||||
|
output.append(f'{self.name} (ID: {self.ID}):')
|
||||||
|
if self.desc:
|
||||||
|
output.append('- ' + self.desc)
|
||||||
|
output.append(f'- Languages: {self.source_lang} -> {self.target_lang}')
|
||||||
|
if with_entries:
|
||||||
|
output.append('- Entries:')
|
||||||
|
for source, target in self.entries.items():
|
||||||
|
output.append(f' {source} : {target}')
|
||||||
|
else:
|
||||||
|
output.append(f'- Entries: {len(self.entries)}')
|
||||||
|
return '\n'.join(output)
|
||||||
+59
-1
@@ -3,6 +3,7 @@
|
|||||||
# vim: set fileencoding=utf-8 :
|
# vim: set fileencoding=utf-8 :
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
|
import os
|
||||||
import argcomplete
|
import argcomplete
|
||||||
import argparse
|
import argparse
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -14,6 +15,7 @@ from .commands.tags import tags_cmd
|
|||||||
from .commands.config import config_cmd
|
from .commands.config import config_cmd
|
||||||
from .commands.hist import hist_cmd
|
from .commands.hist import hist_cmd
|
||||||
from .commands.print import print_cmd
|
from .commands.print import print_cmd
|
||||||
|
from .commands.translation import translation_cmd
|
||||||
from .chat import msg_location
|
from .chat import msg_location
|
||||||
|
|
||||||
|
|
||||||
@@ -102,7 +104,7 @@ def create_parser() -> argparse.ArgumentParser:
|
|||||||
# 'tags' command parser
|
# 'tags' command parser
|
||||||
tags_cmd_parser = cmdparser.add_parser('tags',
|
tags_cmd_parser = cmdparser.add_parser('tags',
|
||||||
help="Manage tags.",
|
help="Manage tags.",
|
||||||
aliases=['t'])
|
aliases=['T'])
|
||||||
tags_cmd_parser.set_defaults(func=tags_cmd)
|
tags_cmd_parser.set_defaults(func=tags_cmd)
|
||||||
tags_group = tags_cmd_parser.add_mutually_exclusive_group(required=True)
|
tags_group = tags_cmd_parser.add_mutually_exclusive_group(required=True)
|
||||||
tags_group.add_argument('-l', '--list', help="List all tags and their frequency",
|
tags_group.add_argument('-l', '--list', help="List all tags and their frequency",
|
||||||
@@ -136,10 +138,65 @@ def create_parser() -> argparse.ArgumentParser:
|
|||||||
print_cmd_modes.add_argument('-a', '--answer', help='Only print the answer', action='store_true')
|
print_cmd_modes.add_argument('-a', '--answer', help='Only print the answer', action='store_true')
|
||||||
print_cmd_modes.add_argument('-S', '--only-source-code', help='Only print embedded source code', action='store_true')
|
print_cmd_modes.add_argument('-S', '--only-source-code', help='Only print embedded source code', action='store_true')
|
||||||
|
|
||||||
|
# 'translation' command parser
|
||||||
|
translation_cmd_parser = cmdparser.add_parser('translation', parents=[ai_parser, tag_parser],
|
||||||
|
help="ask, create and repeat translations.",
|
||||||
|
aliases=['t'])
|
||||||
|
translation_cmd_parser.set_defaults(func=translation_cmd)
|
||||||
|
translation_group = translation_cmd_parser.add_mutually_exclusive_group(required=True)
|
||||||
|
translation_group.add_argument('-a', '--ask', nargs='+', help='Ask to translate the given text', metavar='TEXT')
|
||||||
|
translation_group.add_argument('-c', '--create', nargs='+', help='Create a translation', metavar='TEXT')
|
||||||
|
translation_group.add_argument('-r', '--repeat', nargs='*', help='Repeat a translation', metavar='MESSAGE')
|
||||||
|
translation_cmd_parser.add_argument('-S', '--source-lang', help="Source language", metavar="LANGUAGE", required=True)
|
||||||
|
translation_cmd_parser.add_argument('-T', '--target-lang', help="Target language", metavar="LANGUAGE", required=True)
|
||||||
|
translation_cmd_parser.add_argument('-G', '--glossaries', nargs='+', help="List of glossaries", metavar="GLOSSARY")
|
||||||
|
translation_cmd_parser.add_argument('-d', '--input-document', help="Document to translate", metavar="FILE")
|
||||||
|
translation_cmd_parser.add_argument('-D', '--output-document', help="Path for the translated document", metavar="FILE")
|
||||||
|
|
||||||
argcomplete.autocomplete(parser)
|
argcomplete.autocomplete(parser)
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def create_directories(config: Config) -> None: # noqa: 11
|
||||||
|
"""
|
||||||
|
Create the directories in the given configuration if they don't exist.
|
||||||
|
"""
|
||||||
|
def make_dir(path: Path) -> None:
|
||||||
|
try:
|
||||||
|
os.makedirs(path.absolute())
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Creating directory '{path.absolute()}' failed with: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
# Cache
|
||||||
|
cache_path = Path(config.cache)
|
||||||
|
if not cache_path.exists():
|
||||||
|
answer = input(f"Cache directory '{cache_path}' does not exist. Create it? [y/n]")
|
||||||
|
if answer.lower() in ['y', 'yes']:
|
||||||
|
make_dir(cache_path.absolute())
|
||||||
|
else:
|
||||||
|
print("Can't continue without a valid cache directory!")
|
||||||
|
sys.exit(1)
|
||||||
|
# DB
|
||||||
|
db_path = Path(config.db)
|
||||||
|
if not db_path.exists():
|
||||||
|
answer = input(f"DB directory '{db_path}' does not exist. Create it? [y/n]")
|
||||||
|
if answer.lower() in ['y', 'yes']:
|
||||||
|
make_dir(db_path.absolute())
|
||||||
|
else:
|
||||||
|
print("Can't continue without a valid DB directory!")
|
||||||
|
sys.exit(1)
|
||||||
|
# Glossaries
|
||||||
|
if config.glossaries:
|
||||||
|
glossaries_path = Path(config.glossaries)
|
||||||
|
if not glossaries_path.exists():
|
||||||
|
answer = input(f"Glossaries directory '{glossaries_path}' does not exist. Create it? [y/n]")
|
||||||
|
if answer.lower() in ['y', 'yes']:
|
||||||
|
make_dir(glossaries_path.absolute())
|
||||||
|
else:
|
||||||
|
print("Can't continue without a valid glossaries directory. Create it or remove it from the configuration.")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = create_parser()
|
parser = create_parser()
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
@@ -149,6 +206,7 @@ def main() -> int:
|
|||||||
command.func(command)
|
command.func(command)
|
||||||
else:
|
else:
|
||||||
config = Config.from_file(args.config)
|
config = Config.from_file(args.config)
|
||||||
|
create_directories(config)
|
||||||
command.func(command, config)
|
command.func(command, config)
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
|
|||||||
@@ -71,11 +71,13 @@ class TestConfig(unittest.TestCase):
|
|||||||
'frequency_penalty': 0.7,
|
'frequency_penalty': 0.7,
|
||||||
'presence_penalty': 0.2
|
'presence_penalty': 0.2
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
|
'glossaries': './glossaries/'
|
||||||
}
|
}
|
||||||
config = Config.from_dict(source_dict)
|
config = Config.from_dict(source_dict)
|
||||||
self.assertEqual(config.cache, '.')
|
self.assertEqual(config.cache, '.')
|
||||||
self.assertEqual(config.db, './test_db/')
|
self.assertEqual(config.db, './test_db/')
|
||||||
|
self.assertEqual(config.glossaries, './glossaries/')
|
||||||
self.assertEqual(len(config.ais), 1)
|
self.assertEqual(len(config.ais), 1)
|
||||||
self.assertEqual(config.ais['myopenai'].name, 'openai')
|
self.assertEqual(config.ais['myopenai'].name, 'openai')
|
||||||
self.assertEqual(cast(OpenAIConfig, config.ais['myopenai']).system, 'Custom system')
|
self.assertEqual(cast(OpenAIConfig, config.ais['myopenai']).system, 'Custom system')
|
||||||
@@ -105,6 +107,7 @@ class TestConfig(unittest.TestCase):
|
|||||||
'frequency_penalty': 0.7,
|
'frequency_penalty': 0.7,
|
||||||
'presence_penalty': 0.2
|
'presence_penalty': 0.2
|
||||||
}
|
}
|
||||||
|
# omit glossaries, since it's optional
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
with open(self.test_file.name, 'w') as f:
|
with open(self.test_file.name, 'w') as f:
|
||||||
@@ -113,6 +116,8 @@ class TestConfig(unittest.TestCase):
|
|||||||
self.assertIsInstance(config, Config)
|
self.assertIsInstance(config, Config)
|
||||||
self.assertEqual(config.cache, './test_cache/')
|
self.assertEqual(config.cache, './test_cache/')
|
||||||
self.assertEqual(config.db, './test_db/')
|
self.assertEqual(config.db, './test_db/')
|
||||||
|
# missing 'glossaries' should result in 'None'
|
||||||
|
self.assertEqual(config.glossaries, None)
|
||||||
self.assertEqual(len(config.ais), 1)
|
self.assertEqual(len(config.ais), 1)
|
||||||
self.assertIsInstance(config.ais['default'], AIConfig)
|
self.assertIsInstance(config.ais['default'], AIConfig)
|
||||||
self.assertEqual(cast(OpenAIConfig, config.ais['default']).system, 'Custom system')
|
self.assertEqual(cast(OpenAIConfig, config.ais['default']).system, 'Custom system')
|
||||||
|
|||||||
@@ -0,0 +1,204 @@
|
|||||||
|
import unittest
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
from chatmastermind.glossary import Glossary, GlossaryError
|
||||||
|
|
||||||
|
|
||||||
|
glossary_suffix: str = Glossary.file_suffix
|
||||||
|
|
||||||
|
|
||||||
|
class TestGlossary(unittest.TestCase):
|
||||||
|
|
||||||
|
def test_from_file_yaml_unquoted(self) -> None:
|
||||||
|
"""
|
||||||
|
Test glossary creatiom from YAML with unquoted entries.
|
||||||
|
"""
|
||||||
|
with tempfile.NamedTemporaryFile('w', delete=False, suffix=glossary_suffix) as yaml_file:
|
||||||
|
yaml_file.write("Name: Sample\n"
|
||||||
|
"Description: A brief description\n"
|
||||||
|
"ID: '123'\n"
|
||||||
|
"SourceLang: en\n"
|
||||||
|
"TargetLang: es\n"
|
||||||
|
"Entries:\n"
|
||||||
|
" hello: hola\n"
|
||||||
|
" goodbye: adiós\n"
|
||||||
|
# 'yes' is a YAML keyword and would normally be quoted
|
||||||
|
" yes: sí\n"
|
||||||
|
" I'm going home: me voy a casa\n")
|
||||||
|
yaml_file_path = Path(yaml_file.name)
|
||||||
|
|
||||||
|
glossary = Glossary.from_file(yaml_file_path)
|
||||||
|
self.assertEqual(glossary.name, "Sample")
|
||||||
|
self.assertEqual(glossary.desc, "A brief description")
|
||||||
|
self.assertEqual(glossary.ID, "123")
|
||||||
|
self.assertEqual(glossary.source_lang, "en")
|
||||||
|
self.assertEqual(glossary.target_lang, "es")
|
||||||
|
self.assertEqual(glossary.entries, {"hello": "hola",
|
||||||
|
"goodbye": "adiós",
|
||||||
|
"yes": "sí",
|
||||||
|
"I'm going home": "me voy a casa"})
|
||||||
|
yaml_file_path.unlink() # Remove the temporary file
|
||||||
|
|
||||||
|
def test_from_file_yaml_quoted(self) -> None:
|
||||||
|
"""
|
||||||
|
Test glossary creatiom from YAML with quoted entries.
|
||||||
|
"""
|
||||||
|
with tempfile.NamedTemporaryFile('w', delete=False, suffix=glossary_suffix) as yaml_file:
|
||||||
|
yaml_file.write("Name: Sample\n"
|
||||||
|
"Description: A brief description\n"
|
||||||
|
"ID: '123'\n"
|
||||||
|
"SourceLang: en\n"
|
||||||
|
"TargetLang: es\n"
|
||||||
|
"Entries:\n"
|
||||||
|
" 'hello': 'hola'\n"
|
||||||
|
" 'goodbye': 'adiós'\n"
|
||||||
|
" 'yes': 'sí'\n"
|
||||||
|
" \"I'm going home\": 'me voy a casa'\n")
|
||||||
|
yaml_file_path = Path(yaml_file.name)
|
||||||
|
|
||||||
|
glossary = Glossary.from_file(yaml_file_path)
|
||||||
|
self.assertEqual(glossary.name, "Sample")
|
||||||
|
self.assertEqual(glossary.desc, "A brief description")
|
||||||
|
self.assertEqual(glossary.ID, "123")
|
||||||
|
self.assertEqual(glossary.source_lang, "en")
|
||||||
|
self.assertEqual(glossary.target_lang, "es")
|
||||||
|
self.assertEqual(glossary.entries, {"hello": "hola",
|
||||||
|
"goodbye": "adiós",
|
||||||
|
"yes": "sí",
|
||||||
|
"I'm going home": "me voy a casa"})
|
||||||
|
yaml_file_path.unlink() # Remove the temporary file
|
||||||
|
|
||||||
|
def test_to_file_writes_yaml(self) -> None:
|
||||||
|
# Create glossary instance
|
||||||
|
glossary = Glossary(name="Test",
|
||||||
|
desc="Test description",
|
||||||
|
ID="666",
|
||||||
|
source_lang="en",
|
||||||
|
target_lang="fr",
|
||||||
|
entries={"yes": "oui"})
|
||||||
|
|
||||||
|
with tempfile.NamedTemporaryFile('w', delete=False, suffix=glossary_suffix) as tmp_file:
|
||||||
|
file_path = Path(tmp_file.name)
|
||||||
|
glossary.to_file(file_path)
|
||||||
|
|
||||||
|
with open(file_path, 'r') as file:
|
||||||
|
content = file.read()
|
||||||
|
|
||||||
|
self.assertIn("Name: Test", content)
|
||||||
|
self.assertIn("Description: Test description", content)
|
||||||
|
self.assertIn("ID: '666'", content)
|
||||||
|
self.assertIn("SourceLang: en", content)
|
||||||
|
self.assertIn("TargetLang: fr", content)
|
||||||
|
self.assertIn("Entries", content)
|
||||||
|
# 'yes' is a YAML keyword and therefore quoted
|
||||||
|
self.assertIn("'yes': oui", content)
|
||||||
|
file_path.unlink() # Remove the temporary file
|
||||||
|
|
||||||
|
def test_write_read_glossary(self) -> None:
|
||||||
|
# Create glossary instance
|
||||||
|
# -> use 'yes' in order to test if the YAML quoting is correctly removed when reading the file
|
||||||
|
glossary_write = Glossary(name="Test", source_lang="en", target_lang="fr", entries={"yes": "oui"})
|
||||||
|
|
||||||
|
with tempfile.NamedTemporaryFile('w', delete=False, suffix=glossary_suffix) as tmp_file:
|
||||||
|
file_path = Path(tmp_file.name)
|
||||||
|
glossary_write.to_file(file_path)
|
||||||
|
|
||||||
|
# create new instance from glossary file
|
||||||
|
glossary_read = Glossary.from_file(file_path)
|
||||||
|
self.assertEqual(glossary_write.name, glossary_read.name)
|
||||||
|
self.assertEqual(glossary_write.source_lang, glossary_read.source_lang)
|
||||||
|
self.assertEqual(glossary_write.target_lang, glossary_read.target_lang)
|
||||||
|
self.assertDictEqual(glossary_write.entries, glossary_read.entries)
|
||||||
|
|
||||||
|
file_path.unlink() # Remove the temporary file
|
||||||
|
|
||||||
|
def test_import_export_csv(self) -> None:
|
||||||
|
glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={})
|
||||||
|
|
||||||
|
# First export to CSV
|
||||||
|
with tempfile.NamedTemporaryFile('w', delete=False, suffix=glossary_suffix) as csvfile:
|
||||||
|
csv_file_path = Path(csvfile.name)
|
||||||
|
glossary.entries = {"hello": "salut", "goodbye": "au revoir"}
|
||||||
|
glossary.export_csv(glossary.entries, csv_file_path)
|
||||||
|
|
||||||
|
# Now import CSV
|
||||||
|
glossary.import_csv(csv_file_path)
|
||||||
|
self.assertEqual(glossary.entries, {"hello": "salut", "goodbye": "au revoir"})
|
||||||
|
csv_file_path.unlink() # Remove the temporary file
|
||||||
|
|
||||||
|
def test_import_export_tsv(self) -> None:
|
||||||
|
glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={})
|
||||||
|
|
||||||
|
# First export to TSV
|
||||||
|
with tempfile.NamedTemporaryFile('w', delete=False, suffix=glossary_suffix) as tsvfile:
|
||||||
|
tsv_file_path = Path(tsvfile.name)
|
||||||
|
glossary.entries = {"hello": "salut", "goodbye": "au revoir"}
|
||||||
|
glossary.export_tsv(glossary.entries, tsv_file_path)
|
||||||
|
|
||||||
|
# Now import TSV
|
||||||
|
glossary.import_tsv(tsv_file_path)
|
||||||
|
self.assertEqual(glossary.entries, {"hello": "salut", "goodbye": "au revoir"})
|
||||||
|
tsv_file_path.unlink() # Remove the temporary file
|
||||||
|
|
||||||
|
def test_to_file_wrong_suffix(self) -> None:
|
||||||
|
"""
|
||||||
|
Test for exception if suffix is wrong.
|
||||||
|
"""
|
||||||
|
glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={"yes": "oui"})
|
||||||
|
with tempfile.NamedTemporaryFile('w', delete=False, suffix='.wrong') as tmp_file:
|
||||||
|
file_path = Path(tmp_file.name)
|
||||||
|
with self.assertRaises(GlossaryError) as err:
|
||||||
|
glossary.to_file(file_path)
|
||||||
|
self.assertEqual(str(err.exception), "File suffix '.wrong' is not supported")
|
||||||
|
|
||||||
|
def test_to_file_auto_suffix(self) -> None:
|
||||||
|
"""
|
||||||
|
Test if suffix is auto-generated if omitted.
|
||||||
|
"""
|
||||||
|
glossary = Glossary(name="Test", source_lang="en", target_lang="fr", entries={"yes": "oui"})
|
||||||
|
with tempfile.NamedTemporaryFile('w', delete=False, suffix='') as tmp_file:
|
||||||
|
file_path = Path(tmp_file.name)
|
||||||
|
glossary.to_file(file_path)
|
||||||
|
assert glossary.file_path is not None
|
||||||
|
self.assertEqual(glossary.file_path.suffix, glossary_suffix)
|
||||||
|
|
||||||
|
def test_to_str_with_id(self) -> None:
|
||||||
|
# Create a Glossary instance with an ID
|
||||||
|
glossary_with_id = Glossary(name="TestGlossary", source_lang="en", target_lang="fr",
|
||||||
|
desc="A simple test glossary", ID="1001", entries={"one": "un"})
|
||||||
|
glossary_str = glossary_with_id.to_str()
|
||||||
|
self.assertIn("TestGlossary (ID: 1001):", glossary_str)
|
||||||
|
self.assertIn("- A simple test glossary", glossary_str)
|
||||||
|
self.assertIn("- Languages: en -> fr", glossary_str)
|
||||||
|
self.assertIn("- Entries: 1", glossary_str)
|
||||||
|
|
||||||
|
def test_to_str_with_id_and_entries(self) -> None:
|
||||||
|
# Create a Glossary instance with an ID and include entries
|
||||||
|
glossary_with_entries = Glossary(name="TestGlossaryWithEntries", source_lang="en", target_lang="fr",
|
||||||
|
desc="Another test glossary", ID="2002",
|
||||||
|
entries={"hello": "salut", "goodbye": "au revoir"})
|
||||||
|
glossary_str_with_entries = glossary_with_entries.to_str(with_entries=True)
|
||||||
|
self.assertIn("TestGlossaryWithEntries (ID: 2002):", glossary_str_with_entries)
|
||||||
|
self.assertIn("- Entries:", glossary_str_with_entries)
|
||||||
|
self.assertIn(" hello : salut", glossary_str_with_entries)
|
||||||
|
self.assertIn(" goodbye : au revoir", glossary_str_with_entries)
|
||||||
|
|
||||||
|
def test_to_str_without_id(self) -> None:
|
||||||
|
# Create a Glossary instance without an ID
|
||||||
|
glossary_without_id = Glossary(name="TestGlossaryNoID", source_lang="en", target_lang="fr",
|
||||||
|
desc="A test glossary without an ID", ID=None, entries={"yes": "oui"})
|
||||||
|
glossary_str_no_id = glossary_without_id.to_str()
|
||||||
|
self.assertIn("TestGlossaryNoID (ID: None):", glossary_str_no_id)
|
||||||
|
self.assertIn("- A test glossary without an ID", glossary_str_no_id)
|
||||||
|
self.assertIn("- Languages: en -> fr", glossary_str_no_id)
|
||||||
|
self.assertIn("- Entries: 1", glossary_str_no_id)
|
||||||
|
|
||||||
|
def test_to_str_without_id_and_no_entries(self) -> None:
|
||||||
|
# Create a Glossary instance without an ID and no entries
|
||||||
|
glossary_no_id_no_entries = Glossary(name="EmptyGlossary", source_lang="en", target_lang="fr",
|
||||||
|
desc="An empty test glossary", ID=None, entries={})
|
||||||
|
glossary_str_no_id_no_entries = glossary_no_id_no_entries.to_str()
|
||||||
|
self.assertIn("EmptyGlossary (ID: None):", glossary_str_no_id_no_entries)
|
||||||
|
self.assertIn("- An empty test glossary", glossary_str_no_id_no_entries)
|
||||||
|
self.assertIn("- Languages: en -> fr", glossary_str_no_id_no_entries)
|
||||||
|
self.assertIn("- Entries: 0", glossary_str_no_id_no_entries)
|
||||||
Reference in New Issue
Block a user