feat(translate): multi-language optimization (Phase 11)
- Auto-detection of source language per row via LLM (US6) - Multi-target translation — one LLM call for N languages (US1-US3) - Language-aware storage: TranslationLanguage, per-language stats - Multilingual dictionaries with language-pair-aware filtering (US7) - Inline correction on any run result + submit-to-dictionary (US8) - Context-aware dictionary: auto-capture row context, usage notes, Jaccard similarity, priority flagging in LLM prompts (US8b) - Configurable preview sample size 1-100, cost warning at >30 - Per-language history & metrics with MetricSnapshot preservation - 36 files, +5022/-373, all specs GRACE-Poly v2.6 compliant
This commit is contained in:
149
backend/alembic.ini
Normal file
149
backend/alembic.ini
Normal file
@@ -0,0 +1,149 @@
|
||||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts.
|
||||
# this is typically a path given in POSIX (e.g. forward slashes)
|
||||
# format, relative to the token %(here)s which refers to the location of this
|
||||
# ini file
|
||||
script_location = %(here)s/alembic
|
||||
|
||||
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||
# Uncomment the line below if you want the files to be prepended with date and time
|
||||
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
|
||||
# for all available tokens
|
||||
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||
# Or organize into date-based subdirectories (requires recursive_version_locations = true)
|
||||
# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory. for multiple paths, the path separator
|
||||
# is defined by "path_separator" below.
|
||||
prepend_sys_path = .
|
||||
|
||||
|
||||
# timezone to use when rendering the date within the migration file
|
||||
# as well as the filename.
|
||||
# If specified, requires the tzdata library which can be installed by adding
|
||||
# `alembic[tz]` to the pip requirements.
|
||||
# string value is passed to ZoneInfo()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; This defaults
|
||||
# to <script_location>/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path.
|
||||
# The path separator used here should be the separator specified by "path_separator"
|
||||
# below.
|
||||
# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions
|
||||
|
||||
# path_separator; This indicates what character is used to split lists of file
|
||||
# paths, including version_locations and prepend_sys_path within configparser
|
||||
# files such as alembic.ini.
|
||||
# The default rendered in new alembic.ini files is "os", which uses os.pathsep
|
||||
# to provide os-dependent path splitting.
|
||||
#
|
||||
# Note that in order to support legacy alembic.ini files, this default does NOT
|
||||
# take place if path_separator is not present in alembic.ini. If this
|
||||
# option is omitted entirely, fallback logic is as follows:
|
||||
#
|
||||
# 1. Parsing of the version_locations option falls back to using the legacy
|
||||
# "version_path_separator" key, which if absent then falls back to the legacy
|
||||
# behavior of splitting on spaces and/or commas.
|
||||
# 2. Parsing of the prepend_sys_path option falls back to the legacy
|
||||
# behavior of splitting on spaces, commas, or colons.
|
||||
#
|
||||
# Valid values for path_separator are:
|
||||
#
|
||||
# path_separator = :
|
||||
# path_separator = ;
|
||||
# path_separator = space
|
||||
# path_separator = newline
|
||||
#
|
||||
# Use os.pathsep. Default configuration used for new projects.
|
||||
path_separator = os
|
||||
|
||||
# set to 'true' to search source files recursively
|
||||
# in each "version_locations" directory
|
||||
# new in Alembic version 1.10
|
||||
# recursive_version_locations = false
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
# database URL. This is consumed by the user-maintained env.py script only.
|
||||
# other means of configuring database URLs may be customized within the env.py
|
||||
# file.
|
||||
sqlalchemy.url = sqlite:///./alembic_test.db
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module
|
||||
# hooks = ruff
|
||||
# ruff.type = module
|
||||
# ruff.module = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Alternatively, use the exec runner to execute a binary found on your PATH
|
||||
# hooks = ruff
|
||||
# ruff.type = exec
|
||||
# ruff.executable = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration. This is also consumed by the user-maintained
|
||||
# env.py script only.
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
1
backend/alembic/README
Normal file
1
backend/alembic/README
Normal file
@@ -0,0 +1 @@
|
||||
Generic single-database configuration.
|
||||
100
backend/alembic/env.py
Normal file
100
backend/alembic/env.py
Normal file
@@ -0,0 +1,100 @@
|
||||
import sys
|
||||
from logging.config import fileConfig
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import engine_from_config
|
||||
from sqlalchemy import pool
|
||||
|
||||
from alembic import context
|
||||
|
||||
# Ensure backend/src is importable
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
# This line sets up loggers basically.
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# add your model's MetaData object here
|
||||
# for 'autogenerate' support
|
||||
from src.models.mapping import Base
|
||||
from src.models.translate import (
|
||||
TranslationJob,
|
||||
TranslationRun,
|
||||
TranslationBatch,
|
||||
TranslationRecord,
|
||||
TranslationEvent,
|
||||
TranslationPreviewSession,
|
||||
TranslationPreviewRecord,
|
||||
TerminologyDictionary,
|
||||
DictionaryEntry,
|
||||
TranslationSchedule,
|
||||
TranslationJobDictionary,
|
||||
MetricSnapshot,
|
||||
TranslationLanguage,
|
||||
TranslationPreviewLanguage,
|
||||
TranslationRunLanguageStats,
|
||||
)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
|
||||
"""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
"""
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection, target_metadata=target_metadata
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
28
backend/alembic/script.py.mako
Normal file
28
backend/alembic/script.py.mako
Normal file
@@ -0,0 +1,28 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,392 @@
|
||||
"""multi-language translation tables
|
||||
|
||||
Revision ID: ed310b33f02c
|
||||
Revises:
|
||||
Create Date: 2026-05-14 15:35:03.167031
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'ed310b33f02c'
|
||||
down_revision: Union[str, Sequence[str], None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('environments',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('name', sa.String(), nullable=False),
|
||||
sa.Column('url', sa.String(), nullable=False),
|
||||
sa.Column('credentials_id', sa.String(), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('terminology_dictionaries',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('name', sa.String(), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('source_dialect', sa.String(), nullable=False),
|
||||
sa.Column('target_dialect', sa.String(), nullable=False),
|
||||
sa.Column('source_language', sa.String(), nullable=True, comment='[DEPRECATED: use per-entry source_language]'),
|
||||
sa.Column('target_language', sa.String(), nullable=True, comment='[DEPRECATED: use per-entry target_language]'),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=True),
|
||||
sa.Column('created_by', sa.String(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('translation_jobs',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('name', sa.String(), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('source_dialect', sa.String(), nullable=False),
|
||||
sa.Column('target_dialect', sa.String(), nullable=False),
|
||||
sa.Column('database_dialect', sa.String(), nullable=True, comment='Detected dialect from Superset connection at save time'),
|
||||
sa.Column('status', sa.String(), nullable=False),
|
||||
sa.Column('source_datasource_id', sa.String(), nullable=True, comment='Superset datasource ID'),
|
||||
sa.Column('source_table', sa.String(), nullable=True, comment='Source table name resolved from datasource'),
|
||||
sa.Column('target_schema', sa.String(), nullable=True, comment='Target table schema'),
|
||||
sa.Column('target_table', sa.String(), nullable=True, comment='Target table name'),
|
||||
sa.Column('source_key_cols', sa.JSON(), nullable=True, comment='Source key column names for composite key'),
|
||||
sa.Column('target_key_cols', sa.JSON(), nullable=True, comment='Target key column names for composite key'),
|
||||
sa.Column('translation_column', sa.String(), nullable=True, comment='Source column whose values will be translated'),
|
||||
sa.Column('target_column', sa.String(), nullable=True, comment='Target column for translated output (defaults to translation_column)'),
|
||||
sa.Column('context_columns', sa.JSON(), nullable=True, comment='Context column names included in LLM prompt'),
|
||||
sa.Column('target_language', sa.String(), nullable=True, comment='Target language code (e.g. en, ru) [DEPRECATED: use target_languages]'),
|
||||
sa.Column('source_language', sa.String(), nullable=True, comment='Fallback source language hint [DEPRECATED: auto-detected per row]'),
|
||||
sa.Column('target_languages', sa.JSON(), nullable=True, comment='List of BCP-47 target language codes (multi-language support)'),
|
||||
sa.Column('provider_id', sa.String(), nullable=True, comment='LLM provider ID'),
|
||||
sa.Column('batch_size', sa.Integer(), nullable=False, comment='Records per batch'),
|
||||
sa.Column('upsert_strategy', sa.String(), nullable=False, comment='MERGE, INSERT, UPDATE'),
|
||||
sa.Column('environment_id', sa.String(), nullable=True, comment='Superset environment ID for datasource access'),
|
||||
sa.Column('target_database_id', sa.String(), nullable=True, comment='Superset database ID for SQL Lab insert target'),
|
||||
sa.Column('created_by', sa.String(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('database_mappings',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('source_env_id', sa.String(), nullable=False),
|
||||
sa.Column('target_env_id', sa.String(), nullable=False),
|
||||
sa.Column('source_db_uuid', sa.String(), nullable=False),
|
||||
sa.Column('target_db_uuid', sa.String(), nullable=False),
|
||||
sa.Column('source_db_name', sa.String(), nullable=False),
|
||||
sa.Column('target_db_name', sa.String(), nullable=False),
|
||||
sa.Column('engine', sa.String(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['source_env_id'], ['environments.id'], ),
|
||||
sa.ForeignKeyConstraint(['target_env_id'], ['environments.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('dictionary_entries',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('dictionary_id', sa.String(), nullable=False),
|
||||
sa.Column('source_term', sa.String(), nullable=False),
|
||||
sa.Column('source_term_normalized', sa.String(), nullable=False),
|
||||
sa.Column('target_term', sa.String(), nullable=False),
|
||||
sa.Column('source_language', sa.String(), nullable=False, comment='BCP-47 source language code'),
|
||||
sa.Column('target_language', sa.String(), nullable=False, comment='BCP-47 target language code'),
|
||||
sa.Column('context_notes', sa.Text(), nullable=True),
|
||||
sa.Column('context_data', sa.JSON(), nullable=True, comment='Structured context for term usage'),
|
||||
sa.Column('usage_notes', sa.Text(), nullable=True, comment='Usage guidance for the term mapping'),
|
||||
sa.Column('has_context', sa.Boolean(), nullable=True, comment='Whether context_data is populated'),
|
||||
sa.Column('context_source', sa.String(), nullable=True, comment='auto|auto_with_edits|manual|bulk'),
|
||||
sa.Column('origin_source_language', sa.String(), nullable=True, comment='Original source language of the term'),
|
||||
sa.Column('origin_run_id', sa.String(), nullable=True, comment='Run ID from which this correction originated'),
|
||||
sa.Column('origin_row_key', sa.String(), nullable=True, comment='Row key within the run that triggered this correction'),
|
||||
sa.Column('origin_user_id', sa.String(), nullable=True, comment='User who submitted the correction'),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['dictionary_id'], ['terminology_dictionaries.id'], ),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('dictionary_id', 'source_term_normalized', 'source_language', 'target_language', name='uq_dict_source_term_lang')
|
||||
)
|
||||
op.create_index('idx_dict_entry_lang', 'dictionary_entries', ['source_language', 'target_language'], unique=False)
|
||||
op.create_index('idx_dict_has_context', 'dictionary_entries', ['has_context'], unique=False)
|
||||
op.create_index(op.f('ix_dictionary_entries_dictionary_id'), 'dictionary_entries', ['dictionary_id'], unique=False)
|
||||
op.create_table('migration_jobs',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('source_env_id', sa.String(), nullable=False),
|
||||
sa.Column('target_env_id', sa.String(), nullable=False),
|
||||
sa.Column('status', sa.Enum('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', 'AWAITING_MAPPING', name='migrationstatus'), nullable=True),
|
||||
sa.Column('replace_db', sa.Boolean(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
|
||||
sa.ForeignKeyConstraint(['source_env_id'], ['environments.id'], ),
|
||||
sa.ForeignKeyConstraint(['target_env_id'], ['environments.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('resource_mappings',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('environment_id', sa.String(), nullable=False),
|
||||
sa.Column('resource_type', sa.Enum('CHART', 'DATASET', 'DASHBOARD', name='resourcetype'), nullable=False),
|
||||
sa.Column('uuid', sa.String(), nullable=False),
|
||||
sa.Column('remote_integer_id', sa.String(), nullable=False),
|
||||
sa.Column('resource_name', sa.String(), nullable=True),
|
||||
sa.Column('last_synced_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
|
||||
sa.ForeignKeyConstraint(['environment_id'], ['environments.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_table('translation_job_dictionaries',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('job_id', sa.String(), nullable=False),
|
||||
sa.Column('dictionary_id', sa.String(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['dictionary_id'], ['terminology_dictionaries.id'], ),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('job_id', 'dictionary_id', name='uq_job_dictionary')
|
||||
)
|
||||
op.create_index(op.f('ix_translation_job_dictionaries_dictionary_id'), 'translation_job_dictionaries', ['dictionary_id'], unique=False)
|
||||
op.create_index(op.f('ix_translation_job_dictionaries_job_id'), 'translation_job_dictionaries', ['job_id'], unique=False)
|
||||
op.create_table('translation_runs',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('job_id', sa.String(), nullable=False),
|
||||
sa.Column('status', sa.String(), nullable=False),
|
||||
sa.Column('trigger_type', sa.String(), nullable=True, comment='manual, scheduled, retry, baseline_expired'),
|
||||
sa.Column('started_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('completed_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('error_message', sa.Text(), nullable=True),
|
||||
sa.Column('total_records', sa.Integer(), nullable=True),
|
||||
sa.Column('successful_records', sa.Integer(), nullable=True),
|
||||
sa.Column('failed_records', sa.Integer(), nullable=True),
|
||||
sa.Column('skipped_records', sa.Integer(), nullable=True),
|
||||
sa.Column('insert_status', sa.String(), nullable=True, comment='Status of the Superset insert/update operation'),
|
||||
sa.Column('superset_execution_id', sa.String(), nullable=True, comment='Superset execution/task ID'),
|
||||
sa.Column('superset_execution_log', sa.JSON(), nullable=True, comment='Superset execution log output'),
|
||||
sa.Column('config_snapshot', sa.JSON(), nullable=True, comment='Snapshot of job config at run creation time'),
|
||||
sa.Column('key_hash', sa.String(), nullable=True, comment='Hash of source key fields for dedup'),
|
||||
sa.Column('config_hash', sa.String(), nullable=True, comment='Hash of translation configuration state'),
|
||||
sa.Column('dict_snapshot_hash', sa.String(), nullable=True, comment='Hash of dictionary state at run time'),
|
||||
sa.Column('created_by', sa.String(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_translation_runs_job_id'), 'translation_runs', ['job_id'], unique=False)
|
||||
op.create_table('translation_schedules',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('job_id', sa.String(), nullable=False),
|
||||
sa.Column('cron_expression', sa.String(), nullable=False),
|
||||
sa.Column('timezone', sa.String(), nullable=False),
|
||||
sa.Column('is_active', sa.Boolean(), nullable=True),
|
||||
sa.Column('last_run_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('next_run_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('execution_mode', sa.String(), nullable=False, comment='full, new_key_only'),
|
||||
sa.Column('created_by', sa.String(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_translation_schedules_job_id'), 'translation_schedules', ['job_id'], unique=False)
|
||||
op.create_table('translation_batches',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('run_id', sa.String(), nullable=False),
|
||||
sa.Column('batch_index', sa.Integer(), nullable=False),
|
||||
sa.Column('status', sa.String(), nullable=False),
|
||||
sa.Column('total_records', sa.Integer(), nullable=True),
|
||||
sa.Column('successful_records', sa.Integer(), nullable=True),
|
||||
sa.Column('failed_records', sa.Integer(), nullable=True),
|
||||
sa.Column('started_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('completed_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_translation_batches_run_id'), 'translation_batches', ['run_id'], unique=False)
|
||||
op.create_table('translation_events',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('job_id', sa.String(), nullable=False),
|
||||
sa.Column('run_id', sa.String(), nullable=True),
|
||||
sa.Column('event_type', sa.String(), nullable=False),
|
||||
sa.Column('event_data', sa.JSON(), nullable=True),
|
||||
sa.Column('created_by', sa.String(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ),
|
||||
sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_translation_events_job_id'), 'translation_events', ['job_id'], unique=False)
|
||||
op.create_index(op.f('ix_translation_events_run_id'), 'translation_events', ['run_id'], unique=False)
|
||||
op.create_table('translation_metric_snapshots',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('job_id', sa.String(), nullable=False),
|
||||
sa.Column('run_id', sa.String(), nullable=True),
|
||||
sa.Column('key_hash', sa.String(), nullable=False, comment='Hash of dimension key fields for aggregation'),
|
||||
sa.Column('config_hash', sa.String(), nullable=True, comment='Hash of translation configuration state'),
|
||||
sa.Column('dict_snapshot_hash', sa.String(), nullable=True, comment='Hash of dictionary state at capture time'),
|
||||
sa.Column('covers_events_before', sa.DateTime(), nullable=True, comment='Indicates snapshot covers events before this timestamp'),
|
||||
sa.Column('total_jobs', sa.Integer(), nullable=True),
|
||||
sa.Column('total_runs', sa.Integer(), nullable=True),
|
||||
sa.Column('total_records', sa.Integer(), nullable=True),
|
||||
sa.Column('successful_records', sa.Integer(), nullable=True),
|
||||
sa.Column('failed_records', sa.Integer(), nullable=True),
|
||||
sa.Column('skipped_records', sa.Integer(), nullable=True),
|
||||
sa.Column('avg_duration_ms', sa.Integer(), nullable=True),
|
||||
sa.Column('p50_duration_ms', sa.Integer(), nullable=True),
|
||||
sa.Column('p95_duration_ms', sa.Integer(), nullable=True),
|
||||
sa.Column('p99_duration_ms', sa.Integer(), nullable=True),
|
||||
sa.Column('snapshot_date', sa.DateTime(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ),
|
||||
sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index('ix_metric_snapshots_job_date', 'translation_metric_snapshots', ['job_id', 'snapshot_date'], unique=False)
|
||||
op.create_index(op.f('ix_translation_metric_snapshots_job_id'), 'translation_metric_snapshots', ['job_id'], unique=False)
|
||||
op.create_index(op.f('ix_translation_metric_snapshots_run_id'), 'translation_metric_snapshots', ['run_id'], unique=False)
|
||||
op.create_table('translation_preview_sessions',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('job_id', sa.String(), nullable=False),
|
||||
sa.Column('run_id', sa.String(), nullable=True),
|
||||
sa.Column('status', sa.String(), nullable=False),
|
||||
sa.Column('created_by', sa.String(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('expires_at', sa.DateTime(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ),
|
||||
sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_translation_preview_sessions_job_id'), 'translation_preview_sessions', ['job_id'], unique=False)
|
||||
op.create_table('translation_run_language_stats',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('run_id', sa.String(), nullable=False),
|
||||
sa.Column('language_code', sa.String(), nullable=False, comment='BCP-47 language code'),
|
||||
sa.Column('total_rows', sa.Integer(), nullable=True),
|
||||
sa.Column('translated_rows', sa.Integer(), nullable=True),
|
||||
sa.Column('failed_rows', sa.Integer(), nullable=True),
|
||||
sa.Column('skipped_rows', sa.Integer(), nullable=True),
|
||||
sa.Column('token_count', sa.Integer(), nullable=True),
|
||||
sa.Column('estimated_cost', sa.Float(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('run_id', 'language_code', name='uq_run_language')
|
||||
)
|
||||
op.create_index('idx_rls_run', 'translation_run_language_stats', ['run_id'], unique=False)
|
||||
op.create_table('translation_preview_records',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('session_id', sa.String(), nullable=False),
|
||||
sa.Column('source_sql', sa.Text(), nullable=True),
|
||||
sa.Column('target_sql', sa.Text(), nullable=True),
|
||||
sa.Column('source_object_type', sa.String(), nullable=True),
|
||||
sa.Column('source_object_id', sa.String(), nullable=True),
|
||||
sa.Column('source_object_name', sa.String(), nullable=True),
|
||||
sa.Column('source_data', sa.JSON(), nullable=True, comment='Original source row key columns for upsert matching'),
|
||||
sa.Column('status', sa.String(), nullable=False),
|
||||
sa.Column('feedback', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['session_id'], ['translation_preview_sessions.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_translation_preview_records_session_id'), 'translation_preview_records', ['session_id'], unique=False)
|
||||
op.create_table('translation_records',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('batch_id', sa.String(), nullable=False),
|
||||
sa.Column('run_id', sa.String(), nullable=False),
|
||||
sa.Column('source_sql', sa.Text(), nullable=True),
|
||||
sa.Column('target_sql', sa.Text(), nullable=True),
|
||||
sa.Column('source_object_type', sa.String(), nullable=True),
|
||||
sa.Column('source_object_id', sa.String(), nullable=True),
|
||||
sa.Column('source_object_name', sa.String(), nullable=True),
|
||||
sa.Column('source_data', sa.JSON(), nullable=True, comment='Original source row key columns for upsert matching'),
|
||||
sa.Column('status', sa.String(), nullable=False),
|
||||
sa.Column('error_message', sa.Text(), nullable=True),
|
||||
sa.Column('token_count_input', sa.Integer(), nullable=True),
|
||||
sa.Column('token_count_output', sa.Integer(), nullable=True),
|
||||
sa.Column('translation_duration_ms', sa.Integer(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('llm_translation', sa.Text(), nullable=True, comment='[DEPRECATED: use TranslationLanguage]'),
|
||||
sa.Column('user_edit', sa.Text(), nullable=True, comment='[DEPRECATED: use TranslationLanguage]'),
|
||||
sa.Column('final_value', sa.Text(), nullable=True, comment='[DEPRECATED: use TranslationLanguage]'),
|
||||
sa.ForeignKeyConstraint(['batch_id'], ['translation_batches.id'], ),
|
||||
sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_translation_records_batch_id'), 'translation_records', ['batch_id'], unique=False)
|
||||
op.create_index(op.f('ix_translation_records_run_id'), 'translation_records', ['run_id'], unique=False)
|
||||
op.create_index('ix_translation_records_run_status', 'translation_records', ['run_id', 'status'], unique=False)
|
||||
op.create_table('translation_languages',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('record_id', sa.String(), nullable=False),
|
||||
sa.Column('language_code', sa.String(), nullable=False, comment='BCP-47 language code'),
|
||||
sa.Column('source_language_detected', sa.String(), nullable=True, comment="BCP-47 or 'und' for undetermined"),
|
||||
sa.Column('translated_value', sa.Text(), nullable=True, comment='LLM-generated translation'),
|
||||
sa.Column('user_edit', sa.Text(), nullable=True, comment='User-edited translation'),
|
||||
sa.Column('final_value', sa.Text(), nullable=True, comment='Final resolved value (translated or user edit)'),
|
||||
sa.Column('status', sa.String(), nullable=True, comment='pending|translated|approved|edited|rejected|failed|skipped'),
|
||||
sa.Column('error_message', sa.Text(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.ForeignKeyConstraint(['record_id'], ['translation_records.id'], ),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('record_id', 'language_code', name='uq_record_language')
|
||||
)
|
||||
op.create_index('idx_tl_language', 'translation_languages', ['language_code'], unique=False)
|
||||
op.create_index('idx_tl_record_lang', 'translation_languages', ['record_id', 'language_code'], unique=False)
|
||||
op.create_table('translation_preview_languages',
|
||||
sa.Column('id', sa.String(), nullable=False),
|
||||
sa.Column('preview_record_id', sa.String(), nullable=False),
|
||||
sa.Column('language_code', sa.String(), nullable=False, comment='BCP-47 language code'),
|
||||
sa.Column('source_language_detected', sa.String(), nullable=True, comment="BCP-47 or 'und'"),
|
||||
sa.Column('translated_value', sa.Text(), nullable=True),
|
||||
sa.Column('user_edit', sa.Text(), nullable=True),
|
||||
sa.Column('final_value', sa.Text(), nullable=True),
|
||||
sa.Column('status', sa.String(), nullable=True, comment='pending|approved|edited|rejected'),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.ForeignKeyConstraint(['preview_record_id'], ['translation_preview_records.id'], ),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('preview_record_id', 'language_code', name='uq_preview_record_language')
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('translation_preview_languages')
|
||||
op.drop_index('idx_tl_record_lang', table_name='translation_languages')
|
||||
op.drop_index('idx_tl_language', table_name='translation_languages')
|
||||
op.drop_table('translation_languages')
|
||||
op.drop_index('ix_translation_records_run_status', table_name='translation_records')
|
||||
op.drop_index(op.f('ix_translation_records_run_id'), table_name='translation_records')
|
||||
op.drop_index(op.f('ix_translation_records_batch_id'), table_name='translation_records')
|
||||
op.drop_table('translation_records')
|
||||
op.drop_index(op.f('ix_translation_preview_records_session_id'), table_name='translation_preview_records')
|
||||
op.drop_table('translation_preview_records')
|
||||
op.drop_index('idx_rls_run', table_name='translation_run_language_stats')
|
||||
op.drop_table('translation_run_language_stats')
|
||||
op.drop_index(op.f('ix_translation_preview_sessions_job_id'), table_name='translation_preview_sessions')
|
||||
op.drop_table('translation_preview_sessions')
|
||||
op.drop_index(op.f('ix_translation_metric_snapshots_run_id'), table_name='translation_metric_snapshots')
|
||||
op.drop_index(op.f('ix_translation_metric_snapshots_job_id'), table_name='translation_metric_snapshots')
|
||||
op.drop_index('ix_metric_snapshots_job_date', table_name='translation_metric_snapshots')
|
||||
op.drop_table('translation_metric_snapshots')
|
||||
op.drop_index(op.f('ix_translation_events_run_id'), table_name='translation_events')
|
||||
op.drop_index(op.f('ix_translation_events_job_id'), table_name='translation_events')
|
||||
op.drop_table('translation_events')
|
||||
op.drop_index(op.f('ix_translation_batches_run_id'), table_name='translation_batches')
|
||||
op.drop_table('translation_batches')
|
||||
op.drop_index(op.f('ix_translation_schedules_job_id'), table_name='translation_schedules')
|
||||
op.drop_table('translation_schedules')
|
||||
op.drop_index(op.f('ix_translation_runs_job_id'), table_name='translation_runs')
|
||||
op.drop_table('translation_runs')
|
||||
op.drop_index(op.f('ix_translation_job_dictionaries_job_id'), table_name='translation_job_dictionaries')
|
||||
op.drop_index(op.f('ix_translation_job_dictionaries_dictionary_id'), table_name='translation_job_dictionaries')
|
||||
op.drop_table('translation_job_dictionaries')
|
||||
op.drop_table('resource_mappings')
|
||||
op.drop_table('migration_jobs')
|
||||
op.drop_index(op.f('ix_dictionary_entries_dictionary_id'), table_name='dictionary_entries')
|
||||
op.drop_index('idx_dict_has_context', table_name='dictionary_entries')
|
||||
op.drop_index('idx_dict_entry_lang', table_name='dictionary_entries')
|
||||
op.drop_table('dictionary_entries')
|
||||
op.drop_table('database_mappings')
|
||||
op.drop_table('translation_jobs')
|
||||
op.drop_table('terminology_dictionaries')
|
||||
op.drop_table('environments')
|
||||
# ### end Alembic commands ###
|
||||
@@ -45,6 +45,8 @@ async def submit_correction(
|
||||
origin_run_id=payload.origin_run_id,
|
||||
origin_row_key=payload.origin_row_key,
|
||||
origin_user_id=current_user.username,
|
||||
context_data=payload.context_data,
|
||||
usage_notes=payload.usage_notes,
|
||||
)
|
||||
return result
|
||||
except ValueError as e:
|
||||
|
||||
@@ -163,14 +163,16 @@ async def delete_dictionary(
|
||||
# ============================================================
|
||||
|
||||
# #region list_dictionary_entries [TYPE Function]
|
||||
# @BRIEF List entries for a dictionary.
|
||||
# @BRIEF List entries for a dictionary, optionally filtered by language pair.
|
||||
# @PRE: User has translate.dictionary.view permission.
|
||||
# @POST: Returns paginated list of entries.
|
||||
# @POST: Returns paginated list of entries with language pair fields.
|
||||
@router.get("/dictionaries/{dictionary_id}/entries")
|
||||
async def list_dictionary_entries(
|
||||
dictionary_id: str,
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(50, ge=1, le=500),
|
||||
source_language: str | None = Query(None, description="Filter by BCP-47 source language"),
|
||||
target_language: str | None = Query(None, description="Filter by BCP-47 target language"),
|
||||
current_user: User = Depends(get_current_user),
|
||||
_ = Depends(has_permission("translate.dictionary", "VIEW")),
|
||||
db: Session = Depends(get_db),
|
||||
@@ -180,6 +182,17 @@ async def list_dictionary_entries(
|
||||
logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}")
|
||||
try:
|
||||
entries, total = DictionaryManager.list_entries(db, dictionary_id, page=page, page_size=page_size)
|
||||
# Apply language pair filtering
|
||||
if source_language or target_language:
|
||||
filtered = []
|
||||
for e in entries:
|
||||
if source_language and e.source_language != source_language and e.source_language != "und":
|
||||
continue
|
||||
if target_language and e.target_language != target_language:
|
||||
continue
|
||||
filtered.append(e)
|
||||
entries = filtered
|
||||
total = len(filtered)
|
||||
items = [
|
||||
{
|
||||
"id": e.id,
|
||||
@@ -187,13 +200,20 @@ async def list_dictionary_entries(
|
||||
"source_term": e.source_term,
|
||||
"source_term_normalized": e.source_term_normalized,
|
||||
"target_term": e.target_term,
|
||||
"source_language": e.source_language,
|
||||
"target_language": e.target_language,
|
||||
"context_notes": e.context_notes,
|
||||
"context_data": e.context_data,
|
||||
"usage_notes": e.usage_notes,
|
||||
"has_context": e.has_context,
|
||||
"context_source": e.context_source,
|
||||
"origin_source_language": e.origin_source_language,
|
||||
"created_at": e.created_at,
|
||||
"updated_at": e.updated_at,
|
||||
}
|
||||
for e in entries
|
||||
]
|
||||
logger.reflect("Entries listed", {"dict_id": dictionary_id, "total": total})
|
||||
logger.reflect("Entries listed", {"dict_id": dictionary_id, "total": total, "source_language": source_language, "target_language": target_language})
|
||||
return {"items": items, "total": total, "page": page, "page_size": page_size}
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
|
||||
@@ -221,6 +241,8 @@ async def add_dictionary_entry(
|
||||
source_term=payload.source_term,
|
||||
target_term=payload.target_term,
|
||||
context_notes=payload.context_notes,
|
||||
source_language=payload.source_language,
|
||||
target_language=payload.target_language,
|
||||
)
|
||||
logger.reflect("Entry added", {"entry_id": entry.id})
|
||||
return {
|
||||
@@ -229,7 +251,14 @@ async def add_dictionary_entry(
|
||||
"source_term": entry.source_term,
|
||||
"source_term_normalized": entry.source_term_normalized,
|
||||
"target_term": entry.target_term,
|
||||
"source_language": entry.source_language,
|
||||
"target_language": entry.target_language,
|
||||
"context_notes": entry.context_notes,
|
||||
"context_data": entry.context_data,
|
||||
"usage_notes": entry.usage_notes,
|
||||
"has_context": entry.has_context,
|
||||
"context_source": entry.context_source,
|
||||
"origin_source_language": entry.origin_source_language,
|
||||
"created_at": entry.created_at,
|
||||
"updated_at": entry.updated_at,
|
||||
}
|
||||
@@ -262,6 +291,8 @@ async def edit_dictionary_entry(
|
||||
source_term=payload.source_term,
|
||||
target_term=payload.target_term,
|
||||
context_notes=payload.context_notes,
|
||||
source_language=payload.source_language,
|
||||
target_language=payload.target_language,
|
||||
)
|
||||
logger.reflect("Entry updated", {"entry_id": entry_id})
|
||||
return {
|
||||
@@ -270,7 +301,14 @@ async def edit_dictionary_entry(
|
||||
"source_term": entry.source_term,
|
||||
"source_term_normalized": entry.source_term_normalized,
|
||||
"target_term": entry.target_term,
|
||||
"source_language": entry.source_language,
|
||||
"target_language": entry.target_language,
|
||||
"context_notes": entry.context_notes,
|
||||
"context_data": entry.context_data,
|
||||
"usage_notes": entry.usage_notes,
|
||||
"has_context": entry.has_context,
|
||||
"context_source": entry.context_source,
|
||||
"origin_source_language": entry.origin_source_language,
|
||||
"created_at": entry.created_at,
|
||||
"updated_at": entry.updated_at,
|
||||
}
|
||||
@@ -335,6 +373,8 @@ async def import_dictionary_entries(
|
||||
delimiter=payload.delimiter,
|
||||
on_conflict=payload.on_conflict,
|
||||
preview_only=payload.preview_only,
|
||||
default_source_language=payload.default_source_language,
|
||||
default_target_language=payload.default_target_language,
|
||||
)
|
||||
logger.reflect("Import completed", {
|
||||
"dict_id": dictionary_id,
|
||||
|
||||
@@ -59,7 +59,7 @@ async def preview_translation(
|
||||
|
||||
|
||||
# #region update_preview_row [TYPE Function]
|
||||
# @BRIEF Approve, edit, or reject a preview row.
|
||||
# @BRIEF Approve, edit, or reject a preview row (optionally per language).
|
||||
# @PRE: User has translate.job.execute permission.
|
||||
# @POST: Preview row status is updated.
|
||||
@router.put("/jobs/{job_id}/preview/rows/{row_key}")
|
||||
@@ -72,8 +72,8 @@ async def update_preview_row(
|
||||
db: Session = Depends(get_db),
|
||||
config_manager: ConfigManager = Depends(get_config_manager),
|
||||
):
|
||||
"""Approve, edit, or reject a preview row."""
|
||||
logger.reason(f"update_preview_row — Job: {job_id}, Row: {row_key}, Action: {payload.action}", extra={"src": "translate_routes"})
|
||||
"""Approve, edit, or reject a preview row (optionally per language)."""
|
||||
logger.reason(f"update_preview_row — Job: {job_id}, Row: {row_key}, Action: {payload.action}, Lang: {payload.language_code}", extra={"src": "translate_routes"})
|
||||
try:
|
||||
preview_service = TranslationPreview(db, config_manager, current_user.username)
|
||||
result = preview_service.update_preview_row(
|
||||
@@ -82,6 +82,7 @@ async def update_preview_row(
|
||||
action=payload.action,
|
||||
translation=payload.translation,
|
||||
feedback=payload.feedback,
|
||||
language_code=payload.language_code,
|
||||
)
|
||||
return result
|
||||
except ValueError as e:
|
||||
|
||||
@@ -76,6 +76,22 @@ async def list_runs(
|
||||
|
||||
items = []
|
||||
for r in runs:
|
||||
# Gather per-language statistics from the run's language_stats relationship
|
||||
lang_stats = r.language_stats or []
|
||||
target_languages = [ls.language_code for ls in lang_stats]
|
||||
total_translated = sum((ls.translated_rows or 0) for ls in lang_stats)
|
||||
language_stats = {
|
||||
ls.language_code: {
|
||||
"total_rows": ls.total_rows or 0,
|
||||
"translated_rows": ls.translated_rows or 0,
|
||||
"failed_rows": ls.failed_rows or 0,
|
||||
"skipped_rows": ls.skipped_rows or 0,
|
||||
"token_count": ls.token_count or 0,
|
||||
"estimated_cost": ls.estimated_cost or 0.0,
|
||||
}
|
||||
for ls in lang_stats
|
||||
}
|
||||
|
||||
items.append({
|
||||
"id": r.id,
|
||||
"job_id": r.job_id,
|
||||
@@ -94,6 +110,9 @@ async def list_runs(
|
||||
"dict_snapshot_hash": r.dict_snapshot_hash,
|
||||
"created_by": r.created_by,
|
||||
"created_at": r.created_at.isoformat() if r.created_at else None,
|
||||
"target_languages": target_languages,
|
||||
"total_translated": total_translated,
|
||||
"language_stats": language_stats,
|
||||
})
|
||||
|
||||
return {"items": items, "total": total, "page": page, "page_size": page_size}
|
||||
|
||||
@@ -13,6 +13,11 @@ from ....core.logger import logger
|
||||
from ....dependencies import get_config_manager, get_current_user, has_permission
|
||||
from ....plugins.translate.orchestrator import TranslationOrchestrator
|
||||
from ....schemas.auth import User
|
||||
from ....schemas.translate import (
|
||||
BulkFindReplaceRequest,
|
||||
InlineEditRequest,
|
||||
OverrideLanguageRequest,
|
||||
)
|
||||
from ._helpers import _run_to_response
|
||||
from ._router import router
|
||||
|
||||
@@ -341,4 +346,196 @@ async def get_batches(
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
# #endregion get_batches
|
||||
|
||||
# ============================================================
|
||||
# Manual Language Override
|
||||
# ============================================================
|
||||
|
||||
# #region override_detected_language [TYPE Function]
|
||||
# @BRIEF Manually override the detected source language for a specific translation language entry.
|
||||
# @PRE: User has translate.job.execute permission. Run, record, and language entry exist.
|
||||
# @POST: TranslationLanguage.source_language_detected is updated; language_overridden is set to True.
|
||||
# @SIDE_EFFECT: DB write.
|
||||
@router.put("/runs/{run_id}/records/{record_id}/languages/{language_code}/override-language")
|
||||
async def override_detected_language(
|
||||
run_id: str,
|
||||
record_id: str,
|
||||
language_code: str,
|
||||
payload: OverrideLanguageRequest,
|
||||
current_user: User = Depends(get_current_user),
|
||||
_ = Depends(has_permission("translate.job", "EXECUTE")),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Manually override the detected source language for a translation entry."""
|
||||
logger.reason(f"override_detected_language — Run: {run_id}, Record: {record_id}, Lang: {language_code}, Override: {payload.source_language}", extra={"src": "translate_routes"})
|
||||
try:
|
||||
from ....models.translate import TranslationLanguage, TranslationRecord
|
||||
|
||||
# Verify the record exists and belongs to the run
|
||||
record = (
|
||||
db.query(TranslationRecord)
|
||||
.filter(
|
||||
TranslationRecord.id == record_id,
|
||||
TranslationRecord.run_id == run_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not record:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Translation record '{record_id}' not found in run '{run_id}'",
|
||||
)
|
||||
|
||||
# Find the language entry
|
||||
lang_entry = (
|
||||
db.query(TranslationLanguage)
|
||||
.filter(
|
||||
TranslationLanguage.record_id == record_id,
|
||||
TranslationLanguage.language_code == language_code,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not lang_entry:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Language entry '{language_code}' not found for record '{record_id}'",
|
||||
)
|
||||
|
||||
# Update the language entry
|
||||
lang_entry.source_language_detected = payload.source_language
|
||||
lang_entry.language_overridden = True
|
||||
lang_entry.needs_review = False # Override clears the review flag
|
||||
db.commit()
|
||||
db.refresh(lang_entry)
|
||||
|
||||
logger.reason("Language override applied", {
|
||||
"run_id": run_id,
|
||||
"record_id": record_id,
|
||||
"language_code": language_code,
|
||||
"new_source_language": payload.source_language,
|
||||
"overridden_by": current_user.username,
|
||||
})
|
||||
|
||||
from ....schemas.translate import TranslationLanguageResponse
|
||||
return TranslationLanguageResponse.model_validate(lang_entry)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.explore("override_detected_language failed", extra={"src": "translate_routes", "error": str(e)})
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
# #endregion override_detected_language
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Inline Correction
|
||||
# ============================================================
|
||||
|
||||
# #region inline_edit_translation [C:3] [TYPE Function] [SEMANTICS api,translate,correction]
|
||||
# @BRIEF Apply an inline correction to a translated value on a completed run result.
|
||||
# @PRE: User has translate.job.execute permission. Run, record, and language entry exist.
|
||||
# @POST: TranslationLanguage.final_value and user_edit are updated. Optional dictionary submission.
|
||||
@router.put("/runs/{run_id}/records/{record_id}/languages/{language_code}")
|
||||
async def inline_edit_translation(
|
||||
run_id: str,
|
||||
record_id: str,
|
||||
language_code: str,
|
||||
payload: InlineEditRequest,
|
||||
current_user: User = Depends(get_current_user),
|
||||
_ = Depends(has_permission("translate.job", "EXECUTE")),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Apply an inline correction to a translated value."""
|
||||
logger.reason(
|
||||
f"inline_edit_translation — Run: {run_id}, Record: {record_id}, "
|
||||
f"Lang: {language_code}, User: {current_user.username}",
|
||||
extra={"src": "translate_routes"},
|
||||
)
|
||||
try:
|
||||
from ....plugins.translate.service import InlineCorrectionService
|
||||
|
||||
result = InlineCorrectionService.apply_inline_edit(
|
||||
db=db,
|
||||
run_id=run_id,
|
||||
record_id=record_id,
|
||||
language_code=language_code,
|
||||
final_value=payload.final_value,
|
||||
submit_to_dictionary=payload.submit_to_dictionary,
|
||||
dictionary_id=payload.dictionary_id,
|
||||
current_user=current_user.username,
|
||||
context_data_override=payload.context_data_override,
|
||||
usage_notes=payload.usage_notes,
|
||||
keep_context=payload.keep_context,
|
||||
)
|
||||
return result
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.explore("inline_edit_translation failed", extra={"src": "translate_routes", "error": str(e)})
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
# #endregion inline_edit_translation
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Bulk Find-and-Replace
|
||||
# ============================================================
|
||||
|
||||
# #region bulk_find_replace [C:3] [TYPE Function] [SEMANTICS api,translate,bulk,replace]
|
||||
# @BRIEF Perform bulk find-and-replace on translated values within a run.
|
||||
# @PRE: User has translate.job.execute permission. Run exists.
|
||||
# @POST: If preview=false, matching translations are updated. Optional dictionary submission.
|
||||
@router.post("/runs/{run_id}/bulk-replace")
|
||||
async def bulk_find_replace(
|
||||
run_id: str,
|
||||
payload: BulkFindReplaceRequest,
|
||||
current_user: User = Depends(get_current_user),
|
||||
_ = Depends(has_permission("translate.job", "EXECUTE")),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Perform bulk find-and-replace on translated values within a run."""
|
||||
logger.reason(
|
||||
f"bulk_find_replace — Run: {run_id}, Pattern: '{payload.find_pattern}', "
|
||||
f"Regex: {payload.is_regex}, Lang: {payload.target_language}, "
|
||||
f"User: {current_user.username}",
|
||||
extra={"src": "translate_routes"},
|
||||
)
|
||||
try:
|
||||
from ....plugins.translate.service import BulkFindReplaceService
|
||||
|
||||
if payload.preview:
|
||||
# Preview mode: return matches without applying
|
||||
preview = BulkFindReplaceService.preview(
|
||||
db=db,
|
||||
run_id=run_id,
|
||||
pattern=payload.find_pattern,
|
||||
is_regex=payload.is_regex,
|
||||
target_language=payload.target_language,
|
||||
)
|
||||
return {
|
||||
"rows_affected": len(preview),
|
||||
"corrections_submitted": 0,
|
||||
"preview": preview,
|
||||
}
|
||||
|
||||
# Apply mode: perform replacements
|
||||
result = BulkFindReplaceService.apply(
|
||||
db=db,
|
||||
run_id=run_id,
|
||||
pattern=payload.find_pattern,
|
||||
is_regex=payload.is_regex,
|
||||
replacement_text=payload.replacement_text,
|
||||
target_language=payload.target_language,
|
||||
submit_to_dictionary=payload.submit_to_dictionary,
|
||||
dictionary_id=payload.dictionary_id,
|
||||
usage_notes=payload.usage_notes,
|
||||
current_user=current_user.username,
|
||||
submit_to_dictionary_with_context=payload.submit_to_dictionary_with_context,
|
||||
)
|
||||
return result
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.explore("bulk_find_replace failed", extra={"src": "translate_routes", "error": str(e)})
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
# #endregion bulk_find_replace
|
||||
|
||||
|
||||
# #endregion TranslateRunRoutesModule
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import JSON, Boolean, Column, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import JSON, Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from .mapping import Base
|
||||
|
||||
@@ -44,7 +45,11 @@ class TranslationJob(Base):
|
||||
context_columns = Column(JSON, nullable=True, comment="Context column names included in LLM prompt")
|
||||
|
||||
# LLM & processing settings
|
||||
target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru)")
|
||||
# @DEPRECATED — use target_languages
|
||||
target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru) [DEPRECATED: use target_languages]")
|
||||
# @DEPRECATED — Auto-detected per row by LLM, kept as fallback hint
|
||||
source_language = Column(String, nullable=True, comment="Fallback source language hint [DEPRECATED: auto-detected per row]")
|
||||
target_languages = Column(JSON, nullable=True, comment="List of BCP-47 target language codes (multi-language support)")
|
||||
provider_id = Column(String, nullable=True, comment="LLM provider ID")
|
||||
batch_size = Column(Integer, nullable=False, default=50, comment="Records per batch")
|
||||
upsert_strategy = Column(String, nullable=False, default="MERGE", comment="MERGE, INSERT, UPDATE")
|
||||
@@ -84,6 +89,8 @@ class TranslationRun(Base):
|
||||
dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at run time")
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
||||
|
||||
language_stats = relationship("TranslationRunLanguageStats", back_populates="run")
|
||||
# #endregion TranslationRun
|
||||
|
||||
|
||||
@@ -126,6 +133,15 @@ class TranslationRecord(Base):
|
||||
translation_duration_ms = Column(Integer, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
||||
|
||||
# @DEPRECATED — use TranslationLanguage instead
|
||||
llm_translation = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]")
|
||||
# @DEPRECATED — use TranslationLanguage instead
|
||||
user_edit = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]")
|
||||
# @DEPRECATED — use TranslationLanguage instead
|
||||
final_value = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]")
|
||||
|
||||
languages = relationship("TranslationLanguage", back_populates="record")
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_translation_records_run_status", "run_id", "status"),
|
||||
)
|
||||
@@ -178,6 +194,8 @@ class TranslationPreviewRecord(Base):
|
||||
status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED
|
||||
feedback = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
||||
|
||||
languages = relationship("TranslationPreviewLanguage", back_populates="preview_record")
|
||||
# #endregion TranslationPreviewRecord
|
||||
|
||||
|
||||
@@ -191,6 +209,10 @@ class TerminologyDictionary(Base):
|
||||
description = Column(Text, nullable=True)
|
||||
source_dialect = Column(String, nullable=False)
|
||||
target_dialect = Column(String, nullable=False)
|
||||
# @DEPRECATED — use per-entry source_language instead, kept for backward compat
|
||||
source_language = Column(String, nullable=True, comment="[DEPRECATED: use per-entry source_language]")
|
||||
# @DEPRECATED — use per-entry target_language instead, kept for backward compat
|
||||
target_language = Column(String, nullable=True, comment="[DEPRECATED: use per-entry target_language]")
|
||||
is_active = Column(Boolean, default=True)
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
||||
@@ -208,7 +230,14 @@ class DictionaryEntry(Base):
|
||||
source_term = Column(String, nullable=False)
|
||||
source_term_normalized = Column(String, nullable=False)
|
||||
target_term = Column(String, nullable=False)
|
||||
source_language = Column(String, nullable=False, comment="BCP-47 source language code")
|
||||
target_language = Column(String, nullable=False, comment="BCP-47 target language code")
|
||||
context_notes = Column(Text, nullable=True)
|
||||
context_data = Column(JSON, nullable=True, comment="Structured context for term usage")
|
||||
usage_notes = Column(Text, nullable=True, comment="Usage guidance for the term mapping")
|
||||
has_context = Column(Boolean, default=False, comment="Whether context_data is populated")
|
||||
context_source = Column(String, nullable=True, comment="auto|auto_with_edits|manual|bulk")
|
||||
origin_source_language = Column(String, nullable=True, comment="Original source language of the term")
|
||||
origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated")
|
||||
origin_row_key = Column(String, nullable=True, comment="Row key within the run that triggered this correction")
|
||||
origin_user_id = Column(String, nullable=True, comment="User who submitted the correction")
|
||||
@@ -217,9 +246,11 @@ class DictionaryEntry(Base):
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"dictionary_id", "source_term_normalized",
|
||||
name="uq_dictionary_entry_term"
|
||||
"dictionary_id", "source_term_normalized", "source_language", "target_language",
|
||||
name="uq_dict_source_term_lang"
|
||||
),
|
||||
Index("idx_dict_entry_lang", "source_language", "target_language"),
|
||||
Index("idx_dict_has_context", "has_context"),
|
||||
)
|
||||
# #endregion DictionaryEntry
|
||||
|
||||
@@ -284,6 +315,7 @@ class MetricSnapshot(Base):
|
||||
p50_duration_ms = Column(Integer, nullable=True)
|
||||
p95_duration_ms = Column(Integer, nullable=True)
|
||||
p99_duration_ms = Column(Integer, nullable=True)
|
||||
per_language_metrics = Column(JSON, nullable=True, comment="Per-language cumulative metrics: {lang: {cumulative_tokens, cumulative_cost, runs}}")
|
||||
snapshot_date = Column(DateTime, nullable=False, default=lambda: datetime.now(UTC))
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
||||
|
||||
@@ -292,4 +324,81 @@ class MetricSnapshot(Base):
|
||||
)
|
||||
# #endregion MetricSnapshot
|
||||
|
||||
|
||||
# #region TranslationLanguage [C:1] [TYPE Class]
|
||||
# @BRIEF Per-language translation result for a single record, supporting multi-language output.
|
||||
class TranslationLanguage(Base):
|
||||
__tablename__ = "translation_languages"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
record_id = Column(String, ForeignKey("translation_records.id"), nullable=False)
|
||||
language_code = Column(String, nullable=False, comment="BCP-47 language code")
|
||||
source_language_detected = Column(String, nullable=True, comment="BCP-47 or 'und' for undetermined")
|
||||
translated_value = Column(Text, nullable=True, comment="LLM-generated translation")
|
||||
user_edit = Column(Text, nullable=True, comment="User-edited translation")
|
||||
final_value = Column(Text, nullable=True, comment="Final resolved value (translated or user edit)")
|
||||
status = Column(String, default="pending", comment="pending|translated|approved|edited|rejected|failed|skipped")
|
||||
error_message = Column(Text, nullable=True)
|
||||
needs_review = Column(Boolean, default=False, comment="Flagged because source language could not be determined")
|
||||
language_overridden = Column(Boolean, default=False, comment="Source language was manually overridden by user")
|
||||
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
|
||||
record = relationship("TranslationRecord", back_populates="languages")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("record_id", "language_code", name="uq_record_language"),
|
||||
Index("idx_tl_language", "language_code"),
|
||||
Index("idx_tl_record_lang", "record_id", "language_code"),
|
||||
)
|
||||
# #endregion TranslationLanguage
|
||||
|
||||
|
||||
# #region TranslationPreviewLanguage [C:1] [TYPE Class]
|
||||
# @BRIEF Per-language preview entry within a preview session.
|
||||
class TranslationPreviewLanguage(Base):
|
||||
__tablename__ = "translation_preview_languages"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
preview_record_id = Column(String, ForeignKey("translation_preview_records.id"), nullable=False)
|
||||
language_code = Column(String, nullable=False, comment="BCP-47 language code")
|
||||
source_language_detected = Column(String, nullable=True, comment="BCP-47 or 'und'")
|
||||
translated_value = Column(Text, nullable=True)
|
||||
user_edit = Column(Text, nullable=True)
|
||||
final_value = Column(Text, nullable=True)
|
||||
status = Column(String, default="pending", comment="pending|approved|edited|rejected")
|
||||
needs_review = Column(Boolean, default=False, comment="Flagged because source language could not be determined")
|
||||
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(UTC))
|
||||
|
||||
preview_record = relationship("TranslationPreviewRecord", back_populates="languages")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("preview_record_id", "language_code", name="uq_preview_record_language"),
|
||||
)
|
||||
# #endregion TranslationPreviewLanguage
|
||||
|
||||
|
||||
# #region TranslationRunLanguageStats [C:1] [TYPE Class]
|
||||
# @BRIEF Per-language statistics for a translation run (row counts, tokens, cost).
|
||||
class TranslationRunLanguageStats(Base):
|
||||
__tablename__ = "translation_run_language_stats"
|
||||
|
||||
id = Column(String, primary_key=True, default=generate_uuid)
|
||||
run_id = Column(String, ForeignKey("translation_runs.id"), nullable=False)
|
||||
language_code = Column(String, nullable=False, comment="BCP-47 language code")
|
||||
total_rows = Column(Integer, default=0)
|
||||
translated_rows = Column(Integer, default=0)
|
||||
failed_rows = Column(Integer, default=0)
|
||||
skipped_rows = Column(Integer, default=0)
|
||||
token_count = Column(Integer, default=0)
|
||||
estimated_cost = Column(Float, default=0.0)
|
||||
|
||||
run = relationship("TranslationRun", back_populates="language_stats")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("run_id", "language_code", name="uq_run_language"),
|
||||
Index("idx_rls_run", "run_id"),
|
||||
)
|
||||
# #endregion TranslationRunLanguageStats
|
||||
|
||||
|
||||
# #endregion TranslateModels
|
||||
|
||||
@@ -1,29 +1,37 @@
|
||||
# region DictionaryTests [TYPE Module]
|
||||
# @SEMANTICS: tests, dictionary, crud, import, filter
|
||||
# @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, and batch filtering.
|
||||
# @SEMANTICS: tests, dictionary, crud, import, filter, language_pair
|
||||
# @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, batch filtering, and language-pair support.
|
||||
# @RELATION: BINDS_TO -> [DictionaryManager:Class]
|
||||
#
|
||||
# @TEST_CONTRACT: [DictionaryManager] -> {
|
||||
# invariants: [
|
||||
# "Create/Read/Update/Delete dictionaries works",
|
||||
# "Entry CRUD enforces unique source_term_normalized per dictionary",
|
||||
# "Entry CRUD enforces unique (dictionary_id, source_term_normalized, source_language, target_language)",
|
||||
# "Import CSV/TSV handles overwrite/keep_existing/cancel conflict modes",
|
||||
# "Delete blocked when dictionary attached to active/scheduled jobs",
|
||||
# "filter_for_batch returns matched entries with word-boundary awareness"
|
||||
# "filter_for_batch returns matched entries with word-boundary awareness",
|
||||
# "filter_for_batch supports language-pair filtering",
|
||||
# "Language-pair-aware duplicate detection: same term with different lang pair is allowed"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_EDGE: duplicate_entry -> 409-style ValueError on repeated source_term
|
||||
# @TEST_EDGE: duplicate_entry -> 409-style ValueError on repeated (dictionary_id, source_term_norm, source_lang, target_lang)
|
||||
# @TEST_EDGE: delete_active_job -> ValueError with active/scheduled message
|
||||
# @TEST_EDGE: import_invalid_format -> ValueError for missing columns
|
||||
# @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry]
|
||||
# @TEST_EDGE: same_term_different_lang_pair -> allowed (not duplicate)
|
||||
# @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry, same_term_different_lang_pair allowed]
|
||||
|
||||
|
||||
import csv
|
||||
import io
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, event
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from src.models.translate import (
|
||||
Base,
|
||||
DictionaryEntry,
|
||||
TerminologyDictionary,
|
||||
TranslationJob,
|
||||
TranslationJobDictionary,
|
||||
)
|
||||
@@ -112,8 +120,8 @@ def test_delete_dictionary(db_session: Session):
|
||||
db_session, name="To Delete",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
# Add an entry
|
||||
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
# Add an entry with language pair
|
||||
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
|
||||
assert entry.id is not None
|
||||
|
||||
DictionaryManager.delete_dictionary(db_session, d.id)
|
||||
@@ -144,15 +152,15 @@ def test_add_entry_duplicate(db_session: Session):
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "Hello", "Hola")
|
||||
DictionaryManager.add_entry(db_session, d.id, "Hello", "Hola", source_language="en", target_language="es")
|
||||
|
||||
# Same normalized term should raise
|
||||
# Same normalized term with same language pair should raise
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "Bonjour")
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "Bonjour", source_language="en", target_language="es")
|
||||
|
||||
# Different case, same normalized should also raise
|
||||
# Different case, same normalized, same lang pair should also raise
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
DictionaryManager.add_entry(db_session, d.id, "HELLO", "Ciao")
|
||||
DictionaryManager.add_entry(db_session, d.id, "HELLO", "Ciao", source_language="en", target_language="es")
|
||||
# endregion test_add_entry_duplicate
|
||||
|
||||
|
||||
@@ -167,9 +175,9 @@ def test_add_entry_duplicate_per_dictionary(db_session: Session):
|
||||
db_session, name="Dict2",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola")
|
||||
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola", source_language="en", target_language="es")
|
||||
# Same term in different dictionary should work
|
||||
entry = DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour")
|
||||
entry = DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour", source_language="en", target_language="es")
|
||||
assert entry.id is not None
|
||||
# endregion test_add_entry_duplicate_per_dictionary
|
||||
|
||||
@@ -181,13 +189,13 @@ def test_edit_entry(db_session: Session):
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
|
||||
|
||||
updated = DictionaryManager.edit_entry(db_session, entry.id, target_term="HOLA!")
|
||||
assert updated.target_term == "HOLA!"
|
||||
|
||||
# Edit source term to a value that's already taken should fail
|
||||
DictionaryManager.add_entry(db_session, d.id, "world", "mundo")
|
||||
DictionaryManager.add_entry(db_session, d.id, "world", "mundo", source_language="en", target_language="es")
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
DictionaryManager.edit_entry(db_session, entry.id, source_term="WORLD")
|
||||
# endregion test_edit_entry
|
||||
@@ -200,7 +208,7 @@ def test_delete_entry(db_session: Session):
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
|
||||
DictionaryManager.delete_entry(db_session, entry.id)
|
||||
|
||||
entries, total = DictionaryManager.list_entries(db_session, d.id)
|
||||
@@ -215,10 +223,10 @@ def test_import_csv_overwrite(db_session: Session):
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
# Pre-add an entry
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
# Pre-add an entry with language pair
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
|
||||
|
||||
csv_content = "source_term,target_term,context_notes\nhello,HELLO,\nworld,mundo,"
|
||||
csv_content = "source_term,target_term,context_notes,source_language,target_language\nhello,HELLO,,en,es\nworld,mundo,,en,es"
|
||||
result = DictionaryManager.import_entries(
|
||||
db_session, d.id, csv_content,
|
||||
delimiter=",", on_conflict="overwrite",
|
||||
@@ -242,9 +250,9 @@ def test_import_csv_keep_existing(db_session: Session):
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
|
||||
|
||||
csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo"
|
||||
csv_content = "source_term,target_term,source_language,target_language\nhello,HELLO,en,es\nworld,mundo,en,es"
|
||||
result = DictionaryManager.import_entries(
|
||||
db_session, d.id, csv_content,
|
||||
delimiter=",", on_conflict="keep_existing",
|
||||
@@ -267,9 +275,9 @@ def test_import_csv_cancel_on_conflict(db_session: Session):
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
|
||||
|
||||
csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo"
|
||||
csv_content = "source_term,target_term,source_language,target_language\nhello,HELLO,en,es\nworld,mundo,en,es"
|
||||
result = DictionaryManager.import_entries(
|
||||
db_session, d.id, csv_content,
|
||||
delimiter=",", on_conflict="cancel",
|
||||
@@ -289,7 +297,7 @@ def test_import_tsv(db_session: Session):
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
tsv_content = "source_term\ttarget_term\nhello\thola\nworld\tmundo"
|
||||
tsv_content = "source_term\ttarget_term\tsource_language\ttarget_language\nhello\thola\ten\tes\nworld\tmundo\ten\tes"
|
||||
result = DictionaryManager.import_entries(
|
||||
db_session, d.id, tsv_content,
|
||||
delimiter="\t", on_conflict="overwrite",
|
||||
@@ -339,9 +347,9 @@ def test_import_preview(db_session: Session):
|
||||
db_session, name="Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
|
||||
|
||||
csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo"
|
||||
csv_content = "source_term,target_term,source_language,target_language\nhello,HELLO,en,es\nworld,mundo,en,es"
|
||||
result = DictionaryManager.import_entries(
|
||||
db_session, d.id, csv_content,
|
||||
delimiter=",", on_conflict="overwrite",
|
||||
@@ -446,9 +454,9 @@ def test_filter_for_batch_matches(db_session: Session):
|
||||
db_session, name="Test Dict",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
DictionaryManager.add_entry(db_session, d.id, "world", "mundo")
|
||||
DictionaryManager.add_entry(db_session, d.id, "foo", "bar")
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
|
||||
DictionaryManager.add_entry(db_session, d.id, "world", "mundo", source_language="en", target_language="es")
|
||||
DictionaryManager.add_entry(db_session, d.id, "foo", "bar", source_language="en", target_language="es")
|
||||
|
||||
job = TranslationJob(
|
||||
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
|
||||
@@ -492,7 +500,7 @@ def test_filter_for_batch_case_insensitive(db_session: Session):
|
||||
db_session, name="Test Dict",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "Hello World", "Hola Mundo")
|
||||
DictionaryManager.add_entry(db_session, d.id, "Hello World", "Hola Mundo", source_language="en", target_language="es")
|
||||
|
||||
job = TranslationJob(
|
||||
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
|
||||
@@ -523,7 +531,7 @@ def test_filter_for_batch_word_boundary(db_session: Session):
|
||||
db_session, name="Test Dict",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "cat", "gato")
|
||||
DictionaryManager.add_entry(db_session, d.id, "cat", "gato", source_language="en", target_language="es")
|
||||
|
||||
job = TranslationJob(
|
||||
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
|
||||
@@ -562,8 +570,8 @@ def test_filter_for_batch_multi_dictionary_priority(db_session: Session):
|
||||
d2 = DictionaryManager.create_dictionary(
|
||||
db_session, name="Priority2", source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola")
|
||||
DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour")
|
||||
DictionaryManager.add_entry(db_session, d1.id, "hello", "hola", source_language="en", target_language="es")
|
||||
DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour", source_language="en", target_language="es")
|
||||
|
||||
job = TranslationJob(
|
||||
name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT",
|
||||
@@ -625,8 +633,8 @@ def test_clear_entries(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test", source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola")
|
||||
DictionaryManager.add_entry(db_session, d.id, "world", "mundo")
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es")
|
||||
DictionaryManager.add_entry(db_session, d.id, "world", "mundo", source_language="en", target_language="es")
|
||||
|
||||
deleted = DictionaryManager.clear_entries(db_session, d.id)
|
||||
assert deleted == 2
|
||||
@@ -634,4 +642,558 @@ def test_clear_entries(db_session: Session):
|
||||
entries, total = DictionaryManager.list_entries(db_session, d.id)
|
||||
assert total == 0
|
||||
# endregion test_clear_entries
|
||||
|
||||
# region test_add_entry_with_language_pair [TYPE Function]
|
||||
# @PURPOSE: Verify creating entry with source_language and target_language stores correctly.
|
||||
def test_add_entry_with_language_pair(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Lang Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
entry = DictionaryManager.add_entry(
|
||||
db_session, d.id, "hello", "привет",
|
||||
source_language="en", target_language="ru",
|
||||
)
|
||||
assert entry.source_language == "en"
|
||||
assert entry.target_language == "ru"
|
||||
assert entry.source_term == "hello"
|
||||
|
||||
# Read back
|
||||
entries, total = DictionaryManager.list_entries(db_session, d.id)
|
||||
assert total == 1
|
||||
assert entries[0].source_language == "en"
|
||||
assert entries[0].target_language == "ru"
|
||||
# endregion test_add_entry_with_language_pair
|
||||
|
||||
|
||||
# region test_duplicate_same_language_pair [TYPE Function]
|
||||
# @PURPOSE: Verify duplicate entry with same (dictionary_id, source_term_norm, source_lang, target_lang) raises conflict.
|
||||
def test_duplicate_same_language_pair(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Dup Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(
|
||||
db_session, d.id, "hello", "привет",
|
||||
source_language="en", target_language="ru",
|
||||
)
|
||||
# Same term with same language pair should raise
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
DictionaryManager.add_entry(
|
||||
db_session, d.id, "hello", "hallo",
|
||||
source_language="en", target_language="ru",
|
||||
)
|
||||
# endregion test_duplicate_same_language_pair
|
||||
|
||||
|
||||
# region test_same_term_different_language_pair [TYPE Function]
|
||||
# @PURPOSE: Verify same source_term with different language pair is allowed (not duplicate).
|
||||
def test_same_term_different_language_pair(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Multi Lang",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
entry1 = DictionaryManager.add_entry(
|
||||
db_session, d.id, "hello", "привет",
|
||||
source_language="en", target_language="ru",
|
||||
)
|
||||
entry2 = DictionaryManager.add_entry(
|
||||
db_session, d.id, "hello", "hallo",
|
||||
source_language="en", target_language="de",
|
||||
)
|
||||
assert entry1.id != entry2.id
|
||||
assert entry1.target_language == "ru"
|
||||
assert entry2.target_language == "de"
|
||||
|
||||
entries, total = DictionaryManager.list_entries(db_session, d.id)
|
||||
assert total == 2
|
||||
# endregion test_same_term_different_language_pair
|
||||
|
||||
|
||||
# region test_filter_for_batch_with_language_pair [TYPE Function]
|
||||
# @PURPOSE: Verify filter_for_batch with source_language and target_language filters entries correctly.
|
||||
def test_filter_for_batch_with_language_pair(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Lang Filter",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")
|
||||
DictionaryManager.add_entry(db_session, d.id, "world", "мир", source_language="en", target_language="ru")
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hallo", source_language="en", target_language="de")
|
||||
|
||||
job = TranslationJob(
|
||||
name="Lang Job", source_dialect="a", target_dialect="b", status="DRAFT",
|
||||
)
|
||||
db_session.add(job)
|
||||
db_session.flush()
|
||||
|
||||
link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id)
|
||||
db_session.add(link)
|
||||
db_session.commit()
|
||||
|
||||
# Filter for en → ru: should get 2 entries (hello/privet, world/mir)
|
||||
result = DictionaryManager.filter_for_batch(
|
||||
db_session, ["hello world"], job.id,
|
||||
source_language="en", target_language="ru",
|
||||
)
|
||||
assert len(result) == 2
|
||||
assert all(m["target_language"] == "ru" for m in result)
|
||||
|
||||
# Filter for en → de: should get only the hello/hallo entry
|
||||
result2 = DictionaryManager.filter_for_batch(
|
||||
db_session, ["hello"], job.id,
|
||||
source_language="en", target_language="de",
|
||||
)
|
||||
assert len(result2) == 1
|
||||
assert result2[0]["target_language"] == "de"
|
||||
assert result2[0]["target_term"] == "hallo"
|
||||
# endregion test_filter_for_batch_with_language_pair
|
||||
|
||||
|
||||
# region test_filter_for_batch_target_language_only [TYPE Function]
|
||||
# @PURPOSE: Verify filter_for_batch with only target_language filters by target language only.
|
||||
def test_filter_for_batch_target_language_only(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Target Only",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "hallo", source_language="en", target_language="de")
|
||||
DictionaryManager.add_entry(db_session, d.id, "bonjour", "hallo", source_language="fr", target_language="de")
|
||||
|
||||
job = TranslationJob(
|
||||
name="Tgt Job", source_dialect="a", target_dialect="b", status="DRAFT",
|
||||
)
|
||||
db_session.add(job)
|
||||
db_session.flush()
|
||||
|
||||
link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id)
|
||||
db_session.add(link)
|
||||
db_session.commit()
|
||||
|
||||
# Filter for target=de only: should match both hello→hallo and bonjour→hallo (from any source lang)
|
||||
result = DictionaryManager.filter_for_batch(
|
||||
db_session, ["hello bonjour"], job.id,
|
||||
target_language="de",
|
||||
)
|
||||
assert len(result) == 2
|
||||
assert all(m["target_language"] == "de" for m in result)
|
||||
|
||||
# Filter for target=ru only: should match hello→privet
|
||||
result2 = DictionaryManager.filter_for_batch(
|
||||
db_session, ["hello"], job.id,
|
||||
target_language="ru",
|
||||
)
|
||||
assert len(result2) == 1
|
||||
assert result2[0]["target_language"] == "ru"
|
||||
# endregion test_filter_for_batch_target_language_only
|
||||
|
||||
|
||||
# region test_migrate_old_entries [TYPE Function]
|
||||
# @PURPOSE: Verify migration populates language pair fields for old-style entries.
|
||||
def test_migrate_old_entries(db_session: Session):
|
||||
# Create a dictionary with deprecated target_language set
|
||||
d = TerminologyDictionary(
|
||||
name="Old Dict",
|
||||
source_dialect="a",
|
||||
target_dialect="b",
|
||||
target_language="ru",
|
||||
)
|
||||
db_session.add(d)
|
||||
db_session.flush()
|
||||
|
||||
# Create old-style entries without explicit language pair (source_language=und, target_language=und)
|
||||
entry1 = DictionaryEntry(
|
||||
dictionary_id=d.id,
|
||||
source_term="hello",
|
||||
source_term_normalized="hello",
|
||||
target_term="привет",
|
||||
source_language="und",
|
||||
target_language="und",
|
||||
)
|
||||
entry2 = DictionaryEntry(
|
||||
dictionary_id=d.id,
|
||||
source_term="world",
|
||||
source_term_normalized="world",
|
||||
target_term="мир",
|
||||
source_language="und",
|
||||
target_language="und",
|
||||
origin_source_language="en",
|
||||
)
|
||||
db_session.add(entry1)
|
||||
db_session.add(entry2)
|
||||
db_session.commit()
|
||||
|
||||
# Run migration
|
||||
result = DictionaryManager.migrate_old_entries(db_session)
|
||||
|
||||
# Verify migration counts
|
||||
assert result["total_processed"] >= 2
|
||||
assert result["migrated_source"] >= 1 # entry2 had origin_source_language
|
||||
assert result["migrated_target"] >= 2 # both should get target_language from dictionary
|
||||
|
||||
# Verify entry1 got target_language from dictionary
|
||||
db_session.refresh(entry1)
|
||||
assert entry1.target_language == "ru"
|
||||
# source_language should remain "und" (no origin_source_language)
|
||||
assert entry1.source_language == "und"
|
||||
|
||||
# Verify entry2 got source from origin and target from dictionary
|
||||
db_session.refresh(entry2)
|
||||
assert entry2.source_language == "en"
|
||||
assert entry2.target_language == "ru"
|
||||
# endregion test_migrate_old_entries
|
||||
|
||||
|
||||
# region test_export_entries [TYPE Function]
|
||||
# @PURPOSE: Verify export_entries includes language columns.
|
||||
def test_export_entries(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Export Test",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru")
|
||||
DictionaryManager.add_entry(db_session, d.id, "world", "мир", source_language="en", target_language="ru")
|
||||
|
||||
csv_output = DictionaryManager.export_entries(db_session, d.id)
|
||||
assert "source_language" in csv_output
|
||||
assert "target_language" in csv_output
|
||||
assert "context_data" in csv_output
|
||||
assert "usage_notes" in csv_output
|
||||
assert "en" in csv_output
|
||||
assert "ru" in csv_output
|
||||
|
||||
# Verify it's valid CSV
|
||||
reader = csv.DictReader(io.StringIO(csv_output))
|
||||
rows = list(reader)
|
||||
assert len(rows) == 2
|
||||
assert rows[0]["source_language"] == "en"
|
||||
assert rows[0]["target_language"] == "ru"
|
||||
# endregion test_export_entries
|
||||
|
||||
|
||||
# region test_import_with_default_language [TYPE Function]
|
||||
# @PURPOSE: Verify import uses default_source_language and default_target_language when columns missing.
|
||||
def test_import_with_default_language(db_session: Session):
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Default Lang Import",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
# CSV without language columns
|
||||
csv_content = "source_term,target_term\nhello,привет\nworld,мир"
|
||||
result = DictionaryManager.import_entries(
|
||||
db_session, d.id, csv_content,
|
||||
delimiter=",", on_conflict="overwrite",
|
||||
default_source_language="en",
|
||||
default_target_language="ru",
|
||||
)
|
||||
assert result["created"] == 2
|
||||
|
||||
entries, _ = DictionaryManager.list_entries(db_session, d.id)
|
||||
for e in entries:
|
||||
assert e.source_language == "en"
|
||||
assert e.target_language == "ru"
|
||||
# endregion test_import_with_default_language
|
||||
|
||||
|
||||
# region test_context_capture_in_correction [TYPE Function]
|
||||
# @PURPOSE: Verify context_data is auto-captured when submitting a correction with source row context.
|
||||
def test_context_capture_in_correction(db_session: Session):
|
||||
"""Test T132: Context auto-capture when submitting a correction."""
|
||||
from src.plugins.translate.service import InlineCorrectionService
|
||||
from src.models.translate import TranslationRecord, TranslationRun, TranslationBatch, TranslationLanguage
|
||||
|
||||
d = DictionaryManager.create_dictionary(
|
||||
db_session, name="Test Dict",
|
||||
source_dialect="a", target_dialect="b",
|
||||
)
|
||||
job = TranslationJob(
|
||||
name="Ctx Job", source_dialect="a", target_dialect="b",
|
||||
status="DRAFT",
|
||||
)
|
||||
db_session.add(job)
|
||||
db_session.flush()
|
||||
|
||||
run = TranslationRun(
|
||||
job_id=job.id, status="COMPLETED",
|
||||
)
|
||||
db_session.add(run)
|
||||
db_session.flush()
|
||||
|
||||
batch = TranslationBatch(
|
||||
run_id=run.id, batch_index=0, status="COMPLETED",
|
||||
)
|
||||
db_session.add(batch)
|
||||
db_session.flush()
|
||||
|
||||
# Create a TranslationRecord with source_data (context columns)
|
||||
record = TranslationRecord(
|
||||
batch_id=batch.id,
|
||||
run_id=run.id,
|
||||
source_sql="hello world",
|
||||
source_object_type="table_row",
|
||||
source_object_name="Row 0",
|
||||
source_object_id="0",
|
||||
source_data={"schema": "public", "table": "users", "column": "name"},
|
||||
status="SUCCESS",
|
||||
)
|
||||
db_session.add(record)
|
||||
db_session.flush()
|
||||
|
||||
# Create language entries for test languages
|
||||
for lang_code in ("ru", "de", "fr", "it"):
|
||||
tl = TranslationLanguage(
|
||||
record_id=record.id,
|
||||
language_code=lang_code,
|
||||
source_language_detected="en",
|
||||
translated_value="translated",
|
||||
final_value="translated",
|
||||
status="translated",
|
||||
)
|
||||
db_session.add(tl)
|
||||
db_session.commit()
|
||||
|
||||
# Submit correction to dictionary
|
||||
result = InlineCorrectionService.submit_correction_to_dict(
|
||||
db=db_session,
|
||||
record_id=record.id,
|
||||
language_code="ru",
|
||||
dictionary_id=d.id,
|
||||
corrected_value="привет мир (исправлено)",
|
||||
current_user="test_user",
|
||||
)
|
||||
|
||||
assert result["action"] == "created"
|
||||
assert result["entry_id"] is not None
|
||||
|
||||
# Verify context was auto-captured
|
||||
entry = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result["entry_id"]).first()
|
||||
assert entry is not None
|
||||
assert entry.has_context is True
|
||||
assert entry.context_source == "auto"
|
||||
assert entry.context_data is not None
|
||||
# source_data should contain the row's context columns
|
||||
assert "source_data" in entry.context_data
|
||||
assert entry.context_data["source_object_type"] == "table_row"
|
||||
assert entry.context_data["source_object_name"] == "Row 0"
|
||||
|
||||
# Test context_data_override (use different language to avoid conflict)
|
||||
result2 = InlineCorrectionService.submit_correction_to_dict(
|
||||
db=db_session,
|
||||
record_id=record.id,
|
||||
language_code="de", # different target language avoids conflict
|
||||
dictionary_id=d.id,
|
||||
corrected_value="another correction",
|
||||
current_user="test_user",
|
||||
context_data_override={"custom_key": "custom_value"},
|
||||
)
|
||||
assert result2["action"] == "created"
|
||||
entry2 = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result2["entry_id"]).first()
|
||||
assert entry2.context_data == {"custom_key": "custom_value"}
|
||||
assert entry2.context_source == "manual"
|
||||
|
||||
# Test keep_context=False
|
||||
result3 = InlineCorrectionService.submit_correction_to_dict(
|
||||
db=db_session,
|
||||
record_id=record.id,
|
||||
language_code="fr", # different target language avoids conflict
|
||||
dictionary_id=d.id,
|
||||
corrected_value="no context",
|
||||
current_user="test_user",
|
||||
keep_context=False,
|
||||
)
|
||||
assert result3["action"] == "created"
|
||||
entry3 = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result3["entry_id"]).first()
|
||||
assert entry3.context_data is None
|
||||
assert entry3.has_context is False
|
||||
assert entry3.context_source == "manual"
|
||||
|
||||
# Test usage_notes
|
||||
result4 = InlineCorrectionService.submit_correction_to_dict(
|
||||
db=db_session,
|
||||
record_id=record.id,
|
||||
language_code="it", # different target language avoids conflict
|
||||
dictionary_id=d.id,
|
||||
corrected_value="with notes",
|
||||
current_user="test_user",
|
||||
usage_notes="Use this for user-facing columns only",
|
||||
)
|
||||
assert result4["action"] == "created"
|
||||
entry4 = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result4["entry_id"]).first()
|
||||
assert entry4.usage_notes == "Use this for user-facing columns only"
|
||||
# endregion test_context_capture_in_correction
|
||||
|
||||
|
||||
# region test_jaccard_similarity [TYPE Function]
|
||||
# @PURPOSE: Verify Jaccard similarity computation in ContextAwarePromptBuilder.
|
||||
def test_jaccard_similarity():
|
||||
"""Test T132: Jaccard similarity = 1.0 for identical, 0.0 for disjoint."""
|
||||
from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder
|
||||
|
||||
builder = ContextAwarePromptBuilder()
|
||||
|
||||
# Identical contexts
|
||||
ctx1 = {"schema": "public", "table": "users"}
|
||||
ctx2 = {"schema": "public", "table": "users"}
|
||||
sim = builder.compute_context_similarity(ctx1, ctx2)
|
||||
assert sim == 1.0, f"Expected 1.0, got {sim}"
|
||||
|
||||
# Disjoint contexts
|
||||
ctx3 = {"schema": "private"}
|
||||
sim = builder.compute_context_similarity(ctx1, ctx3)
|
||||
assert sim == 0.0, f"Expected 0.0, got {sim}"
|
||||
|
||||
# Partial overlap
|
||||
ctx4 = {"schema": "public", "table": "orders"}
|
||||
sim = builder.compute_context_similarity(ctx1, ctx4)
|
||||
# intersection: {"public"} union: {"public", "users", "orders"} => 1/3 ≈ 0.33
|
||||
assert 0.33 <= sim <= 0.34, f"Expected ~0.33, got {sim}"
|
||||
|
||||
# Empty/missing contexts
|
||||
assert builder.compute_context_similarity(None, ctx1) == 0.0
|
||||
assert builder.compute_context_similarity(ctx1, None) == 0.0
|
||||
assert builder.compute_context_similarity({}, ctx1) == 0.0
|
||||
|
||||
# Case-insensitive matching (same keys, different case)
|
||||
ctx5 = {"schema": "PUBLIC", "table": "USERS"}
|
||||
sim = builder.compute_context_similarity(ctx1, ctx5)
|
||||
assert sim == 1.0, f"Expected 1.0 for case-insensitive, got {sim}"
|
||||
# endregion test_jaccard_similarity
|
||||
|
||||
|
||||
# region test_context_truncation [TYPE Function]
|
||||
# @PURPOSE: Verify context string truncation at ~2000 chars.
|
||||
def test_context_truncation():
|
||||
"""Test T132: Context string truncation in render_entry."""
|
||||
from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder
|
||||
|
||||
builder = ContextAwarePromptBuilder()
|
||||
|
||||
# Build context data that would produce a very long context string
|
||||
long_val = "x" * 500
|
||||
long_context = {f"key{i}": long_val for i in range(10)}
|
||||
|
||||
# Create an entry-like dict with this context
|
||||
entry = {
|
||||
"source_term": "hello",
|
||||
"target_term": "world",
|
||||
"has_context": True,
|
||||
"context_data": long_context,
|
||||
"usage_notes": None,
|
||||
}
|
||||
|
||||
rendered = builder.render_entry(entry, priority=False)
|
||||
# The context part should be truncated
|
||||
assert len(rendered) < 2200, f"Rendered length {len(rendered)} exceeds expected max"
|
||||
assert "...[truncated]" in rendered, "Expected truncation marker in output"
|
||||
|
||||
# Short context should not be truncated
|
||||
short_entry = {
|
||||
"source_term": "hello",
|
||||
"target_term": "world",
|
||||
"has_context": True,
|
||||
"context_data": {"table": "users"},
|
||||
"usage_notes": None,
|
||||
}
|
||||
short_rendered = builder.render_entry(short_entry, priority=False)
|
||||
assert "...[truncated]" not in short_rendered
|
||||
assert "(context: table=users)" in short_rendered
|
||||
|
||||
# Priority prefix
|
||||
priority_rendered = builder.render_entry(short_entry, priority=True)
|
||||
assert priority_rendered.startswith("# PRIORITY (context match)")
|
||||
|
||||
# Usage notes
|
||||
notes_entry = {
|
||||
"source_term": "hello",
|
||||
"target_term": "world",
|
||||
"has_context": False,
|
||||
"context_data": None,
|
||||
"usage_notes": "Use for admin panels",
|
||||
}
|
||||
notes_rendered = builder.render_entry(notes_entry, priority=False)
|
||||
assert "# Usage: Use for admin panels" in notes_rendered
|
||||
# endregion test_context_truncation
|
||||
|
||||
|
||||
# region test_render_entry_dict_and_object [TYPE Function]
|
||||
# @PURPOSE: Verify render_entry handles both dicts and objects.
|
||||
def test_render_entry_dict_and_object():
|
||||
"""Test T132: render_entry accepts dict or object with same results."""
|
||||
from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder
|
||||
|
||||
builder = ContextAwarePromptBuilder()
|
||||
|
||||
# Dict entry
|
||||
dict_entry = {
|
||||
"source_term": "hello",
|
||||
"target_term": "hola",
|
||||
"has_context": False,
|
||||
"context_data": None,
|
||||
"usage_notes": None,
|
||||
}
|
||||
dict_result = builder.render_entry(dict_entry)
|
||||
|
||||
# Object-like entry using a simple class
|
||||
class FakeEntry:
|
||||
source_term = "hello"
|
||||
target_term = "hola"
|
||||
has_context = False
|
||||
context_data = None
|
||||
usage_notes = None
|
||||
|
||||
obj_result = builder.render_entry(FakeEntry())
|
||||
assert dict_result == obj_result
|
||||
assert dict_result == '"hello" -> "hola"'
|
||||
# endregion test_render_entry_dict_and_object
|
||||
|
||||
|
||||
# region test_build_context_entries_prioritization [TYPE Function]
|
||||
# @PURPOSE: Verify build_context_entries sorts priority entries first.
|
||||
def test_build_context_entries_prioritization():
|
||||
"""Test T132: build_context_entries sorts priority entries before non-priority."""
|
||||
from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder
|
||||
|
||||
builder = ContextAwarePromptBuilder()
|
||||
|
||||
entries = [
|
||||
{
|
||||
"source_term": "high_match",
|
||||
"target_term": "alta_coincidencia",
|
||||
"has_context": True,
|
||||
"context_data": {"schema": "public", "table": "users"},
|
||||
"usage_notes": None,
|
||||
},
|
||||
{
|
||||
"source_term": "no_match",
|
||||
"target_term": "sin_coincidencia",
|
||||
"has_context": True,
|
||||
"context_data": {"schema": "private", "table": "config"},
|
||||
"usage_notes": None,
|
||||
},
|
||||
{
|
||||
"source_term": "no_context",
|
||||
"target_term": "sin_contexto",
|
||||
"has_context": False,
|
||||
"context_data": None,
|
||||
"usage_notes": None,
|
||||
},
|
||||
]
|
||||
|
||||
# Row context that matches the first entry
|
||||
row_context = {"schema": "public", "table": "users"}
|
||||
|
||||
rendered = builder.build_context_entries(entries, row_context)
|
||||
|
||||
# The first entry should be priority (similarity=1.0 >= 0.5)
|
||||
assert len(rendered) == 3
|
||||
assert rendered[0].startswith("# PRIORITY (context match)"), f"Expected priority first, got: {rendered[0]}"
|
||||
assert '"high_match"' in rendered[0]
|
||||
|
||||
# The non-priority entries should not have priority prefix
|
||||
assert not rendered[1].startswith("# PRIORITY")
|
||||
assert not rendered[2].startswith("# PRIORITY")
|
||||
# endregion test_build_context_entries_prioritization
|
||||
|
||||
|
||||
# endregion DictionaryTests
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
# @TEST_EDGE: invalid_run_status -> raises ValueError
|
||||
# @TEST_EDGE: executor_failure -> run is marked FAILED
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -18,11 +19,14 @@ import pytest
|
||||
|
||||
from src.models.translate import (
|
||||
TranslationJob,
|
||||
TranslationLanguage,
|
||||
TranslationPreviewSession,
|
||||
TranslationRecord,
|
||||
TranslationRun,
|
||||
TranslationRunLanguageStats,
|
||||
)
|
||||
from src.plugins.translate.events import TranslationEventLog
|
||||
from src.plugins.translate.executor import TranslationExecutor
|
||||
from src.plugins.translate.orchestrator import TranslationOrchestrator
|
||||
|
||||
|
||||
@@ -598,4 +602,249 @@ class TestTranslationOrchestrator:
|
||||
# endregion test_get_run_records
|
||||
# endregion test_get_run_history
|
||||
# endregion TestTranslationOrchestrator
|
||||
|
||||
|
||||
# region TestTranslationExecutorMultiLang [TYPE Class]
|
||||
# @PURPOSE: Tests for multi-language LLM translation execution: parsing, per-language entries, source-as-reference, backward compat.
|
||||
class TestTranslationExecutorMultiLang:
|
||||
|
||||
# region test_parse_multi_language_response [TYPE Function]
|
||||
# @PURPOSE: _parse_llm_response handles multi-language JSON format correctly.
|
||||
def test_parse_multi_language_response(self) -> None:
|
||||
response = json.dumps({
|
||||
"rows": [
|
||||
{"row_id": 0, "detected_source_language": "fr", "ru": "текст", "en": "text", "de": "Text"},
|
||||
{"row_id": 1, "detected_source_language": "en", "ru": "дашборд", "en": "dashboard", "de": "Dashboard"},
|
||||
]
|
||||
})
|
||||
result = TranslationExecutor._parse_llm_response(
|
||||
response, expected_count=2, target_languages=["ru", "en", "de"]
|
||||
)
|
||||
assert len(result) == 2
|
||||
assert "0" in result
|
||||
assert result["0"]["detected_source_language"] == "fr"
|
||||
assert result["0"]["ru"] == "текст"
|
||||
assert result["0"]["en"] == "text"
|
||||
assert result["0"]["de"] == "Text"
|
||||
assert "1" in result
|
||||
assert result["1"]["en"] == "dashboard"
|
||||
assert result["1"]["ru"] == "дашборд"
|
||||
# endregion test_parse_multi_language_response
|
||||
|
||||
# region test_parse_legacy_format_backward_compat [TYPE Function]
|
||||
# @PURPOSE: _parse_llm_response still handles legacy single "translation" key format.
|
||||
def test_parse_legacy_format_backward_compat(self) -> None:
|
||||
response = json.dumps({
|
||||
"rows": [
|
||||
{"row_id": 0, "translation": "текст", "detected_source_language": "fr"},
|
||||
{"row_id": 1, "translation": "дашборд", "detected_source_language": "en"},
|
||||
]
|
||||
})
|
||||
result = TranslationExecutor._parse_llm_response(
|
||||
response, expected_count=2, target_languages=["ru"]
|
||||
)
|
||||
assert len(result) == 2
|
||||
assert result["0"]["translation"] == "текст"
|
||||
assert result["1"]["detected_source_language"] == "en"
|
||||
# endregion test_parse_legacy_format_backward_compat
|
||||
|
||||
# region test_multi_language_translation_language_entries [TYPE Function]
|
||||
# @PURPOSE: Multi-language LLM response creates per-language TranslationLanguage entries.
|
||||
def test_multi_language_translation_language_entries(self) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
# Mock job with multi-language target_languages
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.id = "job-ml-1"
|
||||
job.target_languages = ["ru", "en"]
|
||||
job.target_language = None
|
||||
job.target_dialect = "postgresql"
|
||||
job.source_dialect = "postgresql"
|
||||
job.translation_column = "name"
|
||||
job.provider_id = "provider-1"
|
||||
job.batch_size = 50
|
||||
|
||||
executor = TranslationExecutor(db, config_manager, "test-user")
|
||||
|
||||
# Patch _call_llm to return multi-language response
|
||||
multi_lang_response = json.dumps({
|
||||
"rows": [
|
||||
{"row_id": "0", "detected_source_language": "fr", "ru": "текст", "en": "text"},
|
||||
{"row_id": "1", "detected_source_language": "en", "ru": "дашборд", "en": "dashboard"},
|
||||
]
|
||||
})
|
||||
batch_rows = [
|
||||
{"row_index": "0", "source_text": "texte", "source_object_name": "Row 0"},
|
||||
{"row_index": "1", "source_text": "dashboard", "source_object_name": "Row 1"},
|
||||
]
|
||||
|
||||
with patch.object(executor, '_call_llm', return_value=multi_lang_response):
|
||||
result = executor._call_llm_for_batch(
|
||||
job=job,
|
||||
run_id="run-ml-1",
|
||||
batch_rows=batch_rows,
|
||||
dict_matches=[],
|
||||
batch_id="batch-ml-1",
|
||||
)
|
||||
|
||||
assert result["successful"] == 2
|
||||
assert result["failed"] == 0
|
||||
assert result["skipped"] == 0
|
||||
|
||||
# Check that TranslationLanguage entries were created for each row per language
|
||||
# Each row should have 2 language entries (ru, en) = 4 total
|
||||
lang_added_calls = [
|
||||
call for call in db.add.call_args_list
|
||||
if isinstance(call[0][0], TranslationLanguage)
|
||||
]
|
||||
# Should have 4 language entries (2 rows x 2 languages)
|
||||
assert len(lang_added_calls) >= 4, f"Expected at least 4 TranslationLanguage entries, got {len(lang_added_calls)}"
|
||||
|
||||
# Group by language code
|
||||
lang_codes: list[str] = []
|
||||
for call in lang_added_calls:
|
||||
lang_entry = call[0][0]
|
||||
lang_codes.append(lang_entry.language_code)
|
||||
assert "ru" in lang_codes
|
||||
assert "en" in lang_codes
|
||||
# Each language should appear twice (once per row)
|
||||
assert lang_codes.count("ru") == 2
|
||||
assert lang_codes.count("en") == 2
|
||||
# endregion test_multi_language_translation_language_entries
|
||||
|
||||
# region test_source_as_reference [TYPE Function]
|
||||
# @PURPOSE: When detected source language matches a target language, store original text verbatim.
|
||||
def test_source_as_reference(self) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.id = "job-sar-1"
|
||||
job.target_languages = ["fr", "en"]
|
||||
job.target_language = None
|
||||
job.target_dialect = "postgresql"
|
||||
job.source_dialect = "postgresql"
|
||||
job.translation_column = "name"
|
||||
job.provider_id = "provider-1"
|
||||
job.batch_size = 50
|
||||
|
||||
executor = TranslationExecutor(db, config_manager, "test-user")
|
||||
|
||||
# Source is French, target_languages include French
|
||||
response = json.dumps({
|
||||
"rows": [
|
||||
{"row_id": "0", "detected_source_language": "fr", "fr": "texte français", "en": "french text"},
|
||||
]
|
||||
})
|
||||
batch_rows = [
|
||||
{"row_index": "0", "source_text": "texte français", "source_object_name": "Row 0"},
|
||||
]
|
||||
|
||||
with patch.object(executor, '_call_llm', return_value=response):
|
||||
result = executor._call_llm_for_batch(
|
||||
job=job,
|
||||
run_id="run-sar-1",
|
||||
batch_rows=batch_rows,
|
||||
dict_matches=[],
|
||||
batch_id="batch-sar-1",
|
||||
)
|
||||
|
||||
assert result["successful"] == 1
|
||||
|
||||
# Find the TranslationLanguage for French — should use source text (original)
|
||||
lang_calls = [
|
||||
call for call in db.add.call_args_list
|
||||
if isinstance(call[0][0], TranslationLanguage)
|
||||
]
|
||||
fr_entry = None
|
||||
en_entry = None
|
||||
for call in lang_calls:
|
||||
le = call[0][0]
|
||||
if le.language_code == "fr":
|
||||
fr_entry = le
|
||||
elif le.language_code == "en":
|
||||
en_entry = le
|
||||
|
||||
assert fr_entry is not None, "FR language entry should exist"
|
||||
# Source-as-reference: fr translation should be the original source text
|
||||
assert fr_entry.translated_value == "texte français", (
|
||||
f"Expected source text 'texte français' for fr, got '{fr_entry.translated_value}'"
|
||||
)
|
||||
assert en_entry is not None, "EN language entry should exist"
|
||||
# EN should get the LLM-translated value
|
||||
assert en_entry.translated_value == "french text"
|
||||
# endregion test_source_as_reference
|
||||
|
||||
# region test_per_language_stats_on_execute_run [TYPE Function]
|
||||
# @PURPOSE: execute_run with multi-language job creates and populates TranslationRunLanguageStats.
|
||||
def test_per_language_stats_on_execute_run(self, mock_job: MagicMock) -> None:
|
||||
db = MagicMock()
|
||||
config_manager = MagicMock()
|
||||
|
||||
# Configure mock_job for multi-language
|
||||
mock_job.target_languages = ["ru", "en"]
|
||||
mock_job.target_language = None
|
||||
mock_job.target_table = "target_tbl"
|
||||
|
||||
# Mock run
|
||||
run = MagicMock()
|
||||
run.id = "run-stats-1"
|
||||
run.job_id = "job-123"
|
||||
run.status = "PENDING"
|
||||
|
||||
# Mock completed run returned by executor
|
||||
completed_run = MagicMock()
|
||||
completed_run.id = "run-stats-1"
|
||||
completed_run.job_id = "job-123"
|
||||
completed_run.status = "COMPLETED"
|
||||
completed_run.total_records = 5
|
||||
completed_run.successful_records = 5
|
||||
completed_run.failed_records = 0
|
||||
completed_run.skipped_records = 0
|
||||
completed_run.completed_at = datetime.now(UTC)
|
||||
completed_run.error_message = None
|
||||
completed_run.insert_status = "success"
|
||||
completed_run.superset_execution_id = "q-1"
|
||||
completed_run.superset_execution_log = {}
|
||||
|
||||
# Mock queries: job -> mock_job, run -> completed_run
|
||||
db.query.return_value.filter.return_value.first.side_effect = [
|
||||
mock_job,
|
||||
completed_run,
|
||||
]
|
||||
|
||||
orch = TranslationOrchestrator(db, config_manager, "test-user")
|
||||
with patch.object(orch, 'event_log'), \
|
||||
patch.object(orch, '_generate_and_insert_sql',
|
||||
return_value={"status": "success", "query_id": "q-1", "rows_affected": 5}), \
|
||||
patch.object(orch, '_update_language_stats') as mock_update_stats, \
|
||||
patch('src.plugins.translate.orchestrator.TranslationExecutor') as MockExecutor:
|
||||
|
||||
mock_executor_instance = MagicMock()
|
||||
mock_executor_instance.execute_run.return_value = completed_run
|
||||
MockExecutor.return_value = mock_executor_instance
|
||||
|
||||
result = orch.execute_run(run)
|
||||
|
||||
assert result.status == "COMPLETED"
|
||||
|
||||
# Verify that TranslationRunLanguageStats entries were created
|
||||
stats_added = [
|
||||
call for call in db.add.call_args_list
|
||||
if isinstance(call[0][0], TranslationRunLanguageStats)
|
||||
]
|
||||
# 2 language stats entries (ru, en)
|
||||
assert len(stats_added) >= 2, (
|
||||
f"Expected at least 2 TranslationRunLanguageStats entries, got {len(stats_added)}"
|
||||
)
|
||||
codes = [call[0][0].language_code for call in stats_added]
|
||||
assert "ru" in codes
|
||||
assert "en" in codes
|
||||
|
||||
# Verify _update_language_stats was called
|
||||
mock_update_stats.assert_called_once()
|
||||
# endregion test_per_language_stats_on_execute_run
|
||||
|
||||
# endregion TestTranslationExecutorMultiLang
|
||||
# endregion OrchestratorTests
|
||||
|
||||
@@ -420,8 +420,13 @@ class TestTranslationPreview:
|
||||
assert tokens > 0
|
||||
assert tokens < len(prompt) # tokens typically fewer than chars
|
||||
|
||||
output_tokens = TokenEstimator.estimate_output_tokens(10)
|
||||
assert output_tokens == 500
|
||||
# Single language
|
||||
output_tokens = TokenEstimator.estimate_output_tokens(10, num_languages=1)
|
||||
assert output_tokens == 600 # 10 * 1 * 50 * 1.2
|
||||
|
||||
# Multi-language
|
||||
output_tokens_multi = TokenEstimator.estimate_output_tokens(10, num_languages=3)
|
||||
assert output_tokens_multi == 1800 # 10 * 3 * 50 * 1.2
|
||||
|
||||
cost = TokenEstimator.estimate_cost(1000, 0.002)
|
||||
assert cost == 0.002
|
||||
@@ -431,28 +436,65 @@ class TestTranslationPreview:
|
||||
# endregion test_cost_estimation
|
||||
|
||||
# region test_preview_parse_llm_response [TYPE Function]
|
||||
# @PURPOSE: Verify LLM JSON response parsing.
|
||||
# @PURPOSE: Verify LLM JSON response parsing includes detected_source_language.
|
||||
def test_preview_parse_llm_response(self):
|
||||
"""Parse LLM response should extract translations keyed by row_id."""
|
||||
"""Parse LLM response should extract translations with detected_source_language per row."""
|
||||
response = json.dumps({
|
||||
"rows": [
|
||||
{"row_id": "0", "translation": "Привет"},
|
||||
{"row_id": "0", "translation": "Привет", "detected_source_language": "en"},
|
||||
{"row_id": "1", "translation": "Мир"},
|
||||
]
|
||||
})
|
||||
result = TranslationPreview._parse_llm_response(response, 2)
|
||||
assert result == {"0": "Привет", "1": "Мир"}
|
||||
assert result == {
|
||||
"0": {"translation": "Привет", "detected_source_language": "en"},
|
||||
"1": {"translation": "Мир", "detected_source_language": "und"},
|
||||
}
|
||||
# endregion test_preview_parse_llm_response
|
||||
|
||||
# region test_preview_parse_llm_response_multilang [TYPE Function]
|
||||
# @PURPOSE: Verify multi-language LLM response parsing.
|
||||
def test_preview_parse_llm_response_multilang(self):
|
||||
"""Parse multi-language LLM response should extract per-language translations."""
|
||||
response = json.dumps({
|
||||
"rows": [
|
||||
{"row_id": "0", "detected_source_language": "fr", "ru": "текст", "en": "text", "de": "Text"},
|
||||
{"row_id": "1", "detected_source_language": "en", "ru": "привет", "en": "hello"},
|
||||
]
|
||||
})
|
||||
result = TranslationPreview._parse_llm_response(response, 2, target_languages=["ru", "en", "de"])
|
||||
assert result == {
|
||||
"0": {"detected_source_language": "fr", "ru": "текст", "en": "text", "de": "Text"},
|
||||
"1": {"detected_source_language": "en", "ru": "привет", "en": "hello"},
|
||||
}
|
||||
# Verify per-language extraction
|
||||
assert result["0"]["ru"] == "текст"
|
||||
assert result["0"]["en"] == "text"
|
||||
assert result["1"]["ru"] == "привет"
|
||||
# endregion test_preview_parse_llm_response_multilang
|
||||
|
||||
# region test_preview_parse_llm_response_with_code_block [TYPE Function]
|
||||
# @PURPOSE: Verify LLM response parsing handles markdown code blocks.
|
||||
def test_preview_parse_llm_response_with_code_block(self):
|
||||
"""Parse LLM response should handle markdown code block wrapping."""
|
||||
response = "```json\n{\n \"rows\": [\n {\"row_id\": \"0\", \"translation\": \"Test\"}\n ]\n}\n```"
|
||||
result = TranslationPreview._parse_llm_response(response, 1)
|
||||
assert result == {"0": "Test"}
|
||||
assert result == {"0": {"translation": "Test", "detected_source_language": "und"}}
|
||||
# endregion test_preview_parse_llm_response_with_code_block
|
||||
|
||||
# region test_cost_warning [TYPE Function]
|
||||
# @PURPOSE: Verify cost warning for large sample sizes.
|
||||
def test_cost_warning(self):
|
||||
"""Cost warning should appear for sample_size > 30."""
|
||||
warning = TokenEstimator.check_cost_warning(31, 2, 5000, 0.01)
|
||||
assert warning is not None
|
||||
assert "Large preview" in warning
|
||||
assert "5000" in warning
|
||||
|
||||
no_warning = TokenEstimator.check_cost_warning(10, 2, 1000, 0.002)
|
||||
assert no_warning is None
|
||||
# endregion test_cost_warning
|
||||
|
||||
# region test_preview_compute_config_hash [TYPE Function]
|
||||
# @PURPOSE: Verify config hash computation is deterministic.
|
||||
def test_preview_compute_config_hash(self):
|
||||
|
||||
@@ -14,7 +14,7 @@ import io
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy import func, or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...core.logger import belief_scope, logger
|
||||
@@ -27,6 +27,23 @@ from ...models.translate import (
|
||||
from ._utils import _detect_delimiter, _normalize_term
|
||||
|
||||
|
||||
# #region _validate_bcp47 [C:2] [TYPE Function] [SEMANTICS validation, bcp47, language]
|
||||
# @BRIEF Validate that a language tag is a non-empty BCP-47 string.
|
||||
def _validate_bcp47(tag: str, field_name: str) -> None:
|
||||
"""Validate that tag is a non-empty BCP-47 string (basic check)."""
|
||||
if not tag or not tag.strip():
|
||||
raise ValueError(f"{field_name} must be a non-empty BCP-47 language tag")
|
||||
tag = tag.strip()
|
||||
# Basic BCP-47 validation: must match lang[-script][-region][-variant]* pattern
|
||||
import re as _re
|
||||
if not _re.match(r'^[a-zA-Z]{2,8}(-[a-zA-Z0-9]{1,8})*$', tag):
|
||||
raise ValueError(
|
||||
f"{field_name} is not a valid BCP-47 tag: '{tag}'. "
|
||||
"Expected format like 'en', 'ru', 'zh-CN', 'pt-BR'."
|
||||
)
|
||||
# #endregion _validate_bcp47
|
||||
|
||||
|
||||
# #region DictionaryManager [C:4] [TYPE Class]
|
||||
# @BRIEF Manages terminology dictionaries and their entries with referential integrity.
|
||||
# @PRE: Database session is open and valid.
|
||||
@@ -34,12 +51,14 @@ from ._utils import _detect_delimiter, _normalize_term
|
||||
# @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
|
||||
class DictionaryManager:
|
||||
# region DictionaryManager.create_dictionary [TYPE Function]
|
||||
# @PURPOSE: Create a new terminology dictionary.
|
||||
# @PRE: payload contains name, source_dialect, target_dialect.
|
||||
# @PURPOSE: Create a new terminology dictionary (language independent at dictionary level; language pairs are per-entry).
|
||||
# @PRE: name is provided.
|
||||
# @POST: New TerminologyDictionary row is created and returned.
|
||||
@staticmethod
|
||||
def create_dictionary(
|
||||
db: Session, name: str, source_dialect: str, target_dialect: str,
|
||||
db: Session, name: str,
|
||||
source_dialect: str = "",
|
||||
target_dialect: str = "",
|
||||
created_by: str | None = None, description: str | None = None,
|
||||
is_active: bool = True,
|
||||
) -> TerminologyDictionary:
|
||||
@@ -48,8 +67,8 @@ class DictionaryManager:
|
||||
dictionary = TerminologyDictionary(
|
||||
name=name,
|
||||
description=description,
|
||||
source_dialect=source_dialect,
|
||||
target_dialect=target_dialect,
|
||||
source_dialect=source_dialect or "",
|
||||
target_dialect=target_dialect or "",
|
||||
is_active=is_active,
|
||||
created_by=created_by,
|
||||
)
|
||||
@@ -166,53 +185,66 @@ class DictionaryManager:
|
||||
# endregion DictionaryManager.list_dictionaries
|
||||
|
||||
# region DictionaryManager.add_entry [TYPE Function]
|
||||
# @PURPOSE: Add an entry to a dictionary, enforcing unique source_term_normalized.
|
||||
# @PRE: dict_id exists. source_term and target_term are non-empty.
|
||||
# @PURPOSE: Add an entry to a dictionary with language-pair-aware uniqueness validation.
|
||||
# @PRE: dict_id exists. source_term, target_term are non-empty. source_language, target_language are valid BCP-47.
|
||||
# @POST: New DictionaryEntry row is created or raises on duplicate.
|
||||
@staticmethod
|
||||
def add_entry(
|
||||
db: Session, dict_id: str, source_term: str, target_term: str,
|
||||
context_notes: str | None = None,
|
||||
source_language: str = "und",
|
||||
target_language: str = "und",
|
||||
) -> DictionaryEntry:
|
||||
with belief_scope("DictionaryManager.add_entry"):
|
||||
_validate_bcp47(source_language, "source_language")
|
||||
_validate_bcp47(target_language, "target_language")
|
||||
|
||||
normalized = _normalize_term(source_term)
|
||||
# Uniqueness is now (dictionary_id, source_term_normalized, source_language, target_language)
|
||||
existing = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(
|
||||
DictionaryEntry.dictionary_id == dict_id,
|
||||
DictionaryEntry.source_term_normalized == normalized,
|
||||
DictionaryEntry.source_language == source_language,
|
||||
DictionaryEntry.target_language == target_language,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
raise ValueError(
|
||||
f"Duplicate entry: '{source_term}' already exists in this dictionary "
|
||||
f"(id={existing.id}). Use overwrite or keep_existing conflict mode."
|
||||
f"Duplicate entry: '{source_term}' (lang: {source_language}→{target_language}) "
|
||||
f"already exists in this dictionary (id={existing.id}). "
|
||||
"Use overwrite or keep_existing conflict mode."
|
||||
)
|
||||
|
||||
logger.reason("Adding dictionary entry", {"dict_id": dict_id, "term": source_term})
|
||||
logger.reason("Adding dictionary entry", {"dict_id": dict_id, "term": source_term, "src_lang": source_language, "tgt_lang": target_language})
|
||||
entry = DictionaryEntry(
|
||||
dictionary_id=dict_id,
|
||||
source_term=source_term.strip(),
|
||||
source_term_normalized=normalized,
|
||||
target_term=target_term.strip(),
|
||||
context_notes=context_notes.strip() if context_notes else None,
|
||||
source_language=source_language.strip(),
|
||||
target_language=target_language.strip(),
|
||||
)
|
||||
db.add(entry)
|
||||
db.commit()
|
||||
db.refresh(entry)
|
||||
logger.reflect("Entry added", {"entry_id": entry.id})
|
||||
logger.reflect("Entry added", {"entry_id": entry.id, "src_lang": source_language, "tgt_lang": target_language})
|
||||
return entry
|
||||
# endregion DictionaryManager.add_entry
|
||||
|
||||
# region DictionaryManager.edit_entry [TYPE Function]
|
||||
# @PURPOSE: Edit an existing dictionary entry with duplicate-aware normalization.
|
||||
# @PURPOSE: Edit an existing dictionary entry with language-pair-aware duplicate check.
|
||||
# @PRE: entry_id exists.
|
||||
# @POST: Entry fields are updated.
|
||||
@staticmethod
|
||||
def edit_entry(
|
||||
db: Session, entry_id: str, source_term: str | None = None,
|
||||
target_term: str | None = None, context_notes: str | None = None,
|
||||
source_language: str | None = None,
|
||||
target_language: str | None = None,
|
||||
) -> DictionaryEntry:
|
||||
with belief_scope("DictionaryManager.edit_entry"):
|
||||
entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first()
|
||||
@@ -220,14 +252,24 @@ class DictionaryManager:
|
||||
raise ValueError(f"Entry not found: {entry_id}")
|
||||
|
||||
logger.reason("Editing dictionary entry", {"entry_id": entry_id})
|
||||
|
||||
if source_language is not None:
|
||||
_validate_bcp47(source_language, "source_language")
|
||||
entry.source_language = source_language.strip()
|
||||
if target_language is not None:
|
||||
_validate_bcp47(target_language, "target_language")
|
||||
entry.target_language = target_language.strip()
|
||||
|
||||
if source_term is not None:
|
||||
normalized = _normalize_term(source_term)
|
||||
# Check uniqueness within the same dictionary
|
||||
# Check uniqueness within the same dictionary + language pair
|
||||
existing = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(
|
||||
DictionaryEntry.dictionary_id == entry.dictionary_id,
|
||||
DictionaryEntry.source_term_normalized == normalized,
|
||||
DictionaryEntry.source_language == entry.source_language,
|
||||
DictionaryEntry.target_language == entry.target_language,
|
||||
DictionaryEntry.id != entry_id,
|
||||
)
|
||||
.first()
|
||||
@@ -318,6 +360,8 @@ class DictionaryManager:
|
||||
delimiter: str | None = None,
|
||||
on_conflict: str = "overwrite",
|
||||
preview_only: bool = False,
|
||||
default_source_language: str | None = None,
|
||||
default_target_language: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
with belief_scope("DictionaryManager.import_entries"):
|
||||
# Validate dictionary
|
||||
@@ -359,6 +403,8 @@ class DictionaryManager:
|
||||
source_term = row.get("source_term", "").strip()
|
||||
target_term = row.get("target_term", "").strip()
|
||||
context_notes = row.get("context_notes", "").strip() or None
|
||||
source_language = row.get("source_language", "").strip() or default_source_language or "und"
|
||||
target_language = row.get("target_language", "").strip() or default_target_language or "und"
|
||||
|
||||
if not source_term or not target_term:
|
||||
result["errors"].append({
|
||||
@@ -376,6 +422,8 @@ class DictionaryManager:
|
||||
.filter(
|
||||
DictionaryEntry.dictionary_id == dict_id,
|
||||
DictionaryEntry.source_term_normalized == normalized,
|
||||
DictionaryEntry.source_language == source_language,
|
||||
DictionaryEntry.target_language == target_language,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
@@ -384,6 +432,8 @@ class DictionaryManager:
|
||||
"source_term": source_term,
|
||||
"target_term": target_term,
|
||||
"context_notes": context_notes,
|
||||
"source_language": source_language,
|
||||
"target_language": target_language,
|
||||
"is_conflict": existing is not None,
|
||||
"existing_target_term": existing.target_term if existing else None,
|
||||
}
|
||||
@@ -395,6 +445,8 @@ class DictionaryManager:
|
||||
.filter(
|
||||
DictionaryEntry.dictionary_id == dict_id,
|
||||
DictionaryEntry.source_term_normalized == normalized,
|
||||
DictionaryEntry.source_language == source_language,
|
||||
DictionaryEntry.target_language == target_language,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
@@ -404,6 +456,8 @@ class DictionaryManager:
|
||||
existing.source_term = source_term
|
||||
existing.target_term = target_term
|
||||
existing.context_notes = context_notes
|
||||
existing.source_language = source_language
|
||||
existing.target_language = target_language
|
||||
result["updated"] += 1
|
||||
elif on_conflict == "keep_existing":
|
||||
result["skipped"] += 1
|
||||
@@ -421,6 +475,8 @@ class DictionaryManager:
|
||||
source_term_normalized=normalized,
|
||||
target_term=target_term,
|
||||
context_notes=context_notes,
|
||||
source_language=source_language,
|
||||
target_language=target_language,
|
||||
)
|
||||
db.add(entry)
|
||||
result["created"] += 1
|
||||
@@ -446,14 +502,122 @@ class DictionaryManager:
|
||||
return result
|
||||
# endregion DictionaryManager.import_entries
|
||||
|
||||
# region DictionaryManager.export_entries [TYPE Function]
|
||||
# @PURPOSE: Export all entries as CSV string with language columns.
|
||||
# @PRE: dict_id exists.
|
||||
# @POST: Returns CSV string with header: source_term, target_term, source_language, target_language, context_notes, context_data, usage_notes.
|
||||
@staticmethod
|
||||
def export_entries(
|
||||
db: Session, dict_id: str, delimiter: str = ",",
|
||||
) -> str:
|
||||
with belief_scope("DictionaryManager.export_entries"):
|
||||
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
|
||||
if not dictionary:
|
||||
raise ValueError(f"Dictionary not found: {dict_id}")
|
||||
|
||||
entries = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(DictionaryEntry.dictionary_id == dict_id)
|
||||
.order_by(DictionaryEntry.source_term.asc())
|
||||
.all()
|
||||
)
|
||||
|
||||
output = io.StringIO()
|
||||
fieldnames = [
|
||||
"source_term", "target_term",
|
||||
"source_language", "target_language",
|
||||
"context_notes", "context_data", "usage_notes",
|
||||
]
|
||||
writer = csv.DictWriter(output, fieldnames=fieldnames, delimiter=delimiter)
|
||||
writer.writeheader()
|
||||
|
||||
for entry in entries:
|
||||
writer.writerow({
|
||||
"source_term": entry.source_term,
|
||||
"target_term": entry.target_term,
|
||||
"source_language": entry.source_language,
|
||||
"target_language": entry.target_language,
|
||||
"context_notes": entry.context_notes or "",
|
||||
"context_data": entry.context_data,
|
||||
"usage_notes": entry.usage_notes or "",
|
||||
})
|
||||
|
||||
result = output.getvalue()
|
||||
logger.reflect("Entries exported", {"dict_id": dict_id, "count": len(entries), "delimiter": repr(delimiter)})
|
||||
return result
|
||||
# endregion DictionaryManager.export_entries
|
||||
|
||||
# region DictionaryManager.migrate_old_entries [TYPE Function]
|
||||
# @PURPOSE: Migrate existing single-language dictionary entries to populate source_language and target_language.
|
||||
# @PRE: db session is open.
|
||||
# @POST: Entries with "und" source_language or target_language are updated with inferred values.
|
||||
# @SIDE_EFFECT: Updates DictionaryEntry rows in bulk.
|
||||
@staticmethod
|
||||
def migrate_old_entries(db: Session) -> dict[str, int]:
|
||||
with belief_scope("DictionaryManager.migrate_old_entries"):
|
||||
logger.reason("Starting migration of old dictionary entries")
|
||||
migrated_source = 0
|
||||
migrated_target = 0
|
||||
skipped = 0
|
||||
|
||||
# Find all entries that may need migration (source_language is "und" or target_language is "und")
|
||||
entries = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(
|
||||
(DictionaryEntry.source_language == "und") |
|
||||
(DictionaryEntry.target_language == "und")
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
for entry in entries:
|
||||
# Try to infer source_language from origin_source_language
|
||||
if entry.source_language == "und":
|
||||
if entry.origin_source_language:
|
||||
entry.source_language = entry.origin_source_language
|
||||
migrated_source += 1
|
||||
|
||||
# Try to infer target_language from the parent dictionary's deprecated target_language
|
||||
if entry.target_language == "und":
|
||||
dictionary = (
|
||||
db.query(TerminologyDictionary)
|
||||
.filter(TerminologyDictionary.id == entry.dictionary_id)
|
||||
.first()
|
||||
)
|
||||
if dictionary and dictionary.target_language:
|
||||
entry.target_language = dictionary.target_language
|
||||
migrated_target += 1
|
||||
|
||||
if entry.source_language == "und" and entry.target_language == "und":
|
||||
skipped += 1
|
||||
|
||||
db.commit()
|
||||
|
||||
logger.reflect("Migration complete", {
|
||||
"migrated_source": migrated_source,
|
||||
"migrated_target": migrated_target,
|
||||
"skipped": skipped,
|
||||
"total_processed": len(entries),
|
||||
})
|
||||
return {
|
||||
"migrated_source": migrated_source,
|
||||
"migrated_target": migrated_target,
|
||||
"skipped": skipped,
|
||||
"total_processed": len(entries),
|
||||
}
|
||||
# endregion DictionaryManager.migrate_old_entries
|
||||
|
||||
# region DictionaryManager.filter_for_batch [TYPE Function]
|
||||
# @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job.
|
||||
# @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job,
|
||||
# optionally filtered by language pair.
|
||||
# @PRE: job_id exists and source_texts is a list of strings.
|
||||
# @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries.
|
||||
# @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
|
||||
@staticmethod
|
||||
def filter_for_batch(
|
||||
db: Session, source_texts: list[str], job_id: str,
|
||||
source_language: str | None = None,
|
||||
target_language: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
with belief_scope("DictionaryManager.filter_for_batch"):
|
||||
# Get dictionaries attached to this job
|
||||
@@ -508,6 +672,18 @@ class DictionaryManager:
|
||||
if not norm:
|
||||
continue
|
||||
|
||||
# Apply language pair filtering
|
||||
if source_language is not None:
|
||||
src = source_language.strip().lower()
|
||||
entry_src = entry.source_language.strip().lower()
|
||||
if entry_src != src and entry_src != "und":
|
||||
continue
|
||||
if target_language is not None:
|
||||
tgt = target_language.strip().lower()
|
||||
entry_tgt = entry.target_language.strip().lower()
|
||||
if entry_tgt != tgt:
|
||||
continue
|
||||
|
||||
# Word-boundary-aware matching
|
||||
# Build pattern: \bterm\b (case-insensitive)
|
||||
# Escape regex special chars in the search term
|
||||
@@ -528,6 +704,12 @@ class DictionaryManager:
|
||||
"dictionary_id": entry.dictionary_id,
|
||||
"dictionary_name": dict_map.get(entry.dictionary_id, ""),
|
||||
"context_notes": entry.context_notes,
|
||||
"context_data": entry.context_data,
|
||||
"has_context": entry.has_context,
|
||||
"context_source": entry.context_source,
|
||||
"usage_notes": entry.usage_notes,
|
||||
"source_language": entry.source_language,
|
||||
"target_language": entry.target_language,
|
||||
})
|
||||
|
||||
# Sort by dictionary link priority (order of dict_ids from link order)
|
||||
@@ -539,6 +721,8 @@ class DictionaryManager:
|
||||
"source_texts": len(source_texts),
|
||||
"matches": len(matched),
|
||||
"dictionaries_used": len(dictionaries),
|
||||
"source_language": source_language,
|
||||
"target_language": target_language,
|
||||
})
|
||||
return matched
|
||||
# endregion DictionaryManager.filter_for_batch
|
||||
@@ -560,6 +744,8 @@ class DictionaryManager:
|
||||
origin_row_key: str | None = None,
|
||||
origin_user_id: str | None = None,
|
||||
on_conflict: str = "overwrite",
|
||||
context_data: dict[str, Any] | None = None,
|
||||
usage_notes: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
with belief_scope("DictionaryManager.submit_correction"):
|
||||
# Validate dictionary exists
|
||||
@@ -568,11 +754,22 @@ class DictionaryManager:
|
||||
raise ValueError(f"Dictionary '{dict_id}' not found")
|
||||
|
||||
normalized = _normalize_term(source_term)
|
||||
entry_src_lang = dictionary.source_dialect or "und"
|
||||
entry_tgt_lang = dictionary.target_dialect or "und"
|
||||
# Find existing entry: match exact language pair OR old-style "und"/"und" entries
|
||||
existing = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(
|
||||
DictionaryEntry.dictionary_id == dict_id,
|
||||
DictionaryEntry.source_term_normalized == normalized,
|
||||
or_(
|
||||
# Exact language pair match
|
||||
(DictionaryEntry.source_language == entry_src_lang) &
|
||||
(DictionaryEntry.target_language == entry_tgt_lang),
|
||||
# Old-style entries without language pair
|
||||
(DictionaryEntry.source_language == "und") &
|
||||
(DictionaryEntry.target_language == "und"),
|
||||
),
|
||||
)
|
||||
.first()
|
||||
)
|
||||
@@ -606,6 +803,12 @@ class DictionaryManager:
|
||||
existing.origin_row_key = origin_row_key
|
||||
if origin_user_id:
|
||||
existing.origin_user_id = origin_user_id
|
||||
if context_data is not None:
|
||||
existing.context_data = context_data
|
||||
existing.has_context = bool(context_data)
|
||||
existing.context_source = "auto"
|
||||
if usage_notes is not None:
|
||||
existing.usage_notes = usage_notes
|
||||
db.flush()
|
||||
result["action"] = "updated"
|
||||
result["entry_id"] = existing.id
|
||||
@@ -621,12 +824,18 @@ class DictionaryManager:
|
||||
result["message"] = "Correction cancelled by conflict mode"
|
||||
return result
|
||||
else:
|
||||
# Create new entry
|
||||
# Create new entry — derive language pair from dictionary
|
||||
entry = DictionaryEntry(
|
||||
dictionary_id=dict_id,
|
||||
source_term=source_term.strip(),
|
||||
source_term_normalized=normalized,
|
||||
target_term=corrected_target_term.strip(),
|
||||
source_language=entry_src_lang,
|
||||
target_language=entry_tgt_lang,
|
||||
context_data=context_data,
|
||||
usage_notes=usage_notes,
|
||||
has_context=bool(context_data),
|
||||
context_source="auto" if context_data else None,
|
||||
origin_run_id=origin_run_id,
|
||||
origin_row_key=origin_row_key,
|
||||
origin_user_id=origin_user_id,
|
||||
@@ -678,11 +887,19 @@ class DictionaryManager:
|
||||
continue
|
||||
|
||||
normalized = _normalize_term(source_term)
|
||||
bulk_src_lang = dictionary.source_dialect or "und"
|
||||
bulk_tgt_lang = dictionary.target_dialect or "und"
|
||||
existing = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(
|
||||
DictionaryEntry.dictionary_id == dict_id,
|
||||
DictionaryEntry.source_term_normalized == normalized,
|
||||
or_(
|
||||
(DictionaryEntry.source_language == bulk_src_lang) &
|
||||
(DictionaryEntry.target_language == bulk_tgt_lang),
|
||||
(DictionaryEntry.source_language == "und") &
|
||||
(DictionaryEntry.target_language == "und"),
|
||||
),
|
||||
)
|
||||
.first()
|
||||
)
|
||||
@@ -729,6 +946,8 @@ class DictionaryManager:
|
||||
source_term=source_term,
|
||||
source_term_normalized=normalized,
|
||||
target_term=corrected_target,
|
||||
source_language=bulk_src_lang,
|
||||
target_language=bulk_tgt_lang,
|
||||
origin_run_id=corr.get("origin_run_id"),
|
||||
origin_row_key=corr.get("origin_row_key"),
|
||||
origin_user_id=origin_user_id,
|
||||
|
||||
@@ -191,6 +191,18 @@ class TranslationEventLog:
|
||||
logger.reflect("No expired events to prune", {})
|
||||
return {"pruned": 0, "snapshot_id": None}
|
||||
|
||||
# Compute per-language metrics from expired events before pruning
|
||||
per_language: dict[str, dict[str, int | float]] = {}
|
||||
expired_events = expired_query.all()
|
||||
for evt in expired_events:
|
||||
event_data = evt.event_data or {}
|
||||
lang = event_data.get("language_code") or event_data.get("target_language") or "_unknown_"
|
||||
if lang not in per_language:
|
||||
per_language[lang] = {"cumulative_tokens": 0, "cumulative_cost": 0.0, "runs": 0}
|
||||
per_language[lang]["cumulative_tokens"] += event_data.get("token_count", 0) or 0
|
||||
per_language[lang]["cumulative_cost"] += event_data.get("cost", 0.0) or 0.0
|
||||
per_language[lang]["runs"] += 1 if event_data.get("run_id") else 0
|
||||
|
||||
# Create MetricSnapshot before pruning
|
||||
snapshot = MetricSnapshot(
|
||||
id=str(uuid.uuid4()),
|
||||
@@ -198,6 +210,7 @@ class TranslationEventLog:
|
||||
key_hash=f"prune_{cutoff.timestamp():.0f}",
|
||||
covers_events_before=cutoff,
|
||||
total_records=total_expired,
|
||||
per_language_metrics=per_language or None,
|
||||
snapshot_date=datetime.now(UTC),
|
||||
)
|
||||
self.db.add(snapshot)
|
||||
|
||||
@@ -27,6 +27,7 @@ from ...core.logger import belief_scope, logger
|
||||
from ...models.translate import (
|
||||
TranslationBatch,
|
||||
TranslationJob,
|
||||
TranslationLanguage,
|
||||
TranslationPreviewRecord,
|
||||
TranslationPreviewSession,
|
||||
TranslationRecord,
|
||||
@@ -36,6 +37,7 @@ from ...services.llm_prompt_templates import render_prompt
|
||||
from ...services.llm_provider import LLMProviderService
|
||||
from .dictionary import DictionaryManager
|
||||
from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
|
||||
from .prompt_builder import ContextAwarePromptBuilder
|
||||
|
||||
# #region MAX_RETRIES_PER_BATCH [TYPE Constant]
|
||||
# @BRIEF Maximum number of retries for a single batch before marking it failed.
|
||||
@@ -62,6 +64,7 @@ class TranslationExecutor:
|
||||
self.current_user = current_user
|
||||
self.on_batch_progress = on_batch_progress
|
||||
self._current_run_id: str | None = None
|
||||
self._preview_edits_cache: dict[str, dict[str, str]] | None = None # key_hash -> {lang_code: edited_value}
|
||||
|
||||
# region execute_run [TYPE Function]
|
||||
# @PURPOSE: Run full translation execution for a TranslationRun.
|
||||
@@ -84,6 +87,9 @@ class TranslationExecutor:
|
||||
"batch_size": job.batch_size,
|
||||
})
|
||||
|
||||
# Load preview edits for carry-forward
|
||||
self._load_preview_edits(job.id)
|
||||
|
||||
# Mark run as RUNNING
|
||||
run.status = "RUNNING"
|
||||
run.started_at = datetime.now(UTC)
|
||||
@@ -410,6 +416,70 @@ class TranslationExecutor:
|
||||
return []
|
||||
# endregion _extract_chart_data_rows
|
||||
|
||||
# region _load_preview_edits [TYPE Function]
|
||||
# @PURPOSE: Load user edits from accepted preview session for carry-forward.
|
||||
# @PRE: job_id exists.
|
||||
# @POST: Populates _preview_edits_cache with key_hash -> {lang_code: edited_value}.
|
||||
# @SIDE_EFFECT: Queries TranslationPreviewLanguage and TranslationPreviewRecord.
|
||||
def _load_preview_edits(self, job_id: str) -> None:
|
||||
"""Load preview edits for carry-forward during execution."""
|
||||
from ...models.translate import TranslationPreviewLanguage, TranslationPreviewRecord, TranslationPreviewSession
|
||||
|
||||
with belief_scope("TranslationExecutor._load_preview_edits"):
|
||||
session = (
|
||||
self.db.query(TranslationPreviewSession)
|
||||
.filter(
|
||||
TranslationPreviewSession.job_id == job_id,
|
||||
TranslationPreviewSession.status == "APPLIED",
|
||||
)
|
||||
.order_by(TranslationPreviewSession.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
if not session:
|
||||
logger.reason("No applied preview session found — no edits to carry forward", {
|
||||
"job_id": job_id,
|
||||
})
|
||||
self._preview_edits_cache = {}
|
||||
return
|
||||
|
||||
records = (
|
||||
self.db.query(TranslationPreviewRecord)
|
||||
.filter(TranslationPreviewRecord.session_id == session.id)
|
||||
.all()
|
||||
)
|
||||
|
||||
edits: dict[str, dict[str, str]] = {}
|
||||
for rec in records:
|
||||
if not rec.source_data:
|
||||
continue
|
||||
# Compute key hash from source_data
|
||||
key_hash = self._compute_key_hash(rec.source_data)
|
||||
edited_langs: dict[str, str] = {}
|
||||
for lang_entry in (rec.languages or []):
|
||||
if lang_entry.status in ("edited", "approved") and lang_entry.user_edit:
|
||||
edited_langs[lang_entry.language_code] = lang_entry.final_value or lang_entry.user_edit
|
||||
logger.reason("Carrying forward preview edit", {
|
||||
"key_hash": key_hash,
|
||||
"language_code": lang_entry.language_code,
|
||||
})
|
||||
if edited_langs:
|
||||
edits[key_hash] = edited_langs
|
||||
|
||||
self._preview_edits_cache = edits
|
||||
logger.reason(f"Loaded {len(edits)} preview edits for carry-forward", {
|
||||
"job_id": job_id,
|
||||
})
|
||||
# endregion _load_preview_edits
|
||||
|
||||
# region _compute_key_hash [TYPE Function]
|
||||
# @PURPOSE: Compute a stable hash from source_data dict for matching preview edits.
|
||||
@staticmethod
|
||||
def _compute_key_hash(source_data: dict) -> str:
|
||||
import hashlib
|
||||
stable = json.dumps(source_data, sort_keys=True)
|
||||
return hashlib.sha256(stable.encode()).hexdigest()[:16]
|
||||
# endregion _compute_key_hash
|
||||
|
||||
# region _process_batch [TYPE Function]
|
||||
# @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records.
|
||||
# @PRE: job and batch_rows are valid.
|
||||
@@ -452,17 +522,39 @@ class TranslationExecutor:
|
||||
self.db, source_texts, job.id
|
||||
)
|
||||
|
||||
# For each row, determine if we need LLM translation or can use approved translation
|
||||
# For each row, determine if we need LLM translation or can use approved/preview edit
|
||||
rows_for_llm = []
|
||||
pre_translated = []
|
||||
|
||||
for row in batch_rows:
|
||||
if row.get("approved_translation"):
|
||||
pre_translated.append(row)
|
||||
else:
|
||||
rows_for_llm.append(row)
|
||||
continue
|
||||
|
||||
# Check for preview edits carry-forward
|
||||
source_data = row.get("source_data") or {}
|
||||
if source_data and self._preview_edits_cache:
|
||||
key_hash = self._compute_key_hash(source_data)
|
||||
preview_edit = self._preview_edits_cache.get(key_hash)
|
||||
if preview_edit:
|
||||
# Use the first edited language's value as the approved translation
|
||||
first_edit = next(iter(preview_edit.values()), None)
|
||||
if first_edit:
|
||||
row["approved_translation"] = first_edit
|
||||
logger.reason("Using preview edit carry-forward", {
|
||||
"key_hash": key_hash,
|
||||
"langs": list(preview_edit.keys()),
|
||||
})
|
||||
pre_translated.append(row)
|
||||
continue
|
||||
|
||||
rows_for_llm.append(row)
|
||||
|
||||
# Handle pre-translated (approved) rows
|
||||
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
|
||||
if not isinstance(target_languages, list):
|
||||
target_languages = [str(target_languages)]
|
||||
|
||||
for row in pre_translated:
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
@@ -477,6 +569,20 @@ class TranslationExecutor:
|
||||
status="SUCCESS",
|
||||
)
|
||||
self.db.add(record)
|
||||
|
||||
# Create per-language entry for each target language (pre-approved)
|
||||
for lang_code in target_languages:
|
||||
lang_entry = TranslationLanguage(
|
||||
id=str(uuid.uuid4()),
|
||||
record_id=record.id,
|
||||
language_code=lang_code,
|
||||
source_language_detected="und",
|
||||
translated_value=row.get("approved_translation"),
|
||||
final_value=row.get("approved_translation"),
|
||||
status="translated",
|
||||
needs_review=False,
|
||||
)
|
||||
self.db.add(lang_entry)
|
||||
result["successful"] += 1
|
||||
|
||||
# Process rows needing LLM translation
|
||||
@@ -524,21 +630,31 @@ class TranslationExecutor:
|
||||
batch_id: str,
|
||||
) -> dict[str, int]:
|
||||
with belief_scope("TranslationExecutor._call_llm_for_batch"):
|
||||
# Build dictionary section
|
||||
# Build dictionary section using ContextAwarePromptBuilder
|
||||
dictionary_section = ""
|
||||
if dict_matches:
|
||||
# Get row_context from first batch row if available
|
||||
row_context = batch_rows[0].get("source_data") if batch_rows else None
|
||||
|
||||
# Use ContextAwarePromptBuilder for context-aware annotations
|
||||
annotated_entries = ContextAwarePromptBuilder.build_context_entries(
|
||||
dict_matches, row_context
|
||||
)
|
||||
glossary_lines = []
|
||||
for m in dict_matches:
|
||||
glossary_lines.append(
|
||||
f"- '{m['source_term']}' -> '{m['target_term']}'"
|
||||
f"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}"
|
||||
)
|
||||
for annotated_line in annotated_entries:
|
||||
glossary_lines.append(f"- {annotated_line}")
|
||||
dictionary_section = (
|
||||
"Terminology dictionary (use these translations when applicable):\n"
|
||||
+ "\n".join(glossary_lines)
|
||||
+ "\n\n"
|
||||
)
|
||||
|
||||
# Resolve target languages
|
||||
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
|
||||
if not isinstance(target_languages, list):
|
||||
target_languages = [str(target_languages)]
|
||||
target_languages_str = ", ".join(target_languages)
|
||||
|
||||
# Build rows JSON for LLM
|
||||
rows_json = json.dumps([
|
||||
{
|
||||
@@ -548,12 +664,13 @@ class TranslationExecutor:
|
||||
for idx, row in enumerate(batch_rows)
|
||||
], indent=2)
|
||||
|
||||
# Build prompt
|
||||
# Build prompt (use multi-language format)
|
||||
prompt = render_prompt(
|
||||
DEFAULT_EXECUTION_PROMPT_TEMPLATE,
|
||||
{
|
||||
"source_language": job.source_dialect or "SQL",
|
||||
"target_language": job.target_language or job.target_dialect or "en",
|
||||
"target_language": target_languages_str,
|
||||
"target_languages": target_languages_str,
|
||||
"source_dialect": job.source_dialect or "",
|
||||
"target_dialect": job.target_dialect or "",
|
||||
"translation_column": job.translation_column or "",
|
||||
@@ -607,9 +724,9 @@ class TranslationExecutor:
|
||||
self.db.add(record)
|
||||
return {"successful": 0, "failed": len(batch_rows), "skipped": 0, "retries": retries}
|
||||
|
||||
# Parse LLM response
|
||||
# Parse LLM response (multi-language aware)
|
||||
try:
|
||||
translations = self._parse_llm_response(llm_response, len(batch_rows))
|
||||
translations = self._parse_llm_response(llm_response, len(batch_rows), target_languages=target_languages)
|
||||
except ValueError as e:
|
||||
# Parse failure — mark all rows as SKIPPED
|
||||
logger.explore("LLM response parse failed", {
|
||||
@@ -646,16 +763,20 @@ class TranslationExecutor:
|
||||
|
||||
for row in batch_rows:
|
||||
row_id = str(row.get("row_index", ""))
|
||||
translation = translations.get(row_id)
|
||||
translation_data = translations.get(row_id)
|
||||
source_text = row.get("source_text", "")
|
||||
detected_lang = "und"
|
||||
if translation_data:
|
||||
detected_lang = translation_data.get("detected_source_language", "und")
|
||||
|
||||
if translation is None:
|
||||
if translation_data is None:
|
||||
# NULL translation — skip
|
||||
skipped += 1
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
source_sql=source_text,
|
||||
target_sql="",
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
@@ -667,14 +788,32 @@ class TranslationExecutor:
|
||||
self.db.add(record)
|
||||
continue
|
||||
|
||||
if translation.strip() == "":
|
||||
# Empty translation — skip
|
||||
# Collect per-language translated values
|
||||
per_lang_values: dict[str, str] = {}
|
||||
has_any_translation = False
|
||||
|
||||
# First try multi-language format (per-language keys)
|
||||
for lang_code in target_languages:
|
||||
lang_val = translation_data.get(lang_code)
|
||||
if lang_val is not None and str(lang_val).strip():
|
||||
per_lang_values[lang_code] = str(lang_val)
|
||||
has_any_translation = True
|
||||
|
||||
# Fallback to legacy single "translation" key format
|
||||
if not has_any_translation:
|
||||
translation_text = translation_data.get("translation", "")
|
||||
if translation_text.strip():
|
||||
per_lang_values[target_languages[0]] = translation_text
|
||||
has_any_translation = True
|
||||
|
||||
if not has_any_translation:
|
||||
# Empty/all-empty translations — skip
|
||||
skipped += 1
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
source_sql=source_text,
|
||||
target_sql="",
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
@@ -686,13 +825,16 @@ class TranslationExecutor:
|
||||
self.db.add(record)
|
||||
continue
|
||||
|
||||
# Use the first language's value as the primary target_sql for backward compat
|
||||
primary_translation = next(iter(per_lang_values.values()), "")
|
||||
|
||||
successful += 1
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
target_sql=translation,
|
||||
source_sql=source_text,
|
||||
target_sql=primary_translation,
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
@@ -701,6 +843,37 @@ class TranslationExecutor:
|
||||
)
|
||||
self.db.add(record)
|
||||
|
||||
# Create per-language entries — one TranslationLanguage per language_code
|
||||
for lang_code in target_languages:
|
||||
lang_translation = per_lang_values.get(lang_code, "")
|
||||
|
||||
# Source-as-reference: if detected source language matches this target,
|
||||
# store the original text verbatim (no translation needed)
|
||||
if detected_lang != "und" and str(lang_code).lower() == str(detected_lang).lower():
|
||||
lang_translation = source_text
|
||||
|
||||
# Check for undetermined source language
|
||||
lang_needs_review = (detected_lang == "und")
|
||||
|
||||
if lang_needs_review:
|
||||
logger.explore("undetected language", {
|
||||
"record_id": row_id,
|
||||
"language_code": lang_code,
|
||||
"source_text": source_text[:100],
|
||||
})
|
||||
|
||||
lang_entry = TranslationLanguage(
|
||||
id=str(uuid.uuid4()),
|
||||
record_id=record.id,
|
||||
language_code=lang_code,
|
||||
source_language_detected=detected_lang,
|
||||
translated_value=lang_translation or "",
|
||||
final_value=lang_translation or "",
|
||||
status="translated",
|
||||
needs_review=lang_needs_review,
|
||||
)
|
||||
self.db.add(lang_entry)
|
||||
|
||||
return {
|
||||
"successful": successful,
|
||||
"failed": failed,
|
||||
@@ -806,11 +979,13 @@ class TranslationExecutor:
|
||||
# endregion _call_openai_compatible
|
||||
|
||||
# region _parse_llm_response [TYPE Function]
|
||||
# @PURPOSE: Parse LLM JSON response into dict of row_id -> translation.
|
||||
# @PURPOSE: Parse LLM JSON response into dict of row_id -> per-language translations.
|
||||
# Supports multi-language format: {"row_id": 0, "detected_source_language": "fr", "ru": "текст", "en": "text"}
|
||||
# Backward-compat: old format {"row_id": 0, "translation": "text", "detected_source_language": "fr"}
|
||||
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
|
||||
# @POST: Returns dict mapping row_id to translation text.
|
||||
# @POST: Returns dict mapping row_id to dict with 'detected_source_language' and per-language codes.
|
||||
@staticmethod
|
||||
def _parse_llm_response(response_text: str, expected_count: int) -> dict[str, str]:
|
||||
def _parse_llm_response(response_text: str, expected_count: int, target_languages: list[str] | None = None) -> dict[str, dict[str, str]]:
|
||||
with belief_scope("TranslationExecutor._parse_llm_response"):
|
||||
try:
|
||||
data = json.loads(response_text)
|
||||
@@ -830,15 +1005,41 @@ class TranslationExecutor:
|
||||
if not isinstance(rows, list):
|
||||
raise ValueError("LLM response missing 'rows' array")
|
||||
|
||||
translations: dict[str, str] = {}
|
||||
translations: dict[str, dict[str, str]] = {}
|
||||
for item in rows:
|
||||
row_id = str(item.get("row_id", ""))
|
||||
translation = item.get("translation")
|
||||
if translation is None:
|
||||
# Skip NULL translations — they'll be handled by caller
|
||||
if not row_id:
|
||||
continue
|
||||
if row_id:
|
||||
translations[row_id] = str(translation)
|
||||
|
||||
detected_lang = str(item.get("detected_source_language", "und")) if item.get("detected_source_language") else "und"
|
||||
result: dict[str, str] = {
|
||||
"detected_source_language": detected_lang,
|
||||
}
|
||||
|
||||
# Multi-language format: extract per-language keys
|
||||
has_language_data = False
|
||||
if target_languages:
|
||||
for lang_code in target_languages:
|
||||
lang_val = item.get(lang_code)
|
||||
if lang_val is not None and str(lang_val).strip():
|
||||
result[lang_code] = str(lang_val)
|
||||
has_language_data = True
|
||||
|
||||
# Fallback to legacy single "translation" key format
|
||||
if not has_language_data:
|
||||
translation = item.get("translation")
|
||||
if translation is not None:
|
||||
result["translation"] = str(translation)
|
||||
has_language_data = True
|
||||
|
||||
if has_language_data:
|
||||
translations[row_id] = result
|
||||
|
||||
if len(translations) < expected_count:
|
||||
logger.explore(
|
||||
f"LLM returned fewer translations expected={expected_count} "
|
||||
f"got={len(translations)}"
|
||||
)
|
||||
|
||||
return translations
|
||||
# endregion _parse_llm_response
|
||||
|
||||
@@ -21,6 +21,7 @@ from ...models.translate import (
|
||||
MetricSnapshot,
|
||||
TranslationEvent,
|
||||
TranslationRun,
|
||||
TranslationRunLanguageStats,
|
||||
TranslationSchedule,
|
||||
)
|
||||
|
||||
@@ -121,17 +122,37 @@ class TranslationMetrics:
|
||||
# indicates cutoff
|
||||
pass
|
||||
|
||||
# Live events (<90 days) for token/cost
|
||||
cutoff = datetime.now(UTC)
|
||||
live_events = (
|
||||
self.db.query(TranslationEvent)
|
||||
.filter(
|
||||
TranslationEvent.job_id == job_id,
|
||||
TranslationEvent.event_type.in_(["TRANSLATION_PHASE_COMPLETED", "RUN_COMPLETED"]),
|
||||
TranslationEvent.created_at > cutoff, # events newer than snapshot
|
||||
)
|
||||
# Per-language metrics from TranslationRunLanguageStats (live runs)
|
||||
per_language: dict[str, dict[str, int | float]] = {}
|
||||
lang_stats_rows = (
|
||||
self.db.query(TranslationRunLanguageStats)
|
||||
.join(TranslationRun, TranslationRunLanguageStats.run_id == TranslationRun.id)
|
||||
.filter(TranslationRun.job_id == job_id)
|
||||
.all()
|
||||
)
|
||||
for ls in lang_stats_rows:
|
||||
lang = ls.language_code
|
||||
if lang not in per_language:
|
||||
per_language[lang] = {"tokens": 0, "cost": 0.0, "runs": 0, "translated_rows": 0}
|
||||
per_language[lang]["tokens"] += ls.token_count or 0
|
||||
per_language[lang]["cost"] += ls.estimated_cost or 0.0
|
||||
per_language[lang]["runs"] += 1
|
||||
per_language[lang]["translated_rows"] += ls.translated_rows or 0
|
||||
|
||||
# Merge in per-language data from MetricSnapshot (pruned period)
|
||||
latest_snapshot = (
|
||||
self.db.query(MetricSnapshot)
|
||||
.filter(MetricSnapshot.job_id == job_id)
|
||||
.order_by(MetricSnapshot.snapshot_date.desc())
|
||||
.first()
|
||||
)
|
||||
if latest_snapshot and latest_snapshot.per_language_metrics:
|
||||
for lang, snap_data in latest_snapshot.per_language_metrics.items():
|
||||
if lang not in per_language:
|
||||
per_language[lang] = {"tokens": 0, "cost": 0.0, "runs": 0, "translated_rows": 0}
|
||||
per_language[lang]["tokens"] += snap_data.get("cumulative_tokens", 0)
|
||||
per_language[lang]["cost"] += snap_data.get("cumulative_cost", 0.0)
|
||||
per_language[lang]["runs"] += snap_data.get("runs", 0)
|
||||
|
||||
return {
|
||||
"job_id": job_id,
|
||||
@@ -148,6 +169,7 @@ class TranslationMetrics:
|
||||
"avg_duration_ms": int(avg_duration) if avg_duration else None,
|
||||
"last_run_at": last_run.created_at.isoformat() if last_run else None,
|
||||
"next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None,
|
||||
"per_language_metrics": per_language,
|
||||
}
|
||||
# endregion get_job_metrics
|
||||
|
||||
|
||||
@@ -30,9 +30,11 @@ from ...core.logger import belief_scope, logger
|
||||
from ...models.translate import (
|
||||
TranslationBatch,
|
||||
TranslationJob,
|
||||
TranslationLanguage,
|
||||
TranslationPreviewSession,
|
||||
TranslationRecord,
|
||||
TranslationRun,
|
||||
TranslationRunLanguageStats,
|
||||
)
|
||||
from .events import TranslationEventLog
|
||||
from .executor import TranslationExecutor
|
||||
@@ -201,6 +203,27 @@ class TranslationOrchestrator:
|
||||
created_by=self.current_user,
|
||||
)
|
||||
|
||||
# Initialize per-language stats
|
||||
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
|
||||
if not isinstance(target_languages, list):
|
||||
target_languages = [str(target_languages)]
|
||||
language_stats_map: dict[str, TranslationRunLanguageStats] = {}
|
||||
for lang_code in target_languages:
|
||||
lang_stat = TranslationRunLanguageStats(
|
||||
id=str(uuid.uuid4()),
|
||||
run_id=run.id,
|
||||
language_code=lang_code,
|
||||
total_rows=0,
|
||||
translated_rows=0,
|
||||
failed_rows=0,
|
||||
skipped_rows=0,
|
||||
token_count=0,
|
||||
estimated_cost=0.0,
|
||||
)
|
||||
self.db.add(lang_stat)
|
||||
language_stats_map[lang_code] = lang_stat
|
||||
self.db.flush()
|
||||
|
||||
# Dispatch executor
|
||||
executor = TranslationExecutor(
|
||||
self.db, self.config_manager, self.current_user,
|
||||
@@ -227,6 +250,9 @@ class TranslationOrchestrator:
|
||||
self.db.commit()
|
||||
return run
|
||||
|
||||
# Aggregate per-language statistics after executor completes
|
||||
self._update_language_stats(run.id, language_stats_map)
|
||||
|
||||
# Record translation phase complete
|
||||
self.event_log.log_event(
|
||||
job_id=job.id,
|
||||
@@ -745,6 +771,25 @@ class TranslationOrchestrator:
|
||||
# Get event summary
|
||||
event_summary = self.event_log.get_run_event_summary(run_id)
|
||||
|
||||
# Get language stats
|
||||
language_stats_entries = (
|
||||
self.db.query(TranslationRunLanguageStats)
|
||||
.filter(TranslationRunLanguageStats.run_id == run_id)
|
||||
.all()
|
||||
)
|
||||
language_stats = [
|
||||
{
|
||||
"language_code": ls.language_code,
|
||||
"total_rows": ls.total_rows or 0,
|
||||
"translated_rows": ls.translated_rows or 0,
|
||||
"failed_rows": ls.failed_rows or 0,
|
||||
"skipped_rows": ls.skipped_rows or 0,
|
||||
"token_count": ls.token_count or 0,
|
||||
"estimated_cost": ls.estimated_cost or 0.0,
|
||||
}
|
||||
for ls in language_stats_entries
|
||||
]
|
||||
|
||||
return {
|
||||
"id": run.id,
|
||||
"job_id": run.job_id,
|
||||
@@ -759,6 +804,7 @@ class TranslationOrchestrator:
|
||||
"insert_status": run.insert_status,
|
||||
"superset_execution_id": run.superset_execution_id,
|
||||
"batch_count": batch_count,
|
||||
"language_stats": language_stats,
|
||||
"event_invariants": {
|
||||
"has_run_started": event_summary["has_run_started"],
|
||||
"terminal_event_count": event_summary["terminal_event_count"],
|
||||
@@ -863,6 +909,80 @@ class TranslationOrchestrator:
|
||||
]
|
||||
# endregion get_run_history
|
||||
|
||||
# region _update_language_stats [TYPE Function]
|
||||
# @PURPOSE: Aggregate TranslationLanguage entries by language_code and update TranslationRunLanguageStats.
|
||||
# @PRE: run_id and language_stats_map are valid. DB session is available.
|
||||
# @POST: Language stats are updated with row counts and estimated tokens/cost.
|
||||
# @SIDE_EFFECT: DB writes on language_stats objects.
|
||||
def _update_language_stats(
|
||||
self,
|
||||
run_id: str,
|
||||
language_stats_map: dict[str, TranslationRunLanguageStats],
|
||||
) -> None:
|
||||
with belief_scope("TranslationOrchestrator._update_language_stats"):
|
||||
# Get all records for this run to join with TranslationLanguage
|
||||
records = (
|
||||
self.db.query(TranslationRecord)
|
||||
.filter(TranslationRecord.run_id == run_id)
|
||||
.all()
|
||||
)
|
||||
record_ids = [r.id for r in records]
|
||||
|
||||
if not record_ids:
|
||||
logger.reason("No records for language stats aggregation", {"run_id": run_id})
|
||||
return
|
||||
|
||||
# Get all language entries for this run's records
|
||||
lang_entries = (
|
||||
self.db.query(TranslationLanguage)
|
||||
.filter(TranslationLanguage.record_id.in_(record_ids))
|
||||
.all()
|
||||
)
|
||||
|
||||
# Aggregate by language_code
|
||||
from collections import defaultdict
|
||||
agg: dict[str, dict[str, int]] = defaultdict(lambda: {"total": 0, "translated": 0, "failed": 0, "skipped": 0})
|
||||
|
||||
for le in lang_entries:
|
||||
code = le.language_code
|
||||
agg[code]["total"] += 1
|
||||
if le.status in ("translated", "approved", "edited"):
|
||||
agg[code]["translated"] += 1
|
||||
elif le.status == "failed":
|
||||
agg[code]["failed"] += 1
|
||||
elif le.status == "skipped":
|
||||
agg[code]["skipped"] += 1
|
||||
|
||||
# Estimate tokens: heuristic based on character count of translated values
|
||||
total_chars = sum(
|
||||
len(le.translated_value or "") for le in lang_entries if le.translated_value
|
||||
)
|
||||
total_tokens = max(1, total_chars // 4) # ~4 chars per token
|
||||
cost_per_token = 0.002 / 1000 # $0.002 per 1K tokens
|
||||
|
||||
# Update each language stat entry
|
||||
for lang_code, lang_stat in language_stats_map.items():
|
||||
data = agg.get(lang_code, {"total": 0, "translated": 0, "failed": 0, "skipped": 0})
|
||||
lang_stat.total_rows = data["total"]
|
||||
lang_stat.translated_rows = data["translated"]
|
||||
lang_stat.failed_rows = data["failed"]
|
||||
lang_stat.skipped_rows = data["skipped"]
|
||||
|
||||
# Proportional token split: share tokens across languages
|
||||
num_langs = len(language_stats_map)
|
||||
if num_langs > 0:
|
||||
lang_stat.token_count = total_tokens // num_langs
|
||||
lang_stat.estimated_cost = round((lang_stat.token_count / 1000) * cost_per_token, 6)
|
||||
|
||||
self.db.flush()
|
||||
|
||||
logger.reason("Language stats updated", {
|
||||
"run_id": run_id,
|
||||
"languages": list(language_stats_map.keys()),
|
||||
"total_tokens_est": total_tokens,
|
||||
})
|
||||
# endregion _update_language_stats
|
||||
|
||||
# region _compute_config_hash [TYPE Function]
|
||||
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
|
||||
@staticmethod
|
||||
|
||||
@@ -29,6 +29,7 @@ from ...core.superset_client import SupersetClient
|
||||
from ...models.translate import (
|
||||
TranslationJob,
|
||||
TranslationJobDictionary,
|
||||
TranslationPreviewLanguage,
|
||||
TranslationPreviewRecord,
|
||||
TranslationPreviewSession,
|
||||
)
|
||||
@@ -38,16 +39,19 @@ from .dictionary import DictionaryManager
|
||||
|
||||
# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [TYPE Constant]
|
||||
# @BRIEF Default prompt template for batch LLM translation execution (no context columns — faster).
|
||||
# Supports both single-language and multi-language modes via {target_languages} placeholder.
|
||||
DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
|
||||
"Translate the following database content from {source_language} to {target_language}.\n\n"
|
||||
"Translate the following database content from {source_language} to the following language(s): {target_languages}.\n\n"
|
||||
"Source dialect: {source_dialect}\n"
|
||||
"Target dialect: {target_dialect}\n"
|
||||
"Target dialect(s): {target_dialect}\n"
|
||||
"Column to translate: {translation_column}\n\n"
|
||||
"{dictionary_section}"
|
||||
"For each row, provide an accurate translation of the text.\n\n"
|
||||
"For each row, provide an accurate translation of the text into each target language.\n\n"
|
||||
"Rows to translate:\n{rows_json}\n\n"
|
||||
"Respond with a JSON object in this exact format:\n"
|
||||
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
|
||||
'{{"rows": [{{"row_id": "<row_index>", "detected_source_language": "<bcp47_or_und>", "<language_code_1>": "<translation_in_lang_1>", "<language_code_2>": "<translation_in_lang_2>"}}]}}\n'
|
||||
"For each row, return the detected source language as a BCP-47 tag (e.g. 'en', 'ru', 'fr'), or 'und' if uncertain.\n"
|
||||
"Include a separate key for EACH target language code with the translated text in that language.\n"
|
||||
"Each row_id must match the index provided. Return exactly {row_count} entries."
|
||||
)
|
||||
# #endregion DEFAULT_EXECUTION_PROMPT_TEMPLATE
|
||||
@@ -56,16 +60,18 @@ DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
|
||||
# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [TYPE Constant]
|
||||
# @BRIEF Default prompt template for LLM translation preview.
|
||||
DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
|
||||
"Translate the following database content from {source_language} to {target_language}.\n\n"
|
||||
"Translate the following database content.\n\n"
|
||||
"Source dialect: {source_dialect}\n"
|
||||
"Target dialect: {target_dialect}\n"
|
||||
"Column to translate: {translation_column}\n\n"
|
||||
"{dictionary_section}"
|
||||
"For each row, provide an accurate translation of the '{translation_column}' value.\n"
|
||||
"Translate to the following languages: {target_languages}\n\n"
|
||||
"For each row, provide an accurate translation of the '{translation_column}' value into each language.\n"
|
||||
"Consider the context columns when determining the meaning of the text.\n\n"
|
||||
"Rows to translate:\n{rows_json}\n\n"
|
||||
"Respond with a JSON object in this exact format:\n"
|
||||
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
|
||||
'{{"rows": [{{"row_id": "<row_index>", "detected_source_language": "<bcp47_or_und>", "language_code_1": "<translation_in_lang_1>", "language_code_2": "<translation_in_lang_2>"}}]}}\n'
|
||||
"For each row, return the detected source language as a BCP-47 tag (e.g. 'en', 'ru', 'fr'), or 'und' if uncertain.\n"
|
||||
"Include a separate key for EACH target language code with the translated text in that language.\n"
|
||||
"Each row_id must match the index provided. Return exactly {row_count} entries."
|
||||
)
|
||||
# #endregion DEFAULT_PREVIEW_PROMPT_TEMPLATE
|
||||
@@ -80,7 +86,9 @@ class TokenEstimator:
|
||||
|
||||
CHARS_PER_TOKEN_ESTIMATE: float = 4.0
|
||||
OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50
|
||||
MULTI_LANG_FACTOR: float = 1.2 # Overhead for multi-language in one call
|
||||
TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens
|
||||
COST_WARNING_THRESHOLD: int = 30 # Show warning above this sample size
|
||||
|
||||
# region estimate_prompt_tokens [TYPE Function]
|
||||
# @PURPOSE: Estimate token count for a prompt string.
|
||||
@@ -94,12 +102,12 @@ class TokenEstimator:
|
||||
# endregion estimate_prompt_tokens
|
||||
|
||||
# region estimate_output_tokens [TYPE Function]
|
||||
# @PURPOSE: Estimate output token count for translating N rows.
|
||||
# @PRE: row_count >= 0.
|
||||
# @PURPOSE: Estimate output token count for translating N rows across N languages.
|
||||
# @PRE: row_count >= 0, num_languages >= 1.
|
||||
# @POST: Returns estimated output token count.
|
||||
@staticmethod
|
||||
def estimate_output_tokens(row_count: int) -> int:
|
||||
return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE
|
||||
def estimate_output_tokens(row_count: int, num_languages: int = 1) -> int:
|
||||
return int(row_count * num_languages * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE * TokenEstimator.MULTI_LANG_FACTOR)
|
||||
# endregion estimate_output_tokens
|
||||
|
||||
# region estimate_cost [TYPE Function]
|
||||
@@ -112,6 +120,20 @@ class TokenEstimator:
|
||||
return round((total_tokens / 1000) * rate, 6)
|
||||
# endregion estimate_cost
|
||||
|
||||
# region check_cost_warning [TYPE Function]
|
||||
# @PURPOSE: Generate cost warning for large previews.
|
||||
# @PRE: sample_size > 0, num_languages >= 1.
|
||||
# @POST: Returns warning string or None.
|
||||
@staticmethod
|
||||
def check_cost_warning(sample_size: int, num_languages: int, estimated_tokens: int, estimated_cost: float) -> str | None:
|
||||
if sample_size > TokenEstimator.COST_WARNING_THRESHOLD:
|
||||
return (
|
||||
f"Large preview — estimated {estimated_tokens} tokens, ~${estimated_cost:.4f} cost "
|
||||
f"(across {num_languages} languages)"
|
||||
)
|
||||
return None
|
||||
# endregion check_cost_warning
|
||||
|
||||
|
||||
# #endregion TokenEstimator
|
||||
|
||||
@@ -134,10 +156,10 @@ class TranslationPreview:
|
||||
self.current_user = current_user
|
||||
|
||||
# region preview_rows [TYPE Function]
|
||||
# @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.
|
||||
# @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for multi-language translation, create preview session with per-language records.
|
||||
# @PRE: job_id exists and job has source_datasource_id, translation_column configured.
|
||||
# @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session.
|
||||
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.
|
||||
# @POST: Returns TranslationPreviewResponse with per-language records, cost estimation, and persistent session.
|
||||
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord + TranslationPreviewLanguage rows.
|
||||
def preview_rows(
|
||||
self,
|
||||
job_id: str,
|
||||
@@ -157,6 +179,12 @@ class TranslationPreview:
|
||||
if not job.translation_column:
|
||||
raise ValueError("Job must have a translation column configured for preview")
|
||||
|
||||
# Resolve target languages: prefer target_languages list, fallback to single target_language
|
||||
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"]
|
||||
if not isinstance(target_languages, list):
|
||||
target_languages = [str(target_languages)]
|
||||
num_languages = len(target_languages)
|
||||
|
||||
# 2. Compute config hash and dict snapshot hash
|
||||
config_hash = self._compute_config_hash(job)
|
||||
dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id)
|
||||
@@ -224,40 +252,48 @@ class TranslationPreview:
|
||||
+ "\n\n"
|
||||
)
|
||||
|
||||
# 6. Build LLM prompt
|
||||
# 6. Build LLM prompt with multi-language instructions
|
||||
rows_json = json.dumps([
|
||||
{"row_id": str(m["row_index"]), "text": m["source_text"], "context": m["context_data"]}
|
||||
for m in row_meta
|
||||
], indent=2)
|
||||
|
||||
target_languages_str = ", ".join(target_languages)
|
||||
template = prompt_template or DEFAULT_PREVIEW_PROMPT_TEMPLATE
|
||||
prompt = render_prompt(template, {
|
||||
"source_language": job.source_dialect or "SQL",
|
||||
"target_language": job.target_language or job.target_dialect or "en",
|
||||
"target_language": target_languages_str,
|
||||
"source_dialect": job.source_dialect or "",
|
||||
"target_dialect": job.target_dialect or "",
|
||||
"target_languages": target_languages_str,
|
||||
"translation_column": job.translation_column or "",
|
||||
"dictionary_section": dictionary_section,
|
||||
"rows_json": rows_json,
|
||||
"row_count": str(actual_row_count),
|
||||
})
|
||||
|
||||
# 7. Estimate tokens/cost for sample and full dataset
|
||||
# 7. Estimate tokens/cost for sample with multi-language factor
|
||||
sample_prompt_tokens = TokenEstimator.estimate_prompt_tokens(prompt)
|
||||
sample_output_tokens = TokenEstimator.estimate_output_tokens(actual_row_count)
|
||||
sample_output_tokens = TokenEstimator.estimate_output_tokens(actual_row_count, num_languages)
|
||||
sample_total_tokens = sample_prompt_tokens + sample_output_tokens
|
||||
sample_cost = TokenEstimator.estimate_cost(sample_total_tokens)
|
||||
|
||||
# Estimate full dataset cost (if we knew total rows)
|
||||
# Estimate full dataset cost (extrapolated)
|
||||
total_est_rows = sample_size * 10 # rough extrapolation
|
||||
total_est_tokens = TokenEstimator.estimate_prompt_tokens(
|
||||
prompt.replace(str(actual_row_count), "{total}")
|
||||
) + TokenEstimator.estimate_output_tokens(sample_size * 10) # rough extrapolation
|
||||
prompt.replace(str(actual_row_count), str(total_est_rows))
|
||||
) + TokenEstimator.estimate_output_tokens(total_est_rows, num_languages)
|
||||
total_est_cost = TokenEstimator.estimate_cost(total_est_tokens)
|
||||
|
||||
# Cost warning for large previews
|
||||
cost_warning = TokenEstimator.check_cost_warning(
|
||||
sample_size, num_languages, sample_total_tokens, sample_cost
|
||||
)
|
||||
|
||||
# 8. Call LLM
|
||||
logger.reason("Calling LLM for preview translation", {
|
||||
"provider_id": job.provider_id,
|
||||
"row_count": actual_row_count,
|
||||
"num_languages": num_languages,
|
||||
"estimated_tokens": sample_total_tokens,
|
||||
})
|
||||
llm_response = self._call_llm(
|
||||
@@ -265,8 +301,10 @@ class TranslationPreview:
|
||||
prompt=prompt,
|
||||
)
|
||||
|
||||
# 9. Parse LLM response
|
||||
translations = self._parse_llm_response(llm_response, actual_row_count)
|
||||
# 9. Parse LLM response (multi-language)
|
||||
translations = self._parse_llm_response(
|
||||
llm_response, actual_row_count, target_languages=target_languages
|
||||
)
|
||||
|
||||
# 10. Create preview session
|
||||
session = TranslationPreviewSession(
|
||||
@@ -280,14 +318,50 @@ class TranslationPreview:
|
||||
self.db.add(session)
|
||||
self.db.flush()
|
||||
|
||||
# 11. Create preview records
|
||||
# 11. Create preview records with per-language entries
|
||||
records = []
|
||||
for meta in row_meta:
|
||||
idx = meta["row_index"]
|
||||
translation = translations.get(str(idx), "")
|
||||
is_rejected = False
|
||||
status = "PENDING"
|
||||
feedback = None
|
||||
source_text = meta["source_text"]
|
||||
translation_data = translations.get(str(idx), {})
|
||||
detected_lang = translation_data.get("detected_source_language", "und") if isinstance(translation_data, dict) else "und"
|
||||
|
||||
# Extract per-language translations
|
||||
lang_entries: list[TranslationPreviewLanguage] = []
|
||||
for lang_code in target_languages:
|
||||
# Get translation for this language from LLM response
|
||||
lang_translation = translation_data.get(lang_code) if isinstance(translation_data, dict) else None
|
||||
|
||||
# Fallback: if no per-language data, use the single "translation" key
|
||||
if not lang_translation:
|
||||
if isinstance(translation_data, dict):
|
||||
lang_translation = translation_data.get("translation", "")
|
||||
elif isinstance(translation_data, str):
|
||||
lang_translation = translation_data
|
||||
else:
|
||||
lang_translation = ""
|
||||
|
||||
# Source-as-reference: if this language code matches source, use original text
|
||||
if str(lang_code).lower() == str(detected_lang).lower() and detected_lang != "und":
|
||||
# Use source text as reference (no translation needed for source language)
|
||||
lang_translation = source_text
|
||||
|
||||
lang_needs_review = detected_lang == "und"
|
||||
|
||||
lang_entry = TranslationPreviewLanguage(
|
||||
id=str(uuid.uuid4()),
|
||||
preview_record_id="", # Will set after record is created
|
||||
language_code=lang_code,
|
||||
source_language_detected=detected_lang,
|
||||
translated_value=str(lang_translation),
|
||||
final_value=str(lang_translation),
|
||||
status="pending",
|
||||
needs_review=lang_needs_review,
|
||||
)
|
||||
lang_entries.append(lang_entry)
|
||||
|
||||
# Determine overall status and needs_review for the record
|
||||
overall_needs_review = any(le.needs_review for le in lang_entries)
|
||||
|
||||
# Extract source_data: store original row key columns for upsert matching
|
||||
source_row = meta.get("source_row", {})
|
||||
@@ -299,24 +373,41 @@ class TranslationPreview:
|
||||
if k in source_row
|
||||
}
|
||||
elif source_row:
|
||||
# No key columns configured — store the full row as fallback
|
||||
source_data = dict(source_row)
|
||||
|
||||
# Use first language's translation as target_sql for backward compat
|
||||
primary_translation = lang_entries[0].translated_value if lang_entries else ""
|
||||
|
||||
record = TranslationPreviewRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
session_id=session.id,
|
||||
source_sql=meta["source_text"],
|
||||
target_sql=translation,
|
||||
source_sql=source_text,
|
||||
target_sql=primary_translation,
|
||||
source_object_type="table_row",
|
||||
source_object_id=str(idx),
|
||||
source_object_name=f"Row {idx + 1}",
|
||||
source_data=source_data,
|
||||
status=status,
|
||||
feedback=feedback,
|
||||
status="PENDING",
|
||||
feedback=None,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
self.db.add(record)
|
||||
self.db.flush()
|
||||
|
||||
# Now set the preview_record_id for all lang entries and persist
|
||||
serialized_langs = []
|
||||
for le in lang_entries:
|
||||
le.preview_record_id = record.id
|
||||
self.db.add(le)
|
||||
serialized_langs.append({
|
||||
"language_code": le.language_code,
|
||||
"source_language_detected": le.source_language_detected,
|
||||
"translated_value": le.translated_value,
|
||||
"final_value": le.final_value,
|
||||
"status": le.status,
|
||||
"needs_review": le.needs_review,
|
||||
})
|
||||
|
||||
records.append({
|
||||
"id": record.id,
|
||||
"source_sql": record.source_sql,
|
||||
@@ -326,6 +417,9 @@ class TranslationPreview:
|
||||
"source_object_name": record.source_object_name,
|
||||
"status": record.status,
|
||||
"feedback": record.feedback,
|
||||
"source_language_detected": detected_lang,
|
||||
"needs_review": overall_needs_review,
|
||||
"languages": serialized_langs,
|
||||
})
|
||||
|
||||
self.db.commit()
|
||||
@@ -338,15 +432,18 @@ class TranslationPreview:
|
||||
"created_at": session.created_at.isoformat(),
|
||||
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
|
||||
"records": records,
|
||||
"target_languages": target_languages,
|
||||
"cost_estimate": {
|
||||
"sample_size": actual_row_count,
|
||||
"num_languages": num_languages,
|
||||
"sample_prompt_tokens": sample_prompt_tokens,
|
||||
"sample_output_tokens": sample_output_tokens,
|
||||
"sample_total_tokens": sample_total_tokens,
|
||||
"sample_cost": sample_cost,
|
||||
"estimated_total_rows": actual_row_count * 10,
|
||||
"estimated_total_rows": total_est_rows,
|
||||
"estimated_tokens": total_est_tokens,
|
||||
"estimated_cost": total_est_cost,
|
||||
"warning": cost_warning,
|
||||
},
|
||||
"config_hash": config_hash,
|
||||
"dict_snapshot_hash": dict_snapshot_hash,
|
||||
@@ -355,15 +452,16 @@ class TranslationPreview:
|
||||
logger.reflect("Preview completed", {
|
||||
"session_id": session.id,
|
||||
"row_count": actual_row_count,
|
||||
"num_languages": num_languages,
|
||||
"sample_cost": sample_cost,
|
||||
})
|
||||
return result
|
||||
# endregion preview_rows
|
||||
|
||||
# region update_preview_row [TYPE Function]
|
||||
# @PURPOSE: Approve, edit, or reject an individual preview row.
|
||||
# @PURPOSE: Approve, edit, or reject an individual preview row (optionally per language).
|
||||
# @PRE: session_id and row_id exist, session is ACTIVE.
|
||||
# @POST: PreviewRecord status is updated.
|
||||
# @POST: PreviewRecord status is updated. If language_code provided, only that TranslationPreviewLanguage is updated.
|
||||
def update_preview_row(
|
||||
self,
|
||||
job_id: str,
|
||||
@@ -371,6 +469,7 @@ class TranslationPreview:
|
||||
action: str,
|
||||
translation: str | None = None,
|
||||
feedback: str | None = None,
|
||||
language_code: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
with belief_scope("TranslationPreview.update_preview_row"):
|
||||
# Find the active session for this job
|
||||
@@ -397,16 +496,76 @@ class TranslationPreview:
|
||||
if not record:
|
||||
raise ValueError(f"Preview record '{row_id}' not found in active session")
|
||||
|
||||
if action == "approve":
|
||||
record.status = "APPROVED"
|
||||
elif action == "reject":
|
||||
record.status = "REJECTED"
|
||||
elif action == "edit":
|
||||
record.status = "APPROVED"
|
||||
if translation is not None:
|
||||
record.target_sql = translation
|
||||
# If language_code specified, operate on the specific TranslationPreviewLanguage
|
||||
if language_code:
|
||||
lang_entry = (
|
||||
self.db.query(TranslationPreviewLanguage)
|
||||
.filter(
|
||||
TranslationPreviewLanguage.preview_record_id == row_id,
|
||||
TranslationPreviewLanguage.language_code == language_code,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not lang_entry:
|
||||
raise ValueError(f"Language entry '{language_code}' not found for record '{row_id}'")
|
||||
|
||||
if action == "approve":
|
||||
lang_entry.status = "approved"
|
||||
elif action == "reject":
|
||||
lang_entry.status = "rejected"
|
||||
elif action == "edit":
|
||||
lang_entry.status = "edited"
|
||||
if translation is not None:
|
||||
lang_entry.translated_value = translation
|
||||
lang_entry.final_value = translation
|
||||
lang_entry.user_edit = translation
|
||||
else:
|
||||
raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.")
|
||||
|
||||
# Update record-level status based on all language entries
|
||||
all_langs = (
|
||||
self.db.query(TranslationPreviewLanguage)
|
||||
.filter(TranslationPreviewLanguage.preview_record_id == row_id)
|
||||
.all()
|
||||
)
|
||||
all_approved = all(le.status in ("approved", "edited") for le in all_langs)
|
||||
any_rejected = any(le.status == "rejected" for le in all_langs)
|
||||
|
||||
if all_approved:
|
||||
record.status = "APPROVED"
|
||||
elif any_rejected and not all_langs:
|
||||
record.status = "REJECTED"
|
||||
|
||||
# Update target_sql to reflect primary (first) language edit
|
||||
if action == "edit" and translation is not None and all_langs:
|
||||
record.target_sql = all_langs[0].final_value or all_langs[0].translated_value
|
||||
|
||||
else:
|
||||
raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.")
|
||||
# Legacy behavior: operate on whole record (all languages)
|
||||
if action == "approve":
|
||||
record.status = "APPROVED"
|
||||
# Approve all pending language entries
|
||||
for lang_entry in record.languages:
|
||||
if lang_entry.status == "pending":
|
||||
lang_entry.status = "approved"
|
||||
elif action == "reject":
|
||||
record.status = "REJECTED"
|
||||
for lang_entry in record.languages:
|
||||
if lang_entry.status == "pending":
|
||||
lang_entry.status = "rejected"
|
||||
elif action == "edit":
|
||||
record.status = "APPROVED"
|
||||
if translation is not None:
|
||||
record.target_sql = translation
|
||||
# Update primary (first) language entry
|
||||
if record.languages:
|
||||
first_lang = record.languages[0]
|
||||
first_lang.translated_value = translation
|
||||
first_lang.final_value = translation
|
||||
first_lang.user_edit = translation
|
||||
first_lang.status = "edited"
|
||||
else:
|
||||
raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.")
|
||||
|
||||
if feedback is not None:
|
||||
record.feedback = feedback
|
||||
@@ -418,21 +577,37 @@ class TranslationPreview:
|
||||
"row_id": row_id,
|
||||
"session_id": session.id,
|
||||
"status": record.status,
|
||||
"language_code": language_code,
|
||||
})
|
||||
|
||||
# Build response with per-language data
|
||||
lang_responses = []
|
||||
if record.languages:
|
||||
for le in record.languages:
|
||||
lang_responses.append({
|
||||
"language_code": le.language_code,
|
||||
"source_language_detected": le.source_language_detected,
|
||||
"translated_value": le.translated_value,
|
||||
"final_value": le.final_value,
|
||||
"user_edit": le.user_edit,
|
||||
"status": le.status,
|
||||
"needs_review": le.needs_review,
|
||||
})
|
||||
|
||||
return {
|
||||
"id": record.id,
|
||||
"source_sql": record.source_sql,
|
||||
"target_sql": record.target_sql,
|
||||
"status": record.status,
|
||||
"feedback": record.feedback,
|
||||
"languages": lang_responses,
|
||||
}
|
||||
# endregion update_preview_row
|
||||
|
||||
# region accept_preview_session [TYPE Function]
|
||||
# @PURPOSE: Mark a preview session as accepted, which gates full execution.
|
||||
# @PRE: job_id has an ACTIVE preview session.
|
||||
# @POST: Session status changes to APPLIED.
|
||||
# @POST: Session status changes to APPLIED. User edits are persisted for carry-forward.
|
||||
# @SIDE_EFFECT: Future full execution calls will check for accepted session.
|
||||
def accept_preview_session(self, job_id: str) -> dict[str, Any]:
|
||||
with belief_scope("TranslationPreview.accept_preview_session"):
|
||||
@@ -464,6 +639,12 @@ class TranslationPreview:
|
||||
.all()
|
||||
)
|
||||
|
||||
# Resolve target languages
|
||||
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
||||
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] if job else ["en"]
|
||||
if not isinstance(target_languages, list):
|
||||
target_languages = [str(target_languages)]
|
||||
|
||||
return {
|
||||
"id": session.id,
|
||||
"job_id": job_id,
|
||||
@@ -471,6 +652,7 @@ class TranslationPreview:
|
||||
"created_by": session.created_by,
|
||||
"created_at": session.created_at.isoformat(),
|
||||
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
|
||||
"target_languages": target_languages,
|
||||
"records": [
|
||||
{
|
||||
"id": r.id,
|
||||
@@ -478,6 +660,28 @@ class TranslationPreview:
|
||||
"target_sql": r.target_sql,
|
||||
"status": r.status,
|
||||
"feedback": r.feedback,
|
||||
"source_language_detected": (
|
||||
r.languages[0].source_language_detected
|
||||
if r.languages and r.languages[0].source_language_detected
|
||||
else None
|
||||
),
|
||||
"needs_review": (
|
||||
r.languages[0].needs_review
|
||||
if r.languages
|
||||
else False
|
||||
),
|
||||
"languages": [
|
||||
{
|
||||
"language_code": le.language_code,
|
||||
"source_language_detected": le.source_language_detected,
|
||||
"translated_value": le.translated_value,
|
||||
"final_value": le.final_value,
|
||||
"user_edit": le.user_edit,
|
||||
"status": le.status,
|
||||
"needs_review": le.needs_review,
|
||||
}
|
||||
for le in (r.languages or [])
|
||||
] if r.languages else [],
|
||||
}
|
||||
for r in records
|
||||
],
|
||||
@@ -487,7 +691,7 @@ class TranslationPreview:
|
||||
# region get_preview_session [TYPE Function]
|
||||
# @PURPOSE: Get the latest preview session for a job with its records.
|
||||
# @PRE: job_id exists.
|
||||
# @POST: Returns session data with records or raises ValueError.
|
||||
# @POST: Returns session data with per-language records or raises ValueError.
|
||||
def get_preview_session(self, job_id: str) -> dict[str, Any]:
|
||||
with belief_scope("TranslationPreview.get_preview_session"):
|
||||
session = (
|
||||
@@ -505,6 +709,12 @@ class TranslationPreview:
|
||||
.all()
|
||||
)
|
||||
|
||||
# Resolve target languages
|
||||
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
||||
target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] if job else ["en"]
|
||||
if not isinstance(target_languages, list):
|
||||
target_languages = [str(target_languages)]
|
||||
|
||||
return {
|
||||
"id": session.id,
|
||||
"job_id": job_id,
|
||||
@@ -512,6 +722,7 @@ class TranslationPreview:
|
||||
"created_by": session.created_by,
|
||||
"created_at": session.created_at.isoformat(),
|
||||
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
|
||||
"target_languages": target_languages,
|
||||
"records": [
|
||||
{
|
||||
"id": r.id,
|
||||
@@ -522,6 +733,28 @@ class TranslationPreview:
|
||||
"source_object_name": r.source_object_name,
|
||||
"status": r.status,
|
||||
"feedback": r.feedback,
|
||||
"source_language_detected": (
|
||||
r.languages[0].source_language_detected
|
||||
if r.languages and r.languages[0].source_language_detected
|
||||
else None
|
||||
),
|
||||
"needs_review": (
|
||||
r.languages[0].needs_review
|
||||
if r.languages
|
||||
else False
|
||||
),
|
||||
"languages": [
|
||||
{
|
||||
"language_code": le.language_code,
|
||||
"source_language_detected": le.source_language_detected,
|
||||
"translated_value": le.translated_value,
|
||||
"final_value": le.final_value,
|
||||
"user_edit": le.user_edit,
|
||||
"status": le.status,
|
||||
"needs_review": le.needs_review,
|
||||
}
|
||||
for le in (r.languages or [])
|
||||
] if r.languages else [],
|
||||
}
|
||||
for r in records
|
||||
],
|
||||
@@ -752,11 +985,11 @@ class TranslationPreview:
|
||||
# endregion _call_openai_compatible
|
||||
|
||||
# region _parse_llm_response [TYPE Function]
|
||||
# @PURPOSE: Parse the LLM JSON response into a dict of row_id -> translation.
|
||||
# @PURPOSE: Parse the LLM JSON response into a dict of row_id -> per-language translations.
|
||||
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
|
||||
# @POST: Returns dict mapping string row_id to translation text.
|
||||
# @POST: Returns dict mapping row_id to dict with 'detected_source_language' and per-language keys.
|
||||
@staticmethod
|
||||
def _parse_llm_response(response_text: str, expected_count: int) -> dict[str, str]:
|
||||
def _parse_llm_response(response_text: str, expected_count: int, target_languages: list[str] | None = None) -> dict[str, dict[str, str]]:
|
||||
with belief_scope("TranslationPreview._parse_llm_response"):
|
||||
logger.reason(f"Raw LLM response length={len(response_text)} preview={response_text[:500]}")
|
||||
|
||||
@@ -779,12 +1012,37 @@ class TranslationPreview:
|
||||
logger.explore(f"LLM response has no 'rows' array, keys={list(data.keys())} text_preview={response_text[:300]}")
|
||||
raise ValueError("LLM response missing 'rows' array")
|
||||
|
||||
translations: dict[str, str] = {}
|
||||
translations: dict[str, dict[str, str]] = {}
|
||||
for item in rows:
|
||||
row_id = str(item.get("row_id", ""))
|
||||
translation = str(item.get("translation", ""))
|
||||
if row_id:
|
||||
translations[row_id] = translation
|
||||
if not row_id:
|
||||
continue
|
||||
|
||||
detected_lang = str(item.get("detected_source_language", "und")) if item.get("detected_source_language") else "und"
|
||||
result: dict[str, str] = {
|
||||
"detected_source_language": detected_lang,
|
||||
}
|
||||
|
||||
# Multi-language format: extract per-language keys
|
||||
# Keys that are NOT row_id, detected_source_language, or translation are language codes
|
||||
has_language_data = False
|
||||
if target_languages:
|
||||
for lang_code in target_languages:
|
||||
lang_val = item.get(lang_code)
|
||||
if lang_val is not None and str(lang_val).strip():
|
||||
result[lang_code] = str(lang_val)
|
||||
has_language_data = True
|
||||
|
||||
# Fallback to old format (single "translation" key)
|
||||
if not has_language_data:
|
||||
translation = item.get("translation")
|
||||
if translation is not None:
|
||||
result["translation"] = str(translation)
|
||||
else:
|
||||
# Skip rows with no translation data at all
|
||||
continue
|
||||
|
||||
translations[row_id] = result
|
||||
|
||||
if len(translations) < expected_count:
|
||||
logger.explore(
|
||||
|
||||
152
backend/src/plugins/translate/prompt_builder.py
Normal file
152
backend/src/plugins/translate/prompt_builder.py
Normal file
@@ -0,0 +1,152 @@
|
||||
# #region ContextAwarePromptBuilder [C:2] [TYPE Module] [SEMANTICS translate, prompt, context, dictionary]
|
||||
# @BRIEF Pure-function prompt builder that enhances dictionary entries with context annotations.
|
||||
# @LAYER: Domain
|
||||
# @RELATION DEPENDS_ON -> [DictionaryEntry:Class]
|
||||
# @RATIONALE: Pure functions only — no I/O, no DB access. Separated from executor for testability.
|
||||
# @REJECTED: Embedding context inline in the executor would make it untestable without mocking DB.
|
||||
#
|
||||
# Typical workflow:
|
||||
# 1. Call build_context_entries(dictionary_entries, row_context) to get annotated, prioritized entries
|
||||
# 2. Each entry is rendered via render_entry() with optional priority flag
|
||||
# 3. Jaccard similarity >= 0.5 triggers priority flagging
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
# #region ContextAwarePromptBuilder [C:2] [TYPE Class]
|
||||
# @BRIEF Build LLM prompts with context-aware dictionary entries and similarity-based priority.
|
||||
|
||||
class ContextAwarePromptBuilder:
|
||||
"""Build LLM prompts with context-aware dictionary entries.
|
||||
|
||||
Pure function — no I/O, no DB access.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def render_entry(entry: Any, priority: bool = False, row_context: dict | None = None) -> str:
|
||||
"""Render a dictionary entry for the LLM prompt.
|
||||
|
||||
Args:
|
||||
entry: DictionaryEntry-like object (must have source_term, target_term, has_context,
|
||||
context_data, usage_notes attrs or dict keys).
|
||||
priority: Whether this entry should be flagged as high priority.
|
||||
row_context: Optional row context dict (unused in rendering, kept for API symmetry).
|
||||
|
||||
Returns:
|
||||
Rendered prompt line string.
|
||||
"""
|
||||
# Support both object attrs and dict access
|
||||
if isinstance(entry, dict):
|
||||
source_term = entry.get("source_term", "")
|
||||
target_term = entry.get("target_term", "")
|
||||
has_context = entry.get("has_context", False)
|
||||
context_data = entry.get("context_data")
|
||||
usage_notes = entry.get("usage_notes")
|
||||
else:
|
||||
source_term = getattr(entry, "source_term", "")
|
||||
target_term = getattr(entry, "target_term", "")
|
||||
has_context = getattr(entry, "has_context", False)
|
||||
context_data = getattr(entry, "context_data", None)
|
||||
usage_notes = getattr(entry, "usage_notes", None)
|
||||
|
||||
# Build base line
|
||||
line = f'"{source_term}" -> "{target_term}"'
|
||||
|
||||
# Add context annotation if present
|
||||
if has_context and context_data:
|
||||
context_items = []
|
||||
if isinstance(context_data, dict):
|
||||
context_items = [f"{k}={v}" for k, v in context_data.items()]
|
||||
elif isinstance(context_data, str):
|
||||
try:
|
||||
parsed = json.loads(context_data)
|
||||
if isinstance(parsed, dict):
|
||||
context_items = [f"{k}={v}" for k, v in parsed.items()]
|
||||
else:
|
||||
context_items = [str(context_data)]
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
context_items = [str(context_data)]
|
||||
|
||||
context_str = ", ".join(context_items)
|
||||
|
||||
# Truncate if too long (500 tokens ≈ 2000 chars)
|
||||
if len(context_str) > 2000:
|
||||
context_str = context_str[:1997] + "...[truncated]"
|
||||
|
||||
line = f'"{source_term}" (context: {context_str}) -> "{target_term}"'
|
||||
|
||||
# Add usage notes
|
||||
if usage_notes:
|
||||
notes = str(usage_notes)[:200] # cap at 200 chars
|
||||
line += f" # Usage: {notes}"
|
||||
|
||||
# Add priority prefix
|
||||
if priority:
|
||||
line = f"# PRIORITY (context match) — {line}"
|
||||
|
||||
return line
|
||||
|
||||
@staticmethod
|
||||
def compute_context_similarity(entry_context: dict | None, row_context: dict | None) -> float:
|
||||
"""Jaccard similarity between entry context and row context. Returns 0.0-1.0.
|
||||
|
||||
Compares the sets of lowercased string values from both contexts.
|
||||
Returns 1.0 for identical contexts, 0.0 for disjoint or missing.
|
||||
"""
|
||||
if not entry_context or not row_context:
|
||||
return 0.0
|
||||
|
||||
entry_vals = set(str(v).lower() for v in entry_context.values() if v is not None)
|
||||
row_vals = set(str(v).lower() for v in row_context.values() if v is not None)
|
||||
|
||||
if not entry_vals or not row_vals:
|
||||
return 0.0
|
||||
|
||||
intersection = entry_vals & row_vals
|
||||
union = entry_vals | row_vals
|
||||
return len(intersection) / len(union)
|
||||
|
||||
@staticmethod
|
||||
def build_context_entries(
|
||||
dictionary_entries: list[Any],
|
||||
row_context: dict | None = None,
|
||||
) -> list[str]:
|
||||
"""Build prioritized dictionary entry list with context annotations.
|
||||
|
||||
Args:
|
||||
dictionary_entries: List of DictionaryEntry-like objects or dicts.
|
||||
row_context: Optional dict of current row's context columns.
|
||||
|
||||
Returns:
|
||||
List of rendered prompt strings, sorted with priority entries first.
|
||||
"""
|
||||
results: list[tuple[Any, bool]] = []
|
||||
|
||||
for entry in dictionary_entries:
|
||||
priority = False
|
||||
if row_context:
|
||||
# Extract entry context_data
|
||||
if isinstance(entry, dict):
|
||||
entry_context = entry.get("context_data")
|
||||
else:
|
||||
entry_context = getattr(entry, "context_data", None)
|
||||
|
||||
if entry_context:
|
||||
similarity = ContextAwarePromptBuilder.compute_context_similarity(
|
||||
entry_context, row_context
|
||||
)
|
||||
priority = similarity >= 0.5
|
||||
|
||||
results.append((entry, priority))
|
||||
|
||||
# Sort: priority first, then non-priority
|
||||
results.sort(key=lambda x: (not x[1]))
|
||||
|
||||
return [
|
||||
ContextAwarePromptBuilder.render_entry(entry, priority, row_context)
|
||||
for entry, priority in results
|
||||
]
|
||||
# #endregion ContextAwarePromptBuilder
|
||||
|
||||
|
||||
# #endregion ContextAwarePromptBuilder
|
||||
@@ -10,6 +10,7 @@
|
||||
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
|
||||
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
@@ -27,6 +28,7 @@ from ...schemas.translate import (
|
||||
TranslateJobResponse,
|
||||
TranslateJobUpdate,
|
||||
)
|
||||
from .dictionary import _validate_bcp47
|
||||
|
||||
# Supported database dialects for translation
|
||||
SUPPORTED_DIALECTS = {
|
||||
@@ -227,6 +229,16 @@ class TranslateJobService:
|
||||
logger.warning(f"[TranslateJobService] Dialect detection failed: {e}")
|
||||
dialect = payload.source_dialect
|
||||
|
||||
# Resolve target_languages: accept new format (list) or legacy single language
|
||||
target_languages = payload.target_languages
|
||||
if not target_languages and payload.target_language:
|
||||
target_languages = [payload.target_language]
|
||||
if target_languages:
|
||||
if not isinstance(target_languages, list):
|
||||
target_languages = [str(target_languages)]
|
||||
for lang in target_languages:
|
||||
_validate_bcp47(lang, "target_languages")
|
||||
|
||||
# Build job instance
|
||||
job = TranslationJob(
|
||||
id=str(uuid.uuid4()),
|
||||
@@ -245,6 +257,7 @@ class TranslateJobService:
|
||||
target_column=payload.target_column,
|
||||
context_columns=payload.context_columns or [],
|
||||
target_language=payload.target_language,
|
||||
target_languages=target_languages,
|
||||
provider_id=payload.provider_id,
|
||||
batch_size=payload.batch_size,
|
||||
upsert_strategy=payload.upsert_strategy,
|
||||
@@ -284,6 +297,22 @@ class TranslateJobService:
|
||||
update_data = payload.model_dump(exclude_unset=True)
|
||||
dict_ids = update_data.pop("dictionary_ids", None)
|
||||
|
||||
# Backward compat: if only target_language is set (old API), wrap into target_languages
|
||||
if "target_language" in update_data and "target_languages" not in update_data:
|
||||
target_languages = [update_data["target_language"]] if update_data["target_language"] else []
|
||||
update_data["target_languages"] = target_languages
|
||||
elif "target_languages" in update_data and "target_language" not in update_data:
|
||||
# Keep deprecated field in sync
|
||||
if update_data["target_languages"]:
|
||||
update_data["target_language"] = update_data["target_languages"][0]
|
||||
|
||||
# Validate BCP-47 for target_languages
|
||||
if update_data.get("target_languages"):
|
||||
if not isinstance(update_data["target_languages"], list):
|
||||
update_data["target_languages"] = [str(update_data["target_languages"])]
|
||||
for lang in update_data["target_languages"]:
|
||||
_validate_bcp47(lang, "target_languages")
|
||||
|
||||
for field, value in update_data.items():
|
||||
if hasattr(job, field):
|
||||
setattr(job, field, value)
|
||||
@@ -366,6 +395,7 @@ class TranslateJobService:
|
||||
target_column=source.target_column,
|
||||
context_columns=source.context_columns,
|
||||
target_language=source.target_language,
|
||||
target_languages=source.target_languages,
|
||||
provider_id=source.provider_id,
|
||||
batch_size=source.batch_size,
|
||||
upsert_strategy=source.upsert_strategy,
|
||||
@@ -468,6 +498,7 @@ def job_to_response(job: TranslationJob, dict_ids: list[str] | None = None) -> T
|
||||
target_column=job.target_column,
|
||||
context_columns=job.context_columns or [],
|
||||
target_language=job.target_language,
|
||||
target_languages=job.target_languages,
|
||||
provider_id=job.provider_id,
|
||||
batch_size=job.batch_size or 50,
|
||||
upsert_strategy=job.upsert_strategy or "MERGE",
|
||||
@@ -546,5 +577,462 @@ def get_datasource_columns(
|
||||
)
|
||||
# #endregion DatasourceColumnsService
|
||||
|
||||
|
||||
# #region InlineCorrectionService [C:3] [TYPE Class] [SEMANTICS translate, correction, inline, dictionary]
|
||||
# @BRIEF Service for inline editing translated values and submitting corrections to dictionaries.
|
||||
class InlineCorrectionService:
|
||||
"""Handle inline correction of translated values with optional dictionary submission."""
|
||||
|
||||
@staticmethod
|
||||
def apply_inline_edit(
|
||||
db: Session,
|
||||
run_id: str,
|
||||
record_id: str,
|
||||
language_code: str,
|
||||
final_value: str,
|
||||
submit_to_dictionary: bool = False,
|
||||
dictionary_id: str | None = None,
|
||||
current_user: str | None = None,
|
||||
context_data_override: dict | None = None,
|
||||
usage_notes: str | None = None,
|
||||
keep_context: bool = True,
|
||||
) -> dict:
|
||||
"""Apply an inline edit to a TranslationLanguage entry.
|
||||
|
||||
Updates final_value and user_edit fields. Optionally submits to dictionary.
|
||||
Returns the updated TranslationLanguage data as a dict.
|
||||
|
||||
Args:
|
||||
context_data_override: If provided, overrides auto-captured context data.
|
||||
usage_notes: Usage notes for the dictionary entry.
|
||||
keep_context: If False, clear context on the dictionary entry.
|
||||
"""
|
||||
from ...models.translate import TranslationLanguage, TranslationRecord
|
||||
|
||||
# Find the language entry
|
||||
lang_entry = (
|
||||
db.query(TranslationLanguage)
|
||||
.filter(
|
||||
TranslationLanguage.record_id == record_id,
|
||||
TranslationLanguage.language_code == language_code,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not lang_entry:
|
||||
raise ValueError(f"Language entry '{language_code}' not found for record '{record_id}'")
|
||||
|
||||
# Verify the record belongs to the run
|
||||
record = (
|
||||
db.query(TranslationRecord)
|
||||
.filter(
|
||||
TranslationRecord.id == record_id,
|
||||
TranslationRecord.run_id == run_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not record:
|
||||
raise ValueError(
|
||||
f"Translation record '{record_id}' not found in run '{run_id}'"
|
||||
)
|
||||
|
||||
# Apply the edit
|
||||
lang_entry.final_value = final_value
|
||||
lang_entry.user_edit = final_value
|
||||
if lang_entry.status in ("pending", "translated"):
|
||||
lang_entry.status = "edited"
|
||||
db.flush()
|
||||
|
||||
# Optionally submit to dictionary
|
||||
dict_result = None
|
||||
if submit_to_dictionary and dictionary_id:
|
||||
try:
|
||||
dict_result = InlineCorrectionService.submit_correction_to_dict(
|
||||
db=db,
|
||||
record_id=record_id,
|
||||
language_code=language_code,
|
||||
dictionary_id=dictionary_id,
|
||||
corrected_value=final_value,
|
||||
current_user=current_user,
|
||||
context_data_override=context_data_override,
|
||||
usage_notes=usage_notes,
|
||||
keep_context=keep_context,
|
||||
)
|
||||
except Exception as e:
|
||||
# Dictionary submission failure should not block the edit
|
||||
dict_result = {"error": str(e), "action": "failed"}
|
||||
|
||||
db.commit()
|
||||
db.refresh(lang_entry)
|
||||
|
||||
from ...schemas.translate import TranslationLanguageResponse
|
||||
response = TranslationLanguageResponse.model_validate(lang_entry)
|
||||
result = response.model_dump()
|
||||
if dict_result:
|
||||
result["dictionary_result"] = dict_result
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def submit_correction_to_dict(
|
||||
db: Session,
|
||||
record_id: str,
|
||||
language_code: str,
|
||||
dictionary_id: str,
|
||||
corrected_value: str,
|
||||
current_user: str | None = None,
|
||||
context_data_override: dict | None = None,
|
||||
usage_notes: str | None = None,
|
||||
keep_context: bool = True,
|
||||
) -> dict:
|
||||
"""Submit a correction from an inline edit to the dictionary.
|
||||
|
||||
Fetches the TranslationLanguage entry, auto-captures source text and context,
|
||||
and creates/updates a DictionaryEntry with correct language pair.
|
||||
Returns conflict info if an entry already exists.
|
||||
|
||||
Args:
|
||||
context_data_override: If provided, overrides auto-captured context_data.
|
||||
usage_notes: Optional usage notes to store on the dictionary entry.
|
||||
keep_context: If False, sets has_context=False and clears context_data.
|
||||
"""
|
||||
from ...core.logger import belief_scope, logger
|
||||
from ...models.translate import DictionaryEntry, TranslationLanguage, TranslationRecord
|
||||
from ._utils import _normalize_term
|
||||
|
||||
with belief_scope("InlineCorrectionService.submit_correction_to_dict"):
|
||||
# Find the language entry
|
||||
lang_entry = (
|
||||
db.query(TranslationLanguage)
|
||||
.filter(
|
||||
TranslationLanguage.record_id == record_id,
|
||||
TranslationLanguage.language_code == language_code,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not lang_entry:
|
||||
raise ValueError(f"Language entry '{language_code}' not found for record '{record_id}'")
|
||||
|
||||
# Find the record for source text and context
|
||||
record = (
|
||||
db.query(TranslationRecord)
|
||||
.filter(TranslationRecord.id == record_id)
|
||||
.first()
|
||||
)
|
||||
if not record:
|
||||
raise ValueError(f"Translation record '{record_id}' not found")
|
||||
|
||||
# Get source text from the record
|
||||
source_term = record.source_sql or record.source_object_name or ""
|
||||
if not source_term:
|
||||
raise ValueError("No source text available for this record")
|
||||
|
||||
# Determine language pair
|
||||
source_language = lang_entry.source_language_detected or "und"
|
||||
target_language = language_code
|
||||
|
||||
# Prepare context data — auto-capture from source row
|
||||
context_data = {}
|
||||
if record.source_data:
|
||||
context_data["source_data"] = record.source_data
|
||||
if record.source_object_type:
|
||||
context_data["source_object_type"] = record.source_object_type
|
||||
if record.source_object_name:
|
||||
context_data["source_object_name"] = record.source_object_name
|
||||
if record.source_object_id:
|
||||
context_data["source_object_id"] = record.source_object_id
|
||||
|
||||
# Apply context_data_override if provided (user edited)
|
||||
if context_data_override is not None:
|
||||
context_data = context_data_override
|
||||
context_source = "manual"
|
||||
else:
|
||||
context_source = "auto"
|
||||
|
||||
# Handle keep_context=False (user explicitly removed context)
|
||||
if not keep_context:
|
||||
context_data = None
|
||||
context_source = "manual"
|
||||
|
||||
# Normalize source term
|
||||
normalized = _normalize_term(source_term)
|
||||
|
||||
# Check for existing entry with same (dictionary_id, source_term_norm, source_language, target_language)
|
||||
existing = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(
|
||||
DictionaryEntry.dictionary_id == dictionary_id,
|
||||
DictionaryEntry.source_term_normalized == normalized,
|
||||
DictionaryEntry.source_language == source_language,
|
||||
DictionaryEntry.target_language == target_language,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
|
||||
result = {
|
||||
"action": "created",
|
||||
"entry_id": None,
|
||||
"conflict": None,
|
||||
"message": None,
|
||||
}
|
||||
|
||||
if existing:
|
||||
# Conflict: return conflict info
|
||||
result["action"] = "conflict_detected"
|
||||
result["conflict"] = {
|
||||
"source_term": source_term,
|
||||
"existing_target_term": existing.target_term,
|
||||
"submitted_target_term": corrected_value,
|
||||
"action": "keep_existing",
|
||||
}
|
||||
result["message"] = (
|
||||
f"Existing entry found: '{existing.target_term}' "
|
||||
f"for source '{source_term}' ({source_language} → {target_language})"
|
||||
)
|
||||
logger.reason("Correction conflict detected", result)
|
||||
return result
|
||||
|
||||
# Create new entry
|
||||
entry = DictionaryEntry(
|
||||
dictionary_id=dictionary_id,
|
||||
source_term=source_term.strip(),
|
||||
source_term_normalized=normalized,
|
||||
target_term=corrected_value.strip(),
|
||||
source_language=source_language,
|
||||
target_language=target_language,
|
||||
context_data=context_data if context_data else None,
|
||||
context_notes=None,
|
||||
has_context=bool(context_data) and keep_context,
|
||||
context_source=context_source,
|
||||
usage_notes=usage_notes,
|
||||
origin_source_language=source_language,
|
||||
origin_run_id=record.run_id,
|
||||
origin_row_key=record.id,
|
||||
origin_user_id=current_user,
|
||||
)
|
||||
db.add(entry)
|
||||
db.flush()
|
||||
result["entry_id"] = entry.id
|
||||
result["message"] = (
|
||||
f"Entry created for '{source_term}' ({source_language}) → "
|
||||
f"'{corrected_value}' ({target_language})"
|
||||
)
|
||||
logger.reflect("Dictionary entry created from correction", result)
|
||||
return result
|
||||
# #endregion InlineCorrectionService
|
||||
|
||||
|
||||
# #region BulkFindReplaceService [C:3] [TYPE Class] [SEMANTICS translate, bulk, find, replace, regex]
|
||||
# @BRIEF Service for bulk find-and-replace operations on translated values.
|
||||
class BulkFindReplaceService:
|
||||
"""Handle bulk find-and-replace on TranslationLanguage entries."""
|
||||
|
||||
@staticmethod
|
||||
def _compile_pattern(pattern: str, is_regex: bool) -> re.Pattern:
|
||||
"""Compile a search pattern (regex or plain text)."""
|
||||
import re
|
||||
if is_regex:
|
||||
return re.compile(pattern)
|
||||
return re.compile(re.escape(pattern))
|
||||
|
||||
@staticmethod
|
||||
def _find_matching_entries(
|
||||
db: Session,
|
||||
run_id: str,
|
||||
pattern: str,
|
||||
is_regex: bool,
|
||||
target_language: str,
|
||||
) -> list[Any]:
|
||||
"""Find all TranslationLanguage entries matching the pattern."""
|
||||
from ...models.translate import TranslationLanguage, TranslationRecord
|
||||
|
||||
# We scan entries for the target language in this run
|
||||
entries = (
|
||||
db.query(TranslationLanguage)
|
||||
.join(
|
||||
TranslationRecord,
|
||||
TranslationRecord.id == TranslationLanguage.record_id,
|
||||
)
|
||||
.filter(
|
||||
TranslationRecord.run_id == run_id,
|
||||
TranslationLanguage.language_code == target_language,
|
||||
)
|
||||
.all()
|
||||
)
|
||||
return entries
|
||||
|
||||
@staticmethod
|
||||
def preview(
|
||||
db: Session,
|
||||
run_id: str,
|
||||
pattern: str,
|
||||
is_regex: bool,
|
||||
target_language: str,
|
||||
) -> list[dict]:
|
||||
"""Scan TranslationLanguage entries and return matching items without applying changes."""
|
||||
from ...models.translate import TranslationRecord
|
||||
|
||||
compiled = BulkFindReplaceService._compile_pattern(pattern, is_regex)
|
||||
entries = BulkFindReplaceService._find_matching_entries(
|
||||
db, run_id, pattern, is_regex, target_language
|
||||
)
|
||||
|
||||
preview_items = []
|
||||
for entry in entries:
|
||||
current_value = entry.final_value or entry.translated_value or ""
|
||||
if compiled.search(current_value):
|
||||
# Get source term for context
|
||||
record = (
|
||||
db.query(TranslationRecord)
|
||||
.filter(TranslationRecord.id == entry.record_id)
|
||||
.first()
|
||||
)
|
||||
source_term = record.source_sql if record else ""
|
||||
|
||||
new_value = compiled.sub(
|
||||
BulkFindReplaceService._get_replacement(pattern, is_regex),
|
||||
current_value,
|
||||
)
|
||||
if new_value != current_value:
|
||||
preview_items.append({
|
||||
"record_id": entry.record_id,
|
||||
"language_code": entry.language_code,
|
||||
"source_term": source_term or "",
|
||||
"current_value": current_value,
|
||||
"new_value": new_value,
|
||||
"source_language_detected": entry.source_language_detected,
|
||||
})
|
||||
|
||||
return preview_items
|
||||
|
||||
@staticmethod
|
||||
def _get_replacement(pattern: str, is_regex: bool) -> str:
|
||||
"""Return replacement pattern (for regex, use backrefs; for plain, use literal)."""
|
||||
# The replacement_text is passed through at apply time
|
||||
return pattern # Placeholder — actual replacement is done in apply()
|
||||
|
||||
@staticmethod
|
||||
def apply(
|
||||
db: Session,
|
||||
run_id: str,
|
||||
pattern: str,
|
||||
is_regex: bool,
|
||||
replacement_text: str,
|
||||
target_language: str,
|
||||
submit_to_dictionary: bool = False,
|
||||
dictionary_id: str | None = None,
|
||||
usage_notes: str | None = None,
|
||||
current_user: str | None = None,
|
||||
submit_to_dictionary_with_context: bool = False,
|
||||
) -> dict:
|
||||
"""Apply find-and-replace to matching TranslationLanguage entries.
|
||||
|
||||
Returns dict with rows_affected, corrections_submitted, and preview list.
|
||||
|
||||
Args:
|
||||
submit_to_dictionary_with_context: If True, auto-capture source row context
|
||||
when submitting to dictionary.
|
||||
"""
|
||||
from ...core.logger import belief_scope, logger
|
||||
from ...models.translate import DictionaryEntry, TranslationRecord
|
||||
|
||||
with belief_scope("BulkFindReplaceService.apply"):
|
||||
compiled = BulkFindReplaceService._compile_pattern(pattern, is_regex)
|
||||
entries = BulkFindReplaceService._find_matching_entries(
|
||||
db, run_id, pattern, is_regex, target_language
|
||||
)
|
||||
|
||||
preview_items = []
|
||||
rows_affected = 0
|
||||
corrections_submitted = 0
|
||||
|
||||
# Track unique (source_term, replacement) -> first record for context capture
|
||||
seen_term_replacements: dict[str, str] = {}
|
||||
|
||||
for entry in entries:
|
||||
current_value = entry.final_value or entry.translated_value or ""
|
||||
if compiled.search(current_value):
|
||||
new_value = compiled.sub(replacement_text, current_value)
|
||||
if new_value != current_value:
|
||||
# Apply the replacement
|
||||
entry.final_value = new_value
|
||||
entry.user_edit = new_value
|
||||
if entry.status in ("pending", "translated"):
|
||||
entry.status = "edited"
|
||||
rows_affected += 1
|
||||
|
||||
# Get source term for preview
|
||||
record = (
|
||||
db.query(TranslationRecord)
|
||||
.filter(TranslationRecord.id == entry.record_id)
|
||||
.first()
|
||||
)
|
||||
source_term = record.source_sql if record else ""
|
||||
|
||||
# Optionally submit to dictionary
|
||||
if submit_to_dictionary and dictionary_id:
|
||||
try:
|
||||
dict_result = InlineCorrectionService.submit_correction_to_dict(
|
||||
db=db,
|
||||
record_id=entry.record_id,
|
||||
language_code=entry.language_code,
|
||||
dictionary_id=dictionary_id,
|
||||
corrected_value=new_value,
|
||||
current_user=current_user,
|
||||
)
|
||||
if dict_result.get("action") in ("created", "updated"):
|
||||
corrections_submitted += 1
|
||||
|
||||
# Track context for unique (source_term, replacement) pairs
|
||||
if submit_to_dictionary_with_context and record:
|
||||
term_key = f"{source_term}::{new_value}"
|
||||
if term_key not in seen_term_replacements:
|
||||
seen_term_replacements[term_key] = entry.record_id
|
||||
# Auto-capture context from this row
|
||||
context_data = {}
|
||||
if record.source_data:
|
||||
context_data["source_data"] = record.source_data
|
||||
if record.source_object_type:
|
||||
context_data["source_object_type"] = record.source_object_type
|
||||
if record.source_object_name:
|
||||
context_data["source_object_name"] = record.source_object_name
|
||||
if record.source_object_id:
|
||||
context_data["source_object_id"] = record.source_object_id
|
||||
# Update the just-created entry's context
|
||||
if dict_result.get("action") == "created" and dict_result.get("entry_id"):
|
||||
d_entry = (
|
||||
db.query(DictionaryEntry)
|
||||
.filter(DictionaryEntry.id == dict_result["entry_id"])
|
||||
.first()
|
||||
)
|
||||
if d_entry:
|
||||
d_entry.context_data = context_data if context_data else None
|
||||
d_entry.has_context = bool(context_data)
|
||||
d_entry.context_source = "bulk"
|
||||
except Exception as e:
|
||||
logger.explore(
|
||||
"Bulk replace: dictionary submission failed",
|
||||
extra={"record_id": entry.record_id, "error": str(e)},
|
||||
)
|
||||
|
||||
preview_items.append({
|
||||
"record_id": entry.record_id,
|
||||
"language_code": entry.language_code,
|
||||
"source_term": source_term or "",
|
||||
"current_value": current_value,
|
||||
"new_value": new_value,
|
||||
"source_language_detected": entry.source_language_detected,
|
||||
})
|
||||
|
||||
db.commit()
|
||||
|
||||
result = {
|
||||
"rows_affected": rows_affected,
|
||||
"corrections_submitted": corrections_submitted,
|
||||
"preview": preview_items,
|
||||
}
|
||||
logger.reason("Bulk replace completed", result)
|
||||
return result
|
||||
# #endregion BulkFindReplaceService
|
||||
|
||||
|
||||
# #endregion TranslateJobService
|
||||
# #endregion TranslateJobService
|
||||
|
||||
@@ -6,7 +6,24 @@
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
import re
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
def _validate_bcp47_list(v: list[str] | None) -> list[str] | None:
|
||||
"""Validate that each item in target_languages is a valid BCP-47 tag."""
|
||||
if not v:
|
||||
return v
|
||||
for tag in v:
|
||||
if not tag or not tag.strip():
|
||||
raise ValueError(f"Invalid BCP-47 tag: '{tag}' — must be a non-empty string")
|
||||
if not re.match(r'^[a-zA-Z]{2,8}(-[a-zA-Z0-9]{1,8})*$', tag.strip()):
|
||||
raise ValueError(
|
||||
f"Invalid BCP-47 tag: '{tag}'. "
|
||||
"Expected format like 'en', 'ru', 'zh-CN', 'pt-BR'."
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
# #region TranslateJobCreate [TYPE Class]
|
||||
@@ -26,8 +43,11 @@ class TranslateJobCreate(BaseModel):
|
||||
translation_column: str | None = Field(None, description="Source column to translate")
|
||||
target_column: str | None = Field(None, description="Target column for translated output (defaults to translation_column)")
|
||||
context_columns: list[str] | None = Field(default_factory=list, description="Context column names")
|
||||
target_language: str | None = Field(None, description="Target language code")
|
||||
target_language: str | None = Field(None, description="Target language code [DEPRECATED: use target_languages]")
|
||||
target_languages: list[str] | None = Field(default_factory=list, description="List of BCP-47 target language codes")
|
||||
provider_id: str | None = Field(None, description="LLM provider ID")
|
||||
|
||||
_validate_target_languages = field_validator("target_languages")(_validate_bcp47_list)
|
||||
batch_size: int = Field(50, description="Records per batch")
|
||||
upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE")
|
||||
dictionary_ids: list[str] | None = Field(default_factory=list, description="Associated terminology dictionary IDs")
|
||||
@@ -54,7 +74,10 @@ class TranslateJobUpdate(BaseModel):
|
||||
target_column: str | None = None
|
||||
context_columns: list[str] | None = None
|
||||
target_language: str | None = None
|
||||
target_languages: list[str] | None = None
|
||||
provider_id: str | None = None
|
||||
|
||||
_validate_target_languages = field_validator("target_languages")(_validate_bcp47_list)
|
||||
batch_size: int | None = None
|
||||
upsert_strategy: str | None = None
|
||||
status: str | None = None
|
||||
@@ -83,6 +106,8 @@ class TranslateJobResponse(BaseModel):
|
||||
target_column: str | None = None
|
||||
context_columns: list[str] | None = None
|
||||
target_language: str | None = None
|
||||
source_language: str | None = None
|
||||
target_languages: list[str] | None = None
|
||||
provider_id: str | None = None
|
||||
batch_size: int = 50
|
||||
upsert_strategy: str = "MERGE"
|
||||
@@ -154,6 +179,8 @@ class DictionaryImport(BaseModel):
|
||||
delimiter: str | None = Field(None, description="Detected or forced delimiter: ',' or '\\t'. Auto-detect if omitted.")
|
||||
on_conflict: str = Field("overwrite", description="'overwrite' or 'keep_existing' or 'cancel'")
|
||||
preview_only: bool = Field(False, description="If true, return preview without applying")
|
||||
default_source_language: str | None = Field(None, description="Default BCP-47 source language when column missing in data")
|
||||
default_target_language: str | None = Field(None, description="Default BCP-47 target language when column missing in data")
|
||||
# #endregion DictionaryImport
|
||||
|
||||
|
||||
@@ -177,11 +204,13 @@ class DictionaryResponse(BaseModel):
|
||||
|
||||
|
||||
# #region DictionaryEntryCreate [TYPE Class]
|
||||
# @BRIEF Schema for adding/editing a dictionary entry.
|
||||
# @BRIEF Schema for adding/editing a dictionary entry with language pair support.
|
||||
class DictionaryEntryCreate(BaseModel):
|
||||
source_term: str = Field(..., description="Source term to translate")
|
||||
target_term: str = Field(..., description="Target/translated term")
|
||||
context_notes: str | None = Field(None, description="Optional context notes")
|
||||
source_language: str = Field("und", description="BCP-47 source language code (default: und)")
|
||||
target_language: str = Field("und", description="BCP-47 target language code (default: und)")
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
@@ -196,7 +225,14 @@ class DictionaryEntryResponse(BaseModel):
|
||||
source_term: str
|
||||
source_term_normalized: str
|
||||
target_term: str
|
||||
source_language: str | None = None
|
||||
target_language: str | None = None
|
||||
context_notes: str | None = None
|
||||
context_data: dict[str, Any] | None = None
|
||||
usage_notes: str | None = None
|
||||
has_context: bool = False
|
||||
context_source: str | None = None
|
||||
origin_source_language: str | None = None
|
||||
created_at: datetime
|
||||
updated_at: datetime | None = None
|
||||
|
||||
@@ -232,6 +268,7 @@ class PreviewRowUpdate(BaseModel):
|
||||
action: str = Field(..., description="'approve', 'reject', or 'edit'")
|
||||
translation: str | None = Field(None, description="Edited translation (required for 'edit' action)")
|
||||
feedback: str | None = Field(None, description="Optional feedback/comment")
|
||||
language_code: str | None = Field(None, description="BCP-47 language code for per-language actions (optional, applies to all if omitted)")
|
||||
# #endregion PreviewRowUpdate
|
||||
|
||||
|
||||
@@ -245,6 +282,7 @@ class PreviewAcceptResponse(BaseModel):
|
||||
created_at: datetime
|
||||
expires_at: datetime | None = None
|
||||
records: list['PreviewRow'] = []
|
||||
target_languages: list[str] = Field(default_factory=list, description="Target languages for this preview")
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
@@ -255,6 +293,7 @@ class PreviewAcceptResponse(BaseModel):
|
||||
# @BRIEF Schema for cost estimation in preview response.
|
||||
class CostEstimate(BaseModel):
|
||||
sample_size: int = 0
|
||||
num_languages: int = 1
|
||||
sample_prompt_tokens: int = 0
|
||||
sample_output_tokens: int = 0
|
||||
sample_total_tokens: int = 0
|
||||
@@ -262,6 +301,7 @@ class CostEstimate(BaseModel):
|
||||
estimated_total_rows: int = 0
|
||||
estimated_tokens: int = 0
|
||||
estimated_cost: float = 0.0
|
||||
warning: str | None = None
|
||||
# #endregion CostEstimate
|
||||
|
||||
|
||||
@@ -274,6 +314,8 @@ class TermCorrectionSubmit(BaseModel):
|
||||
dictionary_id: str | None = Field(None, description="Target dictionary ID (language-filtered)")
|
||||
origin_run_id: str | None = Field(None, description="Run ID from which this correction originated")
|
||||
origin_row_key: str | None = Field(None, description="Row key within the run")
|
||||
context_data: dict[str, Any] | None = Field(None, description="Context data for the dictionary entry")
|
||||
usage_notes: str | None = Field(None, description="Usage notes for the dictionary entry")
|
||||
# #endregion TermCorrectionSubmit
|
||||
|
||||
|
||||
@@ -369,6 +411,7 @@ class TranslationRunResponse(BaseModel):
|
||||
dict_snapshot_hash: str | None = None
|
||||
created_by: str | None = None
|
||||
created_at: datetime
|
||||
language_stats: list['TranslationRunLanguageStatsResponse'] | None = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
@@ -430,6 +473,22 @@ class AggregatedMetricsResponse(BaseModel):
|
||||
# #endregion AggregatedMetricsResponse
|
||||
|
||||
|
||||
# #region TranslationPreviewLanguageResponse [C:1] [TYPE Class]
|
||||
# @BRIEF Schema for per-language preview data in API responses.
|
||||
class TranslationPreviewLanguageResponse(BaseModel):
|
||||
language_code: str
|
||||
source_language_detected: str | None = None
|
||||
translated_value: str | None = None
|
||||
user_edit: str | None = None
|
||||
final_value: str | None = None
|
||||
status: str = "pending"
|
||||
needs_review: bool = False
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# #endregion TranslationPreviewLanguageResponse
|
||||
|
||||
|
||||
# #region PreviewRow [TYPE Class]
|
||||
# @BRIEF A single row in a translation preview showing original and translated content side by side.
|
||||
class PreviewRow(BaseModel):
|
||||
@@ -441,6 +500,9 @@ class PreviewRow(BaseModel):
|
||||
source_object_name: str | None = None
|
||||
status: str = "PENDING"
|
||||
feedback: str | None = None
|
||||
source_language_detected: str | None = None
|
||||
needs_review: bool = False
|
||||
languages: list[TranslationPreviewLanguageResponse] = Field(default_factory=list, description="Per-language translation data")
|
||||
# #endregion PreviewRow
|
||||
|
||||
|
||||
@@ -455,6 +517,8 @@ class TranslationPreviewResponse(BaseModel):
|
||||
created_at: datetime
|
||||
expires_at: datetime | None = None
|
||||
records: list[PreviewRow] = []
|
||||
target_languages: list[str] = Field(default_factory=list, description="Target languages for this preview")
|
||||
cost_estimate: CostEstimate | None = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
@@ -500,4 +564,105 @@ class MetricsResponse(BaseModel):
|
||||
from_attributes = True
|
||||
# #endregion MetricsResponse
|
||||
|
||||
|
||||
# #region TranslationLanguageResponse [C:1] [TYPE Class]
|
||||
# @BRIEF Schema for per-language translation data in API responses.
|
||||
class TranslationLanguageResponse(BaseModel):
|
||||
language_code: str
|
||||
source_language_detected: str | None = None
|
||||
translated_value: str | None = None
|
||||
user_edit: str | None = None
|
||||
final_value: str | None = None
|
||||
status: str = "pending"
|
||||
needs_review: bool = False
|
||||
language_overridden: bool = False
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# #endregion TranslationLanguageResponse
|
||||
|
||||
|
||||
# #region TranslationRunLanguageStatsResponse [C:1] [TYPE Class]
|
||||
# @BRIEF Schema for per-language statistics in run API responses.
|
||||
class TranslationRunLanguageStatsResponse(BaseModel):
|
||||
language_code: str
|
||||
total_rows: int = 0
|
||||
translated_rows: int = 0
|
||||
failed_rows: int = 0
|
||||
skipped_rows: int = 0
|
||||
token_count: int = 0
|
||||
estimated_cost: float = 0.0
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# #endregion TranslationRunLanguageStatsResponse
|
||||
|
||||
|
||||
# #region OverrideLanguageRequest [C:1] [TYPE Class]
|
||||
# @BRIEF Schema for manually overriding the detected source language of a translation.
|
||||
class OverrideLanguageRequest(BaseModel):
|
||||
source_language: str = Field(..., description="BCP-47 language code to override with")
|
||||
# #endregion OverrideLanguageRequest
|
||||
|
||||
|
||||
# #region BulkFindReplaceRequest [C:1] [TYPE Class]
|
||||
# @BRIEF Schema for bulk find-and-replace within translations for a target language.
|
||||
class BulkFindReplaceRequest(BaseModel):
|
||||
find_pattern: str = Field(..., description="Text or regex pattern to find")
|
||||
is_regex: bool = Field(False, description="Whether find_pattern is a regular expression")
|
||||
replacement_text: str = Field(..., description="Replacement text")
|
||||
target_language: str = Field(..., description="BCP-47 language code to target")
|
||||
preview: bool = Field(True, description="If true, return matches without applying replacements")
|
||||
submit_to_dictionary: bool = Field(False, description="If true, also submit corrections to dictionary")
|
||||
dictionary_id: str | None = Field(None, description="Target dictionary ID for submitting corrections")
|
||||
usage_notes: str | None = Field(None, description="Usage notes for dictionary entries")
|
||||
submit_to_dictionary_with_context: bool = Field(False, description="If true, auto-capture source row context when submitting to dictionary")
|
||||
# #endregion BulkFindReplaceRequest
|
||||
|
||||
|
||||
# #region InlineEditRequest [C:1] [TYPE Class]
|
||||
# @BRIEF Schema for inline editing a translated value on a completed run result.
|
||||
class InlineEditRequest(BaseModel):
|
||||
final_value: str = Field(..., description="Corrected/edited translation text")
|
||||
submit_to_dictionary: bool = Field(False, description="If true, also save correction to dictionary")
|
||||
dictionary_id: str | None = Field(None, description="Target dictionary ID for saving correction")
|
||||
context_data_override: dict[str, Any] | None = Field(None, description="Override auto-captured context data for dictionary entry")
|
||||
usage_notes: str | None = Field(None, description="Usage notes for the dictionary entry")
|
||||
keep_context: bool = Field(True, description="If false, clear context_data and set has_context=False on dictionary entry")
|
||||
# #endregion InlineEditRequest
|
||||
|
||||
|
||||
# #region BulkReplacePreviewItem [C:1] [TYPE Class]
|
||||
# @BRIEF A single item in a bulk replace preview list.
|
||||
class BulkReplacePreviewItem(BaseModel):
|
||||
record_id: str
|
||||
language_code: str
|
||||
source_term: str
|
||||
current_value: str
|
||||
new_value: str
|
||||
source_language_detected: str | None = None
|
||||
# #endregion BulkReplacePreviewItem
|
||||
|
||||
|
||||
# #region BulkReplaceResponse [C:1] [TYPE Class]
|
||||
# @BRIEF Response for bulk find-and-replace operation.
|
||||
class BulkReplaceResponse(BaseModel):
|
||||
rows_affected: int = 0
|
||||
corrections_submitted: int = 0
|
||||
preview: list[BulkReplacePreviewItem] = Field(default_factory=list)
|
||||
# #endregion BulkReplaceResponse
|
||||
|
||||
|
||||
# #region InlineCorrectionSubmit [C:1] [TYPE Class]
|
||||
# @BRIEF Schema for submitting an inline correction for a specific translated record.
|
||||
class InlineCorrectionSubmit(BaseModel):
|
||||
record_id: str = Field(..., description="Translation record ID")
|
||||
language_code: str = Field(..., description="BCP-47 language code of the translated value")
|
||||
source_term: str = Field(..., description="Original source term that was incorrectly translated")
|
||||
corrected_term: str = Field(..., description="Corrected translation")
|
||||
dictionary_id: str = Field(..., description="Target dictionary ID to also save the correction")
|
||||
source_language: str = Field(..., description="BCP-47 source language code")
|
||||
# #endregion InlineCorrectionSubmit
|
||||
|
||||
|
||||
# #endregion TranslateSchemas
|
||||
|
||||
@@ -295,4 +295,172 @@ def test_run_detail_not_found(client):
|
||||
response = client.get("/api/translate/runs/non-existent/detail")
|
||||
assert response.status_code == 404
|
||||
# [/DEF:test_run_detail_not_found:Function]
|
||||
# [DEF:test_list_runs_includes_language_info:Function]
|
||||
# @PURPOSE: Verify run list includes target_languages, total_translated, and language_stats.
|
||||
def test_list_runs_includes_language_info(client, mock_api_deps):
|
||||
"""Test run list includes per-language fields."""
|
||||
from src.models.translate import TranslationRunLanguageStats
|
||||
session = mock_api_deps["session"]
|
||||
job = _create_test_job(session)
|
||||
run = _create_test_run(session, job.id)
|
||||
# Add language stats
|
||||
ls = TranslationRunLanguageStats(
|
||||
id=str(uuid.uuid4()),
|
||||
run_id=run.id,
|
||||
language_code="ru",
|
||||
total_rows=100,
|
||||
translated_rows=80,
|
||||
failed_rows=5,
|
||||
skipped_rows=15,
|
||||
token_count=15000,
|
||||
estimated_cost=0.03,
|
||||
)
|
||||
session.add(ls)
|
||||
session.commit()
|
||||
|
||||
response = client.get("/api/translate/runs")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert len(data["items"]) >= 1
|
||||
item = data["items"][0]
|
||||
assert "target_languages" in item
|
||||
assert "total_translated" in item
|
||||
assert "language_stats" in item
|
||||
assert "ru" in item["target_languages"]
|
||||
assert item["total_translated"] == 80
|
||||
assert item["language_stats"]["ru"]["translated_rows"] == 80
|
||||
# [/DEF:test_list_runs_includes_language_info:Function]
|
||||
|
||||
|
||||
# [DEF:test_metrics_per_language_breakdown:Function]
|
||||
# @PURPOSE: Verify metrics endpoint returns per_language_metrics.
|
||||
def test_metrics_per_language_breakdown(client, mock_api_deps):
|
||||
"""Test metrics returns per-language breakdown."""
|
||||
from src.models.translate import TranslationRunLanguageStats
|
||||
session = mock_api_deps["session"]
|
||||
job = _create_test_job(session)
|
||||
run = _create_test_run(session, job.id)
|
||||
ls = TranslationRunLanguageStats(
|
||||
id=str(uuid.uuid4()),
|
||||
run_id=run.id,
|
||||
language_code="ru",
|
||||
total_rows=100,
|
||||
translated_rows=80,
|
||||
token_count=15000,
|
||||
estimated_cost=0.03,
|
||||
)
|
||||
session.add(ls)
|
||||
session.commit()
|
||||
|
||||
response = client.get(f"/api/translate/jobs/{job.id}/metrics")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "per_language_metrics" in data
|
||||
assert "ru" in data["per_language_metrics"]
|
||||
assert data["per_language_metrics"]["ru"]["tokens"] == 15000
|
||||
assert data["per_language_metrics"]["ru"]["cost"] == 0.03
|
||||
# [/DEF:test_metrics_per_language_breakdown:Function]
|
||||
|
||||
|
||||
# [DEF:test_metric_snapshot_stores_per_language:Function]
|
||||
# @PURPOSE: Verify MetricSnapshot stores per_language_metrics via prune_expired.
|
||||
def test_metric_snapshot_stores_per_language(client, mock_api_deps):
|
||||
"""Test MetricSnapshot stores per_language_metrics after prune."""
|
||||
from datetime import timedelta
|
||||
from src.models.translate import MetricSnapshot, TranslationEvent
|
||||
from src.plugins.translate.events import TranslationEventLog
|
||||
|
||||
session = mock_api_deps["session"]
|
||||
job = _create_test_job(session)
|
||||
session.commit()
|
||||
|
||||
# Create an old event with per-language data
|
||||
old_date = datetime.now(UTC) - timedelta(days=100)
|
||||
event = TranslationEvent(
|
||||
id=str(uuid.uuid4()),
|
||||
job_id=job.id,
|
||||
event_type="TRANSLATION_PHASE_COMPLETED",
|
||||
event_data={
|
||||
"language_code": "ru",
|
||||
"token_count": 5000,
|
||||
"cost": 0.01,
|
||||
"run_id": "test-run-id",
|
||||
},
|
||||
created_at=old_date,
|
||||
)
|
||||
session.add(event)
|
||||
session.commit()
|
||||
|
||||
# Prune events older than 30 days
|
||||
event_log = TranslationEventLog(session)
|
||||
result = event_log.prune_expired(retention_days=30)
|
||||
assert result["snapshot_id"] is not None
|
||||
|
||||
# Verify snapshot has per_language_metrics
|
||||
snapshot = session.query(MetricSnapshot).filter(
|
||||
MetricSnapshot.id == result["snapshot_id"]
|
||||
).first()
|
||||
assert snapshot is not None
|
||||
assert snapshot.per_language_metrics is not None
|
||||
assert "ru" in snapshot.per_language_metrics
|
||||
assert snapshot.per_language_metrics["ru"]["cumulative_tokens"] == 5000
|
||||
assert snapshot.per_language_metrics["ru"]["cumulative_cost"] == 0.01
|
||||
# [/DEF:test_metric_snapshot_stores_per_language:Function]
|
||||
|
||||
|
||||
# [DEF:test_combined_metrics_live_and_snapshot:Function]
|
||||
# @PURPOSE: Verify combined metrics merge live + snapshot per-language data.
|
||||
def test_combined_metrics_live_and_snapshot(client, mock_api_deps):
|
||||
"""Test metrics merge live data with MetricSnapshot per-language data."""
|
||||
from datetime import timedelta
|
||||
from src.models.translate import (
|
||||
MetricSnapshot,
|
||||
TranslationEvent,
|
||||
TranslationRunLanguageStats,
|
||||
)
|
||||
from src.plugins.translate.metrics import TranslationMetrics
|
||||
|
||||
session = mock_api_deps["session"]
|
||||
job = _create_test_job(session)
|
||||
run = _create_test_run(session, job.id)
|
||||
session.commit()
|
||||
|
||||
# Add snapshot with per-language data (simulating pruned period)
|
||||
snapshot = MetricSnapshot(
|
||||
id=str(uuid.uuid4()),
|
||||
job_id=job.id,
|
||||
key_hash="test_key",
|
||||
per_language_metrics={
|
||||
"ru": {"cumulative_tokens": 100000, "cumulative_cost": 0.20, "runs": 3},
|
||||
},
|
||||
covers_events_before=datetime.now(UTC) - timedelta(days=90),
|
||||
total_records=100,
|
||||
snapshot_date=datetime.now(UTC),
|
||||
)
|
||||
session.add(snapshot)
|
||||
|
||||
# Add live language stats (recent runs)
|
||||
ls = TranslationRunLanguageStats(
|
||||
id=str(uuid.uuid4()),
|
||||
run_id=run.id,
|
||||
language_code="ru",
|
||||
total_rows=50,
|
||||
translated_rows=40,
|
||||
token_count=5000,
|
||||
estimated_cost=0.01,
|
||||
)
|
||||
session.add(ls)
|
||||
session.commit()
|
||||
|
||||
metrics_svc = TranslationMetrics(session)
|
||||
result = metrics_svc.get_job_metrics(job.id)
|
||||
assert "per_language_metrics" in result
|
||||
assert "ru" in result["per_language_metrics"]
|
||||
# Live 5000 + snapshot 100000 = 105000
|
||||
assert result["per_language_metrics"]["ru"]["tokens"] == 105000
|
||||
# Live 0.01 + snapshot 0.20 = 0.21
|
||||
assert abs(result["per_language_metrics"]["ru"]["cost"] - 0.21) < 0.001
|
||||
# Live 1 + snapshot 3 = 4
|
||||
assert result["per_language_metrics"]["ru"]["runs"] == 4
|
||||
# [/DEF:test_combined_metrics_live_and_snapshot:Function]
|
||||
# [/DEF:TranslateHistoryTests:Module]
|
||||
|
||||
Reference in New Issue
Block a user