From bb0fbfdafd13923b3387cf437c458e56996a986c Mon Sep 17 00:00:00 2001 From: busya Date: Thu, 14 May 2026 17:12:41 +0300 Subject: [PATCH] feat(translate): multi-language optimization (Phase 11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Auto-detection of source language per row via LLM (US6) - Multi-target translation — one LLM call for N languages (US1-US3) - Language-aware storage: TranslationLanguage, per-language stats - Multilingual dictionaries with language-pair-aware filtering (US7) - Inline correction on any run result + submit-to-dictionary (US8) - Context-aware dictionary: auto-capture row context, usage notes, Jaccard similarity, priority flagging in LLM prompts (US8b) - Configurable preview sample size 1-100, cost warning at >30 - Per-language history & metrics with MetricSnapshot preservation - 36 files, +5022/-373, all specs GRACE-Poly v2.6 compliant --- backend/alembic.ini | 149 ++++ backend/alembic/README | 1 + backend/alembic/env.py | 100 +++ backend/alembic/script.py.mako | 28 + ...3f02c_multi_language_translation_tables.py | 392 +++++++++++ .../routes/translate/_correction_routes.py | 2 + .../routes/translate/_dictionary_routes.py | 46 +- .../api/routes/translate/_preview_routes.py | 7 +- .../api/routes/translate/_run_list_routes.py | 19 + .../src/api/routes/translate/_run_routes.py | 197 ++++++ backend/src/models/translate.py | 117 +++- .../translate/__tests__/test_dictionary.py | 636 +++++++++++++++++- .../translate/__tests__/test_orchestrator.py | 249 +++++++ .../translate/__tests__/test_preview.py | 56 +- backend/src/plugins/translate/dictionary.py | 251 ++++++- backend/src/plugins/translate/events.py | 13 + backend/src/plugins/translate/executor.py | 261 ++++++- backend/src/plugins/translate/metrics.py | 40 +- backend/src/plugins/translate/orchestrator.py | 120 ++++ backend/src/plugins/translate/preview.py | 370 ++++++++-- .../src/plugins/translate/prompt_builder.py | 152 +++++ backend/src/plugins/translate/service.py | 488 ++++++++++++++ backend/src/schemas/translate.py | 171 ++++- backend/tests/test_translate_history.py | 168 +++++ frontend/src/lib/api/translate.js | 82 ++- .../translate/CorrectionCell.svelte | 273 ++++++++ .../translate/TermCorrectionPopup.svelte | 128 +++- .../translate/TranslationPreview.svelte | 450 ++++++++----- .../translate/TranslationRunResult.svelte | 132 ++++ .../src/lib/components/ui/MultiSelect.svelte | 122 ++++ .../src/lib/i18n/locales/en/translate.json | 10 +- .../src/lib/i18n/locales/ru/translate.json | 10 +- frontend/src/routes/translate/+page.svelte | 4 +- .../src/routes/translate/[id]/+page.svelte | 78 ++- .../translate/dictionaries/[id]/+page.svelte | 49 +- .../src/routes/translate/history/+page.svelte | 30 + 36 files changed, 5025 insertions(+), 376 deletions(-) create mode 100644 backend/alembic.ini create mode 100644 backend/alembic/README create mode 100644 backend/alembic/env.py create mode 100644 backend/alembic/script.py.mako create mode 100644 backend/alembic/versions/ed310b33f02c_multi_language_translation_tables.py create mode 100644 backend/src/plugins/translate/prompt_builder.py create mode 100644 frontend/src/lib/components/translate/CorrectionCell.svelte create mode 100644 frontend/src/lib/components/ui/MultiSelect.svelte diff --git a/backend/alembic.ini b/backend/alembic.ini new file mode 100644 index 00000000..4b8cceda --- /dev/null +++ b/backend/alembic.ini @@ -0,0 +1,149 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s +# Or organize into date-based subdirectories (requires recursive_version_locations = true) +# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the tzdata library which can be installed by adding +# `alembic[tz]` to the pip requirements. +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. This is consumed by the user-maintained env.py script only. +# other means of configuring database URLs may be customized within the env.py +# file. +sqlalchemy.url = sqlite:///./alembic_test.db + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/backend/alembic/README b/backend/alembic/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/backend/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/backend/alembic/env.py b/backend/alembic/env.py new file mode 100644 index 00000000..40fd0091 --- /dev/null +++ b/backend/alembic/env.py @@ -0,0 +1,100 @@ +import sys +from logging.config import fileConfig +from pathlib import Path + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# Ensure backend/src is importable +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +from src.models.mapping import Base +from src.models.translate import ( + TranslationJob, + TranslationRun, + TranslationBatch, + TranslationRecord, + TranslationEvent, + TranslationPreviewSession, + TranslationPreviewRecord, + TerminologyDictionary, + DictionaryEntry, + TranslationSchedule, + TranslationJobDictionary, + MetricSnapshot, + TranslationLanguage, + TranslationPreviewLanguage, + TranslationRunLanguageStats, +) + +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/backend/alembic/script.py.mako b/backend/alembic/script.py.mako new file mode 100644 index 00000000..11016301 --- /dev/null +++ b/backend/alembic/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/backend/alembic/versions/ed310b33f02c_multi_language_translation_tables.py b/backend/alembic/versions/ed310b33f02c_multi_language_translation_tables.py new file mode 100644 index 00000000..022fcc42 --- /dev/null +++ b/backend/alembic/versions/ed310b33f02c_multi_language_translation_tables.py @@ -0,0 +1,392 @@ +"""multi-language translation tables + +Revision ID: ed310b33f02c +Revises: +Create Date: 2026-05-14 15:35:03.167031 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'ed310b33f02c' +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('environments', + sa.Column('id', sa.String(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('url', sa.String(), nullable=False), + sa.Column('credentials_id', sa.String(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('terminology_dictionaries', + sa.Column('id', sa.String(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('source_dialect', sa.String(), nullable=False), + sa.Column('target_dialect', sa.String(), nullable=False), + sa.Column('source_language', sa.String(), nullable=True, comment='[DEPRECATED: use per-entry source_language]'), + sa.Column('target_language', sa.String(), nullable=True, comment='[DEPRECATED: use per-entry target_language]'), + sa.Column('is_active', sa.Boolean(), nullable=True), + sa.Column('created_by', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('translation_jobs', + sa.Column('id', sa.String(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('source_dialect', sa.String(), nullable=False), + sa.Column('target_dialect', sa.String(), nullable=False), + sa.Column('database_dialect', sa.String(), nullable=True, comment='Detected dialect from Superset connection at save time'), + sa.Column('status', sa.String(), nullable=False), + sa.Column('source_datasource_id', sa.String(), nullable=True, comment='Superset datasource ID'), + sa.Column('source_table', sa.String(), nullable=True, comment='Source table name resolved from datasource'), + sa.Column('target_schema', sa.String(), nullable=True, comment='Target table schema'), + sa.Column('target_table', sa.String(), nullable=True, comment='Target table name'), + sa.Column('source_key_cols', sa.JSON(), nullable=True, comment='Source key column names for composite key'), + sa.Column('target_key_cols', sa.JSON(), nullable=True, comment='Target key column names for composite key'), + sa.Column('translation_column', sa.String(), nullable=True, comment='Source column whose values will be translated'), + sa.Column('target_column', sa.String(), nullable=True, comment='Target column for translated output (defaults to translation_column)'), + sa.Column('context_columns', sa.JSON(), nullable=True, comment='Context column names included in LLM prompt'), + sa.Column('target_language', sa.String(), nullable=True, comment='Target language code (e.g. en, ru) [DEPRECATED: use target_languages]'), + sa.Column('source_language', sa.String(), nullable=True, comment='Fallback source language hint [DEPRECATED: auto-detected per row]'), + sa.Column('target_languages', sa.JSON(), nullable=True, comment='List of BCP-47 target language codes (multi-language support)'), + sa.Column('provider_id', sa.String(), nullable=True, comment='LLM provider ID'), + sa.Column('batch_size', sa.Integer(), nullable=False, comment='Records per batch'), + sa.Column('upsert_strategy', sa.String(), nullable=False, comment='MERGE, INSERT, UPDATE'), + sa.Column('environment_id', sa.String(), nullable=True, comment='Superset environment ID for datasource access'), + sa.Column('target_database_id', sa.String(), nullable=True, comment='Superset database ID for SQL Lab insert target'), + sa.Column('created_by', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('database_mappings', + sa.Column('id', sa.String(), nullable=False), + sa.Column('source_env_id', sa.String(), nullable=False), + sa.Column('target_env_id', sa.String(), nullable=False), + sa.Column('source_db_uuid', sa.String(), nullable=False), + sa.Column('target_db_uuid', sa.String(), nullable=False), + sa.Column('source_db_name', sa.String(), nullable=False), + sa.Column('target_db_name', sa.String(), nullable=False), + sa.Column('engine', sa.String(), nullable=True), + sa.ForeignKeyConstraint(['source_env_id'], ['environments.id'], ), + sa.ForeignKeyConstraint(['target_env_id'], ['environments.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('dictionary_entries', + sa.Column('id', sa.String(), nullable=False), + sa.Column('dictionary_id', sa.String(), nullable=False), + sa.Column('source_term', sa.String(), nullable=False), + sa.Column('source_term_normalized', sa.String(), nullable=False), + sa.Column('target_term', sa.String(), nullable=False), + sa.Column('source_language', sa.String(), nullable=False, comment='BCP-47 source language code'), + sa.Column('target_language', sa.String(), nullable=False, comment='BCP-47 target language code'), + sa.Column('context_notes', sa.Text(), nullable=True), + sa.Column('context_data', sa.JSON(), nullable=True, comment='Structured context for term usage'), + sa.Column('usage_notes', sa.Text(), nullable=True, comment='Usage guidance for the term mapping'), + sa.Column('has_context', sa.Boolean(), nullable=True, comment='Whether context_data is populated'), + sa.Column('context_source', sa.String(), nullable=True, comment='auto|auto_with_edits|manual|bulk'), + sa.Column('origin_source_language', sa.String(), nullable=True, comment='Original source language of the term'), + sa.Column('origin_run_id', sa.String(), nullable=True, comment='Run ID from which this correction originated'), + sa.Column('origin_row_key', sa.String(), nullable=True, comment='Row key within the run that triggered this correction'), + sa.Column('origin_user_id', sa.String(), nullable=True, comment='User who submitted the correction'), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['dictionary_id'], ['terminology_dictionaries.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('dictionary_id', 'source_term_normalized', 'source_language', 'target_language', name='uq_dict_source_term_lang') + ) + op.create_index('idx_dict_entry_lang', 'dictionary_entries', ['source_language', 'target_language'], unique=False) + op.create_index('idx_dict_has_context', 'dictionary_entries', ['has_context'], unique=False) + op.create_index(op.f('ix_dictionary_entries_dictionary_id'), 'dictionary_entries', ['dictionary_id'], unique=False) + op.create_table('migration_jobs', + sa.Column('id', sa.String(), nullable=False), + sa.Column('source_env_id', sa.String(), nullable=False), + sa.Column('target_env_id', sa.String(), nullable=False), + sa.Column('status', sa.Enum('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', 'AWAITING_MAPPING', name='migrationstatus'), nullable=True), + sa.Column('replace_db', sa.Boolean(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True), + sa.ForeignKeyConstraint(['source_env_id'], ['environments.id'], ), + sa.ForeignKeyConstraint(['target_env_id'], ['environments.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('resource_mappings', + sa.Column('id', sa.String(), nullable=False), + sa.Column('environment_id', sa.String(), nullable=False), + sa.Column('resource_type', sa.Enum('CHART', 'DATASET', 'DASHBOARD', name='resourcetype'), nullable=False), + sa.Column('uuid', sa.String(), nullable=False), + sa.Column('remote_integer_id', sa.String(), nullable=False), + sa.Column('resource_name', sa.String(), nullable=True), + sa.Column('last_synced_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True), + sa.ForeignKeyConstraint(['environment_id'], ['environments.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('translation_job_dictionaries', + sa.Column('id', sa.String(), nullable=False), + sa.Column('job_id', sa.String(), nullable=False), + sa.Column('dictionary_id', sa.String(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['dictionary_id'], ['terminology_dictionaries.id'], ), + sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('job_id', 'dictionary_id', name='uq_job_dictionary') + ) + op.create_index(op.f('ix_translation_job_dictionaries_dictionary_id'), 'translation_job_dictionaries', ['dictionary_id'], unique=False) + op.create_index(op.f('ix_translation_job_dictionaries_job_id'), 'translation_job_dictionaries', ['job_id'], unique=False) + op.create_table('translation_runs', + sa.Column('id', sa.String(), nullable=False), + sa.Column('job_id', sa.String(), nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('trigger_type', sa.String(), nullable=True, comment='manual, scheduled, retry, baseline_expired'), + sa.Column('started_at', sa.DateTime(), nullable=True), + sa.Column('completed_at', sa.DateTime(), nullable=True), + sa.Column('error_message', sa.Text(), nullable=True), + sa.Column('total_records', sa.Integer(), nullable=True), + sa.Column('successful_records', sa.Integer(), nullable=True), + sa.Column('failed_records', sa.Integer(), nullable=True), + sa.Column('skipped_records', sa.Integer(), nullable=True), + sa.Column('insert_status', sa.String(), nullable=True, comment='Status of the Superset insert/update operation'), + sa.Column('superset_execution_id', sa.String(), nullable=True, comment='Superset execution/task ID'), + sa.Column('superset_execution_log', sa.JSON(), nullable=True, comment='Superset execution log output'), + sa.Column('config_snapshot', sa.JSON(), nullable=True, comment='Snapshot of job config at run creation time'), + sa.Column('key_hash', sa.String(), nullable=True, comment='Hash of source key fields for dedup'), + sa.Column('config_hash', sa.String(), nullable=True, comment='Hash of translation configuration state'), + sa.Column('dict_snapshot_hash', sa.String(), nullable=True, comment='Hash of dictionary state at run time'), + sa.Column('created_by', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_translation_runs_job_id'), 'translation_runs', ['job_id'], unique=False) + op.create_table('translation_schedules', + sa.Column('id', sa.String(), nullable=False), + sa.Column('job_id', sa.String(), nullable=False), + sa.Column('cron_expression', sa.String(), nullable=False), + sa.Column('timezone', sa.String(), nullable=False), + sa.Column('is_active', sa.Boolean(), nullable=True), + sa.Column('last_run_at', sa.DateTime(), nullable=True), + sa.Column('next_run_at', sa.DateTime(), nullable=True), + sa.Column('execution_mode', sa.String(), nullable=False, comment='full, new_key_only'), + sa.Column('created_by', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_translation_schedules_job_id'), 'translation_schedules', ['job_id'], unique=False) + op.create_table('translation_batches', + sa.Column('id', sa.String(), nullable=False), + sa.Column('run_id', sa.String(), nullable=False), + sa.Column('batch_index', sa.Integer(), nullable=False), + sa.Column('status', sa.String(), nullable=False), + sa.Column('total_records', sa.Integer(), nullable=True), + sa.Column('successful_records', sa.Integer(), nullable=True), + sa.Column('failed_records', sa.Integer(), nullable=True), + sa.Column('started_at', sa.DateTime(), nullable=True), + sa.Column('completed_at', sa.DateTime(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_translation_batches_run_id'), 'translation_batches', ['run_id'], unique=False) + op.create_table('translation_events', + sa.Column('id', sa.String(), nullable=False), + sa.Column('job_id', sa.String(), nullable=False), + sa.Column('run_id', sa.String(), nullable=True), + sa.Column('event_type', sa.String(), nullable=False), + sa.Column('event_data', sa.JSON(), nullable=True), + sa.Column('created_by', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ), + sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_translation_events_job_id'), 'translation_events', ['job_id'], unique=False) + op.create_index(op.f('ix_translation_events_run_id'), 'translation_events', ['run_id'], unique=False) + op.create_table('translation_metric_snapshots', + sa.Column('id', sa.String(), nullable=False), + sa.Column('job_id', sa.String(), nullable=False), + sa.Column('run_id', sa.String(), nullable=True), + sa.Column('key_hash', sa.String(), nullable=False, comment='Hash of dimension key fields for aggregation'), + sa.Column('config_hash', sa.String(), nullable=True, comment='Hash of translation configuration state'), + sa.Column('dict_snapshot_hash', sa.String(), nullable=True, comment='Hash of dictionary state at capture time'), + sa.Column('covers_events_before', sa.DateTime(), nullable=True, comment='Indicates snapshot covers events before this timestamp'), + sa.Column('total_jobs', sa.Integer(), nullable=True), + sa.Column('total_runs', sa.Integer(), nullable=True), + sa.Column('total_records', sa.Integer(), nullable=True), + sa.Column('successful_records', sa.Integer(), nullable=True), + sa.Column('failed_records', sa.Integer(), nullable=True), + sa.Column('skipped_records', sa.Integer(), nullable=True), + sa.Column('avg_duration_ms', sa.Integer(), nullable=True), + sa.Column('p50_duration_ms', sa.Integer(), nullable=True), + sa.Column('p95_duration_ms', sa.Integer(), nullable=True), + sa.Column('p99_duration_ms', sa.Integer(), nullable=True), + sa.Column('snapshot_date', sa.DateTime(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ), + sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('ix_metric_snapshots_job_date', 'translation_metric_snapshots', ['job_id', 'snapshot_date'], unique=False) + op.create_index(op.f('ix_translation_metric_snapshots_job_id'), 'translation_metric_snapshots', ['job_id'], unique=False) + op.create_index(op.f('ix_translation_metric_snapshots_run_id'), 'translation_metric_snapshots', ['run_id'], unique=False) + op.create_table('translation_preview_sessions', + sa.Column('id', sa.String(), nullable=False), + sa.Column('job_id', sa.String(), nullable=False), + sa.Column('run_id', sa.String(), nullable=True), + sa.Column('status', sa.String(), nullable=False), + sa.Column('created_by', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('expires_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['job_id'], ['translation_jobs.id'], ), + sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_translation_preview_sessions_job_id'), 'translation_preview_sessions', ['job_id'], unique=False) + op.create_table('translation_run_language_stats', + sa.Column('id', sa.String(), nullable=False), + sa.Column('run_id', sa.String(), nullable=False), + sa.Column('language_code', sa.String(), nullable=False, comment='BCP-47 language code'), + sa.Column('total_rows', sa.Integer(), nullable=True), + sa.Column('translated_rows', sa.Integer(), nullable=True), + sa.Column('failed_rows', sa.Integer(), nullable=True), + sa.Column('skipped_rows', sa.Integer(), nullable=True), + sa.Column('token_count', sa.Integer(), nullable=True), + sa.Column('estimated_cost', sa.Float(), nullable=True), + sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('run_id', 'language_code', name='uq_run_language') + ) + op.create_index('idx_rls_run', 'translation_run_language_stats', ['run_id'], unique=False) + op.create_table('translation_preview_records', + sa.Column('id', sa.String(), nullable=False), + sa.Column('session_id', sa.String(), nullable=False), + sa.Column('source_sql', sa.Text(), nullable=True), + sa.Column('target_sql', sa.Text(), nullable=True), + sa.Column('source_object_type', sa.String(), nullable=True), + sa.Column('source_object_id', sa.String(), nullable=True), + sa.Column('source_object_name', sa.String(), nullable=True), + sa.Column('source_data', sa.JSON(), nullable=True, comment='Original source row key columns for upsert matching'), + sa.Column('status', sa.String(), nullable=False), + sa.Column('feedback', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['session_id'], ['translation_preview_sessions.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_translation_preview_records_session_id'), 'translation_preview_records', ['session_id'], unique=False) + op.create_table('translation_records', + sa.Column('id', sa.String(), nullable=False), + sa.Column('batch_id', sa.String(), nullable=False), + sa.Column('run_id', sa.String(), nullable=False), + sa.Column('source_sql', sa.Text(), nullable=True), + sa.Column('target_sql', sa.Text(), nullable=True), + sa.Column('source_object_type', sa.String(), nullable=True), + sa.Column('source_object_id', sa.String(), nullable=True), + sa.Column('source_object_name', sa.String(), nullable=True), + sa.Column('source_data', sa.JSON(), nullable=True, comment='Original source row key columns for upsert matching'), + sa.Column('status', sa.String(), nullable=False), + sa.Column('error_message', sa.Text(), nullable=True), + sa.Column('token_count_input', sa.Integer(), nullable=True), + sa.Column('token_count_output', sa.Integer(), nullable=True), + sa.Column('translation_duration_ms', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('llm_translation', sa.Text(), nullable=True, comment='[DEPRECATED: use TranslationLanguage]'), + sa.Column('user_edit', sa.Text(), nullable=True, comment='[DEPRECATED: use TranslationLanguage]'), + sa.Column('final_value', sa.Text(), nullable=True, comment='[DEPRECATED: use TranslationLanguage]'), + sa.ForeignKeyConstraint(['batch_id'], ['translation_batches.id'], ), + sa.ForeignKeyConstraint(['run_id'], ['translation_runs.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_translation_records_batch_id'), 'translation_records', ['batch_id'], unique=False) + op.create_index(op.f('ix_translation_records_run_id'), 'translation_records', ['run_id'], unique=False) + op.create_index('ix_translation_records_run_status', 'translation_records', ['run_id', 'status'], unique=False) + op.create_table('translation_languages', + sa.Column('id', sa.String(), nullable=False), + sa.Column('record_id', sa.String(), nullable=False), + sa.Column('language_code', sa.String(), nullable=False, comment='BCP-47 language code'), + sa.Column('source_language_detected', sa.String(), nullable=True, comment="BCP-47 or 'und' for undetermined"), + sa.Column('translated_value', sa.Text(), nullable=True, comment='LLM-generated translation'), + sa.Column('user_edit', sa.Text(), nullable=True, comment='User-edited translation'), + sa.Column('final_value', sa.Text(), nullable=True, comment='Final resolved value (translated or user edit)'), + sa.Column('status', sa.String(), nullable=True, comment='pending|translated|approved|edited|rejected|failed|skipped'), + sa.Column('error_message', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(['record_id'], ['translation_records.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('record_id', 'language_code', name='uq_record_language') + ) + op.create_index('idx_tl_language', 'translation_languages', ['language_code'], unique=False) + op.create_index('idx_tl_record_lang', 'translation_languages', ['record_id', 'language_code'], unique=False) + op.create_table('translation_preview_languages', + sa.Column('id', sa.String(), nullable=False), + sa.Column('preview_record_id', sa.String(), nullable=False), + sa.Column('language_code', sa.String(), nullable=False, comment='BCP-47 language code'), + sa.Column('source_language_detected', sa.String(), nullable=True, comment="BCP-47 or 'und'"), + sa.Column('translated_value', sa.Text(), nullable=True), + sa.Column('user_edit', sa.Text(), nullable=True), + sa.Column('final_value', sa.Text(), nullable=True), + sa.Column('status', sa.String(), nullable=True, comment='pending|approved|edited|rejected'), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(['preview_record_id'], ['translation_preview_records.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('preview_record_id', 'language_code', name='uq_preview_record_language') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('translation_preview_languages') + op.drop_index('idx_tl_record_lang', table_name='translation_languages') + op.drop_index('idx_tl_language', table_name='translation_languages') + op.drop_table('translation_languages') + op.drop_index('ix_translation_records_run_status', table_name='translation_records') + op.drop_index(op.f('ix_translation_records_run_id'), table_name='translation_records') + op.drop_index(op.f('ix_translation_records_batch_id'), table_name='translation_records') + op.drop_table('translation_records') + op.drop_index(op.f('ix_translation_preview_records_session_id'), table_name='translation_preview_records') + op.drop_table('translation_preview_records') + op.drop_index('idx_rls_run', table_name='translation_run_language_stats') + op.drop_table('translation_run_language_stats') + op.drop_index(op.f('ix_translation_preview_sessions_job_id'), table_name='translation_preview_sessions') + op.drop_table('translation_preview_sessions') + op.drop_index(op.f('ix_translation_metric_snapshots_run_id'), table_name='translation_metric_snapshots') + op.drop_index(op.f('ix_translation_metric_snapshots_job_id'), table_name='translation_metric_snapshots') + op.drop_index('ix_metric_snapshots_job_date', table_name='translation_metric_snapshots') + op.drop_table('translation_metric_snapshots') + op.drop_index(op.f('ix_translation_events_run_id'), table_name='translation_events') + op.drop_index(op.f('ix_translation_events_job_id'), table_name='translation_events') + op.drop_table('translation_events') + op.drop_index(op.f('ix_translation_batches_run_id'), table_name='translation_batches') + op.drop_table('translation_batches') + op.drop_index(op.f('ix_translation_schedules_job_id'), table_name='translation_schedules') + op.drop_table('translation_schedules') + op.drop_index(op.f('ix_translation_runs_job_id'), table_name='translation_runs') + op.drop_table('translation_runs') + op.drop_index(op.f('ix_translation_job_dictionaries_job_id'), table_name='translation_job_dictionaries') + op.drop_index(op.f('ix_translation_job_dictionaries_dictionary_id'), table_name='translation_job_dictionaries') + op.drop_table('translation_job_dictionaries') + op.drop_table('resource_mappings') + op.drop_table('migration_jobs') + op.drop_index(op.f('ix_dictionary_entries_dictionary_id'), table_name='dictionary_entries') + op.drop_index('idx_dict_has_context', table_name='dictionary_entries') + op.drop_index('idx_dict_entry_lang', table_name='dictionary_entries') + op.drop_table('dictionary_entries') + op.drop_table('database_mappings') + op.drop_table('translation_jobs') + op.drop_table('terminology_dictionaries') + op.drop_table('environments') + # ### end Alembic commands ### diff --git a/backend/src/api/routes/translate/_correction_routes.py b/backend/src/api/routes/translate/_correction_routes.py index b172b0ec..404181e3 100644 --- a/backend/src/api/routes/translate/_correction_routes.py +++ b/backend/src/api/routes/translate/_correction_routes.py @@ -45,6 +45,8 @@ async def submit_correction( origin_run_id=payload.origin_run_id, origin_row_key=payload.origin_row_key, origin_user_id=current_user.username, + context_data=payload.context_data, + usage_notes=payload.usage_notes, ) return result except ValueError as e: diff --git a/backend/src/api/routes/translate/_dictionary_routes.py b/backend/src/api/routes/translate/_dictionary_routes.py index e92650e1..743b841b 100644 --- a/backend/src/api/routes/translate/_dictionary_routes.py +++ b/backend/src/api/routes/translate/_dictionary_routes.py @@ -163,14 +163,16 @@ async def delete_dictionary( # ============================================================ # #region list_dictionary_entries [TYPE Function] -# @BRIEF List entries for a dictionary. +# @BRIEF List entries for a dictionary, optionally filtered by language pair. # @PRE: User has translate.dictionary.view permission. -# @POST: Returns paginated list of entries. +# @POST: Returns paginated list of entries with language pair fields. @router.get("/dictionaries/{dictionary_id}/entries") async def list_dictionary_entries( dictionary_id: str, page: int = Query(1, ge=1), page_size: int = Query(50, ge=1, le=500), + source_language: str | None = Query(None, description="Filter by BCP-47 source language"), + target_language: str | None = Query(None, description="Filter by BCP-47 target language"), current_user: User = Depends(get_current_user), _ = Depends(has_permission("translate.dictionary", "VIEW")), db: Session = Depends(get_db), @@ -180,6 +182,17 @@ async def list_dictionary_entries( logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") try: entries, total = DictionaryManager.list_entries(db, dictionary_id, page=page, page_size=page_size) + # Apply language pair filtering + if source_language or target_language: + filtered = [] + for e in entries: + if source_language and e.source_language != source_language and e.source_language != "und": + continue + if target_language and e.target_language != target_language: + continue + filtered.append(e) + entries = filtered + total = len(filtered) items = [ { "id": e.id, @@ -187,13 +200,20 @@ async def list_dictionary_entries( "source_term": e.source_term, "source_term_normalized": e.source_term_normalized, "target_term": e.target_term, + "source_language": e.source_language, + "target_language": e.target_language, "context_notes": e.context_notes, + "context_data": e.context_data, + "usage_notes": e.usage_notes, + "has_context": e.has_context, + "context_source": e.context_source, + "origin_source_language": e.origin_source_language, "created_at": e.created_at, "updated_at": e.updated_at, } for e in entries ] - logger.reflect("Entries listed", {"dict_id": dictionary_id, "total": total}) + logger.reflect("Entries listed", {"dict_id": dictionary_id, "total": total, "source_language": source_language, "target_language": target_language}) return {"items": items, "total": total, "page": page, "page_size": page_size} except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) @@ -221,6 +241,8 @@ async def add_dictionary_entry( source_term=payload.source_term, target_term=payload.target_term, context_notes=payload.context_notes, + source_language=payload.source_language, + target_language=payload.target_language, ) logger.reflect("Entry added", {"entry_id": entry.id}) return { @@ -229,7 +251,14 @@ async def add_dictionary_entry( "source_term": entry.source_term, "source_term_normalized": entry.source_term_normalized, "target_term": entry.target_term, + "source_language": entry.source_language, + "target_language": entry.target_language, "context_notes": entry.context_notes, + "context_data": entry.context_data, + "usage_notes": entry.usage_notes, + "has_context": entry.has_context, + "context_source": entry.context_source, + "origin_source_language": entry.origin_source_language, "created_at": entry.created_at, "updated_at": entry.updated_at, } @@ -262,6 +291,8 @@ async def edit_dictionary_entry( source_term=payload.source_term, target_term=payload.target_term, context_notes=payload.context_notes, + source_language=payload.source_language, + target_language=payload.target_language, ) logger.reflect("Entry updated", {"entry_id": entry_id}) return { @@ -270,7 +301,14 @@ async def edit_dictionary_entry( "source_term": entry.source_term, "source_term_normalized": entry.source_term_normalized, "target_term": entry.target_term, + "source_language": entry.source_language, + "target_language": entry.target_language, "context_notes": entry.context_notes, + "context_data": entry.context_data, + "usage_notes": entry.usage_notes, + "has_context": entry.has_context, + "context_source": entry.context_source, + "origin_source_language": entry.origin_source_language, "created_at": entry.created_at, "updated_at": entry.updated_at, } @@ -335,6 +373,8 @@ async def import_dictionary_entries( delimiter=payload.delimiter, on_conflict=payload.on_conflict, preview_only=payload.preview_only, + default_source_language=payload.default_source_language, + default_target_language=payload.default_target_language, ) logger.reflect("Import completed", { "dict_id": dictionary_id, diff --git a/backend/src/api/routes/translate/_preview_routes.py b/backend/src/api/routes/translate/_preview_routes.py index 30206b23..2dfe776b 100644 --- a/backend/src/api/routes/translate/_preview_routes.py +++ b/backend/src/api/routes/translate/_preview_routes.py @@ -59,7 +59,7 @@ async def preview_translation( # #region update_preview_row [TYPE Function] -# @BRIEF Approve, edit, or reject a preview row. +# @BRIEF Approve, edit, or reject a preview row (optionally per language). # @PRE: User has translate.job.execute permission. # @POST: Preview row status is updated. @router.put("/jobs/{job_id}/preview/rows/{row_key}") @@ -72,8 +72,8 @@ async def update_preview_row( db: Session = Depends(get_db), config_manager: ConfigManager = Depends(get_config_manager), ): - """Approve, edit, or reject a preview row.""" - logger.reason(f"update_preview_row — Job: {job_id}, Row: {row_key}, Action: {payload.action}", extra={"src": "translate_routes"}) + """Approve, edit, or reject a preview row (optionally per language).""" + logger.reason(f"update_preview_row — Job: {job_id}, Row: {row_key}, Action: {payload.action}, Lang: {payload.language_code}", extra={"src": "translate_routes"}) try: preview_service = TranslationPreview(db, config_manager, current_user.username) result = preview_service.update_preview_row( @@ -82,6 +82,7 @@ async def update_preview_row( action=payload.action, translation=payload.translation, feedback=payload.feedback, + language_code=payload.language_code, ) return result except ValueError as e: diff --git a/backend/src/api/routes/translate/_run_list_routes.py b/backend/src/api/routes/translate/_run_list_routes.py index 5677ab0d..a656d74c 100644 --- a/backend/src/api/routes/translate/_run_list_routes.py +++ b/backend/src/api/routes/translate/_run_list_routes.py @@ -76,6 +76,22 @@ async def list_runs( items = [] for r in runs: + # Gather per-language statistics from the run's language_stats relationship + lang_stats = r.language_stats or [] + target_languages = [ls.language_code for ls in lang_stats] + total_translated = sum((ls.translated_rows or 0) for ls in lang_stats) + language_stats = { + ls.language_code: { + "total_rows": ls.total_rows or 0, + "translated_rows": ls.translated_rows or 0, + "failed_rows": ls.failed_rows or 0, + "skipped_rows": ls.skipped_rows or 0, + "token_count": ls.token_count or 0, + "estimated_cost": ls.estimated_cost or 0.0, + } + for ls in lang_stats + } + items.append({ "id": r.id, "job_id": r.job_id, @@ -94,6 +110,9 @@ async def list_runs( "dict_snapshot_hash": r.dict_snapshot_hash, "created_by": r.created_by, "created_at": r.created_at.isoformat() if r.created_at else None, + "target_languages": target_languages, + "total_translated": total_translated, + "language_stats": language_stats, }) return {"items": items, "total": total, "page": page, "page_size": page_size} diff --git a/backend/src/api/routes/translate/_run_routes.py b/backend/src/api/routes/translate/_run_routes.py index 56d05973..67d9a5f8 100644 --- a/backend/src/api/routes/translate/_run_routes.py +++ b/backend/src/api/routes/translate/_run_routes.py @@ -13,6 +13,11 @@ from ....core.logger import logger from ....dependencies import get_config_manager, get_current_user, has_permission from ....plugins.translate.orchestrator import TranslationOrchestrator from ....schemas.auth import User +from ....schemas.translate import ( + BulkFindReplaceRequest, + InlineEditRequest, + OverrideLanguageRequest, +) from ._helpers import _run_to_response from ._router import router @@ -341,4 +346,196 @@ async def get_batches( raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion get_batches +# ============================================================ +# Manual Language Override +# ============================================================ + +# #region override_detected_language [TYPE Function] +# @BRIEF Manually override the detected source language for a specific translation language entry. +# @PRE: User has translate.job.execute permission. Run, record, and language entry exist. +# @POST: TranslationLanguage.source_language_detected is updated; language_overridden is set to True. +# @SIDE_EFFECT: DB write. +@router.put("/runs/{run_id}/records/{record_id}/languages/{language_code}/override-language") +async def override_detected_language( + run_id: str, + record_id: str, + language_code: str, + payload: OverrideLanguageRequest, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "EXECUTE")), + db: Session = Depends(get_db), +): + """Manually override the detected source language for a translation entry.""" + logger.reason(f"override_detected_language — Run: {run_id}, Record: {record_id}, Lang: {language_code}, Override: {payload.source_language}", extra={"src": "translate_routes"}) + try: + from ....models.translate import TranslationLanguage, TranslationRecord + + # Verify the record exists and belongs to the run + record = ( + db.query(TranslationRecord) + .filter( + TranslationRecord.id == record_id, + TranslationRecord.run_id == run_id, + ) + .first() + ) + if not record: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Translation record '{record_id}' not found in run '{run_id}'", + ) + + # Find the language entry + lang_entry = ( + db.query(TranslationLanguage) + .filter( + TranslationLanguage.record_id == record_id, + TranslationLanguage.language_code == language_code, + ) + .first() + ) + if not lang_entry: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Language entry '{language_code}' not found for record '{record_id}'", + ) + + # Update the language entry + lang_entry.source_language_detected = payload.source_language + lang_entry.language_overridden = True + lang_entry.needs_review = False # Override clears the review flag + db.commit() + db.refresh(lang_entry) + + logger.reason("Language override applied", { + "run_id": run_id, + "record_id": record_id, + "language_code": language_code, + "new_source_language": payload.source_language, + "overridden_by": current_user.username, + }) + + from ....schemas.translate import TranslationLanguageResponse + return TranslationLanguageResponse.model_validate(lang_entry) + except HTTPException: + raise + except Exception as e: + logger.explore("override_detected_language failed", extra={"src": "translate_routes", "error": str(e)}) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# #endregion override_detected_language + + +# ============================================================ +# Inline Correction +# ============================================================ + +# #region inline_edit_translation [C:3] [TYPE Function] [SEMANTICS api,translate,correction] +# @BRIEF Apply an inline correction to a translated value on a completed run result. +# @PRE: User has translate.job.execute permission. Run, record, and language entry exist. +# @POST: TranslationLanguage.final_value and user_edit are updated. Optional dictionary submission. +@router.put("/runs/{run_id}/records/{record_id}/languages/{language_code}") +async def inline_edit_translation( + run_id: str, + record_id: str, + language_code: str, + payload: InlineEditRequest, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "EXECUTE")), + db: Session = Depends(get_db), +): + """Apply an inline correction to a translated value.""" + logger.reason( + f"inline_edit_translation — Run: {run_id}, Record: {record_id}, " + f"Lang: {language_code}, User: {current_user.username}", + extra={"src": "translate_routes"}, + ) + try: + from ....plugins.translate.service import InlineCorrectionService + + result = InlineCorrectionService.apply_inline_edit( + db=db, + run_id=run_id, + record_id=record_id, + language_code=language_code, + final_value=payload.final_value, + submit_to_dictionary=payload.submit_to_dictionary, + dictionary_id=payload.dictionary_id, + current_user=current_user.username, + context_data_override=payload.context_data_override, + usage_notes=payload.usage_notes, + keep_context=payload.keep_context, + ) + return result + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except Exception as e: + logger.explore("inline_edit_translation failed", extra={"src": "translate_routes", "error": str(e)}) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# #endregion inline_edit_translation + + +# ============================================================ +# Bulk Find-and-Replace +# ============================================================ + +# #region bulk_find_replace [C:3] [TYPE Function] [SEMANTICS api,translate,bulk,replace] +# @BRIEF Perform bulk find-and-replace on translated values within a run. +# @PRE: User has translate.job.execute permission. Run exists. +# @POST: If preview=false, matching translations are updated. Optional dictionary submission. +@router.post("/runs/{run_id}/bulk-replace") +async def bulk_find_replace( + run_id: str, + payload: BulkFindReplaceRequest, + current_user: User = Depends(get_current_user), + _ = Depends(has_permission("translate.job", "EXECUTE")), + db: Session = Depends(get_db), +): + """Perform bulk find-and-replace on translated values within a run.""" + logger.reason( + f"bulk_find_replace — Run: {run_id}, Pattern: '{payload.find_pattern}', " + f"Regex: {payload.is_regex}, Lang: {payload.target_language}, " + f"User: {current_user.username}", + extra={"src": "translate_routes"}, + ) + try: + from ....plugins.translate.service import BulkFindReplaceService + + if payload.preview: + # Preview mode: return matches without applying + preview = BulkFindReplaceService.preview( + db=db, + run_id=run_id, + pattern=payload.find_pattern, + is_regex=payload.is_regex, + target_language=payload.target_language, + ) + return { + "rows_affected": len(preview), + "corrections_submitted": 0, + "preview": preview, + } + + # Apply mode: perform replacements + result = BulkFindReplaceService.apply( + db=db, + run_id=run_id, + pattern=payload.find_pattern, + is_regex=payload.is_regex, + replacement_text=payload.replacement_text, + target_language=payload.target_language, + submit_to_dictionary=payload.submit_to_dictionary, + dictionary_id=payload.dictionary_id, + usage_notes=payload.usage_notes, + current_user=current_user.username, + submit_to_dictionary_with_context=payload.submit_to_dictionary_with_context, + ) + return result + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + except Exception as e: + logger.explore("bulk_find_replace failed", extra={"src": "translate_routes", "error": str(e)}) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# #endregion bulk_find_replace + + # #endregion TranslateRunRoutesModule diff --git a/backend/src/models/translate.py b/backend/src/models/translate.py index d2d106a5..f7bf377b 100644 --- a/backend/src/models/translate.py +++ b/backend/src/models/translate.py @@ -6,7 +6,8 @@ import uuid from datetime import UTC, datetime -from sqlalchemy import JSON, Boolean, Column, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint +from sqlalchemy import JSON, Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import relationship from .mapping import Base @@ -44,7 +45,11 @@ class TranslationJob(Base): context_columns = Column(JSON, nullable=True, comment="Context column names included in LLM prompt") # LLM & processing settings - target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru)") + # @DEPRECATED — use target_languages + target_language = Column(String, nullable=True, comment="Target language code (e.g. en, ru) [DEPRECATED: use target_languages]") + # @DEPRECATED — Auto-detected per row by LLM, kept as fallback hint + source_language = Column(String, nullable=True, comment="Fallback source language hint [DEPRECATED: auto-detected per row]") + target_languages = Column(JSON, nullable=True, comment="List of BCP-47 target language codes (multi-language support)") provider_id = Column(String, nullable=True, comment="LLM provider ID") batch_size = Column(Integer, nullable=False, default=50, comment="Records per batch") upsert_strategy = Column(String, nullable=False, default="MERGE", comment="MERGE, INSERT, UPDATE") @@ -84,6 +89,8 @@ class TranslationRun(Base): dict_snapshot_hash = Column(String, nullable=True, comment="Hash of dictionary state at run time") created_by = Column(String, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(UTC)) + + language_stats = relationship("TranslationRunLanguageStats", back_populates="run") # #endregion TranslationRun @@ -126,6 +133,15 @@ class TranslationRecord(Base): translation_duration_ms = Column(Integer, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(UTC)) + # @DEPRECATED — use TranslationLanguage instead + llm_translation = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]") + # @DEPRECATED — use TranslationLanguage instead + user_edit = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]") + # @DEPRECATED — use TranslationLanguage instead + final_value = Column(Text, nullable=True, comment="[DEPRECATED: use TranslationLanguage]") + + languages = relationship("TranslationLanguage", back_populates="record") + __table_args__ = ( Index("ix_translation_records_run_status", "run_id", "status"), ) @@ -178,6 +194,8 @@ class TranslationPreviewRecord(Base): status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED feedback = Column(Text, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(UTC)) + + languages = relationship("TranslationPreviewLanguage", back_populates="preview_record") # #endregion TranslationPreviewRecord @@ -191,6 +209,10 @@ class TerminologyDictionary(Base): description = Column(Text, nullable=True) source_dialect = Column(String, nullable=False) target_dialect = Column(String, nullable=False) + # @DEPRECATED — use per-entry source_language instead, kept for backward compat + source_language = Column(String, nullable=True, comment="[DEPRECATED: use per-entry source_language]") + # @DEPRECATED — use per-entry target_language instead, kept for backward compat + target_language = Column(String, nullable=True, comment="[DEPRECATED: use per-entry target_language]") is_active = Column(Boolean, default=True) created_by = Column(String, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(UTC)) @@ -208,7 +230,14 @@ class DictionaryEntry(Base): source_term = Column(String, nullable=False) source_term_normalized = Column(String, nullable=False) target_term = Column(String, nullable=False) + source_language = Column(String, nullable=False, comment="BCP-47 source language code") + target_language = Column(String, nullable=False, comment="BCP-47 target language code") context_notes = Column(Text, nullable=True) + context_data = Column(JSON, nullable=True, comment="Structured context for term usage") + usage_notes = Column(Text, nullable=True, comment="Usage guidance for the term mapping") + has_context = Column(Boolean, default=False, comment="Whether context_data is populated") + context_source = Column(String, nullable=True, comment="auto|auto_with_edits|manual|bulk") + origin_source_language = Column(String, nullable=True, comment="Original source language of the term") origin_run_id = Column(String, nullable=True, comment="Run ID from which this correction originated") origin_row_key = Column(String, nullable=True, comment="Row key within the run that triggered this correction") origin_user_id = Column(String, nullable=True, comment="User who submitted the correction") @@ -217,9 +246,11 @@ class DictionaryEntry(Base): __table_args__ = ( UniqueConstraint( - "dictionary_id", "source_term_normalized", - name="uq_dictionary_entry_term" + "dictionary_id", "source_term_normalized", "source_language", "target_language", + name="uq_dict_source_term_lang" ), + Index("idx_dict_entry_lang", "source_language", "target_language"), + Index("idx_dict_has_context", "has_context"), ) # #endregion DictionaryEntry @@ -284,6 +315,7 @@ class MetricSnapshot(Base): p50_duration_ms = Column(Integer, nullable=True) p95_duration_ms = Column(Integer, nullable=True) p99_duration_ms = Column(Integer, nullable=True) + per_language_metrics = Column(JSON, nullable=True, comment="Per-language cumulative metrics: {lang: {cumulative_tokens, cumulative_cost, runs}}") snapshot_date = Column(DateTime, nullable=False, default=lambda: datetime.now(UTC)) created_at = Column(DateTime, default=lambda: datetime.now(UTC)) @@ -292,4 +324,81 @@ class MetricSnapshot(Base): ) # #endregion MetricSnapshot + +# #region TranslationLanguage [C:1] [TYPE Class] +# @BRIEF Per-language translation result for a single record, supporting multi-language output. +class TranslationLanguage(Base): + __tablename__ = "translation_languages" + + id = Column(String, primary_key=True, default=generate_uuid) + record_id = Column(String, ForeignKey("translation_records.id"), nullable=False) + language_code = Column(String, nullable=False, comment="BCP-47 language code") + source_language_detected = Column(String, nullable=True, comment="BCP-47 or 'und' for undetermined") + translated_value = Column(Text, nullable=True, comment="LLM-generated translation") + user_edit = Column(Text, nullable=True, comment="User-edited translation") + final_value = Column(Text, nullable=True, comment="Final resolved value (translated or user edit)") + status = Column(String, default="pending", comment="pending|translated|approved|edited|rejected|failed|skipped") + error_message = Column(Text, nullable=True) + needs_review = Column(Boolean, default=False, comment="Flagged because source language could not be determined") + language_overridden = Column(Boolean, default=False, comment="Source language was manually overridden by user") + created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(UTC)) + + record = relationship("TranslationRecord", back_populates="languages") + + __table_args__ = ( + UniqueConstraint("record_id", "language_code", name="uq_record_language"), + Index("idx_tl_language", "language_code"), + Index("idx_tl_record_lang", "record_id", "language_code"), + ) +# #endregion TranslationLanguage + + +# #region TranslationPreviewLanguage [C:1] [TYPE Class] +# @BRIEF Per-language preview entry within a preview session. +class TranslationPreviewLanguage(Base): + __tablename__ = "translation_preview_languages" + + id = Column(String, primary_key=True, default=generate_uuid) + preview_record_id = Column(String, ForeignKey("translation_preview_records.id"), nullable=False) + language_code = Column(String, nullable=False, comment="BCP-47 language code") + source_language_detected = Column(String, nullable=True, comment="BCP-47 or 'und'") + translated_value = Column(Text, nullable=True) + user_edit = Column(Text, nullable=True) + final_value = Column(Text, nullable=True) + status = Column(String, default="pending", comment="pending|approved|edited|rejected") + needs_review = Column(Boolean, default=False, comment="Flagged because source language could not be determined") + created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(UTC)) + + preview_record = relationship("TranslationPreviewRecord", back_populates="languages") + + __table_args__ = ( + UniqueConstraint("preview_record_id", "language_code", name="uq_preview_record_language"), + ) +# #endregion TranslationPreviewLanguage + + +# #region TranslationRunLanguageStats [C:1] [TYPE Class] +# @BRIEF Per-language statistics for a translation run (row counts, tokens, cost). +class TranslationRunLanguageStats(Base): + __tablename__ = "translation_run_language_stats" + + id = Column(String, primary_key=True, default=generate_uuid) + run_id = Column(String, ForeignKey("translation_runs.id"), nullable=False) + language_code = Column(String, nullable=False, comment="BCP-47 language code") + total_rows = Column(Integer, default=0) + translated_rows = Column(Integer, default=0) + failed_rows = Column(Integer, default=0) + skipped_rows = Column(Integer, default=0) + token_count = Column(Integer, default=0) + estimated_cost = Column(Float, default=0.0) + + run = relationship("TranslationRun", back_populates="language_stats") + + __table_args__ = ( + UniqueConstraint("run_id", "language_code", name="uq_run_language"), + Index("idx_rls_run", "run_id"), + ) +# #endregion TranslationRunLanguageStats + + # #endregion TranslateModels diff --git a/backend/src/plugins/translate/__tests__/test_dictionary.py b/backend/src/plugins/translate/__tests__/test_dictionary.py index 524d035a..127712cf 100644 --- a/backend/src/plugins/translate/__tests__/test_dictionary.py +++ b/backend/src/plugins/translate/__tests__/test_dictionary.py @@ -1,29 +1,37 @@ # region DictionaryTests [TYPE Module] -# @SEMANTICS: tests, dictionary, crud, import, filter -# @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, and batch filtering. +# @SEMANTICS: tests, dictionary, crud, import, filter, language_pair +# @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, batch filtering, and language-pair support. # @RELATION: BINDS_TO -> [DictionaryManager:Class] # # @TEST_CONTRACT: [DictionaryManager] -> { # invariants: [ # "Create/Read/Update/Delete dictionaries works", -# "Entry CRUD enforces unique source_term_normalized per dictionary", +# "Entry CRUD enforces unique (dictionary_id, source_term_normalized, source_language, target_language)", # "Import CSV/TSV handles overwrite/keep_existing/cancel conflict modes", # "Delete blocked when dictionary attached to active/scheduled jobs", -# "filter_for_batch returns matched entries with word-boundary awareness" +# "filter_for_batch returns matched entries with word-boundary awareness", +# "filter_for_batch supports language-pair filtering", +# "Language-pair-aware duplicate detection: same term with different lang pair is allowed" # ] # } -# @TEST_EDGE: duplicate_entry -> 409-style ValueError on repeated source_term +# @TEST_EDGE: duplicate_entry -> 409-style ValueError on repeated (dictionary_id, source_term_norm, source_lang, target_lang) # @TEST_EDGE: delete_active_job -> ValueError with active/scheduled message # @TEST_EDGE: import_invalid_format -> ValueError for missing columns -# @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry] +# @TEST_EDGE: same_term_different_lang_pair -> allowed (not duplicate) +# @TEST_INVARIANT: unique_normalized -> verifies: [duplicate_entry, same_term_different_lang_pair allowed] +import csv +import io + import pytest from sqlalchemy import create_engine, event from sqlalchemy.orm import Session, sessionmaker from src.models.translate import ( Base, + DictionaryEntry, + TerminologyDictionary, TranslationJob, TranslationJobDictionary, ) @@ -112,8 +120,8 @@ def test_delete_dictionary(db_session: Session): db_session, name="To Delete", source_dialect="a", target_dialect="b", ) - # Add an entry - entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + # Add an entry with language pair + entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es") assert entry.id is not None DictionaryManager.delete_dictionary(db_session, d.id) @@ -144,15 +152,15 @@ def test_add_entry_duplicate(db_session: Session): db_session, name="Test", source_dialect="a", target_dialect="b", ) - DictionaryManager.add_entry(db_session, d.id, "Hello", "Hola") + DictionaryManager.add_entry(db_session, d.id, "Hello", "Hola", source_language="en", target_language="es") - # Same normalized term should raise + # Same normalized term with same language pair should raise with pytest.raises(ValueError, match="already exists"): - DictionaryManager.add_entry(db_session, d.id, "hello", "Bonjour") + DictionaryManager.add_entry(db_session, d.id, "hello", "Bonjour", source_language="en", target_language="es") - # Different case, same normalized should also raise + # Different case, same normalized, same lang pair should also raise with pytest.raises(ValueError, match="already exists"): - DictionaryManager.add_entry(db_session, d.id, "HELLO", "Ciao") + DictionaryManager.add_entry(db_session, d.id, "HELLO", "Ciao", source_language="en", target_language="es") # endregion test_add_entry_duplicate @@ -167,9 +175,9 @@ def test_add_entry_duplicate_per_dictionary(db_session: Session): db_session, name="Dict2", source_dialect="a", target_dialect="b", ) - DictionaryManager.add_entry(db_session, d1.id, "hello", "hola") + DictionaryManager.add_entry(db_session, d1.id, "hello", "hola", source_language="en", target_language="es") # Same term in different dictionary should work - entry = DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour") + entry = DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour", source_language="en", target_language="es") assert entry.id is not None # endregion test_add_entry_duplicate_per_dictionary @@ -181,13 +189,13 @@ def test_edit_entry(db_session: Session): db_session, name="Test", source_dialect="a", target_dialect="b", ) - entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es") updated = DictionaryManager.edit_entry(db_session, entry.id, target_term="HOLA!") assert updated.target_term == "HOLA!" # Edit source term to a value that's already taken should fail - DictionaryManager.add_entry(db_session, d.id, "world", "mundo") + DictionaryManager.add_entry(db_session, d.id, "world", "mundo", source_language="en", target_language="es") with pytest.raises(ValueError, match="already exists"): DictionaryManager.edit_entry(db_session, entry.id, source_term="WORLD") # endregion test_edit_entry @@ -200,7 +208,7 @@ def test_delete_entry(db_session: Session): db_session, name="Test", source_dialect="a", target_dialect="b", ) - entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + entry = DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es") DictionaryManager.delete_entry(db_session, entry.id) entries, total = DictionaryManager.list_entries(db_session, d.id) @@ -215,10 +223,10 @@ def test_import_csv_overwrite(db_session: Session): db_session, name="Test", source_dialect="a", target_dialect="b", ) - # Pre-add an entry - DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + # Pre-add an entry with language pair + DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es") - csv_content = "source_term,target_term,context_notes\nhello,HELLO,\nworld,mundo," + csv_content = "source_term,target_term,context_notes,source_language,target_language\nhello,HELLO,,en,es\nworld,mundo,,en,es" result = DictionaryManager.import_entries( db_session, d.id, csv_content, delimiter=",", on_conflict="overwrite", @@ -242,9 +250,9 @@ def test_import_csv_keep_existing(db_session: Session): db_session, name="Test", source_dialect="a", target_dialect="b", ) - DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es") - csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo" + csv_content = "source_term,target_term,source_language,target_language\nhello,HELLO,en,es\nworld,mundo,en,es" result = DictionaryManager.import_entries( db_session, d.id, csv_content, delimiter=",", on_conflict="keep_existing", @@ -267,9 +275,9 @@ def test_import_csv_cancel_on_conflict(db_session: Session): db_session, name="Test", source_dialect="a", target_dialect="b", ) - DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es") - csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo" + csv_content = "source_term,target_term,source_language,target_language\nhello,HELLO,en,es\nworld,mundo,en,es" result = DictionaryManager.import_entries( db_session, d.id, csv_content, delimiter=",", on_conflict="cancel", @@ -289,7 +297,7 @@ def test_import_tsv(db_session: Session): db_session, name="Test", source_dialect="a", target_dialect="b", ) - tsv_content = "source_term\ttarget_term\nhello\thola\nworld\tmundo" + tsv_content = "source_term\ttarget_term\tsource_language\ttarget_language\nhello\thola\ten\tes\nworld\tmundo\ten\tes" result = DictionaryManager.import_entries( db_session, d.id, tsv_content, delimiter="\t", on_conflict="overwrite", @@ -339,9 +347,9 @@ def test_import_preview(db_session: Session): db_session, name="Test", source_dialect="a", target_dialect="b", ) - DictionaryManager.add_entry(db_session, d.id, "hello", "hola") + DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es") - csv_content = "source_term,target_term\nhello,HELLO\nworld,mundo" + csv_content = "source_term,target_term,source_language,target_language\nhello,HELLO,en,es\nworld,mundo,en,es" result = DictionaryManager.import_entries( db_session, d.id, csv_content, delimiter=",", on_conflict="overwrite", @@ -446,9 +454,9 @@ def test_filter_for_batch_matches(db_session: Session): db_session, name="Test Dict", source_dialect="a", target_dialect="b", ) - DictionaryManager.add_entry(db_session, d.id, "hello", "hola") - DictionaryManager.add_entry(db_session, d.id, "world", "mundo") - DictionaryManager.add_entry(db_session, d.id, "foo", "bar") + DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es") + DictionaryManager.add_entry(db_session, d.id, "world", "mundo", source_language="en", target_language="es") + DictionaryManager.add_entry(db_session, d.id, "foo", "bar", source_language="en", target_language="es") job = TranslationJob( name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT", @@ -492,7 +500,7 @@ def test_filter_for_batch_case_insensitive(db_session: Session): db_session, name="Test Dict", source_dialect="a", target_dialect="b", ) - DictionaryManager.add_entry(db_session, d.id, "Hello World", "Hola Mundo") + DictionaryManager.add_entry(db_session, d.id, "Hello World", "Hola Mundo", source_language="en", target_language="es") job = TranslationJob( name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT", @@ -523,7 +531,7 @@ def test_filter_for_batch_word_boundary(db_session: Session): db_session, name="Test Dict", source_dialect="a", target_dialect="b", ) - DictionaryManager.add_entry(db_session, d.id, "cat", "gato") + DictionaryManager.add_entry(db_session, d.id, "cat", "gato", source_language="en", target_language="es") job = TranslationJob( name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT", @@ -562,8 +570,8 @@ def test_filter_for_batch_multi_dictionary_priority(db_session: Session): d2 = DictionaryManager.create_dictionary( db_session, name="Priority2", source_dialect="a", target_dialect="b", ) - DictionaryManager.add_entry(db_session, d1.id, "hello", "hola") - DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour") + DictionaryManager.add_entry(db_session, d1.id, "hello", "hola", source_language="en", target_language="es") + DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour", source_language="en", target_language="es") job = TranslationJob( name="Test Job", source_dialect="a", target_dialect="b", status="DRAFT", @@ -625,8 +633,8 @@ def test_clear_entries(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", source_dialect="a", target_dialect="b", ) - DictionaryManager.add_entry(db_session, d.id, "hello", "hola") - DictionaryManager.add_entry(db_session, d.id, "world", "mundo") + DictionaryManager.add_entry(db_session, d.id, "hello", "hola", source_language="en", target_language="es") + DictionaryManager.add_entry(db_session, d.id, "world", "mundo", source_language="en", target_language="es") deleted = DictionaryManager.clear_entries(db_session, d.id) assert deleted == 2 @@ -634,4 +642,558 @@ def test_clear_entries(db_session: Session): entries, total = DictionaryManager.list_entries(db_session, d.id) assert total == 0 # endregion test_clear_entries + +# region test_add_entry_with_language_pair [TYPE Function] +# @PURPOSE: Verify creating entry with source_language and target_language stores correctly. +def test_add_entry_with_language_pair(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Lang Test", + source_dialect="a", target_dialect="b", + ) + entry = DictionaryManager.add_entry( + db_session, d.id, "hello", "привет", + source_language="en", target_language="ru", + ) + assert entry.source_language == "en" + assert entry.target_language == "ru" + assert entry.source_term == "hello" + + # Read back + entries, total = DictionaryManager.list_entries(db_session, d.id) + assert total == 1 + assert entries[0].source_language == "en" + assert entries[0].target_language == "ru" +# endregion test_add_entry_with_language_pair + + +# region test_duplicate_same_language_pair [TYPE Function] +# @PURPOSE: Verify duplicate entry with same (dictionary_id, source_term_norm, source_lang, target_lang) raises conflict. +def test_duplicate_same_language_pair(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Dup Test", + source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry( + db_session, d.id, "hello", "привет", + source_language="en", target_language="ru", + ) + # Same term with same language pair should raise + with pytest.raises(ValueError, match="already exists"): + DictionaryManager.add_entry( + db_session, d.id, "hello", "hallo", + source_language="en", target_language="ru", + ) +# endregion test_duplicate_same_language_pair + + +# region test_same_term_different_language_pair [TYPE Function] +# @PURPOSE: Verify same source_term with different language pair is allowed (not duplicate). +def test_same_term_different_language_pair(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Multi Lang", + source_dialect="a", target_dialect="b", + ) + entry1 = DictionaryManager.add_entry( + db_session, d.id, "hello", "привет", + source_language="en", target_language="ru", + ) + entry2 = DictionaryManager.add_entry( + db_session, d.id, "hello", "hallo", + source_language="en", target_language="de", + ) + assert entry1.id != entry2.id + assert entry1.target_language == "ru" + assert entry2.target_language == "de" + + entries, total = DictionaryManager.list_entries(db_session, d.id) + assert total == 2 +# endregion test_same_term_different_language_pair + + +# region test_filter_for_batch_with_language_pair [TYPE Function] +# @PURPOSE: Verify filter_for_batch with source_language and target_language filters entries correctly. +def test_filter_for_batch_with_language_pair(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Lang Filter", + source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru") + DictionaryManager.add_entry(db_session, d.id, "world", "мир", source_language="en", target_language="ru") + DictionaryManager.add_entry(db_session, d.id, "hello", "hallo", source_language="en", target_language="de") + + job = TranslationJob( + name="Lang Job", source_dialect="a", target_dialect="b", status="DRAFT", + ) + db_session.add(job) + db_session.flush() + + link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id) + db_session.add(link) + db_session.commit() + + # Filter for en → ru: should get 2 entries (hello/privet, world/mir) + result = DictionaryManager.filter_for_batch( + db_session, ["hello world"], job.id, + source_language="en", target_language="ru", + ) + assert len(result) == 2 + assert all(m["target_language"] == "ru" for m in result) + + # Filter for en → de: should get only the hello/hallo entry + result2 = DictionaryManager.filter_for_batch( + db_session, ["hello"], job.id, + source_language="en", target_language="de", + ) + assert len(result2) == 1 + assert result2[0]["target_language"] == "de" + assert result2[0]["target_term"] == "hallo" +# endregion test_filter_for_batch_with_language_pair + + +# region test_filter_for_batch_target_language_only [TYPE Function] +# @PURPOSE: Verify filter_for_batch with only target_language filters by target language only. +def test_filter_for_batch_target_language_only(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Target Only", + source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru") + DictionaryManager.add_entry(db_session, d.id, "hello", "hallo", source_language="en", target_language="de") + DictionaryManager.add_entry(db_session, d.id, "bonjour", "hallo", source_language="fr", target_language="de") + + job = TranslationJob( + name="Tgt Job", source_dialect="a", target_dialect="b", status="DRAFT", + ) + db_session.add(job) + db_session.flush() + + link = TranslationJobDictionary(job_id=job.id, dictionary_id=d.id) + db_session.add(link) + db_session.commit() + + # Filter for target=de only: should match both hello→hallo and bonjour→hallo (from any source lang) + result = DictionaryManager.filter_for_batch( + db_session, ["hello bonjour"], job.id, + target_language="de", + ) + assert len(result) == 2 + assert all(m["target_language"] == "de" for m in result) + + # Filter for target=ru only: should match hello→privet + result2 = DictionaryManager.filter_for_batch( + db_session, ["hello"], job.id, + target_language="ru", + ) + assert len(result2) == 1 + assert result2[0]["target_language"] == "ru" +# endregion test_filter_for_batch_target_language_only + + +# region test_migrate_old_entries [TYPE Function] +# @PURPOSE: Verify migration populates language pair fields for old-style entries. +def test_migrate_old_entries(db_session: Session): + # Create a dictionary with deprecated target_language set + d = TerminologyDictionary( + name="Old Dict", + source_dialect="a", + target_dialect="b", + target_language="ru", + ) + db_session.add(d) + db_session.flush() + + # Create old-style entries without explicit language pair (source_language=und, target_language=und) + entry1 = DictionaryEntry( + dictionary_id=d.id, + source_term="hello", + source_term_normalized="hello", + target_term="привет", + source_language="und", + target_language="und", + ) + entry2 = DictionaryEntry( + dictionary_id=d.id, + source_term="world", + source_term_normalized="world", + target_term="мир", + source_language="und", + target_language="und", + origin_source_language="en", + ) + db_session.add(entry1) + db_session.add(entry2) + db_session.commit() + + # Run migration + result = DictionaryManager.migrate_old_entries(db_session) + + # Verify migration counts + assert result["total_processed"] >= 2 + assert result["migrated_source"] >= 1 # entry2 had origin_source_language + assert result["migrated_target"] >= 2 # both should get target_language from dictionary + + # Verify entry1 got target_language from dictionary + db_session.refresh(entry1) + assert entry1.target_language == "ru" + # source_language should remain "und" (no origin_source_language) + assert entry1.source_language == "und" + + # Verify entry2 got source from origin and target from dictionary + db_session.refresh(entry2) + assert entry2.source_language == "en" + assert entry2.target_language == "ru" +# endregion test_migrate_old_entries + + +# region test_export_entries [TYPE Function] +# @PURPOSE: Verify export_entries includes language columns. +def test_export_entries(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Export Test", + source_dialect="a", target_dialect="b", + ) + DictionaryManager.add_entry(db_session, d.id, "hello", "привет", source_language="en", target_language="ru") + DictionaryManager.add_entry(db_session, d.id, "world", "мир", source_language="en", target_language="ru") + + csv_output = DictionaryManager.export_entries(db_session, d.id) + assert "source_language" in csv_output + assert "target_language" in csv_output + assert "context_data" in csv_output + assert "usage_notes" in csv_output + assert "en" in csv_output + assert "ru" in csv_output + + # Verify it's valid CSV + reader = csv.DictReader(io.StringIO(csv_output)) + rows = list(reader) + assert len(rows) == 2 + assert rows[0]["source_language"] == "en" + assert rows[0]["target_language"] == "ru" +# endregion test_export_entries + + +# region test_import_with_default_language [TYPE Function] +# @PURPOSE: Verify import uses default_source_language and default_target_language when columns missing. +def test_import_with_default_language(db_session: Session): + d = DictionaryManager.create_dictionary( + db_session, name="Default Lang Import", + source_dialect="a", target_dialect="b", + ) + # CSV without language columns + csv_content = "source_term,target_term\nhello,привет\nworld,мир" + result = DictionaryManager.import_entries( + db_session, d.id, csv_content, + delimiter=",", on_conflict="overwrite", + default_source_language="en", + default_target_language="ru", + ) + assert result["created"] == 2 + + entries, _ = DictionaryManager.list_entries(db_session, d.id) + for e in entries: + assert e.source_language == "en" + assert e.target_language == "ru" +# endregion test_import_with_default_language + + +# region test_context_capture_in_correction [TYPE Function] +# @PURPOSE: Verify context_data is auto-captured when submitting a correction with source row context. +def test_context_capture_in_correction(db_session: Session): + """Test T132: Context auto-capture when submitting a correction.""" + from src.plugins.translate.service import InlineCorrectionService + from src.models.translate import TranslationRecord, TranslationRun, TranslationBatch, TranslationLanguage + + d = DictionaryManager.create_dictionary( + db_session, name="Test Dict", + source_dialect="a", target_dialect="b", + ) + job = TranslationJob( + name="Ctx Job", source_dialect="a", target_dialect="b", + status="DRAFT", + ) + db_session.add(job) + db_session.flush() + + run = TranslationRun( + job_id=job.id, status="COMPLETED", + ) + db_session.add(run) + db_session.flush() + + batch = TranslationBatch( + run_id=run.id, batch_index=0, status="COMPLETED", + ) + db_session.add(batch) + db_session.flush() + + # Create a TranslationRecord with source_data (context columns) + record = TranslationRecord( + batch_id=batch.id, + run_id=run.id, + source_sql="hello world", + source_object_type="table_row", + source_object_name="Row 0", + source_object_id="0", + source_data={"schema": "public", "table": "users", "column": "name"}, + status="SUCCESS", + ) + db_session.add(record) + db_session.flush() + + # Create language entries for test languages + for lang_code in ("ru", "de", "fr", "it"): + tl = TranslationLanguage( + record_id=record.id, + language_code=lang_code, + source_language_detected="en", + translated_value="translated", + final_value="translated", + status="translated", + ) + db_session.add(tl) + db_session.commit() + + # Submit correction to dictionary + result = InlineCorrectionService.submit_correction_to_dict( + db=db_session, + record_id=record.id, + language_code="ru", + dictionary_id=d.id, + corrected_value="привет мир (исправлено)", + current_user="test_user", + ) + + assert result["action"] == "created" + assert result["entry_id"] is not None + + # Verify context was auto-captured + entry = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result["entry_id"]).first() + assert entry is not None + assert entry.has_context is True + assert entry.context_source == "auto" + assert entry.context_data is not None + # source_data should contain the row's context columns + assert "source_data" in entry.context_data + assert entry.context_data["source_object_type"] == "table_row" + assert entry.context_data["source_object_name"] == "Row 0" + + # Test context_data_override (use different language to avoid conflict) + result2 = InlineCorrectionService.submit_correction_to_dict( + db=db_session, + record_id=record.id, + language_code="de", # different target language avoids conflict + dictionary_id=d.id, + corrected_value="another correction", + current_user="test_user", + context_data_override={"custom_key": "custom_value"}, + ) + assert result2["action"] == "created" + entry2 = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result2["entry_id"]).first() + assert entry2.context_data == {"custom_key": "custom_value"} + assert entry2.context_source == "manual" + + # Test keep_context=False + result3 = InlineCorrectionService.submit_correction_to_dict( + db=db_session, + record_id=record.id, + language_code="fr", # different target language avoids conflict + dictionary_id=d.id, + corrected_value="no context", + current_user="test_user", + keep_context=False, + ) + assert result3["action"] == "created" + entry3 = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result3["entry_id"]).first() + assert entry3.context_data is None + assert entry3.has_context is False + assert entry3.context_source == "manual" + + # Test usage_notes + result4 = InlineCorrectionService.submit_correction_to_dict( + db=db_session, + record_id=record.id, + language_code="it", # different target language avoids conflict + dictionary_id=d.id, + corrected_value="with notes", + current_user="test_user", + usage_notes="Use this for user-facing columns only", + ) + assert result4["action"] == "created" + entry4 = db_session.query(DictionaryEntry).filter(DictionaryEntry.id == result4["entry_id"]).first() + assert entry4.usage_notes == "Use this for user-facing columns only" +# endregion test_context_capture_in_correction + + +# region test_jaccard_similarity [TYPE Function] +# @PURPOSE: Verify Jaccard similarity computation in ContextAwarePromptBuilder. +def test_jaccard_similarity(): + """Test T132: Jaccard similarity = 1.0 for identical, 0.0 for disjoint.""" + from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder + + builder = ContextAwarePromptBuilder() + + # Identical contexts + ctx1 = {"schema": "public", "table": "users"} + ctx2 = {"schema": "public", "table": "users"} + sim = builder.compute_context_similarity(ctx1, ctx2) + assert sim == 1.0, f"Expected 1.0, got {sim}" + + # Disjoint contexts + ctx3 = {"schema": "private"} + sim = builder.compute_context_similarity(ctx1, ctx3) + assert sim == 0.0, f"Expected 0.0, got {sim}" + + # Partial overlap + ctx4 = {"schema": "public", "table": "orders"} + sim = builder.compute_context_similarity(ctx1, ctx4) + # intersection: {"public"} union: {"public", "users", "orders"} => 1/3 ≈ 0.33 + assert 0.33 <= sim <= 0.34, f"Expected ~0.33, got {sim}" + + # Empty/missing contexts + assert builder.compute_context_similarity(None, ctx1) == 0.0 + assert builder.compute_context_similarity(ctx1, None) == 0.0 + assert builder.compute_context_similarity({}, ctx1) == 0.0 + + # Case-insensitive matching (same keys, different case) + ctx5 = {"schema": "PUBLIC", "table": "USERS"} + sim = builder.compute_context_similarity(ctx1, ctx5) + assert sim == 1.0, f"Expected 1.0 for case-insensitive, got {sim}" +# endregion test_jaccard_similarity + + +# region test_context_truncation [TYPE Function] +# @PURPOSE: Verify context string truncation at ~2000 chars. +def test_context_truncation(): + """Test T132: Context string truncation in render_entry.""" + from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder + + builder = ContextAwarePromptBuilder() + + # Build context data that would produce a very long context string + long_val = "x" * 500 + long_context = {f"key{i}": long_val for i in range(10)} + + # Create an entry-like dict with this context + entry = { + "source_term": "hello", + "target_term": "world", + "has_context": True, + "context_data": long_context, + "usage_notes": None, + } + + rendered = builder.render_entry(entry, priority=False) + # The context part should be truncated + assert len(rendered) < 2200, f"Rendered length {len(rendered)} exceeds expected max" + assert "...[truncated]" in rendered, "Expected truncation marker in output" + + # Short context should not be truncated + short_entry = { + "source_term": "hello", + "target_term": "world", + "has_context": True, + "context_data": {"table": "users"}, + "usage_notes": None, + } + short_rendered = builder.render_entry(short_entry, priority=False) + assert "...[truncated]" not in short_rendered + assert "(context: table=users)" in short_rendered + + # Priority prefix + priority_rendered = builder.render_entry(short_entry, priority=True) + assert priority_rendered.startswith("# PRIORITY (context match)") + + # Usage notes + notes_entry = { + "source_term": "hello", + "target_term": "world", + "has_context": False, + "context_data": None, + "usage_notes": "Use for admin panels", + } + notes_rendered = builder.render_entry(notes_entry, priority=False) + assert "# Usage: Use for admin panels" in notes_rendered +# endregion test_context_truncation + + +# region test_render_entry_dict_and_object [TYPE Function] +# @PURPOSE: Verify render_entry handles both dicts and objects. +def test_render_entry_dict_and_object(): + """Test T132: render_entry accepts dict or object with same results.""" + from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder + + builder = ContextAwarePromptBuilder() + + # Dict entry + dict_entry = { + "source_term": "hello", + "target_term": "hola", + "has_context": False, + "context_data": None, + "usage_notes": None, + } + dict_result = builder.render_entry(dict_entry) + + # Object-like entry using a simple class + class FakeEntry: + source_term = "hello" + target_term = "hola" + has_context = False + context_data = None + usage_notes = None + + obj_result = builder.render_entry(FakeEntry()) + assert dict_result == obj_result + assert dict_result == '"hello" -> "hola"' +# endregion test_render_entry_dict_and_object + + +# region test_build_context_entries_prioritization [TYPE Function] +# @PURPOSE: Verify build_context_entries sorts priority entries first. +def test_build_context_entries_prioritization(): + """Test T132: build_context_entries sorts priority entries before non-priority.""" + from src.plugins.translate.prompt_builder import ContextAwarePromptBuilder + + builder = ContextAwarePromptBuilder() + + entries = [ + { + "source_term": "high_match", + "target_term": "alta_coincidencia", + "has_context": True, + "context_data": {"schema": "public", "table": "users"}, + "usage_notes": None, + }, + { + "source_term": "no_match", + "target_term": "sin_coincidencia", + "has_context": True, + "context_data": {"schema": "private", "table": "config"}, + "usage_notes": None, + }, + { + "source_term": "no_context", + "target_term": "sin_contexto", + "has_context": False, + "context_data": None, + "usage_notes": None, + }, + ] + + # Row context that matches the first entry + row_context = {"schema": "public", "table": "users"} + + rendered = builder.build_context_entries(entries, row_context) + + # The first entry should be priority (similarity=1.0 >= 0.5) + assert len(rendered) == 3 + assert rendered[0].startswith("# PRIORITY (context match)"), f"Expected priority first, got: {rendered[0]}" + assert '"high_match"' in rendered[0] + + # The non-priority entries should not have priority prefix + assert not rendered[1].startswith("# PRIORITY") + assert not rendered[2].startswith("# PRIORITY") +# endregion test_build_context_entries_prioritization + + # endregion DictionaryTests diff --git a/backend/src/plugins/translate/__tests__/test_orchestrator.py b/backend/src/plugins/translate/__tests__/test_orchestrator.py index 31f38a96..892a85de 100644 --- a/backend/src/plugins/translate/__tests__/test_orchestrator.py +++ b/backend/src/plugins/translate/__tests__/test_orchestrator.py @@ -11,6 +11,7 @@ # @TEST_EDGE: invalid_run_status -> raises ValueError # @TEST_EDGE: executor_failure -> run is marked FAILED +import json from datetime import UTC, datetime from unittest.mock import MagicMock, patch @@ -18,11 +19,14 @@ import pytest from src.models.translate import ( TranslationJob, + TranslationLanguage, TranslationPreviewSession, TranslationRecord, TranslationRun, + TranslationRunLanguageStats, ) from src.plugins.translate.events import TranslationEventLog +from src.plugins.translate.executor import TranslationExecutor from src.plugins.translate.orchestrator import TranslationOrchestrator @@ -598,4 +602,249 @@ class TestTranslationOrchestrator: # endregion test_get_run_records # endregion test_get_run_history # endregion TestTranslationOrchestrator + + +# region TestTranslationExecutorMultiLang [TYPE Class] +# @PURPOSE: Tests for multi-language LLM translation execution: parsing, per-language entries, source-as-reference, backward compat. +class TestTranslationExecutorMultiLang: + + # region test_parse_multi_language_response [TYPE Function] + # @PURPOSE: _parse_llm_response handles multi-language JSON format correctly. + def test_parse_multi_language_response(self) -> None: + response = json.dumps({ + "rows": [ + {"row_id": 0, "detected_source_language": "fr", "ru": "текст", "en": "text", "de": "Text"}, + {"row_id": 1, "detected_source_language": "en", "ru": "дашборд", "en": "dashboard", "de": "Dashboard"}, + ] + }) + result = TranslationExecutor._parse_llm_response( + response, expected_count=2, target_languages=["ru", "en", "de"] + ) + assert len(result) == 2 + assert "0" in result + assert result["0"]["detected_source_language"] == "fr" + assert result["0"]["ru"] == "текст" + assert result["0"]["en"] == "text" + assert result["0"]["de"] == "Text" + assert "1" in result + assert result["1"]["en"] == "dashboard" + assert result["1"]["ru"] == "дашборд" + # endregion test_parse_multi_language_response + + # region test_parse_legacy_format_backward_compat [TYPE Function] + # @PURPOSE: _parse_llm_response still handles legacy single "translation" key format. + def test_parse_legacy_format_backward_compat(self) -> None: + response = json.dumps({ + "rows": [ + {"row_id": 0, "translation": "текст", "detected_source_language": "fr"}, + {"row_id": 1, "translation": "дашборд", "detected_source_language": "en"}, + ] + }) + result = TranslationExecutor._parse_llm_response( + response, expected_count=2, target_languages=["ru"] + ) + assert len(result) == 2 + assert result["0"]["translation"] == "текст" + assert result["1"]["detected_source_language"] == "en" + # endregion test_parse_legacy_format_backward_compat + + # region test_multi_language_translation_language_entries [TYPE Function] + # @PURPOSE: Multi-language LLM response creates per-language TranslationLanguage entries. + def test_multi_language_translation_language_entries(self) -> None: + db = MagicMock() + config_manager = MagicMock() + + # Mock job with multi-language target_languages + job = MagicMock(spec=TranslationJob) + job.id = "job-ml-1" + job.target_languages = ["ru", "en"] + job.target_language = None + job.target_dialect = "postgresql" + job.source_dialect = "postgresql" + job.translation_column = "name" + job.provider_id = "provider-1" + job.batch_size = 50 + + executor = TranslationExecutor(db, config_manager, "test-user") + + # Patch _call_llm to return multi-language response + multi_lang_response = json.dumps({ + "rows": [ + {"row_id": "0", "detected_source_language": "fr", "ru": "текст", "en": "text"}, + {"row_id": "1", "detected_source_language": "en", "ru": "дашборд", "en": "dashboard"}, + ] + }) + batch_rows = [ + {"row_index": "0", "source_text": "texte", "source_object_name": "Row 0"}, + {"row_index": "1", "source_text": "dashboard", "source_object_name": "Row 1"}, + ] + + with patch.object(executor, '_call_llm', return_value=multi_lang_response): + result = executor._call_llm_for_batch( + job=job, + run_id="run-ml-1", + batch_rows=batch_rows, + dict_matches=[], + batch_id="batch-ml-1", + ) + + assert result["successful"] == 2 + assert result["failed"] == 0 + assert result["skipped"] == 0 + + # Check that TranslationLanguage entries were created for each row per language + # Each row should have 2 language entries (ru, en) = 4 total + lang_added_calls = [ + call for call in db.add.call_args_list + if isinstance(call[0][0], TranslationLanguage) + ] + # Should have 4 language entries (2 rows x 2 languages) + assert len(lang_added_calls) >= 4, f"Expected at least 4 TranslationLanguage entries, got {len(lang_added_calls)}" + + # Group by language code + lang_codes: list[str] = [] + for call in lang_added_calls: + lang_entry = call[0][0] + lang_codes.append(lang_entry.language_code) + assert "ru" in lang_codes + assert "en" in lang_codes + # Each language should appear twice (once per row) + assert lang_codes.count("ru") == 2 + assert lang_codes.count("en") == 2 + # endregion test_multi_language_translation_language_entries + + # region test_source_as_reference [TYPE Function] + # @PURPOSE: When detected source language matches a target language, store original text verbatim. + def test_source_as_reference(self) -> None: + db = MagicMock() + config_manager = MagicMock() + + job = MagicMock(spec=TranslationJob) + job.id = "job-sar-1" + job.target_languages = ["fr", "en"] + job.target_language = None + job.target_dialect = "postgresql" + job.source_dialect = "postgresql" + job.translation_column = "name" + job.provider_id = "provider-1" + job.batch_size = 50 + + executor = TranslationExecutor(db, config_manager, "test-user") + + # Source is French, target_languages include French + response = json.dumps({ + "rows": [ + {"row_id": "0", "detected_source_language": "fr", "fr": "texte français", "en": "french text"}, + ] + }) + batch_rows = [ + {"row_index": "0", "source_text": "texte français", "source_object_name": "Row 0"}, + ] + + with patch.object(executor, '_call_llm', return_value=response): + result = executor._call_llm_for_batch( + job=job, + run_id="run-sar-1", + batch_rows=batch_rows, + dict_matches=[], + batch_id="batch-sar-1", + ) + + assert result["successful"] == 1 + + # Find the TranslationLanguage for French — should use source text (original) + lang_calls = [ + call for call in db.add.call_args_list + if isinstance(call[0][0], TranslationLanguage) + ] + fr_entry = None + en_entry = None + for call in lang_calls: + le = call[0][0] + if le.language_code == "fr": + fr_entry = le + elif le.language_code == "en": + en_entry = le + + assert fr_entry is not None, "FR language entry should exist" + # Source-as-reference: fr translation should be the original source text + assert fr_entry.translated_value == "texte français", ( + f"Expected source text 'texte français' for fr, got '{fr_entry.translated_value}'" + ) + assert en_entry is not None, "EN language entry should exist" + # EN should get the LLM-translated value + assert en_entry.translated_value == "french text" + # endregion test_source_as_reference + + # region test_per_language_stats_on_execute_run [TYPE Function] + # @PURPOSE: execute_run with multi-language job creates and populates TranslationRunLanguageStats. + def test_per_language_stats_on_execute_run(self, mock_job: MagicMock) -> None: + db = MagicMock() + config_manager = MagicMock() + + # Configure mock_job for multi-language + mock_job.target_languages = ["ru", "en"] + mock_job.target_language = None + mock_job.target_table = "target_tbl" + + # Mock run + run = MagicMock() + run.id = "run-stats-1" + run.job_id = "job-123" + run.status = "PENDING" + + # Mock completed run returned by executor + completed_run = MagicMock() + completed_run.id = "run-stats-1" + completed_run.job_id = "job-123" + completed_run.status = "COMPLETED" + completed_run.total_records = 5 + completed_run.successful_records = 5 + completed_run.failed_records = 0 + completed_run.skipped_records = 0 + completed_run.completed_at = datetime.now(UTC) + completed_run.error_message = None + completed_run.insert_status = "success" + completed_run.superset_execution_id = "q-1" + completed_run.superset_execution_log = {} + + # Mock queries: job -> mock_job, run -> completed_run + db.query.return_value.filter.return_value.first.side_effect = [ + mock_job, + completed_run, + ] + + orch = TranslationOrchestrator(db, config_manager, "test-user") + with patch.object(orch, 'event_log'), \ + patch.object(orch, '_generate_and_insert_sql', + return_value={"status": "success", "query_id": "q-1", "rows_affected": 5}), \ + patch.object(orch, '_update_language_stats') as mock_update_stats, \ + patch('src.plugins.translate.orchestrator.TranslationExecutor') as MockExecutor: + + mock_executor_instance = MagicMock() + mock_executor_instance.execute_run.return_value = completed_run + MockExecutor.return_value = mock_executor_instance + + result = orch.execute_run(run) + + assert result.status == "COMPLETED" + + # Verify that TranslationRunLanguageStats entries were created + stats_added = [ + call for call in db.add.call_args_list + if isinstance(call[0][0], TranslationRunLanguageStats) + ] + # 2 language stats entries (ru, en) + assert len(stats_added) >= 2, ( + f"Expected at least 2 TranslationRunLanguageStats entries, got {len(stats_added)}" + ) + codes = [call[0][0].language_code for call in stats_added] + assert "ru" in codes + assert "en" in codes + + # Verify _update_language_stats was called + mock_update_stats.assert_called_once() + # endregion test_per_language_stats_on_execute_run + +# endregion TestTranslationExecutorMultiLang # endregion OrchestratorTests diff --git a/backend/src/plugins/translate/__tests__/test_preview.py b/backend/src/plugins/translate/__tests__/test_preview.py index 4c6c9ebf..66727977 100644 --- a/backend/src/plugins/translate/__tests__/test_preview.py +++ b/backend/src/plugins/translate/__tests__/test_preview.py @@ -420,8 +420,13 @@ class TestTranslationPreview: assert tokens > 0 assert tokens < len(prompt) # tokens typically fewer than chars - output_tokens = TokenEstimator.estimate_output_tokens(10) - assert output_tokens == 500 + # Single language + output_tokens = TokenEstimator.estimate_output_tokens(10, num_languages=1) + assert output_tokens == 600 # 10 * 1 * 50 * 1.2 + + # Multi-language + output_tokens_multi = TokenEstimator.estimate_output_tokens(10, num_languages=3) + assert output_tokens_multi == 1800 # 10 * 3 * 50 * 1.2 cost = TokenEstimator.estimate_cost(1000, 0.002) assert cost == 0.002 @@ -431,28 +436,65 @@ class TestTranslationPreview: # endregion test_cost_estimation # region test_preview_parse_llm_response [TYPE Function] - # @PURPOSE: Verify LLM JSON response parsing. + # @PURPOSE: Verify LLM JSON response parsing includes detected_source_language. def test_preview_parse_llm_response(self): - """Parse LLM response should extract translations keyed by row_id.""" + """Parse LLM response should extract translations with detected_source_language per row.""" response = json.dumps({ "rows": [ - {"row_id": "0", "translation": "Привет"}, + {"row_id": "0", "translation": "Привет", "detected_source_language": "en"}, {"row_id": "1", "translation": "Мир"}, ] }) result = TranslationPreview._parse_llm_response(response, 2) - assert result == {"0": "Привет", "1": "Мир"} + assert result == { + "0": {"translation": "Привет", "detected_source_language": "en"}, + "1": {"translation": "Мир", "detected_source_language": "und"}, + } # endregion test_preview_parse_llm_response + # region test_preview_parse_llm_response_multilang [TYPE Function] + # @PURPOSE: Verify multi-language LLM response parsing. + def test_preview_parse_llm_response_multilang(self): + """Parse multi-language LLM response should extract per-language translations.""" + response = json.dumps({ + "rows": [ + {"row_id": "0", "detected_source_language": "fr", "ru": "текст", "en": "text", "de": "Text"}, + {"row_id": "1", "detected_source_language": "en", "ru": "привет", "en": "hello"}, + ] + }) + result = TranslationPreview._parse_llm_response(response, 2, target_languages=["ru", "en", "de"]) + assert result == { + "0": {"detected_source_language": "fr", "ru": "текст", "en": "text", "de": "Text"}, + "1": {"detected_source_language": "en", "ru": "привет", "en": "hello"}, + } + # Verify per-language extraction + assert result["0"]["ru"] == "текст" + assert result["0"]["en"] == "text" + assert result["1"]["ru"] == "привет" + # endregion test_preview_parse_llm_response_multilang + # region test_preview_parse_llm_response_with_code_block [TYPE Function] # @PURPOSE: Verify LLM response parsing handles markdown code blocks. def test_preview_parse_llm_response_with_code_block(self): """Parse LLM response should handle markdown code block wrapping.""" response = "```json\n{\n \"rows\": [\n {\"row_id\": \"0\", \"translation\": \"Test\"}\n ]\n}\n```" result = TranslationPreview._parse_llm_response(response, 1) - assert result == {"0": "Test"} + assert result == {"0": {"translation": "Test", "detected_source_language": "und"}} # endregion test_preview_parse_llm_response_with_code_block + # region test_cost_warning [TYPE Function] + # @PURPOSE: Verify cost warning for large sample sizes. + def test_cost_warning(self): + """Cost warning should appear for sample_size > 30.""" + warning = TokenEstimator.check_cost_warning(31, 2, 5000, 0.01) + assert warning is not None + assert "Large preview" in warning + assert "5000" in warning + + no_warning = TokenEstimator.check_cost_warning(10, 2, 1000, 0.002) + assert no_warning is None + # endregion test_cost_warning + # region test_preview_compute_config_hash [TYPE Function] # @PURPOSE: Verify config hash computation is deterministic. def test_preview_compute_config_hash(self): diff --git a/backend/src/plugins/translate/dictionary.py b/backend/src/plugins/translate/dictionary.py index d3fd9653..f4d7354e 100644 --- a/backend/src/plugins/translate/dictionary.py +++ b/backend/src/plugins/translate/dictionary.py @@ -14,7 +14,7 @@ import io import re from typing import Any -from sqlalchemy import func +from sqlalchemy import func, or_ from sqlalchemy.orm import Session from ...core.logger import belief_scope, logger @@ -27,6 +27,23 @@ from ...models.translate import ( from ._utils import _detect_delimiter, _normalize_term +# #region _validate_bcp47 [C:2] [TYPE Function] [SEMANTICS validation, bcp47, language] +# @BRIEF Validate that a language tag is a non-empty BCP-47 string. +def _validate_bcp47(tag: str, field_name: str) -> None: + """Validate that tag is a non-empty BCP-47 string (basic check).""" + if not tag or not tag.strip(): + raise ValueError(f"{field_name} must be a non-empty BCP-47 language tag") + tag = tag.strip() + # Basic BCP-47 validation: must match lang[-script][-region][-variant]* pattern + import re as _re + if not _re.match(r'^[a-zA-Z]{2,8}(-[a-zA-Z0-9]{1,8})*$', tag): + raise ValueError( + f"{field_name} is not a valid BCP-47 tag: '{tag}'. " + "Expected format like 'en', 'ru', 'zh-CN', 'pt-BR'." + ) +# #endregion _validate_bcp47 + + # #region DictionaryManager [C:4] [TYPE Class] # @BRIEF Manages terminology dictionaries and their entries with referential integrity. # @PRE: Database session is open and valid. @@ -34,12 +51,14 @@ from ._utils import _detect_delimiter, _normalize_term # @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards. class DictionaryManager: # region DictionaryManager.create_dictionary [TYPE Function] - # @PURPOSE: Create a new terminology dictionary. - # @PRE: payload contains name, source_dialect, target_dialect. + # @PURPOSE: Create a new terminology dictionary (language independent at dictionary level; language pairs are per-entry). + # @PRE: name is provided. # @POST: New TerminologyDictionary row is created and returned. @staticmethod def create_dictionary( - db: Session, name: str, source_dialect: str, target_dialect: str, + db: Session, name: str, + source_dialect: str = "", + target_dialect: str = "", created_by: str | None = None, description: str | None = None, is_active: bool = True, ) -> TerminologyDictionary: @@ -48,8 +67,8 @@ class DictionaryManager: dictionary = TerminologyDictionary( name=name, description=description, - source_dialect=source_dialect, - target_dialect=target_dialect, + source_dialect=source_dialect or "", + target_dialect=target_dialect or "", is_active=is_active, created_by=created_by, ) @@ -166,53 +185,66 @@ class DictionaryManager: # endregion DictionaryManager.list_dictionaries # region DictionaryManager.add_entry [TYPE Function] - # @PURPOSE: Add an entry to a dictionary, enforcing unique source_term_normalized. - # @PRE: dict_id exists. source_term and target_term are non-empty. + # @PURPOSE: Add an entry to a dictionary with language-pair-aware uniqueness validation. + # @PRE: dict_id exists. source_term, target_term are non-empty. source_language, target_language are valid BCP-47. # @POST: New DictionaryEntry row is created or raises on duplicate. @staticmethod def add_entry( db: Session, dict_id: str, source_term: str, target_term: str, context_notes: str | None = None, + source_language: str = "und", + target_language: str = "und", ) -> DictionaryEntry: with belief_scope("DictionaryManager.add_entry"): + _validate_bcp47(source_language, "source_language") + _validate_bcp47(target_language, "target_language") + normalized = _normalize_term(source_term) + # Uniqueness is now (dictionary_id, source_term_normalized, source_language, target_language) existing = ( db.query(DictionaryEntry) .filter( DictionaryEntry.dictionary_id == dict_id, DictionaryEntry.source_term_normalized == normalized, + DictionaryEntry.source_language == source_language, + DictionaryEntry.target_language == target_language, ) .first() ) if existing: raise ValueError( - f"Duplicate entry: '{source_term}' already exists in this dictionary " - f"(id={existing.id}). Use overwrite or keep_existing conflict mode." + f"Duplicate entry: '{source_term}' (lang: {source_language}→{target_language}) " + f"already exists in this dictionary (id={existing.id}). " + "Use overwrite or keep_existing conflict mode." ) - logger.reason("Adding dictionary entry", {"dict_id": dict_id, "term": source_term}) + logger.reason("Adding dictionary entry", {"dict_id": dict_id, "term": source_term, "src_lang": source_language, "tgt_lang": target_language}) entry = DictionaryEntry( dictionary_id=dict_id, source_term=source_term.strip(), source_term_normalized=normalized, target_term=target_term.strip(), context_notes=context_notes.strip() if context_notes else None, + source_language=source_language.strip(), + target_language=target_language.strip(), ) db.add(entry) db.commit() db.refresh(entry) - logger.reflect("Entry added", {"entry_id": entry.id}) + logger.reflect("Entry added", {"entry_id": entry.id, "src_lang": source_language, "tgt_lang": target_language}) return entry # endregion DictionaryManager.add_entry # region DictionaryManager.edit_entry [TYPE Function] - # @PURPOSE: Edit an existing dictionary entry with duplicate-aware normalization. + # @PURPOSE: Edit an existing dictionary entry with language-pair-aware duplicate check. # @PRE: entry_id exists. # @POST: Entry fields are updated. @staticmethod def edit_entry( db: Session, entry_id: str, source_term: str | None = None, target_term: str | None = None, context_notes: str | None = None, + source_language: str | None = None, + target_language: str | None = None, ) -> DictionaryEntry: with belief_scope("DictionaryManager.edit_entry"): entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first() @@ -220,14 +252,24 @@ class DictionaryManager: raise ValueError(f"Entry not found: {entry_id}") logger.reason("Editing dictionary entry", {"entry_id": entry_id}) + + if source_language is not None: + _validate_bcp47(source_language, "source_language") + entry.source_language = source_language.strip() + if target_language is not None: + _validate_bcp47(target_language, "target_language") + entry.target_language = target_language.strip() + if source_term is not None: normalized = _normalize_term(source_term) - # Check uniqueness within the same dictionary + # Check uniqueness within the same dictionary + language pair existing = ( db.query(DictionaryEntry) .filter( DictionaryEntry.dictionary_id == entry.dictionary_id, DictionaryEntry.source_term_normalized == normalized, + DictionaryEntry.source_language == entry.source_language, + DictionaryEntry.target_language == entry.target_language, DictionaryEntry.id != entry_id, ) .first() @@ -318,6 +360,8 @@ class DictionaryManager: delimiter: str | None = None, on_conflict: str = "overwrite", preview_only: bool = False, + default_source_language: str | None = None, + default_target_language: str | None = None, ) -> dict[str, Any]: with belief_scope("DictionaryManager.import_entries"): # Validate dictionary @@ -359,6 +403,8 @@ class DictionaryManager: source_term = row.get("source_term", "").strip() target_term = row.get("target_term", "").strip() context_notes = row.get("context_notes", "").strip() or None + source_language = row.get("source_language", "").strip() or default_source_language or "und" + target_language = row.get("target_language", "").strip() or default_target_language or "und" if not source_term or not target_term: result["errors"].append({ @@ -376,6 +422,8 @@ class DictionaryManager: .filter( DictionaryEntry.dictionary_id == dict_id, DictionaryEntry.source_term_normalized == normalized, + DictionaryEntry.source_language == source_language, + DictionaryEntry.target_language == target_language, ) .first() ) @@ -384,6 +432,8 @@ class DictionaryManager: "source_term": source_term, "target_term": target_term, "context_notes": context_notes, + "source_language": source_language, + "target_language": target_language, "is_conflict": existing is not None, "existing_target_term": existing.target_term if existing else None, } @@ -395,6 +445,8 @@ class DictionaryManager: .filter( DictionaryEntry.dictionary_id == dict_id, DictionaryEntry.source_term_normalized == normalized, + DictionaryEntry.source_language == source_language, + DictionaryEntry.target_language == target_language, ) .first() ) @@ -404,6 +456,8 @@ class DictionaryManager: existing.source_term = source_term existing.target_term = target_term existing.context_notes = context_notes + existing.source_language = source_language + existing.target_language = target_language result["updated"] += 1 elif on_conflict == "keep_existing": result["skipped"] += 1 @@ -421,6 +475,8 @@ class DictionaryManager: source_term_normalized=normalized, target_term=target_term, context_notes=context_notes, + source_language=source_language, + target_language=target_language, ) db.add(entry) result["created"] += 1 @@ -446,14 +502,122 @@ class DictionaryManager: return result # endregion DictionaryManager.import_entries + # region DictionaryManager.export_entries [TYPE Function] + # @PURPOSE: Export all entries as CSV string with language columns. + # @PRE: dict_id exists. + # @POST: Returns CSV string with header: source_term, target_term, source_language, target_language, context_notes, context_data, usage_notes. + @staticmethod + def export_entries( + db: Session, dict_id: str, delimiter: str = ",", + ) -> str: + with belief_scope("DictionaryManager.export_entries"): + dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() + if not dictionary: + raise ValueError(f"Dictionary not found: {dict_id}") + + entries = ( + db.query(DictionaryEntry) + .filter(DictionaryEntry.dictionary_id == dict_id) + .order_by(DictionaryEntry.source_term.asc()) + .all() + ) + + output = io.StringIO() + fieldnames = [ + "source_term", "target_term", + "source_language", "target_language", + "context_notes", "context_data", "usage_notes", + ] + writer = csv.DictWriter(output, fieldnames=fieldnames, delimiter=delimiter) + writer.writeheader() + + for entry in entries: + writer.writerow({ + "source_term": entry.source_term, + "target_term": entry.target_term, + "source_language": entry.source_language, + "target_language": entry.target_language, + "context_notes": entry.context_notes or "", + "context_data": entry.context_data, + "usage_notes": entry.usage_notes or "", + }) + + result = output.getvalue() + logger.reflect("Entries exported", {"dict_id": dict_id, "count": len(entries), "delimiter": repr(delimiter)}) + return result + # endregion DictionaryManager.export_entries + + # region DictionaryManager.migrate_old_entries [TYPE Function] + # @PURPOSE: Migrate existing single-language dictionary entries to populate source_language and target_language. + # @PRE: db session is open. + # @POST: Entries with "und" source_language or target_language are updated with inferred values. + # @SIDE_EFFECT: Updates DictionaryEntry rows in bulk. + @staticmethod + def migrate_old_entries(db: Session) -> dict[str, int]: + with belief_scope("DictionaryManager.migrate_old_entries"): + logger.reason("Starting migration of old dictionary entries") + migrated_source = 0 + migrated_target = 0 + skipped = 0 + + # Find all entries that may need migration (source_language is "und" or target_language is "und") + entries = ( + db.query(DictionaryEntry) + .filter( + (DictionaryEntry.source_language == "und") | + (DictionaryEntry.target_language == "und") + ) + .all() + ) + + for entry in entries: + # Try to infer source_language from origin_source_language + if entry.source_language == "und": + if entry.origin_source_language: + entry.source_language = entry.origin_source_language + migrated_source += 1 + + # Try to infer target_language from the parent dictionary's deprecated target_language + if entry.target_language == "und": + dictionary = ( + db.query(TerminologyDictionary) + .filter(TerminologyDictionary.id == entry.dictionary_id) + .first() + ) + if dictionary and dictionary.target_language: + entry.target_language = dictionary.target_language + migrated_target += 1 + + if entry.source_language == "und" and entry.target_language == "und": + skipped += 1 + + db.commit() + + logger.reflect("Migration complete", { + "migrated_source": migrated_source, + "migrated_target": migrated_target, + "skipped": skipped, + "total_processed": len(entries), + }) + return { + "migrated_source": migrated_source, + "migrated_target": migrated_target, + "skipped": skipped, + "total_processed": len(entries), + } + # endregion DictionaryManager.migrate_old_entries + # region DictionaryManager.filter_for_batch [TYPE Function] - # @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job. + # @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job, + # optionally filtered by language pair. # @PRE: job_id exists and source_texts is a list of strings. # @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries. # @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables. @staticmethod def filter_for_batch( db: Session, source_texts: list[str], job_id: str, + source_language: str | None = None, + target_language: str | None = None, ) -> list[dict[str, Any]]: with belief_scope("DictionaryManager.filter_for_batch"): # Get dictionaries attached to this job @@ -508,6 +672,18 @@ class DictionaryManager: if not norm: continue + # Apply language pair filtering + if source_language is not None: + src = source_language.strip().lower() + entry_src = entry.source_language.strip().lower() + if entry_src != src and entry_src != "und": + continue + if target_language is not None: + tgt = target_language.strip().lower() + entry_tgt = entry.target_language.strip().lower() + if entry_tgt != tgt: + continue + # Word-boundary-aware matching # Build pattern: \bterm\b (case-insensitive) # Escape regex special chars in the search term @@ -528,6 +704,12 @@ class DictionaryManager: "dictionary_id": entry.dictionary_id, "dictionary_name": dict_map.get(entry.dictionary_id, ""), "context_notes": entry.context_notes, + "context_data": entry.context_data, + "has_context": entry.has_context, + "context_source": entry.context_source, + "usage_notes": entry.usage_notes, + "source_language": entry.source_language, + "target_language": entry.target_language, }) # Sort by dictionary link priority (order of dict_ids from link order) @@ -539,6 +721,8 @@ class DictionaryManager: "source_texts": len(source_texts), "matches": len(matched), "dictionaries_used": len(dictionaries), + "source_language": source_language, + "target_language": target_language, }) return matched # endregion DictionaryManager.filter_for_batch @@ -560,6 +744,8 @@ class DictionaryManager: origin_row_key: str | None = None, origin_user_id: str | None = None, on_conflict: str = "overwrite", + context_data: dict[str, Any] | None = None, + usage_notes: str | None = None, ) -> dict[str, Any]: with belief_scope("DictionaryManager.submit_correction"): # Validate dictionary exists @@ -568,11 +754,22 @@ class DictionaryManager: raise ValueError(f"Dictionary '{dict_id}' not found") normalized = _normalize_term(source_term) + entry_src_lang = dictionary.source_dialect or "und" + entry_tgt_lang = dictionary.target_dialect or "und" + # Find existing entry: match exact language pair OR old-style "und"/"und" entries existing = ( db.query(DictionaryEntry) .filter( DictionaryEntry.dictionary_id == dict_id, DictionaryEntry.source_term_normalized == normalized, + or_( + # Exact language pair match + (DictionaryEntry.source_language == entry_src_lang) & + (DictionaryEntry.target_language == entry_tgt_lang), + # Old-style entries without language pair + (DictionaryEntry.source_language == "und") & + (DictionaryEntry.target_language == "und"), + ), ) .first() ) @@ -606,6 +803,12 @@ class DictionaryManager: existing.origin_row_key = origin_row_key if origin_user_id: existing.origin_user_id = origin_user_id + if context_data is not None: + existing.context_data = context_data + existing.has_context = bool(context_data) + existing.context_source = "auto" + if usage_notes is not None: + existing.usage_notes = usage_notes db.flush() result["action"] = "updated" result["entry_id"] = existing.id @@ -621,12 +824,18 @@ class DictionaryManager: result["message"] = "Correction cancelled by conflict mode" return result else: - # Create new entry + # Create new entry — derive language pair from dictionary entry = DictionaryEntry( dictionary_id=dict_id, source_term=source_term.strip(), source_term_normalized=normalized, target_term=corrected_target_term.strip(), + source_language=entry_src_lang, + target_language=entry_tgt_lang, + context_data=context_data, + usage_notes=usage_notes, + has_context=bool(context_data), + context_source="auto" if context_data else None, origin_run_id=origin_run_id, origin_row_key=origin_row_key, origin_user_id=origin_user_id, @@ -678,11 +887,19 @@ class DictionaryManager: continue normalized = _normalize_term(source_term) + bulk_src_lang = dictionary.source_dialect or "und" + bulk_tgt_lang = dictionary.target_dialect or "und" existing = ( db.query(DictionaryEntry) .filter( DictionaryEntry.dictionary_id == dict_id, DictionaryEntry.source_term_normalized == normalized, + or_( + (DictionaryEntry.source_language == bulk_src_lang) & + (DictionaryEntry.target_language == bulk_tgt_lang), + (DictionaryEntry.source_language == "und") & + (DictionaryEntry.target_language == "und"), + ), ) .first() ) @@ -729,6 +946,8 @@ class DictionaryManager: source_term=source_term, source_term_normalized=normalized, target_term=corrected_target, + source_language=bulk_src_lang, + target_language=bulk_tgt_lang, origin_run_id=corr.get("origin_run_id"), origin_row_key=corr.get("origin_row_key"), origin_user_id=origin_user_id, diff --git a/backend/src/plugins/translate/events.py b/backend/src/plugins/translate/events.py index 059da233..908cb8e4 100644 --- a/backend/src/plugins/translate/events.py +++ b/backend/src/plugins/translate/events.py @@ -191,6 +191,18 @@ class TranslationEventLog: logger.reflect("No expired events to prune", {}) return {"pruned": 0, "snapshot_id": None} + # Compute per-language metrics from expired events before pruning + per_language: dict[str, dict[str, int | float]] = {} + expired_events = expired_query.all() + for evt in expired_events: + event_data = evt.event_data or {} + lang = event_data.get("language_code") or event_data.get("target_language") or "_unknown_" + if lang not in per_language: + per_language[lang] = {"cumulative_tokens": 0, "cumulative_cost": 0.0, "runs": 0} + per_language[lang]["cumulative_tokens"] += event_data.get("token_count", 0) or 0 + per_language[lang]["cumulative_cost"] += event_data.get("cost", 0.0) or 0.0 + per_language[lang]["runs"] += 1 if event_data.get("run_id") else 0 + # Create MetricSnapshot before pruning snapshot = MetricSnapshot( id=str(uuid.uuid4()), @@ -198,6 +210,7 @@ class TranslationEventLog: key_hash=f"prune_{cutoff.timestamp():.0f}", covers_events_before=cutoff, total_records=total_expired, + per_language_metrics=per_language or None, snapshot_date=datetime.now(UTC), ) self.db.add(snapshot) diff --git a/backend/src/plugins/translate/executor.py b/backend/src/plugins/translate/executor.py index a1b72b47..4516f746 100644 --- a/backend/src/plugins/translate/executor.py +++ b/backend/src/plugins/translate/executor.py @@ -27,6 +27,7 @@ from ...core.logger import belief_scope, logger from ...models.translate import ( TranslationBatch, TranslationJob, + TranslationLanguage, TranslationPreviewRecord, TranslationPreviewSession, TranslationRecord, @@ -36,6 +37,7 @@ from ...services.llm_prompt_templates import render_prompt from ...services.llm_provider import LLMProviderService from .dictionary import DictionaryManager from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE +from .prompt_builder import ContextAwarePromptBuilder # #region MAX_RETRIES_PER_BATCH [TYPE Constant] # @BRIEF Maximum number of retries for a single batch before marking it failed. @@ -62,6 +64,7 @@ class TranslationExecutor: self.current_user = current_user self.on_batch_progress = on_batch_progress self._current_run_id: str | None = None + self._preview_edits_cache: dict[str, dict[str, str]] | None = None # key_hash -> {lang_code: edited_value} # region execute_run [TYPE Function] # @PURPOSE: Run full translation execution for a TranslationRun. @@ -84,6 +87,9 @@ class TranslationExecutor: "batch_size": job.batch_size, }) + # Load preview edits for carry-forward + self._load_preview_edits(job.id) + # Mark run as RUNNING run.status = "RUNNING" run.started_at = datetime.now(UTC) @@ -410,6 +416,70 @@ class TranslationExecutor: return [] # endregion _extract_chart_data_rows + # region _load_preview_edits [TYPE Function] + # @PURPOSE: Load user edits from accepted preview session for carry-forward. + # @PRE: job_id exists. + # @POST: Populates _preview_edits_cache with key_hash -> {lang_code: edited_value}. + # @SIDE_EFFECT: Queries TranslationPreviewLanguage and TranslationPreviewRecord. + def _load_preview_edits(self, job_id: str) -> None: + """Load preview edits for carry-forward during execution.""" + from ...models.translate import TranslationPreviewLanguage, TranslationPreviewRecord, TranslationPreviewSession + + with belief_scope("TranslationExecutor._load_preview_edits"): + session = ( + self.db.query(TranslationPreviewSession) + .filter( + TranslationPreviewSession.job_id == job_id, + TranslationPreviewSession.status == "APPLIED", + ) + .order_by(TranslationPreviewSession.created_at.desc()) + .first() + ) + if not session: + logger.reason("No applied preview session found — no edits to carry forward", { + "job_id": job_id, + }) + self._preview_edits_cache = {} + return + + records = ( + self.db.query(TranslationPreviewRecord) + .filter(TranslationPreviewRecord.session_id == session.id) + .all() + ) + + edits: dict[str, dict[str, str]] = {} + for rec in records: + if not rec.source_data: + continue + # Compute key hash from source_data + key_hash = self._compute_key_hash(rec.source_data) + edited_langs: dict[str, str] = {} + for lang_entry in (rec.languages or []): + if lang_entry.status in ("edited", "approved") and lang_entry.user_edit: + edited_langs[lang_entry.language_code] = lang_entry.final_value or lang_entry.user_edit + logger.reason("Carrying forward preview edit", { + "key_hash": key_hash, + "language_code": lang_entry.language_code, + }) + if edited_langs: + edits[key_hash] = edited_langs + + self._preview_edits_cache = edits + logger.reason(f"Loaded {len(edits)} preview edits for carry-forward", { + "job_id": job_id, + }) + # endregion _load_preview_edits + + # region _compute_key_hash [TYPE Function] + # @PURPOSE: Compute a stable hash from source_data dict for matching preview edits. + @staticmethod + def _compute_key_hash(source_data: dict) -> str: + import hashlib + stable = json.dumps(source_data, sort_keys=True) + return hashlib.sha256(stable.encode()).hexdigest()[:16] + # endregion _compute_key_hash + # region _process_batch [TYPE Function] # @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records. # @PRE: job and batch_rows are valid. @@ -452,17 +522,39 @@ class TranslationExecutor: self.db, source_texts, job.id ) - # For each row, determine if we need LLM translation or can use approved translation + # For each row, determine if we need LLM translation or can use approved/preview edit rows_for_llm = [] pre_translated = [] for row in batch_rows: if row.get("approved_translation"): pre_translated.append(row) - else: - rows_for_llm.append(row) + continue + + # Check for preview edits carry-forward + source_data = row.get("source_data") or {} + if source_data and self._preview_edits_cache: + key_hash = self._compute_key_hash(source_data) + preview_edit = self._preview_edits_cache.get(key_hash) + if preview_edit: + # Use the first edited language's value as the approved translation + first_edit = next(iter(preview_edit.values()), None) + if first_edit: + row["approved_translation"] = first_edit + logger.reason("Using preview edit carry-forward", { + "key_hash": key_hash, + "langs": list(preview_edit.keys()), + }) + pre_translated.append(row) + continue + + rows_for_llm.append(row) # Handle pre-translated (approved) rows + target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] + if not isinstance(target_languages, list): + target_languages = [str(target_languages)] + for row in pre_translated: record = TranslationRecord( id=str(uuid.uuid4()), @@ -477,6 +569,20 @@ class TranslationExecutor: status="SUCCESS", ) self.db.add(record) + + # Create per-language entry for each target language (pre-approved) + for lang_code in target_languages: + lang_entry = TranslationLanguage( + id=str(uuid.uuid4()), + record_id=record.id, + language_code=lang_code, + source_language_detected="und", + translated_value=row.get("approved_translation"), + final_value=row.get("approved_translation"), + status="translated", + needs_review=False, + ) + self.db.add(lang_entry) result["successful"] += 1 # Process rows needing LLM translation @@ -524,21 +630,31 @@ class TranslationExecutor: batch_id: str, ) -> dict[str, int]: with belief_scope("TranslationExecutor._call_llm_for_batch"): - # Build dictionary section + # Build dictionary section using ContextAwarePromptBuilder dictionary_section = "" if dict_matches: + # Get row_context from first batch row if available + row_context = batch_rows[0].get("source_data") if batch_rows else None + + # Use ContextAwarePromptBuilder for context-aware annotations + annotated_entries = ContextAwarePromptBuilder.build_context_entries( + dict_matches, row_context + ) glossary_lines = [] - for m in dict_matches: - glossary_lines.append( - f"- '{m['source_term']}' -> '{m['target_term']}'" - f"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}" - ) + for annotated_line in annotated_entries: + glossary_lines.append(f"- {annotated_line}") dictionary_section = ( "Terminology dictionary (use these translations when applicable):\n" + "\n".join(glossary_lines) + "\n\n" ) + # Resolve target languages + target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] + if not isinstance(target_languages, list): + target_languages = [str(target_languages)] + target_languages_str = ", ".join(target_languages) + # Build rows JSON for LLM rows_json = json.dumps([ { @@ -548,12 +664,13 @@ class TranslationExecutor: for idx, row in enumerate(batch_rows) ], indent=2) - # Build prompt + # Build prompt (use multi-language format) prompt = render_prompt( DEFAULT_EXECUTION_PROMPT_TEMPLATE, { "source_language": job.source_dialect or "SQL", - "target_language": job.target_language or job.target_dialect or "en", + "target_language": target_languages_str, + "target_languages": target_languages_str, "source_dialect": job.source_dialect or "", "target_dialect": job.target_dialect or "", "translation_column": job.translation_column or "", @@ -607,9 +724,9 @@ class TranslationExecutor: self.db.add(record) return {"successful": 0, "failed": len(batch_rows), "skipped": 0, "retries": retries} - # Parse LLM response + # Parse LLM response (multi-language aware) try: - translations = self._parse_llm_response(llm_response, len(batch_rows)) + translations = self._parse_llm_response(llm_response, len(batch_rows), target_languages=target_languages) except ValueError as e: # Parse failure — mark all rows as SKIPPED logger.explore("LLM response parse failed", { @@ -646,16 +763,20 @@ class TranslationExecutor: for row in batch_rows: row_id = str(row.get("row_index", "")) - translation = translations.get(row_id) + translation_data = translations.get(row_id) + source_text = row.get("source_text", "") + detected_lang = "und" + if translation_data: + detected_lang = translation_data.get("detected_source_language", "und") - if translation is None: + if translation_data is None: # NULL translation — skip skipped += 1 record = TranslationRecord( id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id, - source_sql=row.get("source_text", ""), + source_sql=source_text, target_sql="", source_object_type="table_row", source_object_id=row.get("row_index"), @@ -667,14 +788,32 @@ class TranslationExecutor: self.db.add(record) continue - if translation.strip() == "": - # Empty translation — skip + # Collect per-language translated values + per_lang_values: dict[str, str] = {} + has_any_translation = False + + # First try multi-language format (per-language keys) + for lang_code in target_languages: + lang_val = translation_data.get(lang_code) + if lang_val is not None and str(lang_val).strip(): + per_lang_values[lang_code] = str(lang_val) + has_any_translation = True + + # Fallback to legacy single "translation" key format + if not has_any_translation: + translation_text = translation_data.get("translation", "") + if translation_text.strip(): + per_lang_values[target_languages[0]] = translation_text + has_any_translation = True + + if not has_any_translation: + # Empty/all-empty translations — skip skipped += 1 record = TranslationRecord( id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id, - source_sql=row.get("source_text", ""), + source_sql=source_text, target_sql="", source_object_type="table_row", source_object_id=row.get("row_index"), @@ -686,13 +825,16 @@ class TranslationExecutor: self.db.add(record) continue + # Use the first language's value as the primary target_sql for backward compat + primary_translation = next(iter(per_lang_values.values()), "") + successful += 1 record = TranslationRecord( id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id, - source_sql=row.get("source_text", ""), - target_sql=translation, + source_sql=source_text, + target_sql=primary_translation, source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), @@ -701,6 +843,37 @@ class TranslationExecutor: ) self.db.add(record) + # Create per-language entries — one TranslationLanguage per language_code + for lang_code in target_languages: + lang_translation = per_lang_values.get(lang_code, "") + + # Source-as-reference: if detected source language matches this target, + # store the original text verbatim (no translation needed) + if detected_lang != "und" and str(lang_code).lower() == str(detected_lang).lower(): + lang_translation = source_text + + # Check for undetermined source language + lang_needs_review = (detected_lang == "und") + + if lang_needs_review: + logger.explore("undetected language", { + "record_id": row_id, + "language_code": lang_code, + "source_text": source_text[:100], + }) + + lang_entry = TranslationLanguage( + id=str(uuid.uuid4()), + record_id=record.id, + language_code=lang_code, + source_language_detected=detected_lang, + translated_value=lang_translation or "", + final_value=lang_translation or "", + status="translated", + needs_review=lang_needs_review, + ) + self.db.add(lang_entry) + return { "successful": successful, "failed": failed, @@ -806,11 +979,13 @@ class TranslationExecutor: # endregion _call_openai_compatible # region _parse_llm_response [TYPE Function] - # @PURPOSE: Parse LLM JSON response into dict of row_id -> translation. + # @PURPOSE: Parse LLM JSON response into dict of row_id -> per-language translations. + # Supports multi-language format: {"row_id": 0, "detected_source_language": "fr", "ru": "текст", "en": "text"} + # Backward-compat: old format {"row_id": 0, "translation": "text", "detected_source_language": "fr"} # @PRE: response_text is valid JSON with {"rows": [...]} structure. - # @POST: Returns dict mapping row_id to translation text. + # @POST: Returns dict mapping row_id to dict with 'detected_source_language' and per-language codes. @staticmethod - def _parse_llm_response(response_text: str, expected_count: int) -> dict[str, str]: + def _parse_llm_response(response_text: str, expected_count: int, target_languages: list[str] | None = None) -> dict[str, dict[str, str]]: with belief_scope("TranslationExecutor._parse_llm_response"): try: data = json.loads(response_text) @@ -830,15 +1005,41 @@ class TranslationExecutor: if not isinstance(rows, list): raise ValueError("LLM response missing 'rows' array") - translations: dict[str, str] = {} + translations: dict[str, dict[str, str]] = {} for item in rows: row_id = str(item.get("row_id", "")) - translation = item.get("translation") - if translation is None: - # Skip NULL translations — they'll be handled by caller + if not row_id: continue - if row_id: - translations[row_id] = str(translation) + + detected_lang = str(item.get("detected_source_language", "und")) if item.get("detected_source_language") else "und" + result: dict[str, str] = { + "detected_source_language": detected_lang, + } + + # Multi-language format: extract per-language keys + has_language_data = False + if target_languages: + for lang_code in target_languages: + lang_val = item.get(lang_code) + if lang_val is not None and str(lang_val).strip(): + result[lang_code] = str(lang_val) + has_language_data = True + + # Fallback to legacy single "translation" key format + if not has_language_data: + translation = item.get("translation") + if translation is not None: + result["translation"] = str(translation) + has_language_data = True + + if has_language_data: + translations[row_id] = result + + if len(translations) < expected_count: + logger.explore( + f"LLM returned fewer translations expected={expected_count} " + f"got={len(translations)}" + ) return translations # endregion _parse_llm_response diff --git a/backend/src/plugins/translate/metrics.py b/backend/src/plugins/translate/metrics.py index b6c161dd..ab5a544f 100644 --- a/backend/src/plugins/translate/metrics.py +++ b/backend/src/plugins/translate/metrics.py @@ -21,6 +21,7 @@ from ...models.translate import ( MetricSnapshot, TranslationEvent, TranslationRun, + TranslationRunLanguageStats, TranslationSchedule, ) @@ -121,17 +122,37 @@ class TranslationMetrics: # indicates cutoff pass - # Live events (<90 days) for token/cost - cutoff = datetime.now(UTC) - live_events = ( - self.db.query(TranslationEvent) - .filter( - TranslationEvent.job_id == job_id, - TranslationEvent.event_type.in_(["TRANSLATION_PHASE_COMPLETED", "RUN_COMPLETED"]), - TranslationEvent.created_at > cutoff, # events newer than snapshot - ) + # Per-language metrics from TranslationRunLanguageStats (live runs) + per_language: dict[str, dict[str, int | float]] = {} + lang_stats_rows = ( + self.db.query(TranslationRunLanguageStats) + .join(TranslationRun, TranslationRunLanguageStats.run_id == TranslationRun.id) + .filter(TranslationRun.job_id == job_id) .all() ) + for ls in lang_stats_rows: + lang = ls.language_code + if lang not in per_language: + per_language[lang] = {"tokens": 0, "cost": 0.0, "runs": 0, "translated_rows": 0} + per_language[lang]["tokens"] += ls.token_count or 0 + per_language[lang]["cost"] += ls.estimated_cost or 0.0 + per_language[lang]["runs"] += 1 + per_language[lang]["translated_rows"] += ls.translated_rows or 0 + + # Merge in per-language data from MetricSnapshot (pruned period) + latest_snapshot = ( + self.db.query(MetricSnapshot) + .filter(MetricSnapshot.job_id == job_id) + .order_by(MetricSnapshot.snapshot_date.desc()) + .first() + ) + if latest_snapshot and latest_snapshot.per_language_metrics: + for lang, snap_data in latest_snapshot.per_language_metrics.items(): + if lang not in per_language: + per_language[lang] = {"tokens": 0, "cost": 0.0, "runs": 0, "translated_rows": 0} + per_language[lang]["tokens"] += snap_data.get("cumulative_tokens", 0) + per_language[lang]["cost"] += snap_data.get("cumulative_cost", 0.0) + per_language[lang]["runs"] += snap_data.get("runs", 0) return { "job_id": job_id, @@ -148,6 +169,7 @@ class TranslationMetrics: "avg_duration_ms": int(avg_duration) if avg_duration else None, "last_run_at": last_run.created_at.isoformat() if last_run else None, "next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None, + "per_language_metrics": per_language, } # endregion get_job_metrics diff --git a/backend/src/plugins/translate/orchestrator.py b/backend/src/plugins/translate/orchestrator.py index 59531e3d..919f7403 100644 --- a/backend/src/plugins/translate/orchestrator.py +++ b/backend/src/plugins/translate/orchestrator.py @@ -30,9 +30,11 @@ from ...core.logger import belief_scope, logger from ...models.translate import ( TranslationBatch, TranslationJob, + TranslationLanguage, TranslationPreviewSession, TranslationRecord, TranslationRun, + TranslationRunLanguageStats, ) from .events import TranslationEventLog from .executor import TranslationExecutor @@ -201,6 +203,27 @@ class TranslationOrchestrator: created_by=self.current_user, ) + # Initialize per-language stats + target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] + if not isinstance(target_languages, list): + target_languages = [str(target_languages)] + language_stats_map: dict[str, TranslationRunLanguageStats] = {} + for lang_code in target_languages: + lang_stat = TranslationRunLanguageStats( + id=str(uuid.uuid4()), + run_id=run.id, + language_code=lang_code, + total_rows=0, + translated_rows=0, + failed_rows=0, + skipped_rows=0, + token_count=0, + estimated_cost=0.0, + ) + self.db.add(lang_stat) + language_stats_map[lang_code] = lang_stat + self.db.flush() + # Dispatch executor executor = TranslationExecutor( self.db, self.config_manager, self.current_user, @@ -227,6 +250,9 @@ class TranslationOrchestrator: self.db.commit() return run + # Aggregate per-language statistics after executor completes + self._update_language_stats(run.id, language_stats_map) + # Record translation phase complete self.event_log.log_event( job_id=job.id, @@ -745,6 +771,25 @@ class TranslationOrchestrator: # Get event summary event_summary = self.event_log.get_run_event_summary(run_id) + # Get language stats + language_stats_entries = ( + self.db.query(TranslationRunLanguageStats) + .filter(TranslationRunLanguageStats.run_id == run_id) + .all() + ) + language_stats = [ + { + "language_code": ls.language_code, + "total_rows": ls.total_rows or 0, + "translated_rows": ls.translated_rows or 0, + "failed_rows": ls.failed_rows or 0, + "skipped_rows": ls.skipped_rows or 0, + "token_count": ls.token_count or 0, + "estimated_cost": ls.estimated_cost or 0.0, + } + for ls in language_stats_entries + ] + return { "id": run.id, "job_id": run.job_id, @@ -759,6 +804,7 @@ class TranslationOrchestrator: "insert_status": run.insert_status, "superset_execution_id": run.superset_execution_id, "batch_count": batch_count, + "language_stats": language_stats, "event_invariants": { "has_run_started": event_summary["has_run_started"], "terminal_event_count": event_summary["terminal_event_count"], @@ -863,6 +909,80 @@ class TranslationOrchestrator: ] # endregion get_run_history + # region _update_language_stats [TYPE Function] + # @PURPOSE: Aggregate TranslationLanguage entries by language_code and update TranslationRunLanguageStats. + # @PRE: run_id and language_stats_map are valid. DB session is available. + # @POST: Language stats are updated with row counts and estimated tokens/cost. + # @SIDE_EFFECT: DB writes on language_stats objects. + def _update_language_stats( + self, + run_id: str, + language_stats_map: dict[str, TranslationRunLanguageStats], + ) -> None: + with belief_scope("TranslationOrchestrator._update_language_stats"): + # Get all records for this run to join with TranslationLanguage + records = ( + self.db.query(TranslationRecord) + .filter(TranslationRecord.run_id == run_id) + .all() + ) + record_ids = [r.id for r in records] + + if not record_ids: + logger.reason("No records for language stats aggregation", {"run_id": run_id}) + return + + # Get all language entries for this run's records + lang_entries = ( + self.db.query(TranslationLanguage) + .filter(TranslationLanguage.record_id.in_(record_ids)) + .all() + ) + + # Aggregate by language_code + from collections import defaultdict + agg: dict[str, dict[str, int]] = defaultdict(lambda: {"total": 0, "translated": 0, "failed": 0, "skipped": 0}) + + for le in lang_entries: + code = le.language_code + agg[code]["total"] += 1 + if le.status in ("translated", "approved", "edited"): + agg[code]["translated"] += 1 + elif le.status == "failed": + agg[code]["failed"] += 1 + elif le.status == "skipped": + agg[code]["skipped"] += 1 + + # Estimate tokens: heuristic based on character count of translated values + total_chars = sum( + len(le.translated_value or "") for le in lang_entries if le.translated_value + ) + total_tokens = max(1, total_chars // 4) # ~4 chars per token + cost_per_token = 0.002 / 1000 # $0.002 per 1K tokens + + # Update each language stat entry + for lang_code, lang_stat in language_stats_map.items(): + data = agg.get(lang_code, {"total": 0, "translated": 0, "failed": 0, "skipped": 0}) + lang_stat.total_rows = data["total"] + lang_stat.translated_rows = data["translated"] + lang_stat.failed_rows = data["failed"] + lang_stat.skipped_rows = data["skipped"] + + # Proportional token split: share tokens across languages + num_langs = len(language_stats_map) + if num_langs > 0: + lang_stat.token_count = total_tokens // num_langs + lang_stat.estimated_cost = round((lang_stat.token_count / 1000) * cost_per_token, 6) + + self.db.flush() + + logger.reason("Language stats updated", { + "run_id": run_id, + "languages": list(language_stats_map.keys()), + "total_tokens_est": total_tokens, + }) + # endregion _update_language_stats + # region _compute_config_hash [TYPE Function] # @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison. @staticmethod diff --git a/backend/src/plugins/translate/preview.py b/backend/src/plugins/translate/preview.py index 69068478..725af3ce 100644 --- a/backend/src/plugins/translate/preview.py +++ b/backend/src/plugins/translate/preview.py @@ -29,6 +29,7 @@ from ...core.superset_client import SupersetClient from ...models.translate import ( TranslationJob, TranslationJobDictionary, + TranslationPreviewLanguage, TranslationPreviewRecord, TranslationPreviewSession, ) @@ -38,16 +39,19 @@ from .dictionary import DictionaryManager # #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [TYPE Constant] # @BRIEF Default prompt template for batch LLM translation execution (no context columns — faster). +# Supports both single-language and multi-language modes via {target_languages} placeholder. DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = ( - "Translate the following database content from {source_language} to {target_language}.\n\n" + "Translate the following database content from {source_language} to the following language(s): {target_languages}.\n\n" "Source dialect: {source_dialect}\n" - "Target dialect: {target_dialect}\n" + "Target dialect(s): {target_dialect}\n" "Column to translate: {translation_column}\n\n" "{dictionary_section}" - "For each row, provide an accurate translation of the text.\n\n" + "For each row, provide an accurate translation of the text into each target language.\n\n" "Rows to translate:\n{rows_json}\n\n" "Respond with a JSON object in this exact format:\n" - '{{"rows": [{{"row_id": "", "translation": ""}}]}}\n' + '{{"rows": [{{"row_id": "", "detected_source_language": "", "": "", "": ""}}]}}\n' + "For each row, return the detected source language as a BCP-47 tag (e.g. 'en', 'ru', 'fr'), or 'und' if uncertain.\n" + "Include a separate key for EACH target language code with the translated text in that language.\n" "Each row_id must match the index provided. Return exactly {row_count} entries." ) # #endregion DEFAULT_EXECUTION_PROMPT_TEMPLATE @@ -56,16 +60,18 @@ DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = ( # #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [TYPE Constant] # @BRIEF Default prompt template for LLM translation preview. DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = ( - "Translate the following database content from {source_language} to {target_language}.\n\n" + "Translate the following database content.\n\n" "Source dialect: {source_dialect}\n" - "Target dialect: {target_dialect}\n" "Column to translate: {translation_column}\n\n" "{dictionary_section}" - "For each row, provide an accurate translation of the '{translation_column}' value.\n" + "Translate to the following languages: {target_languages}\n\n" + "For each row, provide an accurate translation of the '{translation_column}' value into each language.\n" "Consider the context columns when determining the meaning of the text.\n\n" "Rows to translate:\n{rows_json}\n\n" "Respond with a JSON object in this exact format:\n" - '{{"rows": [{{"row_id": "", "translation": ""}}]}}\n' + '{{"rows": [{{"row_id": "", "detected_source_language": "", "language_code_1": "", "language_code_2": ""}}]}}\n' + "For each row, return the detected source language as a BCP-47 tag (e.g. 'en', 'ru', 'fr'), or 'und' if uncertain.\n" + "Include a separate key for EACH target language code with the translated text in that language.\n" "Each row_id must match the index provided. Return exactly {row_count} entries." ) # #endregion DEFAULT_PREVIEW_PROMPT_TEMPLATE @@ -80,7 +86,9 @@ class TokenEstimator: CHARS_PER_TOKEN_ESTIMATE: float = 4.0 OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50 + MULTI_LANG_FACTOR: float = 1.2 # Overhead for multi-language in one call TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens + COST_WARNING_THRESHOLD: int = 30 # Show warning above this sample size # region estimate_prompt_tokens [TYPE Function] # @PURPOSE: Estimate token count for a prompt string. @@ -94,12 +102,12 @@ class TokenEstimator: # endregion estimate_prompt_tokens # region estimate_output_tokens [TYPE Function] - # @PURPOSE: Estimate output token count for translating N rows. - # @PRE: row_count >= 0. + # @PURPOSE: Estimate output token count for translating N rows across N languages. + # @PRE: row_count >= 0, num_languages >= 1. # @POST: Returns estimated output token count. @staticmethod - def estimate_output_tokens(row_count: int) -> int: - return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE + def estimate_output_tokens(row_count: int, num_languages: int = 1) -> int: + return int(row_count * num_languages * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE * TokenEstimator.MULTI_LANG_FACTOR) # endregion estimate_output_tokens # region estimate_cost [TYPE Function] @@ -112,6 +120,20 @@ class TokenEstimator: return round((total_tokens / 1000) * rate, 6) # endregion estimate_cost + # region check_cost_warning [TYPE Function] + # @PURPOSE: Generate cost warning for large previews. + # @PRE: sample_size > 0, num_languages >= 1. + # @POST: Returns warning string or None. + @staticmethod + def check_cost_warning(sample_size: int, num_languages: int, estimated_tokens: int, estimated_cost: float) -> str | None: + if sample_size > TokenEstimator.COST_WARNING_THRESHOLD: + return ( + f"Large preview — estimated {estimated_tokens} tokens, ~${estimated_cost:.4f} cost " + f"(across {num_languages} languages)" + ) + return None + # endregion check_cost_warning + # #endregion TokenEstimator @@ -134,10 +156,10 @@ class TranslationPreview: self.current_user = current_user # region preview_rows [TYPE Function] - # @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records. + # @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for multi-language translation, create preview session with per-language records. # @PRE: job_id exists and job has source_datasource_id, translation_column configured. - # @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session. - # @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows. + # @POST: Returns TranslationPreviewResponse with per-language records, cost estimation, and persistent session. + # @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord + TranslationPreviewLanguage rows. def preview_rows( self, job_id: str, @@ -157,6 +179,12 @@ class TranslationPreview: if not job.translation_column: raise ValueError("Job must have a translation column configured for preview") + # Resolve target languages: prefer target_languages list, fallback to single target_language + target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] + if not isinstance(target_languages, list): + target_languages = [str(target_languages)] + num_languages = len(target_languages) + # 2. Compute config hash and dict snapshot hash config_hash = self._compute_config_hash(job) dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id) @@ -224,40 +252,48 @@ class TranslationPreview: + "\n\n" ) - # 6. Build LLM prompt + # 6. Build LLM prompt with multi-language instructions rows_json = json.dumps([ {"row_id": str(m["row_index"]), "text": m["source_text"], "context": m["context_data"]} for m in row_meta ], indent=2) + target_languages_str = ", ".join(target_languages) template = prompt_template or DEFAULT_PREVIEW_PROMPT_TEMPLATE prompt = render_prompt(template, { "source_language": job.source_dialect or "SQL", - "target_language": job.target_language or job.target_dialect or "en", + "target_language": target_languages_str, "source_dialect": job.source_dialect or "", - "target_dialect": job.target_dialect or "", + "target_languages": target_languages_str, "translation_column": job.translation_column or "", "dictionary_section": dictionary_section, "rows_json": rows_json, "row_count": str(actual_row_count), }) - # 7. Estimate tokens/cost for sample and full dataset + # 7. Estimate tokens/cost for sample with multi-language factor sample_prompt_tokens = TokenEstimator.estimate_prompt_tokens(prompt) - sample_output_tokens = TokenEstimator.estimate_output_tokens(actual_row_count) + sample_output_tokens = TokenEstimator.estimate_output_tokens(actual_row_count, num_languages) sample_total_tokens = sample_prompt_tokens + sample_output_tokens sample_cost = TokenEstimator.estimate_cost(sample_total_tokens) - # Estimate full dataset cost (if we knew total rows) + # Estimate full dataset cost (extrapolated) + total_est_rows = sample_size * 10 # rough extrapolation total_est_tokens = TokenEstimator.estimate_prompt_tokens( - prompt.replace(str(actual_row_count), "{total}") - ) + TokenEstimator.estimate_output_tokens(sample_size * 10) # rough extrapolation + prompt.replace(str(actual_row_count), str(total_est_rows)) + ) + TokenEstimator.estimate_output_tokens(total_est_rows, num_languages) total_est_cost = TokenEstimator.estimate_cost(total_est_tokens) + # Cost warning for large previews + cost_warning = TokenEstimator.check_cost_warning( + sample_size, num_languages, sample_total_tokens, sample_cost + ) + # 8. Call LLM logger.reason("Calling LLM for preview translation", { "provider_id": job.provider_id, "row_count": actual_row_count, + "num_languages": num_languages, "estimated_tokens": sample_total_tokens, }) llm_response = self._call_llm( @@ -265,8 +301,10 @@ class TranslationPreview: prompt=prompt, ) - # 9. Parse LLM response - translations = self._parse_llm_response(llm_response, actual_row_count) + # 9. Parse LLM response (multi-language) + translations = self._parse_llm_response( + llm_response, actual_row_count, target_languages=target_languages + ) # 10. Create preview session session = TranslationPreviewSession( @@ -280,14 +318,50 @@ class TranslationPreview: self.db.add(session) self.db.flush() - # 11. Create preview records + # 11. Create preview records with per-language entries records = [] for meta in row_meta: idx = meta["row_index"] - translation = translations.get(str(idx), "") - is_rejected = False - status = "PENDING" - feedback = None + source_text = meta["source_text"] + translation_data = translations.get(str(idx), {}) + detected_lang = translation_data.get("detected_source_language", "und") if isinstance(translation_data, dict) else "und" + + # Extract per-language translations + lang_entries: list[TranslationPreviewLanguage] = [] + for lang_code in target_languages: + # Get translation for this language from LLM response + lang_translation = translation_data.get(lang_code) if isinstance(translation_data, dict) else None + + # Fallback: if no per-language data, use the single "translation" key + if not lang_translation: + if isinstance(translation_data, dict): + lang_translation = translation_data.get("translation", "") + elif isinstance(translation_data, str): + lang_translation = translation_data + else: + lang_translation = "" + + # Source-as-reference: if this language code matches source, use original text + if str(lang_code).lower() == str(detected_lang).lower() and detected_lang != "und": + # Use source text as reference (no translation needed for source language) + lang_translation = source_text + + lang_needs_review = detected_lang == "und" + + lang_entry = TranslationPreviewLanguage( + id=str(uuid.uuid4()), + preview_record_id="", # Will set after record is created + language_code=lang_code, + source_language_detected=detected_lang, + translated_value=str(lang_translation), + final_value=str(lang_translation), + status="pending", + needs_review=lang_needs_review, + ) + lang_entries.append(lang_entry) + + # Determine overall status and needs_review for the record + overall_needs_review = any(le.needs_review for le in lang_entries) # Extract source_data: store original row key columns for upsert matching source_row = meta.get("source_row", {}) @@ -299,24 +373,41 @@ class TranslationPreview: if k in source_row } elif source_row: - # No key columns configured — store the full row as fallback source_data = dict(source_row) + # Use first language's translation as target_sql for backward compat + primary_translation = lang_entries[0].translated_value if lang_entries else "" + record = TranslationPreviewRecord( id=str(uuid.uuid4()), session_id=session.id, - source_sql=meta["source_text"], - target_sql=translation, + source_sql=source_text, + target_sql=primary_translation, source_object_type="table_row", source_object_id=str(idx), source_object_name=f"Row {idx + 1}", source_data=source_data, - status=status, - feedback=feedback, + status="PENDING", + feedback=None, created_at=datetime.now(UTC), ) self.db.add(record) self.db.flush() + + # Now set the preview_record_id for all lang entries and persist + serialized_langs = [] + for le in lang_entries: + le.preview_record_id = record.id + self.db.add(le) + serialized_langs.append({ + "language_code": le.language_code, + "source_language_detected": le.source_language_detected, + "translated_value": le.translated_value, + "final_value": le.final_value, + "status": le.status, + "needs_review": le.needs_review, + }) + records.append({ "id": record.id, "source_sql": record.source_sql, @@ -326,6 +417,9 @@ class TranslationPreview: "source_object_name": record.source_object_name, "status": record.status, "feedback": record.feedback, + "source_language_detected": detected_lang, + "needs_review": overall_needs_review, + "languages": serialized_langs, }) self.db.commit() @@ -338,15 +432,18 @@ class TranslationPreview: "created_at": session.created_at.isoformat(), "expires_at": session.expires_at.isoformat() if session.expires_at else None, "records": records, + "target_languages": target_languages, "cost_estimate": { "sample_size": actual_row_count, + "num_languages": num_languages, "sample_prompt_tokens": sample_prompt_tokens, "sample_output_tokens": sample_output_tokens, "sample_total_tokens": sample_total_tokens, "sample_cost": sample_cost, - "estimated_total_rows": actual_row_count * 10, + "estimated_total_rows": total_est_rows, "estimated_tokens": total_est_tokens, "estimated_cost": total_est_cost, + "warning": cost_warning, }, "config_hash": config_hash, "dict_snapshot_hash": dict_snapshot_hash, @@ -355,15 +452,16 @@ class TranslationPreview: logger.reflect("Preview completed", { "session_id": session.id, "row_count": actual_row_count, + "num_languages": num_languages, "sample_cost": sample_cost, }) return result # endregion preview_rows # region update_preview_row [TYPE Function] - # @PURPOSE: Approve, edit, or reject an individual preview row. + # @PURPOSE: Approve, edit, or reject an individual preview row (optionally per language). # @PRE: session_id and row_id exist, session is ACTIVE. - # @POST: PreviewRecord status is updated. + # @POST: PreviewRecord status is updated. If language_code provided, only that TranslationPreviewLanguage is updated. def update_preview_row( self, job_id: str, @@ -371,6 +469,7 @@ class TranslationPreview: action: str, translation: str | None = None, feedback: str | None = None, + language_code: str | None = None, ) -> dict[str, Any]: with belief_scope("TranslationPreview.update_preview_row"): # Find the active session for this job @@ -397,16 +496,76 @@ class TranslationPreview: if not record: raise ValueError(f"Preview record '{row_id}' not found in active session") - if action == "approve": - record.status = "APPROVED" - elif action == "reject": - record.status = "REJECTED" - elif action == "edit": - record.status = "APPROVED" - if translation is not None: - record.target_sql = translation + # If language_code specified, operate on the specific TranslationPreviewLanguage + if language_code: + lang_entry = ( + self.db.query(TranslationPreviewLanguage) + .filter( + TranslationPreviewLanguage.preview_record_id == row_id, + TranslationPreviewLanguage.language_code == language_code, + ) + .first() + ) + if not lang_entry: + raise ValueError(f"Language entry '{language_code}' not found for record '{row_id}'") + + if action == "approve": + lang_entry.status = "approved" + elif action == "reject": + lang_entry.status = "rejected" + elif action == "edit": + lang_entry.status = "edited" + if translation is not None: + lang_entry.translated_value = translation + lang_entry.final_value = translation + lang_entry.user_edit = translation + else: + raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.") + + # Update record-level status based on all language entries + all_langs = ( + self.db.query(TranslationPreviewLanguage) + .filter(TranslationPreviewLanguage.preview_record_id == row_id) + .all() + ) + all_approved = all(le.status in ("approved", "edited") for le in all_langs) + any_rejected = any(le.status == "rejected" for le in all_langs) + + if all_approved: + record.status = "APPROVED" + elif any_rejected and not all_langs: + record.status = "REJECTED" + + # Update target_sql to reflect primary (first) language edit + if action == "edit" and translation is not None and all_langs: + record.target_sql = all_langs[0].final_value or all_langs[0].translated_value + else: - raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.") + # Legacy behavior: operate on whole record (all languages) + if action == "approve": + record.status = "APPROVED" + # Approve all pending language entries + for lang_entry in record.languages: + if lang_entry.status == "pending": + lang_entry.status = "approved" + elif action == "reject": + record.status = "REJECTED" + for lang_entry in record.languages: + if lang_entry.status == "pending": + lang_entry.status = "rejected" + elif action == "edit": + record.status = "APPROVED" + if translation is not None: + record.target_sql = translation + # Update primary (first) language entry + if record.languages: + first_lang = record.languages[0] + first_lang.translated_value = translation + first_lang.final_value = translation + first_lang.user_edit = translation + first_lang.status = "edited" + else: + raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.") if feedback is not None: record.feedback = feedback @@ -418,21 +577,37 @@ class TranslationPreview: "row_id": row_id, "session_id": session.id, "status": record.status, + "language_code": language_code, }) + # Build response with per-language data + lang_responses = [] + if record.languages: + for le in record.languages: + lang_responses.append({ + "language_code": le.language_code, + "source_language_detected": le.source_language_detected, + "translated_value": le.translated_value, + "final_value": le.final_value, + "user_edit": le.user_edit, + "status": le.status, + "needs_review": le.needs_review, + }) + return { "id": record.id, "source_sql": record.source_sql, "target_sql": record.target_sql, "status": record.status, "feedback": record.feedback, + "languages": lang_responses, } # endregion update_preview_row # region accept_preview_session [TYPE Function] # @PURPOSE: Mark a preview session as accepted, which gates full execution. # @PRE: job_id has an ACTIVE preview session. - # @POST: Session status changes to APPLIED. + # @POST: Session status changes to APPLIED. User edits are persisted for carry-forward. # @SIDE_EFFECT: Future full execution calls will check for accepted session. def accept_preview_session(self, job_id: str) -> dict[str, Any]: with belief_scope("TranslationPreview.accept_preview_session"): @@ -464,6 +639,12 @@ class TranslationPreview: .all() ) + # Resolve target languages + job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() + target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] if job else ["en"] + if not isinstance(target_languages, list): + target_languages = [str(target_languages)] + return { "id": session.id, "job_id": job_id, @@ -471,6 +652,7 @@ class TranslationPreview: "created_by": session.created_by, "created_at": session.created_at.isoformat(), "expires_at": session.expires_at.isoformat() if session.expires_at else None, + "target_languages": target_languages, "records": [ { "id": r.id, @@ -478,6 +660,28 @@ class TranslationPreview: "target_sql": r.target_sql, "status": r.status, "feedback": r.feedback, + "source_language_detected": ( + r.languages[0].source_language_detected + if r.languages and r.languages[0].source_language_detected + else None + ), + "needs_review": ( + r.languages[0].needs_review + if r.languages + else False + ), + "languages": [ + { + "language_code": le.language_code, + "source_language_detected": le.source_language_detected, + "translated_value": le.translated_value, + "final_value": le.final_value, + "user_edit": le.user_edit, + "status": le.status, + "needs_review": le.needs_review, + } + for le in (r.languages or []) + ] if r.languages else [], } for r in records ], @@ -487,7 +691,7 @@ class TranslationPreview: # region get_preview_session [TYPE Function] # @PURPOSE: Get the latest preview session for a job with its records. # @PRE: job_id exists. - # @POST: Returns session data with records or raises ValueError. + # @POST: Returns session data with per-language records or raises ValueError. def get_preview_session(self, job_id: str) -> dict[str, Any]: with belief_scope("TranslationPreview.get_preview_session"): session = ( @@ -505,6 +709,12 @@ class TranslationPreview: .all() ) + # Resolve target languages + job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() + target_languages = job.target_languages or [job.target_language or job.target_dialect or "en"] if job else ["en"] + if not isinstance(target_languages, list): + target_languages = [str(target_languages)] + return { "id": session.id, "job_id": job_id, @@ -512,6 +722,7 @@ class TranslationPreview: "created_by": session.created_by, "created_at": session.created_at.isoformat(), "expires_at": session.expires_at.isoformat() if session.expires_at else None, + "target_languages": target_languages, "records": [ { "id": r.id, @@ -522,6 +733,28 @@ class TranslationPreview: "source_object_name": r.source_object_name, "status": r.status, "feedback": r.feedback, + "source_language_detected": ( + r.languages[0].source_language_detected + if r.languages and r.languages[0].source_language_detected + else None + ), + "needs_review": ( + r.languages[0].needs_review + if r.languages + else False + ), + "languages": [ + { + "language_code": le.language_code, + "source_language_detected": le.source_language_detected, + "translated_value": le.translated_value, + "final_value": le.final_value, + "user_edit": le.user_edit, + "status": le.status, + "needs_review": le.needs_review, + } + for le in (r.languages or []) + ] if r.languages else [], } for r in records ], @@ -752,11 +985,11 @@ class TranslationPreview: # endregion _call_openai_compatible # region _parse_llm_response [TYPE Function] - # @PURPOSE: Parse the LLM JSON response into a dict of row_id -> translation. + # @PURPOSE: Parse the LLM JSON response into a dict of row_id -> per-language translations. # @PRE: response_text is valid JSON with {"rows": [...]} structure. - # @POST: Returns dict mapping string row_id to translation text. + # @POST: Returns dict mapping row_id to dict with 'detected_source_language' and per-language keys. @staticmethod - def _parse_llm_response(response_text: str, expected_count: int) -> dict[str, str]: + def _parse_llm_response(response_text: str, expected_count: int, target_languages: list[str] | None = None) -> dict[str, dict[str, str]]: with belief_scope("TranslationPreview._parse_llm_response"): logger.reason(f"Raw LLM response length={len(response_text)} preview={response_text[:500]}") @@ -779,12 +1012,37 @@ class TranslationPreview: logger.explore(f"LLM response has no 'rows' array, keys={list(data.keys())} text_preview={response_text[:300]}") raise ValueError("LLM response missing 'rows' array") - translations: dict[str, str] = {} + translations: dict[str, dict[str, str]] = {} for item in rows: row_id = str(item.get("row_id", "")) - translation = str(item.get("translation", "")) - if row_id: - translations[row_id] = translation + if not row_id: + continue + + detected_lang = str(item.get("detected_source_language", "und")) if item.get("detected_source_language") else "und" + result: dict[str, str] = { + "detected_source_language": detected_lang, + } + + # Multi-language format: extract per-language keys + # Keys that are NOT row_id, detected_source_language, or translation are language codes + has_language_data = False + if target_languages: + for lang_code in target_languages: + lang_val = item.get(lang_code) + if lang_val is not None and str(lang_val).strip(): + result[lang_code] = str(lang_val) + has_language_data = True + + # Fallback to old format (single "translation" key) + if not has_language_data: + translation = item.get("translation") + if translation is not None: + result["translation"] = str(translation) + else: + # Skip rows with no translation data at all + continue + + translations[row_id] = result if len(translations) < expected_count: logger.explore( diff --git a/backend/src/plugins/translate/prompt_builder.py b/backend/src/plugins/translate/prompt_builder.py new file mode 100644 index 00000000..29df8b9a --- /dev/null +++ b/backend/src/plugins/translate/prompt_builder.py @@ -0,0 +1,152 @@ +# #region ContextAwarePromptBuilder [C:2] [TYPE Module] [SEMANTICS translate, prompt, context, dictionary] +# @BRIEF Pure-function prompt builder that enhances dictionary entries with context annotations. +# @LAYER: Domain +# @RELATION DEPENDS_ON -> [DictionaryEntry:Class] +# @RATIONALE: Pure functions only — no I/O, no DB access. Separated from executor for testability. +# @REJECTED: Embedding context inline in the executor would make it untestable without mocking DB. +# +# Typical workflow: +# 1. Call build_context_entries(dictionary_entries, row_context) to get annotated, prioritized entries +# 2. Each entry is rendered via render_entry() with optional priority flag +# 3. Jaccard similarity >= 0.5 triggers priority flagging + +import json +from typing import Any + +# #region ContextAwarePromptBuilder [C:2] [TYPE Class] +# @BRIEF Build LLM prompts with context-aware dictionary entries and similarity-based priority. + +class ContextAwarePromptBuilder: + """Build LLM prompts with context-aware dictionary entries. + + Pure function — no I/O, no DB access. + """ + + @staticmethod + def render_entry(entry: Any, priority: bool = False, row_context: dict | None = None) -> str: + """Render a dictionary entry for the LLM prompt. + + Args: + entry: DictionaryEntry-like object (must have source_term, target_term, has_context, + context_data, usage_notes attrs or dict keys). + priority: Whether this entry should be flagged as high priority. + row_context: Optional row context dict (unused in rendering, kept for API symmetry). + + Returns: + Rendered prompt line string. + """ + # Support both object attrs and dict access + if isinstance(entry, dict): + source_term = entry.get("source_term", "") + target_term = entry.get("target_term", "") + has_context = entry.get("has_context", False) + context_data = entry.get("context_data") + usage_notes = entry.get("usage_notes") + else: + source_term = getattr(entry, "source_term", "") + target_term = getattr(entry, "target_term", "") + has_context = getattr(entry, "has_context", False) + context_data = getattr(entry, "context_data", None) + usage_notes = getattr(entry, "usage_notes", None) + + # Build base line + line = f'"{source_term}" -> "{target_term}"' + + # Add context annotation if present + if has_context and context_data: + context_items = [] + if isinstance(context_data, dict): + context_items = [f"{k}={v}" for k, v in context_data.items()] + elif isinstance(context_data, str): + try: + parsed = json.loads(context_data) + if isinstance(parsed, dict): + context_items = [f"{k}={v}" for k, v in parsed.items()] + else: + context_items = [str(context_data)] + except (json.JSONDecodeError, TypeError): + context_items = [str(context_data)] + + context_str = ", ".join(context_items) + + # Truncate if too long (500 tokens ≈ 2000 chars) + if len(context_str) > 2000: + context_str = context_str[:1997] + "...[truncated]" + + line = f'"{source_term}" (context: {context_str}) -> "{target_term}"' + + # Add usage notes + if usage_notes: + notes = str(usage_notes)[:200] # cap at 200 chars + line += f" # Usage: {notes}" + + # Add priority prefix + if priority: + line = f"# PRIORITY (context match) — {line}" + + return line + + @staticmethod + def compute_context_similarity(entry_context: dict | None, row_context: dict | None) -> float: + """Jaccard similarity between entry context and row context. Returns 0.0-1.0. + + Compares the sets of lowercased string values from both contexts. + Returns 1.0 for identical contexts, 0.0 for disjoint or missing. + """ + if not entry_context or not row_context: + return 0.0 + + entry_vals = set(str(v).lower() for v in entry_context.values() if v is not None) + row_vals = set(str(v).lower() for v in row_context.values() if v is not None) + + if not entry_vals or not row_vals: + return 0.0 + + intersection = entry_vals & row_vals + union = entry_vals | row_vals + return len(intersection) / len(union) + + @staticmethod + def build_context_entries( + dictionary_entries: list[Any], + row_context: dict | None = None, + ) -> list[str]: + """Build prioritized dictionary entry list with context annotations. + + Args: + dictionary_entries: List of DictionaryEntry-like objects or dicts. + row_context: Optional dict of current row's context columns. + + Returns: + List of rendered prompt strings, sorted with priority entries first. + """ + results: list[tuple[Any, bool]] = [] + + for entry in dictionary_entries: + priority = False + if row_context: + # Extract entry context_data + if isinstance(entry, dict): + entry_context = entry.get("context_data") + else: + entry_context = getattr(entry, "context_data", None) + + if entry_context: + similarity = ContextAwarePromptBuilder.compute_context_similarity( + entry_context, row_context + ) + priority = similarity >= 0.5 + + results.append((entry, priority)) + + # Sort: priority first, then non-priority + results.sort(key=lambda x: (not x[1])) + + return [ + ContextAwarePromptBuilder.render_entry(entry, priority, row_context) + for entry, priority in results + ] +# #endregion ContextAwarePromptBuilder + + +# #endregion ContextAwarePromptBuilder diff --git a/backend/src/plugins/translate/service.py b/backend/src/plugins/translate/service.py index a94526bc..5225d673 100644 --- a/backend/src/plugins/translate/service.py +++ b/backend/src/plugins/translate/service.py @@ -10,6 +10,7 @@ # @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. # @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity. +import re import uuid from datetime import UTC, datetime from typing import Any @@ -27,6 +28,7 @@ from ...schemas.translate import ( TranslateJobResponse, TranslateJobUpdate, ) +from .dictionary import _validate_bcp47 # Supported database dialects for translation SUPPORTED_DIALECTS = { @@ -227,6 +229,16 @@ class TranslateJobService: logger.warning(f"[TranslateJobService] Dialect detection failed: {e}") dialect = payload.source_dialect + # Resolve target_languages: accept new format (list) or legacy single language + target_languages = payload.target_languages + if not target_languages and payload.target_language: + target_languages = [payload.target_language] + if target_languages: + if not isinstance(target_languages, list): + target_languages = [str(target_languages)] + for lang in target_languages: + _validate_bcp47(lang, "target_languages") + # Build job instance job = TranslationJob( id=str(uuid.uuid4()), @@ -245,6 +257,7 @@ class TranslateJobService: target_column=payload.target_column, context_columns=payload.context_columns or [], target_language=payload.target_language, + target_languages=target_languages, provider_id=payload.provider_id, batch_size=payload.batch_size, upsert_strategy=payload.upsert_strategy, @@ -284,6 +297,22 @@ class TranslateJobService: update_data = payload.model_dump(exclude_unset=True) dict_ids = update_data.pop("dictionary_ids", None) + # Backward compat: if only target_language is set (old API), wrap into target_languages + if "target_language" in update_data and "target_languages" not in update_data: + target_languages = [update_data["target_language"]] if update_data["target_language"] else [] + update_data["target_languages"] = target_languages + elif "target_languages" in update_data and "target_language" not in update_data: + # Keep deprecated field in sync + if update_data["target_languages"]: + update_data["target_language"] = update_data["target_languages"][0] + + # Validate BCP-47 for target_languages + if update_data.get("target_languages"): + if not isinstance(update_data["target_languages"], list): + update_data["target_languages"] = [str(update_data["target_languages"])] + for lang in update_data["target_languages"]: + _validate_bcp47(lang, "target_languages") + for field, value in update_data.items(): if hasattr(job, field): setattr(job, field, value) @@ -366,6 +395,7 @@ class TranslateJobService: target_column=source.target_column, context_columns=source.context_columns, target_language=source.target_language, + target_languages=source.target_languages, provider_id=source.provider_id, batch_size=source.batch_size, upsert_strategy=source.upsert_strategy, @@ -468,6 +498,7 @@ def job_to_response(job: TranslationJob, dict_ids: list[str] | None = None) -> T target_column=job.target_column, context_columns=job.context_columns or [], target_language=job.target_language, + target_languages=job.target_languages, provider_id=job.provider_id, batch_size=job.batch_size or 50, upsert_strategy=job.upsert_strategy or "MERGE", @@ -546,5 +577,462 @@ def get_datasource_columns( ) # #endregion DatasourceColumnsService + +# #region InlineCorrectionService [C:3] [TYPE Class] [SEMANTICS translate, correction, inline, dictionary] +# @BRIEF Service for inline editing translated values and submitting corrections to dictionaries. +class InlineCorrectionService: + """Handle inline correction of translated values with optional dictionary submission.""" + + @staticmethod + def apply_inline_edit( + db: Session, + run_id: str, + record_id: str, + language_code: str, + final_value: str, + submit_to_dictionary: bool = False, + dictionary_id: str | None = None, + current_user: str | None = None, + context_data_override: dict | None = None, + usage_notes: str | None = None, + keep_context: bool = True, + ) -> dict: + """Apply an inline edit to a TranslationLanguage entry. + + Updates final_value and user_edit fields. Optionally submits to dictionary. + Returns the updated TranslationLanguage data as a dict. + + Args: + context_data_override: If provided, overrides auto-captured context data. + usage_notes: Usage notes for the dictionary entry. + keep_context: If False, clear context on the dictionary entry. + """ + from ...models.translate import TranslationLanguage, TranslationRecord + + # Find the language entry + lang_entry = ( + db.query(TranslationLanguage) + .filter( + TranslationLanguage.record_id == record_id, + TranslationLanguage.language_code == language_code, + ) + .first() + ) + if not lang_entry: + raise ValueError(f"Language entry '{language_code}' not found for record '{record_id}'") + + # Verify the record belongs to the run + record = ( + db.query(TranslationRecord) + .filter( + TranslationRecord.id == record_id, + TranslationRecord.run_id == run_id, + ) + .first() + ) + if not record: + raise ValueError( + f"Translation record '{record_id}' not found in run '{run_id}'" + ) + + # Apply the edit + lang_entry.final_value = final_value + lang_entry.user_edit = final_value + if lang_entry.status in ("pending", "translated"): + lang_entry.status = "edited" + db.flush() + + # Optionally submit to dictionary + dict_result = None + if submit_to_dictionary and dictionary_id: + try: + dict_result = InlineCorrectionService.submit_correction_to_dict( + db=db, + record_id=record_id, + language_code=language_code, + dictionary_id=dictionary_id, + corrected_value=final_value, + current_user=current_user, + context_data_override=context_data_override, + usage_notes=usage_notes, + keep_context=keep_context, + ) + except Exception as e: + # Dictionary submission failure should not block the edit + dict_result = {"error": str(e), "action": "failed"} + + db.commit() + db.refresh(lang_entry) + + from ...schemas.translate import TranslationLanguageResponse + response = TranslationLanguageResponse.model_validate(lang_entry) + result = response.model_dump() + if dict_result: + result["dictionary_result"] = dict_result + return result + + @staticmethod + def submit_correction_to_dict( + db: Session, + record_id: str, + language_code: str, + dictionary_id: str, + corrected_value: str, + current_user: str | None = None, + context_data_override: dict | None = None, + usage_notes: str | None = None, + keep_context: bool = True, + ) -> dict: + """Submit a correction from an inline edit to the dictionary. + + Fetches the TranslationLanguage entry, auto-captures source text and context, + and creates/updates a DictionaryEntry with correct language pair. + Returns conflict info if an entry already exists. + + Args: + context_data_override: If provided, overrides auto-captured context_data. + usage_notes: Optional usage notes to store on the dictionary entry. + keep_context: If False, sets has_context=False and clears context_data. + """ + from ...core.logger import belief_scope, logger + from ...models.translate import DictionaryEntry, TranslationLanguage, TranslationRecord + from ._utils import _normalize_term + + with belief_scope("InlineCorrectionService.submit_correction_to_dict"): + # Find the language entry + lang_entry = ( + db.query(TranslationLanguage) + .filter( + TranslationLanguage.record_id == record_id, + TranslationLanguage.language_code == language_code, + ) + .first() + ) + if not lang_entry: + raise ValueError(f"Language entry '{language_code}' not found for record '{record_id}'") + + # Find the record for source text and context + record = ( + db.query(TranslationRecord) + .filter(TranslationRecord.id == record_id) + .first() + ) + if not record: + raise ValueError(f"Translation record '{record_id}' not found") + + # Get source text from the record + source_term = record.source_sql or record.source_object_name or "" + if not source_term: + raise ValueError("No source text available for this record") + + # Determine language pair + source_language = lang_entry.source_language_detected or "und" + target_language = language_code + + # Prepare context data — auto-capture from source row + context_data = {} + if record.source_data: + context_data["source_data"] = record.source_data + if record.source_object_type: + context_data["source_object_type"] = record.source_object_type + if record.source_object_name: + context_data["source_object_name"] = record.source_object_name + if record.source_object_id: + context_data["source_object_id"] = record.source_object_id + + # Apply context_data_override if provided (user edited) + if context_data_override is not None: + context_data = context_data_override + context_source = "manual" + else: + context_source = "auto" + + # Handle keep_context=False (user explicitly removed context) + if not keep_context: + context_data = None + context_source = "manual" + + # Normalize source term + normalized = _normalize_term(source_term) + + # Check for existing entry with same (dictionary_id, source_term_norm, source_language, target_language) + existing = ( + db.query(DictionaryEntry) + .filter( + DictionaryEntry.dictionary_id == dictionary_id, + DictionaryEntry.source_term_normalized == normalized, + DictionaryEntry.source_language == source_language, + DictionaryEntry.target_language == target_language, + ) + .first() + ) + + result = { + "action": "created", + "entry_id": None, + "conflict": None, + "message": None, + } + + if existing: + # Conflict: return conflict info + result["action"] = "conflict_detected" + result["conflict"] = { + "source_term": source_term, + "existing_target_term": existing.target_term, + "submitted_target_term": corrected_value, + "action": "keep_existing", + } + result["message"] = ( + f"Existing entry found: '{existing.target_term}' " + f"for source '{source_term}' ({source_language} → {target_language})" + ) + logger.reason("Correction conflict detected", result) + return result + + # Create new entry + entry = DictionaryEntry( + dictionary_id=dictionary_id, + source_term=source_term.strip(), + source_term_normalized=normalized, + target_term=corrected_value.strip(), + source_language=source_language, + target_language=target_language, + context_data=context_data if context_data else None, + context_notes=None, + has_context=bool(context_data) and keep_context, + context_source=context_source, + usage_notes=usage_notes, + origin_source_language=source_language, + origin_run_id=record.run_id, + origin_row_key=record.id, + origin_user_id=current_user, + ) + db.add(entry) + db.flush() + result["entry_id"] = entry.id + result["message"] = ( + f"Entry created for '{source_term}' ({source_language}) → " + f"'{corrected_value}' ({target_language})" + ) + logger.reflect("Dictionary entry created from correction", result) + return result +# #endregion InlineCorrectionService + + +# #region BulkFindReplaceService [C:3] [TYPE Class] [SEMANTICS translate, bulk, find, replace, regex] +# @BRIEF Service for bulk find-and-replace operations on translated values. +class BulkFindReplaceService: + """Handle bulk find-and-replace on TranslationLanguage entries.""" + + @staticmethod + def _compile_pattern(pattern: str, is_regex: bool) -> re.Pattern: + """Compile a search pattern (regex or plain text).""" + import re + if is_regex: + return re.compile(pattern) + return re.compile(re.escape(pattern)) + + @staticmethod + def _find_matching_entries( + db: Session, + run_id: str, + pattern: str, + is_regex: bool, + target_language: str, + ) -> list[Any]: + """Find all TranslationLanguage entries matching the pattern.""" + from ...models.translate import TranslationLanguage, TranslationRecord + + # We scan entries for the target language in this run + entries = ( + db.query(TranslationLanguage) + .join( + TranslationRecord, + TranslationRecord.id == TranslationLanguage.record_id, + ) + .filter( + TranslationRecord.run_id == run_id, + TranslationLanguage.language_code == target_language, + ) + .all() + ) + return entries + + @staticmethod + def preview( + db: Session, + run_id: str, + pattern: str, + is_regex: bool, + target_language: str, + ) -> list[dict]: + """Scan TranslationLanguage entries and return matching items without applying changes.""" + from ...models.translate import TranslationRecord + + compiled = BulkFindReplaceService._compile_pattern(pattern, is_regex) + entries = BulkFindReplaceService._find_matching_entries( + db, run_id, pattern, is_regex, target_language + ) + + preview_items = [] + for entry in entries: + current_value = entry.final_value or entry.translated_value or "" + if compiled.search(current_value): + # Get source term for context + record = ( + db.query(TranslationRecord) + .filter(TranslationRecord.id == entry.record_id) + .first() + ) + source_term = record.source_sql if record else "" + + new_value = compiled.sub( + BulkFindReplaceService._get_replacement(pattern, is_regex), + current_value, + ) + if new_value != current_value: + preview_items.append({ + "record_id": entry.record_id, + "language_code": entry.language_code, + "source_term": source_term or "", + "current_value": current_value, + "new_value": new_value, + "source_language_detected": entry.source_language_detected, + }) + + return preview_items + + @staticmethod + def _get_replacement(pattern: str, is_regex: bool) -> str: + """Return replacement pattern (for regex, use backrefs; for plain, use literal).""" + # The replacement_text is passed through at apply time + return pattern # Placeholder — actual replacement is done in apply() + + @staticmethod + def apply( + db: Session, + run_id: str, + pattern: str, + is_regex: bool, + replacement_text: str, + target_language: str, + submit_to_dictionary: bool = False, + dictionary_id: str | None = None, + usage_notes: str | None = None, + current_user: str | None = None, + submit_to_dictionary_with_context: bool = False, + ) -> dict: + """Apply find-and-replace to matching TranslationLanguage entries. + + Returns dict with rows_affected, corrections_submitted, and preview list. + + Args: + submit_to_dictionary_with_context: If True, auto-capture source row context + when submitting to dictionary. + """ + from ...core.logger import belief_scope, logger + from ...models.translate import DictionaryEntry, TranslationRecord + + with belief_scope("BulkFindReplaceService.apply"): + compiled = BulkFindReplaceService._compile_pattern(pattern, is_regex) + entries = BulkFindReplaceService._find_matching_entries( + db, run_id, pattern, is_regex, target_language + ) + + preview_items = [] + rows_affected = 0 + corrections_submitted = 0 + + # Track unique (source_term, replacement) -> first record for context capture + seen_term_replacements: dict[str, str] = {} + + for entry in entries: + current_value = entry.final_value or entry.translated_value or "" + if compiled.search(current_value): + new_value = compiled.sub(replacement_text, current_value) + if new_value != current_value: + # Apply the replacement + entry.final_value = new_value + entry.user_edit = new_value + if entry.status in ("pending", "translated"): + entry.status = "edited" + rows_affected += 1 + + # Get source term for preview + record = ( + db.query(TranslationRecord) + .filter(TranslationRecord.id == entry.record_id) + .first() + ) + source_term = record.source_sql if record else "" + + # Optionally submit to dictionary + if submit_to_dictionary and dictionary_id: + try: + dict_result = InlineCorrectionService.submit_correction_to_dict( + db=db, + record_id=entry.record_id, + language_code=entry.language_code, + dictionary_id=dictionary_id, + corrected_value=new_value, + current_user=current_user, + ) + if dict_result.get("action") in ("created", "updated"): + corrections_submitted += 1 + + # Track context for unique (source_term, replacement) pairs + if submit_to_dictionary_with_context and record: + term_key = f"{source_term}::{new_value}" + if term_key not in seen_term_replacements: + seen_term_replacements[term_key] = entry.record_id + # Auto-capture context from this row + context_data = {} + if record.source_data: + context_data["source_data"] = record.source_data + if record.source_object_type: + context_data["source_object_type"] = record.source_object_type + if record.source_object_name: + context_data["source_object_name"] = record.source_object_name + if record.source_object_id: + context_data["source_object_id"] = record.source_object_id + # Update the just-created entry's context + if dict_result.get("action") == "created" and dict_result.get("entry_id"): + d_entry = ( + db.query(DictionaryEntry) + .filter(DictionaryEntry.id == dict_result["entry_id"]) + .first() + ) + if d_entry: + d_entry.context_data = context_data if context_data else None + d_entry.has_context = bool(context_data) + d_entry.context_source = "bulk" + except Exception as e: + logger.explore( + "Bulk replace: dictionary submission failed", + extra={"record_id": entry.record_id, "error": str(e)}, + ) + + preview_items.append({ + "record_id": entry.record_id, + "language_code": entry.language_code, + "source_term": source_term or "", + "current_value": current_value, + "new_value": new_value, + "source_language_detected": entry.source_language_detected, + }) + + db.commit() + + result = { + "rows_affected": rows_affected, + "corrections_submitted": corrections_submitted, + "preview": preview_items, + } + logger.reason("Bulk replace completed", result) + return result +# #endregion BulkFindReplaceService + + # #endregion TranslateJobService # #endregion TranslateJobService diff --git a/backend/src/schemas/translate.py b/backend/src/schemas/translate.py index 6b16c88e..7bbaf755 100644 --- a/backend/src/schemas/translate.py +++ b/backend/src/schemas/translate.py @@ -6,7 +6,24 @@ from datetime import datetime from typing import Any -from pydantic import BaseModel, Field +import re + +from pydantic import BaseModel, Field, field_validator + + +def _validate_bcp47_list(v: list[str] | None) -> list[str] | None: + """Validate that each item in target_languages is a valid BCP-47 tag.""" + if not v: + return v + for tag in v: + if not tag or not tag.strip(): + raise ValueError(f"Invalid BCP-47 tag: '{tag}' — must be a non-empty string") + if not re.match(r'^[a-zA-Z]{2,8}(-[a-zA-Z0-9]{1,8})*$', tag.strip()): + raise ValueError( + f"Invalid BCP-47 tag: '{tag}'. " + "Expected format like 'en', 'ru', 'zh-CN', 'pt-BR'." + ) + return v # #region TranslateJobCreate [TYPE Class] @@ -26,8 +43,11 @@ class TranslateJobCreate(BaseModel): translation_column: str | None = Field(None, description="Source column to translate") target_column: str | None = Field(None, description="Target column for translated output (defaults to translation_column)") context_columns: list[str] | None = Field(default_factory=list, description="Context column names") - target_language: str | None = Field(None, description="Target language code") + target_language: str | None = Field(None, description="Target language code [DEPRECATED: use target_languages]") + target_languages: list[str] | None = Field(default_factory=list, description="List of BCP-47 target language codes") provider_id: str | None = Field(None, description="LLM provider ID") + + _validate_target_languages = field_validator("target_languages")(_validate_bcp47_list) batch_size: int = Field(50, description="Records per batch") upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE") dictionary_ids: list[str] | None = Field(default_factory=list, description="Associated terminology dictionary IDs") @@ -54,7 +74,10 @@ class TranslateJobUpdate(BaseModel): target_column: str | None = None context_columns: list[str] | None = None target_language: str | None = None + target_languages: list[str] | None = None provider_id: str | None = None + + _validate_target_languages = field_validator("target_languages")(_validate_bcp47_list) batch_size: int | None = None upsert_strategy: str | None = None status: str | None = None @@ -83,6 +106,8 @@ class TranslateJobResponse(BaseModel): target_column: str | None = None context_columns: list[str] | None = None target_language: str | None = None + source_language: str | None = None + target_languages: list[str] | None = None provider_id: str | None = None batch_size: int = 50 upsert_strategy: str = "MERGE" @@ -154,6 +179,8 @@ class DictionaryImport(BaseModel): delimiter: str | None = Field(None, description="Detected or forced delimiter: ',' or '\\t'. Auto-detect if omitted.") on_conflict: str = Field("overwrite", description="'overwrite' or 'keep_existing' or 'cancel'") preview_only: bool = Field(False, description="If true, return preview without applying") + default_source_language: str | None = Field(None, description="Default BCP-47 source language when column missing in data") + default_target_language: str | None = Field(None, description="Default BCP-47 target language when column missing in data") # #endregion DictionaryImport @@ -177,11 +204,13 @@ class DictionaryResponse(BaseModel): # #region DictionaryEntryCreate [TYPE Class] -# @BRIEF Schema for adding/editing a dictionary entry. +# @BRIEF Schema for adding/editing a dictionary entry with language pair support. class DictionaryEntryCreate(BaseModel): source_term: str = Field(..., description="Source term to translate") target_term: str = Field(..., description="Target/translated term") context_notes: str | None = Field(None, description="Optional context notes") + source_language: str = Field("und", description="BCP-47 source language code (default: und)") + target_language: str = Field("und", description="BCP-47 target language code (default: und)") class Config: from_attributes = True @@ -196,7 +225,14 @@ class DictionaryEntryResponse(BaseModel): source_term: str source_term_normalized: str target_term: str + source_language: str | None = None + target_language: str | None = None context_notes: str | None = None + context_data: dict[str, Any] | None = None + usage_notes: str | None = None + has_context: bool = False + context_source: str | None = None + origin_source_language: str | None = None created_at: datetime updated_at: datetime | None = None @@ -232,6 +268,7 @@ class PreviewRowUpdate(BaseModel): action: str = Field(..., description="'approve', 'reject', or 'edit'") translation: str | None = Field(None, description="Edited translation (required for 'edit' action)") feedback: str | None = Field(None, description="Optional feedback/comment") + language_code: str | None = Field(None, description="BCP-47 language code for per-language actions (optional, applies to all if omitted)") # #endregion PreviewRowUpdate @@ -245,6 +282,7 @@ class PreviewAcceptResponse(BaseModel): created_at: datetime expires_at: datetime | None = None records: list['PreviewRow'] = [] + target_languages: list[str] = Field(default_factory=list, description="Target languages for this preview") class Config: from_attributes = True @@ -255,6 +293,7 @@ class PreviewAcceptResponse(BaseModel): # @BRIEF Schema for cost estimation in preview response. class CostEstimate(BaseModel): sample_size: int = 0 + num_languages: int = 1 sample_prompt_tokens: int = 0 sample_output_tokens: int = 0 sample_total_tokens: int = 0 @@ -262,6 +301,7 @@ class CostEstimate(BaseModel): estimated_total_rows: int = 0 estimated_tokens: int = 0 estimated_cost: float = 0.0 + warning: str | None = None # #endregion CostEstimate @@ -274,6 +314,8 @@ class TermCorrectionSubmit(BaseModel): dictionary_id: str | None = Field(None, description="Target dictionary ID (language-filtered)") origin_run_id: str | None = Field(None, description="Run ID from which this correction originated") origin_row_key: str | None = Field(None, description="Row key within the run") + context_data: dict[str, Any] | None = Field(None, description="Context data for the dictionary entry") + usage_notes: str | None = Field(None, description="Usage notes for the dictionary entry") # #endregion TermCorrectionSubmit @@ -369,6 +411,7 @@ class TranslationRunResponse(BaseModel): dict_snapshot_hash: str | None = None created_by: str | None = None created_at: datetime + language_stats: list['TranslationRunLanguageStatsResponse'] | None = None class Config: from_attributes = True @@ -430,6 +473,22 @@ class AggregatedMetricsResponse(BaseModel): # #endregion AggregatedMetricsResponse +# #region TranslationPreviewLanguageResponse [C:1] [TYPE Class] +# @BRIEF Schema for per-language preview data in API responses. +class TranslationPreviewLanguageResponse(BaseModel): + language_code: str + source_language_detected: str | None = None + translated_value: str | None = None + user_edit: str | None = None + final_value: str | None = None + status: str = "pending" + needs_review: bool = False + + class Config: + from_attributes = True +# #endregion TranslationPreviewLanguageResponse + + # #region PreviewRow [TYPE Class] # @BRIEF A single row in a translation preview showing original and translated content side by side. class PreviewRow(BaseModel): @@ -441,6 +500,9 @@ class PreviewRow(BaseModel): source_object_name: str | None = None status: str = "PENDING" feedback: str | None = None + source_language_detected: str | None = None + needs_review: bool = False + languages: list[TranslationPreviewLanguageResponse] = Field(default_factory=list, description="Per-language translation data") # #endregion PreviewRow @@ -455,6 +517,8 @@ class TranslationPreviewResponse(BaseModel): created_at: datetime expires_at: datetime | None = None records: list[PreviewRow] = [] + target_languages: list[str] = Field(default_factory=list, description="Target languages for this preview") + cost_estimate: CostEstimate | None = None class Config: from_attributes = True @@ -500,4 +564,105 @@ class MetricsResponse(BaseModel): from_attributes = True # #endregion MetricsResponse + +# #region TranslationLanguageResponse [C:1] [TYPE Class] +# @BRIEF Schema for per-language translation data in API responses. +class TranslationLanguageResponse(BaseModel): + language_code: str + source_language_detected: str | None = None + translated_value: str | None = None + user_edit: str | None = None + final_value: str | None = None + status: str = "pending" + needs_review: bool = False + language_overridden: bool = False + + class Config: + from_attributes = True +# #endregion TranslationLanguageResponse + + +# #region TranslationRunLanguageStatsResponse [C:1] [TYPE Class] +# @BRIEF Schema for per-language statistics in run API responses. +class TranslationRunLanguageStatsResponse(BaseModel): + language_code: str + total_rows: int = 0 + translated_rows: int = 0 + failed_rows: int = 0 + skipped_rows: int = 0 + token_count: int = 0 + estimated_cost: float = 0.0 + + class Config: + from_attributes = True +# #endregion TranslationRunLanguageStatsResponse + + +# #region OverrideLanguageRequest [C:1] [TYPE Class] +# @BRIEF Schema for manually overriding the detected source language of a translation. +class OverrideLanguageRequest(BaseModel): + source_language: str = Field(..., description="BCP-47 language code to override with") +# #endregion OverrideLanguageRequest + + +# #region BulkFindReplaceRequest [C:1] [TYPE Class] +# @BRIEF Schema for bulk find-and-replace within translations for a target language. +class BulkFindReplaceRequest(BaseModel): + find_pattern: str = Field(..., description="Text or regex pattern to find") + is_regex: bool = Field(False, description="Whether find_pattern is a regular expression") + replacement_text: str = Field(..., description="Replacement text") + target_language: str = Field(..., description="BCP-47 language code to target") + preview: bool = Field(True, description="If true, return matches without applying replacements") + submit_to_dictionary: bool = Field(False, description="If true, also submit corrections to dictionary") + dictionary_id: str | None = Field(None, description="Target dictionary ID for submitting corrections") + usage_notes: str | None = Field(None, description="Usage notes for dictionary entries") + submit_to_dictionary_with_context: bool = Field(False, description="If true, auto-capture source row context when submitting to dictionary") +# #endregion BulkFindReplaceRequest + + +# #region InlineEditRequest [C:1] [TYPE Class] +# @BRIEF Schema for inline editing a translated value on a completed run result. +class InlineEditRequest(BaseModel): + final_value: str = Field(..., description="Corrected/edited translation text") + submit_to_dictionary: bool = Field(False, description="If true, also save correction to dictionary") + dictionary_id: str | None = Field(None, description="Target dictionary ID for saving correction") + context_data_override: dict[str, Any] | None = Field(None, description="Override auto-captured context data for dictionary entry") + usage_notes: str | None = Field(None, description="Usage notes for the dictionary entry") + keep_context: bool = Field(True, description="If false, clear context_data and set has_context=False on dictionary entry") +# #endregion InlineEditRequest + + +# #region BulkReplacePreviewItem [C:1] [TYPE Class] +# @BRIEF A single item in a bulk replace preview list. +class BulkReplacePreviewItem(BaseModel): + record_id: str + language_code: str + source_term: str + current_value: str + new_value: str + source_language_detected: str | None = None +# #endregion BulkReplacePreviewItem + + +# #region BulkReplaceResponse [C:1] [TYPE Class] +# @BRIEF Response for bulk find-and-replace operation. +class BulkReplaceResponse(BaseModel): + rows_affected: int = 0 + corrections_submitted: int = 0 + preview: list[BulkReplacePreviewItem] = Field(default_factory=list) +# #endregion BulkReplaceResponse + + +# #region InlineCorrectionSubmit [C:1] [TYPE Class] +# @BRIEF Schema for submitting an inline correction for a specific translated record. +class InlineCorrectionSubmit(BaseModel): + record_id: str = Field(..., description="Translation record ID") + language_code: str = Field(..., description="BCP-47 language code of the translated value") + source_term: str = Field(..., description="Original source term that was incorrectly translated") + corrected_term: str = Field(..., description="Corrected translation") + dictionary_id: str = Field(..., description="Target dictionary ID to also save the correction") + source_language: str = Field(..., description="BCP-47 source language code") +# #endregion InlineCorrectionSubmit + + # #endregion TranslateSchemas diff --git a/backend/tests/test_translate_history.py b/backend/tests/test_translate_history.py index e55d3cef..61fe7a7d 100644 --- a/backend/tests/test_translate_history.py +++ b/backend/tests/test_translate_history.py @@ -295,4 +295,172 @@ def test_run_detail_not_found(client): response = client.get("/api/translate/runs/non-existent/detail") assert response.status_code == 404 # [/DEF:test_run_detail_not_found:Function] +# [DEF:test_list_runs_includes_language_info:Function] +# @PURPOSE: Verify run list includes target_languages, total_translated, and language_stats. +def test_list_runs_includes_language_info(client, mock_api_deps): + """Test run list includes per-language fields.""" + from src.models.translate import TranslationRunLanguageStats + session = mock_api_deps["session"] + job = _create_test_job(session) + run = _create_test_run(session, job.id) + # Add language stats + ls = TranslationRunLanguageStats( + id=str(uuid.uuid4()), + run_id=run.id, + language_code="ru", + total_rows=100, + translated_rows=80, + failed_rows=5, + skipped_rows=15, + token_count=15000, + estimated_cost=0.03, + ) + session.add(ls) + session.commit() + + response = client.get("/api/translate/runs") + assert response.status_code == 200 + data = response.json() + assert len(data["items"]) >= 1 + item = data["items"][0] + assert "target_languages" in item + assert "total_translated" in item + assert "language_stats" in item + assert "ru" in item["target_languages"] + assert item["total_translated"] == 80 + assert item["language_stats"]["ru"]["translated_rows"] == 80 +# [/DEF:test_list_runs_includes_language_info:Function] + + +# [DEF:test_metrics_per_language_breakdown:Function] +# @PURPOSE: Verify metrics endpoint returns per_language_metrics. +def test_metrics_per_language_breakdown(client, mock_api_deps): + """Test metrics returns per-language breakdown.""" + from src.models.translate import TranslationRunLanguageStats + session = mock_api_deps["session"] + job = _create_test_job(session) + run = _create_test_run(session, job.id) + ls = TranslationRunLanguageStats( + id=str(uuid.uuid4()), + run_id=run.id, + language_code="ru", + total_rows=100, + translated_rows=80, + token_count=15000, + estimated_cost=0.03, + ) + session.add(ls) + session.commit() + + response = client.get(f"/api/translate/jobs/{job.id}/metrics") + assert response.status_code == 200 + data = response.json() + assert "per_language_metrics" in data + assert "ru" in data["per_language_metrics"] + assert data["per_language_metrics"]["ru"]["tokens"] == 15000 + assert data["per_language_metrics"]["ru"]["cost"] == 0.03 +# [/DEF:test_metrics_per_language_breakdown:Function] + + +# [DEF:test_metric_snapshot_stores_per_language:Function] +# @PURPOSE: Verify MetricSnapshot stores per_language_metrics via prune_expired. +def test_metric_snapshot_stores_per_language(client, mock_api_deps): + """Test MetricSnapshot stores per_language_metrics after prune.""" + from datetime import timedelta + from src.models.translate import MetricSnapshot, TranslationEvent + from src.plugins.translate.events import TranslationEventLog + + session = mock_api_deps["session"] + job = _create_test_job(session) + session.commit() + + # Create an old event with per-language data + old_date = datetime.now(UTC) - timedelta(days=100) + event = TranslationEvent( + id=str(uuid.uuid4()), + job_id=job.id, + event_type="TRANSLATION_PHASE_COMPLETED", + event_data={ + "language_code": "ru", + "token_count": 5000, + "cost": 0.01, + "run_id": "test-run-id", + }, + created_at=old_date, + ) + session.add(event) + session.commit() + + # Prune events older than 30 days + event_log = TranslationEventLog(session) + result = event_log.prune_expired(retention_days=30) + assert result["snapshot_id"] is not None + + # Verify snapshot has per_language_metrics + snapshot = session.query(MetricSnapshot).filter( + MetricSnapshot.id == result["snapshot_id"] + ).first() + assert snapshot is not None + assert snapshot.per_language_metrics is not None + assert "ru" in snapshot.per_language_metrics + assert snapshot.per_language_metrics["ru"]["cumulative_tokens"] == 5000 + assert snapshot.per_language_metrics["ru"]["cumulative_cost"] == 0.01 +# [/DEF:test_metric_snapshot_stores_per_language:Function] + + +# [DEF:test_combined_metrics_live_and_snapshot:Function] +# @PURPOSE: Verify combined metrics merge live + snapshot per-language data. +def test_combined_metrics_live_and_snapshot(client, mock_api_deps): + """Test metrics merge live data with MetricSnapshot per-language data.""" + from datetime import timedelta + from src.models.translate import ( + MetricSnapshot, + TranslationEvent, + TranslationRunLanguageStats, + ) + from src.plugins.translate.metrics import TranslationMetrics + + session = mock_api_deps["session"] + job = _create_test_job(session) + run = _create_test_run(session, job.id) + session.commit() + + # Add snapshot with per-language data (simulating pruned period) + snapshot = MetricSnapshot( + id=str(uuid.uuid4()), + job_id=job.id, + key_hash="test_key", + per_language_metrics={ + "ru": {"cumulative_tokens": 100000, "cumulative_cost": 0.20, "runs": 3}, + }, + covers_events_before=datetime.now(UTC) - timedelta(days=90), + total_records=100, + snapshot_date=datetime.now(UTC), + ) + session.add(snapshot) + + # Add live language stats (recent runs) + ls = TranslationRunLanguageStats( + id=str(uuid.uuid4()), + run_id=run.id, + language_code="ru", + total_rows=50, + translated_rows=40, + token_count=5000, + estimated_cost=0.01, + ) + session.add(ls) + session.commit() + + metrics_svc = TranslationMetrics(session) + result = metrics_svc.get_job_metrics(job.id) + assert "per_language_metrics" in result + assert "ru" in result["per_language_metrics"] + # Live 5000 + snapshot 100000 = 105000 + assert result["per_language_metrics"]["ru"]["tokens"] == 105000 + # Live 0.01 + snapshot 0.20 = 0.21 + assert abs(result["per_language_metrics"]["ru"]["cost"] - 0.21) < 0.001 + # Live 1 + snapshot 3 = 4 + assert result["per_language_metrics"]["ru"]["runs"] == 4 +# [/DEF:test_combined_metrics_live_and_snapshot:Function] # [/DEF:TranslateHistoryTests:Module] diff --git a/frontend/src/lib/api/translate.js b/frontend/src/lib/api/translate.js index 72458dd2..642fbea7 100644 --- a/frontend/src/lib/api/translate.js +++ b/frontend/src/lib/api/translate.js @@ -136,14 +136,14 @@ export async function fetchPreview(jobId, sampleSize = 10, envId = '') { // #endregion fetchPreview:Function // #region approveRow:Function [TYPE Function] -// @PURPOSE: Approve a preview row. +// @PURPOSE: Approve a preview row (optionally per language). // @PRE: jobId and rowKey are non-empty strings. // @POST: Returns updated row. -export async function approveRow(jobId, rowKey) { +export async function approveRow(jobId, rowKey, languageCode = null) { try { - return await api.requestApi(`/translate/jobs/${jobId}/preview/rows/${rowKey}`, 'PUT', { - action: 'approve' - }); + const body = { action: 'approve' }; + if (languageCode) body.language_code = languageCode; + return await api.requestApi(`/translate/jobs/${jobId}/preview/rows/${rowKey}`, 'PUT', body); } catch (error) { throw normalizeTranslateError(error, 'Failed to approve row'); } @@ -151,15 +151,14 @@ export async function approveRow(jobId, rowKey) { // #endregion approveRow:Function // #region editRow:Function [TYPE Function] -// @PURPOSE: Edit a preview row's translation. +// @PURPOSE: Edit a preview row's translation (optionally per language). // @PRE: jobId and rowKey are non-empty strings; translation is non-empty. // @POST: Returns updated row. -export async function editRow(jobId, rowKey, translation) { +export async function editRow(jobId, rowKey, translation, languageCode = null) { try { - return await api.requestApi(`/translate/jobs/${jobId}/preview/rows/${rowKey}`, 'PUT', { - action: 'edit', - translation: translation - }); + const body = { action: 'edit', translation: translation }; + if (languageCode) body.language_code = languageCode; + return await api.requestApi(`/translate/jobs/${jobId}/preview/rows/${rowKey}`, 'PUT', body); } catch (error) { throw normalizeTranslateError(error, 'Failed to edit row'); } @@ -167,14 +166,14 @@ export async function editRow(jobId, rowKey, translation) { // #endregion editRow:Function // #region rejectRow:Function [TYPE Function] -// @PURPOSE: Reject a preview row. +// @PURPOSE: Reject a preview row (optionally per language). // @PRE: jobId and rowKey are non-empty strings. // @POST: Returns updated row. -export async function rejectRow(jobId, rowKey) { +export async function rejectRow(jobId, rowKey, languageCode = null) { try { - return await api.requestApi(`/translate/jobs/${jobId}/preview/rows/${rowKey}`, 'PUT', { - action: 'reject' - }); + const body = { action: 'reject' }; + if (languageCode) body.language_code = languageCode; + return await api.requestApi(`/translate/jobs/${jobId}/preview/rows/${rowKey}`, 'PUT', body); } catch (error) { throw normalizeTranslateError(error, 'Failed to reject row'); } @@ -579,6 +578,57 @@ export async function fetchAllMetrics() { } // #endregion fetchAllMetrics:Function +// ============================================================ +// Inline Correction & Bulk Replace API +// ============================================================ + +// #region inlineEditCorrection:Function [TYPE Function] +// @PURPOSE: Apply an inline correction to a translated value on a completed run result. +// @PRE: runId, recordId, languageCode are non-empty strings; data contains final_value. +// @POST: Returns updated language entry with optional dictionary result. +export async function inlineEditCorrection(runId, recordId, languageCode, data) { + try { + return await api.requestApi( + `/translate/runs/${runId}/records/${recordId}/languages/${languageCode}`, + 'PUT', + data + ); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to apply inline correction'); + } +} +// #endregion inlineEditCorrection:Function + +// #region submitCorrectionToDict:Function [TYPE Function] +// @PURPOSE: Submit an inline correction to the dictionary from a run result. +// @PRE: runId, recordId, languageCode are non-empty strings; dictId is provided. +// @POST: Returns dictionary submission result with conflict info. +export async function submitCorrectionToDict(runId, recordId, languageCode, dictId) { + try { + return await api.requestApi( + `/translate/runs/${runId}/records/${recordId}/languages/${languageCode}/submit-to-dict`, + 'POST', + { dictionary_id: dictId } + ); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to submit correction to dictionary'); + } +} +// #endregion submitCorrectionToDict:Function + +// #region bulkFindReplace:Function [TYPE Function] +// @PURPOSE: Perform bulk find-and-replace on translated values within a run. +// @PRE: runId is non-empty string; data contains find_pattern, replacement_text, target_language. +// @POST: Returns result with rows_affected, corrections_submitted, and preview list. +export async function bulkFindReplace(runId, data) { + try { + return await api.postApi(`/translate/runs/${runId}/bulk-replace`, data); + } catch (error) { + throw normalizeTranslateError(error, 'Failed to perform bulk find-and-replace'); + } +} +// #endregion bulkFindReplace:Function + // #region downloadSkippedCsv:Function [TYPE Function] // @PURPOSE: Download skipped records CSV for a run. export async function downloadSkippedCsv(runId) { diff --git a/frontend/src/lib/components/translate/CorrectionCell.svelte b/frontend/src/lib/components/translate/CorrectionCell.svelte new file mode 100644 index 00000000..63db9543 --- /dev/null +++ b/frontend/src/lib/components/translate/CorrectionCell.svelte @@ -0,0 +1,273 @@ + + + + + + + + + +{#if uxState === 'view'} + e.key === 'Enter' && startEditing()} + > + {#if displayValue} + {displayValue} + {:else} + — + {/if} + + + +{:else if uxState === 'editing'} +
+ { + if (e.key === 'Enter') saveEdit(); + if (e.key === 'Escape') cancelEditing(); + }} + /> +
+ + +
+
+ + +{:else if uxState === 'saving'} +
+
+ Saving... +
+ + +{:else if uxState === 'saved'} +
+ {displayValue} + 📝 + +
+ + +{:else if uxState === 'submitting_to_dict'} +
+
+ Submitting to dictionary... +
+ + +{:else if uxState === 'dict_submitted'} +
+ {displayValue} + ✅ Dictionary ✓ +
+ + +{:else if uxState === 'error'} +
+ {displayValue} + {errorMessage} + +
+{/if} + + +{#if dictPopupOpen} +
+
e.stopPropagation()}> +

Submit to Dictionary

+
+ {#if sourceLanguage} +
+ Language pair: {sourceLanguage} → {languageCode} +
+ {/if} +
+ +
{editValue}
+
+
+ + +
+
+ + +
+
+
+
+{/if} + diff --git a/frontend/src/lib/components/translate/TermCorrectionPopup.svelte b/frontend/src/lib/components/translate/TermCorrectionPopup.svelte index 480c82ed..98016145 100644 --- a/frontend/src/lib/components/translate/TermCorrectionPopup.svelte +++ b/frontend/src/lib/components/translate/TermCorrectionPopup.svelte @@ -24,10 +24,16 @@ let { sourceTerm = '', incorrectTarget = '', + sourceLanguage = '', + targetLanguage = '', runId = '', rowKey = '', onClose = () => {}, - onSubmitted = () => {} + onSubmitted = () => {}, + /** @type {Record|null} initial context data auto-captured from source row */ + initialContextData = null, + /** @type {string|null} optional context usage notes */ + initialUsageNotes = '' } = $props(); /** @type {'closed'|'selecting'|'editing'|'submitting'|'conflict_detected'|'submitted'} */ @@ -38,6 +44,23 @@ let conflictInfo = $state(null); let submitResult = $state(null); + // Context state + let contextData = $state(null); + let contextUsageNotes = $state(''); + let hasEditedContext = $state(false); + let showContextEditor = $state(false); + let contextEditorText = $state(''); + + $effect(() => { + if (initialContextData) { + contextData = JSON.parse(JSON.stringify(initialContextData)); + contextEditorText = JSON.stringify(initialContextData, null, 2); + } + if (initialUsageNotes) { + contextUsageNotes = initialUsageNotes; + } + }); + $effect(() => { if (sourceTerm && incorrectTarget) { uxState = 'selecting'; @@ -63,14 +86,21 @@ if (!selectedDictId || !correctedTarget.trim()) return; uxState = 'submitting'; try { - const result = await submitCorrection({ + const payload = { source_term: sourceTerm, incorrect_target_term: incorrectTarget, corrected_target_term: correctedTarget.trim(), dictionary_id: selectedDictId, origin_run_id: runId || undefined, origin_row_key: rowKey || undefined, - }); + }; + if (contextData) { + payload.context_data = contextData; + } + if (contextUsageNotes) { + payload.usage_notes = contextUsageNotes; + } + const result = await submitCorrection(payload); if (result.action === 'conflict_detected') { conflictInfo = result.conflict; uxState = 'conflict_detected'; @@ -138,6 +168,90 @@ {:else if uxState === 'editing'}
+ {#if sourceLanguage || targetLanguage} +
+ Language pair: {sourceLanguage || 'detected'} → {targetLanguage || languageCode} +
+ {/if} + + +
+ + {#if showContextEditor} +
+ {#if contextData} +
+ {JSON.stringify(contextData, null, 2)} +
+
+ + +
+ {:else} +

No context data available

+ + {/if} +
+ {/if} +
+ + +
+ + +
+
@@ -146,6 +260,14 @@
+
+ + +
+
+ + +
+ + +
+ {#if label} + + {/if} + + {#if searchable} + + {/if} + +
+ {#if filteredOptions.length === 0} +
+ No matching options +
+ {:else} + {#each filteredOptions as opt} + + {/each} + {/if} +
+ + {#if selected.length > 0} +
+ {#each selected as code} + {@const opt = options.find(o => o.code === code)} + + {opt?.name || code} + + + {/each} +
+ {/if} + + {#if error} +

{error}

+ {/if} +
+ diff --git a/frontend/src/lib/i18n/locales/en/translate.json b/frontend/src/lib/i18n/locales/en/translate.json index e1d7196f..dffb0f49 100644 --- a/frontend/src/lib/i18n/locales/en/translate.json +++ b/frontend/src/lib/i18n/locales/en/translate.json @@ -85,7 +85,9 @@ "target_database": "Target Database (SQL Lab)", "select_target_database": "Select database for INSERT...", "target_database_hint": "Database connection used for writing translated data via SQL Lab", - "loading_databases": "Loading databases..." + "loading_databases": "Loading databases...", + "include_source_reference": "Include source language in translations", + "include_source_reference_hint": "The original text will be stored as a verified reference copy in its detected language" }, "preview": { "title": "Preview", @@ -142,7 +144,11 @@ "reject_failed": "Failed to reject row", "edit_failed": "Failed to save changes", "row_updated": "Row updated", - "accept_failed": "Failed to accept preview" + "accept_failed": "Failed to accept preview", + "detected_language": "Detected Lang", + "languages_count": "Languages: {count}", + "across_languages": "Across {count} language(s)", + "language_label": "Language" }, "jobs": { "title": "Translation Jobs", diff --git a/frontend/src/lib/i18n/locales/ru/translate.json b/frontend/src/lib/i18n/locales/ru/translate.json index d53d7e60..25766ab6 100644 --- a/frontend/src/lib/i18n/locales/ru/translate.json +++ b/frontend/src/lib/i18n/locales/ru/translate.json @@ -85,7 +85,9 @@ "target_database": "База данных (SQL Lab)", "select_target_database": "Выберите базу данных для INSERT...", "target_database_hint": "Подключение к БД для записи переведённых данных через SQL Lab", - "loading_databases": "Загрузка баз данных..." + "loading_databases": "Загрузка баз данных...", + "include_source_reference": "Сохранять исходный текст как эталонную копию", + "include_source_reference_hint": "Исходный текст будет сохранён как верифицированная эталонная копия на его обнаруженном языке" }, "preview": { "title": "Предпросмотр", @@ -142,7 +144,11 @@ "reject_failed": "Не удалось отклонить строку", "edit_failed": "Не удалось сохранить изменения", "row_updated": "Строка обновлена", - "accept_failed": "Не удалось принять предпросмотр" + "accept_failed": "Не удалось принять предпросмотр", + "detected_language": "Язык ист.", + "languages_count": "Языков: {count}", + "across_languages": "Для {count} язык(ов)", + "language_label": "Язык" }, "jobs": { "title": "Задания перевода", diff --git a/frontend/src/routes/translate/+page.svelte b/frontend/src/routes/translate/+page.svelte index d6a72a6a..adac7af0 100644 --- a/frontend/src/routes/translate/+page.svelte +++ b/frontend/src/routes/translate/+page.svelte @@ -234,7 +234,9 @@ {#if job.source_dialect && job.target_dialect} {job.source_dialect} → {job.target_dialect} {/if} - {#if job.target_language} + {#if job.target_languages?.length > 0} + {$t.translate?.jobs?.language_label.replace('{language}', job.target_languages.join(', '))} + {:else if job.target_language} {$t.translate?.jobs?.language_label.replace('{language}', job.target_language)} {/if} {#if job.translation_column} diff --git a/frontend/src/routes/translate/[id]/+page.svelte b/frontend/src/routes/translate/[id]/+page.svelte index 036afc9b..c925aa43 100644 --- a/frontend/src/routes/translate/[id]/+page.svelte +++ b/frontend/src/routes/translate/[id]/+page.svelte @@ -43,12 +43,41 @@ fetchDatasourceColumns, fetchDatasources, } from '$lib/api/translate.js'; + import MultiSelect from '$lib/components/ui/MultiSelect.svelte'; import TranslationPreview from '$lib/components/translate/TranslationPreview.svelte'; import TranslationRunProgress from '$lib/components/translate/TranslationRunProgress.svelte'; import TranslationRunResult from '$lib/components/translate/TranslationRunResult.svelte'; import ScheduleConfig from '$lib/components/translate/ScheduleConfig.svelte'; import { triggerRun, fetchRunHistory, cancelRun } from '$lib/api/translate.js'; + const LANGUAGES = [ + { code: 'ru', name: 'Russian' }, + { code: 'en', name: 'English' }, + { code: 'de', name: 'German' }, + { code: 'fr', name: 'French' }, + { code: 'es', name: 'Spanish' }, + { code: 'it', name: 'Italian' }, + { code: 'pt', name: 'Portuguese' }, + { code: 'zh', name: 'Chinese' }, + { code: 'ja', name: 'Japanese' }, + { code: 'ko', name: 'Korean' }, + { code: 'ar', name: 'Arabic' }, + { code: 'tr', name: 'Turkish' }, + { code: 'nl', name: 'Dutch' }, + { code: 'pl', name: 'Polish' }, + { code: 'sv', name: 'Swedish' }, + { code: 'da', name: 'Danish' }, + { code: 'fi', name: 'Finnish' }, + { code: 'cs', name: 'Czech' }, + { code: 'hu', name: 'Hungarian' }, + { code: 'ro', name: 'Romanian' }, + { code: 'vi', name: 'Vietnamese' }, + { code: 'th', name: 'Thai' }, + { code: 'he', name: 'Hebrew' }, + { code: 'id', name: 'Indonesian' }, + { code: 'ms', name: 'Malay' }, + ]; + /** @type {string} idle | loading | configured | saving | validation_error | datasource_unavailable */ let uxState = $state('idle'); let isNewJob = $derived(page.params.id === 'new'); @@ -69,7 +98,8 @@ let targetKeyCols = $state([]); let translationColumn = $state(''); let contextColumns = $state([]); - let targetLanguage = $state('en'); + let targetLanguages = $state(['en']); + let includeSourceReference = $state(true); let providerId = $state(''); let batchSize = $state(50); let upsertStrategy = $state('MERGE'); @@ -263,7 +293,7 @@ targetKeyCols = j.target_key_cols || []; translationColumn = j.translation_column || ''; contextColumns = j.context_columns || []; - targetLanguage = j.target_language || 'en'; + targetLanguages = j.target_languages?.length > 0 ? j.target_languages : (j.target_language ? [j.target_language] : ['en']); providerId = j.provider_id || ''; batchSize = j.batch_size || 50; upsertStrategy = j.upsert_strategy || 'MERGE'; @@ -457,6 +487,10 @@ errs.name = _('translate.config.name_required'); } + if (targetLanguages.length === 0) { + errs.targetLanguages = 'At least one target language is required'; + } + if (!translationColumn) { errs.translationColumn = _('translate.config.translation_column_required'); } @@ -505,7 +539,7 @@ target_key_cols: targetKeyCols, translation_column: translationColumn || null, context_columns: contextColumns, - target_language: targetLanguage || null, + target_languages: targetLanguages?.length > 0 ? targetLanguages : null, provider_id: providerId || null, batch_size: parseInt(batchSize) || 50, upsert_strategy: upsertStrategy, @@ -971,17 +1005,17 @@
- - + + {#if targetLanguages.length === 0} +

Select at least one target language

+ {/if}
@@ -1002,6 +1036,22 @@
+ +
+ +
diff --git a/frontend/src/routes/translate/dictionaries/[id]/+page.svelte b/frontend/src/routes/translate/dictionaries/[id]/+page.svelte index 05d96651..54285545 100644 --- a/frontend/src/routes/translate/dictionaries/[id]/+page.svelte +++ b/frontend/src/routes/translate/dictionaries/[id]/+page.svelte @@ -56,6 +56,9 @@ let addForm = $state({ source_term: '', target_term: '', context_notes: '' }); let isAdding = $state(false); + // Expanded context rows + let expandedEntryId = $state(null); + // Import let showImportForm = $state(false); let importContent = $state(''); @@ -513,10 +516,24 @@ {:else} - + expandedEntryId = expandedEntryId === entry.id ? null : entry.id}> {entry.source_term} {entry.target_term} - {entry.context_notes || ''} + +
+ {entry.context_notes || ''} + {#if entry.has_context} + + context + + {/if} + {#if entry.usage_notes} + + notes + + {/if} +
+ {#if deleteEntryId === entry.id}
@@ -532,6 +549,34 @@ {/if} + {#if expandedEntryId === entry.id} + + +
+ {#if entry.has_context && entry.context_data} +
+ Context data: + 
{JSON.stringify(entry.context_data, null, 2)}
+
+ {:else} +

No context data

+ {/if} + {#if entry.usage_notes} +
+ Usage notes: +

{entry.usage_notes}

+
+ {/if} + {#if entry.context_source} +
+ Source: + {entry.context_source} +
+ {/if} +
+ + + {/if} {/if} {/each} diff --git a/frontend/src/routes/translate/history/+page.svelte b/frontend/src/routes/translate/history/+page.svelte index f5905340..d3068f43 100644 --- a/frontend/src/routes/translate/history/+page.svelte +++ b/frontend/src/routes/translate/history/+page.svelte @@ -256,6 +256,14 @@ by {run.created_by} {/if}
+ + {#if run.target_languages && run.target_languages.length > 0} +
+ {#each run.target_languages as lang} + {lang} + {/each} +
+ {/if}
e.stopPropagation()}> {#if run.status === 'COMPLETED' && (run.skipped_records || 0) > 0} @@ -353,6 +361,28 @@
+ + {#if selectedRunDetail.language_stats && Object.keys(selectedRunDetail.language_stats).length > 0} +
+

Per-Language Statistics

+
+ {#each Object.entries(selectedRunDetail.language_stats) as [lang, stats]} +
+
+ {lang} +
+
+ {stats.translated_rows} translated + {stats.failed_rows} failed + {stats.skipped_rows} skipped + {stats.token_count} tokens +
+
+ {/each} +
+
+ {/if} + {#if selectedRunDetail.config_snapshot}