64 lines
3.2 KiB
Python
64 lines
3.2 KiB
Python
# #region TextCleaner [C:3] [TYPE Module] [SEMANTICS translate, text, cleaning, whitespace, truncation]
|
|
# @BRIEF Text cleaning utilities for the translation pipeline — whitespace normalization
|
|
# and character-length truncation. Used as a preprocessing step before LLM calls.
|
|
# @LAYER Domain
|
|
# @REJECTED Doing normalization inside the LLM prompt — wastes tokens and produces
|
|
# inconsistent results. Preprocessing text before the LLM call is more reliable.
|
|
# @RATIONALE Centralizes text preprocessing logic used across the translation pipeline.
|
|
# Normalizing whitespace before truncation ensures accurate character counts.
|
|
|
|
# #region normalize_whitespace [C:2] [TYPE Function] [SEMANTICS text, whitespace, normalize]
|
|
# @BRIEF Collapse multiple spaces, tabs, and newlines into single spaces; trim leading/trailing
|
|
# whitespace. Returns empty string for empty or whitespace-only input.
|
|
# @PRE text is a string (empty string allowed).
|
|
# @POST Returns a string with no leading/trailing whitespace and single internal spaces.
|
|
# @EXAMPLE normalize_whitespace(" hello world ") -> "hello world"
|
|
# @EXAMPLE normalize_whitespace("") -> ""
|
|
def normalize_whitespace(text: str) -> str:
|
|
"""Collapse multiple spaces/newlines, trim, return clean text.
|
|
|
|
All Unicode whitespace characters (spaces, tabs, newlines, etc.) are
|
|
collapsed into single ASCII spaces. Leading/trailing whitespace is removed.
|
|
"""
|
|
if not text:
|
|
return ""
|
|
return " ".join(text.split())
|
|
# #endregion normalize_whitespace
|
|
|
|
|
|
# #region truncate_text [C:2] [TYPE Function] [SEMANTICS text, truncation, length]
|
|
# @BRIEF Truncate text to max_length characters, appending "..." if truncation occurs.
|
|
# @PRE text is a string. max_length >= 0.
|
|
# @POST When len(text) <= max_length, returns text unchanged.
|
|
# When len(text) > max_length, returns text[:max_length] + "...".
|
|
# @EXAMPLE truncate_text("hello world", 5) -> "hello..."
|
|
# @EXAMPLE truncate_text("hi", 5) -> "hi"
|
|
def truncate_text(text: str, max_length: int = 500) -> str:
|
|
"""Truncate text to max_length characters, adding '...' if truncated.
|
|
|
|
If the text is shorter than or equal to max_length, it is returned unchanged.
|
|
If it exceeds max_length, it is cut at max_length and '...' is appended.
|
|
"""
|
|
if len(text) > max_length:
|
|
return text[:max_length] + "..."
|
|
return text
|
|
# #endregion truncate_text
|
|
|
|
|
|
# #region clean_text [C:2] [TYPE Function] [SEMANTICS text, clean, normalize, truncate]
|
|
# @BRIEF Combine whitespace normalization and truncation into one step.
|
|
# Applies normalize_whitespace first, then truncate_text.
|
|
# @PRE text is a string. max_length >= 0.
|
|
# @POST Returns normalized and optionally truncated text.
|
|
# @RELATION CALLS -> [normalize_whitespace]
|
|
# @RELATION CALLS -> [truncate_text]
|
|
# @EXAMPLE clean_text(" hello world ", 5) -> "hello..." (normalized to "hello world",
|
|
# then truncated since "hello world" has 11 chars > 5)
|
|
def clean_text(text: str, max_length: int = 500) -> str:
|
|
"""Normalize whitespace then truncate to max_length."""
|
|
normalized = normalize_whitespace(text)
|
|
return truncate_text(normalized, max_length)
|
|
# #endregion clean_text
|
|
|
|
# #endregion TextCleaner
|