busya
c6189876b3
Auto-fixed categories: - F401: unused imports removed - I001: import blocks sorted - W293: trailing whitespace stripped - UP035: deprecated typing imports replaced - SIM: simplify suggestions applied - ARG: unused args prefixed with underscore - T201: print statements removed - F841: unused variables removed - RUF059: unpacked variables prefixed Remaining ~1500 unfixable errors (C901, B904, N806, E402) require manual work. Backend smoke tests: 13/13 passed.
34 lines
1.4 KiB
Python
34 lines
1.4 KiB
Python
# #region DictionaryUtils [C:1] [TYPE Module] [SEMANTICS translate, term, normalize, delimiter, csv]
|
|
# @BRIEF Utility helpers for dictionary management (term normalization and delimiter detection).
|
|
# @LAYER: Domain
|
|
|
|
import unicodedata
|
|
|
|
|
|
# #region _normalize_term [TYPE Function]
|
|
# @BRIEF Normalize a term for case-insensitive unique constraint lookup.
|
|
# @RATIONALE: NFC normalization is applied before lowercasing to ensure consistent
|
|
# comparison of Unicode characters (e.g. precomposed vs decomposed forms).
|
|
# @REJECTED: Lowercasing without NFC normalization — would cause duplicate entries
|
|
# for semantically identical Unicode strings in different normalization forms.
|
|
def _normalize_term(term: str) -> str:
|
|
"""Normalize a term by NFC, lowercasing, and removing extra whitespace."""
|
|
if not term:
|
|
return ""
|
|
term = unicodedata.normalize('NFC', term)
|
|
return " ".join(term.lower().split())
|
|
# #endregion _normalize_term
|
|
|
|
|
|
# #region _detect_delimiter [TYPE Function]
|
|
# @BRIEF Detect the delimiter used in a CSV/TSV header line.
|
|
def _detect_delimiter(header_line: str) -> str:
|
|
"""Detect delimiter by counting tabs vs commas in the first line."""
|
|
if not header_line:
|
|
return ","
|
|
tab_count = header_line.count("\t")
|
|
comma_count = header_line.count(",")
|
|
return "\t" if tab_count > comma_count else ","
|
|
# #endregion _detect_delimiter
|
|
# #endregion DictionaryUtils
|