busya
ec6421de35
- Replace all occurrences of 'ss-tools' with 'superset-tools' in 104 files - Rename git bundle file ss-tools.bundle → superset-tools.bundle - Update .gitignore pattern accordingly - Preserve variable names (hasSsTools etc.) and code identifiers
306 lines
11 KiB
Python
306 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Regenerate [SEMANTICS ...] tags on MODULE-LEVEL #region headers only.
|
|
Generates MEANINGFUL, content-derived keywords for semantic graph search/discovery.
|
|
Uses word-boundary matching to prevent false positives from substrings.
|
|
"""
|
|
import subprocess, re, os, sys
|
|
|
|
WORKSPACE = "/home/busya/dev/superset-tools"
|
|
|
|
# Domain → tag mapping (path fragments)
|
|
PATH_DOMAINS = {
|
|
'translate': 'translate', 'git': 'git', 'auth': 'auth',
|
|
'dashboard': 'dashboard', 'migration': 'migration',
|
|
'dataset': 'dataset', 'superset': 'superset', 'llm': 'llm',
|
|
'storage': 'storage', 'backup': 'backup', 'health': 'health',
|
|
'notification': 'notification', 'report': 'report',
|
|
'assistant': 'assistant', 'rbac': 'rbac', 'role': 'rbac',
|
|
'permission': 'rbac', 'clean_release': 'clean-release',
|
|
'task': 'task', 'mapping': 'mapping', 'profile': 'profile',
|
|
'scheduler': 'scheduler', 'plugin': 'plugin',
|
|
'encrypt': 'encryption', 'resource': 'resource',
|
|
'provider': 'provider', 'environ': 'environment', 'gitea': 'gitea',
|
|
'connection': 'connection', 'preview': 'preview',
|
|
'clarification': 'clarification', 'finding': 'finding',
|
|
'execution': 'execution', 'session': 'session', 'filter': 'filter',
|
|
'admin': 'admin', 'user': 'user', 'logs': 'logs',
|
|
'policy': 'policy', 'compliance': 'compliance',
|
|
'approval': 'approval', 'publication': 'publication',
|
|
'artifact': 'artifact', 'manifest': 'manifest',
|
|
'candidate': 'candidate', 'audit': 'audit', 'review': 'review',
|
|
'datasets': 'dataset', 'dto': 'dto', 'enums': 'enum',
|
|
'exceptions': 'exception', 'facade': 'facade',
|
|
'clickhouse': 'clickhouse', 'sqlite': 'sqlite',
|
|
'sidebar': 'sidebar', 'activity': 'activity', 'history': 'history',
|
|
'orchestrator': 'orchestration', 'model': 'model',
|
|
}
|
|
|
|
# Operation → patterns (word-boundary matched against region text)
|
|
OP_MAP = {
|
|
'sync': ['sync', 'synchronize', 'syncing'],
|
|
'validate': ['validate', 'validation', 'validator'],
|
|
'transform': ['transform', 'conver', 'parse', 'normalize'],
|
|
'schedule': ['schedule', 'scheduler', 'cron', 'periodic'],
|
|
'upload': ['upload'],
|
|
'download': ['download'],
|
|
'search': ['search', 'find', 'query'],
|
|
}
|
|
|
|
# Framework patterns (checked via word-boundary match)
|
|
PY_FRAMEWORKS = [
|
|
('fastapi', ['FastAPI', 'APIRouter']),
|
|
('sqlalchemy', ['SQLAlchemy', 'declarative_base']),
|
|
('pydantic', ['BaseModel', 'Field']),
|
|
('clickhouse', ['ClickHouse']),
|
|
]
|
|
FRT_FRAMEWORKS = [
|
|
('svelte', ['Svelte', '{#each', '{#if', 'on:click']),
|
|
('sveltekit', ['SvelteKit', '$page', '$app']),
|
|
]
|
|
COMMON_FRAMEWORKS = [
|
|
('tenacity', ['tenacity', 'retry']),
|
|
]
|
|
|
|
# Brief extra tags (word-boundary matched against @BRIEF text)
|
|
BRIEF_EXTRA = {
|
|
'config': 'config', 'template': 'template', 'render': 'render',
|
|
'key': 'encryption', 'jwt': 'jwt', 'oauth': 'oauth',
|
|
'token': 'token', 'webhook': 'webhook',
|
|
'notification': 'notification', 'alert': 'notification',
|
|
'metric': 'metrics', 'logging': 'logging', 'tracing': 'tracing',
|
|
'deploy': 'deploy', 'release': 'release',
|
|
'docker': 'docker', 'batch': 'batch',
|
|
'cli': 'cli', 'scaffold': 'scaffold', 'generator': 'generator',
|
|
'diff': 'diff', 'patch': 'diff',
|
|
'i18n': 'i18n', 'localization': 'i18n', 'locale': 'i18n',
|
|
'navigation': 'navigation', 'sidebar': 'sidebar',
|
|
'drawer': 'drawer', 'chat': 'chat',
|
|
'sort': 'sort', 'pagination': 'pagination',
|
|
'selection': 'selection', 'dry-run': 'dry-run',
|
|
'form': 'form', 'table': 'table', 'modal': 'modal',
|
|
'filter': 'filter', 'search': 'search',
|
|
'history': 'history', 'activity': 'activity',
|
|
'encryption': 'encryption', 'credential': 'credential',
|
|
'archive': 'archive', 'restore': 'restore',
|
|
'schedule': 'schedule', 'sync': 'sync',
|
|
}
|
|
|
|
|
|
def word_in_text(word, text):
|
|
"""Check if word appears as a whole word in text (case-insensitive, word-boundary)."""
|
|
return bool(re.search(r'\b' + re.escape(word) + r'\b', text, re.IGNORECASE))
|
|
|
|
|
|
def any_word_in_text(words, text):
|
|
"""Check if any word from list appears as whole word in text."""
|
|
return any(word_in_text(w, text) for w in words)
|
|
|
|
|
|
def is_frontend(fp):
|
|
return os.path.splitext(fp)[1].lower() in ('.js', '.svelte', '.ts', '.css', '.html')
|
|
|
|
|
|
def is_python(fp):
|
|
return fp.endswith('.py')
|
|
|
|
|
|
def read_region_text(content, line_num):
|
|
lines = content.split('\n')
|
|
result = []
|
|
for i in range(line_num, min(line_num + 80, len(lines))):
|
|
s = lines[i].strip()
|
|
# Stop at region/def closers (use 'in' to handle '# #endregion' style)
|
|
if '#endregion' in s or '[/DEF' in s:
|
|
break
|
|
result.append(lines[i])
|
|
return '\n'.join(result)
|
|
|
|
|
|
def analyze(file_path, region_text):
|
|
is_fe = is_frontend(file_path)
|
|
pl = file_path.lower()
|
|
bl = region_text.lower()
|
|
kw = []
|
|
|
|
# 1. Framework detection (whole-word or known identifier)
|
|
if is_python(file_path):
|
|
for fw, pats in PY_FRAMEWORKS:
|
|
if any_word_in_text(pats, region_text):
|
|
kw.append(fw)
|
|
break
|
|
if is_fe:
|
|
for fw, pats in FRT_FRAMEWORKS:
|
|
if any_word_in_text(pats, region_text):
|
|
kw.append(fw)
|
|
break
|
|
for fw, pats in COMMON_FRAMEWORKS:
|
|
if any_word_in_text(pats, region_text):
|
|
kw.append(fw)
|
|
break
|
|
|
|
# 2. Domain from path
|
|
for key, tag in PATH_DOMAINS.items():
|
|
if key in pl and tag not in kw:
|
|
kw.append(tag)
|
|
|
|
# 3. Role from path
|
|
if '/routes/' in pl or '/api/' in pl:
|
|
kw.append('api')
|
|
if '/services/' in pl:
|
|
kw.append('service')
|
|
if '/plugins/' in pl:
|
|
kw.append('plugin')
|
|
if '/models/' in pl or '/schemas/' in pl:
|
|
kw.append('schema')
|
|
if '/components/' in pl:
|
|
kw.append('ui')
|
|
if '/stores/' in pl:
|
|
kw.append('store')
|
|
if '/pages/' in pl or '+page' in pl:
|
|
kw.append('page')
|
|
if '/__tests__/' in pl or '/tests/' in pl or 'test_' in os.path.basename(file_path):
|
|
kw.append('test')
|
|
if '/repositories/' in pl:
|
|
kw.append('repository')
|
|
if '/orchestrator' in pl:
|
|
kw.append('orchestration')
|
|
if file_path.endswith('__init__.py') and 'package' not in kw:
|
|
kw.append('package')
|
|
|
|
# 4. Operations (word-boundary matched)
|
|
for tag, pats in OP_MAP.items():
|
|
if tag not in kw and any_word_in_text(pats, region_text):
|
|
kw.append(tag)
|
|
|
|
# 5. @BRIEF scanning
|
|
bm = re.search(r'@BRIEF\s+(.*?)(?:\n|$)', region_text)
|
|
if bm:
|
|
brief = bm.group(1).strip()
|
|
brief_lower = brief.lower()
|
|
for key, tag in PATH_DOMAINS.items():
|
|
if word_in_text(key, brief) and tag not in kw:
|
|
kw.append(tag)
|
|
for key, tag in BRIEF_EXTRA.items():
|
|
if word_in_text(key, brief) and tag not in kw:
|
|
kw.append(tag)
|
|
|
|
# 6. Key classes as concepts
|
|
for cls in re.findall(r'class\s+(\w+)', region_text)[:2]:
|
|
clean = re.sub(r'([A-Z])', r' \1', cls).strip().lower()
|
|
cw = [w for w in clean.split() if len(w) > 1]
|
|
ch = '-'.join(cw)
|
|
if ch and ch not in kw and len(ch) > 2:
|
|
kw.append(ch)
|
|
|
|
# 7. Deduplicate, limit to 6
|
|
seen = set()
|
|
result = []
|
|
for k in kw:
|
|
if k not in seen:
|
|
seen.add(k)
|
|
result.append(k)
|
|
return result[:6]
|
|
|
|
|
|
def main():
|
|
r = subprocess.run(['grep', '-rln', r'\[SEMANTICS',
|
|
os.path.join(WORKSPACE, 'backend/src'),
|
|
os.path.join(WORKSPACE, 'frontend/src')],
|
|
capture_output=True, text=True, cwd=WORKSPACE)
|
|
files = sorted(f.strip() for f in r.stdout.split('\n') if f.strip())
|
|
print(f"Found {len(files)} files with [SEMANTICS] tags")
|
|
|
|
stats = {'updated': 0, 'skipped': 0, 'errors': 0, 'examples': []}
|
|
|
|
for fp in files:
|
|
if not os.path.exists(fp):
|
|
stats['errors'] += 1
|
|
continue
|
|
rel = os.path.relpath(fp, WORKSPACE)
|
|
|
|
with open(fp, 'r') as f:
|
|
content = f.read()
|
|
|
|
lines = content.split('\n')
|
|
modified = False
|
|
|
|
for i, line in enumerate(lines):
|
|
s = line.strip()
|
|
type_match = re.search(r'\[TYPE\s+(\w+)\]', s)
|
|
def_match = re.match(r'^\s*(?:#|//|<!--)\s*\[DEF:(\w+):(\w+)\]', s)
|
|
region_match = re.match(r'^\s*(?:#|//|<!--)\s*#region\s+(\S+)', s)
|
|
|
|
if not region_match and not def_match:
|
|
continue
|
|
|
|
# Determine if this is a module-level header
|
|
is_module = False
|
|
if type_match and type_match.group(1) in (
|
|
'Module', 'Page', 'Service', 'Store', 'Package',
|
|
'Component', 'Agent', 'Skill'
|
|
):
|
|
is_module = True
|
|
if def_match:
|
|
is_module = True
|
|
if region_match and not type_match:
|
|
prev_regions = [j for j in range(i) if '#region' in lines[j] or '[DEF:' in lines[j]]
|
|
if not prev_regions:
|
|
is_module = True
|
|
|
|
if not is_module:
|
|
continue
|
|
|
|
sem_match = re.search(r'\[SEMANTICS\s*([^\]]*)\]', s)
|
|
if not sem_match:
|
|
continue
|
|
|
|
old_sem = sem_match.group(1).strip()
|
|
region_text = read_region_text(content, i)
|
|
keywords = analyze(fp, region_text)
|
|
new_sem = ', '.join(keywords) if keywords else old_sem
|
|
|
|
old_set = set(k.strip().lower() for k in old_sem.split(','))
|
|
new_set = set(keywords)
|
|
if old_set == new_set and len(old_set) == len(new_set):
|
|
continue
|
|
|
|
new_line = re.sub(r'\[SEMANTICS\s*([^\]]*)\]', f'[SEMANTICS {new_sem}]', line)
|
|
lines[i] = new_line
|
|
modified = True
|
|
|
|
if len(stats['examples']) < 20:
|
|
stats['examples'].append((rel, old_sem, new_sem))
|
|
|
|
if modified:
|
|
with open(fp, 'w') as f:
|
|
f.write('\n'.join(lines))
|
|
stats['updated'] += 1
|
|
if stats['updated'] % 100 == 0:
|
|
print(f" ... {stats['updated']} files updated")
|
|
else:
|
|
stats['skipped'] += 1
|
|
|
|
print(f"\n{'='*60}")
|
|
print(f"COMPLETE: {stats['updated']} updated, {stats['skipped']} skipped, {stats['errors']} errors")
|
|
print(f"{'='*60}")
|
|
print("\n--- SAMPLE CHANGES (first 20) ---")
|
|
for rel, old, new in stats['examples']:
|
|
print(f"\n FILE: {rel}")
|
|
print(f" OLD: [{old}]")
|
|
print(f" NEW: [{new}]")
|
|
|
|
os.makedirs(os.path.join(WORKSPACE, 'semantics/reports'), exist_ok=True)
|
|
rp = os.path.join(WORKSPACE, 'semantics/reports/semantics_update_report.txt')
|
|
with open(rp, 'w') as f:
|
|
f.write(f"SEMANTICS Tags Update Report\n{'='*60}\n")
|
|
f.write(f"Updated: {stats['updated']}\nSkipped: {stats['skipped']}\nErrors: {stats['errors']}\n\n")
|
|
f.write("SAMPLE CHANGES:\n")
|
|
for rel, old, new in stats['examples']:
|
|
f.write(f"\n {rel}\n OLD: [{old}]\n NEW: [{new}]\n")
|
|
print(f"\nReport: {rp}")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|