busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

add debug for retention

Browse Source
This commit is contained in:
Volobuev Andrey
2025-07-24 10:21:24 +03:00
parent d23eef096a
commit 672be4fd19
5 changed files with 58 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
superset_tool/utils/fileio.py
View File

@@ -283,10 +283,18 @@ def archive_exports(
# [CLEANUP] Удаление устаревших файлов
deleted_count = 0
for file, _ in files_with_dates:
files_to_delete = []
files_to_keep = []
for file, file_date in files_with_dates:
# [DEBUG_ARCHIVE] Check file for deletion
logger.debug(f"[DEBUG_ARCHIVE] Checking file for deletion: {file.name}. Should keep: {file in keep_files}")
if file not in keep_files:
should_keep = file in keep_files
logger.debug(f"[DEBUG_ARCHIVE] Checking file for deletion: {file.name} (date: {file_date}). Should keep: {should_keep}")
if should_keep:
files_to_keep.append(file.name)
else:
files_to_delete.append(file.name)
try:
# [DEBUG_ARCHIVE][FILE_REMOVED_ATTEMPT] Log deletion attempt
logger.info(f"[DEBUG_ARCHIVE][FILE_REMOVED_ATTEMPT] Attempting to delete archive: {file.name}")
@@ -296,6 +304,9 @@ def archive_exports(
except OSError as e:
# [DEBUG_ARCHIVE][FILE_ERROR] Log deletion error
logger.error(f"[DEBUG_ARCHIVE][FILE_ERROR] Error deleting {file.name}: {str(e)}", exc_info=True)
logger.debug(f"[DEBUG_ARCHIVE] Summary - Files to keep: {files_to_keep}")
logger.debug(f"[DEBUG_ARCHIVE] Summary - Files to delete: {files_to_delete}")
logger.info(f"[ARCHIVE_RESULT] Cleanup completed. Deleted {deleted_count} archives.")
@@ -323,30 +334,38 @@ def apply_retention_policy(
weekly_groups = defaultdict(list)
monthly_groups = defaultdict(list)
logger.debug(f"[RETENTION_DEBUG] Processing {len(files_with_dates)} files for retention policy")
for file, file_date in files_with_dates:
daily_groups[file_date].append(file)
weekly_groups[(file_date.isocalendar().year, file_date.isocalendar().week)].append(file)
monthly_groups[(file_date.year, file_date.month)].append(file)
logger.debug(f"[RETENTION_DEBUG] Grouped into {len(daily_groups)} daily groups, {len(weekly_groups)} weekly groups, {len(monthly_groups)} monthly groups")
# [SELECTION] Выбор файлов для сохранения
keep_files = set()
# Daily - последние N дней
sorted_daily = sorted(daily_groups.keys(), reverse=True)[:daily]
logger.debug(f"[RETENTION_DEBUG] Daily groups to keep: {sorted_daily}")
for day in sorted_daily:
keep_files.update(daily_groups[day])
# Weekly - последние N недель
sorted_weekly = sorted(weekly_groups.keys(), reverse=True)[:weekly]
logger.debug(f"[RETENTION_DEBUG] Weekly groups to keep: {sorted_weekly}")
for week in sorted_weekly:
keep_files.update(weekly_groups[week])
# Monthly - последние N месяцев
sorted_monthly = sorted(monthly_groups.keys(), reverse=True)[:monthly]
logger.debug(f"[RETENTION_DEBUG] Monthly groups to keep: {sorted_monthly}")
for month in sorted_monthly:
keep_files.update(monthly_groups[month])
logger.debug(f"[RETENTION] Сохранено файлов: {len(keep_files)}")
logger.debug(f"[RETENTION_DEBUG] Files to keep: {[f.name for f in keep_files]}")
return keep_files
# [CONTRACT] Сохранение и распаковка дашборда