busya
bcc69f4bbe
Add versioned metric graph authority, owned browser evidence, paginated and all-tab traversal, deterministic sampling policies, and analyst-facing run inspection. Provision the DEV/PREPROD/PROD Superset, Gitea and million-row ClickHouse lab; retain reproducible lifecycle evidence and explicit incomplete-traversal limits.
201 lines
11 KiB
Python
201 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
# #region FullFlow.FinancePage242Probe [C:4] [TYPE Module] [SEMANTICS fixture,readonly,browser,pagination,diagnostic]
|
|
# @BRIEF Reproduce actual exact-chart navigation through page242 with production reader stage observations.
|
|
# @INVARIANT Read-only UI/query observations; no public run, lease, receipt, data mutation or completeness result is manufactured.
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
from hashlib import sha256
|
|
|
|
from playwright.async_api import async_playwright
|
|
|
|
from release import private_environment
|
|
from browser_memory import sample
|
|
from browser_reconstruct import reconstruct
|
|
|
|
sys.path.insert(0,str(Path(__file__).resolve().parents[2]/'backend'))
|
|
from src.services.dashboard_testing.execution.providers.browser_pagination import SupersetPageReader # noqa: E402
|
|
from src.services.dashboard_testing.execution.providers.browser_traversal_inputs import PaginationTraversalInput # noqa: E402
|
|
from src.services.dashboard_testing.execution.providers.browser_pagination_response import page_query_context # noqa: E402
|
|
from browser_page_replace import replace # noqa: E402
|
|
|
|
|
|
# #region FullFlow.FinancePage242Probe.ContextHashes [C:3] [TYPE Function]
|
|
# @POST Semantic field identities are retained as hashes; arbitrary query/filter values and credentials are excluded.
|
|
def context_hashes(value, prefix=''):
|
|
if isinstance(value,dict):
|
|
return {path:digest for key,item in value.items() for path,digest in context_hashes(item,f'{prefix}.{key}').items()}
|
|
if isinstance(value,list) and value:
|
|
return {path:digest for index,item in enumerate(value) for path,digest in context_hashes(item,f'{prefix}[{index}]').items()}
|
|
return {prefix:sha256(json.dumps(value,sort_keys=True).encode()).hexdigest()}
|
|
# #endregion FullFlow.FinancePage242Probe.ContextHashes
|
|
|
|
|
|
# #region FullFlow.FinancePage242Probe.Retain [C:1] [TYPE Function]
|
|
# @POST Only safe bounded observations are retained; no authentication values or arbitrary exception text is printed.
|
|
def retain(output, value):
|
|
with (output/'observations.jsonl').open('a') as stream:
|
|
stream.write(json.dumps(value,ensure_ascii=False)+'\n')
|
|
# #endregion FullFlow.FinancePage242Probe.Retain
|
|
|
|
|
|
# #region FullFlow.FinancePage242Probe.GarbageCollection [C:2] [TYPE Function]
|
|
# @POST Before/after numeric samples distinguish collectible garbage from retained renderer memory without changing dashboard state.
|
|
# @RATIONALE Actual renderer heap/DOM grew rapidly despite constant bounded page payloads; an explicit collection is a causal diagnostic before selecting a runtime policy.
|
|
# @REJECTED Increasing page timeout or assuming a reachable frontend leak from pre-collection heap growth would not establish the cause.
|
|
async def collect_garbage(reader, args, cdp, ordinal):
|
|
before = await sample(cdp)
|
|
started = time.monotonic()
|
|
collected = False
|
|
try:
|
|
await asyncio.wait_for(reader.page.request_gc(),timeout=10)
|
|
collected = True
|
|
except Exception:
|
|
pass
|
|
retain(args.output,{'scope':'diagnostic only','ordinal':ordinal,'garbage_collection':{
|
|
'completed':collected,'elapsed_seconds':round(time.monotonic()-started,3),
|
|
'before':before,'after':await sample(cdp)}})
|
|
# #endregion FullFlow.FinancePage242Probe.GarbageCollection
|
|
|
|
|
|
# #region FullFlow.FinancePage242Probe.Renew [C:2] [TYPE Function]
|
|
# @POST Real document renewal is measured with source-derived diagnostic coordinates; no owned-run frontier is created.
|
|
async def renew_document(reader, args, cdp, ordinal):
|
|
source = {key:reader.exchange[key] for key in ('source_total','dataset_id','context_digest')}
|
|
source.update(chart_id=reader.limits.chart_id,page_size=reader.limits.page_size)
|
|
before = await sample(cdp)
|
|
started = time.monotonic()
|
|
old_hashes = dict(reader.context_hashes)
|
|
try:
|
|
if args.renew_mode == 'new-page':
|
|
await asyncio.wait_for(replace(reader,ordinal+1,source),timeout=120)
|
|
try:
|
|
await cdp.detach()
|
|
except Exception:
|
|
pass
|
|
cdp = await reader.page.context.new_cdp_session(reader.page)
|
|
else:
|
|
await asyncio.wait_for(reader.maintain(ordinal+1,{'source':source}),timeout=120)
|
|
except Exception:
|
|
retain(args.output,{'scope':'diagnostic only','ordinal':ordinal,'renewal_failure':{
|
|
'original_source':source,'fresh_source':{key:reader.exchange.get(key) for key in source} if reader.exchange else None,
|
|
'changed_context_paths':[key for key in sorted(old_hashes.keys() | reader.context_hashes.keys())
|
|
if old_hashes.get(key) != reader.context_hashes.get(key)],
|
|
'stage':reader.stage,'browser_position':reader.position}})
|
|
raise
|
|
retain(args.output,{'scope':'diagnostic only','ordinal':ordinal,'document_renewal':{
|
|
'mode':args.renew_mode,'elapsed_seconds':round(time.monotonic()-started,3),
|
|
'open_page_count':len(reader.page.context.pages),'before':before,'after':await sample(cdp)}})
|
|
return cdp
|
|
# #endregion FullFlow.FinancePage242Probe.Renew
|
|
|
|
|
|
# #region FullFlow.FinancePage242Probe.Walk [C:3] [TYPE Function]
|
|
# @POST Genuine server response/context/offset and ordered key boundaries are observed separately for each UI page.
|
|
async def walk(reader, args, cdp):
|
|
await asyncio.wait_for(reader.prepare(),timeout=120)
|
|
first = 1
|
|
if args.reconstruct_target:
|
|
await asyncio.wait_for(reconstruct(reader,args.reconstruct_target,
|
|
lambda value:retain(args.output,value)),timeout=120)
|
|
first = args.reconstruct_target
|
|
for ordinal in range(first,args.through_page+1):
|
|
started = time.monotonic()
|
|
try:
|
|
observed = await asyncio.wait_for(reader.read_page(ordinal,timeout_seconds=70),timeout=70)
|
|
except Exception as exc:
|
|
code = str(exc) if isinstance(exc,ValueError) and str(exc).startswith('BROWSER_') else 'BROWSER_TRAVERSAL_PAGE_TIMEOUT'
|
|
diagnostic = reader.diagnostic_state(code)
|
|
try:
|
|
diagnostic = await asyncio.wait_for(reader.diagnostic_snapshot(code),timeout=2)
|
|
except Exception:
|
|
pass
|
|
retain(args.output,{'scope':'diagnostic only','diagnostic':diagnostic,
|
|
'elapsed_seconds':round(time.monotonic()-started,3),'memory':await sample(cdp)})
|
|
return False
|
|
retain(args.output,{'scope':'diagnostic only','ordinal':ordinal,'row_count':len(observed['rows']),
|
|
'source_total':observed['source_total'],'row_offset':observed['row_offset'],
|
|
'first_key':observed['ordering_keys'][0] if observed['ordering_keys'] else None,
|
|
'last_key':observed['ordering_keys'][-1] if observed['ordering_keys'] else None,
|
|
'context_digest':observed['context_digest'],'response_sha256':observed['response_sha256'],
|
|
'elapsed_seconds':round(time.monotonic()-started,3),
|
|
'request_byte_length':reader.last_request_byte_length,'response_byte_length':reader.last_response_byte_length})
|
|
if ordinal % 10 == 0 or ordinal == 1:
|
|
retain(args.output,{'scope':'diagnostic only','ordinal':ordinal,'memory':await sample(cdp)})
|
|
if ordinal in args.gc_at:
|
|
await collect_garbage(reader,args,cdp,ordinal)
|
|
if ordinal in args.renew_at and ordinal < args.through_page:
|
|
cdp = await renew_document(reader,args,cdp,ordinal)
|
|
return True
|
|
# #endregion FullFlow.FinancePage242Probe.Walk
|
|
|
|
|
|
# #region FullFlow.FinancePage242Probe.Run [C:3] [TYPE Function]
|
|
# @PRE Original fixture restored and frozen; fresh output/private env and authenticated local DEV user.
|
|
async def run(args):
|
|
private_environment(args.env_file)
|
|
args.output.mkdir(parents=True,exist_ok=False)
|
|
async with async_playwright() as playwright:
|
|
browser = await playwright.chromium.launch(headless=True)
|
|
context = await browser.new_context(viewport={'width':1600,'height':1000})
|
|
page = await context.new_page()
|
|
try:
|
|
await page.goto('http://127.0.0.1:18111/login/',wait_until='domcontentloaded')
|
|
await page.locator('#username').fill('admin')
|
|
await page.locator('#password').fill(os.environ['FIXTURE_PASSWORD'])
|
|
await page.locator('input[type="submit"],button[type="submit"]').click()
|
|
await page.wait_for_url(lambda url:'/login' not in url,timeout=30000)
|
|
await page.goto('http://127.0.0.1:18111/superset/dashboard/2/',wait_until='domcontentloaded')
|
|
await page.locator('#chart-id-6 tbody tr').first.wait_for(state='visible',timeout=90000)
|
|
limits = PaginationTraversalInput(chart_id=6,table_selector='table:has(tbody tr)',
|
|
header_selector='table:has(thead th)',pagination_selector='.pagination',current_page_selector='li.active a',
|
|
next_page_selector='a[role="button"][href="#page-{next_page_index}"]',first_page_selector='a[href="#page-0"]',
|
|
ordering_column='counterparty_id',page_size=200)
|
|
cdp = await page.context.new_cdp_session(page)
|
|
reader = SupersetPageReader(page,limits)
|
|
reader.context_hashes = {}
|
|
# #region FullFlow.FinancePage242Probe.Run.RequestContext [C:2] [TYPE Function]
|
|
# @POST Only exact chart semantic hashes are observed; no request bytes persist.
|
|
def observe_context(request):
|
|
if reader.chart_request(request):
|
|
reader.context_hashes = context_hashes(page_query_context(request.post_data_json))
|
|
# #endregion FullFlow.FinancePage242Probe.Run.RequestContext
|
|
reader.context_observer = observe_context
|
|
page.on('request',observe_context)
|
|
success = await walk(reader,args,cdp)
|
|
return 0 if success else 1
|
|
finally:
|
|
await browser.close()
|
|
# #endregion FullFlow.FinancePage242Probe.Run
|
|
|
|
|
|
# #region FullFlow.FinancePage242Probe.Main [C:2] [TYPE Function]
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--env-file',type=Path,required=True)
|
|
parser.add_argument('--output',type=Path,required=True)
|
|
parser.add_argument('--through-page',type=int,default=242)
|
|
parser.add_argument('--gc-at',type=int,nargs='*',default=[])
|
|
parser.add_argument('--reconstruct-target',type=int,default=0)
|
|
parser.add_argument('--renew-at',type=int,nargs='*',default=[])
|
|
parser.add_argument('--renew-mode',choices=['document','new-page'],default='document')
|
|
args = parser.parse_args()
|
|
if not 1 <= args.through_page <= 625:
|
|
raise SystemExit('Bounded diagnostic page required')
|
|
if any(not 1 <= ordinal <= args.through_page for ordinal in args.gc_at):
|
|
raise SystemExit('Collection page must be within diagnostic scope')
|
|
if args.reconstruct_target and not 1 <= args.reconstruct_target <= args.through_page:
|
|
raise SystemExit('Reconstruction target must be within diagnostic scope')
|
|
if any(not 1 <= ordinal < args.through_page for ordinal in args.renew_at):
|
|
raise SystemExit('Renewal page must precede diagnostic end')
|
|
return asyncio.run(run(args))
|
|
# #endregion FullFlow.FinancePage242Probe.Main
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|
|
# #endregion FullFlow.FinancePage242Probe
|