"""Reproduce the September rebuild's aggregate charts from a private catalog export.
Usage: MPLCONFIGDIR=/private/tmp/prismatic-mpl python3 scripts/analyze-rebuild-catalog.py /path/to/strategies.json
The raw feed is never copied into the website or preview.
"""
import argparse, collections, csv, hashlib, json, pathlib, re
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

ROOT = pathlib.Path(__file__).resolve().parents[1]
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('source', type=pathlib.Path)
parser.add_argument('--output', type=pathlib.Path, default=ROOT / 'deliverables/rebuild-preview')
parser.add_argument('--audit', type=pathlib.Path, default=ROOT / 'research/rebuild-data-2026-09-23')
args = parser.parse_args()
OUT = args.output
AUDIT = args.audit
OUT.mkdir(parents=True, exist_ok=True)
(OUT/'charts').mkdir(exist_ok=True)
AUDIT.mkdir(parents=True, exist_ok=True)
source = args.source; raw = source.read_bytes(); data = json.loads(raw); rows=data['strategies']; n=len(rows)
assert n == data['strategy_count']
assert len({r['id'] for r in rows}) == n, 'Duplicate record IDs'
labels={'A':'Trend & directional','B':'Mean reversion & counter-trend','C':'Grid, martingale & averaging','D':'Scalping & short-horizon','E':'Price action & structure','F':'Time, session & seasonality','G':'Volatility-based','H':'Carry, swap & fundamental','I':'Statistical, quantitative & ML','J':'Risk & portfolio overlays','K':'Trade & execution management','L':'Confluence & multi-timeframe',None:'Unclassified'}
counts=collections.Counter(r.get('family') for r in rows)
assert set(counts).issubset(labels)
classified_n=n-counts[None]
families=[{'key':k,'label':labels[k],'count':v,'pct':round(v/classified_n*100,1)} for k,v in counts.most_common() if k is not None]
assert sum(x['count'] for x in families)==classified_n
# A reproducible text-presence check, NOT a judgment that rules are sufficient or correct.
empty={'','null','none','n/a','na','unknown','unspecified','not specified','not stated','not provided','not available','not explicitly stated','not explicitly specified','none specified','...','…','-','[]','{}'}
def present(v):
 if isinstance(v,list): return any(present(x) for x in v)
 if v is None: return False
 s=re.sub(r'\s+',' ',str(v).strip().lower()).rstrip('.')
 if s in empty or not s: return False
 if re.fullmatch(r'(?:not (?:explicitly )?(?:stated|specified|provided|available|mentioned|defined|described)|none(?: specified)?|unspecified)(?: (?:in|by|from) (?:the )?(?:source|text|article|code|description|document))?',s): return False
 return True
checks=[('Source URL recorded',lambda r:bool(re.match(r'^https?://[^/ ]+',r.get('doc_url') or ''))),('Entry rule text',lambda r:present(r.get('entry_long')) or present(r.get('entry_short'))),('Exit rule text',lambda r:present(r.get('exit_rules'))),('Position-sizing text',lambda r:present(r.get('position_size'))),('Stop-loss text',lambda r:present(r.get('stop_loss'))),('Code capture flagged',lambda r:r.get('code_captured') is True)]
coverage=[{'label':label,'count':sum(bool(fn(r)) for r in rows)} for label,fn in checks]
for x in coverage:x['pct']=round(x['count']/n*100,1)
from urllib.parse import urlparse
hosts=collections.Counter(urlparse(r.get('doc_url','')).hostname for r in rows)
groups=collections.Counter(r.get('version_group') or f"id:{r['id']}" for r in rows)
result={'snapshot':data['generated_utc'],'commit':'375327aef33950300bee89377b7cc134d62cab3a','retrieved':'2026-09-23','sha256':hashlib.sha256(raw).hexdigest(),'recordCount':n,'documentsCrawled':data['documents_crawled'],'versionGroupCount':len(groups),'recordsBeyondFirstInGroup':n-len(groups),'unclassified':counts[None],'familyDenominator':classified_n,'familyScope':'Classified records only; records without a family label excluded. Percentages use this subset.','families':families,'coverage':coverage,'sourceHosts':[{'host':k,'count':v} for k,v in hosts.most_common()], 'scope':'Full exported catalog; record counts, not unique validated algorithms. Labels retained as indexed. Includes utilities, overlays and versions.','coverageMeaning':'Non-placeholder structured text or metadata flag; not verified completeness, source accuracy, executable code or performance.'}
(OUT/'data.json').write_text(json.dumps(result,indent=2)+'\n')
(AUDIT/'aggregate-results.json').write_text(json.dumps(result,indent=2)+'\n')
for name,items in [('families',families),('coverage',coverage)]:
 with (OUT/'charts'/f'{name}.csv').open('w') as f:
  w=csv.DictWriter(f,fieldnames=['label','count','pct']);w.writeheader();w.writerows({k:x[k] for k in ['label','count','pct']} for x in items)
 plt.rcParams.update({'font.family':'DejaVu Sans','font.size':11,'axes.labelcolor':'#485366','text.color':'#172638','svg.fonttype':'none'})
 fig,ax=plt.subplots(figsize=(12,8.4 if name=='families' else 6.8),facecolor='#f8f7f2');ax.set_facecolor('#f8f7f2')
 y=list(range(len(items))); values=[x['count'] for x in items]
 colors=['#285d52' for x in items]
 ax.barh(y,values,color=colors,height=.57);ax.set_yticks(y,[x['label'] for x in items]);ax.invert_yaxis();ax.set_xlim(0,(max(values)*1.22 if name=='families' else n*1.2));ax.set_xlabel(f'Classified catalog records · n = {classified_n:,}' if name=='families' else f'Catalog records · full snapshot, n = {n:,}',labelpad=12)
 for i,x in enumerate(items):ax.text(x['count']+max(values)*.02,i,f"{x['count']:,}  /  {x['pct']:.1f}%",va='center',fontsize=10)
 for side in ['top','right','left']:ax.spines[side].set_visible(False)
 ax.spines['bottom'].set_color('#d1d5da');ax.tick_params(axis='y',length=0,pad=10);ax.grid(axis='x',color='#dde0e5',linewidth=.6);ax.set_axisbelow(True)
 title='How the catalog is organized' if name=='families' else 'What the structured records contain'
 fig.text(.035,.96,'PRISMATIC RESEARCH  /  CATALOG NOTES 01',fontsize=10,color='#285d52',weight='bold');fig.text(.035,.91,title,fontsize=22,weight='bold')
 note='Classified records only; unlabeled records excluded. Labels as indexed, not independently validated.' if name=='families' else 'Text fields exclude defined placeholders. Presence does not establish accuracy, sufficient rules or effectiveness.'
 fig.text(.035,.055,note,fontsize=9);fig.text(.035,.025,'Source: Prismatic catalog export · 21 Sep 2026, 15:09 UTC · retrieved 23 Sep · commit 375327a',fontsize=9,color='#536176')
 fig.subplots_adjust(left=.32,right=.94,top=.83,bottom=.17)
 for ext in ['png','svg']:fig.savefig(OUT/'charts'/f'{name}.{ext}',dpi=180,facecolor=fig.get_facecolor())
 plt.close(fig)
print(json.dumps({k:result[k] for k in ['recordCount','versionGroupCount','unclassified','coverage']},indent=2))