Add sample scenario data and reporting updates

mockups/realistic_seed.py

@@ -35,219 +35,7 @@ def reset_sample(conn):
         conn.execute(sql.SQL('DELETE FROM {} WHERE game_id=ANY(%s)').format(sql.Identifier(table)), (gids,))
 
 
-def populate(conn, players_per_game=6000):
-    rng = random.Random(20261103)
-    now = datetime.now(timezone.utc)
-    midnight = now.replace(hour=0, minute=0, second=0, microsecond=0)
-    first_day = midnight - 89 * DAY
-    conn.execute((Path(__file__).parent / 'db' / 'seed.sql').read_text(encoding='utf-8'))
-    games = conn.execute('SELECT * FROM games WHERE sample_key=ANY(%s) ORDER BY id', (list(SAMPLE_KEYS),)).fetchall()
-    for game in games:
-        gid = game['id']
-        # Different genres, not three copies of the same graph.
-        scale, session_scale = {'willow-v1': (1., 1.), 'orbit-v1': (.78, .78), 'dungeon-v1': (1.35, 1.35)}[game['sample_key']]
-        defs = {r['key']: r for r in conn.execute('SELECT * FROM activity_definitions WHERE game_id=%s', (gid,))}
-        products = conn.execute('SELECT * FROM products WHERE game_id=%s ORDER BY id', (gid,)).fetchall()
-        weights = []
-        for day in range(90):
-            date = first_day + day * DAY
-            # Gradual growth, weekend lift, acquisition campaign, and update bump.
-            weight = (.7 + day / 140) * (1.28 if date.weekday() >= 5 else 1)
-            weight *= 1 + .75 * math.exp(-((day-61)/3)**2) + .4 * math.exp(-((day-79)/2)**2)
-            if day == 89:
-                weight *= max(.02, (now-midnight).total_seconds()/86400)
-            weights.append(weight)
-        players = []
-        for index in range(players_per_game):
-            cohort = rng.choices(range(90), weights=weights)[0]
-            date = first_day + cohort * DAY
-            # Global audience with an evening peak, not a uniform hourly trace.
-            hour = rng.gauss(18, 4) % 24 if rng.random() < .7 else rng.uniform(0, 24)
-            if cohort == 89:
-                hour = min(hour, max(0, (now-midnight).total_seconds()/3600-.02)) * rng.uniform(.1, 1)
-            start = date + timedelta(hours=hour)
-            kind = rng.choices(['bounce', 'casual', 'regular', 'whale'], [.55, .32, .127, .003])[0]
-            name = rng.choice(['Pixel','Cloud','Nova','Maple','Moss','Luna','Echo','Aero','Fern','Snow'])
-            name += rng.choice(['Fox','Builder','Runner','Cat','Plays','_']) + str(rng.randrange(10, 9999))
-            players.append((gid, 1000000000+gid*100000+index, name if rng.random()>.008 else None,
-                            start, start, {'seedVersion': 2, 'seedIndex': index, 'simulationGroup': kind}))
-        copy_rows(conn, 'players', ['game_id','roblox_user_id','display_name','first_seen','last_seen','metadata'], players)
-        stored = conn.execute('SELECT * FROM players WHERE game_id=%s ORDER BY id', (gid,)).fetchall()
-        session_rows, traits = [], {}
-        for player in stored:
-            pid, start = player['id'], player['first_seen']
-            kind = player['metadata']['simulationGroup']
-            affinity = rng.uniform(.7, 1.3)
-            payer = rng.random() < {'bounce': .0015, 'casual': .035, 'regular': .18, 'whale': 1}[kind]
-            traits[pid] = {'kind':kind, 'payer':payer, 'device':rng.choices(['phone','desktop','tablet'],[.65,.28,.07])[0],
-                           'spendIntensity':min(12, rng.paretovariate(1.7)) if kind=='whale' else 1}
-            dates = [start]
-            age = (midnight - start.replace(hour=0, minute=0, second=0, microsecond=0)).days
-            for offset in range(1, age+1):
-                if kind == 'bounce':
-                    break
-                if kind == 'casual':
-                    probability = .20 * math.exp(-(offset-1)/1.8)
-                elif kind == 'regular':
-                    probability = .40*math.exp(-(offset-1)/2.7) + .10*math.exp(-(offset-1)/15)
-                else:
-                    probability = .85*math.exp(-(offset-1)/70)
-                date = start.replace(hour=0, minute=0, second=0, microsecond=0) + offset * DAY
-                weekend = 1.12 if date.weekday() >= 5 else .96
-                if rng.random() < min(.96, probability * affinity * scale * weekend):
-                    dates.append(date + timedelta(hours=rng.gauss(18,4)%24))
-            last_end = None
-            for day_index, date in enumerate(dates):
-                count = 1
-                if kind != 'bounce' and rng.random() < (.65 if kind=='whale' else .18):
-                    count += rng.choices([1,2], [.85,.15])[0]
-                previous_end = None
-                for number in range(count):
-                    current = date if previous_end is None else previous_end+timedelta(minutes=rng.uniform(25,180))
-                    if last_end is not None and current < last_end:
-                        current = last_end + timedelta(minutes=20)
-                    if current > now or current.date() != date.date():
-                        continue
-                    median = {'bounce':2.5,'casual':6,'regular':11,'whale':28}[kind]
-                    if day_index:
-                        median *= 1.2
-                    duration = max(15, min(3*3600, rng.lognormvariate(math.log(median*60*session_scale), .85)))
-                    end = current+timedelta(seconds=duration)
-                    # Preserve genuinely open sessions, plus rare lost end reports.
-                    recorded_end = None if end>now or rng.random()<.003 else end
-                    session_rows.append([gid,pid,current,recorded_end,
-                                         rng.choice(['Main Island','Forest Arena','Trading Hub']),
-                                         {'device':traits[pid]['device'], 'source':'synthetic-v2'}])
-                    previous_end = end
-                    last_end = end
-        session_rows.sort(key=lambda row: row[2])
-        ids = [r['id'] for r in conn.execute("SELECT nextval(pg_get_serial_sequence('sessions','id')) id FROM generate_series(1,%s)", (len(session_rows),))]
-        copy_rows(conn, 'sessions', ['id','game_id','player_id','started_at','ended_at','place_name','stats'],
-                  [[sid]+row for sid,row in zip(ids,session_rows)])
-        events, sales, snapshots, runs = [], [], [], []
-        balances, spent, units, playtime, levels, visits = defaultdict(int), defaultdict(int), defaultdict(int), defaultdict(float), defaultdict(lambda:1), Counter()
-        cash_earned, cash_spent = defaultdict(int), defaultdict(int)
-        intervals = []
-
-        def event(name, pid, sid, at, props=None, category='events'):
-            events.append((gid,category,name,pid,sid,at,props or {}))
-
-        def activity(key, pid, sid, start, end, outcome, progress, captured, prompts=0):
-            definition=defs[key]
-            runs.append((gid,definition['id'],pid,sid,start,end,outcome,
-                         sum(e in definition['configuration']['steps'] for e in progress),prompts,captured,progress))
-            # The ordered observations also appear as individually inspectable events.
-            for i, name in enumerate(progress):
-                at=start+(end-start)*(i/max(1,len(progress)-1))
-                event(name,pid,sid,at,captured,'activities')
-
-        for sid, row in zip(ids, session_rows):
-            _,pid,start,end,place,stats=row
-            trait=traits[pid]
-            visits[start.replace(hour=0,minute=0,second=0,microsecond=0)] += 1
-            intervals.append((start,end or min(now,start+timedelta(minutes=10))))
-            event('session_start',pid,sid,start,{'place':place,'device':trait['device']})
-            if end is None:
-                continue
-            seconds=(end-start).total_seconds()
-            playtime[pid] += seconds
-            earned=round(seconds*rng.uniform(1.2,3.5))
-            cash_earned[pid] += earned
-            balances[pid] += earned
-            expense=round(balances[pid]*rng.uniform(0,.25))
-            cash_spent[pid] += expense
-            balances[pid] -= expense
-            before=levels[pid]
-            levels[pid]=1+int((playtime[pid]/240)**.72)
-            if levels[pid]>before:
-                event('level_up',pid,sid,start+(end-start)*.6,{'level':levels[pid],'levelsGained':levels[pid]-before})
-            if rng.random()<.18 and seconds>300:
-                event('daily_reward_claimed',pid,sid,start+timedelta(seconds=50),{'reward':100})
-            buys=[]
-            if trait['payer'] and seconds>90 and rng.random()<(.82 if trait['kind']=='whale' else .35):
-                n=rng.randint(1,max(2,round(2*trait['spendIntensity']))) if trait['kind']=='whale' else 1
-                for j in range(n):
-                    product=rng.choices(products,weights=[.03,.12,.3,.55] if trait['kind']=='whale' else [.52,.27,.16,.05])[0]
-                    quantity=rng.randint(1,4) if trait['kind']=='whale' else 1
-                    # A temporary promotion changes historical sale prices.
-                    price=round(product['current_price_robux']*(.8 if (start-first_day).days in range(60,64) else 1))
-                    at=start+(end-start)*(.2+.5*(j+1)/(n+1))
-                    sales.append((gid,product['id'],pid,sid,f'synthetic-v2-{sid}-{j}',at,price,quantity,{'source':'synthetic_receipt'}))
-                    spent[pid] += price*quantity
-                    units[pid] += quantity
-                    buys.append((product,quantity,price,at))
-            if seconds>60 and (buys or rng.random()<.23):
-                definition=defs['shop']
-                progress=[definition['start_event']]
-                if buys or rng.random()<.85:
-                    progress.append('product_viewed')
-                activity('shop',pid,sid,start+(end-start)*.15,start+(end-start)*.3,'closed',
-                         progress+[definition['end_event']],{'currency_balance':balances[pid]})
-            for key, chance in [('minigame',.6),('quest',.35)]:
-                if seconds<180 or rng.random()>chance:
-                    continue
-                definition=defs[key]
-                maximum=len(definition['configuration']['steps'])
-                count=maximum if rng.random()<(.7 if seconds>600 else .4) else rng.randrange(maximum)
-                progress=[definition['start_event']]+definition['configuration']['steps'][:count]+[definition['end_event']]
-                outcome=('won' if rng.random()<.4 else 'lost') if key=='minigame' and count==maximum else 'completed' if count==maximum else 'abandoned'
-                captured={'score':round(seconds*rng.uniform(.3,2)),'rounds':1} if key=='minigame' else {'quest_id':f'quest_{rng.randrange(12)}','reward':100 if count==maximum else 0}
-                activity(key,pid,sid,start+(end-start)*.35,start+(end-start)*.9,outcome,progress,captured)
-            event('session_end',pid,sid,end,{'playtime':round(seconds),'level':levels[pid]})
-            data={'level':levels[pid],'cash':balances[pid], 'cashEarned':cash_earned[pid],
-                  'cashSpent':cash_spent[pid], 'robux':spent[pid],'purchases':units[pid],
-                  'levelsGained':levels[pid]-1,'quests':int(playtime[pid]/1800),'achievements':int(math.sqrt(playtime[pid]/600))}
-            if game['sample_key']=='orbit-v1':
-                data['roundWins']=int(playtime[pid]/1200)
-            # Realistic partial payloads: a small stable set lacks a field or all snapshots.
-            if pid%47==0:
-                data.pop('cash')
-            if pid%61:
-                snapshots.append((gid,pid,end,data))
-        copy_rows(conn,'events',['game_id','category','name','player_id','session_id','occurred_at','properties'],events)
-        copy_rows(conn,'product_sales',['game_id','product_id','player_id','session_id','receipt_id','purchased_at','unit_price_robux','quantity','properties'],sales)
-        copy_rows(conn,'player_snapshots',['game_id','player_id','captured_at','data'],snapshots)
-        copy_rows(conn,'activity_runs',['game_id','definition_id','player_id','session_id','started_at','ended_at','outcome','step_count','prompt_count','captured_values','stage_progress'],runs)
-        conn.execute('''UPDATE players p SET first_seen=s.first_seen,last_seen=s.last_seen FROM
-          (SELECT player_id,min(started_at) first_seen,max(coalesce(ended_at,started_at)) last_seen
-           FROM sessions WHERE game_id=%s GROUP BY player_id) s WHERE p.game_id=%s AND p.id=s.player_id''',(gid,gid))
-        # CCU and visits come from these same sessions, instead of unrelated sine-wave values.
-        starts=sorted(s for s,e in intervals)
-        ends=sorted(e for s,e in intervals)
-        metrics=[]
-        from bisect import bisect_right
-        for hour in range(90*24):
-            at=first_day+timedelta(hours=hour)
-            if at>now:
-                break
-            ccu=bisect_right(starts,at)-bisect_right(ends,at)
-            metrics.append((gid,'ccu','concurrent_players',at,None if rng.random()<.004 else ccu,{'source':'synthetic_session_intervals'}))
-            if hour%3==0:
-                metrics.append((gid,'performance','server_fps',at,round(max(35,min(60,59-ccu*.12+rng.gauss(0,1.1))),1),{}))
-        metrics.extend((gid,'visits','daily_visits',day,count,{'source':'session_starts'}) for day,count in visits.items())
-        copy_rows(conn,'game_metrics',['game_id','category','name','recorded_at','value','properties'],metrics)
-        root_events=[]
-        for day in range(90):
-            at=first_day+day*DAY+timedelta(hours=17)
-            if at<=now:
-                root_events.append((gid,'server','weather_changed',at,{'weather':rng.choice(['clear','rain','fog'])}))
-                if day%7==4:
-                    root_events.append((gid,'events','global_announcement',at,{'message':'Weekend event'}))
-        copy_rows(conn,'events',['game_id','category','name','occurred_at','properties'],root_events)
-        conn.execute('''INSERT INTO gamepasses(game_id,roblox_gamepass_id,name)
-          SELECT %s,9000000+%s*100+n,(ARRAY['VIP','Double cash','Extra inventory','Early access'])[n]
-          FROM generate_series(1,4) n''',(gid,gid))
-        passes=conn.execute('SELECT * FROM gamepasses WHERE game_id=%s ORDER BY id',(gid,)).fetchall()
-        ownership=[]
-        for player in stored:
-            pid=player['id']
-            for index,gp in enumerate(passes):
-                # Previously owned / gifted passes do not require a receipt in this 90-day window.
-                probability=([.82,.92,.78,.6] if traits[pid]['kind']=='whale' else [.25,.4,.2,.08] if spent[pid] else [.012,.018,.009,.003])[index]
-                if rng.random() < (.04 if index==3 else 0):
-                    continue
-                ownership.append((gid,gp['id'],pid,rng.random()<probability,now))
-        copy_rows(conn,'gamepass_ownership',['game_id','gamepass_id','player_id','is_owned','checked_at'],ownership)
-        conn.execute("UPDATE games SET properties=properties || %s WHERE id=%s",
-                     (Jsonb({'sampleAsOf':now.isoformat(),'sampleVersion':2,'sampleData':'benchmark-informed synthetic; not actual Roblox users'}),gid))
-        print(f"Generated {game['name']}: {len(stored):,} players, {len(session_rows):,} sessions, {len(sales):,} receipts.",flush=True)
+def populate(conn,players_per_game=None):
+    """Generate the current three-genre scenario fixtures; preserve shared COPY/reset helpers."""
+    from scenario_seed import populate as generate_scenarios
+    return generate_scenarios(conn,players_per_game)