how do i write a subquery which references the original table for an aggregation?

I think you can use window functions – but you need a row frame so that only prior games are taken into account:

select g.id, 
    avg(ta.stats_a) over(
        partition by tsa.team_id 
        order by g.date rows between unbounded preceding and 1 preceding
    ) team_a_avg_stat_a,
    avg(ta.stats_b) over(
        partition by tsa.team_id 
        order by g.date rows between unbounded preceding and 1 preceding
    ) team_a_avg_stat_b,
    avg(tb.stats_a) over(
        partition by tsb.team_id 
        order by g.date rows between unbounded preceding and 1 preceding
    ) team_b_avg_stat_a,
    avg(ta.stats_b) over(
        partition by tsb.team_id 
        order by g.date rows between unbounded preceding and 1 preceding
    ) team_b_avg_stat_b
from game g
inner join teamstats tsa 
    on  tsa.game_id = g.game_id
    and tsa.team_id = g.team_a_id
inner join teamstats tsb
    on  tsb.game_id = g.game_id
    and tsb.team_id = g.team_b_id