Subversion Repositories planix.SVN

/*

 * PL310 level 2 cache (non-architectural bag on the side)

 *

 * guaranteed to work incorrectly with default settings; must set Sharovr.

 *

 * clean & invalidate (wbinv) is buggy, so we work around erratum 588369

 * by disabling write-back and cache line-fill before, and restoring after.

 */

#include "u.h"

#include "../port/lib.h"

#include "mem.h"

#include "dat.h"

#include "fns.h"

#include "io.h"

#include "../port/error.h"

#include "arm.h"

#define NWAYS(l2p)	((l2p)->auxctl & Assoc16way? 16: 8)

#define L2P		((L2pl310 *)soc.l2cache)

enum {

	L2size		= 1024 * 1024,	/* according to the tegra 2 manual */

	Wayszgran	= 16 * KiB,	/* granularity of way sizes */

};

typedef struct L2pl310 L2pl310;

typedef struct Pl310op Pl310op;

struct Pl310op {

	ulong	pa;

	ulong	_pad;

	ulong	indexway;

	ulong	way;

};

struct L2pl310 {

	ulong	id;

	ulong	type;

	uchar	_pad0[0x100 - 0x8];

	ulong	ctl;

	ulong	auxctl;

	uchar	_pad1[0x730 - 0x108];	/* boring regs */

	ulong	sync;

	uchar	_pad2[0x740 - 0x734];

	ulong	r3p0sync;		/* workaround for r3p0 bug */

	uchar	_pad3[0x770 - 0x744];

	Pl310op	inv;			/* inv.indexway doesn't exist */

	uchar	_pad4[0x7b0 - 0x780];

	Pl310op	clean;

	uchar	_pad5[0x7f0 - 0x7c0];

	Pl310op	cleaninv;

	uchar	_pad6[0xc00 - 0x7d0];

	ulong	filtstart;

	ulong	filtend;

	uchar	_pad6[0xf40 - 0xc08];

	ulong	debug;

	/* ... */

};

enum {

	/* ctl bits */

	L2enable = 1,

	/* auxctl bits */

	Ipref	= 1<<29,		/* prefetch enables */

	Dpref	= 1<<28,

	Mbo	= 1<<25,

	Sharovr	= 1<<22, /* shared attribute override (i.e., work right!) */

	Parity	= 1<<21,

	Waycfgshift= 17,

	Waycfgmask = (1<<3) - 1,

	Assoc16way = 1<<16,

	/*

	 * optim'n to 0 cache lines; must be enabled in a9(?!).

	 * set CpAClwr0line on all cpus 1st.

	 */

	Fullline0= 1<<0,

	/* debug bits */

	Wt	= 1<<1,			/* write-through, not write-back */

	Nolinefill= 1<<0,

	Basecfg = Wt | Nolinefill,

};

static Lock l2lock;

static int disallowed;			/* by user: *l2off= in plan9.ini */

static int l2ison;

static int bg_op_running;

static ulong waysmask;

static Cacheimpl l2cacheimpl;

static void

awaitbgop(void)

{

	while (bg_op_running)

		;

}

static void

getlock(void)

{

	awaitbgop();		/* wait at normal PL first */

	ilock(&l2lock);

	awaitbgop();		/* wait under lock */

}

static void

l2pl310sync(void)

{

	L2P->sync = 0;

	coherence();

}

/* call this first to set sets/ways configuration */

void

l2pl310init(void)

{

	int waysz, nways;

	ulong new;

	L2pl310 *l2p = L2P;

	static int configed;

	if (getconf("*l2off") != nil) {

//		iprint("l2 cache (pl310) disabled\n");

		disallowed = 1;

		return;

	}

	if (l2ison || configed)

		return;

	l2cache = &l2cacheimpl;

	cachedwb();

	/*

	 * default config is:

	 * l2: ext unified, 8 ways 512 sets 32 bytes/line => 128KB

	 * but the tegra 2 manual says there's 1MB available.

	 * ways or way-size may be fixed by hardware; the only way to tell

	 * is to try to change the setting and read it back.

	 */

	l2pl310sync();

	l2cache->inv();

	/* figure out number of ways */

	l2pl310sync();

	nways = NWAYS(l2p);

	if (!(l2p->auxctl & Assoc16way)) {

		l2p->auxctl |= Assoc16way;

		coherence();

		l2pl310sync();

		nways = NWAYS(l2p);

//		iprint("\nl2: was set for 8 ways, asked for 16, got %d\n", nways);

	}

	waysmask = MASK(nways);

	/* figure out way size (and thus number of sets) */

	waysz = L2size / nways;

	new = l2p->auxctl & ~(Waycfgmask << Waycfgshift) |

		(log2(waysz / Wayszgran) + 1) << Waycfgshift;

	l2p->auxctl = new;

	coherence();

	l2pl310sync();

	l2cache->inv();

//	iprint("\nl2: configed %d ways, %d sets (way size %d)\n", nways,

//		waysz / CACHELINESZ, waysz);

	if (l2p->auxctl != new)

		iprint("l2 config %#8.8lux didn't stick; is now %#8.8lux\n",

			new, l2p->auxctl);

	configed++;

}

void

l2pl310info(Memcache *cp)

{

	int pow2;

	ulong waysz;

	L2pl310 *l2p = L2P;

	memset(cp, 0, sizeof *cp);

	if (!l2ison)

		return;

	l2pl310init();

	assert((l2p->id >> 24) == 'A');

	cp->level = 2;

	cp->type = Unified;

	cp->external = Extcache;

	cp->setsways = Cara | Cawa | Cawt | Cawb;

	cp->l1ip = 3<<14;				/* PIPT */

	cp->setsh = cp->waysh = 0;			/* bag on the side */

	cp->linelen = CACHELINESZ;

	cp->log2linelen = log2(CACHELINESZ);

	cp->nways = NWAYS(l2p);

	pow2 = ((l2p->auxctl >> Waycfgshift) & Waycfgmask) - 1;

	if (pow2 < 0)

		pow2 = 0;

	waysz = (1 << pow2) * Wayszgran;

	cp->nsets = waysz / CACHELINESZ;

}

void

l2pl310on(void)

{

	ulong ctl;

	L2pl310 *l2p = L2P;

	if (getconf("*l2off") != nil) {

//		iprint("l2 cache (pl310) disabled\n");

		disallowed = 1;

		return;

	}

	if (l2ison)

		return;

	l2pl310init();

	l2cache->inv();

	/*

	 * drain l1.  can't turn it off (which would make locks not work)

	 * because doing so makes references below to the l2 registers wedge

	 * the system.

	 */

	cacheuwbinv();

	cacheiinv();

	/*

	 * this is only called once, on cpu0 at startup,

	 * so we don't need locks here.

	 * must do all configuration before enabling l2 cache.

	 */

	l2p->filtend = 0;

	coherence();

	l2p->filtstart = 0;		/* no enable bit */

	l2p->debug = 0;			/* write-back, line fills allowed */

	coherence();

	ctl = l2p->auxctl;

	/* don't change number of sets & ways, but reset all else. */

	ctl &= Waycfgmask << Waycfgshift | Assoc16way;

	ctl |= Sharovr;		/* actually work correctly for a change */

	ctl |= Mbo | Ipref | Dpref | Parity | Fullline0;

	l2p->auxctl = ctl;

	coherence();

	l2p->ctl |= L2enable;

	coherence();

	l2ison = 1;

//	iprint("l2 cache (pl310) now on\n");

}

void

l2pl310off(void)

{

	if (!l2ison)

		return;

	l2cache->wbinv();

	getlock();

	L2P->ctl &= ~L2enable;

	coherence();

	l2ison = 0;

	iunlock(&l2lock);

}

static void

applyrange(ulong *reg, void *ava, int len)

{

	uintptr va, endva;

	if (disallowed || !l2ison)

		return;

	if (len < 0)

		panic("l2cache*se called with negative length");

	endva = (uintptr)ava + len;

	for (va = (uintptr)ava & ~(CACHELINESZ-1); va < endva;

	     va += CACHELINESZ)

		*reg = PADDR(va);

	l2pl310sync();

}

void

l2pl310invse(void *va, int bytes)

{

	uintptr start, end;

	L2pl310 *l2p = L2P;

	/*

	 * if start & end addresses are not on cache-line boundaries,

	 * flush first & last cachelines before invalidating.

	 */

	start = (uintptr)va;

	end = start + bytes;

	getlock();

	if (start % CACHELINESZ != 0) {

//		iprint("l2pl310invse: unaligned start %#p from %#p\n", start,

//			getcallerpc(&va));

		applyrange(&l2p->clean.pa, va, 1);

	}

	if (end % CACHELINESZ != 0) {

//		iprint("l2pl310invse: unaligned end %#p from %#p\n", end,

//			getcallerpc(&va));

		applyrange(&l2p->clean.pa, (char *)va + bytes, 1);

	}

	applyrange(&l2p->inv.pa, va, bytes);

	iunlock(&l2lock);

}

void

l2pl310wbse(void *va, int bytes)

{

	getlock();

	applyrange(&L2P->clean.pa, va, bytes);

	iunlock(&l2lock);

}

/*

 * assume that ldrex/strex (thus locks) won't work when Wt in is effect,

 * so don't manipulate locks between setting and clearing Wt.

 */

void

l2pl310wbinvse(void *va, int bytes)

{

	int odb;

	L2pl310 *l2p = L2P;

	if (!l2ison)

		return;

	getlock();

	applyrange(&l2p->clean.pa, va, bytes);	/* paranoia */

	odb = l2p->debug;

	l2p->debug |= Wt | Nolinefill;		/* erratum workaround */

	coherence();

	applyrange(&l2p->cleaninv.pa, va, bytes);

	l2p->debug = odb;

	iunlock(&l2lock);

}

/*

 * we want to wait for completion at normal PL.

 * if waiting is interrupted, interrupt code that calls

 * these ops could deadlock on a uniprocessor, so we only

 * give up l2lock before waiting on multiprocessors.

 * in this port, only cpu 0 gets interrupts other than local timer ones.

 */

void

l2pl310inv(void)

{

	L2pl310 *l2p = L2P;

	if (disallowed)

		return;

	getlock();

	bg_op_running = 1;

	l2p->inv.way = waysmask;

	coherence();

	if (conf.nmach > 1)

		iunlock(&l2lock);

	while (l2p->inv.way & waysmask)

		;

	if (conf.nmach > 1)

		ilock(&l2lock);

	l2pl310sync();

	bg_op_running = 0;

	iunlock(&l2lock);

}

/*

 * maximum time seen is 2542µs, typical is 625µs.

 */

void

l2pl310wb(void)

{

	L2pl310 *l2p = L2P;

	if (disallowed || !l2ison)

		return;

	getlock();

	bg_op_running = 1;

	l2p->clean.way = waysmask;

	coherence();

	if (conf.nmach > 1)

		iunlock(&l2lock);

	while (l2p->clean.way & waysmask)

		;

	if (conf.nmach > 1)

		ilock(&l2lock);

	l2pl310sync();

	bg_op_running = 0;

	iunlock(&l2lock);

}

void

l2pl310wbinv(void)

{

	int odb;

	L2pl310 *l2p = L2P;

	if (disallowed || !l2ison)

		return;

	l2pl310wb();			/* paranoia */

	getlock();

	bg_op_running = 1;

	odb = l2p->debug;

	l2p->debug |= Wt | Nolinefill;	/* erratum workaround */

	coherence();

	l2p->cleaninv.way = waysmask;

	coherence();

	if (conf.nmach > 1)

		iunlock(&l2lock);

	while (l2p->cleaninv.way & waysmask)

		;

	if (conf.nmach > 1)

		ilock(&l2lock);

	l2pl310sync();

	l2p->debug = odb;

	bg_op_running = 0;

	iunlock(&l2lock);

}

static Cacheimpl l2cacheimpl = {

	.info	= l2pl310info,

	.on	= l2pl310on,

	.off	= l2pl310off,

	.inv	= l2pl310inv,

	.wb	= l2pl310wb,

	.wbinv	= l2pl310wbinv,

	.invse	= l2pl310invse,

	.wbse	= l2pl310wbse,

	.wbinvse= l2pl310wbinvse,

};