Subversion Repositories planix.SVN

/*

 * Atheros 71xx ethernets for rb450g.

 *

 * all 5 PHYs are accessible only through first ether's register space.

 *

 * TODO:

 *	promiscuous mode.

 *	make ether1 work: probably needs mii/phy initialisation,

 *	maybe needs 8316 switch code too (which requires mdio, phy, etc. glop).

 * to maybe do some day:

 *	dig mac addresses out & config phy/mii via spi or other grot and swill

 *	(instead of editing rb config file).

 */

#include	"u.h"

#include	"../port/lib.h"

#include	"mem.h"

#include	"dat.h"

#include	"fns.h"

#include	"io.h"

#include	"../port/error.h"

#include	"../port/netif.h"

#include	"etherif.h"

#include	"ethermii.h"

#include	<pool.h>

enum {

	Ntd	= 64,

	Nrd	= 256,

	Nrb	= 1024,

	Bufalign= 4,

	Rbsz	= ETHERMAXTU + 4,	/* 4 for CRC */

};

extern uchar arge0mac[Eaddrlen];	/* see rb config file */

extern uchar arge1mac[Eaddrlen];

typedef struct Arge Arge;

typedef struct Ctlr Ctlr;

typedef struct Desc Desc;

typedef struct Etherif Etherif;

/*

 *  device registers

 */

struct Arge {

	ulong	cfg1;

	ulong	cfg2;

	ulong	ifg;

	ulong	hduplex;

	ulong	maxframelen;

	uchar	_pad0[0x20 - 0x14];

	ulong	miicfg;

	ulong	miicmd;

	ulong	miiaddr;

	ulong	miictl;

	ulong	miists;

	ulong	miiindic;

	ulong	ifctl;

	ulong	_pad1;

	ulong	staaddr1;

	ulong	staaddr2;

	ulong	fifocfg[3];

	ulong	fifotxthresh;

	ulong	fiforxfiltmatch;

	ulong	fiforxfiltmask;

	ulong	fiforam[7];

	uchar	_pad2[0x180 - 0x7c];

	/* dma */

	ulong	txctl;

	ulong	txdesc;

	ulong	txsts;

	ulong	rxctl;

	ulong	rxdesc;

	ulong	rxsts;

	ulong	dmaintr;

	ulong	dmaintrsts;

};

enum {

	Cfg1softrst		= 1 << 31,

	Cfg1simulrst		= 1 << 30,

	Cfg1macrxblkrst		= 1 << 19,

	Cfg1mactxblkrst		= 1 << 18,

	Cfg1rxfuncrst		= 1 << 17,

	Cfg1txfuncrst		= 1 << 16,

	Cfg1loopback		= 1 <<  8,

	Cfg1rxflowctl		= 1 <<  5,

	Cfg1txflowctl		= 1 <<  4,

	Cfg1syncrx		= 1 <<  3,

	Cfg1rxen		= 1 <<  2,

	Cfg1synctx		= 1 <<  1,

	Cfg1txen		= 1 <<  0,

	Cfg2preamblelenmask	= 0xf,

	Cfg2preamblelenshift	= 12,

	Cfg2ifmode1000		= 2 << 8,

	Cfg2ifmode10_100	= 1 << 8,

	Cfg2ifmodeshift		= 8,

	Cfg2ifmodemask		= 3,

	Cfg2hugeframe		= 1 << 5,

	Cfg2lenfield		= 1 << 4,

	Cfg2enpadcrc		= 1 << 2,

	Cfg2encrc		= 1 << 1,

	Cfg2fdx			= 1 << 0,

	Miicfgrst		= 1 << 31,

	Miicfgscanautoinc	= 1 <<  5,

	Miicfgpreamblesup	= 1 <<  4,

	Miicfgclkselmask	= 0x7,

	Miicfgclkdiv4		= 0,

	Miicfgclkdiv6		= 2,

	Miicfgclkdiv8		= 3,

	Miicfgclkdiv10		= 4,

	Miicfgclkdiv14		= 5,

	Miicfgclkdiv20		= 6,

	Miicfgclkdiv28		= 7,

	Miicmdscancycle		= 1 << 1,

	Miicmdread		= 1,

	Miicmdwrite		= 0,

	Miiphyaddrshift		= 8,

	Miiphyaddrmask		= 0xff,

	Miiregmask		= 0x1f,

	Miictlmask		= 0xffff,

	Miistsmask		= 0xffff,

	Miiindicinvalid		= 1 << 2,

	Miiindicscanning	= 1 << 1,

	Miiindicbusy		= 1 << 0,

	Ifctlspeed		= 1 << 16,

	Fifocfg0txfabric	= 1 << 4,

	Fifocfg0txsys		= 1 << 3,

	Fifocfg0rxfabric	= 1 << 2,

	Fifocfg0rxsys		= 1 << 1,

	Fifocfg0watermark	= 1 << 0,

	Fifocfg0all		= MASK(5),

	Fifocfg0enshift		= 8,

	/*

	 * these flags applicable both to filter mask and to filter match.

	 * `Ff' is for `fifo filter'.

	 */

	Ffunicast		= 1 << 17,

	Fftruncframe		= 1 << 16,

	Ffvlantag		= 1 << 15,

	Ffunsupopcode		= 1 << 14,

	Ffpauseframe		= 1 << 13,

	Ffctlframe		= 1 << 12,

	Fflongevent		= 1 << 11,

	Ffdribblenibble		= 1 << 10,

	Ffbcast			= 1 <<  9,

	Ffmcast			= 1 <<  8,

	Ffok			= 1 <<  7,

	Ffoorange		= 1 <<  6,

	Fflenmsmtch		= 1 <<  5,

	Ffcrcerr		= 1 <<  4,

	Ffcodeerr		= 1 <<  3,

	Fffalsecarrier		= 1 <<  2,

	Ffrxdvevent		= 1 <<  1,

	Ffdropevent		= 1 <<  0,

	/*

	 * exclude unicast and truncated frames from matching.

	 */

	Ffmatchdflt = Ffvlantag | Ffunsupopcode | Ffpauseframe | Ffctlframe |

		Fflongevent | Ffdribblenibble | Ffbcast | Ffmcast | Ffok |

		Ffoorange | Fflenmsmtch | Ffcrcerr | Ffcodeerr |

		Fffalsecarrier | Ffrxdvevent | Ffdropevent,

	/* `Frm' is for `fifo receive mask'. */

	Frmbytemode		= 1 << 19,

	Frmnoshortframe		= 1 << 18,

	Frmbit17		= 1 << 17,

	Frmbit16		= 1 << 16,

	Frmtruncframe		= 1 << 15,

	Frmlongevent		= 1 << 14,

	Frmvlantag		= 1 << 13,

	Frmunsupopcode		= 1 << 12,

	Frmpauseframe		= 1 << 11,

	Frmctlframe		= 1 << 10,

	Frmdribblenibble	= 1 <<  9,

	Frmbcast		= 1 <<  8,

	Frmmcast		= 1 <<  7,

	Frmok			= 1 <<  6,

	Frmoorange		= 1 <<  5,

	Frmlenmsmtch		= 1 <<  4,

	Frmcodeerr		= 1 <<  3,

	Frmfalsecarrier		= 1 <<  2,

	Frmrxdvevent		= 1 <<  1,

	Frmdropevent		= 1 <<  0,

	/*

	 *  len. mismatch, unsupp. opcode and short frame bits excluded

	 */

	Ffmaskdflt = Frmnoshortframe | Frmbit17 | Frmbit16 | Frmtruncframe |

		Frmlongevent | Frmvlantag | Frmpauseframe | Frmctlframe |

		Frmdribblenibble | Frmbcast | Frmmcast | Frmok | Frmoorange |

		Frmcodeerr | Frmfalsecarrier | Frmrxdvevent | Frmdropevent,

	Dmatxctlen	= 1 << 0,

	/* dma tx status */

	Txpcountmask	= 0xff,

	Txpcountshift	= 16,

	Txbuserr	= 1 << 3,

	Txunderrun	= 1 << 1,

	Txpktsent	= 1 << 0,

	Dmarxctlen	= 1 << 0,

	/* dma rx status */

	Rxpcountmask	= 0xff,

	Rxpcountshift	= 16,

	Rxbuserr	= 1 << 3,

	Rxovflo		= 1 << 2,

	Rxpktrcvd	= 1 << 0,

	/* dmaintr & dmaintrsts bits */

	Dmarxbuserr	= 1 << 7,

	Dmarxovflo	= 1 << 6,

	Dmarxpktrcvd	= 1 << 4,

	Dmatxbuserr	= 1 << 3,

	Dmatxunderrun	= 1 << 1,

	Dmatxpktsent	= 1 << 0,

	/* we don't really need most tx interrupts */

	Dmaall		= Dmarxbuserr | Dmarxovflo | Dmarxpktrcvd | Dmatxbuserr,

	Spictlremapdisable	= 1 << 6,

	Spictlclkdividermask	= MASK(6),

	Spiioctlcs2		= 1 << 18,

	Spiioctlcs1		= 1 << 17,

	Spiioctlcs0		= 1 << 16,

	Spiioctlcsmask		= 7 << 16,

	Spiioctlclk		= 1 << 8,

	Spiioctldo		= 1,

};

struct Spi {			/* at 0x1f000000 */

	ulong	fs;

	ulong	ctl;

	ulong	ioctl;

	ulong	rds;

};

/* hw descriptors of buffer rings (rx and tx), need to be uncached */

struct Desc {

	ulong	addr;		/* of packet buffer */

	ulong	ctl;

	Desc	*next;

	ulong	_pad;

};

enum {

	Descempty	= 1 << 31,

	Descmore	= 1 << 24,

	Descszmask	= MASK(12),

};

#define DMASIZE(len)	((len) & Descszmask)

struct Ctlr {

	Arge	*regs;

	Ether*	edev;			/* backward pointer */

	Lock;				/* attach */

	int	init;

	int	attached;

	Mii*	mii;

	Rendez	lrendez;

	int	lim;

	int	link;

	int	phymask;

	/* receiver */

	Rendez	rrendez;

	uint	rintr;			/* count */

	int	pktstoread;		/* flag */

	int	discard;

	/* rx descriptors */

	Desc*	rdba;			/* base address */

	Block**	rd;

	uint	rdh;			/* head */

	uint	rdt;			/* tail */

	uint	nrdfree;		/* rd's awaiting pkts (sort of) */

	/* transmitter */

	Rendez	trendez;

	uint	tintr;			/* count */

	int	pktstosend;		/* flag */

	int	ntq;

	/* tx descriptors */

	Desc*	tdba;			/* base address */

	Block**	td;

	uint	tdh;			/* head */

	uint	tdt;			/* tail */

};

struct Etherif {

	uintptr	regs;

	int	irq;

	uchar	*mac;

	int	phymask;

};

static Etherif etherifs[] = {

	{ 0x1a000000, ILenet0, arge0mac, 1<<4 },

	{ 0x19000000, ILenet1, arge1mac, MASK(4) },

};

static Ether *etherxx[MaxEther];

static Lock athrblock;		/* free receive Blocks */

static Block* athrbpool;	/* receive Blocks for all ath controllers */

static void	athrbfree(Block* bp);

/*

 * ar8316 ether switch

 */

enum {

	Swrgmii	= 0,

	Swgmii	= 1,

	Swphy4cpu = 0, /* flag: port 4 connected to CPU (not internal switch) */

};

typedef struct Switch Switch;

struct Switch {

	int	page;

	int	scdev;

};

enum {

	/* atheros-specific mii registers */

	Miiathdbgaddr	= 0x1d,

	Miiathdbgdata	= 0x1e,

	Swregmask	= 0,

		Swmaskrevmask	= 0x00ff,

		Swmaskvermask	= 0xff00,

		Swmaskvershift	= 8,

		Swmasksoftreset	= 1 << 31,

	Swregmode	= 8,

		Swdir615uboot	= 0x8d1003e0,

		/* from ubiquiti rspro */

		Swrgmiiport4iso	= 0x81461bea,

		Swrgmiiport4sw	= 0x01261be2,

		/* avm fritz!box 7390 */

		Swgmiiavm	= 0x010e5b71,

		Swmac0gmiien	= 1 <<  0,

		Swmac0rgmiien	= 1 <<  1,

		Swphy4gmiien	= 1 <<  2,

		Swphy4rgmiien	= 1 <<  3,

		Swmac0macmode	= 1 <<  4,

		Swrgmiirxclkdelayen= 1 <<  6,

		Swrgmiitxclkdelayen= 1 <<  7,

		Swmac5macmode	= 1 << 14,

		Swmac5phymode	= 1 << 15,

		Swtxdelays0	= 1 << 21,

		Swtxdelays1	= 1 << 22,

		Swrxdelays0	= 1 << 23,

		Swledopenen	= 1 << 24,

		Swspien		= 1 << 25,

		Swrxdelays1	= 1 << 26,

		Swpoweronsel	= 1 << 31,

	Swregfloodmask	= 0x2c,

		Swfloodmaskbcast2cpu= 1 << 26,

	Swregglobal	= 0x30,

		Swglobalmtumask	= 0x7fff,

};

#ifdef NOTYET

void *

devicegetparent(int)

{

	static int glop;

	return &glop;

}

static void

arswsplitsetpage(int dev, ulong addr, ushort *phy, ushort *reg)

{

	static Switch ar8316;

	Switch *sc = &ar8316;

	ushort page;

	page = ((addr) >> 9) & 0xffff;

	*phy = (((addr) >> 6) & 0x7) | 0x10;

	*reg = ((addr) >> 1) & 0x1f;

	MDIOWRREG(devicegetparent(dev), 0x18, 0, page);

	sc->page = page;

}

/*

 * Read half a register.  Some of the registers define control bits, and

 * the sequence of half-word accesses matters.  The register addresses

 * are word-even (mod 4).

 */

static int

arswrdreg16(int dev, int addr)

{

	ushort phy, reg;

	arswsplitsetpage(dev, addr, &phy, &reg);

	return MDIORDREG(devicegetparent(dev), phy, reg);

}

void

arswwritedbg(int dev, int phy, ushort dbgaddr, ushort dbgdata)

{

	MDIOWRREG(devicegetparent(dev), phy, Miiathdbgaddr, dbgaddr);

	MDIOWRREG(devicegetparent(dev), phy, Miiathdbgdata, dbgdata);

}

/*

 * Write half a register

 */

static inline int

arswwrreg16(int dev, int addr, int data)

{

	ushort phy, reg;

	arswsplitsetpage(dev, addr, &phy, &reg);

	return MDIOWRREG(devicegetparent(dev), phy, reg, data);

}

/* arsw??reglsb routines operate on lower 16 bits; *msb on upper ones */

int

arswrdreg(int dev, int addr)

{

	return arswrdreglsb(dev, addr) | arswrdregmsb(dev, addr);

}

int

arswwrreg(int dev, int addr, int value)

{

	arswwrreglsb(dev, addr, value);		/* XXX check this write too? */

	return arswwrregmsb(dev, addr, value);

}

int

arswmodifyreg(int dev, int addr, int mask, int set)

{

	return arswwrreg(dev, addr, (arswrdreg(dev, addr) & ~mask) | set);

}

/*

 * initialise the switch

 */

static int

ar8316init(Switch *sc)

{

	if (Swrgmii && Swphy4cpu) {

		arswwrreg(sc->scdev, Swregmode, Swrgmiiport4iso);

		iprint("ar8316: MAC port == RGMII, port 4 = dedicated PHY\n");

	} else if (Swrgmii) {

		arswwrreg(sc->scdev, Swregmode, Swrgmiiport4sw);

		iprint("ar8316: MAC port == RGMII, port 4 = switch port\n");

	} else if (Swgmii) {

		arswwrreg(sc->scdev, Swregmode, Swgmiiavm);

		iprint("ar8316: MAC port == GMII\n");

	} else {

		iprint("ar8316: unknown switch PHY config\n");

		return -1;

	}

	delay(1);			/* wait for things to settle */

	if (Swrgmii && Swphy4cpu) {

		iprint("ar8316: port 4 RGMII hack\n");

		/* work around for phy4 rgmii mode */

		arswwritedbg(sc->scdev, 4, 0x12, 0x480c);

		arswwritedbg(sc->scdev, 4, 0x0, 0x824e);	/* rx delay */

		arswwritedbg(sc->scdev, 4, 0x5, 0x3d47);	/* tx delay */

		delay(1);		/* again to let things settle */

	}

	arswwrreg(sc->scdev, 0x38, 0xc000050e);	/* mystery */

	/*

	 * Flood address table misses to all ports, and enable forwarding of

	 * broadcasts to the cpu port.

	 */

	arswwrreg(sc->scdev, Swregfloodmask, Swfloodmaskbcast2cpu | 0x003f003f);

	arswmodifyreg(sc->scdev, Swregglobal, Swglobalmtumask, ETHERMAXTU+8+2);

	return 0;

}

#endif			/* NOTYET */

static long

ifstat(Ether* edev, void* a, long n, ulong offset)

{

	int l, i, r;

	char *p;

	Ctlr *ctlr;

	ctlr = edev->ctlr;

	p = malloc(READSTR);

	if(p == nil)

		error(Enomem);

	l = 0;

	l += snprint(p+l, READSTR-l, "tintr: %ud\n", ctlr->tintr);

	l += snprint(p+l, READSTR-l, "rintr: %ud\n", ctlr->rintr);

	l += snprint(p+l, READSTR-l, "discarded: %ud\n", ctlr->discard);

	if(ctlr->mii != nil && ctlr->mii->curphy != nil){

		l += snprint(p+l, READSTR-l, "phy:   ");

		for(i = 0; i < NMiiPhyr; i++){

			if(i && ((i & 0x07) == 0))

				l += snprint(p+l, READSTR-l, "\n       ");

			r = miimir(ctlr->mii, i);

			l += snprint(p+l, READSTR-l, " %4.4uX", r);

		}

		snprint(p+l, READSTR-l, "\n");

	}

	n = readstr(offset, a, n, p);

	free(p);

	return n;

}

static void

etherrtrace(Netfile* f, Etherpkt* pkt, int len)

{

	int i, n;

	Block *bp;

	if(qwindow(f->in) <= 0)

		return;

	if(len > 58)

		n = 58;

	else

		n = len;

	bp = iallocb(64);

	if(bp == nil)

		return;

	memmove(bp->wp, pkt->d, n);

	i = TK2MS(MACHP(0)->ticks);

	bp->wp[58] = len>>8;

	bp->wp[59] = len;

	bp->wp[60] = i>>24;

	bp->wp[61] = i>>16;

	bp->wp[62] = i>>8;

	bp->wp[63] = i;

	bp->wp += 64;

	qpass(f->in, bp);

}

Block*

etheriq(Ether* ether, Block* bp, int fromwire)

{

	Etherpkt *pkt;

	ushort type;

	int len, multi, tome, fromme;

	Netfile **ep, *f, **fp, *fx;

	Block *xbp;

	Ctlr *ctlr;

	ether->inpackets++;

	ctlr = ether->ctlr;

	pkt = (Etherpkt*)bp->rp;

	len = BLEN(bp);

	type = (pkt->type[0]<<8)|pkt->type[1];

	fx = 0;

	ep = &ether->f[Ntypes];

	multi = pkt->d[0] & 1;

	/* check for valid multicast addresses */

	if(multi && memcmp(pkt->d, ether->bcast, sizeof(pkt->d)) != 0 &&

	    ether->prom == 0)

		if(!activemulti(ether, pkt->d, sizeof(pkt->d))){

			if(fromwire){

				ctlr->discard++;

				freeb(bp);

				bp = 0;

			}

			return bp;

		}

	/* is it for me? */

	tome   = memcmp(pkt->d, ether->ea, sizeof(pkt->d)) == 0;

	fromme = memcmp(pkt->s, ether->ea, sizeof(pkt->s)) == 0;

	/*

	 * Multiplex the packet to all the connections which want it.

	 * If the packet is not to be used subsequently (fromwire != 0),

	 * attempt to simply pass it into one of the connections, thereby

	 * saving a copy of the data (usual case hopefully).

	 */

	for(fp = ether->f; fp < ep; fp++)

		if((f = *fp) != nil && (f->type == type || f->type < 0))

		if(tome || multi || f->prom)

			/* Don't want to hear bridged packets */

			if(f->bridge && !fromwire && !fromme)

				continue;

			else if(f->headersonly)

				etherrtrace(f, pkt, len);

			else if(fromwire && fx == 0)

				fx = f;

			else if(xbp = iallocb(len)){

				memmove(xbp->wp, pkt, len);

				xbp->wp += len;

				if(qpass(f->in, xbp) < 0){

					iprint("soverflow for f->in\n");

					ether->soverflows++;

				}

			}else{

				iprint("soverflow iallocb\n");

				ether->soverflows++;

			}

	if(fx){

		if(qpass(fx->in, bp) < 0){

			iprint("soverflow for fx->in\n");

			ether->soverflows++;

		}

		return 0;

	}

	if(fromwire){

		ctlr->discard++;

		freeb(bp);

		return 0;

	}

	return bp;

}

static void

athhwreset(Ether *ether)

{

	Ctlr *ctlr;

	Arge *arge;

	ctlr = ether->ctlr;

	if (ctlr == nil)

		return;

	arge = ctlr->regs;

	if (arge == nil)

		return;

	arge->dmaintr = 0;

	arge->rxctl = 0;

	arge->txctl = 0;

	coherence();

	/*

	 * give tx & rx time to stop, otherwise clearing desc registers

	 * too early will cause random memory corruption.

	 */

	delay(1);

	arge->rxdesc = 0;

	arge->txdesc = 0;

	coherence();

	/* clear all interrupts */

	while (arge->rxsts & Rxpktrcvd)

		arge->rxsts = Rxpktrcvd;

	while (arge->txsts & Txpktsent)

		arge->txsts = Txpktsent;

	/* and errors */

	arge->rxsts = Rxbuserr | Rxovflo;

	arge->txsts = Txbuserr | Txunderrun;

}

static void

txreclaim(Ctlr *ctlr)

{

	uint tdh;

	Arge *arge;

	Block *bp;

	arge = ctlr->regs;

	tdh = ctlr->tdh;

	while (tdh != ctlr->tdt && ctlr->tdba[tdh].ctl & Descempty){

		arge->txsts = Txpktsent;

		bp = ctlr->td[tdh];

		ctlr->td[tdh] = nil;

		if (bp)

			freeb(bp);

		ctlr->tdba[tdh].addr = 0;

		ctlr->ntq--;

		tdh = NEXT(tdh, Ntd);

	}

	ctlr->tdh = tdh;

}

static Block*

athrballoc(void)

{

	Block *bp;

	ilock(&athrblock);

	if((bp = athrbpool) != nil){

		athrbpool = bp->next;

		bp->next = nil;

		_xinc(&bp->ref);	/* prevent bp from being freed */

	}

	iunlock(&athrblock);

	return bp;

}

static void

athrbfree(Block* bp)

{

	bp->wp = bp->rp = bp->lim - ROUND(Rbsz, BLOCKALIGN);

	bp->flag &= ~(Bipck | Budpck | Btcpck | Bpktck);

	ilock(&athrblock);

	bp->next = athrbpool;

	athrbpool = bp;

	iunlock(&athrblock);

}

static void

rxnewbuf(Ctlr *ctlr, int i)

{

	Block *bp;

	Desc *rd;

	if (ctlr->rd[i] != nil)

		return;

	ctlr->rd[i] = bp = athrballoc();

	if(bp == nil)

		panic("#l%d: can't allocate receive buffer",

			ctlr->edev->ctlrno);

	dcflush(bp->rp, Rbsz);		/* writeback & invalidate */

	rd = &ctlr->rdba[i];

	rd->addr = PADDR(bp->rp);

	rd->ctl = Descempty | DMASIZE(Rbsz);

	ctlr->nrdfree++;

}

static void

rxreclaim(Ctlr *ctlr)

{

	uint rdt;

	rdt = ctlr->rdt;

	while (rdt != ctlr->rdh && !(ctlr->rdba[rdt].ctl & Descempty)){

		rxnewbuf(ctlr, rdt);

		rdt = NEXT(rdt, Nrd);

	}

	ctlr->rdt = rdt;

}

static void

etherintr(void *arg)

{

	int sts;

	Arge *arge;

	Ctlr *ctlr;

	Ether *ether;

	ether = arg;

	ctlr = ether->ctlr;

	arge = ctlr->regs;

	ilock(ctlr);

	sts = arge->dmaintrsts;

	if (sts & Dmarxpktrcvd) {

		arge->dmaintr &= ~Dmarxpktrcvd;

		ctlr->pktstoread = 1;

		wakeup(&ctlr->rrendez);

		ctlr->rintr++;

		sts &= ~Dmarxpktrcvd;

	}

	if (sts & (Dmatxpktsent | Dmatxunderrun)) {

		arge->dmaintr &= ~(Dmatxpktsent | Dmatxunderrun);

		ctlr->pktstosend = 1;

		wakeup(&ctlr->trendez);

		ctlr->tintr++;

		sts &= ~(Dmatxpktsent | Dmatxunderrun);

	}

	iunlock(ctlr);

	if (sts)

		iprint("#l%d: sts %#ux\n", ether->ctlrno, sts);

}

static int

pktstoread(void* v)

{

	Ctlr *ctlr = v;

	return ctlr->pktstoread || !(ctlr->rdba[ctlr->rdh].ctl & Descempty);

}

static void

rproc(void* arg)

{

	uint rdh, sz;

	Arge *arge;

	Block *bp;

	Ctlr *ctlr;

	Desc *rd;

	Ether *edev;

	edev = arg;

	ctlr = edev->ctlr;

	arge = ctlr->regs;

	for(;;){

		/* wait for next interrupt */

		ilock(ctlr);

		arge->dmaintr |= Dmarxpktrcvd;

		iunlock(ctlr);

		sleep(&ctlr->rrendez, pktstoread, ctlr);

		ctlr->pktstoread = 0;

		rxreclaim(ctlr);

		rdh = ctlr->rdh;

		for (rd = &ctlr->rdba[rdh]; !(rd->ctl & Descempty);

		     rd = &ctlr->rdba[rdh]){

			bp = ctlr->rd[rdh];

			assert(bp != nil);

			ctlr->rd[rdh] = nil;

			/* omit final 4 bytes (crc), pass pkt upstream */

			sz = DMASIZE(rd->ctl) - 4;

			assert(sz > 0 && sz <= Rbsz);

			bp->wp = bp->rp + sz;

			bp = etheriq(edev, bp, 1);

			assert(bp == nil);		/* Block was consumed */

			arge->rxsts = Rxpktrcvd;

			ctlr->nrdfree--;

			rdh = NEXT(rdh, Nrd);

			if(ctlr->nrdfree < Nrd/2) {

				/* rxreclaim reads ctlr->rdh */

				ctlr->rdh = rdh;

				rxreclaim(edev->ctlr);

			}

		}

		ctlr->rdh = rdh;

	}

}

static int

pktstosend(void* v)

{

	Ether *edev = v;

	Ctlr *ctlr = edev->ctlr;

	return ctlr->pktstosend || ctlr->ntq > 0 || qlen(edev->oq) > 0;

}

static void

tproc(void* arg)

{

	uint tdt, added;

	Arge *arge;

	Block *bp;

	Ctlr *ctlr;

	Desc *td;

	Ether *edev;

	edev = arg;

	ctlr = edev->ctlr;

	arge = ctlr->regs;

	for(;;){

		/* wait for next free buffer and output queue block */

		sleep(&ctlr->trendez, pktstosend, edev);

		ctlr->pktstosend = 0;

		txreclaim(ctlr);

		/* copy as much of my output q as possible into output ring */

		added = 0;

		tdt = ctlr->tdt;

		while(ctlr->ntq < Ntd - 1){

			td = &ctlr->tdba[tdt];

			if (!(td->ctl & Descempty))

				break;

			bp = qget(edev->oq);

			if(bp == nil)

				break;

			/* make sure the whole packet is in ram */

			dcflush(bp->rp, BLEN(bp));

			/*

			 * Give ownership of the descriptor to the chip,

			 * increment the software ring descriptor pointer.

			 */

			ctlr->td[tdt] = bp;

			td->addr = PADDR(bp->rp);

			td->ctl = DMASIZE(BLEN(bp));

			coherence();