WebSVN – planix.SVN – Blame – /os/branches/planix-v0/sys/src/9/pc/usbuhci.c

Rev	Author	Line No.	Line
2	-	1	`/*`
		2	`* USB Universal Host Controller Interface (sic) driver.`
		3	`*`
		4	`* BUGS:`
		5	`* - Too many delays and ilocks.`
		6	`* - bandwidth admission control must be done per-frame.`
		7	`* - interrupt endpoints should go on a tree like [oe]hci.`
		8	`* - must warn of power overruns.`
		9	`*/`
		10
		11	`#include "u.h"`
		12	`#include "../port/lib.h"`
		13	`#include "mem.h"`
		14	`#include "dat.h"`
		15	`#include "fns.h"`
		16	`#include "io.h"`
		17	`#include "../port/error.h"`
		18	`#include "../port/usb.h"`
		19
		20	`typedef struct Ctlio Ctlio;`
		21	`typedef struct Ctlr Ctlr;`
		22	`typedef struct Isoio Isoio;`
		23	`typedef struct Qh Qh;`
		24	`typedef struct Qhpool Qhpool;`
		25	`typedef struct Qio Qio;`
		26	`typedef struct Td Td;`
		27	`typedef struct Tdpool Tdpool;`
		28
		29	`enum`
		30	`{`
		31	`Resetdelay = 100, /* delay after a controller reset (ms) */`
		32	`Enabledelay = 100, /* waiting for a port to enable */`
		33	`Abortdelay = 5, /* delay after cancelling Tds (ms) */`
		34	`Incr = 64, /* for Td and Qh pools */`
		35
		36	`Tdatomic = 8, /* max nb. of Tds per bulk I/O op. */`
		37
		38	`/* Queue states (software) */`
		39	`Qidle = 0,`
		40	`Qinstall,`
		41	`Qrun,`
		42	`Qdone,`
		43	`Qclose,`
		44	`Qfree,`
		45
		46	`/*`
		47	`* HW constants`
		48	`*/`
		49
		50	`Nframes = 1024, /* 2ⁿ for xspanalloc; max 1024 */`
		51	`Align = 16, /* for data structures */`
		52
		53	`/* Size of small buffer kept within Tds. (software) */`
		54	`/* Keep as a multiple of Align to maintain alignment of Tds in pool */`
		55	`Tdndata = 1*Align,`
		56
		57	`/* i/o space`
		58	`* Some ports are short, some are long, some are byte.`
		59	`* We use ins[bsl] and not vmap.`
		60	`*/`
		61	`Cmd = 0,`
		62	`Crun = 0x01,`
		63	`Chcreset = 0x02, /* host controller reset */`
		64	`Cgreset = 0x04, /* global reset */`
		65	`Cegsm = 0x08, /* enter global suspend */`
		66	`Cfgr = 0x10, /* forge global resume */`
		67	`Cdbg = 0x20, /* single step, debug */`
		68	`Cmaxp = 0x80, /* max packet */`
		69
		70	`Status = 2,`
		71	`Susbintr = 0x01, /* interrupt */`
		72	`Seintr = 0x02, /* error interrupt */`
		73	`Sresume = 0x04, /* resume detect */`
		74	`Shserr = 0x08, /* host system error */`
		75	`Shcerr = 0x10, /* host controller error */`
		76	`Shalted = 0x20, /* controller halted */`
		77	`Sall = 0x3F,`
		78
		79	`Usbintr = 4,`
		80	`Itmout = 0x01, /* timeout or crc */`
		81	`Iresume = 0x02, /* resume interrupt enable */`
		82	`Ioc = 0x04, /* interrupt on complete */`
		83	`Ishort = 0x08, /* short packet interrupt */`
		84	`Iall = 0x0F,`
		85	`Frnum = 6,`
		86	`Flbaseadd = 8,`
		87	`SOFmod = 0xC, /* start of frame modifier register */`
		88
		89	`Portsc0 = 0x10,`
		90	`PSpresent = 0x0001, /* device present */`
		91	`PSstatuschg = 0x0002, /* PSpresent changed */`
		92	`PSenable = 0x0004, /* device enabled */`
		93	`PSchange = 0x0008, /* PSenable changed */`
		94	`PSresume = 0x0040, /* resume detected */`
		95	`PSreserved1 = 0x0080, /* always read as 1; reserved */`
		96	`PSslow = 0x0100, /* device has low speed */`
		97	`PSreset = 0x0200, /* port reset */`
		98	`PSsuspend = 0x1000, /* port suspended */`
		99
		100	`/* Transfer descriptor link */`
		101	`Tdterm = 0x1, /* nil (terminate) */`
		102	`Tdlinkqh = 0x2, /* link refers to a QH */`
		103	`Tdvf = 0x4, /* run linked Tds first (depth-first)*/`
		104
		105	`/* Transfer status bits */`
		106	`Tdbitstuff = 0x00020000, /* bit stuffing error */`
		107	`Tdcrcto = 0x00040000, /* crc or timeout error */`
		108	`Tdnak = 0x00080000, /* nak packet received */`
		109	`Tdbabble = 0x00100000, /* babble detected */`
		110	`Tddberr = 0x00200000, /* data buf. error */`
		111	`Tdstalled = 0x00400000, /* serious error to ep. */`
		112	`Tdactive = 0x00800000, /* enabled/in use by hw */`
		113	`/* Transfer control bits */`
		114	`Tdioc = 0x01000000, /* interrupt on complete */`
		115	`Tdiso = 0x02000000, /* isochronous select */`
		116	`Tdlow = 0x04000000, /* low speed device */`
		117	`Tderr1 = 0x08000000, /* bit 0 of error counter */`
		118	`Tderr2 = 0x10000000, /* bit 1 of error counter */`
		119	`Tdspd = 0x20000000, /* short packet detect */`
		120
		121	`Tdlen = 0x000003FF, /* actual length field */`
		122
		123	`Tdfatalerr = Tdnak\|Tdbabble\|Tdstalled, /* hw retries others */`
		124	`Tderrors = Tdfatalerr\|Tdbitstuff\|Tdcrcto\|Tddberr,`
		125
		126	`/* Transfer descriptor token bits */`
		127	`Tddata0 = 0,`
		128	`Tddata1 = 0x80000, /* data toggle (1==DATA1) */`
		129	`Tdtokin = 0x69,`
		130	`Tdtokout = 0xE1,`
		131	`Tdtoksetup = 0x2D,`
		132
		133	`Tdmaxpkt = 0x800, /* max packet size */`
		134
		135	`/* Queue head bits */`
		136	`QHterm = 1<<0, /* nil (terminate) */`
		137	`QHlinkqh = 1<<1, /* link refers to a QH */`
		138	`QHvf = 1<<2, /* vertical first (depth first) */`
		139	`};`
		140
		141	`struct Ctlr`
		142	`{`
		143	`Lock; /* for ilock. qh lists and basic ctlr I/O */`
		144	`QLock portlck; /* for port resets/enable... */`
		145	`Pcidev* pcidev;`
		146	`int active;`
		147	`int port; /* I/O address */`
		148	`Qh* qhs; /* list of Qhs for this controller */`
		149	`Qh* qh[Tmax]; /* Dummy Qhs to insert Qhs after */`
		150	`Isoio* iso; /* list of active iso I/O */`
		151	`ulong* frames; /* frame list (used by hw) */`
		152	`ulong load; /* max load for a single frame */`
		153	`ulong isoload; /* max iso load for a single frame */`
		154	`int nintr; /* number of interrupts attended */`
		155	`int ntdintr; /* number of intrs. with something to do */`
		156	`int nqhintr; /* number of intrs. for Qhs */`
		157	`int nisointr; /* number of intrs. for iso transfers */`
		158	`};`
		159
		160	`struct Qio`
		161	`{`
		162	`QLock; /* for the entire I/O process */`
		163	`Rendez; /* wait for completion */`
		164	`Qh* qh; /* Td list (field const after init) */`
		165	`int usbid; /* usb address for endpoint/device */`
		166	`int toggle; /* Tddata0/Tddata1 */`
		167	`int tok; /* Tdtoksetup, Tdtokin, Tdtokout */`
		168	`ulong iotime; /* time of last I/O */`
		169	`int debug; /* debug flag from the endpoint */`
		170	`char* err; /* error string */`
		171	`};`
		172
		173	`struct Ctlio`
		174	`{`
		175	`Qio; /* a single Qio for each RPC */`
		176	`uchar* data; /* read from last ctl req. */`
		177	`int ndata; /* number of bytes read */`
		178	`};`
		179
		180	`struct Isoio`
		181	`{`
		182	`QLock;`
		183	`Rendez; /* wait for space/completion/errors */`
		184	`int usbid; /* address used for device/endpoint */`
		185	`int tok; /* Tdtokin or Tdtokout */`
		186	`int state; /* Qrun -> Qdone -> Qrun... -> Qclose */`
		187	`int nframes; /* Nframes/ep->pollival */`
		188	`uchar* data; /* iso data buffers if not embedded */`
		189	`int td0frno; /* frame number for first Td */`
		190	`Td* tdu; /* next td for user I/O in tdps */`
		191	`Td* tdi; /* next td processed by interrupt */`
		192	`char* err; /* error string */`
		193	`int nerrs; /* nb of consecutive I/O errors */`
		194	`long nleft; /* number of bytes left from last write */`
		195	`int debug; /* debug flag from the endpoint */`
		196	`Isoio* next; /* in list of active Isoios */`
		197	`Td* tdps[Nframes]; /* pointer to Td used for i-th frame or nil */`
		198	`};`
		199
		200	`struct Tdpool`
		201	`{`
		202	`Lock;`
		203	`Td* free;`
		204	`int nalloc;`
		205	`int ninuse;`
		206	`int nfree;`
		207	`};`
		208
		209	`struct Qhpool`
		210	`{`
		211	`Lock;`
		212	`Qh* free;`
		213	`int nalloc;`
		214	`int ninuse;`
		215	`int nfree;`
		216	`};`
		217
		218	`/*`
		219	`* HW data structures`
		220	`*/`
		221
		222	`/*`
		223	`* Queue header (known by hw).`
		224	`* 16-byte aligned. first two words used by hw.`
		225	`* They are taken from the pool upon endpoint opening and`
		226	`* queued after the dummy queue header for the endpoint type`
		227	`* in the controller. Actual I/O happens as Tds are linked into it.`
		228	`* The driver does I/O in lock-step.`
		229	`* The user builds a list of Tds and links it into the Qh,`
		230	`* then the Qh goes from Qidle to Qrun and nobody touches it until`
		231	`* it becomes Qdone at interrupt time.`
		232	`* At that point the user collects the Tds and it goes Qidle.`
		233	`* A premature cancel may set the state to Qclose and abort I/O.`
		234	`* The Ctlr lock protects change of state for Qhs in use.`
		235	`*/`
		236	`struct Qh`
		237	`{`
		238	`ulong link; /* link to next horiz. item (eg. Qh) */`
		239	`ulong elink; /* link to element (eg. Td; updated by hw) */`
		240
		241	`ulong state; /* Qidle -> Qinstall -> Qrun -> Qdone \| Qclose */`
		242	`Qio* io; /* for this queue */`
		243
		244	`Qh* next; /* in active or free list */`
		245	`Td* tds; /* Td list in this Qh (initially, elink) */`
		246	`char* tag; /* debug and align, mostly */`
		247	`ulong align;`
		248	`};`
		249
		250	`/*`
		251	`* Transfer descriptor.`
		252	`* 16-byte aligned. first two words used by hw. Next 4 by sw.`
		253	`* We keep an embedded buffer for small I/O transfers.`
		254	`* They are taken from the pool when buffers are needed for I/O`
		255	`* and linked at the Qh/Isoio for the endpoint and direction requiring it.`
		256	`* The block keeps actual data. They are protected from races by`
		257	`* the queue or the pool keeping it. The owner of the link to the Td`
		258	`* is free to use it and can be the only one using it.`
		259	`*/`
		260	`struct Td`
		261	`{`
		262	`ulong link; /* Link to next Td or Qh */`
		263	`ulong csw; /* control and status word (updated by hw) */`
		264	`ulong token; /* endpt, device, pid */`
		265	`ulong buffer; /* buffer pointer */`
		266
		267	`Td* next; /* in qh or Isoio or free list */`
		268	`ulong ndata; /* bytes available/used at data */`
		269	`uchar* data; /* pointer to actual data */`
		270	`void* buff; /* allocated data, for large transfers */`
		271
		272	`uchar sbuff[Tdndata]; /* embedded buffer, for small transfers */`
		273	`};`
		274
		275	`#define INB(x) inb(ctlr->port+(x))`
		276	`#define INS(x) ins(ctlr->port+(x))`
		277	`#define INL(x) inl(ctlr->port+(x))`
		278	`#define OUTB(x, v) outb(ctlr->port+(x), (v))`
		279	`#define OUTS(x, v) outs(ctlr->port+(x), (v))`
		280	`#define OUTL(x, v) outl(ctlr->port+(x), (v))`
		281	`#define TRUNC(x, sz) ((x) & ((sz)-1))`
		282	`#define PTR(q) ((void*)KADDR((ulong)(q) & ~ (0xF\|PCIWINDOW)))`
		283	`#define QPTR(q) ((Qh*)PTR(q))`
		284	`#define TPTR(q) ((Td*)PTR(q))`
		285	`#define PORT(p) (Portsc0 + 2*(p))`
		286	`#define diprint if(debug \|\| iso->debug)print`
		287	`#define ddiprint if(debug>1 \|\| iso->debug>1)print`
		288	`#define dqprint if(debug \|\| (qh->io && qh->io->debug))print`
		289	`#define ddqprint if(debug>1 \|\| (qh->io && qh->io->debug>1))print`
		290
		291	`static Ctlr* ctlrs[Nhcis];`
		292
		293	`static Tdpool tdpool;`
		294	`static Qhpool qhpool;`
		295	`static int debug;`
		296
		297	`static char* qhsname[] = { "idle", "install", "run", "done", "close", "FREE" };`
		298
		299	`static void`
		300	`uhcicmd(Ctlr *ctlr, int c)`
		301	`{`
		302	`OUTS(Cmd, c);`
		303	`}`
		304
		305	`static void`
		306	`uhcirun(Ctlr *ctlr, int on)`
		307	`{`
		308	`int i;`
		309
		310	`ddprint("uhci %#ux setting run to %d\n", ctlr->port, on);`
		311
		312	`if(on)`
		313	`uhcicmd(ctlr, INS(Cmd)\|Crun);`
		314	`else`
		315	`uhcicmd(ctlr, INS(Cmd) & ~Crun);`
		316	`for(i = 0; i < 100; i++)`
		317	`if(on == 0 && (INS(Status) & Shalted) != 0)`
		318	`break;`
		319	`else if(on != 0 && (INS(Status) & Shalted) == 0)`
		320	`break;`
		321	`else`
		322	`delay(1);`
		323	`if(i == 100)`
		324	`dprint("uhci %#x run cmd timed out\n", ctlr->port);`
		325	`ddprint("uhci %#ux cmd %#ux sts %#ux\n",`
		326	`ctlr->port, INS(Cmd), INS(Status));`
		327	`}`
		328
		329	`static int`
		330	`tdlen(Td *td)`
		331	`{`
		332	`return (td->csw+1) & Tdlen;`
		333	`}`
		334
		335	`static int`
		336	`maxtdlen(Td *td)`
		337	`{`
		338	`return ((td->token>>21)+1) & (Tdmaxpkt-1);`
		339	`}`
		340
		341	`static int`
		342	`tdtok(Td *td)`
		343	`{`
		344	`return td->token & 0xFF;`
		345	`}`
		346
		347	`static char*`
		348	`seprinttd(char s, char se, Td *td)`
		349	`{`
		350	`s = seprint(s, se, "%#p link %#ulx", td, td->link);`
		351	`if((td->link & Tdvf) != 0)`
		352	`s = seprint(s, se, "V");`
		353	`if((td->link & Tdterm) != 0)`
		354	`s = seprint(s, se, "T");`
		355	`if((td->link & Tdlinkqh) != 0)`
		356	`s = seprint(s, se, "Q");`
		357	`s = seprint(s, se, " csw %#ulx ", td->csw);`
		358	`if(td->csw & Tdactive)`
		359	`s = seprint(s, se, "a");`
		360	`if(td->csw & Tdiso)`
		361	`s = seprint(s, se, "I");`
		362	`if(td->csw & Tdioc)`
		363	`s = seprint(s, se, "i");`
		364	`if(td->csw & Tdlow)`
		365	`s = seprint(s, se, "l");`
		366	`if((td->csw & (Tderr1\|Tderr2)) == 0)`
		367	`s = seprint(s, se, "z");`
		368	`if(td->csw & Tderrors)`
		369	`s = seprint(s, se, " err %#ulx", td->csw & Tderrors);`
		370	`if(td->csw & Tdstalled)`
		371	`s = seprint(s, se, "s");`
		372	`if(td->csw & Tddberr)`
		373	`s = seprint(s, se, "d");`
		374	`if(td->csw & Tdbabble)`
		375	`s = seprint(s, se, "b");`
		376	`if(td->csw & Tdnak)`
		377	`s = seprint(s, se, "n");`
		378	`if(td->csw & Tdcrcto)`
		379	`s = seprint(s, se, "c");`
		380	`if(td->csw & Tdbitstuff)`
		381	`s = seprint(s, se, "B");`
		382	`s = seprint(s, se, " stslen %d", tdlen(td));`
		383
		384	`s = seprint(s, se, " token %#ulx", td->token);`
		385	`if(td->token == 0) /* the BWS loopback Td, ignore rest */`
		386	`return s;`
		387	`s = seprint(s, se, " maxlen %d", maxtdlen(td));`
		388	`if(td->token & Tddata1)`
		389	`s = seprint(s, se, " d1");`
		390	`else`
		391	`s = seprint(s, se, " d0");`
		392	`s = seprint(s, se, " id %#ulx:", (td->token>>15) & Epmax);`
		393	`s = seprint(s, se, "%#ulx", (td->token>>8) & Devmax);`
		394	`switch(tdtok(td)){`
		395	`case Tdtokin:`
		396	`s = seprint(s, se, " in");`
		397	`break;`
		398	`case Tdtokout:`
		399	`s = seprint(s, se, " out");`
		400	`break;`
		401	`case Tdtoksetup:`
		402	`s = seprint(s, se, " setup");`
		403	`break;`
		404	`default:`
		405	`s = seprint(s, se, " BADPID");`
		406	`}`
		407	`s = seprint(s, se, "\n\t buffer %#ulx data %#p", td->buffer, td->data);`
		408	`s = seprint(s, se, " ndata %uld sbuff %#p buff %#p",`
		409	`td->ndata, td->sbuff, td->buff);`
		410	`if(td->ndata > 0)`
		411	`s = seprintdata(s, se, td->data, td->ndata);`
		412	`return s;`
		413	`}`
		414
		415	`static void`
		416	`isodump(Isoio *iso, int all)`
		417	`{`
		418	`char buf[256];`
		419	`Td *td;`
		420	`int i;`
		421
		422	`print("iso %#p %s state %d nframes %d"`
		423	`" td0 %#p tdu %#p tdi %#p data %#p\n",`
		424	`iso, iso->tok == Tdtokin ? "in" : "out",`
		425	`iso->state, iso->nframes, iso->tdps[iso->td0frno],`
		426	`iso->tdu, iso->tdi, iso->data);`
		427	`if(iso->err != nil)`
		428	`print("\terr='%s'\n", iso->err);`
		429	`if(all == 0){`
		430	`seprinttd(buf, buf+sizeof(buf), iso->tdu);`
		431	`print("\ttdu %s\n", buf);`
		432	`seprinttd(buf, buf+sizeof(buf), iso->tdi);`
		433	`print("\ttdi %s\n", buf);`
		434	`}else{`
		435	`td = iso->tdps[iso->td0frno];`
		436	`for(i = 0; i < iso->nframes; i++){`
		437	`seprinttd(buf, buf+sizeof(buf), td);`
		438	`if(td == iso->tdi)`
		439	`print("i->");`
		440	`if(td == iso->tdu)`
		441	`print("u->");`
		442	`print("\t%s\n", buf);`
		443	`td = td->next;`
		444	`}`
		445	`}`
		446	`}`
		447
		448	`static int`
		449	`sameptr(void *p, ulong l)`
		450	`{`
		451	`if(l & QHterm)`
		452	`return p == nil;`
		453	`return PTR(l) == p;`
		454	`}`
		455
		456	`static void`
		457	`dumptd(Td td, char pref)`
		458	`{`
		459	`char buf[256];`
		460	`char *s;`
		461	`char *se;`
		462	`int i;`
		463
		464	`i = 0;`
		465	`se = buf+sizeof(buf);`
		466	`for(; td != nil; td = td->next){`
		467	`s = seprinttd(buf, se, td);`
		468	`if(!sameptr(td->next, td->link))`
		469	`seprint(s, se, " next %#p != link %#ulx %#p",`
		470	`td->next, td->link, TPTR(td->link));`
		471	`print("%std %s\n", pref, buf);`
		472	`if(i++ > 20){`
		473	`print("...more tds...\n");`
		474	`break;`
		475	`}`
		476	`}`
		477	`}`
		478
		479	`static void`
		480	`qhdump(Qh qh, char pref)`
		481	`{`
		482	`char buf[256];`
		483	`char *s;`
		484	`char *se;`
		485	`ulong td;`
		486	`int i;`
		487
		488	`s = buf;`
		489	`se = buf+sizeof(buf);`
		490	`s = seprint(s, se, "%sqh %s %#p state %s link %#ulx", pref,`
		491	`qh->tag, qh, qhsname[qh->state], qh->link);`
		492	`if(!sameptr(qh->tds, qh->elink))`
		493	`s = seprint(s, se, " [tds %#p != elink %#ulx %#p]",`
		494	`qh->tds, qh->elink, TPTR(qh->elink));`
		495	`if(!sameptr(qh->next, qh->link))`
		496	`s = seprint(s, se, " [next %#p != link %#ulx %#p]",`
		497	`qh->next, qh->link, QPTR(qh->link));`
		498	`if((qh->link & Tdterm) != 0)`
		499	`s = seprint(s, se, "T");`
		500	`if((qh->link & Tdlinkqh) != 0)`
		501	`s = seprint(s, se, "Q");`
		502	`s = seprint(s, se, " elink %#ulx", qh->elink);`
		503	`if((qh->elink & Tdterm) != 0)`
		504	`s = seprint(s, se, "T");`
		505	`if((qh->elink & Tdlinkqh) != 0)`
		506	`s = seprint(s, se, "Q");`
		507	`s = seprint(s, se, " io %#p", qh->io);`
		508	`if(qh->io != nil && qh->io->err != nil)`
		509	`seprint(s, se, " err='%s'", qh->io->err);`
		510	`print("%s\n", buf);`
		511	`dumptd(qh->tds, "\t");`
		512	`if((qh->elink & QHterm) == 0){`
		513	`print("\thw tds:");`
		514	`i = 0;`
		515	`for(td = qh->elink; (td & Tdterm) == 0; td = TPTR(td)->link){`
		516	`print(" %#ulx", td);`
		517	`if(td == TPTR(td)->link) /* BWS Td */`
		518	`break;`
		519	`if(i++ > 40){`
		520	`print("...");`
		521	`break;`
		522	`}`
		523	`}`
		524	`print("\n");`
		525	`}`
		526	`}`
		527
		528	`static void`
		529	`xdump(Ctlr *ctlr, int doilock)`
		530	`{`
		531	`Isoio *iso;`
		532	`Qh *qh;`
		533	`int i;`
		534
		535	`if(doilock){`
		536	`if(ctlr == ctlrs[0]){`
		537	`lock(&tdpool);`
		538	`print("tds: alloc %d = inuse %d + free %d\n",`
		539	`tdpool.nalloc, tdpool.ninuse, tdpool.nfree);`
		540	`unlock(&tdpool);`
		541	`lock(&qhpool);`
		542	`print("qhs: alloc %d = inuse %d + free %d\n",`
		543	`qhpool.nalloc, qhpool.ninuse, qhpool.nfree);`
		544	`unlock(&qhpool);`
		545	`}`
		546	`ilock(ctlr);`
		547	`}`
		548	`print("uhci port %#x frames %#p nintr %d ntdintr %d",`
		549	`ctlr->port, ctlr->frames, ctlr->nintr, ctlr->ntdintr);`
		550	`print(" nqhintr %d nisointr %d\n", ctlr->nqhintr, ctlr->nisointr);`
		551	`print("cmd %#ux sts %#ux fl %#ulx ps1 %#ux ps2 %#ux frames[0] %#ulx\n",`
		552	`INS(Cmd), INS(Status),`
		553	`INL(Flbaseadd), INS(PORT(0)), INS(PORT(1)),`
		554	`ctlr->frames[0]);`
		555	`for(iso = ctlr->iso; iso != nil; iso = iso->next)`
		556	`isodump(iso, 1);`
		557	`i = 0;`
		558	`for(qh = ctlr->qhs; qh != nil; qh = qh->next){`
		559	`qhdump(qh, "");`
		560	`if(i++ > 20){`
		561	`print("qhloop\n");`
		562	`break;`
		563	`}`
		564	`}`
		565	`print("\n");`
		566	`if(doilock)`
		567	`iunlock(ctlr);`
		568	`}`
		569
		570	`static void`
		571	`dump(Hci *hp)`
		572	`{`
		573	`xdump(hp->aux, 1);`
		574	`}`
		575
		576	`static Td*`
		577	`tdalloc(void)`
		578	`{`
		579	`int i;`
		580	`Td *td;`
		581	`Td *pool;`
		582
		583	`lock(&tdpool);`
		584	`if(tdpool.free == nil){`
		585	`ddprint("uhci: tdalloc %d Tds\n", Incr);`
		586	`pool = xspanalloc(Incr*sizeof(Td), Align, 0);`
		587	`if(pool == nil)`
		588	`panic("tdalloc");`
		589	`for(i=Incr; --i>=0;){`
		590	`pool[i].next = tdpool.free;`
		591	`tdpool.free = &pool[i];`
		592	`}`
		593	`tdpool.nalloc += Incr;`
		594	`tdpool.nfree += Incr;`
		595	`}`
		596	`td = tdpool.free;`
		597	`tdpool.free = td->next;`
		598	`tdpool.ninuse++;`
		599	`tdpool.nfree--;`
		600	`unlock(&tdpool);`
		601
		602	`memset(td, 0, sizeof(Td));`
		603	`td->link = Tdterm;`
		604	`assert(((ulong)td & 0xF) == 0);`
		605	`return td;`
		606	`}`
		607
		608	`static void`
		609	`tdfree(Td *td)`
		610	`{`
		611	`if(td == nil)`
		612	`return;`
		613	`free(td->buff);`
		614	`td->buff = nil;`
		615	`lock(&tdpool);`
		616	`td->next = tdpool.free;`
		617	`tdpool.free = td;`
		618	`tdpool.ninuse--;`
		619	`tdpool.nfree++;`
		620	`unlock(&tdpool);`
		621	`}`
		622
		623	`static void`
		624	`qhlinkqh(Qh* qh, Qh* next)`
		625	`{`
		626	`if(next == nil)`
		627	`qh->link = QHterm;`
		628	`else{`
		629	`next->link = qh->link;`
		630	`next->next = qh->next;`
		631	`qh->link = PCIWADDR(next)\|QHlinkqh;`
		632	`}`
		633	`qh->next = next;`
		634	`}`
		635
		636	`static void`
		637	`qhlinktd(Qh qh, Td td)`
		638	`{`
		639	`qh->tds = td;`
		640	`if(td == nil)`
		641	`qh->elink = QHvf\|QHterm;`
		642	`else`
		643	`qh->elink = PCIWADDR(td);`
		644	`}`
		645
		646	`static void`
		647	`tdlinktd(Td td, Td next)`
		648	`{`
		649	`td->next = next;`
		650	`if(next == nil)`
		651	`td->link = Tdterm;`
		652	`else`
		653	`td->link = PCIWADDR(next)\|Tdvf;`
		654	`}`
		655
		656	`static Qh*`
		657	`qhalloc(Ctlr ctlr, Qh prev, Qio io, char tag)`
		658	`{`
		659	`int i;`
		660	`Qh *qh;`
		661	`Qh *pool;`
		662
		663	`lock(&qhpool);`
		664	`if(qhpool.free == nil){`
		665	`ddprint("uhci: qhalloc %d Qhs\n", Incr);`
		666	`pool = xspanalloc(Incr*sizeof(Qh), Align, 0);`
		667	`if(pool == nil)`
		668	`panic("qhalloc");`
		669	`for(i=Incr; --i>=0;){`
		670	`pool[i].next = qhpool.free;`
		671	`qhpool.free = &pool[i];`
		672	`}`
		673	`qhpool.nalloc += Incr;`
		674	`qhpool.nfree += Incr;`
		675	`}`
		676	`qh = qhpool.free;`
		677	`qhpool.free = qh->next;`
		678	`qh->next = nil;`
		679	`qh->link = QHterm;`
		680	`qhpool.ninuse++;`
		681	`qhpool.nfree--;`
		682	`unlock(&qhpool);`
		683
		684	`qh->tds = nil;`
		685	`qh->elink = QHterm;`
		686	`qh->state = Qidle;`
		687	`qh->io = io;`
		688	`qh->tag = nil;`
		689	`kstrdup(&qh->tag, tag);`
		690
		691	`if(prev != nil){`
		692	`coherence();`
		693	`ilock(ctlr);`
		694	`qhlinkqh(prev, qh);`
		695	`iunlock(ctlr);`
		696	`}`
		697
		698	`assert(((ulong)qh & 0xF) == 0);`
		699	`return qh;`
		700	`}`
		701
		702	`static void`
		703	`qhfree(Ctlr ctlr, Qh qh)`
		704	`{`
		705	`Td *td;`
		706	`Td *ltd;`
		707	`Qh *q;`
		708
		709	`if(qh == nil)`
		710	`return;`
		711
		712	`ilock(ctlr);`
		713	`for(q = ctlr->qhs; q != nil; q = q->next)`
		714	`if(q->next == qh)`
		715	`break;`
		716	`if(q == nil)`
		717	`panic("qhfree: nil q");`
		718	`q->next = qh->next;`
		719	`q->link = qh->link;`
		720	`iunlock(ctlr);`
		721
		722	`for(td = qh->tds; td != nil; td = ltd){`
		723	`ltd = td->next;`
		724	`tdfree(td);`
		725	`}`
		726	`lock(&qhpool);`
		727	`qh->state = Qfree; /* paranoia */`
		728	`qh->next = qhpool.free;`
		729	`qh->tag = nil;`
		730	`qh->io = nil;`
		731	`qhpool.free = qh;`
		732	`qhpool.ninuse--;`
		733	`qhpool.nfree++;`
		734	`unlock(&qhpool);`
		735	`ddprint("qhfree: qh %#p\n", qh);`
		736	`}`
		737
		738	`static char*`
		739	`errmsg(int err)`
		740	`{`
		741	`if(err == 0)`
		742	`return "ok";`
		743	`if(err & Tdcrcto)`
		744	`return "crc/timeout error";`
		745	`if(err & Tdbabble)`
		746	`return "babble detected";`
		747	`if(err & Tddberr)`
		748	`return "db error";`
		749	`if(err & Tdbitstuff)`
		750	`return "bit stuffing error";`
		751	`if(err & Tdstalled)`
		752	`return Estalled;`
		753	`return Eio;`
		754	`}`
		755
		756	`static int`
		757	`isocanread(void *a)`
		758	`{`
		759	`Isoio *iso;`
		760
		761	`iso = a;`
		762	`return iso->state == Qclose \|\|`
		763	`(iso->state == Qrun &&`
		764	`iso->tok == Tdtokin && iso->tdi != iso->tdu);`
		765	`}`
		766
		767	`static int`
		768	`isocanwrite(void *a)`
		769	`{`
		770	`Isoio *iso;`
		771
		772	`iso = a;`
		773	`return iso->state == Qclose \|\|`
		774	`(iso->state == Qrun &&`
		775	`iso->tok == Tdtokout && iso->tdu->next != iso->tdi);`
		776	`}`
		777
		778	`static void`
		779	`tdisoinit(Isoio iso, Td td, long count)`
		780	`{`
		781	`td->ndata = count;`
		782	`td->token = ((count-1)<<21)\| ((iso->usbid & 0x7FF)<<8) \| iso->tok;`
		783	`td->csw = Tderr1\|Tdiso\|Tdactive\|Tdioc;`
		784	`}`
		785
		786	`/*`
		787	`* Process Iso i/o on interrupt. For writes update just error status.`
		788	`* For reads update tds to reflect data and also error status.`
		789	`* When tdi aproaches tdu, advance tdu; data may be lost.`
		790	`* (If nframes is << Nframes tdu might be far away but this avoids`
		791	`* races regarding frno.)`
		792	`* If we suffer errors for more than half the frames we stall.`
		793	`*/`
		794	`static void`
		795	`isointerrupt(Ctlr ctlr, Isoio iso)`
		796	`{`
		797	`Td *tdi;`
		798	`int err;`
		799	`int i;`
		800	`int nframes;`
		801
		802	`tdi = iso->tdi;`
		803	`if((tdi->csw & Tdactive) != 0) /* nothing new done */`
		804	`return;`
		805	`ctlr->nisointr++;`
		806	`ddiprint("isointr: iso %#p: tdi %#p tdu %#p\n", iso, tdi, iso->tdu);`
		807	`if(iso->state != Qrun && iso->state != Qdone)`
		808	`panic("isointr: iso state");`
		809	`if(debug > 1 \|\| iso->debug > 1)`
		810	`isodump(iso, 0);`
		811
		812	`nframes = iso->nframes / 2; /* limit how many we look */`
		813	`if(nframes > 64)`
		814	`nframes = 64;`
		815
		816	`for(i = 0; i < nframes && (tdi->csw & Tdactive) == 0; i++){`
		817	`tdi->csw &= ~Tdioc;`
		818	`err = tdi->csw & Tderrors;`
		819	`if(err == 0)`
		820	`iso->nerrs = 0;`
		821	`else if(iso->nerrs++ > iso->nframes/2)`
		822	`tdi->csw \|= Tdstalled;`
		823	`if((tdi->csw & Tdstalled) != 0){`
		824	`if(iso->err == nil){`
		825	`iso->err = errmsg(err);`
		826	`diprint("isointerrupt: tdi %#p error %#ux %s\n",`
		827	`tdi, err, iso->err);`
		828	`diprint("ctlr load %uld\n", ctlr->load);`
		829	`}`
		830	`tdi->ndata = 0;`
		831	`}else`
		832	`tdi->ndata = tdlen(tdi);`
		833
		834	`if(tdi->next == iso->tdu \|\| tdi->next->next == iso->tdu){`
		835	`memset(iso->tdu->data, 0, maxtdlen(iso->tdu));`
		836	`tdisoinit(iso, iso->tdu, maxtdlen(iso->tdu));`
		837	`iso->tdu = iso->tdu->next;`
		838	`iso->nleft = 0;`
		839	`}`
		840	`tdi = tdi->next;`
		841	`}`
		842	`ddiprint("isointr: %d frames processed\n", nframes);`
		843	`if(i == nframes)`
		844	`tdi->csw \|= Tdioc;`
		845	`iso->tdi = tdi;`
		846	`if(isocanwrite(iso) \|\| isocanread(iso)){`
		847	`diprint("wakeup iso %#p tdi %#p tdu %#p\n", iso,`
		848	`iso->tdi, iso->tdu);`
		849	`wakeup(iso);`
		850	`}`
		851
		852	`}`
		853
		854	`/*`
		855	`* Process a Qh upon interrupt. There's one per ongoing user I/O.`
		856	`* User process releases resources later, that is not done here.`
		857	`* We may find in this order one or more Tds:`
		858	`* - none/many non active and completed Tds`
		859	`* - none/one (usually(!) not active) and failed Td`
		860	`* - none/many active Tds.`
		861	`* Upon errors the entire transfer is aborted and error reported.`
		862	`* Otherwise, the transfer is complete only when all Tds are done or`
		863	`* when a read with less than maxpkt is found.`
		864	`* Use the software list and not qh->elink to avoid races.`
		865	`* We could use qh->elink to see if there's something new or not.`
		866	`*/`
		867	`static void`
		868	`qhinterrupt(Ctlr ctlr, Qh qh)`
		869	`{`
		870	`Td *td;`
		871	`int err;`
		872
		873	`ctlr->nqhintr++;`
		874	`if(qh->state != Qrun)`
		875	`panic("qhinterrupt: qh state");`
		876	`if(qh->tds == nil)`
		877	`panic("qhinterrupt: no tds");`
		878	`if((qh->tds->csw & Tdactive) == 0)`
		879	`ddqprint("qhinterrupt port %#ux qh %#p p0 %#x p1 %#x\n",`
		880	`ctlr->port, qh, INS(PORT(0)), INS(PORT(1)));`
		881	`for(td = qh->tds; td != nil; td = td->next){`
		882	`if(td->csw & Tdactive)`
		883	`return;`
		884	`td->csw &= ~Tdioc;`
		885	`if((td->csw & Tdstalled) != 0){`
		886	`err = td->csw & Tderrors;`
		887	`/* just stalled is end of xfer but not an error */`
		888	`if(err != Tdstalled && qh->io->err == nil){`
		889	`qh->io->err = errmsg(td->csw & Tderrors);`
		890	`dqprint("qhinterrupt: td %#p error %#ux %s\n",`
		891	`td, err, qh->io->err);`
		892	`dqprint("ctlr load %uld\n", ctlr->load);`
		893	`}`
		894	`break;`
		895	`}`
		896	`if((td->csw & Tdnak) != 0){ /* retransmit; not serious */`
		897	`td->csw &= ~Tdnak;`
		898	`if(td->next == nil)`
		899	`td->csw \|= Tdioc;`
		900	`}`
		901	`td->ndata = tdlen(td);`
		902	`if(td->ndata < maxtdlen(td)){ /* EOT */`
		903	`td = td->next;`
		904	`break;`
		905	`}`
		906	`}`
		907
		908	`/*`
		909	`* Done. Make void the Tds not used (errors or EOT) and wakeup epio.`
		910	`*/`
		911	`qh->elink = QHterm;`
		912	`for(; td != nil; td = td->next)`
		913	`td->ndata = 0;`
		914	`qh->state = Qdone;`
		915	`wakeup(qh->io);`
		916	`}`
		917
		918	`static void`
		919	`interrupt(Ureg, void a)`
		920	`{`
		921	`Hci *hp;`
		922	`Ctlr *ctlr;`
		923	`int frptr;`
		924	`int frno;`
		925	`Qh *qh;`
		926	`Isoio *iso;`
		927	`int sts;`
		928	`int cmd;`
		929
		930	`hp = a;`
		931	`ctlr = hp->aux;`
		932	`ilock(ctlr);`
		933	`ctlr->nintr++;`
		934	`sts = INS(Status);`
		935	`if((sts & Sall) == 0){ /* not for us; sharing irq */`
		936	`iunlock(ctlr);`
		937	`return;`
		938	`}`
		939	`OUTS(Status, sts & Sall);`
		940	`cmd = INS(Cmd);`
		941	`if(cmd & Crun == 0){`
		942	`print("uhci %#ux: not running: uhci bug?\n", ctlr->port);`
		943	`/* BUG: should abort everything in this case */`
		944	`}`
		945	`if(debug > 1){`
		946	`frptr = INL(Flbaseadd);`
		947	`frno = INL(Frnum);`
		948	`frno = TRUNC(frno, Nframes);`
		949	`print("cmd %#ux sts %#ux frptr %#ux frno %d\n",`
		950	`cmd, sts, frptr, frno);`
		951	`}`
		952	`ctlr->ntdintr++;`
		953	`/*`
		954	`* Will we know in USB 3.0 who the interrupt was for?.`
		955	`* Do they still teach indexing in CS?`
		956	`* This is Intel's doing.`
		957	`*/`
		958	`for(iso = ctlr->iso; iso != nil; iso = iso->next)`
		959	`if(iso->state == Qrun \|\| iso->state == Qdone)`
		960	`isointerrupt(ctlr, iso);`
		961	`for(qh = ctlr->qhs; qh != nil; qh = qh->next)`
		962	`if(qh->state == Qrun)`
		963	`qhinterrupt(ctlr, qh);`
		964	`else if(qh->state == Qclose)`
		965	`qhlinktd(qh, nil);`
		966	`iunlock(ctlr);`
		967	`}`
		968
		969	`/*`
		970	`* iso->tdu is the next place to put data. When it gets full`
		971	`* it is activated and tdu advanced.`
		972	`*/`
		973	`static long`
		974	`putsamples(Isoio iso, uchar b, long count)`
		975	`{`
		976	`long tot;`
		977	`long n;`
		978
		979	`for(tot = 0; isocanwrite(iso) && tot < count; tot += n){`
		980	`n = count-tot;`
		981	`if(n > maxtdlen(iso->tdu) - iso->nleft)`
		982	`n = maxtdlen(iso->tdu) - iso->nleft;`
		983	`memmove(iso->tdu->data+iso->nleft, b+tot, n);`
		984	`iso->nleft += n;`
		985	`if(iso->nleft == maxtdlen(iso->tdu)){`
		986	`tdisoinit(iso, iso->tdu, iso->nleft);`
		987	`iso->nleft = 0;`
		988	`iso->tdu = iso->tdu->next;`
		989	`}`
		990	`}`
		991	`return tot;`
		992	`}`
		993
		994	`/*`
		995	`* Queue data for writing and return error status from`
		996	`* last writes done, to maintain buffered data.`
		997	`*/`
		998	`static long`
		999	`episowrite(Ep ep, Isoio iso, void *a, long count)`
		1000	`{`
		1001	`Ctlr *ctlr;`
		1002	`uchar *b;`
		1003	`int tot;`
		1004	`int nw;`
		1005	`char *err;`
		1006
		1007	`iso->debug = ep->debug;`
		1008	`diprint("uhci: episowrite: %#p ep%d.%d\n", iso, ep->dev->nb, ep->nb);`
		1009
		1010	`ctlr = ep->hp->aux;`
		1011	`qlock(iso);`
		1012	`if(waserror()){`
		1013	`qunlock(iso);`
		1014	`nexterror();`
		1015	`}`
		1016	`ilock(ctlr);`
		1017	`if(iso->state == Qclose){`
		1018	`iunlock(ctlr);`
		1019	`error(iso->err ? iso->err : Eio);`
		1020	`}`
		1021	`iso->state = Qrun;`
		1022	`b = a;`
		1023	`for(tot = 0; tot < count; tot += nw){`
		1024	`while(isocanwrite(iso) == 0){`
		1025	`iunlock(ctlr);`
		1026	`diprint("uhci: episowrite: %#p sleep\n", iso);`
		1027	`if(waserror()){`
		1028	`if(iso->err == nil)`
		1029	`iso->err = "I/O timed out";`
		1030	`ilock(ctlr);`
		1031	`break;`
		1032	`}`
		1033	`tsleep(iso, isocanwrite, iso, ep->tmout);`
		1034	`poperror();`
		1035	`ilock(ctlr);`
		1036	`}`
		1037	`err = iso->err;`
		1038	`iso->err = nil;`
		1039	`if(iso->state == Qclose \|\| err != nil){`
		1040	`iunlock(ctlr);`
		1041	`error(err ? err : Eio);`
		1042	`}`
		1043	`if(iso->state != Qrun)`
		1044	`panic("episowrite: iso not running");`
		1045	`iunlock(ctlr); /* We could page fault here */`
		1046	`nw = putsamples(iso, b+tot, count-tot);`
		1047	`ilock(ctlr);`
		1048	`}`
		1049	`if(iso->state != Qclose)`
		1050	`iso->state = Qdone;`
		1051	`iunlock(ctlr);`
		1052	`err = iso->err; /* in case it failed early */`
		1053	`iso->err = nil;`
		1054	`qunlock(iso);`
		1055	`poperror();`
		1056	`if(err != nil)`
		1057	`error(err);`
		1058	`diprint("uhci: episowrite: %#p %d bytes\n", iso, tot);`
		1059	`return tot;`
		1060	`}`
		1061
		1062	`/*`
		1063	`* Available data is kept at tdu and following tds, up to tdi (excluded).`
		1064	`*/`
		1065	`static long`
		1066	`episoread(Ep ep, Isoio iso, void *a, int count)`
		1067	`{`
		1068	`Ctlr *ctlr;`
		1069	`uchar *b;`
		1070	`int nr;`
		1071	`int tot;`
		1072	`Td *tdu;`
		1073
		1074	`iso->debug = ep->debug;`
		1075	`diprint("uhci: episoread: %#p ep%d.%d\n", iso, ep->dev->nb, ep->nb);`
		1076
		1077	`b = a;`
		1078	`ctlr = ep->hp->aux;`
		1079	`qlock(iso);`
		1080	`if(waserror()){`
		1081	`qunlock(iso);`
		1082	`nexterror();`
		1083	`}`
		1084	`iso->err = nil;`
		1085	`iso->nerrs = 0;`
		1086	`ilock(ctlr);`
		1087	`if(iso->state == Qclose){`
		1088	`iunlock(ctlr);`
		1089	`error(iso->err ? iso->err : Eio);`
		1090	`}`
		1091	`iso->state = Qrun;`
		1092	`while(isocanread(iso) == 0){`
		1093	`iunlock(ctlr);`
		1094	`diprint("uhci: episoread: %#p sleep\n", iso);`
		1095	`if(waserror()){`
		1096	`if(iso->err == nil)`
		1097	`iso->err = "I/O timed out";`
		1098	`ilock(ctlr);`
		1099	`break;`
		1100	`}`
		1101	`tsleep(iso, isocanread, iso, ep->tmout);`
		1102	`poperror();`
		1103	`ilock(ctlr);`
		1104	`}`
		1105	`if(iso->state == Qclose){`
		1106	`iunlock(ctlr);`
		1107	`error(iso->err ? iso->err : Eio);`
		1108	`}`
		1109	`iso->state = Qdone;`
		1110	`assert(iso->tdu != iso->tdi);`
		1111
		1112	`for(tot = 0; iso->tdi != iso->tdu && tot < count; tot += nr){`
		1113	`tdu = iso->tdu;`
		1114	`if(tdu->csw & Tdactive){`
		1115	`diprint("uhci: episoread: %#p tdu active\n", iso);`
		1116	`break;`
		1117	`}`
		1118	`nr = tdu->ndata;`
		1119	`if(tot + nr > count)`
		1120	`nr = count - tot;`
		1121	`if(nr == 0)`
		1122	`print("uhci: ep%d.%d: too many polls\n",`
		1123	`ep->dev->nb, ep->nb);`
		1124	`else{`
		1125	`iunlock(ctlr); /* We could page fault here */`
		1126	`memmove(b+tot, tdu->data, nr);`
		1127	`ilock(ctlr);`
		1128	`if(nr < tdu->ndata)`
		1129	`memmove(tdu->data, tdu->data+nr, tdu->ndata - nr);`
		1130	`tdu->ndata -= nr;`
		1131	`}`
		1132	`if(tdu->ndata == 0){`
		1133	`tdisoinit(iso, tdu, ep->maxpkt);`
		1134	`iso->tdu = tdu->next;`
		1135	`}`
		1136	`}`
		1137	`iunlock(ctlr);`
		1138	`qunlock(iso);`
		1139	`poperror();`
		1140	`diprint("uhci: episoread: %#p %d bytes err '%s'\n", iso, tot, iso->err);`
		1141	`if(iso->err != nil)`
		1142	`error(iso->err);`
		1143	`return tot;`
		1144	`}`
		1145
		1146	`static int`
		1147	`nexttoggle(int tog)`
		1148	`{`
		1149	`if(tog == Tddata0)`
		1150	`return Tddata1;`
		1151	`else`
		1152	`return Tddata0;`
		1153	`}`
		1154
		1155	`static Td*`
		1156	`epgettd(Ep ep, Qio io, int flags, void *a, int count)`
		1157	`{`
		1158	`Td *td;`
		1159	`int tok;`
		1160
		1161	`if(ep->maxpkt < count)`
		1162	`error("maxpkt too short");`
		1163	`td = tdalloc();`
		1164	`if(count <= Tdndata)`
		1165	`td->data = td->sbuff;`
		1166	`else`
		1167	`td->data = td->buff = smalloc(ep->maxpkt);`
		1168	`td->buffer = PCIWADDR(td->data);`
		1169	`td->ndata = count;`
		1170	`if(a != nil && count > 0)`
		1171	`memmove(td->data, a, count);`
		1172	`td->csw = Tderr2\|Tderr1\|flags;`
		1173	`if(ep->dev->speed == Lowspeed)`
		1174	`td->csw \|= Tdlow;`
		1175	`tok = io->tok \| io->toggle;`
		1176	`io->toggle = nexttoggle(io->toggle);`
		1177	`td->token = ((count-1)<<21) \| ((io->usbid&0x7FF)<<8) \| tok;`
		1178
		1179	`return td;`
		1180	`}`
		1181
		1182	`/*`
		1183	`* Try to get them idle`
		1184	`*/`
		1185	`static void`
		1186	`aborttds(Qh *qh)`
		1187	`{`
		1188	`Td *td;`
		1189
		1190	`qh->state = Qdone;`
		1191	`qh->elink = QHterm;`
		1192	`for(td = qh->tds; td != nil; td = td->next){`
		1193	`if(td->csw & Tdactive)`
		1194	`td->ndata = 0;`
		1195	`td->csw &= ~(Tdactive\|Tdioc);`
		1196	`}`
		1197	`}`
		1198
		1199	`static int`
		1200	`epiodone(void *a)`
		1201	`{`
		1202	`Qh *qh;`
		1203
		1204	`qh = a;`
		1205	`return qh->state != Qrun;`
		1206	`}`
		1207
		1208	`static void`
		1209	`epiowait(Ctlr ctlr, Qio io, int tmout, ulong load)`
		1210	`{`
		1211	`Qh *qh;`
		1212	`int timedout;`
		1213
		1214	`qh = io->qh;`
		1215	`ddqprint("uhci io %#p sleep on qh %#p state %uld\n", io, qh, qh->state);`
		1216	`timedout = 0;`
		1217	`if(waserror()){`
		1218	`dqprint("uhci io %#p qh %#p timed out\n", io, qh);`
		1219	`timedout++;`
		1220	`}else{`
		1221	`if(tmout == 0)`
		1222	`sleep(io, epiodone, qh);`
		1223	`else`
		1224	`tsleep(io, epiodone, qh, tmout);`
		1225	`poperror();`
		1226	`}`
		1227	`ilock(ctlr);`
		1228	`if(qh->state == Qrun)`
		1229	`timedout = 1;`
		1230	`else if(qh->state != Qdone && qh->state != Qclose)`
		1231	`panic("epio: queue not done and not closed");`
		1232	`if(timedout){`
		1233	`aborttds(io->qh);`
		1234	`io->err = "request timed out";`
		1235	`iunlock(ctlr);`
		1236	`if(!waserror()){`
		1237	`tsleep(&up->sleep, return0, 0, Abortdelay);`
		1238	`poperror();`
		1239	`}`
		1240	`ilock(ctlr);`
		1241	`}`
		1242	`if(qh->state != Qclose)`
		1243	`qh->state = Qidle;`
		1244	`qhlinktd(qh, nil);`
		1245	`ctlr->load -= load;`
		1246	`iunlock(ctlr);`
		1247	`}`
		1248
		1249	`/*`
		1250	`* Non iso I/O.`
		1251	`* To make it work for control transfers, the caller may`
		1252	`* lock the Qio for the entire control transfer.`
		1253	`*/`
		1254	`static long`
		1255	`epio(Ep ep, Qio io, void *a, long count, int mustlock)`
		1256	`{`
		1257	`Td td, ltd, td0, ntd;`
		1258	`Ctlr *ctlr;`
		1259	`Qh* qh;`
		1260	`long n, tot;`
		1261	`char buf[128];`
		1262	`uchar *c;`
		1263	`int saved, ntds, tmout;`
		1264	`ulong load;`
		1265	`char *err;`
		1266
		1267	`qh = io->qh;`
		1268	`ctlr = ep->hp->aux;`
		1269	`io->debug = ep->debug;`
		1270	`tmout = ep->tmout;`
		1271	`ddeprint("epio: %s ep%d.%d io %#p count %ld load %uld\n",`
		1272	`io->tok == Tdtokin ? "in" : "out",`
		1273	`ep->dev->nb, ep->nb, io, count, ctlr->load);`
		1274	`if((debug > 1 \|\| ep->debug > 1) && io->tok != Tdtokin){`
		1275	`seprintdata(buf, buf+sizeof(buf), a, count);`
		1276	`print("uchi epio: user data: %s\n", buf);`
		1277	`}`
		1278	`if(mustlock){`
		1279	`qlock(io);`
		1280	`if(waserror()){`
		1281	`qunlock(io);`
		1282	`nexterror();`
		1283	`}`
		1284	`}`
		1285	`io->err = nil;`
		1286	`ilock(ctlr);`
		1287	`if(qh->state == Qclose){ /* Tds released by cancelio */`
		1288	`iunlock(ctlr);`
		1289	`error(io->err ? io->err : Eio);`
		1290	`}`
		1291	`if(qh->state != Qidle)`
		1292	`panic("epio: qh not idle");`
		1293	`qh->state = Qinstall;`
		1294	`iunlock(ctlr);`
		1295
		1296	`c = a;`
		1297	`td0 = ltd = nil;`
		1298	`load = tot = 0;`
		1299	`do{`
		1300	`n = ep->maxpkt;`
		1301	`if(count-tot < n)`
		1302	`n = count-tot;`
		1303	`if(c != nil && io->tok != Tdtokin)`
		1304	`td = epgettd(ep, io, Tdactive, c+tot, n);`
		1305	`else`
		1306	`td = epgettd(ep, io, Tdactive\|Tdspd, nil, n);`
		1307	`if(td0 == nil)`
		1308	`td0 = td;`
		1309	`else`
		1310	`tdlinktd(ltd, td);`
		1311	`ltd = td;`
		1312	`tot += n;`
		1313	`load += ep->load;`
		1314	`}while(tot < count);`
		1315	`if(td0 == nil \|\| ltd == nil)`
		1316	`panic("epio: no td");`
		1317
		1318	`ltd->csw \|= Tdioc; /* the last one interrupts */`
		1319	`ddeprint("uhci: load %uld ctlr load %uld\n", load, ctlr->load);`
		1320	`ilock(ctlr);`
		1321	`if(qh->state != Qclose){`
		1322	`io->iotime = TK2MS(MACHP(0)->ticks);`
		1323	`qh->state = Qrun;`
		1324	`coherence();`
		1325	`qhlinktd(qh, td0);`
		1326	`ctlr->load += load;`
		1327	`}`
		1328	`iunlock(ctlr);`
		1329
		1330	`epiowait(ctlr, io, tmout, load);`
		1331
		1332	`if(debug > 1 \|\| ep->debug > 1)`
		1333	`dumptd(td0, "epio: got tds: ");`
		1334
		1335	`tot = 0;`
		1336	`c = a;`
		1337	`saved = 0;`
		1338	`ntds = 0;`
		1339	`for(td = td0; td != nil; td = ntd){`
		1340	`ntds++;`
		1341	`/*`
		1342	`* Use td tok, not io tok, because of setup packets.`
		1343	`* Also, if the Td was stalled or active (previous Td`
		1344	`* was a short packet), we must save the toggle as it is.`
		1345	`*/`
		1346	`if(td->csw & (Tdstalled\|Tdactive)){`
		1347	`if(saved++ == 0)`
		1348	`io->toggle = td->token & Tddata1;`
		1349	`}else{`
		1350	`tot += td->ndata;`
		1351	`if(c != nil && tdtok(td) == Tdtokin && td->ndata > 0){`
		1352	`memmove(c, td->data, td->ndata);`
		1353	`c += td->ndata;`
		1354	`}`
		1355	`}`
		1356	`ntd = td->next;`
		1357	`tdfree(td);`
		1358	`}`
		1359	`err = io->err;`
		1360	`if(mustlock){`
		1361	`qunlock(io);`
		1362	`poperror();`
		1363	`}`
		1364	`ddeprint("epio: io %#p: %d tds: return %ld err '%s'\n",`
		1365	`io, ntds, tot, err);`
		1366	`if(err != nil)`
		1367	`error(err);`
		1368	`if(tot < 0)`
		1369	`error(Eio);`
		1370	`return tot;`
		1371	`}`
		1372
		1373	`/*`
		1374	`* halt condition was cleared on the endpoint. update our toggles.`
		1375	`*/`
		1376	`static void`
		1377	`clrhalt(Ep *ep)`
		1378	`{`
		1379	`Qio *io;`
		1380
		1381	`ep->clrhalt = 0;`
		1382	`switch(ep->ttype){`
		1383	`case Tbulk:`
		1384	`case Tintr:`
		1385	`io = ep->aux;`
		1386	`if(ep->mode != OREAD){`
		1387	`qlock(&io[OWRITE]);`
		1388	`io[OWRITE].toggle = Tddata0;`
		1389	`deprint("ep clrhalt for io %#p\n", io+OWRITE);`
		1390	`qunlock(&io[OWRITE]);`
		1391	`}`
		1392	`if(ep->mode != OWRITE){`
		1393	`qlock(&io[OREAD]);`
		1394	`io[OREAD].toggle = Tddata0;`
		1395	`deprint("ep clrhalt for io %#p\n", io+OREAD);`
		1396	`qunlock(&io[OREAD]);`
		1397	`}`
		1398	`break;`
		1399	`}`
		1400	`}`
		1401
		1402	`static long`
		1403	`epread(Ep ep, void a, long count)`
		1404	`{`
		1405	`Ctlio *cio;`
		1406	`Qio *io;`
		1407	`Isoio *iso;`
		1408	`char buf[160];`
		1409	`ulong delta;`
		1410
		1411	`ddeprint("uhci: epread\n");`
		1412	`if(ep->aux == nil)`
		1413	`panic("epread: not open");`
		1414
		1415	`switch(ep->ttype){`
		1416	`case Tctl:`
		1417	`cio = ep->aux;`
		1418	`qlock(cio);`
		1419	`if(waserror()){`
		1420	`qunlock(cio);`
		1421	`nexterror();`
		1422	`}`
		1423	`ddeprint("epread ctl ndata %d\n", cio->ndata);`
		1424	`if(cio->ndata < 0)`
		1425	`error("request expected");`
		1426	`else if(cio->ndata == 0){`
		1427	`cio->ndata = -1;`
		1428	`count = 0;`
		1429	`}else{`
		1430	`if(count > cio->ndata)`
		1431	`count = cio->ndata;`
		1432	`if(count > 0)`
		1433	`memmove(a, cio->data, count);`
		1434	`/* BUG for big transfers */`
		1435	`free(cio->data);`
		1436	`cio->data = nil;`
		1437	`cio->ndata = 0; /* signal EOF next time */`
		1438	`}`
		1439	`qunlock(cio);`
		1440	`poperror();`
		1441	`if(debug>1 \|\| ep->debug){`
		1442	`seprintdata(buf, buf+sizeof(buf), a, count);`
		1443	`print("epread: %s\n", buf);`
		1444	`}`
		1445	`return count;`
		1446	`case Tbulk:`
		1447	`io = ep->aux;`
		1448	`if(ep->clrhalt)`
		1449	`clrhalt(ep);`
		1450	`return epio(ep, &io[OREAD], a, count, 1);`
		1451	`case Tintr:`
		1452	`io = ep->aux;`
		1453	`delta = TK2MS(MACHP(0)->ticks) - io[OREAD].iotime + 1;`
		1454	`if(delta < ep->pollival / 2)`
		1455	`tsleep(&up->sleep, return0, 0, ep->pollival/2 - delta);`
		1456	`if(ep->clrhalt)`
		1457	`clrhalt(ep);`
		1458	`return epio(ep, &io[OREAD], a, count, 1);`
		1459	`case Tiso:`
		1460	`iso = ep->aux;`
		1461	`return episoread(ep, iso, a, count);`
		1462	`default:`
		1463	`panic("epread: bad ep ttype %d", ep->ttype);`
		1464	`}`
		1465	`return -1;`
		1466	`}`
		1467
		1468	`/*`
		1469	`* Control transfers are one setup write (data0)`
		1470	`* plus zero or more reads/writes (data1, data0, ...)`
		1471	`* plus a final write/read with data1 to ack.`
		1472	`* For both host to device and device to host we perform`
		1473	`* the entire transfer when the user writes the request,`
		1474	`* and keep any data read from the device for a later read.`
		1475	`* We call epio three times instead of placing all Tds at`
		1476	`* the same time because doing so leads to crc/tmout errors`
		1477	`* for some devices.`
		1478	`* Upon errors on the data phase we must still run the status`
		1479	`* phase or the device may cease responding in the future.`
		1480	`*/`
		1481	`static long`
		1482	`epctlio(Ep ep, Ctlio cio, void *a, long count)`
		1483	`{`
		1484	`uchar *c;`
		1485	`long len;`
		1486
		1487	`ddeprint("epctlio: cio %#p ep%d.%d count %ld\n",`
		1488	`cio, ep->dev->nb, ep->nb, count);`
		1489	`if(count < Rsetuplen)`
		1490	`error("short usb comand");`
		1491	`qlock(cio);`
		1492	`free(cio->data);`
		1493	`cio->data = nil;`
		1494	`cio->ndata = 0;`
		1495	`if(waserror()){`
		1496	`qunlock(cio);`
		1497	`free(cio->data);`
		1498	`cio->data = nil;`
		1499	`cio->ndata = 0;`
		1500	`nexterror();`
		1501	`}`
		1502
		1503	`/* set the address if unset and out of configuration state */`
		1504	`if(ep->dev->state != Dconfig && ep->dev->state != Dreset)`
		1505	`if(cio->usbid == 0)`
		1506	`cio->usbid = ((ep->nb&Epmax)<<7)\|(ep->dev->nb&Devmax);`
		1507	`c = a;`
		1508	`cio->tok = Tdtoksetup;`
		1509	`cio->toggle = Tddata0;`
		1510	`if(epio(ep, cio, a, Rsetuplen, 0) < Rsetuplen)`
		1511	`error(Eio);`
		1512	`a = c + Rsetuplen;`
		1513	`count -= Rsetuplen;`
		1514
		1515	`cio->toggle = Tddata1;`
		1516	`if(c[Rtype] & Rd2h){`
		1517	`cio->tok = Tdtokin;`
		1518	`len = GET2(c+Rcount);`
		1519	`if(len <= 0)`
		1520	`error("bad length in d2h request");`
		1521	`if(len > Maxctllen)`
		1522	`error("d2h data too large to fit in uhci");`
		1523	`a = cio->data = smalloc(len+1);`
		1524	`}else{`
		1525	`cio->tok = Tdtokout;`
		1526	`len = count;`
		1527	`}`
		1528	`if(len > 0)`
		1529	`if(waserror())`
		1530	`len = -1;`
		1531	`else{`
		1532	`len = epio(ep, cio, a, len, 0);`
		1533	`poperror();`
		1534	`}`
		1535	`if(c[Rtype] & Rd2h){`
		1536	`count = Rsetuplen;`
		1537	`cio->ndata = len;`
		1538	`cio->tok = Tdtokout;`
		1539	`}else{`
		1540	`if(len < 0)`
		1541	`count = -1;`
		1542	`else`
		1543	`count = Rsetuplen + len;`
		1544	`cio->tok = Tdtokin;`
		1545	`}`
		1546	`cio->toggle = Tddata1;`
		1547	`epio(ep, cio, nil, 0, 0);`
		1548	`qunlock(cio);`
		1549	`poperror();`
		1550	`ddeprint("epctlio cio %#p return %ld\n", cio, count);`
		1551	`return count;`
		1552	`}`
		1553
		1554	`static long`
		1555	`epwrite(Ep ep, void a, long count)`
		1556	`{`
		1557	`Ctlio *cio;`
		1558	`Isoio *iso;`
		1559	`Qio *io;`
		1560	`ulong delta;`
		1561	`char *b;`
		1562	`int tot;`
		1563	`int nw;`
		1564
		1565	`ddeprint("uhci: epwrite ep%d.%d\n", ep->dev->nb, ep->nb);`
		1566	`if(ep->aux == nil)`
		1567	`panic("uhci: epwrite: not open");`
		1568	`switch(ep->ttype){`
		1569	`case Tctl:`
		1570	`cio = ep->aux;`
		1571	`return epctlio(ep, cio, a, count);`
		1572	`case Tbulk:`
		1573	`io = ep->aux;`
		1574	`if(ep->clrhalt)`
		1575	`clrhalt(ep);`
		1576	`/*`
		1577	`* Put at most Tdatomic Tds (512 bytes) at a time.`
		1578	`* Otherwise some devices produce babble errors.`
		1579	`*/`
		1580	`b = a;`
		1581	`for(tot = 0; tot < count ; tot += nw){`
		1582	`nw = count - tot;`
		1583	`if(nw > Tdatomic * ep->maxpkt)`
		1584	`nw = Tdatomic * ep->maxpkt;`
		1585	`nw = epio(ep, &io[OWRITE], b+tot, nw, 1);`
		1586	`}`
		1587	`return tot;`
		1588	`case Tintr:`
		1589	`io = ep->aux;`
		1590	`delta = TK2MS(MACHP(0)->ticks) - io[OWRITE].iotime + 1;`
		1591	`if(delta < ep->pollival)`
		1592	`tsleep(&up->sleep, return0, 0, ep->pollival - delta);`
		1593	`if(ep->clrhalt)`
		1594	`clrhalt(ep);`
		1595	`return epio(ep, &io[OWRITE], a, count, 1);`
		1596	`case Tiso:`
		1597	`iso = ep->aux;`
		1598	`return episowrite(ep, iso, a, count);`
		1599	`default:`
		1600	`panic("uhci: epwrite: bad ep ttype %d", ep->ttype);`
		1601	`}`
		1602	`return -1;`
		1603	`}`
		1604
		1605	`static void`
		1606	`isoopen(Ep *ep)`
		1607	`{`
		1608	`Ctlr *ctlr;`
		1609	`Isoio *iso;`
		1610	`int frno;`
		1611	`int i;`
		1612	`Td* td;`
		1613	`Td* ltd;`
		1614	`int size;`
		1615	`int left;`
		1616
		1617	`if(ep->mode == ORDWR)`
		1618	`error("iso i/o is half-duplex");`
		1619	`ctlr = ep->hp->aux;`
		1620	`iso = ep->aux;`
		1621	`iso->debug = ep->debug;`
		1622	`iso->next = nil; /* paranoia */`
		1623	`if(ep->mode == OREAD)`
		1624	`iso->tok = Tdtokin;`
		1625	`else`
		1626	`iso->tok = Tdtokout;`
		1627	`iso->usbid = ((ep->nb & Epmax)<<7)\|(ep->dev->nb & Devmax);`
		1628	`iso->state = Qidle;`
		1629	`iso->nframes = Nframes/ep->pollival;`
		1630	`if(iso->nframes < 3)`
		1631	`error("uhci isoopen bug"); /* we need at least 3 tds */`
		1632
		1633	`ilock(ctlr);`
		1634	`if(ctlr->load + ep->load > 800)`
		1635	`print("usb: uhci: bandwidth may be exceeded\n");`
		1636	`ctlr->load += ep->load;`
		1637	`ctlr->isoload += ep->load;`
		1638	`dprint("uhci: load %uld isoload %uld\n", ctlr->load, ctlr->isoload);`
		1639	`iunlock(ctlr);`
		1640
		1641	`/*`
		1642	`* From here on this cannot raise errors`
		1643	`* unless we catch them and release here all memory allocated.`
		1644	`*/`
		1645	`if(ep->maxpkt > Tdndata)`
		1646	`iso->data = smalloc(iso->nframes*ep->maxpkt);`
		1647	`ilock(ctlr);`
		1648	`frno = INS(Frnum) + 10; /* start 10ms ahead */`
		1649	`frno = TRUNC(frno, Nframes);`
		1650	`iunlock(ctlr);`
		1651	`iso->td0frno = frno;`
		1652	`ltd = nil;`
		1653	`left = 0;`
		1654	`for(i = 0; i < iso->nframes; i++){`
		1655	`td = iso->tdps[frno] = tdalloc();`
		1656	`if(ep->mode == OREAD)`
		1657	`size = ep->maxpkt;`
		1658	`else{`
		1659	`size = (ep->hz+left) * ep->pollival / 1000;`
		1660	`size *= ep->samplesz;`
		1661	`left = (ep->hz+left) * ep->pollival % 1000;`
		1662	`if(size > ep->maxpkt){`
		1663	`print("uhci: ep%d.%d: size > maxpkt\n",`
		1664	`ep->dev->nb, ep->nb);`
		1665	`print("size = %d max = %ld\n", size, ep->maxpkt);`
		1666	`size = ep->maxpkt;`
		1667	`}`
		1668	`}`
		1669	`if(size > Tdndata)`
		1670	`td->data = iso->data + i * ep->maxpkt;`
		1671	`else`
		1672	`td->data = td->sbuff;`
		1673	`td->buffer = PCIWADDR(td->data);`
		1674	`tdisoinit(iso, td, size);`
		1675	`if(ltd != nil)`
		1676	`ltd->next = td;`
		1677	`ltd = td;`
		1678	`frno = TRUNC(frno+ep->pollival, Nframes);`
		1679	`}`
		1680	`ltd->next = iso->tdps[iso->td0frno];`
		1681	`iso->tdi = iso->tdps[iso->td0frno];`
		1682	`iso->tdu = iso->tdi; /* read: right now; write: 1s ahead */`
		1683	`ilock(ctlr);`
		1684	`frno = iso->td0frno;`
		1685	`for(i = 0; i < iso->nframes; i++){`
		1686	`iso->tdps[frno]->link = ctlr->frames[frno];`
		1687	`frno = TRUNC(frno+ep->pollival, Nframes);`
		1688	`}`
		1689	`coherence();`
		1690	`frno = iso->td0frno;`
		1691	`for(i = 0; i < iso->nframes; i++){`
		1692	`ctlr->frames[frno] = PCIWADDR(iso->tdps[frno]);`
		1693	`frno = TRUNC(frno+ep->pollival, Nframes);`
		1694	`}`
		1695	`iso->next = ctlr->iso;`
		1696	`ctlr->iso = iso;`
		1697	`iso->state = Qdone;`
		1698	`iunlock(ctlr);`
		1699	`if(debug > 1 \|\| iso->debug >1)`
		1700	`isodump(iso, 0);`
		1701	`}`
		1702
		1703	`/*`
		1704	`* Allocate the endpoint and set it up for I/O`
		1705	`* in the controller. This must follow what's said`
		1706	`* in Ep regarding configuration, including perhaps`
		1707	`* the saved toggles (saved on a previous close of`
		1708	`* the endpoint data file by epclose).`
		1709	`*/`
		1710	`static void`
		1711	`epopen(Ep *ep)`
		1712	`{`
		1713	`Ctlr *ctlr;`
		1714	`Qh *cqh;`
		1715	`Qio *io;`
		1716	`Ctlio *cio;`
		1717	`int usbid;`
		1718
		1719	`ctlr = ep->hp->aux;`
		1720	`deprint("uhci: epopen ep%d.%d\n", ep->dev->nb, ep->nb);`
		1721	`if(ep->aux != nil)`
		1722	`panic("uhci: epopen called with open ep");`
		1723	`if(waserror()){`
		1724	`free(ep->aux);`
		1725	`ep->aux = nil;`
		1726	`nexterror();`
		1727	`}`
		1728	`if(ep->maxpkt > Tdmaxpkt){`
		1729	`print("uhci: maxkpkt too large: using %d\n", Tdmaxpkt);`
		1730	`ep->maxpkt = Tdmaxpkt;`
		1731	`}`
		1732	`cqh = ctlr->qh[ep->ttype];`
		1733	`switch(ep->ttype){`
		1734	`case Tnone:`
		1735	`error("endpoint not configured");`
		1736	`case Tiso:`
		1737	`ep->aux = smalloc(sizeof(Isoio));`
		1738	`isoopen(ep);`
		1739	`break;`
		1740	`case Tctl:`
		1741	`cio = ep->aux = smalloc(sizeof(Ctlio));`
		1742	`cio->debug = ep->debug;`
		1743	`cio->ndata = -1;`
		1744	`cio->data = nil;`
		1745	`if(ep->dev->isroot != 0 && ep->nb == 0) /* root hub */`
		1746	`break;`
		1747	`cio->qh = qhalloc(ctlr, cqh, cio, "epc");`
		1748	`break;`
		1749	`case Tbulk:`
		1750	`case Tintr:`
		1751	`io = ep->aux = smalloc(sizeof(Qio)*2);`
		1752	`io[OREAD].debug = io[OWRITE].debug = ep->debug;`
		1753	`usbid = ((ep->nb&Epmax)<<7)\|(ep->dev->nb &Devmax);`
		1754	`if(ep->mode != OREAD){`
		1755	`if(ep->toggle[OWRITE] != 0)`
		1756	`io[OWRITE].toggle = Tddata1;`
		1757	`else`
		1758	`io[OWRITE].toggle = Tddata0;`
		1759	`io[OWRITE].tok = Tdtokout;`
		1760	`io[OWRITE].qh = qhalloc(ctlr, cqh, io+OWRITE, "epw");`
		1761	`io[OWRITE].usbid = usbid;`
		1762	`}`
		1763	`if(ep->mode != OWRITE){`
		1764	`if(ep->toggle[OREAD] != 0)`
		1765	`io[OREAD].toggle = Tddata1;`
		1766	`else`
		1767	`io[OREAD].toggle = Tddata0;`
		1768	`io[OREAD].tok = Tdtokin;`
		1769	`io[OREAD].qh = qhalloc(ctlr, cqh, io+OREAD, "epr");`
		1770	`io[OREAD].usbid = usbid;`
		1771	`}`
		1772	`break;`
		1773	`}`
		1774	`if(debug>1 \|\| ep->debug)`
		1775	`dump(ep->hp);`
		1776	`deprint("uhci: epopen done\n");`
		1777	`poperror();`
		1778	`}`
		1779
		1780	`static void`
		1781	`cancelio(Ctlr ctlr, Qio io)`
		1782	`{`
		1783	`Qh *qh;`
		1784
		1785	`ilock(ctlr);`
		1786	`qh = io->qh;`
		1787	`if(io == nil \|\| io->qh == nil \|\| io->qh->state == Qclose){`
		1788	`iunlock(ctlr);`
		1789	`return;`
		1790	`}`
		1791	`dqprint("uhci: cancelio for qh %#p state %s\n",`
		1792	`qh, qhsname[qh->state]);`
		1793	`aborttds(qh);`
		1794	`qh->state = Qclose;`
		1795	`iunlock(ctlr);`
		1796	`if(!waserror()){`
		1797	`tsleep(&up->sleep, return0, 0, Abortdelay);`
		1798	`poperror();`
		1799	`}`
		1800
		1801	`wakeup(io);`
		1802	`qlock(io);`
		1803	`/* wait for epio if running */`
		1804	`qunlock(io);`
		1805
		1806	`qhfree(ctlr, qh);`
		1807	`io->qh = nil;`
		1808	`}`
		1809
		1810	`static void`
		1811	`cancelisoio(Ctlr ctlr, Isoio iso, int pollival, ulong load)`
		1812	`{`
		1813	`Isoio **il;`
		1814	`ulong *lp;`
		1815	`int i;`
		1816	`int frno;`
		1817	`Td *td;`
		1818
		1819	`ilock(ctlr);`
		1820	`if(iso->state == Qclose){`
		1821	`iunlock(ctlr);`
		1822	`return;`
		1823	`}`
		1824	`if(iso->state != Qrun && iso->state != Qdone)`
		1825	`panic("bad iso state");`
		1826	`iso->state = Qclose;`
		1827	`if(ctlr->isoload < load)`
		1828	`panic("uhci: low isoload");`
		1829	`ctlr->isoload -= load;`
		1830	`ctlr->load -= load;`
		1831	`for(il = &ctlr->iso; il != nil; il = &(il)->next)`
		1832	`if(*il == iso)`
		1833	`break;`
		1834	`if(*il == nil)`
		1835	`panic("isocancel: not found");`
		1836	`*il = iso->next;`
		1837	`frno = iso->td0frno;`
		1838	`for(i = 0; i < iso->nframes; i++){`
		1839	`td = iso->tdps[frno];`
		1840	`td->csw &= ~(Tdioc\|Tdactive);`
		1841	`for(lp=&ctlr->frames[frno]; !(*lp & Tdterm);`
		1842	`lp = &TPTR(*lp)->link)`
		1843	`if(TPTR(*lp) == td)`
		1844	`break;`
		1845	`if(*lp & Tdterm)`
		1846	`panic("cancelisoio: td not found");`
		1847	`*lp = td->link;`
		1848	`frno = TRUNC(frno+pollival, Nframes);`
		1849	`}`
		1850	`iunlock(ctlr);`
		1851
		1852	`/*`
		1853	`* wakeup anyone waiting for I/O and`
		1854	`* wait to be sure no I/O is in progress in the controller.`
		1855	`* and then wait to be sure episo-io is no longer running.`
		1856	`*/`
		1857	`wakeup(iso);`
		1858	`diprint("cancelisoio iso %#p waiting for I/O to cease\n", iso);`
		1859	`tsleep(&up->sleep, return0, 0, 5);`
		1860	`qlock(iso);`
		1861	`qunlock(iso);`
		1862	`diprint("cancelisoio iso %#p releasing iso\n", iso);`
		1863
		1864	`frno = iso->td0frno;`
		1865	`for(i = 0; i < iso->nframes; i++){`
		1866	`tdfree(iso->tdps[frno]);`
		1867	`iso->tdps[frno] = nil;`
		1868	`frno = TRUNC(frno+pollival, Nframes);`
		1869	`}`
		1870	`free(iso->data);`
		1871	`iso->data = nil;`
		1872	`}`
		1873
		1874	`static void`
		1875	`epclose(Ep *ep)`
		1876	`{`
		1877	`Ctlr *ctlr;`
		1878	`Ctlio *cio;`
		1879	`Isoio *iso;`
		1880	`Qio *io;`
		1881
		1882	`ctlr = ep->hp->aux;`
		1883	`deprint("uhci: epclose ep%d.%d\n", ep->dev->nb, ep->nb);`
		1884
		1885	`if(ep->aux == nil)`
		1886	`panic("uhci: epclose called with closed ep");`
		1887	`switch(ep->ttype){`
		1888	`case Tctl:`
		1889	`cio = ep->aux;`
		1890	`cancelio(ctlr, cio);`
		1891	`free(cio->data);`
		1892	`cio->data = nil;`
		1893	`break;`
		1894	`case Tbulk:`
		1895	`case Tintr:`
		1896	`io = ep->aux;`
		1897	`ep->toggle[OREAD] = ep->toggle[OWRITE] = 0;`
		1898	`if(ep->mode != OWRITE){`
		1899	`cancelio(ctlr, &io[OREAD]);`
		1900	`if(io[OREAD].toggle == Tddata1)`
		1901	`ep->toggle[OREAD] = 1;`
		1902	`}`
		1903	`if(ep->mode != OREAD){`
		1904	`cancelio(ctlr, &io[OWRITE]);`
		1905	`if(io[OWRITE].toggle == Tddata1)`
		1906	`ep->toggle[OWRITE] = 1;`
		1907	`}`
		1908	`break;`
		1909	`case Tiso:`
		1910	`iso = ep->aux;`
		1911	`cancelisoio(ctlr, iso, ep->pollival, ep->load);`
		1912	`break;`
		1913	`default:`
		1914	`panic("epclose: bad ttype %d", ep->ttype);`
		1915	`}`
		1916
		1917	`free(ep->aux);`
		1918	`ep->aux = nil;`
		1919
		1920	`}`
		1921
		1922	`static char*`
		1923	`seprintep(char s, char e, Ep *ep)`
		1924	`{`
		1925	`Ctlio *cio;`
		1926	`Qio *io;`
		1927	`Isoio *iso;`
		1928	`Ctlr *ctlr;`
		1929
		1930	`ctlr = ep->hp->aux;`
		1931	`ilock(ctlr);`
		1932	`if(ep->aux == nil){`
		1933	`*s = 0;`
		1934	`iunlock(ctlr);`
		1935	`return s;`
		1936	`}`
		1937	`switch(ep->ttype){`
		1938	`case Tctl:`
		1939	`cio = ep->aux;`
		1940	`s = seprint(s,e,"cio %#p qh %#p"`
		1941	`" id %#x tog %#x tok %#x err %s\n",`
		1942	`cio, cio->qh, cio->usbid, cio->toggle,`
		1943	`cio->tok, cio->err);`
		1944	`break;`
		1945	`case Tbulk:`
		1946	`case Tintr:`
		1947	`io = ep->aux;`
		1948	`if(ep->mode != OWRITE)`
		1949	`s = seprint(s,e,"r: qh %#p id %#x tog %#x tok %#x err %s\n",`
		1950	`io[OREAD].qh, io[OREAD].usbid, io[OREAD].toggle,`
		1951	`io[OREAD].tok, io[OREAD].err);`
		1952	`if(ep->mode != OREAD)`
		1953	`s = seprint(s,e,"w: qh %#p id %#x tog %#x tok %#x err %s\n",`
		1954	`io[OWRITE].qh, io[OWRITE].usbid, io[OWRITE].toggle,`
		1955	`io[OWRITE].tok, io[OWRITE].err);`
		1956	`break;`
		1957	`case Tiso:`
		1958	`iso = ep->aux;`
		1959	`s = seprint(s,e,"iso %#p id %#x tok %#x tdu %#p tdi %#p err %s\n",`
		1960	`iso, iso->usbid, iso->tok, iso->tdu, iso->tdi, iso->err);`
		1961	`break;`
		1962	`}`
		1963	`iunlock(ctlr);`
		1964	`return s;`
		1965	`}`
		1966
		1967	`static int`
		1968	`portenable(Hci *hp, int port, int on)`
		1969	`{`
		1970	`int s;`
		1971	`int ioport;`
		1972	`Ctlr *ctlr;`
		1973
		1974	`ctlr = hp->aux;`
		1975	`dprint("uhci: %#x port %d enable=%d\n", ctlr->port, port, on);`
		1976	`ioport = PORT(port-1);`
		1977	`qlock(&ctlr->portlck);`
		1978	`if(waserror()){`
		1979	`qunlock(&ctlr->portlck);`
		1980	`nexterror();`
		1981	`}`
		1982	`ilock(ctlr);`
		1983	`s = INS(ioport);`
		1984	`if(on)`
		1985	`OUTS(ioport, s \| PSenable);`
		1986	`else`
		1987	`OUTS(ioport, s & ~PSenable);`
		1988	`microdelay(64);`
		1989	`iunlock(ctlr);`
		1990	`tsleep(&up->sleep, return0, 0, Enabledelay);`
		1991	`dprint("uhci %#ux port %d enable=%d: sts %#x\n",`
		1992	`ctlr->port, port, on, INS(ioport));`
		1993	`qunlock(&ctlr->portlck);`
		1994	`poperror();`
		1995	`return 0;`
		1996	`}`
		1997
		1998	`static int`
		1999	`portreset(Hci *hp, int port, int on)`
		2000	`{`
		2001	`int i, p;`
		2002	`Ctlr *ctlr;`
		2003
		2004	`if(on == 0)`
		2005	`return 0;`
		2006	`ctlr = hp->aux;`
		2007	`dprint("uhci: %#ux port %d reset\n", ctlr->port, port);`
		2008	`p = PORT(port-1);`
		2009	`ilock(ctlr);`
		2010	`OUTS(p, PSreset);`
		2011	`delay(50);`
		2012	`OUTS(p, INS(p) & ~PSreset);`
		2013	`OUTS(p, INS(p) \| PSenable);`
		2014	`microdelay(64);`
		2015	`for(i=0; i<1000 && (INS(p) & PSenable) == 0; i++)`
		2016	`;`
		2017	`OUTS(p, (INS(p) & ~PSreset)\|PSenable);`
		2018	`iunlock(ctlr);`
		2019	`dprint("uhci %#ux after port %d reset: sts %#x\n",`
		2020	`ctlr->port, port, INS(p));`
		2021	`return 0;`
		2022	`}`
		2023
		2024	`static int`
		2025	`portstatus(Hci *hp, int port)`
		2026	`{`
		2027	`int s;`
		2028	`int r;`
		2029	`int ioport;`
		2030	`Ctlr *ctlr;`
		2031
		2032	`ctlr = hp->aux;`
		2033	`ioport = PORT(port-1);`
		2034	`qlock(&ctlr->portlck);`
		2035	`if(waserror()){`
		2036	`iunlock(ctlr);`
		2037	`qunlock(&ctlr->portlck);`
		2038	`nexterror();`
		2039	`}`
		2040	`ilock(ctlr);`
		2041	`s = INS(ioport);`
		2042	`if(s & (PSstatuschg \| PSchange)){`
		2043	`OUTS(ioport, s);`
		2044	`ddprint("uhci %#ux port %d status %#x\n", ctlr->port, port, s);`
		2045	`}`
		2046	`iunlock(ctlr);`
		2047	`qunlock(&ctlr->portlck);`
		2048	`poperror();`
		2049
		2050	`/*`
		2051	`* We must return status bits as a`
		2052	`* get port status hub request would do.`
		2053	`*/`
		2054	`r = 0;`
		2055	`if(s & PSpresent)`
		2056	`r \|= HPpresent;`
		2057	`if(s & PSenable)`
		2058	`r \|= HPenable;`
		2059	`if(s & PSsuspend)`
		2060	`r \|= HPsuspend;`
		2061	`if(s & PSreset)`
		2062	`r \|= HPreset;`
		2063	`if(s & PSslow)`
		2064	`r \|= HPslow;`
		2065	`if(s & PSstatuschg)`
		2066	`r \|= HPstatuschg;`
		2067	`if(s & PSchange)`
		2068	`r \|= HPchange;`
		2069	`return r;`
		2070	`}`
		2071
		2072	`static void`
		2073	`scanpci(void)`
		2074	`{`
		2075	`static int already = 0;`
		2076	`int io;`
		2077	`int i;`
		2078	`Ctlr *ctlr;`
		2079	`Pcidev *p;`
		2080
		2081	`if(already)`
		2082	`return;`
		2083	`already = 1;`
		2084	`p = nil;`
		2085	`while(p = pcimatch(p, 0, 0)){`
		2086	`/*`
		2087	`* Find UHCI controllers (Programming Interface = 0).`
		2088	`*/`
		2089	`if(p->ccrb != Pcibcserial \|\| p->ccru != Pciscusb)`
		2090	`continue;`
		2091	`switch(p->ccrp){`
		2092	`case 0:`
		2093	`io = p->mem[4].bar & ~0x0F;`
		2094	`break;`
		2095	`default:`
		2096	`continue;`
		2097	`}`
		2098	`if(io == 0){`
		2099	`print("usbuhci: %#x %#x: failed to map registers\n",`
		2100	`p->vid, p->did);`
		2101	`continue;`
		2102	`}`
		2103	`if(ioalloc(io, p->mem[4].size, 0, "usbuhci") < 0){`
		2104	`print("usbuhci: port %#ux in use\n", io);`
		2105	`continue;`
		2106	`}`
		2107	`if(p->intl == 0xFF \|\| p->intl == 0){`
		2108	`print("usbuhci: no irq assigned for port %#ux\n", io);`
		2109	`continue;`
		2110	`}`
		2111
		2112	`dprint("uhci: %#x %#x: port %#ux size %#x irq %d\n",`
		2113	`p->vid, p->did, io, p->mem[4].size, p->intl);`
		2114
		2115	`ctlr = malloc(sizeof(Ctlr));`
		2116	`if (ctlr == nil)`
		2117	`panic("uhci: out of memory");`
		2118	`ctlr->pcidev = p;`
		2119	`ctlr->port = io;`
		2120	`for(i = 0; i < Nhcis; i++)`
		2121	`if(ctlrs[i] == nil){`
		2122	`ctlrs[i] = ctlr;`
		2123	`break;`
		2124	`}`
		2125	`if(i == Nhcis)`
		2126	`print("uhci: bug: no more controllers\n");`
		2127	`}`
		2128	`}`
		2129
		2130	`static void`
		2131	`uhcimeminit(Ctlr *ctlr)`
		2132	`{`
		2133	`Td* td;`
		2134	`Qh *qh;`
		2135	`int frsize;`
		2136	`int i;`
		2137
		2138	`ctlr->qhs = ctlr->qh[Tctl] = qhalloc(ctlr, nil, nil, "CTL");`
		2139	`ctlr->qh[Tintr] = qhalloc(ctlr, ctlr->qh[Tctl], nil, "INT");`
		2140	`ctlr->qh[Tbulk] = qhalloc(ctlr, ctlr->qh[Tintr], nil, "BLK");`
		2141
		2142	`/* idle Td from dummy Qh at the end. looped back to itself */`
		2143	`/* This is a workaround for PIIX4 errata 29773804.pdf */`
		2144	`qh = qhalloc(ctlr, ctlr->qh[Tbulk], nil, "BWS");`
		2145	`td = tdalloc();`
		2146	`td->link = PCIWADDR(td);`
		2147	`qhlinktd(qh, td);`
		2148
		2149	`/* loop (hw only) from the last qh back to control xfers.`
		2150	`* this may be done only for some of them. Disable until ehci comes.`
		2151	`*/`
		2152	`if(0)`
		2153	`qh->link = PCIWADDR(ctlr->qhs);`
		2154
		2155	`frsize = Nframes*sizeof(ulong);`
		2156	`ctlr->frames = xspanalloc(frsize, frsize, 0);`
		2157	`if(ctlr->frames == nil)`
		2158	`panic("uhci reset: no memory");`
		2159
		2160	`ctlr->iso = nil;`
		2161	`for(i = 0; i < Nframes; i++)`
		2162	`ctlr->frames[i] = PCIWADDR(ctlr->qhs)\|QHlinkqh;`
		2163	`OUTL(Flbaseadd, PCIWADDR(ctlr->frames));`
		2164	`OUTS(Frnum, 0);`
		2165	`dprint("uhci %#ux flb %#ulx frno %#ux\n", ctlr->port,`
		2166	`INL(Flbaseadd), INS(Frnum));`
		2167	`}`
		2168
		2169	`static void`
		2170	`init(Hci *hp)`
		2171	`{`
		2172	`Ctlr *ctlr;`
		2173	`int sts;`
		2174	`int i;`
		2175
		2176	`ctlr = hp->aux;`
		2177	`dprint("uhci %#ux init\n", ctlr->port);`
		2178	`coherence();`
		2179	`ilock(ctlr);`
		2180	`OUTS(Usbintr, Itmout\|Iresume\|Ioc\|Ishort);`
		2181	`uhcirun(ctlr, 1);`
		2182	`dprint("uhci: init: cmd %#ux sts %#ux sof %#ux",`
		2183	`INS(Cmd), INS(Status), INS(SOFmod));`
		2184	`dprint(" flb %#ulx frno %#ux psc0 %#ux psc1 %#ux",`
		2185	`INL(Flbaseadd), INS(Frnum), INS(PORT(0)), INS(PORT(1)));`
		2186	`/* guess other ports */`
		2187	`for(i = 2; i < 6; i++){`
		2188	`sts = INS(PORT(i));`
		2189	`if(sts != 0xFFFF && (sts & PSreserved1) == 1){`
		2190	`dprint(" psc%d %#ux", i, sts);`
		2191	`hp->nports++;`
		2192	`}else`
		2193	`break;`
		2194	`}`
		2195	`for(i = 0; i < hp->nports; i++)`
		2196	`OUTS(PORT(i), 0);`
		2197	`iunlock(ctlr);`
		2198	`}`
		2199
		2200	`static void`
		2201	`uhcireset(Ctlr *ctlr)`
		2202	`{`
		2203	`int i;`
		2204	`int sof;`
		2205
		2206	`ilock(ctlr);`
		2207	`dprint("uhci %#ux reset\n", ctlr->port);`
		2208
		2209	`/*`
		2210	`* Turn off legacy mode. Some controllers won't`
		2211	`* interrupt us as expected otherwise.`
		2212	`*/`
		2213	`uhcirun(ctlr, 0);`
		2214	`pcicfgw16(ctlr->pcidev, 0xc0, 0x2000);`
		2215
		2216	`OUTS(Usbintr, 0);`
		2217	`sof = INB(SOFmod);`
		2218	`uhcicmd(ctlr, Cgreset); /* global reset */`
		2219	`delay(Resetdelay);`
		2220	`uhcicmd(ctlr, 0); /* all halt */`
		2221	`uhcicmd(ctlr, Chcreset); /* controller reset */`
		2222	`for(i = 0; i < 100; i++){`
		2223	`if((INS(Cmd) & Chcreset) == 0)`
		2224	`break;`
		2225	`delay(1);`
		2226	`}`
		2227	`if(i == 100)`
		2228	`print("uhci %#x controller reset timed out\n", ctlr->port);`
		2229	`OUTB(SOFmod, sof);`
		2230	`iunlock(ctlr);`
		2231	`}`
		2232
		2233	`static void`
		2234	`setdebug(Hci*, int d)`
		2235	`{`
		2236	`debug = d;`
		2237	`}`
		2238
		2239	`static void`
		2240	`shutdown(Hci *hp)`
		2241	`{`
		2242	`Ctlr *ctlr;`
		2243
		2244	`ctlr = hp->aux;`
		2245
		2246	`ilock(ctlr);`
		2247	`uhcirun(ctlr, 0);`
		2248	`delay(100);`
		2249	`iunlock(ctlr);`
		2250	`}`
		2251
		2252	`static int`
		2253	`reset(Hci *hp)`
		2254	`{`
		2255	`static Lock resetlck;`
		2256	`int i;`
		2257	`Ctlr *ctlr;`
		2258	`Pcidev *p;`
		2259
		2260	`if(getconf("*nousbuhci"))`
		2261	`return -1;`
		2262
		2263	`ilock(&resetlck);`
		2264	`scanpci();`
		2265
		2266	`/*`
		2267	`* Any adapter matches if no hp->port is supplied,`
		2268	`* otherwise the ports must match.`
		2269	`*/`
		2270	`ctlr = nil;`
		2271	`for(i = 0; i < Nhcis && ctlrs[i] != nil; i++){`
		2272	`ctlr = ctlrs[i];`
		2273	`if(ctlr->active == 0)`
		2274	`if(hp->port == 0 \|\| hp->port == ctlr->port){`
		2275	`ctlr->active = 1;`
		2276	`break;`
		2277	`}`
		2278	`}`
		2279	`iunlock(&resetlck);`
		2280	`if(ctlrs[i] == nil \|\| i == Nhcis)`
		2281	`return -1;`
		2282
		2283	`p = ctlr->pcidev;`
		2284	`hp->aux = ctlr;`
		2285	`hp->port = ctlr->port;`
		2286	`hp->irq = p->intl;`
		2287	`hp->tbdf = p->tbdf;`
		2288	`hp->nports = 2; /* default */`
		2289
		2290	`uhcireset(ctlr);`
		2291	`uhcimeminit(ctlr);`
		2292
		2293	`/*`
		2294	`* Linkage to the generic HCI driver.`
		2295	`*/`
		2296	`hp->init = init;`
		2297	`hp->dump = dump;`
		2298	`hp->interrupt = interrupt;`
		2299	`hp->epopen = epopen;`
		2300	`hp->epclose = epclose;`
		2301	`hp->epread = epread;`
		2302	`hp->epwrite = epwrite;`
		2303	`hp->seprintep = seprintep;`
		2304	`hp->portenable = portenable;`
		2305	`hp->portreset = portreset;`
		2306	`hp->portstatus = portstatus;`
		2307	`hp->shutdown = shutdown;`
		2308	`hp->debug = setdebug;`
		2309	`hp->type = "uhci";`
		2310	`return 0;`
		2311	`}`
		2312
		2313	`void`
		2314	`usbuhcilink(void)`
		2315	`{`
		2316	`addhcitype("uhci", reset);`
		2317	`}`

Subversion Repositories planix.SVN

(root)/os/branches/planix-v0/sys/src/9/pc/usbuhci.c – Rev 9