Subversion Repositories planix.SVN

Rev

Details | Last modification | View Log | RSS feed

Rev Author Line No. Line
2 - 1
/*
2
 * USB Enhanced Host Controller Interface (EHCI) driver
3
 * High speed USB 2.0.
4
 *
5
 * Note that all of our unlock routines call coherence.
6
 *
7
 * BUGS:
8
 * - Too many delays and ilocks.
9
 * - bandwidth admission control must be done per-frame.
10
 * - requires polling (some controllers miss interrupts).
11
 * - must warn of power overruns.
12
 */
13
 
14
#include	"u.h"
15
#include	"../port/lib.h"
16
#include	"mem.h"
17
#include	"dat.h"
18
#include	"fns.h"
19
#include	"io.h"
20
#include	"../port/error.h"
21
#include	"../port/usb.h"
22
#include	"../port/portusbehci.h"
23
#include	"usbehci.h"
24
#include	"uncached.h"
25
 
26
#define diprint		if(ehcidebug || iso->debug)print
27
#define ddiprint	if(ehcidebug>1 || iso->debug>1)print
28
#define dqprint		if(ehcidebug || (qh->io && qh->io->debug))print
29
#define ddqprint	if(ehcidebug>1 || (qh->io && qh->io->debug>1))print
30
 
31
#define TRUNC(x, sz)	((x) & ((sz)-1))
32
#define LPTR(q)		((ulong*)KADDR((q) & ~0x1F))
33
 
34
typedef struct Ctlio Ctlio;
35
typedef union Ed Ed;
36
typedef struct Edpool Edpool;
37
typedef struct Itd Itd;
38
typedef struct Qio Qio;
39
typedef struct Qtd Qtd;
40
typedef struct Sitd Sitd;
41
typedef struct Td Td;
42
 
43
/*
44
 * EHCI interface registers and bits
45
 */
46
enum
47
{
48
	/* Queue states (software) */
49
	Qidle		= 0,
50
	Qinstall,
51
	Qrun,
52
	Qdone,
53
	Qclose,
54
	Qfree,
55
 
56
	Enabledelay	= 100,		/* waiting for a port to enable */
57
	Abortdelay	= 5,		/* delay after cancelling Tds (ms) */
58
 
59
	Incr		= 64,		/* for pools of Tds, Qhs, etc. */
60
	Align		= 128,		/* in bytes for all those descriptors */
61
 
62
	/* Keep them as a power of 2, lower than ctlr->nframes */
63
	/* Also, keep Nisoframes >= Nintrleafs */
64
	Nintrleafs	= 32,		/* nb. of leaf frames in intr. tree */
65
	Nisoframes	= 64,		/* nb. of iso frames (in window) */
66
 
67
	/*
68
	 * HW constants
69
	 */
70
 
71
	/* Itd bits (csw[]) */
72
	Itdactive	= 0x80000000,	/* execution enabled */
73
	Itddberr	= 0x40000000,	/* data buffer error */
74
	Itdbabble	= 0x20000000,	/* babble error */
75
	Itdtrerr	= 0x10000000,	/* transaction error */
76
	Itdlenshift	= 16,		/* transaction length */
77
	Itdlenmask	= 0xFFF,
78
	Itdioc		= 0x00008000,	/* interrupt on complete */
79
	Itdpgshift	= 12,		/* page select field */
80
	Itdoffshift	= 0,		/* transaction offset */
81
	/* Itd bits, buffer[] */
82
	Itdepshift	= 8,		/* endpoint address (buffer[0]) */
83
	Itddevshift	= 0,		/* device address (buffer[0]) */
84
	Itdin		= 0x800,	/* is input (buffer[1]) */
85
	Itdout		= 0,
86
	Itdmaxpktshift	= 0,		/* max packet (buffer[1]) */
87
	Itdntdsshift	= 0,		/* nb. of tds per µframe (buffer[2]) */
88
 
89
	Itderrors	= Itddberr|Itdbabble|Itdtrerr,
90
 
91
	/* Sitd bits (epc) */
92
	Stdin		= 0x80000000,	/* input direction */
93
	Stdportshift	= 24,		/* hub port number */
94
	Stdhubshift	= 16,		/* hub address */
95
	Stdepshift	= 8,		/* endpoint address */
96
	Stddevshift	= 0,		/* device address */
97
	/* Sitd bits (mfs) */
98
	Stdssmshift	= 0,		/* split start mask */
99
	Stdscmshift	= 8,		/* split complete mask */
100
	/* Sitd bits (csw) */
101
	Stdioc		= 0x80000000,	/* interrupt on complete */
102
	Stdpg		= 0x40000000,	/* page select */
103
	Stdlenshift	= 16,		/* total bytes to transfer */
104
	Stdlenmask	= 0x3FF,
105
	Stdactive	= 0x00000080,	/* active */
106
	Stderr		= 0x00000040,	/* tr. translator error */
107
	Stddberr	= 0x00000020,	/* data buffer error */
108
	Stdbabble	= 0x00000010,	/* babble error */
109
	Stdtrerr	= 0x00000008,	/* transaction error */
110
	Stdmmf		= 0x00000004,	/* missed µframe */
111
	Stddcs		= 0x00000002,	/* do complete split */
112
 
113
	Stderrors	= Stderr|Stddberr|Stdbabble|Stdtrerr|Stdmmf,
114
 
115
	/* Sitd bits buffer[1] */
116
	Stdtpall	= 0x00000000,	/* all payload here (188 bytes) */
117
	Stdtpbegin	= 0x00000008,	/* first payload for fs trans. */
118
	Stdtcntmask	= 0x00000007,	/* T-count */
119
 
120
	/* Td bits (csw) */
121
	Tddata1		= 0x80000000,	/* data toggle 1 */
122
	Tddata0		= 0x00000000,	/* data toggle 0 */
123
	Tdlenshift	= 16,		/* total bytes to transfer */
124
	Tdlenmask	= 0x7FFF,
125
	Tdmaxpkt	= 0x5000,	/* max buffer for a Td */
126
	Tdioc		= 0x00008000,	/* interrupt on complete */
127
	Tdpgshift	= 12,		/* current page */
128
	Tdpgmask	= 7,
129
	Tderr1		= 0x00000400,	/* bit 0 of error counter */
130
	Tderr2		= 0x00000800,	/* bit 1 of error counter */
131
	Tdtokout	= 0x00000000,	/* direction out */
132
	Tdtokin		= 0x00000100,	/* direction in */
133
	Tdtoksetup	= 0x00000200,	/* setup packet */
134
	Tdtok		= 0x00000300,	/* token bits */
135
	Tdactive		= 0x00000080,	/* active */
136
	Tdhalt		= 0x00000040,	/* halted */
137
	Tddberr		= 0x00000020,	/* data buffer error */
138
	Tdbabble	= 0x00000010,	/* babble error */
139
	Tdtrerr		= 0x00000008,	/* transaction error */
140
	Tdmmf		= 0x00000004,	/* missed µframe */
141
	Tddcs		= 0x00000002,	/* do complete split */
142
	Tdping		= 0x00000001,	/* do ping */
143
 
144
	Tderrors	= Tdhalt|Tddberr|Tdbabble|Tdtrerr|Tdmmf,
145
 
146
	/* Qh bits (eps0) */
147
	Qhrlcmask	= 0xF,		/* nak reload count */
148
	Qhrlcshift	= 28,		/* nak reload count */
149
	Qhnhctl		= 0x08000000,	/* not-high speed ctl */
150
	Qhmplmask	= 0x7FF,	/* max packet */
151
	Qhmplshift	= 16,
152
	Qhhrl		= 0x00008000,	/* head of reclamation list */
153
	Qhdtc		= 0x00004000,	/* data toggle ctl. */
154
	Qhint		= 0x00000080,	/* inactivate on next transition */
155
	Qhspeedmask	= 0x00003000,	/* speed bits */
156
	Qhfull		= 0x00000000,	/* full speed */
157
	Qhlow		= 0x00001000,	/* low speed */
158
	Qhhigh		= 0x00002000,	/* high speed */
159
 
160
	/* Qh bits (eps1) */
161
	Qhmultshift	= 30,		/* multiple tds per µframe */
162
	Qhmultmask	= 3,
163
	Qhportshift	= 23,		/* hub port number */
164
	Qhhubshift	= 16,		/* hub address */
165
	Qhscmshift	= 8,		/* split completion mask bits */
166
	Qhismshift	= 0,		/* interrupt sched. mask bits */
167
};
168
 
169
/*
170
 * Endpoint tree (software)
171
 */
172
struct Qtree
173
{
174
	int	nel;
175
	int	depth;
176
	ulong*	bw;
177
	Qh**	root;
178
};
179
 
180
/*
181
 * One per endpoint per direction, to control I/O.
182
 */
183
struct Qio
184
{
185
	QLock;			/* for the entire I/O process */
186
	Rendez;			/* wait for completion */
187
	Qh*	qh;		/* Td list (field const after init) */
188
	int	usbid;		/* usb address for endpoint/device */
189
	int	toggle;		/* Tddata0/Tddata1 */
190
	int	tok;		/* Tdtoksetup, Tdtokin, Tdtokout */
191
	ulong	iotime;		/* last I/O time; to hold interrupt polls */
192
	int	debug;		/* debug flag from the endpoint */
193
	char*	err;		/* error string */
194
	char*	tag;		/* debug (no room in Qh for this) */
195
	ulong	bw;
196
};
197
 
198
struct Ctlio
199
{
200
	Qio;			/* a single Qio for each RPC */
201
	uchar*	data;		/* read from last ctl req. */
202
	int	ndata;		/* number of bytes read */
203
};
204
 
205
struct Isoio
206
{
207
	QLock;
208
	Rendez;			/* wait for space/completion/errors */
209
	int	usbid;		/* address used for device/endpoint */
210
	int	tok;		/* Tdtokin or Tdtokout */
211
	int	state;		/* Qrun -> Qdone -> Qrun... -> Qclose */
212
	int	nframes;	/* number of frames ([S]Itds) used */
213
	uchar*	data;		/* iso data buffers if not embedded */
214
	char*	err;		/* error string */
215
	int	nerrs;		/* nb of consecutive I/O errors */
216
	ulong	maxsize;	/* ntds * ep->maxpkt */
217
	long	nleft;		/* number of bytes left from last write */
218
	int	debug;		/* debug flag from the endpoint */
219
	int	hs;		/* is high speed? */
220
	Isoio*	next;		/* in list of active Isoios */
221
	ulong	td0frno;	/* first frame used in ctlr */
222
	union{
223
		Itd*	tdi;	/* next td processed by interrupt */
224
		Sitd*	stdi;
225
	};
226
	union{
227
		Itd*	tdu;	/* next td for user I/O in tdps */
228
		Sitd*	stdu;
229
	};
230
	union{
231
		Itd**	itdps;	/* itdps[i]: ptr to Itd for i-th frame or nil */
232
		Sitd**	sitdps;	/* sitdps[i]: ptr to Sitd for i-th frame or nil */
233
		ulong**	tdps;	/* same thing, as seen by hw */
234
	};
235
};
236
 
237
struct Edpool
238
{
239
	Lock;
240
	Ed*	free;
241
	int	nalloc;
242
	int	ninuse;
243
	int	nfree;
244
};
245
 
246
/*
247
 * We use the 64-bit version for Itd, Sitd, Td, and Qh.
248
 * If the ehci is 64-bit capable it assumes we are using those
249
 * structures even when the system is 32 bits.
250
 */
251
 
252
/*
253
 * Iso transfer descriptor.  hw: 92 bytes, 108 bytes total
254
 * aligned to 32.
255
 */
256
struct Itd
257
{
258
	ulong	link;		/* to next hw struct */
259
	ulong	csw[8];		/* sts/length/pg/off. updated by hw */
260
	ulong	buffer[7];	/* buffer pointers, addrs, maxsz */
261
	ulong	xbuffer[7];	/* high 32 bits of buffer for 64-bits */
262
 
263
	ulong	_pad0;		/* pad to next cache line */
264
	/* cache-line boundary here */
265
 
266
	/* software */
267
	Itd*	next;
268
	ulong	ndata;		/* number of bytes in data */
269
	ulong	mdata;		/* max number of bytes in data */
270
	uchar*	data;
271
};
272
 
273
/*
274
 * Split transaction iso transfer descriptor.
275
 * hw: 36 bytes, 52 bytes total. aligned to 32.
276
 */
277
struct Sitd
278
{
279
	ulong	link;		/* to next hw struct */
280
	ulong	epc;		/* static endpoint state. addrs */
281
	ulong	mfs;		/* static endpoint state. µ-frame sched. */
282
	ulong	csw;		/* transfer state. updated by hw */
283
	ulong	buffer[2];	/* buf. ptr/offset. offset updated by hw */
284
				/* buf ptr/TP/Tcnt. TP/Tcnt updated by hw */
285
	ulong	blink;		/* back pointer */
286
	/* cache-line boundary after xbuffer[0] */
287
	ulong	xbuffer[2];	/* high 32 bits of buffer for 64-bits */
288
 
289
	/* software */
290
	Sitd*	next;
291
	ulong	ndata;		/* number of bytes in data */
292
	ulong	mdata;		/* max number of bytes in data */
293
	uchar*	data;
294
};
295
 
296
/*
297
 * Queue element transfer descriptor.
298
 * hw: first 52 bytes, total 68+sbuff bytes.  aligned to 32 bytes.
299
 */
300
struct Td
301
{
302
	ulong	nlink;		/* to next Td */
303
	ulong	alink;		/* alternate link to next Td */
304
	ulong	csw;		/* cmd/sts. updated by hw */
305
	ulong	buffer[5];	/* buf ptrs. offset updated by hw */
306
	/* cache-line boundary here */
307
	ulong	xbuffer[5];	/* high 32 bits of buffer for 64-bits */
308
 
309
	/* software */
310
	Td*	next;		/* in qh or Isoio or free list */
311
	ulong	ndata;		/* bytes available/used at data */
312
	uchar*	data;		/* pointer to actual data */
313
	uchar*	buff;		/* allocated data buffer or nil */
314
	uchar	sbuff[1];	/* first byte of embedded buffer */
315
};
316
 
317
/*
318
 * Queue head. Aligned to 32 bytes.
319
 * hw: first 68 bytes, 92 total.
320
 */
321
struct Qh
322
{
323
	ulong	link;		/* to next Qh in round robin */
324
	ulong	eps0;		/* static endpoint state. addrs */
325
	ulong	eps1;		/* static endpoint state. µ-frame sched. */
326
 
327
	/* updated by hw */
328
	ulong	tclink;		/* current Td (No Term bit here!) */
329
	ulong	nlink;		/* to next Td */
330
	ulong	alink;		/* alternate link to next Td */
331
	ulong	csw;		/* cmd/sts. updated by hw */
332
	/* cache-line boundary after buffer[0] */
333
	ulong	buffer[5];	/* buf ptrs. offset updated by hw */
334
	ulong	xbuffer[5];	/* high 32 bits of buffer for 64-bits */
335
 
336
	/* software */
337
	Qh*	next;		/* in controller list/tree of Qhs */
338
	int	state;		/* Qidle -> Qinstall -> Qrun -> Qdone | Qclose */
339
	Qio*	io;		/* for this queue */
340
	Td*	tds;		/* for this queue */
341
	int	sched;		/* slot for for intr. Qhs */
342
	Qh*	inext;		/* next in list of intr. qhs */
343
};
344
 
345
/*
346
 * We can avoid frame span traversal nodes if we don't span frames.
347
 * Just schedule transfers that can fit on the current frame and
348
 * wait a little bit otherwise.
349
 */
350
 
351
/*
352
 * Software. Ehci descriptors provided by pool.
353
 * There are soo few because we avoid using Fstn.
354
 */
355
union Ed
356
{
357
	Ed*	next;		/* in free list */
358
	Qh	qh;
359
	Td	td;
360
	Itd	itd;
361
	Sitd	sitd;
362
	uchar	align[Align];
363
};
364
 
365
int ehcidebug = 0;
366
 
367
static Edpool edpool;
368
static char* qhsname[] = { "idle", "install", "run", "done", "close", "FREE" };
369
 
370
Ecapio* ehcidebugcapio;
371
int ehcidebugport;
372
 
373
void
374
ehcirun(Ctlr *ctlr, int on)
375
{
376
	int i;
377
	Eopio *opio;
378
 
379
	ddprint("ehci %#p %s\n", ctlr->capio, on ? "starting" : "halting");
380
	opio = ctlr->opio;
381
	if(on)
382
		opio->cmd |= Crun;
383
	else
384
		opio->cmd = Cstop;
385
	coherence();
386
	for(i = 0; i < 100; i++)
387
		if(on == 0 && (opio->sts & Shalted) != 0)
388
			break;
389
		else if(on != 0 && (opio->sts & Shalted) == 0)
390
			break;
391
		else
392
			delay(1);
393
	if(i == 100)
394
		print("ehci %#p %s cmd timed out\n",
395
			ctlr->capio, on ? "run" : "halt");
396
	ddprint("ehci %#p cmd %#lux sts %#lux\n",
397
		ctlr->capio, opio->cmd, opio->sts);
398
}
399
 
400
static void*
401
edalloc(void)
402
{
403
	Ed *ed, *pool;
404
	int i;
405
 
406
	lock(&edpool);
407
	if(edpool.free == nil){
408
		pool = xspanalloc(Incr*sizeof(Ed), Align, 0);
409
		if(pool == nil)
410
			panic("edalloc");
411
		for(i=Incr; --i>=0;){
412
			pool[i].next = edpool.free;
413
			edpool.free = &pool[i];
414
		}
415
		edpool.nalloc += Incr;
416
		edpool.nfree += Incr;
417
		dprint("ehci: edalloc: %d eds\n", edpool.nalloc);
418
	}
419
	ed = edpool.free;
420
	edpool.free = ed->next;
421
	edpool.ninuse++;
422
	edpool.nfree--;
423
	unlock(&edpool);
424
 
425
	memset(ed, 0, sizeof(Ed));	/* safety */
426
	assert(((ulong)ed & 0xF) == 0);
427
	return ed;
428
}
429
 
430
static void
431
edfree(void *a)
432
{
433
	Ed *ed;
434
 
435
	ed = a;
436
	lock(&edpool);
437
	ed->next = edpool.free;
438
	edpool.free = ed;
439
	edpool.ninuse--;
440
	edpool.nfree++;
441
	unlock(&edpool);
442
}
443
 
444
/*
445
 * Allocate and do some initialization.
446
 * Free after releasing buffers used.
447
 */
448
 
449
static Itd*
450
itdalloc(void)
451
{
452
	Itd *td;
453
 
454
	td = edalloc();
455
	td->link = Lterm;
456
	return td;
457
}
458
 
459
static void
460
itdfree(Itd *td)
461
{
462
	edfree(td);
463
}
464
 
465
static Sitd*
466
sitdalloc(void)
467
{
468
	Sitd *td;
469
 
470
	td = edalloc();
471
	td->link = td->blink = Lterm;
472
	return td;
473
}
474
 
475
static void
476
sitdfree(Sitd *td)
477
{
478
	edfree(td);
479
}
480
 
481
static Td*
482
tdalloc(void)
483
{
484
	Td *td;
485
 
486
	td = edalloc();
487
	td->nlink = td->alink = Lterm;
488
	return td;
489
}
490
 
491
static void
492
tdfree(Td *td)
493
{
494
	if(td == nil)
495
		return;
496
	free(td->buff);
497
	edfree(td);
498
}
499
 
500
static void
501
tdlinktd(Td *td, Td *next)
502
{
503
	td->next = next;
504
	td->alink = Lterm;
505
	if(next == nil)
506
		td->nlink = Lterm;
507
	else
508
		td->nlink = PADDR(next);
509
	coherence();
510
}
511
 
512
static Qh*
513
qhlinkqh(Qh *qh, Qh *next)
514
{
515
	qh->next = next;
516
	if(next == nil)
517
		qh->link = Lterm;
518
	else
519
		qh->link = PADDR(next)|Lqh;
520
	coherence();
521
	return qh;
522
}
523
 
524
static void
525
qhsetaddr(Qh *qh, ulong addr)
526
{
527
	ulong eps0;
528
 
529
	eps0 = qh->eps0 & ~((Epmax<<8)|Devmax);
530
	qh->eps0 = eps0 | addr & Devmax | ((addr >> 7) & Epmax) << 8;
531
	coherence();
532
}
533
 
534
/*
535
 * return largest power of 2 <= n
536
 */
537
static int
538
flog2lower(int n)
539
{
540
	int i;
541
 
542
	for(i = 0; (1 << (i + 1)) <= n; i++)
543
		;
544
	return i;
545
}
546
 
547
static int
548
pickschedq(Qtree *qt, int pollival, ulong bw, ulong limit)
549
{
550
	int i, j, d, upperb, q;
551
	ulong best, worst, total;
552
 
553
	d = flog2lower(pollival);
554
	if(d > qt->depth)
555
		d = qt->depth;
556
	q = -1;
557
	worst = 0;
558
	best = ~0;
559
	upperb = (1 << (d+1)) - 1;
560
	for(i = (1 << d) - 1; i < upperb; i++){
561
		total = qt->bw[0];
562
		for(j = i; j > 0; j = (j - 1) / 2)
563
			total += qt->bw[j];
564
		if(total < best){
565
			best = total;
566
			q = i;
567
		}
568
		if(total > worst)
569
			worst = total;
570
	}
571
	if(worst + bw >= limit)
572
		return -1;
573
	return q;
574
}
575
 
576
static int
577
schedq(Ctlr *ctlr, Qh *qh, int pollival)
578
{
579
	int q;
580
	Qh *tqh;
581
	ulong bw;
582
 
583
	bw = qh->io->bw;
584
	q = pickschedq(ctlr->tree, pollival, 0, ~0);
585
	ddqprint("ehci: sched %#p q %d, ival %d, bw %uld\n",
586
		qh->io, q, pollival, bw);
587
	if(q < 0){
588
		print("ehci: no room for ed\n");
589
		return -1;
590
	}
591
	ctlr->tree->bw[q] += bw;
592
	tqh = ctlr->tree->root[q];
593
	qh->sched = q;
594
	qhlinkqh(qh, tqh->next);
595
	qhlinkqh(tqh, qh);
596
	coherence();
597
	qh->inext = ctlr->intrqhs;
598
	ctlr->intrqhs = qh;
599
	coherence();
600
	return 0;
601
}
602
 
603
static void
604
unschedq(Ctlr *ctlr, Qh *qh)
605
{
606
	int q;
607
	Qh *prev, *this, *next;
608
	Qh **l;
609
	ulong bw;
610
 
611
	bw = qh->io->bw;
612
	q = qh->sched;
613
	if(q < 0)
614
		return;
615
	ctlr->tree->bw[q] -= bw;
616
 
617
	prev = ctlr->tree->root[q];
618
	this = prev->next;
619
	while(this != nil && this != qh){
620
		prev = this;
621
		this = this->next;
622
	}
623
	if(this == nil)
624
		print("ehci: unschedq %d: not found\n", q);
625
	else{
626
		next = this->next;
627
		qhlinkqh(prev, next);
628
	}
629
	for(l = &ctlr->intrqhs; *l != nil; l = &(*l)->inext)
630
		if(*l == qh){
631
			*l = (*l)->inext;
632
			return;
633
		}
634
	print("ehci: unschedq: qh %#p not found\n", qh);
635
}
636
 
637
static ulong
638
qhmaxpkt(Qh *qh)
639
{
640
	return (qh->eps0 >> Qhmplshift) & Qhmplmask;
641
}
642
 
643
static void
644
qhsetmaxpkt(Qh *qh, int maxpkt)
645
{
646
	ulong eps0;
647
 
648
	eps0 = qh->eps0 & ~(Qhmplmask << Qhmplshift);
649
	qh->eps0 = eps0 | (maxpkt & Qhmplmask) << Qhmplshift;
650
	coherence();
651
}
652
 
653
/*
654
 * Initialize the round-robin circular list of ctl/bulk Qhs
655
 * if ep is nil. Otherwise, allocate and link a new Qh in the ctlr.
656
 */
657
static Qh*
658
qhalloc(Ctlr *ctlr, Ep *ep, Qio *io, char* tag)
659
{
660
	Qh *qh;
661
	int ttype;
662
 
663
	qh = edalloc();
664
	qh->nlink = Lterm;
665
	qh->alink = Lterm;
666
	qh->csw = Tdhalt;
667
	qh->state = Qidle;
668
	qh->sched = -1;
669
	qh->io = io;
670
	if(ep != nil){
671
		qh->eps0 = 0;
672
		qhsetmaxpkt(qh, ep->maxpkt);
673
		if(ep->dev->speed == Lowspeed)
674
			qh->eps0 |= Qhlow;
675
		if(ep->dev->speed == Highspeed)
676
			qh->eps0 |= Qhhigh;
677
		else if(ep->ttype == Tctl)
678
			qh->eps0 |= Qhnhctl;
679
		qh->eps0 |= Qhdtc | 8 << Qhrlcshift;	/* 8 naks max */
680
		coherence();
681
		qhsetaddr(qh, io->usbid);
682
		qh->eps1 = (ep->ntds & Qhmultmask) << Qhmultshift;
683
		qh->eps1 |= ep->dev->port << Qhportshift;
684
		qh->eps1 |= ep->dev->hub << Qhhubshift;
685
		qh->eps1 |= 034 << Qhscmshift;
686
		if(ep->ttype == Tintr)
687
			qh->eps1 |= 1 << Qhismshift;	/* intr. start µf. */
688
		coherence();
689
		if(io != nil)
690
			io->tag = tag;
691
	}
692
	ilock(ctlr);
693
	ttype = Tctl;
694
	if(ep != nil)
695
		ttype = ep->ttype;
696
	switch(ttype){
697
	case Tctl:
698
	case Tbulk:
699
		if(ctlr->qhs == nil){
700
			ctlr->qhs = qhlinkqh(qh, qh);
701
			qh->eps0 |= Qhhigh | Qhhrl;
702
			coherence();
703
			ctlr->opio->link = PADDR(qh)|Lqh;
704
			coherence();
705
		}else{
706
			qhlinkqh(qh, ctlr->qhs->next);
707
			qhlinkqh(ctlr->qhs, qh);
708
		}
709
		break;
710
	case Tintr:
711
		schedq(ctlr, qh, ep->pollival);
712
		break;
713
	default:
714
		print("ehci: qhalloc called for ttype != ctl/bulk\n");
715
	}
716
	iunlock(ctlr);
717
	return qh;
718
}
719
 
720
static int
721
qhadvanced(void *a)
722
{
723
	Ctlr *ctlr;
724
 
725
	ctlr = a;
726
	return (ctlr->opio->cmd & Ciasync) == 0;
727
}
728
 
729
/*
730
 * called when a qh is removed, to be sure the hw is not
731
 * keeping pointers into it.
732
 */
733
static void
734
qhcoherency(Ctlr *ctlr)
735
{
736
	int i;
737
 
738
	qlock(&ctlr->portlck);
739
	ctlr->opio->cmd |= Ciasync;	/* ask for intr. on async advance */
740
	coherence();
741
	for(i = 0; i < 3 && qhadvanced(ctlr) == 0; i++)
742
		if(!waserror()){
743
			tsleep(ctlr, qhadvanced, ctlr, Abortdelay);
744
			poperror();
745
		}
746
	dprint("ehci: qhcoherency: doorbell %d\n", qhadvanced(ctlr));
747
	if(i == 3)
748
		print("ehci: async advance doorbell did not ring\n");
749
	ctlr->opio->cmd &= ~Ciasync;	/* try to clean */
750
	qunlock(&ctlr->portlck);
751
}
752
 
753
static void
754
qhfree(Ctlr *ctlr, Qh *qh)
755
{
756
	Td *td, *ltd;
757
	Qh *q;
758
 
759
	if(qh == nil)
760
		return;
761
	ilock(ctlr);
762
	if(qh->sched < 0){
763
		for(q = ctlr->qhs; q != nil; q = q->next)
764
			if(q->next == qh)
765
				break;
766
		if(q == nil)
767
			panic("qhfree: nil q");
768
		q->next = qh->next;
769
		q->link = qh->link;
770
		coherence();
771
	}else
772
		unschedq(ctlr, qh);
773
	iunlock(ctlr);
774
 
775
	qhcoherency(ctlr);
776
 
777
	for(td = qh->tds; td != nil; td = ltd){
778
		ltd = td->next;
779
		tdfree(td);
780
	}
781
 
782
	edfree(qh);
783
}
784
 
785
static void
786
qhlinktd(Qh *qh, Td *td)
787
{
788
	ulong csw;
789
	int i;
790
 
791
	csw = qh->csw;
792
	qh->tds = td;
793
	if(td == nil)
794
		qh->csw = (csw & ~Tdactive) | Tdhalt;
795
	else{
796
		csw &= Tddata1 | Tdping;	/* save */
797
		qh->csw = Tdhalt;
798
		coherence();
799
		qh->tclink = 0;
800
		qh->alink = Lterm;
801
		qh->nlink = PADDR(td);
802
		for(i = 0; i < nelem(qh->buffer); i++)
803
			qh->buffer[i] = 0;
804
		coherence();
805
		qh->csw = csw & ~(Tdhalt|Tdactive);	/* activate next */
806
	}
807
	coherence();
808
}
809
 
810
static char*
811
seprintlink(char *s, char *se, char *name, ulong l, int typed)
812
{
813
	s = seprint(s, se, "%s %ulx", name, l);
814
	if((l & Lterm) != 0)
815
		return seprint(s, se, "T");
816
	if(typed == 0)
817
		return s;
818
	switch(l & (3<<1)){
819
	case Litd:
820
		return seprint(s, se, "I");
821
	case Lqh:
822
		return seprint(s, se, "Q");
823
	case Lsitd:
824
		return seprint(s, se, "S");
825
	default:
826
		return seprint(s, se, "F");
827
	}
828
}
829
 
830
static char*
831
seprintitd(char *s, char *se, Itd *td)
832
{
833
	int i;
834
	ulong b0, b1;
835
	char flags[6];
836
	char *rw;
837
 
838
	if(td == nil)
839
		return seprint(s, se, "<nil itd>\n");
840
	b0 = td->buffer[0];
841
	b1 = td->buffer[1];
842
 
843
	s = seprint(s, se, "itd %#p", td);
844
	rw = (b1 & Itdin) ? "in" : "out";
845
	s = seprint(s, se, " %s ep %uld dev %uld max %uld mult %uld",
846
		rw, (b0>>8)&Epmax, (b0&Devmax),
847
		td->buffer[1] & 0x7ff, b1 & 3);
848
	s = seprintlink(s, se, " link", td->link, 1);
849
	s = seprint(s, se, "\n");
850
	for(i = 0; i < nelem(td->csw); i++){
851
		memset(flags, '-', 5);
852
		if((td->csw[i] & Itdactive) != 0)
853
			flags[0] = 'a';
854
		if((td->csw[i] & Itdioc) != 0)
855
			flags[1] = 'i';
856
		if((td->csw[i] & Itddberr) != 0)
857
			flags[2] = 'd';
858
		if((td->csw[i] & Itdbabble) != 0)
859
			flags[3] = 'b';
860
		if((td->csw[i] & Itdtrerr) != 0)
861
			flags[4] = 't';
862
		flags[5] = 0;
863
		s = seprint(s, se, "\ttd%d %s", i, flags);
864
		s = seprint(s, se, " len %uld", (td->csw[i] >> 16) & 0x7ff);
865
		s = seprint(s, se, " pg %uld", (td->csw[i] >> 12) & 0x7);
866
		s = seprint(s, se, " off %uld\n", td->csw[i] & 0xfff);
867
	}
868
	s = seprint(s, se, "\tbuffs:");
869
	for(i = 0; i < nelem(td->buffer); i++)
870
		s = seprint(s, se, " %#lux", td->buffer[i] >> 12);
871
	return seprint(s, se, "\n");
872
}
873
 
874
static char*
875
seprintsitd(char *s, char *se, Sitd *td)
876
{
877
	char rw, pg, ss;
878
	char flags[8];
879
	static char pc[4] = { 'a', 'b', 'm', 'e' };
880
 
881
	if(td == nil)
882
		return seprint(s, se, "<nil sitd>\n");
883
	s = seprint(s, se, "sitd %#p", td);
884
	rw = (td->epc & Stdin) ? 'r' : 'w';
885
	s = seprint(s, se, " %c ep %uld dev %uld",
886
		rw, (td->epc>>8)&0xf, td->epc&0x7f);
887
	s = seprint(s, se, " max %uld", (td->csw >> 16) & 0x3ff);
888
	s = seprint(s, se, " hub %uld", (td->epc >> 16) & 0x7f);
889
	s = seprint(s, se, " port %uld\n", (td->epc >> 24) & 0x7f);
890
	memset(flags, '-', 7);
891
	if((td->csw & Stdactive) != 0)
892
		flags[0] = 'a';
893
	if((td->csw & Stdioc) != 0)
894
		flags[1] = 'i';
895
	if((td->csw & Stderr) != 0)
896
		flags[2] = 'e';
897
	if((td->csw & Stddberr) != 0)
898
		flags[3] = 'd';
899
	if((td->csw & Stdbabble) != 0)
900
		flags[4] = 'b';
901
	if((td->csw & Stdtrerr) != 0)
902
		flags[5] = 't';
903
	if((td->csw & Stdmmf) != 0)
904
		flags[6] = 'n';
905
	flags[7] = 0;
906
	ss = (td->csw & Stddcs) ? 'c' : 's';
907
	pg = (td->csw & Stdpg) ? '1' : '0';
908
	s = seprint(s, se, "\t%s %cs pg%c", flags, ss, pg);
909
	s = seprint(s, se, " b0 %#lux b1 %#lux off %uld\n",
910
		td->buffer[0] >> 12, td->buffer[1] >> 12, td->buffer[0] & 0xfff);
911
	s = seprint(s, se, "\ttpos %c tcnt %uld",
912
		pc[(td->buffer[0]>>3)&3], td->buffer[1] & 7);
913
	s = seprint(s, se, " ssm %#lux csm %#lux cspm %#lux",
914
		td->mfs & 0xff, (td->mfs>>8) & 0xff, (td->csw>>8) & 0xff);
915
	s = seprintlink(s, se, " link", td->link, 1);
916
	s = seprintlink(s, se, " blink", td->blink, 0);
917
	return seprint(s, se, "\n");
918
}
919
 
920
static long
921
maxtdlen(Td *td)
922
{
923
	return (td->csw >> Tdlenshift) & Tdlenmask;
924
}
925
 
926
static long
927
tdlen(Td *td)
928
{
929
	if(td->data == nil)
930
		return 0;
931
	return td->ndata - maxtdlen(td);
932
}
933
 
934
static char*
935
seprinttd(char *s, char *se, Td *td, char *tag)
936
{
937
	int i;
938
	char t, ss;
939
	char flags[9];
940
	static char *tok[4] = { "out", "in", "setup", "BUG" };
941
 
942
	if(td == nil)
943
		return seprint(s, se, "%s <nil td>\n", tag);
944
	s = seprint(s, se, "%s %#p", tag, td);
945
	s = seprintlink(s, se, " nlink", td->nlink, 0);
946
	s = seprintlink(s, se, " alink", td->alink, 0);
947
	s = seprint(s, se, " %s", tok[(td->csw & Tdtok) >> 8]);
948
	if((td->csw & Tdping) != 0)
949
		s = seprint(s, se, " png");
950
	memset(flags, '-', 8);
951
	if((td->csw & Tdactive) != 0)
952
		flags[0] = 'a';
953
	if((td->csw & Tdioc) != 0)
954
		flags[1] = 'i';
955
	if((td->csw & Tdhalt) != 0)
956
		flags[2] = 'h';
957
	if((td->csw & Tddberr) != 0)
958
		flags[3] = 'd';
959
	if((td->csw & Tdbabble) != 0)
960
		flags[4] = 'b';
961
	if((td->csw & Tdtrerr) != 0)
962
		flags[5] = 't';
963
	if((td->csw & Tdmmf) != 0)
964
		flags[6] = 'n';
965
	if((td->csw & (Tderr2|Tderr1)) == 0)
966
		flags[7] = 'z';
967
	flags[8] = 0;
968
	t = (td->csw & Tddata1) ? '1' : '0';
969
	ss = (td->csw & Tddcs) ? 'c' : 's';
970
	s = seprint(s, se, "\n\td%c %s %cs", t, flags, ss);
971
	s = seprint(s, se, " max %uld", maxtdlen(td));
972
	s = seprint(s, se, " pg %uld off %#lux\n",
973
		(td->csw >> Tdpgshift) & Tdpgmask, td->buffer[0] & 0xFFF);
974
	s = seprint(s, se, "\tbuffs:");
975
	for(i = 0; i < nelem(td->buffer); i++)
976
		s = seprint(s, se, " %#lux", td->buffer[i]>>12);
977
	if(td->data != nil)
978
		s = seprintdata(s, se, td->data, td->ndata);
979
	return seprint(s, se, "\n");
980
}
981
 
982
static void
983
dumptd(Td *td, char *pref)
984
{
985
	char buf[256];
986
	char *se;
987
	int i;
988
 
989
	i = 0;
990
	se = buf+sizeof(buf);
991
	for(; td != nil; td = td->next){
992
		seprinttd(buf, se, td, pref);
993
		print("%s", buf);
994
		if(i++ > 20){
995
			print("...more tds...\n");
996
			break;
997
		}
998
	}
999
}
1000
 
1001
static void
1002
qhdump(Qh *qh)
1003
{
1004
	char buf[256];
1005
	char *s, *se, *tag;
1006
	Td td;
1007
	static char *speed[] = {"full", "low", "high", "BUG"};
1008
 
1009
	if(qh == nil){
1010
		print("<nil qh>\n");
1011
		return;
1012
	}
1013
	if(qh->io == nil)
1014
		tag = "qh";
1015
	else
1016
		tag = qh->io->tag;
1017
	se = buf+sizeof(buf);
1018
	s = seprint(buf, se, "%s %#p", tag, qh);
1019
	s = seprint(s, se, " ep %uld dev %uld",
1020
		(qh->eps0>>8)&0xf, qh->eps0&0x7f);
1021
	s = seprint(s, se, " hub %uld", (qh->eps1 >> 16) & 0x7f);
1022
	s = seprint(s, se, " port %uld", (qh->eps1 >> 23) & 0x7f);
1023
	s = seprintlink(s, se, " link", qh->link, 1);
1024
	seprint(s, se, "  clink %#lux", qh->tclink);
1025
	print("%s\n", buf);
1026
	s = seprint(buf, se, "\tnrld %uld", (qh->eps0 >> Qhrlcshift) & Qhrlcmask);
1027
	s = seprint(s, se, " nak %uld", (qh->alink >> 1) & 0xf);
1028
	s = seprint(s, se, " max %uld ", qhmaxpkt(qh));
1029
	if((qh->eps0 & Qhnhctl) != 0)
1030
		s = seprint(s, se, "c");
1031
	if((qh->eps0 & Qhhrl) != 0)
1032
		s = seprint(s, se, "h");
1033
	if((qh->eps0 & Qhdtc) != 0)
1034
		s = seprint(s, se, "d");
1035
	if((qh->eps0 & Qhint) != 0)
1036
		s = seprint(s, se, "i");
1037
	s = seprint(s, se, " %s", speed[(qh->eps0 >> 12) & 3]);
1038
	s = seprint(s, se, " mult %uld", (qh->eps1 >> Qhmultshift) & Qhmultmask);
1039
	seprint(s, se, " scm %#lux ism %#lux\n",
1040
		(qh->eps1 >> 8 & 0xff), qh->eps1 & 0xff);
1041
	print("%s\n", buf);
1042
	memset(&td, 0, sizeof(td));
1043
	memmove(&td, &qh->nlink, 32);	/* overlay area */
1044
	seprinttd(buf, se, &td, "\tovl");
1045
	print("%s", buf);
1046
}
1047
 
1048
static void
1049
isodump(Isoio* iso, int all)
1050
{
1051
	Itd *td, *tdi, *tdu;
1052
	Sitd *std, *stdi, *stdu;
1053
	char buf[256];
1054
	int i;
1055
 
1056
	if(iso == nil){
1057
		print("<nil iso>\n");
1058
		return;
1059
	}
1060
	print("iso %#p %s %s speed state %d nframes %d maxsz %uld",
1061
		iso, iso->tok == Tdtokin ? "in" : "out",
1062
		iso->hs ? "high" : "full",
1063
		iso->state, iso->nframes, iso->maxsize);
1064
	print(" td0 %uld tdi %#p tdu %#p data %#p\n",
1065
		iso->td0frno, iso->tdi, iso->tdu, iso->data);
1066
	if(iso->err != nil)
1067
		print("\terr %s\n", iso->err);
1068
	if(iso->err != nil)
1069
		print("\terr='%s'\n", iso->err);
1070
	if(all == 0)
1071
		if(iso->hs != 0){
1072
			tdi = iso->tdi;
1073
			seprintitd(buf, buf+sizeof(buf), tdi);
1074
			print("\ttdi %s\n", buf);
1075
			tdu = iso->tdu;
1076
			seprintitd(buf, buf+sizeof(buf), tdu);
1077
			print("\ttdu %s\n", buf);
1078
		}else{
1079
			stdi = iso->stdi;
1080
			seprintsitd(buf, buf+sizeof(buf), stdi);
1081
			print("\tstdi %s\n", buf);
1082
			stdu = iso->stdu;
1083
			seprintsitd(buf, buf+sizeof(buf), stdu);
1084
			print("\tstdu %s\n", buf);
1085
		}
1086
	else
1087
		for(i = 0; i < Nisoframes; i++)
1088
			if(iso->tdps[i] != nil)
1089
				if(iso->hs != 0){
1090
					td = iso->itdps[i];
1091
					seprintitd(buf, buf+sizeof(buf), td);
1092
					if(td == iso->tdi)
1093
						print("i->");
1094
					if(td == iso->tdu)
1095
						print("i->");
1096
					print("[%d]\t%s", i, buf);
1097
				}else{
1098
					std = iso->sitdps[i];
1099
					seprintsitd(buf, buf+sizeof(buf), std);
1100
					if(std == iso->stdi)
1101
						print("i->");
1102
					if(std == iso->stdu)
1103
						print("u->");
1104
					print("[%d]\t%s", i, buf);
1105
				}
1106
}
1107
 
1108
static void
1109
dump(Hci *hp)
1110
{
1111
	int i;
1112
	char *s, *se;
1113
	char buf[128];
1114
	Ctlr *ctlr;
1115
	Eopio *opio;
1116
	Isoio *iso;
1117
	Qh *qh;
1118
 
1119
	ctlr = hp->aux;
1120
	opio = ctlr->opio;
1121
	ilock(ctlr);
1122
	print("ehci port %#p frames %#p (%d fr.) nintr %d ntdintr %d",
1123
		ctlr->capio, ctlr->frames, ctlr->nframes,
1124
		ctlr->nintr, ctlr->ntdintr);
1125
	print(" nqhintr %d nisointr %d\n", ctlr->nqhintr, ctlr->nisointr);
1126
	print("\tcmd %#lux sts %#lux intr %#lux frno %uld",
1127
		opio->cmd, opio->sts, opio->intr, opio->frno);
1128
	print(" base %#lux link %#lux fr0 %#lux\n",
1129
		opio->frbase, opio->link, ctlr->frames[0]);
1130
	se = buf+sizeof(buf);
1131
	s = seprint(buf, se, "\t");
1132
	for(i = 0; i < hp->nports; i++){
1133
		s = seprint(s, se, "p%d %#lux ", i, opio->portsc[i]);
1134
		if(hp->nports > 4 && i == hp->nports/2 - 1)
1135
			s = seprint(s, se, "\n\t");
1136
	}
1137
	print("%s\n", buf);
1138
	qh = ctlr->qhs;
1139
	i = 0;
1140
	do{
1141
		qhdump(qh);
1142
		qh = qh->next;
1143
	}while(qh != ctlr->qhs && i++ < 100);
1144
	if(i > 100)
1145
		print("...too many Qhs...\n");
1146
	if(ctlr->intrqhs != nil)
1147
		print("intr qhs:\n");
1148
	for(qh = ctlr->intrqhs; qh != nil; qh = qh->inext)
1149
		qhdump(qh);
1150
	if(ctlr->iso != nil)
1151
		print("iso:\n");
1152
	for(iso = ctlr->iso; iso != nil; iso = iso->next)
1153
		isodump(ctlr->iso, 0);
1154
	print("%d eds in tree\n", ctlr->ntree);
1155
	iunlock(ctlr);
1156
	lock(&edpool);
1157
	print("%d eds allocated = %d in use + %d free\n",
1158
		edpool.nalloc, edpool.ninuse, edpool.nfree);
1159
	unlock(&edpool);
1160
}
1161
 
1162
static char*
1163
errmsg(int err)
1164
{
1165
	if(err == 0)
1166
		return "ok";
1167
	if(err & Tddberr)
1168
		return "data buffer error";
1169
	if(err & Tdbabble)
1170
		return "babble detected";
1171
	if(err & Tdtrerr)
1172
		return "transaction error";
1173
	if(err & Tdmmf)
1174
		return "missed µframe";
1175
	if(err & Tdhalt)
1176
		return Estalled;	/* [uo]hci report this error */
1177
	return Eio;
1178
}
1179
 
1180
static char*
1181
ierrmsg(int err)
1182
{
1183
	if(err == 0)
1184
		return "ok";
1185
	if(err & Itddberr)
1186
		return "data buffer error";
1187
	if(err & Itdbabble)
1188
		return "babble detected";
1189
	if(err & Itdtrerr)
1190
		return "transaction error";
1191
	return Eio;
1192
}
1193
 
1194
static char*
1195
serrmsg(int err)
1196
{
1197
	if(err & Stderr)
1198
		return "translation translator error";
1199
	/* other errors have same numbers than Td errors */
1200
	return errmsg(err);
1201
}
1202
 
1203
static int
1204
isocanread(void *a)
1205
{
1206
	Isoio *iso;
1207
 
1208
	iso = a;
1209
	if(iso->state == Qclose)
1210
		return 1;
1211
	if(iso->state == Qrun && iso->tok == Tdtokin){
1212
		if(iso->hs != 0 && iso->tdi != iso->tdu)
1213
			return 1;
1214
		if(iso->hs == 0 && iso->stdi != iso->stdu)
1215
			return 1;
1216
	}
1217
	return 0;
1218
}
1219
 
1220
static int
1221
isocanwrite(void *a)
1222
{
1223
	Isoio *iso;
1224
 
1225
	iso = a;
1226
	if(iso->state == Qclose)
1227
		return 1;
1228
	if(iso->state == Qrun && iso->tok == Tdtokout){
1229
		if(iso->hs != 0 && iso->tdu->next != iso->tdi)
1230
			return 1;
1231
		if(iso->hs == 0 && iso->stdu->next != iso->stdi)
1232
			return 1;
1233
	}
1234
	return 0;
1235
}
1236
 
1237
static void
1238
itdinit(Isoio *iso, Itd *td)
1239
{
1240
	int p, t;
1241
	ulong pa, tsize, size;
1242
 
1243
	/*
1244
	 * BUG: This does not put an integral number of samples
1245
	 * on each µframe unless samples per packet % 8 == 0
1246
	 * Also, all samples are packed early on each frame.
1247
	 */
1248
	p = 0;
1249
	size = td->ndata = td->mdata;
1250
	pa = PADDR(td->data);
1251
	for(t = 0; size > 0 && t < 8; t++){
1252
		tsize = size;
1253
		if(tsize > iso->maxsize)
1254
			tsize = iso->maxsize;
1255
		size -= tsize;
1256
		assert(p < nelem(td->buffer));
1257
		td->csw[t] = tsize << Itdlenshift | p << Itdpgshift |
1258
			(pa & 0xFFF) << Itdoffshift | Itdactive | Itdioc;
1259
		coherence();
1260
		if(((pa+tsize) & ~0xFFF) != (pa & ~0xFFF))
1261
			p++;
1262
		pa += tsize;
1263
	}
1264
}
1265
 
1266
static void
1267
sitdinit(Isoio *iso, Sitd *td)
1268
{
1269
	td->ndata = td->mdata & Stdlenmask;
1270
	td->buffer[0] = PADDR(td->data);
1271
	td->buffer[1] = (td->buffer[0] & ~0xFFF) + 0x1000;
1272
	if(iso->tok == Tdtokin || td->ndata <= 188)
1273
		td->buffer[1] |= Stdtpall;
1274
	else
1275
		td->buffer[1] |= Stdtpbegin;
1276
	if(iso->tok == Tdtokin)
1277
		td->buffer[1] |= 1;
1278
	else
1279
		td->buffer[1] |= ((td->ndata + 187) / 188) & Stdtcntmask;
1280
	coherence();
1281
	td->csw = td->ndata << Stdlenshift | Stdactive | Stdioc;
1282
	coherence();
1283
}
1284
 
1285
static int
1286
itdactive(Itd *td)
1287
{
1288
	int i;
1289
 
1290
	for(i = 0; i < nelem(td->csw); i++)
1291
		if((td->csw[i] & Itdactive) != 0)
1292
			return 1;
1293
	return 0;
1294
}
1295
 
1296
static int
1297
isohsinterrupt(Ctlr *ctlr, Isoio *iso)
1298
{
1299
	int err, i, nframes, t;
1300
	Itd *tdi;
1301
 
1302
	tdi = iso->tdi;
1303
	assert(tdi != nil);
1304
	if(itdactive(tdi))			/* not all tds are done */
1305
		return 0;
1306
	ctlr->nisointr++;
1307
	ddiprint("isohsintr: iso %#p: tdi %#p tdu %#p\n", iso, tdi, iso->tdu);
1308
	if(iso->state != Qrun && iso->state != Qdone)
1309
		panic("isofsintr: iso state");
1310
	if(ehcidebug > 1 || iso->debug > 1)
1311
		isodump(iso, 0);
1312
 
1313
	nframes = iso->nframes / 2;		/* limit how many we look */
1314
	if(nframes > Nisoframes)
1315
		nframes = Nisoframes;
1316
 
1317
	if(iso->tok == Tdtokin)
1318
		tdi->ndata = 0;
1319
	/* else, it has the number of bytes transferred */
1320
 
1321
	for(i = 0; i < nframes && itdactive(tdi) == 0; i++){
1322
		if(iso->tok == Tdtokin)
1323
			tdi->ndata += (tdi->csw[i] >> Itdlenshift) & Itdlenmask;
1324
		err = 0;
1325
		coherence();
1326
		for(t = 0; t < nelem(tdi->csw); t++){
1327
			tdi->csw[t] &= ~Itdioc;
1328
			coherence();
1329
			err |= tdi->csw[t] & Itderrors;
1330
		}
1331
		if(err == 0)
1332
			iso->nerrs = 0;
1333
		else if(iso->nerrs++ > iso->nframes/2){
1334
			if(iso->err == nil){
1335
				iso->err = ierrmsg(err);
1336
				diprint("isohsintr: tdi %#p error %#ux %s\n",
1337
					tdi, err, iso->err);
1338
				diprint("ctlr load %uld\n", ctlr->load);
1339
			}
1340
			tdi->ndata = 0;
1341
		}else
1342
			tdi->ndata = 0;
1343
		if(tdi->next == iso->tdu || tdi->next->next == iso->tdu){
1344
			memset(iso->tdu->data, 0, iso->tdu->mdata);
1345
			itdinit(iso, iso->tdu);
1346
			iso->tdu = iso->tdu->next;
1347
			iso->nleft = 0;
1348
		}
1349
		tdi = tdi->next;
1350
		coherence();
1351
	}
1352
	ddiprint("isohsintr: %d frames processed\n", nframes);
1353
	if(i == nframes){
1354
		tdi->csw[0] |= Itdioc;
1355
		coherence();
1356
	}
1357
	iso->tdi = tdi;
1358
	coherence();
1359
	if(isocanwrite(iso) || isocanread(iso)){
1360
		diprint("wakeup iso %#p tdi %#p tdu %#p\n", iso,
1361
			iso->tdi, iso->tdu);
1362
		wakeup(iso);
1363
	}
1364
	return 1;
1365
}
1366
 
1367
static int
1368
isofsinterrupt(Ctlr *ctlr, Isoio *iso)
1369
{
1370
	int err, i, nframes;
1371
	Sitd *stdi;
1372
 
1373
	stdi = iso->stdi;
1374
	assert(stdi != nil);
1375
	if((stdi->csw & Stdactive) != 0)		/* nothing new done */
1376
		return 0;
1377
	ctlr->nisointr++;
1378
	ddiprint("isofsintr: iso %#p: tdi %#p tdu %#p\n", iso, stdi, iso->stdu);
1379
	if(iso->state != Qrun && iso->state != Qdone)
1380
		panic("isofsintr: iso state");
1381
	if(ehcidebug > 1 || iso->debug > 1)
1382
		isodump(iso, 0);
1383
 
1384
	nframes = iso->nframes / 2;		/* limit how many we look */
1385
	if(nframes > Nisoframes)
1386
		nframes = Nisoframes;
1387
 
1388
	for(i = 0; i < nframes && (stdi->csw & Stdactive) == 0; i++){
1389
		stdi->csw &= ~Stdioc;
1390
		/* write back csw and see if it produces errors */
1391
		coherence();
1392
		err = stdi->csw & Stderrors;
1393
		if(err == 0){
1394
			iso->nerrs = 0;
1395
			if(iso->tok == Tdtokin)
1396
				stdi->ndata = (stdi->csw>>Stdlenshift)&Stdlenmask;
1397
			/* else len is assumed correct */
1398
		}else if(iso->nerrs++ > iso->nframes/2){
1399
			if(iso->err == nil){
1400
				iso->err = serrmsg(err);
1401
				diprint("isofsintr: tdi %#p error %#ux %s\n",
1402
					stdi, err, iso->err);
1403
				diprint("ctlr load %uld\n", ctlr->load);
1404
			}
1405
			stdi->ndata = 0;
1406
		}else
1407
			stdi->ndata = 0;
1408
 
1409
		if(stdi->next == iso->stdu || stdi->next->next == iso->stdu){
1410
			memset(iso->stdu->data, 0, iso->stdu->mdata);
1411
			coherence();
1412
			sitdinit(iso, iso->stdu);
1413
			iso->stdu = iso->stdu->next;
1414
			iso->nleft = 0;
1415
		}
1416
		coherence();
1417
		stdi = stdi->next;
1418
	}
1419
	ddiprint("isofsintr: %d frames processed\n", nframes);
1420
	if(i == nframes){
1421
		stdi->csw |= Stdioc;
1422
		coherence();
1423
	}
1424
	iso->stdi = stdi;
1425
	coherence();
1426
	if(isocanwrite(iso) || isocanread(iso)){
1427
		diprint("wakeup iso %#p tdi %#p tdu %#p\n", iso,
1428
			iso->stdi, iso->stdu);
1429
		wakeup(iso);
1430
	}
1431
	return 1;
1432
}
1433
 
1434
static int
1435
qhinterrupt(Ctlr *ctlr, Qh *qh)
1436
{
1437
	Td *td;
1438
	int err;
1439
 
1440
	if(qh->state != Qrun)
1441
		panic("qhinterrupt: qh state");
1442
	td = qh->tds;
1443
	if(td == nil)
1444
		panic("qhinterrupt: no tds");
1445
	if((td->csw & Tdactive) == 0)
1446
		ddqprint("qhinterrupt port %#p qh %#p\n", ctlr->capio, qh);
1447
	for(; td != nil; td = td->next){
1448
		if(td->csw & Tdactive)
1449
			return 0;
1450
		err = td->csw & Tderrors;
1451
		if(err != 0){
1452
			if(qh->io->err == nil){
1453
				qh->io->err = errmsg(err);
1454
				dqprint("qhintr: td %#p csw %#lux error %#ux %s\n",
1455
					td, td->csw, err, qh->io->err);
1456
			}
1457
			break;
1458
		}
1459
		td->ndata = tdlen(td);
1460
		coherence();
1461
		if(td->ndata < maxtdlen(td)){	/* EOT */
1462
			td = td->next;
1463
			break;
1464
		}
1465
	}
1466
	/*
1467
	 * Done. Make void the Tds not used (errors or EOT) and wakeup epio.
1468
	 */
1469
	for(; td != nil; td = td->next)
1470
		td->ndata = 0;
1471
	coherence();
1472
	qh->state = Qdone;
1473
	coherence();
1474
	wakeup(qh->io);
1475
	return 1;
1476
}
1477
 
1478
static int
1479
ehciintr(Hci *hp)
1480
{
1481
	Ctlr *ctlr;
1482
	Eopio *opio;
1483
	Isoio *iso;
1484
	ulong sts;
1485
	Qh *qh;
1486
	int i, some;
1487
 
1488
	ctlr = hp->aux;
1489
	opio = ctlr->opio;
1490
 
1491
	/*
1492
	 * Will we know in USB 3.0 who the interrupt was for?.
1493
	 * Do they still teach indexing in CS?
1494
	 * This is Intel's doing.
1495
	 */
1496
	ilock(ctlr);
1497
	ctlr->nintr++;
1498
	sts = opio->sts & Sintrs;
1499
	if(sts == 0){		/* not ours; shared intr. */
1500
		iunlock(ctlr);
1501
		return 0;
1502
	}
1503
	opio->sts = sts;
1504
	coherence();
1505
	if((sts & Sherr) != 0)
1506
		print("ehci: port %#p fatal host system error\n", ctlr->capio);
1507
	if((sts & Shalted) != 0)
1508
		print("ehci: port %#p: halted\n", ctlr->capio);
1509
	if((sts & Sasync) != 0){
1510
		dprint("ehci: doorbell\n");
1511
		wakeup(ctlr);
1512
	}
1513
	/*
1514
	 * We enter always this if, even if it seems the
1515
	 * interrupt does not report anything done/failed.
1516
	 * Some controllers don't post interrupts right.
1517
	 */
1518
	some = 0;
1519
	if((sts & (Serrintr|Sintr)) != 0){
1520
		ctlr->ntdintr++;
1521
		if(ehcidebug > 1){
1522
			print("ehci port %#p frames %#p nintr %d ntdintr %d",
1523
				ctlr->capio, ctlr->frames,
1524
				ctlr->nintr, ctlr->ntdintr);
1525
			print(" nqhintr %d nisointr %d\n",
1526
				ctlr->nqhintr, ctlr->nisointr);
1527
			print("\tcmd %#lux sts %#lux intr %#lux frno %uld",
1528
				opio->cmd, opio->sts, opio->intr, opio->frno);
1529
		}
1530
 
1531
		/* process the Iso transfers */
1532
		for(iso = ctlr->iso; iso != nil; iso = iso->next)
1533
			if(iso->state == Qrun || iso->state == Qdone)
1534
				if(iso->hs != 0)
1535
					some += isohsinterrupt(ctlr, iso);
1536
				else
1537
					some += isofsinterrupt(ctlr, iso);
1538
 
1539
		/* process the qhs in the periodic tree */
1540
		for(qh = ctlr->intrqhs; qh != nil; qh = qh->inext)
1541
			if(qh->state == Qrun)
1542
				some += qhinterrupt(ctlr, qh);
1543
 
1544
		/* process the async Qh circular list */
1545
		qh = ctlr->qhs;
1546
		i = 0;
1547
		do{
1548
			if (qh == nil)
1549
				panic("ehciintr: nil qh");
1550
			if(qh->state == Qrun)
1551
				some += qhinterrupt(ctlr, qh);
1552
			qh = qh->next;
1553
		}while(qh != ctlr->qhs && i++ < 100);
1554
		if(i > 100)
1555
			print("echi: interrupt: qh loop?\n");
1556
	}
1557
//	if (some == 0)
1558
//		panic("ehciintr: no work");
1559
	iunlock(ctlr);
1560
	return some;
1561
}
1562
 
1563
static void
1564
interrupt(Ureg*, void* a)
1565
{
1566
	ehciintr(a);
1567
}
1568
 
1569
static int
1570
portenable(Hci *hp, int port, int on)
1571
{
1572
	Ctlr *ctlr;
1573
	Eopio *opio;
1574
	int s;
1575
 
1576
	ctlr = hp->aux;
1577
	opio = ctlr->opio;
1578
	s = opio->portsc[port-1];
1579
	qlock(&ctlr->portlck);
1580
	if(waserror()){
1581
		qunlock(&ctlr->portlck);
1582
		nexterror();
1583
	}
1584
	dprint("ehci %#p port %d enable=%d; sts %#x\n",
1585
		ctlr->capio, port, on, s);
1586
	ilock(ctlr);
1587
	if(s & (Psstatuschg | Pschange))
1588
		opio->portsc[port-1] = s;
1589
	if(on)
1590
		opio->portsc[port-1] |= Psenable;
1591
	else
1592
		opio->portsc[port-1] &= ~Psenable;
1593
	coherence();
1594
	microdelay(64);
1595
	iunlock(ctlr);
1596
	tsleep(&up->sleep, return0, 0, Enabledelay);
1597
	dprint("ehci %#p port %d enable=%d: sts %#lux\n",
1598
		ctlr->capio, port, on, opio->portsc[port-1]);
1599
	qunlock(&ctlr->portlck);
1600
	poperror();
1601
	return 0;
1602
}
1603
 
1604
/*
1605
 * If we detect during status that the port is low-speed or
1606
 * during reset that it's full-speed, the device is not for
1607
 * ourselves. The companion controller will take care.
1608
 * Low-speed devices will not be seen by usbd. Full-speed
1609
 * ones are seen because it's only after reset that we know what
1610
 * they are (usbd may notice a device not enabled in this case).
1611
 */
1612
static void
1613
portlend(Ctlr *ctlr, int port, char *ss)
1614
{
1615
	Eopio *opio;
1616
	ulong s;
1617
 
1618
	opio = ctlr->opio;
1619
 
1620
	dprint("ehci %#p port %d: %s speed device: no longer owned\n",
1621
		ctlr->capio, port, ss);
1622
	s = opio->portsc[port-1] & ~(Pschange|Psstatuschg);
1623
	opio->portsc[port-1] = s | Psowner;
1624
	coherence();
1625
}
1626
 
1627
static int
1628
portreset(Hci *hp, int port, int on)
1629
{
1630
	ulong *portscp;
1631
	Eopio *opio;
1632
	Ctlr *ctlr;
1633
	int i;
1634
 
1635
	if(on == 0)
1636
		return 0;
1637
 
1638
	ctlr = hp->aux;
1639
	opio = ctlr->opio;
1640
	qlock(&ctlr->portlck);
1641
	if(waserror()){
1642
		iunlock(ctlr);
1643
		qunlock(&ctlr->portlck);
1644
		nexterror();
1645
	}
1646
	portscp = &opio->portsc[port-1];
1647
	dprint("ehci %#p port %d reset; sts %#lux\n", ctlr->capio, port, *portscp);
1648
	ilock(ctlr);
1649
	/* Shalted must be zero, else Psreset will stay set */
1650
	if (opio->sts & Shalted)
1651
		iprint("ehci %#p: halted yet trying to reset port\n",
1652
			ctlr->capio);
1653
	*portscp = (*portscp & ~Psenable) | Psreset;	/* initiate reset */
1654
	coherence();
1655
 
1656
	/*
1657
	 * usb 2 spec: reset must finish within 20 ms.
1658
	 * linux says spec says it can take 50 ms. for hubs.
1659
	 */
1660
	for(i = 0; *portscp & Psreset && i < 50; i++)
1661
		delay(10);
1662
	if (*portscp & Psreset)
1663
		iprint("ehci %#p: port %d didn't reset within %d ms; sts %#lux\n",
1664
			ctlr->capio, port, i * 10, *portscp);
1665
	*portscp &= ~Psreset;		/* force appearance of reset done */
1666
	coherence();
1667
	delay(10);			/* ehci spec: enable within 2 ms. */
1668
 
1669
	if((*portscp & Psenable) == 0)
1670
		portlend(ctlr, port, "full");
1671
 
1672
	iunlock(ctlr);
1673
	dprint("ehci %#p after port %d reset; sts %#lux\n",
1674
		ctlr->capio, port, *portscp);
1675
	qunlock(&ctlr->portlck);
1676
	poperror();
1677
	return 0;
1678
}
1679
 
1680
static int
1681
portstatus(Hci *hp, int port)
1682
{
1683
	int s, r;
1684
	Eopio *opio;
1685
	Ctlr *ctlr;
1686
 
1687
	ctlr = hp->aux;
1688
	opio = ctlr->opio;
1689
	qlock(&ctlr->portlck);
1690
	if(waserror()){
1691
		iunlock(ctlr);
1692
		qunlock(&ctlr->portlck);
1693
		nexterror();
1694
	}
1695
	ilock(ctlr);
1696
	s = opio->portsc[port-1];
1697
	if(s & (Psstatuschg | Pschange)){
1698
		opio->portsc[port-1] = s;
1699
		coherence();
1700
		ddprint("ehci %#p port %d status %#x\n", ctlr->capio, port, s);
1701
	}
1702
	/*
1703
	 * If the port is a low speed port we yield ownership now
1704
	 * to the [uo]hci companion controller and pretend it's not here.
1705
	 */
1706
	if((s & Pspresent) != 0 && (s & Pslinemask) == Pslow){
1707
		portlend(ctlr, port, "low");
1708
		s &= ~Pspresent;		/* not for us this time */
1709
	}
1710
	iunlock(ctlr);
1711
	qunlock(&ctlr->portlck);
1712
	poperror();
1713
 
1714
	/*
1715
	 * We must return status bits as a
1716
	 * get port status hub request would do.
1717
	 */
1718
	r = 0;
1719
	if(s & Pspresent)
1720
		r |= HPpresent|HPhigh;
1721
	if(s & Psenable)
1722
		r |= HPenable;
1723
	if(s & Pssuspend)
1724
		r |= HPsuspend;
1725
	if(s & Psreset)
1726
		r |= HPreset;
1727
	if(s & Psstatuschg)
1728
		r |= HPstatuschg;
1729
	if(s & Pschange)
1730
		r |= HPchange;
1731
	return r;
1732
}
1733
 
1734
static char*
1735
seprintio(char *s, char *e, Qio *io, char *pref)
1736
{
1737
	s = seprint(s,e,"%s io %#p qh %#p id %#x", pref, io, io->qh, io->usbid);
1738
	s = seprint(s,e," iot %ld", io->iotime);
1739
	s = seprint(s,e," tog %#x tok %#x err %s", io->toggle, io->tok, io->err);
1740
	return s;
1741
}
1742
 
1743
static char*
1744
seprintep(char *s, char *e, Ep *ep)
1745
{
1746
	Qio *io;
1747
	Ctlio *cio;
1748
	Ctlr *ctlr;
1749
 
1750
	ctlr = ep->hp->aux;
1751
	ilock(ctlr);
1752
	if(ep->aux == nil){
1753
		*s = 0;
1754
		iunlock(ctlr);
1755
		return s;
1756
	}
1757
	switch(ep->ttype){
1758
	case Tctl:
1759
		cio = ep->aux;
1760
		s = seprintio(s, e, cio, "c");
1761
		s = seprint(s, e, "\trepl %d ndata %d\n", ep->rhrepl, cio->ndata);
1762
		break;
1763
	case Tbulk:
1764
	case Tintr:
1765
		io = ep->aux;
1766
		if(ep->mode != OWRITE)
1767
			s = seprintio(s, e, &io[OREAD], "r");
1768
		if(ep->mode != OREAD)
1769
			s = seprintio(s, e, &io[OWRITE], "w");
1770
		break;
1771
	case Tiso:
1772
		*s = 0;
1773
		break;
1774
	}
1775
	iunlock(ctlr);
1776
	return s;
1777
}
1778
 
1779
/*
1780
 * halt condition was cleared on the endpoint. update our toggles.
1781
 */
1782
static void
1783
clrhalt(Ep *ep)
1784
{
1785
	Qio *io;
1786
 
1787
	ep->clrhalt = 0;
1788
	coherence();
1789
	switch(ep->ttype){
1790
	case Tintr:
1791
	case Tbulk:
1792
		io = ep->aux;
1793
		if(ep->mode != OREAD){
1794
			qlock(&io[OWRITE]);
1795
			io[OWRITE].toggle = Tddata0;
1796
			deprint("ep clrhalt for io %#p\n", io+OWRITE);
1797
			qunlock(&io[OWRITE]);
1798
		}
1799
		if(ep->mode != OWRITE){
1800
			qlock(&io[OREAD]);
1801
			io[OREAD].toggle = Tddata0;
1802
			deprint("ep clrhalt for io %#p\n", io+OREAD);
1803
			qunlock(&io[OREAD]);
1804
		}
1805
		break;
1806
	}
1807
}
1808
 
1809
static void
1810
xdump(char* pref, void *qh)
1811
{
1812
	int i;
1813
	ulong *u;
1814
 
1815
	u = qh;
1816
	print("%s %#p:", pref, u);
1817
	for(i = 0; i < 16; i++)
1818
		if((i%4) == 0)
1819
			print("\n %#8.8ulx", u[i]);
1820
		else
1821
			print(" %#8.8ulx", u[i]);
1822
	print("\n");
1823
}
1824
 
1825
static long
1826
episohscpy(Ctlr *ctlr, Ep *ep, Isoio* iso, uchar *b, long count)
1827
{
1828
	int nr;
1829
	long tot;
1830
	Itd *tdu;
1831
 
1832
	for(tot = 0; iso->tdi != iso->tdu && tot < count; tot += nr){
1833
		tdu = iso->tdu;
1834
		if(itdactive(tdu))
1835
			break;
1836
		nr = tdu->ndata;
1837
		if(tot + nr > count)
1838
			nr = count - tot;
1839
		if(nr == 0)
1840
			print("ehci: ep%d.%d: too many polls\n",
1841
				ep->dev->nb, ep->nb);
1842
		else{
1843
			iunlock(ctlr);		/* We could page fault here */
1844
			memmove(b+tot, tdu->data, nr);
1845
			ilock(ctlr);
1846
			if(nr < tdu->ndata)
1847
				memmove(tdu->data, tdu->data+nr, tdu->ndata - nr);
1848
			tdu->ndata -= nr;
1849
			coherence();
1850
		}
1851
		if(tdu->ndata == 0){
1852
			itdinit(iso, tdu);
1853
			iso->tdu = tdu->next;
1854
		}
1855
	}
1856
	return tot;
1857
}
1858
 
1859
static long
1860
episofscpy(Ctlr *ctlr, Ep *ep, Isoio* iso, uchar *b, long count)
1861
{
1862
	int nr;
1863
	long tot;
1864
	Sitd *stdu;
1865
 
1866
	for(tot = 0; iso->stdi != iso->stdu && tot < count; tot += nr){
1867
		stdu = iso->stdu;
1868
		if(stdu->csw & Stdactive){
1869
			diprint("ehci: episoread: %#p tdu active\n", iso);
1870
			break;
1871
		}
1872
		nr = stdu->ndata;
1873
		if(tot + nr > count)
1874
			nr = count - tot;
1875
		if(nr == 0)
1876
			print("ehci: ep%d.%d: too many polls\n",
1877
				ep->dev->nb, ep->nb);
1878
		else{
1879
			iunlock(ctlr);		/* We could page fault here */
1880
			memmove(b+tot, stdu->data, nr);
1881
			ilock(ctlr);
1882
			if(nr < stdu->ndata)
1883
				memmove(stdu->data, stdu->data+nr,
1884
					stdu->ndata - nr);
1885
			stdu->ndata -= nr;
1886
			coherence();
1887
		}
1888
		if(stdu->ndata == 0){
1889
			sitdinit(iso, stdu);
1890
			iso->stdu = stdu->next;
1891
		}
1892
	}
1893
	return tot;
1894
}
1895
 
1896
static long
1897
episoread(Ep *ep, Isoio *iso, void *a, long count)
1898
{
1899
	Ctlr *ctlr;
1900
	uchar *b;
1901
	long tot;
1902
 
1903
	iso->debug = ep->debug;
1904
	diprint("ehci: episoread: %#p ep%d.%d\n", iso, ep->dev->nb, ep->nb);
1905
 
1906
	b = a;
1907
	ctlr = ep->hp->aux;
1908
	qlock(iso);
1909
	if(waserror()){
1910
		qunlock(iso);
1911
		nexterror();
1912
	}
1913
	iso->err = nil;
1914
	iso->nerrs = 0;
1915
	ilock(ctlr);
1916
	if(iso->state == Qclose){
1917
		iunlock(ctlr);
1918
		error(iso->err ? iso->err : Eio);
1919
	}
1920
	iso->state = Qrun;
1921
	coherence();
1922
	while(isocanread(iso) == 0){
1923
		iunlock(ctlr);
1924
		diprint("ehci: episoread: %#p sleep\n", iso);
1925
		if(waserror()){
1926
			if(iso->err == nil)
1927
				iso->err = "I/O timed out";
1928
			ilock(ctlr);
1929
			break;
1930
		}
1931
		tsleep(iso, isocanread, iso, ep->tmout);
1932
		poperror();
1933
		ilock(ctlr);
1934
	}
1935
	if(iso->state == Qclose){
1936
		iunlock(ctlr);
1937
		error(iso->err ? iso->err : Eio);
1938
	}
1939
	iso->state = Qdone;
1940
	coherence();
1941
	assert(iso->tdu != iso->tdi);
1942
 
1943
	if(iso->hs != 0)
1944
		tot = episohscpy(ctlr, ep, iso, b, count);
1945
	else
1946
		tot = episofscpy(ctlr, ep, iso, b, count);
1947
	iunlock(ctlr);
1948
	qunlock(iso);
1949
	poperror();
1950
	diprint("uhci: episoread: %#p %uld bytes err '%s'\n", iso, tot, iso->err);
1951
	if(iso->err != nil)
1952
		error(iso->err);
1953
	return tot;
1954
}
1955
 
1956
/*
1957
 * iso->tdu is the next place to put data. When it gets full
1958
 * it is activated and tdu advanced.
1959
 */
1960
static long
1961
putsamples(Isoio *iso, uchar *b, long count)
1962
{
1963
	long tot, n;
1964
 
1965
	for(tot = 0; isocanwrite(iso) && tot < count; tot += n){
1966
		n = count-tot;
1967
		if(iso->hs != 0){
1968
			if(n > iso->tdu->mdata - iso->nleft)
1969
				n = iso->tdu->mdata - iso->nleft;
1970
			memmove(iso->tdu->data + iso->nleft, b + tot, n);
1971
			coherence();
1972
			iso->nleft += n;
1973
			if(iso->nleft == iso->tdu->mdata){
1974
				itdinit(iso, iso->tdu);
1975
				iso->nleft = 0;
1976
				iso->tdu = iso->tdu->next;
1977
			}
1978
		}else{
1979
			if(n > iso->stdu->mdata - iso->nleft)
1980
				n = iso->stdu->mdata - iso->nleft;
1981
			memmove(iso->stdu->data + iso->nleft, b + tot, n);
1982
			coherence();
1983
			iso->nleft += n;
1984
			if(iso->nleft == iso->stdu->mdata){
1985
				sitdinit(iso, iso->stdu);
1986
				iso->nleft = 0;
1987
				iso->stdu = iso->stdu->next;
1988
			}
1989
		}
1990
	}
1991
	return tot;
1992
}
1993
 
1994
/*
1995
 * Queue data for writing and return error status from
1996
 * last writes done, to maintain buffered data.
1997
 */
1998
static long
1999
episowrite(Ep *ep, Isoio *iso, void *a, long count)
2000
{
2001
	Ctlr *ctlr;
2002
	uchar *b;
2003
	int tot, nw;
2004
	char *err;
2005
 
2006
	iso->debug = ep->debug;
2007
	diprint("ehci: episowrite: %#p ep%d.%d\n", iso, ep->dev->nb, ep->nb);
2008
 
2009
	ctlr = ep->hp->aux;
2010
	qlock(iso);
2011
	if(waserror()){
2012
		qunlock(iso);
2013
		nexterror();
2014
	}
2015
	ilock(ctlr);
2016
	if(iso->state == Qclose){
2017
		iunlock(ctlr);
2018
		error(iso->err ? iso->err : Eio);
2019
	}
2020
	iso->state = Qrun;
2021
	coherence();
2022
	b = a;
2023
	for(tot = 0; tot < count; tot += nw){
2024
		while(isocanwrite(iso) == 0){
2025
			iunlock(ctlr);
2026
			diprint("ehci: episowrite: %#p sleep\n", iso);
2027
			if(waserror()){
2028
				if(iso->err == nil)
2029
					iso->err = "I/O timed out";
2030
				ilock(ctlr);
2031
				break;
2032
			}
2033
			tsleep(iso, isocanwrite, iso, ep->tmout);
2034
			poperror();
2035
			ilock(ctlr);
2036
		}
2037
		err = iso->err;
2038
		iso->err = nil;
2039
		if(iso->state == Qclose || err != nil){
2040
			iunlock(ctlr);
2041
			error(err ? err : Eio);
2042
		}
2043
		if(iso->state != Qrun)
2044
			panic("episowrite: iso not running");
2045
		iunlock(ctlr);		/* We could page fault here */
2046
		nw = putsamples(iso, b+tot, count-tot);
2047
		ilock(ctlr);
2048
	}
2049
	if(iso->state != Qclose)
2050
		iso->state = Qdone;
2051
	iunlock(ctlr);
2052
	err = iso->err;		/* in case it failed early */
2053
	iso->err = nil;
2054
	qunlock(iso);
2055
	poperror();
2056
	if(err != nil)
2057
		error(err);
2058
	diprint("ehci: episowrite: %#p %d bytes\n", iso, tot);
2059
	return tot;
2060
}
2061
 
2062
static int
2063
nexttoggle(int toggle, int count, int maxpkt)
2064
{
2065
	int np;
2066
 
2067
	np = count / maxpkt;
2068
	if(np == 0)
2069
		np = 1;
2070
	if((np % 2) == 0)
2071
		return toggle;
2072
	if(toggle == Tddata1)
2073
		return Tddata0;
2074
	else
2075
		return Tddata1;
2076
}
2077
 
2078
static Td*
2079
epgettd(Qio *io, int flags, void *a, int count, int maxpkt)
2080
{
2081
	Td *td;
2082
	ulong pa;
2083
	int i;
2084
 
2085
	if(count > Tdmaxpkt)
2086
		panic("ehci: epgettd: too many bytes");
2087
	td = tdalloc();
2088
	td->csw = flags | io->toggle | io->tok | count << Tdlenshift |
2089
		Tderr2 | Tderr1;
2090
 
2091
	/*
2092
	 * use the space wasted by alignment as an
2093
	 * embedded buffer if count bytes fit in there.
2094
	 */
2095
	assert(Align > sizeof(Td));
2096
	if(count <= Align - sizeof(Td)){
2097
		td->data = td->sbuff;
2098
		td->buff = nil;
2099
	}else
2100
		td->data = td->buff = smalloc(Tdmaxpkt);
2101
 
2102
	pa = PADDR(td->data);
2103
	for(i = 0; i < nelem(td->buffer); i++){
2104
		td->buffer[i] = pa;
2105
		if(i > 0)
2106
			td->buffer[i] &= ~0xFFF;
2107
		pa += 0x1000;
2108
	}
2109
	td->ndata = count;
2110
	if(a != nil && count > 0)
2111
		memmove(td->data, a, count);
2112
	coherence();
2113
	io->toggle = nexttoggle(io->toggle, count, maxpkt);
2114
	coherence();
2115
	return td;
2116
}
2117
 
2118
/*
2119
 * Try to get them idle
2120
 */
2121
static void
2122
aborttds(Qh *qh)
2123
{
2124
	Td *td;
2125
 
2126
	qh->state = Qdone;
2127
	coherence();
2128
	if(qh->sched >= 0 && (qh->eps0 & Qhspeedmask) != Qhhigh)
2129
		qh->eps0 |= Qhint;	/* inactivate on next pass */
2130
	coherence();
2131
	for(td = qh->tds; td != nil; td = td->next){
2132
		if(td->csw & Tdactive)
2133
			td->ndata = 0;
2134
		td->csw |= Tdhalt;
2135
		coherence();
2136
	}
2137
}
2138
 
2139
/*
2140
 * Some controllers do not post the usb/error interrupt after
2141
 * the work has been done. It seems that we must poll for them.
2142
 */
2143
static int
2144
workpending(void *a)
2145
{
2146
	Ctlr *ctlr;
2147
 
2148
	ctlr = a;
2149
	return ctlr->nreqs > 0;
2150
}
2151
 
2152
static void
2153
ehcipoll(void* a)
2154
{
2155
	Hci *hp;
2156
	Ctlr *ctlr;
2157
	Poll *poll;
2158
	int i;
2159
 
2160
	hp = a;
2161
	ctlr = hp->aux;
2162
	poll = &ctlr->poll;
2163
	for(;;){
2164
		if(ctlr->nreqs == 0){
2165
			if(0)ddprint("ehcipoll %#p sleep\n", ctlr->capio);
2166
			sleep(poll, workpending, ctlr);
2167
			if(0)ddprint("ehcipoll %#p awaken\n", ctlr->capio);
2168
		}
2169
		for(i = 0; i < 16 && ctlr->nreqs > 0; i++)
2170
			if(ehciintr(hp) == 0)
2171
				 break;
2172
		do{
2173
			tsleep(&up->sleep, return0, 0, 1);
2174
			ehciintr(hp);
2175
		}while(ctlr->nreqs > 0);
2176
	}
2177
}
2178
 
2179
static void
2180
pollcheck(Hci *hp)
2181
{
2182
	Ctlr *ctlr;
2183
	Poll *poll;
2184
 
2185
	ctlr = hp->aux;
2186
	poll = &ctlr->poll;
2187
 
2188
	if(poll->must != 0 && poll->does == 0){
2189
		lock(poll);
2190
		if(poll->must != 0 && poll->does == 0){
2191
			poll->does++;
2192
			print("ehci %#p: polling\n", ctlr->capio);
2193
			kproc("ehcipoll", ehcipoll, hp);
2194
		}
2195
		unlock(poll);
2196
	}
2197
}
2198
 
2199
static int
2200
epiodone(void *a)
2201
{
2202
	Qh *qh;
2203
 
2204
	qh = a;
2205
	return qh->state != Qrun;
2206
}
2207
 
2208
static void
2209
epiowait(Hci *hp, Qio *io, int tmout, ulong load)
2210
{
2211
	Qh *qh;
2212
	int timedout;
2213
	Ctlr *ctlr;
2214
 
2215
	ctlr = hp->aux;
2216
	qh = io->qh;
2217
	ddqprint("ehci %#p: io %#p sleep on qh %#p state %s\n",
2218
		ctlr->capio, io, qh, qhsname[qh->state]);
2219
	timedout = 0;
2220
	if(waserror()){
2221
		dqprint("ehci %#p: io %#p qh %#p timed out\n",
2222
			ctlr->capio, io, qh);
2223
		timedout++;
2224
	}else{
2225
		if(tmout == 0)
2226
			sleep(io, epiodone, qh);
2227
		else
2228
			tsleep(io, epiodone, qh, tmout);
2229
		poperror();
2230
	}
2231
 
2232
	ilock(ctlr);
2233
	/* Are we missing interrupts? */
2234
	if(qh->state == Qrun){
2235
		iunlock(ctlr);
2236
		ehciintr(hp);
2237
		ilock(ctlr);
2238
		if(qh->state == Qdone){
2239
			dqprint("ehci %#p: polling required\n", ctlr->capio);
2240
			ctlr->poll.must = 1;
2241
			pollcheck(hp);
2242
		}
2243
	}
2244
 
2245
	if(qh->state == Qrun){
2246
//		dqprint("ehci %#p: io %#p qh %#p timed out (no intr?)\n",
2247
		iprint("ehci %#p: io %#p qh %#p timed out (no intr?)\n",
2248
			ctlr->capio, io, qh);
2249
		timedout = 1;
2250
	}else if(qh->state != Qdone && qh->state != Qclose)
2251
		panic("ehci: epio: queue state %d", qh->state);
2252
	if(timedout){
2253
		aborttds(io->qh);
2254
		io->err = "request timed out";
2255
		iunlock(ctlr);
2256
		if(!waserror()){
2257
			tsleep(&up->sleep, return0, 0, Abortdelay);
2258
			poperror();
2259
		}
2260
		ilock(ctlr);
2261
	}
2262
	if(qh->state != Qclose)
2263
		qh->state = Qidle;
2264
	coherence();
2265
	qhlinktd(qh, nil);
2266
	ctlr->load -= load;
2267
	ctlr->nreqs--;
2268
	iunlock(ctlr);
2269
}
2270
 
2271
/*
2272
 * Non iso I/O.
2273
 * To make it work for control transfers, the caller may
2274
 * lock the Qio for the entire control transfer.
2275
 */
2276
static long
2277
epio(Ep *ep, Qio *io, void *a, long count, int mustlock)
2278
{
2279
	int saved, ntds, tmout;
2280
	long n, tot;
2281
	ulong load;
2282
	char *err;
2283
	char buf[128];
2284
	uchar *c;
2285
	Ctlr *ctlr;
2286
	Qh* qh;
2287
	Td *td, *ltd, *td0, *ntd;
2288
 
2289
	qh = io->qh;
2290
	ctlr = ep->hp->aux;
2291
	io->debug = ep->debug;
2292
	tmout = ep->tmout;
2293
	ddeprint("epio: %s ep%d.%d io %#p count %ld load %uld\n",
2294
		io->tok == Tdtokin ? "in" : "out",
2295
		ep->dev->nb, ep->nb, io, count, ctlr->load);
2296
	if((ehcidebug > 1 || ep->debug > 1) && io->tok != Tdtokin){
2297
		seprintdata(buf, buf+sizeof(buf), a, count);
2298
		print("echi epio: user data: %s\n", buf);
2299
	}
2300
	if(mustlock){
2301
		qlock(io);
2302
		if(waserror()){
2303
			qunlock(io);
2304
			nexterror();
2305
		}
2306
	}
2307
	io->err = nil;
2308
	ilock(ctlr);
2309
	if(qh->state == Qclose){	/* Tds released by cancelio */
2310
		iunlock(ctlr);
2311
		error(io->err ? io->err : Eio);
2312
	}
2313
	if(qh->state != Qidle)
2314
		panic("epio: qh not idle");
2315
	qh->state = Qinstall;
2316
	iunlock(ctlr);
2317
 
2318
	c = a;
2319
	td0 = ltd = nil;
2320
	load = tot = 0;
2321
	do{
2322
		n = (Tdmaxpkt / ep->maxpkt) * ep->maxpkt;
2323
		if(count-tot < n)
2324
			n = count-tot;
2325
		if(c != nil && io->tok != Tdtokin)
2326
			td = epgettd(io, Tdactive, c+tot, n, ep->maxpkt);
2327
		else
2328
			td = epgettd(io, Tdactive, nil, n, ep->maxpkt);
2329
		if(td0 == nil)
2330
			td0 = td;
2331
		else
2332
			tdlinktd(ltd, td);
2333
		ltd = td;
2334
		tot += n;
2335
		load += ep->load;
2336
	}while(tot < count);
2337
	if(td0 == nil || ltd == nil)
2338
		panic("epio: no td");
2339
 
2340
	ltd->csw |= Tdioc;		/* the last one interrupts */
2341
	coherence();
2342
 
2343
	ddeprint("ehci: load %uld ctlr load %uld\n", load, ctlr->load);
2344
	if(ehcidebug > 1 || ep->debug > 1)
2345
		dumptd(td0, "epio: put: ");
2346
 
2347
	ilock(ctlr);
2348
	if(qh->state != Qclose){
2349
		io->iotime = TK2MS(MACHP(0)->ticks);
2350
		qh->state = Qrun;
2351
		coherence();
2352
		qhlinktd(qh, td0);
2353
		ctlr->nreqs++;
2354
		ctlr->load += load;
2355
	}
2356
	iunlock(ctlr);
2357
 
2358
	if(ctlr->poll.does)
2359
		wakeup(&ctlr->poll);
2360
 
2361
	epiowait(ep->hp, io, tmout, load);
2362
	if(ehcidebug > 1 || ep->debug > 1){
2363
		dumptd(td0, "epio: got: ");
2364
		qhdump(qh);
2365
	}
2366
 
2367
	tot = 0;
2368
	c = a;
2369
	saved = 0;
2370
	ntds = 0;
2371
	for(td = td0; td != nil; td = ntd){
2372
		ntds++;
2373
		/*
2374
		 * Use td tok, not io tok, because of setup packets.
2375
		 * Also, we must save the next toggle value from the
2376
		 * last completed Td (in case of a short packet, or
2377
		 * fewer than the requested number of packets in the
2378
		 * Td being transferred).
2379
		 */
2380
		if(td->csw & (Tdhalt|Tdactive))
2381
			saved++;
2382
		else{
2383
			if(!saved){
2384
				io->toggle = td->csw & Tddata1;
2385
				coherence();
2386
			}
2387
			tot += td->ndata;
2388
			if(c != nil && (td->csw & Tdtok) == Tdtokin && td->ndata > 0){
2389
				memmove(c, td->data, td->ndata);
2390
				c += td->ndata;
2391
			}
2392
		}
2393
		ntd = td->next;
2394
		tdfree(td);
2395
	}
2396
	err = io->err;
2397
	if(mustlock){
2398
		qunlock(io);
2399
		poperror();
2400
	}
2401
	ddeprint("epio: io %#p: %d tds: return %ld err '%s'\n",
2402
		io, ntds, tot, err);
2403
	if(err == Estalled)
2404
		return 0;	/* that's our convention */
2405
	if(err != nil)
2406
		error(err);
2407
	if(tot < 0)
2408
		error(Eio);
2409
	return tot;
2410
}
2411
 
2412
static long
2413
epread(Ep *ep, void *a, long count)
2414
{
2415
	Ctlio *cio;
2416
	Qio *io;
2417
	Isoio *iso;
2418
	char buf[160];
2419
	ulong delta;
2420
 
2421
	ddeprint("ehci: epread\n");
2422
	if(ep->aux == nil)
2423
		panic("epread: not open");
2424
 
2425
	pollcheck(ep->hp);
2426
 
2427
	switch(ep->ttype){
2428
	case Tctl:
2429
		cio = ep->aux;
2430
		qlock(cio);
2431
		if(waserror()){
2432
			qunlock(cio);
2433
			nexterror();
2434
		}
2435
		ddeprint("epread ctl ndata %d\n", cio->ndata);
2436
		if(cio->ndata < 0)
2437
			error("request expected");
2438
		else if(cio->ndata == 0){
2439
			cio->ndata = -1;
2440
			count = 0;
2441
		}else{
2442
			if(count > cio->ndata)
2443
				count = cio->ndata;
2444
			if(count > 0)
2445
				memmove(a, cio->data, count);
2446
			/* BUG for big transfers */
2447
			free(cio->data);
2448
			cio->data = nil;
2449
			cio->ndata = 0;	/* signal EOF next time */
2450
		}
2451
		qunlock(cio);
2452
		poperror();
2453
		if(ehcidebug>1 || ep->debug){
2454
			seprintdata(buf, buf+sizeof(buf), a, count);
2455
			print("epread: %s\n", buf);
2456
		}
2457
		return count;
2458
	case Tbulk:
2459
		io = ep->aux;
2460
		if(ep->clrhalt)
2461
			clrhalt(ep);
2462
		return epio(ep, &io[OREAD], a, count, 1);
2463
	case Tintr:
2464
		io = ep->aux;
2465
		delta = TK2MS(MACHP(0)->ticks) - io[OREAD].iotime + 1;
2466
		if(delta < ep->pollival / 2)
2467
			tsleep(&up->sleep, return0, 0, ep->pollival/2 - delta);
2468
		if(ep->clrhalt)
2469
			clrhalt(ep);
2470
		return epio(ep, &io[OREAD], a, count, 1);
2471
	case Tiso:
2472
		iso = ep->aux;
2473
		return episoread(ep, iso, a, count);
2474
	}
2475
	return -1;
2476
}
2477
 
2478
/*
2479
 * Control transfers are one setup write (data0)
2480
 * plus zero or more reads/writes (data1, data0, ...)
2481
 * plus a final write/read with data1 to ack.
2482
 * For both host to device and device to host we perform
2483
 * the entire transfer when the user writes the request,
2484
 * and keep any data read from the device for a later read.
2485
 * We call epio three times instead of placing all Tds at
2486
 * the same time because doing so leads to crc/tmout errors
2487
 * for some devices.
2488
 * Upon errors on the data phase we must still run the status
2489
 * phase or the device may cease responding in the future.
2490
 */
2491
static long
2492
epctlio(Ep *ep, Ctlio *cio, void *a, long count)
2493
{
2494
	uchar *c;
2495
	long len;
2496
 
2497
	ddeprint("epctlio: cio %#p ep%d.%d count %ld\n",
2498
		cio, ep->dev->nb, ep->nb, count);
2499
	if(count < Rsetuplen)
2500
		error("short usb comand");
2501
	qlock(cio);
2502
	free(cio->data);
2503
	cio->data = nil;
2504
	cio->ndata = 0;
2505
	if(waserror()){
2506
		free(cio->data);
2507
		cio->data = nil;
2508
		cio->ndata = 0;
2509
		qunlock(cio);
2510
		nexterror();
2511
	}
2512
 
2513
	/* set the address if unset and out of configuration state */
2514
	if(ep->dev->state != Dconfig && ep->dev->state != Dreset)
2515
		if(cio->usbid == 0){
2516
			cio->usbid = (ep->nb&Epmax) << 7 | ep->dev->nb&Devmax;
2517
			coherence();
2518
			qhsetaddr(cio->qh, cio->usbid);
2519
		}
2520
	/* adjust maxpkt if the user has learned a different one */
2521
	if(qhmaxpkt(cio->qh) != ep->maxpkt)
2522
		qhsetmaxpkt(cio->qh, ep->maxpkt);
2523
	c = a;
2524
	cio->tok = Tdtoksetup;
2525
	cio->toggle = Tddata0;
2526
	coherence();
2527
	if(epio(ep, cio, a, Rsetuplen, 0) < Rsetuplen)
2528
		error(Eio);
2529
	a = c + Rsetuplen;
2530
	count -= Rsetuplen;
2531
 
2532
	cio->toggle = Tddata1;
2533
	if(c[Rtype] & Rd2h){
2534
		cio->tok = Tdtokin;
2535
		len = GET2(c+Rcount);
2536
		if(len <= 0)
2537
			error("bad length in d2h request");
2538
		if(len > Maxctllen)
2539
			error("d2h data too large to fit in ehci");
2540
		a = cio->data = smalloc(len+1);
2541
	}else{
2542
		cio->tok = Tdtokout;
2543
		len = count;
2544
	}
2545
	coherence();
2546
	if(len > 0)
2547
		if(waserror())
2548
			len = -1;
2549
		else{
2550
			len = epio(ep, cio, a, len, 0);
2551
			poperror();
2552
		}
2553
	if(c[Rtype] & Rd2h){
2554
		count = Rsetuplen;
2555
		cio->ndata = len;
2556
		cio->tok = Tdtokout;
2557
	}else{
2558
		if(len < 0)
2559
			count = -1;
2560
		else
2561
			count = Rsetuplen + len;
2562
		cio->tok = Tdtokin;
2563
	}
2564
	cio->toggle = Tddata1;
2565
	coherence();
2566
	epio(ep, cio, nil, 0, 0);
2567
	qunlock(cio);
2568
	poperror();
2569
	ddeprint("epctlio cio %#p return %ld\n", cio, count);
2570
	return count;
2571
}
2572
 
2573
static long
2574
epwrite(Ep *ep, void *a, long count)
2575
{
2576
	Qio *io;
2577
	Ctlio *cio;
2578
	Isoio *iso;
2579
	ulong delta;
2580
 
2581
	pollcheck(ep->hp);
2582
 
2583
	ddeprint("ehci: epwrite ep%d.%d\n", ep->dev->nb, ep->nb);
2584
	if(ep->aux == nil)
2585
		panic("ehci: epwrite: not open");
2586
	switch(ep->ttype){
2587
	case Tctl:
2588
		cio = ep->aux;
2589
		return epctlio(ep, cio, a, count);
2590
	case Tbulk:
2591
		io = ep->aux;
2592
		if(ep->clrhalt)
2593
			clrhalt(ep);
2594
		return epio(ep, &io[OWRITE], a, count, 1);
2595
	case Tintr:
2596
		io = ep->aux;
2597
		delta = TK2MS(MACHP(0)->ticks) - io[OWRITE].iotime + 1;
2598
		if(delta < ep->pollival)
2599
			tsleep(&up->sleep, return0, 0, ep->pollival - delta);
2600
		if(ep->clrhalt)
2601
			clrhalt(ep);
2602
		return epio(ep, &io[OWRITE], a, count, 1);
2603
	case Tiso:
2604
		iso = ep->aux;
2605
		return episowrite(ep, iso, a, count);
2606
	}
2607
	return -1;
2608
}
2609
 
2610
static void
2611
isofsinit(Ep *ep, Isoio *iso)
2612
{
2613
	long left;
2614
	Sitd *td, *ltd;
2615
	int i;
2616
	ulong frno;
2617
 
2618
	left = 0;
2619
	ltd = nil;
2620
	frno = iso->td0frno;
2621
	for(i = 0; i < iso->nframes; i++){
2622
		td = sitdalloc();
2623
		td->data = iso->data + i * ep->maxpkt;
2624
		td->epc = ep->dev->port << Stdportshift;
2625
		td->epc |= ep->dev->hub << Stdhubshift;
2626
		td->epc |= ep->nb << Stdepshift;
2627
		td->epc |= ep->dev->nb << Stddevshift;
2628
		td->mfs = 034 << Stdscmshift | 1 << Stdssmshift;
2629
		if(ep->mode == OREAD){
2630
			td->epc |= Stdin;
2631
			td->mdata = ep->maxpkt;
2632
		}else{
2633
			td->mdata = (ep->hz+left) * ep->pollival / 1000;
2634
			td->mdata *= ep->samplesz;
2635
			left = (ep->hz+left) * ep->pollival % 1000;
2636
			if(td->mdata > ep->maxpkt){
2637
				print("ehci: ep%d.%d: size > maxpkt\n",
2638
					ep->dev->nb, ep->nb);
2639
				print("size = %ld max = %ld\n",
2640
					td->mdata,ep->maxpkt);
2641
				td->mdata = ep->maxpkt;
2642
			}
2643
		}
2644
		coherence();
2645
 
2646
		iso->sitdps[frno] = td;
2647
		coherence();
2648
		sitdinit(iso, td);
2649
		if(ltd != nil)
2650
			ltd->next = td;
2651
		ltd = td;
2652
		frno = TRUNC(frno+ep->pollival, Nisoframes);
2653
	}
2654
	ltd->next = iso->sitdps[iso->td0frno];
2655
	coherence();
2656
}
2657
 
2658
static void
2659
isohsinit(Ep *ep, Isoio *iso)
2660
{
2661
	int ival, p;
2662
	long left;
2663
	ulong frno, i, pa;
2664
	Itd *ltd, *td;
2665
 
2666
	iso->hs = 1;
2667
	ival = 1;
2668
	if(ep->pollival > 8)
2669
		ival = ep->pollival/8;
2670
	left = 0;
2671
	ltd = nil;
2672
	frno = iso->td0frno;
2673
	for(i = 0; i < iso->nframes; i++){
2674
		td = itdalloc();
2675
		td->data = iso->data + i * 8 * iso->maxsize;
2676
		pa = PADDR(td->data) & ~0xFFF;
2677
		for(p = 0; p < 8; p++)
2678
			td->buffer[i] = pa + p * 0x1000;
2679
		td->buffer[0] = PADDR(iso->data) & ~0xFFF |
2680
			ep->nb << Itdepshift | ep->dev->nb << Itddevshift;
2681
		if(ep->mode == OREAD)
2682
			td->buffer[1] |= Itdin;
2683
		else
2684
			td->buffer[1] |= Itdout;
2685
		td->buffer[1] |= ep->maxpkt << Itdmaxpktshift;
2686
		td->buffer[2] |= ep->ntds << Itdntdsshift;
2687
 
2688
		if(ep->mode == OREAD)
2689
			td->mdata = 8 * iso->maxsize;
2690
		else{
2691
			td->mdata = (ep->hz + left) * ep->pollival / 1000;
2692
			td->mdata *= ep->samplesz;
2693
			left = (ep->hz + left) * ep->pollival % 1000;
2694
		}
2695
		coherence();
2696
		iso->itdps[frno] = td;
2697
		coherence();
2698
		itdinit(iso, td);
2699
		if(ltd != nil)
2700
			ltd->next = td;
2701
		ltd = td;
2702
		frno = TRUNC(frno + ival, Nisoframes);
2703
	}
2704
}
2705
 
2706
static void
2707
isoopen(Ctlr *ctlr, Ep *ep)
2708
{
2709
	int ival;		/* pollival in ms */
2710
	int tpf;		/* tds per frame */
2711
	int i, n, w, woff;
2712
	ulong frno;
2713
	Isoio *iso;
2714
 
2715
	iso = ep->aux;
2716
	switch(ep->mode){
2717
	case OREAD:
2718
		iso->tok = Tdtokin;
2719
		break;
2720
	case OWRITE:
2721
		iso->tok = Tdtokout;
2722
		break;
2723
	default:
2724
		error("iso i/o is half-duplex");
2725
	}
2726
	iso->usbid = ep->nb << 7 | ep->dev->nb & Devmax;
2727
	iso->state = Qidle;
2728
	coherence();
2729
	iso->debug = ep->debug;
2730
	ival = ep->pollival;
2731
	tpf = 1;
2732
	if(ep->dev->speed == Highspeed){
2733
		tpf = 8;
2734
		if(ival <= 8)
2735
			ival = 1;
2736
		else
2737
			ival /= 8;
2738
	}
2739
	assert(ival != 0);
2740
	iso->nframes = Nisoframes / ival;
2741
	if(iso->nframes < 3)
2742
		error("uhci isoopen bug");	/* we need at least 3 tds */
2743
	iso->maxsize = ep->ntds * ep->maxpkt;
2744
	if(ctlr->load + ep->load > 800)
2745
		print("usb: ehci: bandwidth may be exceeded\n");
2746
	ilock(ctlr);
2747
	ctlr->load += ep->load;
2748
	ctlr->isoload += ep->load;
2749
	ctlr->nreqs++;
2750
	dprint("ehci: load %uld isoload %uld\n", ctlr->load, ctlr->isoload);
2751
	diprint("iso nframes %d pollival %uld ival %d maxpkt %uld ntds %d\n",
2752
		iso->nframes, ep->pollival, ival, ep->maxpkt, ep->ntds);
2753
	iunlock(ctlr);
2754
	if(ctlr->poll.does)
2755
		wakeup(&ctlr->poll);
2756
 
2757
	/*
2758
	 * From here on this cannot raise errors
2759
	 * unless we catch them and release here all memory allocated.
2760
	 */
2761
	assert(ep->maxpkt > 0 && ep->ntds > 0 && ep->ntds < 4);
2762
	assert(ep->maxpkt <= 1024);
2763
	iso->tdps = smalloc(sizeof(uintptr) * Nisoframes);
2764
	iso->data = smalloc(iso->nframes * tpf * ep->ntds * ep->maxpkt);
2765
	iso->td0frno = TRUNC(ctlr->opio->frno + 10, Nisoframes);
2766
	/* read: now; write: 1s ahead */
2767
 
2768
	if(ep->dev->speed == Highspeed)
2769
		isohsinit(ep, iso);
2770
	else
2771
		isofsinit(ep, iso);
2772
	iso->tdu = iso->tdi = iso->itdps[iso->td0frno];
2773
	iso->stdu = iso->stdi = iso->sitdps[iso->td0frno];
2774
	coherence();
2775
 
2776
	ilock(ctlr);
2777
	frno = iso->td0frno;
2778
	for(i = 0; i < iso->nframes; i++){
2779
		*iso->tdps[frno] = ctlr->frames[frno];
2780
		frno = TRUNC(frno+ival, Nisoframes);
2781
	}
2782
 
2783
	/*
2784
	 * Iso uses a virtual frame window of Nisoframes, and we must
2785
	 * fill the actual ctlr frame array by placing ctlr->nframes/Nisoframes
2786
	 * copies of the window in the frame array.
2787
	 */
2788
	assert(ctlr->nframes >= Nisoframes && Nisoframes >= iso->nframes);
2789
	assert(Nisoframes >= Nintrleafs);
2790
	n = ctlr->nframes / Nisoframes;
2791
	for(w = 0; w < n; w++){
2792
		frno = iso->td0frno;
2793
		woff = w * Nisoframes;
2794
		for(i = 0; i < iso->nframes ; i++){
2795
			assert(woff+frno < ctlr->nframes);
2796
			assert(iso->tdps[frno] != nil);
2797
			if(ep->dev->speed == Highspeed)
2798
				ctlr->frames[woff+frno] = PADDR(iso->tdps[frno])
2799
					|Litd;
2800
			else
2801
				ctlr->frames[woff+frno] = PADDR(iso->tdps[frno])
2802
					|Lsitd;
2803
			coherence();
2804
			frno = TRUNC(frno+ep->pollival, Nisoframes);
2805
		}
2806
	}
2807
	coherence();
2808
	iso->next = ctlr->iso;
2809
	ctlr->iso = iso;
2810
	coherence();
2811
	iso->state = Qdone;
2812
	iunlock(ctlr);
2813
	if(ehcidebug > 1 || iso->debug >1)
2814
		isodump(iso, 0);
2815
}
2816
 
2817
/*
2818
 * Allocate the endpoint and set it up for I/O
2819
 * in the controller. This must follow what's said
2820
 * in Ep regarding configuration, including perhaps
2821
 * the saved toggles (saved on a previous close of
2822
 * the endpoint data file by epclose).
2823
 */
2824
static void
2825
epopen(Ep *ep)
2826
{
2827
	Ctlr *ctlr;
2828
	Ctlio *cio;
2829
	Qio *io;
2830
	int usbid;
2831
 
2832
	ctlr = ep->hp->aux;
2833
	deprint("ehci: epopen ep%d.%d\n", ep->dev->nb, ep->nb);
2834
	if(ep->aux != nil)
2835
		panic("ehci: epopen called with open ep");
2836
	if(waserror()){
2837
		free(ep->aux);
2838
		ep->aux = nil;
2839
		nexterror();
2840
	}
2841
	switch(ep->ttype){
2842
	case Tnone:
2843
		error("endpoint not configured");
2844
	case Tiso:
2845
		ep->aux = smalloc(sizeof(Isoio));
2846
		isoopen(ctlr, ep);
2847
		break;
2848
	case Tctl:
2849
		cio = ep->aux = smalloc(sizeof(Ctlio));
2850
		cio->debug = ep->debug;
2851
		cio->ndata = -1;
2852
		cio->data = nil;
2853
		if(ep->dev->isroot != 0 && ep->nb == 0)	/* root hub */
2854
			break;
2855
		cio->qh = qhalloc(ctlr, ep, cio, "epc");
2856
		break;
2857
	case Tbulk:
2858
		ep->pollival = 1;	/* assume this; doesn't really matter */
2859
		/* and fall... */
2860
	case Tintr:
2861
		io = ep->aux = smalloc(sizeof(Qio)*2);
2862
		io[OREAD].debug = io[OWRITE].debug = ep->debug;
2863
		usbid = (ep->nb&Epmax) << 7 | ep->dev->nb &Devmax;
2864
		assert(ep->pollival != 0);
2865
		if(ep->mode != OREAD){
2866
			if(ep->toggle[OWRITE] != 0)
2867
				io[OWRITE].toggle = Tddata1;
2868
			else
2869
				io[OWRITE].toggle = Tddata0;
2870
			io[OWRITE].tok = Tdtokout;
2871
			io[OWRITE].usbid = usbid;
2872
			io[OWRITE].bw = ep->maxpkt*1000/ep->pollival; /* bytes/s */
2873
			io[OWRITE].qh = qhalloc(ctlr, ep, io+OWRITE, "epw");
2874
		}
2875
		if(ep->mode != OWRITE){
2876
			if(ep->toggle[OREAD] != 0)
2877
				io[OREAD].toggle = Tddata1;
2878
			else
2879
				io[OREAD].toggle = Tddata0;
2880
			io[OREAD].tok = Tdtokin;
2881
			io[OREAD].usbid = usbid;
2882
			io[OREAD].bw = ep->maxpkt*1000/ep->pollival; /* bytes/s */
2883
			io[OREAD].qh = qhalloc(ctlr, ep, io+OREAD, "epr");
2884
		}
2885
		break;
2886
	}
2887
	coherence();
2888
	if(ehcidebug>1 || ep->debug)
2889
		dump(ep->hp);
2890
	deprint("ehci: epopen done\n");
2891
	poperror();
2892
}
2893
 
2894
static void
2895
cancelio(Ctlr *ctlr, Qio *io)
2896
{
2897
	Qh *qh;
2898
 
2899
	ilock(ctlr);
2900
	qh = io->qh;
2901
	if(io == nil || io->qh == nil || io->qh->state == Qclose){
2902
		iunlock(ctlr);
2903
		return;
2904
	}
2905
	dqprint("ehci: cancelio for qh %#p state %s\n",
2906
		qh, qhsname[qh->state]);
2907
	aborttds(qh);
2908
	qh->state = Qclose;
2909
	iunlock(ctlr);
2910
	if(!waserror()){
2911
		tsleep(&up->sleep, return0, 0, Abortdelay);
2912
		poperror();
2913
	}
2914
	wakeup(io);
2915
	qlock(io);
2916
	/* wait for epio if running */
2917
	qunlock(io);
2918
 
2919
	qhfree(ctlr, qh);
2920
	io->qh = nil;
2921
}
2922
 
2923
static void
2924
cancelisoio(Ctlr *ctlr, Isoio *iso, int pollival, ulong load)
2925
{
2926
	int frno, i, n, t, w, woff;
2927
	ulong *lp, *tp;
2928
	Isoio **il;
2929
	Itd *td;
2930
	Sitd *std;
2931
 
2932
	ilock(ctlr);
2933
	if(iso->state == Qclose){
2934
		iunlock(ctlr);
2935
		return;
2936
	}
2937
	ctlr->nreqs--;
2938
	if(iso->state != Qrun && iso->state != Qdone)
2939
		panic("bad iso state");
2940
	iso->state = Qclose;
2941
	coherence();
2942
	if(ctlr->isoload < load)
2943
		panic("ehci: low isoload");
2944
	ctlr->isoload -= load;
2945
	ctlr->load -= load;
2946
	for(il = &ctlr->iso; *il != nil; il = &(*il)->next)
2947
		if(*il == iso)
2948
			break;
2949
	if(*il == nil)
2950
		panic("cancleiso: not found");
2951
	*il = iso->next;
2952
 
2953
	frno = iso->td0frno;
2954
	for(i = 0; i < iso->nframes; i++){
2955
		tp = iso->tdps[frno];
2956
		if(iso->hs != 0){
2957
			td = iso->itdps[frno];
2958
			for(t = 0; t < nelem(td->csw); t++)
2959
				td->csw[t] &= ~(Itdioc|Itdactive);
2960
		}else{
2961
			std = iso->sitdps[frno];
2962
			std->csw &= ~(Stdioc|Stdactive);
2963
		}
2964
		coherence();
2965
		for(lp = &ctlr->frames[frno]; !(*lp & Lterm);
2966
		    lp = &LPTR(*lp)[0])
2967
			if(LPTR(*lp) == tp)
2968
				break;
2969
		if(*lp & Lterm)
2970
			panic("cancelisoio: td not found");
2971
		*lp = tp[0];
2972
		/*
2973
		 * Iso uses a virtual frame window of Nisoframes, and we must
2974
		 * restore pointers in copies of the window kept at ctlr->frames.
2975
		 */
2976
		if(lp == &ctlr->frames[frno]){
2977
			n = ctlr->nframes / Nisoframes;
2978
			for(w = 1; w < n; w++){
2979
				woff = w * Nisoframes;
2980
				ctlr->frames[woff+frno] = *lp;
2981
			}
2982
		}
2983
		coherence();
2984
		frno = TRUNC(frno+pollival, Nisoframes);
2985
	}
2986
	iunlock(ctlr);
2987
 
2988
	/*
2989
	 * wakeup anyone waiting for I/O and
2990
	 * wait to be sure no I/O is in progress in the controller.
2991
	 * and then wait to be sure episo* is no longer running.
2992
	 */
2993
	wakeup(iso);
2994
	diprint("cancelisoio iso %#p waiting for I/O to cease\n", iso);
2995
	tsleep(&up->sleep, return0, 0, 5);
2996
	qlock(iso);
2997
	qunlock(iso);
2998
	diprint("cancelisoio iso %#p releasing iso\n", iso);
2999
 
3000
	frno = iso->td0frno;
3001
	for(i = 0; i < iso->nframes; i++){
3002
		if(iso->hs != 0)
3003
			itdfree(iso->itdps[frno]);
3004
		else
3005
			sitdfree(iso->sitdps[frno]);
3006
		iso->tdps[frno] = nil;
3007
		frno = TRUNC(frno+pollival, Nisoframes);
3008
	}
3009
	free(iso->tdps);
3010
	iso->tdps = nil;
3011
	free(iso->data);
3012
	iso->data = nil;
3013
	coherence();
3014
}
3015
 
3016
static void
3017
epclose(Ep *ep)
3018
{
3019
	Qio *io;
3020
	Ctlio *cio;
3021
	Isoio *iso;
3022
	Ctlr *ctlr;
3023
 
3024
	ctlr = ep->hp->aux;
3025
	deprint("ehci: epclose ep%d.%d\n", ep->dev->nb, ep->nb);
3026
 
3027
	if(ep->aux == nil)
3028
		panic("ehci: epclose called with closed ep");
3029
	switch(ep->ttype){
3030
	case Tctl:
3031
		cio = ep->aux;
3032
		cancelio(ctlr, cio);
3033
		free(cio->data);
3034
		cio->data = nil;
3035
		break;
3036
	case Tintr:
3037
	case Tbulk:
3038
		io = ep->aux;
3039
		ep->toggle[OREAD] = ep->toggle[OWRITE] = 0;
3040
		if(ep->mode != OWRITE){
3041
			cancelio(ctlr, &io[OREAD]);
3042
			if(io[OREAD].toggle == Tddata1)
3043
				ep->toggle[OREAD] = 1;
3044
		}
3045
		if(ep->mode != OREAD){
3046
			cancelio(ctlr, &io[OWRITE]);
3047
			if(io[OWRITE].toggle == Tddata1)
3048
				ep->toggle[OWRITE] = 1;
3049
		}
3050
		coherence();
3051
		break;
3052
	case Tiso:
3053
		iso = ep->aux;
3054
		cancelisoio(ctlr, iso, ep->pollival, ep->load);
3055
		break;
3056
	default:
3057
		panic("epclose: bad ttype");
3058
	}
3059
	free(ep->aux);
3060
	ep->aux = nil;
3061
}
3062
 
3063
/*
3064
 * return smallest power of 2 >= n
3065
 */
3066
static int
3067
flog2(int n)
3068
{
3069
	int i;
3070
 
3071
	for(i = 0; (1 << i) < n; i++)
3072
		;
3073
	return i;
3074
}
3075
 
3076
/*
3077
 * build the periodic scheduling tree:
3078
 * framesize must be a multiple of the tree size
3079
 */
3080
static void
3081
mkqhtree(Ctlr *ctlr)
3082
{
3083
	int i, n, d, o, leaf0, depth;
3084
	ulong leafs[Nintrleafs];
3085
	Qh *qh;
3086
	Qh **tree;
3087
	Qtree *qt;
3088
 
3089
	depth = flog2(Nintrleafs);
3090
	n = (1 << (depth+1)) - 1;
3091
	qt = mallocz(sizeof(*qt), 1);
3092
	if(qt == nil)
3093
		panic("ehci: mkqhtree: no memory");
3094
	qt->nel = n;
3095
	qt->depth = depth;
3096
	qt->bw = mallocz(n * sizeof(qt->bw), 1);
3097
	qt->root = tree = mallocz(n * sizeof(Qh *), 1);
3098
	if(qt->bw == nil || tree == nil)
3099
		panic("ehci: mkqhtree: no memory");
3100
	for(i = 0; i < n; i++){
3101
		tree[i] = qh = edalloc();
3102
		if(qh == nil)
3103
			panic("ehci: mkqhtree: no memory");
3104
		qh->nlink = qh->alink = qh->link = Lterm;
3105
		qh->csw = Tdhalt;
3106
		qh->state = Qidle;
3107
		coherence();
3108
		if(i > 0)
3109
			qhlinkqh(tree[i], tree[(i-1)/2]);
3110
	}
3111
	ctlr->ntree = i;
3112
	dprint("ehci: tree: %d endpoints allocated\n", i);
3113
 
3114
	/* distribute leaves evenly round the frame list */
3115
	leaf0 = n / 2;
3116
	for(i = 0; i < Nintrleafs; i++){
3117
		o = 0;
3118
		for(d = 0; d < depth; d++){
3119
			o <<= 1;
3120
			if(i & (1 << d))
3121
				o |= 1;
3122
		}
3123
		if(leaf0 + o >= n){
3124
			print("leaf0=%d o=%d i=%d n=%d\n", leaf0, o, i, n);
3125
			break;
3126
		}
3127
		leafs[i] = PADDR(tree[leaf0 + o]) | Lqh;
3128
	}
3129
	assert((ctlr->nframes % Nintrleafs) == 0);
3130
	for(i = 0; i < ctlr->nframes; i += Nintrleafs){
3131
		memmove(ctlr->frames + i, leafs, sizeof leafs);
3132
		coherence();
3133
	}
3134
	ctlr->tree = qt;
3135
	coherence();
3136
}
3137
 
3138
void
3139
ehcimeminit(Ctlr *ctlr)
3140
{
3141
	int i, frsize;
3142
	Eopio *opio;
3143
 
3144
	opio = ctlr->opio;
3145
	frsize = ctlr->nframes * sizeof(ulong);
3146
	assert((frsize & 0xFFF) == 0);		/* must be 4k aligned */
3147
	ctlr->frames = xspanalloc(frsize, frsize, 0);
3148
	if(ctlr->frames == nil)
3149
		panic("ehci reset: no memory");
3150
 
3151
	for (i = 0; i < ctlr->nframes; i++)
3152
		ctlr->frames[i] = Lterm;
3153
	opio->frbase = PADDR(ctlr->frames);
3154
	opio->frno = 0;
3155
	coherence();
3156
 
3157
	qhalloc(ctlr, nil, nil, nil);	/* init async list */
3158
	mkqhtree(ctlr);			/* init sync list */
3159
	edfree(edalloc());		/* try to get some ones pre-allocated */
3160
 
3161
	dprint("ehci %#p flb %#lux frno %#lux\n",
3162
		ctlr->capio, opio->frbase, opio->frno);
3163
}
3164
 
3165
static void
3166
init(Hci *hp)
3167
{
3168
	Ctlr *ctlr;
3169
	Eopio *opio;
3170
	int i;
3171
	static int ctlrno;
3172
 
3173
	hp->highspeed = 1;
3174
	ctlr = hp->aux;
3175
	opio = ctlr->opio;
3176
	dprint("ehci %#p init\n", ctlr->capio);
3177
 
3178
	ilock(ctlr);
3179
	/*
3180
	 * Unless we activate frroll interrupt
3181
	 * some machines won't post other interrupts.
3182
	 */
3183
	opio->intr = Iusb|Ierr|Iportchg|Ihcerr|Iasync;
3184
	coherence();
3185
	opio->cmd |= Cpse;
3186
	coherence();
3187
	opio->cmd |= Case;
3188
	coherence();
3189
	ehcirun(ctlr, 1);
3190
	/*
3191
	 * route all ports by default to only one ehci (the first).
3192
	 * it's not obvious how multiple ehcis could work and on some
3193
	 * machines, setting Callmine on all ehcis makes the machine seize up.
3194
	 */
3195
	opio->config = (ctlrno == 0? Callmine: 0);
3196
	coherence();
3197
 
3198
	for (i = 0; i < hp->nports; i++)
3199
		opio->portsc[i] = Pspower;
3200
	iunlock(ctlr);
3201
	if(ehcidebug > 1)
3202
		dump(hp);
3203
	ctlrno++;
3204
}
3205
 
3206
void
3207
ehcilinkage(Hci *hp)
3208
{
3209
	hp->init = init;
3210
	hp->dump = dump;
3211
	hp->interrupt = interrupt;
3212
	hp->epopen = epopen;
3213
	hp->epclose = epclose;
3214
	hp->epread = epread;
3215
	hp->epwrite = epwrite;
3216
	hp->seprintep = seprintep;
3217
	hp->portenable = portenable;
3218
	hp->portreset = portreset;
3219
	hp->portstatus = portstatus;
3220
//	hp->shutdown = shutdown;
3221
//	hp->debug = setdebug;
3222
	hp->type = "ehci";
3223
}