ohci-q.c 31.3 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2
/*
 * OHCI HCD (Host Controller Driver) for USB.
3
 *
Linus Torvalds's avatar
Linus Torvalds committed
4 5
 * (C) Copyright 1999 Roman Weissgaerber <weissg@vienna.at>
 * (C) Copyright 2000-2002 David Brownell <dbrownell@users.sourceforge.net>
6
 *
Linus Torvalds's avatar
Linus Torvalds committed
7 8 9
 * This file is licenced under the GPL.
 */

10 11
#include <linux/irq.h>

Linus Torvalds's avatar
Linus Torvalds committed
12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
static void urb_free_priv (struct ohci_hcd *hc, urb_priv_t *urb_priv)
{
	int		last = urb_priv->length - 1;

	if (last >= 0) {
		int		i;
		struct td	*td;

		for (i = 0; i <= last; i++) {
			td = urb_priv->td [i];
			if (td)
				td_free (hc, td);
		}
	}

	list_del (&urb_priv->pending);
	kfree (urb_priv);
}

/*-------------------------------------------------------------------------*/

/*
 * URB goes back to driver, and isn't reissued.
 * It's completely gone from HC data structures.
 * PRECONDITION:  ohci lock held, irqs blocked.
 */
static void
39
finish_urb(struct ohci_hcd *ohci, struct urb *urb, int status)
Linus Torvalds's avatar
Linus Torvalds committed
40 41 42 43 44 45
__releases(ohci->lock)
__acquires(ohci->lock)
{
	// ASSERT (urb->hcpriv != 0);

	urb_free_priv (ohci, urb->hcpriv);
46 47
	if (likely(status == -EINPROGRESS))
		status = 0;
Linus Torvalds's avatar
Linus Torvalds committed
48 49 50 51 52 53 54 55 56 57 58

	switch (usb_pipetype (urb->pipe)) {
	case PIPE_ISOCHRONOUS:
		ohci_to_hcd(ohci)->self.bandwidth_isoc_reqs--;
		break;
	case PIPE_INTERRUPT:
		ohci_to_hcd(ohci)->self.bandwidth_int_reqs--;
		break;
	}

#ifdef OHCI_VERBOSE_DEBUG
59
	urb_print(urb, "RET", usb_pipeout (urb->pipe), status);
Linus Torvalds's avatar
Linus Torvalds committed
60 61 62
#endif

	/* urb->complete() can reenter this HCD */
63
	usb_hcd_unlink_urb_from_ep(ohci_to_hcd(ohci), urb);
Linus Torvalds's avatar
Linus Torvalds committed
64
	spin_unlock (&ohci->lock);
Alan Stern's avatar
Alan Stern committed
65
	usb_hcd_giveback_urb(ohci_to_hcd(ohci), urb, status);
Linus Torvalds's avatar
Linus Torvalds committed
66 67 68 69 70 71 72 73 74 75 76 77 78
	spin_lock (&ohci->lock);

	/* stop periodic dma if it's not needed */
	if (ohci_to_hcd(ohci)->self.bandwidth_isoc_reqs == 0
			&& ohci_to_hcd(ohci)->self.bandwidth_int_reqs == 0) {
		ohci->hc_control &= ~(OHCI_CTRL_PLE|OHCI_CTRL_IE);
		ohci_writel (ohci, ohci->hc_control, &ohci->regs->control);
	}
}


/*-------------------------------------------------------------------------*
 * ED handling functions
79
 *-------------------------------------------------------------------------*/
Linus Torvalds's avatar
Linus Torvalds committed
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105

/* search for the right schedule branch to use for a periodic ed.
 * does some load balancing; returns the branch, or negative errno.
 */
static int balance (struct ohci_hcd *ohci, int interval, int load)
{
	int	i, branch = -ENOSPC;

	/* iso periods can be huge; iso tds specify frame numbers */
	if (interval > NUM_INTS)
		interval = NUM_INTS;

	/* search for the least loaded schedule branch of that period
	 * that has enough bandwidth left unreserved.
	 */
	for (i = 0; i < interval ; i++) {
		if (branch < 0 || ohci->load [branch] > ohci->load [i]) {
			int	j;

			/* usb 1.1 says 90% of one frame */
			for (j = i; j < NUM_INTS; j += interval) {
				if ((ohci->load [j] + load) > 900)
					break;
			}
			if (j < NUM_INTS)
				continue;
106
			branch = i;
Linus Torvalds's avatar
Linus Torvalds committed
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
		}
	}
	return branch;
}

/*-------------------------------------------------------------------------*/

/* both iso and interrupt requests have periods; this routine puts them
 * into the schedule tree in the apppropriate place.  most iso devices use
 * 1msec periods, but that's not required.
 */
static void periodic_link (struct ohci_hcd *ohci, struct ed *ed)
{
	unsigned	i;

	ohci_vdbg (ohci, "link %sed %p branch %d [%dus.], interval %d\n",
		(ed->hwINFO & cpu_to_hc32 (ohci, ED_ISO)) ? "iso " : "",
		ed, ed->branch, ed->load, ed->interval);

	for (i = ed->branch; i < NUM_INTS; i += ed->interval) {
		struct ed	**prev = &ohci->periodic [i];
		__hc32		*prev_p = &ohci->hcca->int_table [i];
		struct ed	*here = *prev;

		/* sorting each branch by period (slow before fast)
		 * lets us share the faster parts of the tree.
		 * (plus maybe: put interrupt eds before iso)
		 */
		while (here && ed != here) {
			if (ed->interval > here->interval)
				break;
			prev = &here->ed_next;
			prev_p = &here->hwNextED;
			here = *prev;
		}
		if (ed != here) {
			ed->ed_next = here;
			if (here)
				ed->hwNextED = *prev_p;
			wmb ();
			*prev = ed;
			*prev_p = cpu_to_hc32(ohci, ed->dma);
			wmb();
		}
		ohci->load [i] += ed->load;
	}
	ohci_to_hcd(ohci)->self.bandwidth_allocated += ed->load / ed->interval;
}

/* link an ed into one of the HC chains */

static int ed_schedule (struct ohci_hcd *ohci, struct ed *ed)
159
{
Linus Torvalds's avatar
Linus Torvalds committed
160 161 162 163 164 165 166 167 168
	int	branch;

	if (ohci_to_hcd(ohci)->state == HC_STATE_QUIESCING)
		return -EAGAIN;

	ed->state = ED_OPER;
	ed->ed_prev = NULL;
	ed->ed_next = NULL;
	ed->hwNextED = 0;
169 170 171
	if (quirk_zfmicro(ohci)
			&& (ed->type == PIPE_INTERRUPT)
			&& !(ohci->eds_scheduled++))
172
		mod_timer(&ohci->unlink_watchdog, round_jiffies(jiffies + HZ));
Linus Torvalds's avatar
Linus Torvalds committed
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
	wmb ();

	/* we care about rm_list when setting CLE/BLE in case the HC was at
	 * work on some TD when CLE/BLE was turned off, and isn't quiesced
	 * yet.  finish_unlinks() restarts as needed, some upcoming INTR_SF.
	 *
	 * control and bulk EDs are doubly linked (ed_next, ed_prev), but
	 * periodic ones are singly linked (ed_next). that's because the
	 * periodic schedule encodes a tree like figure 3-5 in the ohci
	 * spec:  each qh can have several "previous" nodes, and the tree
	 * doesn't have unused/idle descriptors.
	 */
	switch (ed->type) {
	case PIPE_CONTROL:
		if (ohci->ed_controltail == NULL) {
			WARN_ON (ohci->hc_control & OHCI_CTRL_CLE);
			ohci_writel (ohci, ed->dma,
					&ohci->regs->ed_controlhead);
		} else {
			ohci->ed_controltail->ed_next = ed;
			ohci->ed_controltail->hwNextED = cpu_to_hc32 (ohci,
								ed->dma);
		}
		ed->ed_prev = ohci->ed_controltail;
		if (!ohci->ed_controltail && !ohci->ed_rm_list) {
			wmb();
			ohci->hc_control |= OHCI_CTRL_CLE;
			ohci_writel (ohci, 0, &ohci->regs->ed_controlcurrent);
			ohci_writel (ohci, ohci->hc_control,
					&ohci->regs->control);
		}
		ohci->ed_controltail = ed;
		break;

	case PIPE_BULK:
		if (ohci->ed_bulktail == NULL) {
			WARN_ON (ohci->hc_control & OHCI_CTRL_BLE);
			ohci_writel (ohci, ed->dma, &ohci->regs->ed_bulkhead);
		} else {
			ohci->ed_bulktail->ed_next = ed;
			ohci->ed_bulktail->hwNextED = cpu_to_hc32 (ohci,
								ed->dma);
		}
		ed->ed_prev = ohci->ed_bulktail;
		if (!ohci->ed_bulktail && !ohci->ed_rm_list) {
			wmb();
			ohci->hc_control |= OHCI_CTRL_BLE;
			ohci_writel (ohci, 0, &ohci->regs->ed_bulkcurrent);
			ohci_writel (ohci, ohci->hc_control,
					&ohci->regs->control);
		}
		ohci->ed_bulktail = ed;
		break;

	// case PIPE_INTERRUPT:
	// case PIPE_ISOCHRONOUS:
	default:
		branch = balance (ohci, ed->interval, ed->load);
		if (branch < 0) {
			ohci_dbg (ohci,
				"ERR %d, interval %d msecs, load %d\n",
				branch, ed->interval, ed->load);
			// FIXME if there are TDs queued, fail them!
			return branch;
		}
		ed->branch = branch;
		periodic_link (ohci, ed);
240
	}
Linus Torvalds's avatar
Linus Torvalds committed
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268

	/* the HC may not see the schedule updates yet, but if it does
	 * then they'll be properly ordered.
	 */
	return 0;
}

/*-------------------------------------------------------------------------*/

/* scan the periodic table to find and unlink this ED */
static void periodic_unlink (struct ohci_hcd *ohci, struct ed *ed)
{
	int	i;

	for (i = ed->branch; i < NUM_INTS; i += ed->interval) {
		struct ed	*temp;
		struct ed	**prev = &ohci->periodic [i];
		__hc32		*prev_p = &ohci->hcca->int_table [i];

		while (*prev && (temp = *prev) != ed) {
			prev_p = &temp->hwNextED;
			prev = &temp->ed_next;
		}
		if (*prev) {
			*prev_p = ed->hwNextED;
			*prev = ed->ed_next;
		}
		ohci->load [i] -= ed->load;
269
	}
Linus Torvalds's avatar
Linus Torvalds committed
270 271 272 273 274 275 276
	ohci_to_hcd(ohci)->self.bandwidth_allocated -= ed->load / ed->interval;

	ohci_vdbg (ohci, "unlink %sed %p branch %d [%dus.], interval %d\n",
		(ed->hwINFO & cpu_to_hc32 (ohci, ED_ISO)) ? "iso " : "",
		ed, ed->branch, ed->load, ed->interval);
}

277
/* unlink an ed from one of the HC chains.
Linus Torvalds's avatar
Linus Torvalds committed
278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
 * just the link to the ed is unlinked.
 * the link from the ed still points to another operational ed or 0
 * so the HC can eventually finish the processing of the unlinked ed
 * (assuming it already started that, which needn't be true).
 *
 * ED_UNLINK is a transient state: the HC may still see this ED, but soon
 * it won't.  ED_SKIP means the HC will finish its current transaction,
 * but won't start anything new.  The TD queue may still grow; device
 * drivers don't know about this HCD-internal state.
 *
 * When the HC can't see the ED, something changes ED_UNLINK to one of:
 *
 *  - ED_OPER: when there's any request queued, the ED gets rescheduled
 *    immediately.  HC should be working on them.
 *
 *  - ED_IDLE:  when there's no TD queue. there's no reason for the HC
 *    to care about this ED; safe to disable the endpoint.
 *
 * When finish_unlinks() runs later, after SOF interrupt, it will often
 * complete one or more URB unlinks before making that state change.
 */
299
static void ed_deschedule (struct ohci_hcd *ohci, struct ed *ed)
Linus Torvalds's avatar
Linus Torvalds committed
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
{
	ed->hwINFO |= cpu_to_hc32 (ohci, ED_SKIP);
	wmb ();
	ed->state = ED_UNLINK;

	/* To deschedule something from the control or bulk list, just
	 * clear CLE/BLE and wait.  There's no safe way to scrub out list
	 * head/current registers until later, and "later" isn't very
	 * tightly specified.  Figure 6-5 and Section 6.4.2.2 show how
	 * the HC is reading the ED queues (while we modify them).
	 *
	 * For now, ed_schedule() is "later".  It might be good paranoia
	 * to scrub those registers in finish_unlinks(), in case of bugs
	 * that make the HC try to use them.
	 */
	switch (ed->type) {
	case PIPE_CONTROL:
		/* remove ED from the HC's list: */
		if (ed->ed_prev == NULL) {
			if (!ed->hwNextED) {
				ohci->hc_control &= ~OHCI_CTRL_CLE;
				ohci_writel (ohci, ohci->hc_control,
						&ohci->regs->control);
				// a ohci_readl() later syncs CLE with the HC
			} else
				ohci_writel (ohci,
					hc32_to_cpup (ohci, &ed->hwNextED),
					&ohci->regs->ed_controlhead);
		} else {
			ed->ed_prev->ed_next = ed->ed_next;
			ed->ed_prev->hwNextED = ed->hwNextED;
		}
		/* remove ED from the HCD's list: */
		if (ohci->ed_controltail == ed) {
			ohci->ed_controltail = ed->ed_prev;
			if (ohci->ed_controltail)
				ohci->ed_controltail->ed_next = NULL;
		} else if (ed->ed_next) {
			ed->ed_next->ed_prev = ed->ed_prev;
		}
		break;

	case PIPE_BULK:
		/* remove ED from the HC's list: */
		if (ed->ed_prev == NULL) {
			if (!ed->hwNextED) {
				ohci->hc_control &= ~OHCI_CTRL_BLE;
				ohci_writel (ohci, ohci->hc_control,
						&ohci->regs->control);
				// a ohci_readl() later syncs BLE with the HC
			} else
				ohci_writel (ohci,
					hc32_to_cpup (ohci, &ed->hwNextED),
					&ohci->regs->ed_bulkhead);
		} else {
			ed->ed_prev->ed_next = ed->ed_next;
			ed->ed_prev->hwNextED = ed->hwNextED;
		}
		/* remove ED from the HCD's list: */
		if (ohci->ed_bulktail == ed) {
			ohci->ed_bulktail = ed->ed_prev;
			if (ohci->ed_bulktail)
				ohci->ed_bulktail->ed_next = NULL;
		} else if (ed->ed_next) {
			ed->ed_next->ed_prev = ed->ed_prev;
		}
		break;

	// case PIPE_INTERRUPT:
	// case PIPE_ISOCHRONOUS:
	default:
		periodic_unlink (ohci, ed);
		break;
	}
}


/*-------------------------------------------------------------------------*/

/* get and maybe (re)init an endpoint. init _should_ be done only as part
 * of enumeration, usb_set_configuration() or usb_set_interface().
 */
static struct ed *ed_get (
	struct ohci_hcd		*ohci,
	struct usb_host_endpoint *ep,
	struct usb_device	*udev,
	unsigned int		pipe,
	int			interval
) {
389
	struct ed		*ed;
Linus Torvalds's avatar
Linus Torvalds committed
390 391 392 393 394 395 396 397 398 399 400 401 402 403 404
	unsigned long		flags;

	spin_lock_irqsave (&ohci->lock, flags);

	if (!(ed = ep->hcpriv)) {
		struct td	*td;
		int		is_out;
		u32		info;

		ed = ed_alloc (ohci, GFP_ATOMIC);
		if (!ed) {
			/* out of memory */
			goto done;
		}

405
		/* dummy td; end of td list for ed */
Linus Torvalds's avatar
Linus Torvalds committed
406
		td = td_alloc (ohci, GFP_ATOMIC);
407
		if (!td) {
Linus Torvalds's avatar
Linus Torvalds committed
408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453
			/* out of memory */
			ed_free (ohci, ed);
			ed = NULL;
			goto done;
		}
		ed->dummy = td;
		ed->hwTailP = cpu_to_hc32 (ohci, td->td_dma);
		ed->hwHeadP = ed->hwTailP;	/* ED_C, ED_H zeroed */
		ed->state = ED_IDLE;

		is_out = !(ep->desc.bEndpointAddress & USB_DIR_IN);

		/* FIXME usbcore changes dev->devnum before SET_ADDRESS
		 * suceeds ... otherwise we wouldn't need "pipe".
		 */
		info = usb_pipedevice (pipe);
		ed->type = usb_pipetype(pipe);

		info |= (ep->desc.bEndpointAddress & ~USB_DIR_IN) << 7;
		info |= le16_to_cpu(ep->desc.wMaxPacketSize) << 16;
		if (udev->speed == USB_SPEED_LOW)
			info |= ED_LOWSPEED;
		/* only control transfers store pids in tds */
		if (ed->type != PIPE_CONTROL) {
			info |= is_out ? ED_OUT : ED_IN;
			if (ed->type != PIPE_BULK) {
				/* periodic transfers... */
				if (ed->type == PIPE_ISOCHRONOUS)
					info |= ED_ISO;
				else if (interval > 32)	/* iso can be bigger */
					interval = 32;
				ed->interval = interval;
				ed->load = usb_calc_bus_time (
					udev->speed, !is_out,
					ed->type == PIPE_ISOCHRONOUS,
					le16_to_cpu(ep->desc.wMaxPacketSize))
						/ 1000;
			}
		}
		ed->hwINFO = cpu_to_hc32(ohci, info);

		ep->hcpriv = ed;
	}

done:
	spin_unlock_irqrestore (&ohci->lock, flags);
454
	return ed;
Linus Torvalds's avatar
Linus Torvalds committed
455 456 457 458 459 460 461 462 463 464 465
}

/*-------------------------------------------------------------------------*/

/* request unlinking of an endpoint from an operational HC.
 * put the ep on the rm_list
 * real work is done at the next start frame (SF) hardware interrupt
 * caller guarantees HCD is running, so hardware access is safe,
 * and that ed->state is ED_OPER
 */
static void start_ed_unlink (struct ohci_hcd *ohci, struct ed *ed)
466
{
Linus Torvalds's avatar
Linus Torvalds committed
467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532
	ed->hwINFO |= cpu_to_hc32 (ohci, ED_DEQUEUE);
	ed_deschedule (ohci, ed);

	/* rm_list is just singly linked, for simplicity */
	ed->ed_next = ohci->ed_rm_list;
	ed->ed_prev = NULL;
	ohci->ed_rm_list = ed;

	/* enable SOF interrupt */
	ohci_writel (ohci, OHCI_INTR_SF, &ohci->regs->intrstatus);
	ohci_writel (ohci, OHCI_INTR_SF, &ohci->regs->intrenable);
	// flush those writes, and get latest HCCA contents
	(void) ohci_readl (ohci, &ohci->regs->control);

	/* SF interrupt might get delayed; record the frame counter value that
	 * indicates when the HC isn't looking at it, so concurrent unlinks
	 * behave.  frame_no wraps every 2^16 msec, and changes right before
	 * SF is triggered.
	 */
	ed->tick = ohci_frame_no(ohci) + 1;

}

/*-------------------------------------------------------------------------*
 * TD handling functions
 *-------------------------------------------------------------------------*/

/* enqueue next TD for this URB (OHCI spec 5.2.8.2) */

static void
td_fill (struct ohci_hcd *ohci, u32 info,
	dma_addr_t data, int len,
	struct urb *urb, int index)
{
	struct td		*td, *td_pt;
	struct urb_priv		*urb_priv = urb->hcpriv;
	int			is_iso = info & TD_ISO;
	int			hash;

	// ASSERT (index < urb_priv->length);

	/* aim for only one interrupt per urb.  mostly applies to control
	 * and iso; other urbs rarely need more than one TD per urb.
	 * this way, only final tds (or ones with an error) cause IRQs.
	 * at least immediately; use DI=6 in case any control request is
	 * tempted to die part way through.  (and to force the hc to flush
	 * its donelist soonish, even on unlink paths.)
	 *
	 * NOTE: could delay interrupts even for the last TD, and get fewer
	 * interrupts ... increasing per-urb latency by sharing interrupts.
	 * Drivers that queue bulk urbs may request that behavior.
	 */
	if (index != (urb_priv->length - 1)
			|| (urb->transfer_flags & URB_NO_INTERRUPT))
		info |= TD_DI_SET (6);

	/* use this td as the next dummy */
	td_pt = urb_priv->td [index];

	/* fill the old dummy TD */
	td = urb_priv->td [index] = urb_priv->ed->dummy;
	urb_priv->ed->dummy = td_pt;

	td->ed = urb_priv->ed;
	td->next_dl_td = NULL;
	td->index = index;
533
	td->urb = urb;
Linus Torvalds's avatar
Linus Torvalds committed
534 535 536 537 538 539 540 541 542 543 544
	td->data_dma = data;
	if (!len)
		data = 0;

	td->hwINFO = cpu_to_hc32 (ohci, info);
	if (is_iso) {
		td->hwCBP = cpu_to_hc32 (ohci, data & 0xFFFFF000);
		*ohci_hwPSWp(ohci, td, 0) = cpu_to_hc16 (ohci,
						(data & 0x0FFF) | 0xE000);
		td->ed->last_iso = info & 0xffff;
	} else {
545 546
		td->hwCBP = cpu_to_hc32 (ohci, data);
	}
Linus Torvalds's avatar
Linus Torvalds committed
547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588
	if (data)
		td->hwBE = cpu_to_hc32 (ohci, data + len - 1);
	else
		td->hwBE = 0;
	td->hwNextTD = cpu_to_hc32 (ohci, td_pt->td_dma);

	/* append to queue */
	list_add_tail (&td->td_list, &td->ed->td_list);

	/* hash it for later reverse mapping */
	hash = TD_HASH_FUNC (td->td_dma);
	td->td_hash = ohci->td_hash [hash];
	ohci->td_hash [hash] = td;

	/* HC might read the TD (or cachelines) right away ... */
	wmb ();
	td->ed->hwTailP = td->hwNextTD;
}

/*-------------------------------------------------------------------------*/

/* Prepare all TDs of a transfer, and queue them onto the ED.
 * Caller guarantees HC is active.
 * Usually the ED is already on the schedule, so TDs might be
 * processed as soon as they're queued.
 */
static void td_submit_urb (
	struct ohci_hcd	*ohci,
	struct urb	*urb
) {
	struct urb_priv	*urb_priv = urb->hcpriv;
	dma_addr_t	data;
	int		data_len = urb->transfer_buffer_length;
	int		cnt = 0;
	u32		info = 0;
	int		is_out = usb_pipeout (urb->pipe);
	int		periodic = 0;

	/* OHCI handles the bulk/interrupt data toggles itself.  We just
	 * use the device toggle bits for resetting, and rely on the fact
	 * that resetting toggle is meaningless if the endpoint is active.
	 */
589
	if (!usb_gettoggle (urb->dev, usb_pipeendpoint (urb->pipe), is_out)) {
Linus Torvalds's avatar
Linus Torvalds committed
590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
		usb_settoggle (urb->dev, usb_pipeendpoint (urb->pipe),
			is_out, 1);
		urb_priv->ed->hwHeadP &= ~cpu_to_hc32 (ohci, ED_C);
	}

	urb_priv->td_cnt = 0;
	list_add (&urb_priv->pending, &ohci->pending);

	if (data_len)
		data = urb->transfer_dma;
	else
		data = 0;

	/* NOTE:  TD_CC is set so we can tell which TDs the HC processed by
	 * using TD_CC_GET, as well as by seeing them on the done list.
	 * (CC = NotAccessed ... 0x0F, or 0x0E in PSWs for ISO.)
	 */
	switch (urb_priv->ed->type) {

	/* Bulk and interrupt are identical except for where in the schedule
	 * their EDs live.
	 */
	case PIPE_INTERRUPT:
		/* ... and periodic urbs have extra accounting */
		periodic = ohci_to_hcd(ohci)->self.bandwidth_int_reqs++ == 0
			&& ohci_to_hcd(ohci)->self.bandwidth_isoc_reqs == 0;
		/* FALLTHROUGH */
	case PIPE_BULK:
		info = is_out
			? TD_T_TOGGLE | TD_CC | TD_DP_OUT
			: TD_T_TOGGLE | TD_CC | TD_DP_IN;
		/* TDs _could_ transfer up to 8K each */
		while (data_len > 4096) {
			td_fill (ohci, info, data, 4096, urb, cnt);
			data += 4096;
			data_len -= 4096;
			cnt++;
		}
		/* maybe avoid ED halt on final TD short read */
		if (!(urb->transfer_flags & URB_SHORT_NOT_OK))
			info |= TD_R;
		td_fill (ohci, info, data, data_len, urb, cnt);
		cnt++;
		if ((urb->transfer_flags & URB_ZERO_PACKET)
				&& cnt < urb_priv->length) {
			td_fill (ohci, info, 0, 0, urb, cnt);
			cnt++;
		}
		/* maybe kickstart bulk list */
		if (urb_priv->ed->type == PIPE_BULK) {
			wmb ();
			ohci_writel (ohci, OHCI_BLF, &ohci->regs->cmdstatus);
		}
		break;

	/* control manages DATA0/DATA1 toggle per-request; SETUP resets it,
	 * any DATA phase works normally, and the STATUS ack is special.
	 */
	case PIPE_CONTROL:
		info = TD_CC | TD_DP_SETUP | TD_T_DATA0;
		td_fill (ohci, info, urb->setup_dma, 8, urb, cnt++);
		if (data_len > 0) {
			info = TD_CC | TD_R | TD_T_DATA1;
			info |= is_out ? TD_DP_OUT : TD_DP_IN;
			/* NOTE:  mishandles transfers >8K, some >4K */
			td_fill (ohci, info, data, data_len, urb, cnt++);
		}
		info = (is_out || data_len == 0)
			? TD_CC | TD_DP_IN | TD_T_DATA1
			: TD_CC | TD_DP_OUT | TD_T_DATA1;
		td_fill (ohci, info, data, 0, urb, cnt++);
		/* maybe kickstart control list */
		wmb ();
		ohci_writel (ohci, OHCI_CLF, &ohci->regs->cmdstatus);
		break;

	/* ISO has no retransmit, so no toggle; and it uses special TDs.
	 * Each TD could handle multiple consecutive frames (interval 1);
	 * we could often reduce the number of TDs here.
	 */
	case PIPE_ISOCHRONOUS:
		for (cnt = 0; cnt < urb->number_of_packets; cnt++) {
			int	frame = urb->start_frame;

			// FIXME scheduling should handle frame counter
			// roll-around ... exotic case (and OHCI has
			// a 2^16 iso range, vs other HCs max of 2^10)
			frame += cnt * urb->interval;
			frame &= 0xffff;
			td_fill (ohci, TD_CC | TD_ISO | frame,
				data + urb->iso_frame_desc [cnt].offset,
				urb->iso_frame_desc [cnt].length, urb, cnt);
		}
		periodic = ohci_to_hcd(ohci)->self.bandwidth_isoc_reqs++ == 0
			&& ohci_to_hcd(ohci)->self.bandwidth_int_reqs == 0;
		break;
	}

	/* start periodic dma if needed */
	if (periodic) {
		wmb ();
		ohci->hc_control |= OHCI_CTRL_PLE|OHCI_CTRL_IE;
		ohci_writel (ohci, ohci->hc_control, &ohci->regs->control);
	}

	// ASSERT (urb_priv->length == cnt);
}

/*-------------------------------------------------------------------------*
 * Done List handling functions
 *-------------------------------------------------------------------------*/

702 703
/* calculate transfer length/status and update the urb */
static int td_done(struct ohci_hcd *ohci, struct urb *urb, struct td *td)
Linus Torvalds's avatar
Linus Torvalds committed
704 705 706
{
	u32	tdINFO = hc32_to_cpup (ohci, &td->hwINFO);
	int	cc = 0;
707
	int	status = -EINPROGRESS;
Linus Torvalds's avatar
Linus Torvalds committed
708 709 710 711

	list_del (&td->td_list);

	/* ISO ... drivers see per-TD length/status */
712
	if (tdINFO & TD_ISO) {
713
		u16	tdPSW = ohci_hwPSW(ohci, td, 0);
Linus Torvalds's avatar
Linus Torvalds committed
714 715 716 717 718 719
		int	dlen = 0;

		/* NOTE:  assumes FC in tdINFO == 0, and that
		 * only the first of 0..MAXPSW psws is used.
		 */

720 721
		cc = (tdPSW >> 12) & 0xF;
		if (tdINFO & TD_CC)	/* hc didn't touch? */
722
			return status;
Linus Torvalds's avatar
Linus Torvalds committed
723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748

		if (usb_pipeout (urb->pipe))
			dlen = urb->iso_frame_desc [td->index].length;
		else {
			/* short reads are always OK for ISO */
			if (cc == TD_DATAUNDERRUN)
				cc = TD_CC_NOERROR;
			dlen = tdPSW & 0x3ff;
		}
		urb->actual_length += dlen;
		urb->iso_frame_desc [td->index].actual_length = dlen;
		urb->iso_frame_desc [td->index].status = cc_to_error [cc];

		if (cc != TD_CC_NOERROR)
			ohci_vdbg (ohci,
				"urb %p iso td %p (%d) len %d cc %d\n",
				urb, td, 1 + td->index, dlen, cc);

	/* BULK, INT, CONTROL ... drivers see aggregate length/status,
	 * except that "setup" bytes aren't counted and "short" transfers
	 * might not be reported as errors.
	 */
	} else {
		int	type = usb_pipetype (urb->pipe);
		u32	tdBE = hc32_to_cpup (ohci, &td->hwBE);

749
		cc = TD_CC_GET (tdINFO);
Linus Torvalds's avatar
Linus Torvalds committed
750 751 752 753 754

		/* update packet status if needed (short is normally ok) */
		if (cc == TD_DATAUNDERRUN
				&& !(urb->transfer_flags & URB_SHORT_NOT_OK))
			cc = TD_CC_NOERROR;
755 756
		if (cc != TD_CC_NOERROR && cc < 0x0E)
			status = cc_to_error[cc];
Linus Torvalds's avatar
Linus Torvalds committed
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773

		/* count all non-empty packets except control SETUP packet */
		if ((type != PIPE_CONTROL || td->index != 0) && tdBE != 0) {
			if (td->hwCBP == 0)
				urb->actual_length += tdBE - td->data_dma + 1;
			else
				urb->actual_length +=
					  hc32_to_cpup (ohci, &td->hwCBP)
					- td->data_dma;
		}

		if (cc != TD_CC_NOERROR && cc < 0x0E)
			ohci_vdbg (ohci,
				"urb %p td %p (%d) cc %d, len=%d/%d\n",
				urb, td, 1 + td->index, cc,
				urb->actual_length,
				urb->transfer_buffer_length);
774
	}
775
	return status;
Linus Torvalds's avatar
Linus Torvalds committed
776 777 778 779
}

/*-------------------------------------------------------------------------*/

780
static void ed_halted(struct ohci_hcd *ohci, struct td *td, int cc)
Linus Torvalds's avatar
Linus Torvalds committed
781
{
782
	struct urb		*urb = td->urb;
783
	urb_priv_t		*urb_priv = urb->hcpriv;
Linus Torvalds's avatar
Linus Torvalds committed
784 785 786 787 788 789 790 791 792
	struct ed		*ed = td->ed;
	struct list_head	*tmp = td->td_list.next;
	__hc32			toggle = ed->hwHeadP & cpu_to_hc32 (ohci, ED_C);

	/* clear ed halt; this is the td that caused it, but keep it inactive
	 * until its urb->complete() has a chance to clean up.
	 */
	ed->hwINFO |= cpu_to_hc32 (ohci, ED_SKIP);
	wmb ();
793
	ed->hwHeadP &= ~cpu_to_hc32 (ohci, ED_H);
Linus Torvalds's avatar
Linus Torvalds committed
794

795 796 797
	/* Get rid of all later tds from this urb.  We don't have
	 * to be careful: no errors and nothing was transferred.
	 * Also patch the ed so it looks as if those tds completed normally.
Linus Torvalds's avatar
Linus Torvalds committed
798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815
	 */
	while (tmp != &ed->td_list) {
		struct td	*next;

		next = list_entry (tmp, struct td, td_list);
		tmp = next->td_list.next;

		if (next->urb != urb)
			break;

		/* NOTE: if multi-td control DATA segments get supported,
		 * this urb had one of them, this td wasn't the last td
		 * in that segment (TD_R clear), this ed halted because
		 * of a short read, _and_ URB_SHORT_NOT_OK is clear ...
		 * then we need to leave the control STATUS packet queued
		 * and clear ED_SKIP.
		 */

816 817
		list_del(&next->td_list);
		urb_priv->td_cnt++;
Linus Torvalds's avatar
Linus Torvalds committed
818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860
		ed->hwHeadP = next->hwNextTD | toggle;
	}

	/* help for troubleshooting:  report anything that
	 * looks odd ... that doesn't include protocol stalls
	 * (or maybe some other things)
	 */
	switch (cc) {
	case TD_DATAUNDERRUN:
		if ((urb->transfer_flags & URB_SHORT_NOT_OK) == 0)
			break;
		/* fallthrough */
	case TD_CC_STALL:
		if (usb_pipecontrol (urb->pipe))
			break;
		/* fallthrough */
	default:
		ohci_dbg (ohci,
			"urb %p path %s ep%d%s %08x cc %d --> status %d\n",
			urb, urb->dev->devpath,
			usb_pipeendpoint (urb->pipe),
			usb_pipein (urb->pipe) ? "in" : "out",
			hc32_to_cpu (ohci, td->hwINFO),
			cc, cc_to_error [cc]);
	}
}

/* replies to the request have to be on a FIFO basis so
 * we unreverse the hc-reversed done-list
 */
static struct td *dl_reverse_done_list (struct ohci_hcd *ohci)
{
	u32		td_dma;
	struct td	*td_rev = NULL;
	struct td	*td = NULL;

	td_dma = hc32_to_cpup (ohci, &ohci->hcca->done_head);
	ohci->hcca->done_head = 0;
	wmb();

	/* get TD from hc's singly linked list, and
	 * prepend to ours.  ed->td_list changes later.
	 */
861 862
	while (td_dma) {
		int		cc;
Linus Torvalds's avatar
Linus Torvalds committed
863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878

		td = dma_to_td (ohci, td_dma);
		if (!td) {
			ohci_err (ohci, "bad entry %8x\n", td_dma);
			break;
		}

		td->hwINFO |= cpu_to_hc32 (ohci, TD_DONE);
		cc = TD_CC_GET (hc32_to_cpup (ohci, &td->hwINFO));

		/* Non-iso endpoints can halt on error; un-halt,
		 * and dequeue any other TDs from this urb.
		 * No other TD could have caused the halt.
		 */
		if (cc != TD_CC_NOERROR
				&& (td->ed->hwHeadP & cpu_to_hc32 (ohci, ED_H)))
879
			ed_halted(ohci, td, cc);
Linus Torvalds's avatar
Linus Torvalds committed
880

881
		td->next_dl_td = td_rev;
Linus Torvalds's avatar
Linus Torvalds committed
882 883
		td_rev = td;
		td_dma = hc32_to_cpup (ohci, &td->hwNextTD);
884
	}
Linus Torvalds's avatar
Linus Torvalds committed
885 886 887 888 889 890 891
	return td_rev;
}

/*-------------------------------------------------------------------------*/

/* there are some urbs/eds to unlink; called in_irq(), with HCD locked */
static void
892
finish_unlinks (struct ohci_hcd *ohci, u16 tick)
Linus Torvalds's avatar
Linus Torvalds committed
893 894 895 896 897 898 899 900 901 902 903 904
{
	struct ed	*ed, **last;

rescan_all:
	for (last = &ohci->ed_rm_list, ed = *last; ed != NULL; ed = *last) {
		struct list_head	*entry, *tmp;
		int			completed, modified;
		__hc32			*prev;

		/* only take off EDs that the HC isn't using, accounting for
		 * frame counter wraps and EDs with partially retired TDs
		 */
905
		if (likely (HC_IS_RUNNING(ohci_to_hcd(ohci)->state))) {
Linus Torvalds's avatar
Linus Torvalds committed
906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921
			if (tick_before (tick, ed->tick)) {
skip_ed:
				last = &ed->ed_next;
				continue;
			}

			if (!list_empty (&ed->td_list)) {
				struct td	*td;
				u32		head;

				td = list_entry (ed->td_list.next, struct td,
							td_list);
				head = hc32_to_cpu (ohci, ed->hwHeadP) &
								TD_MASK;

				/* INTR_WDH may need to clean up first */
922 923 924 925 926 927
				if (td->td_dma != head) {
					if (ed == ohci->ed_to_check)
						ohci->ed_to_check = NULL;
					else
						goto skip_ed;
				}
Linus Torvalds's avatar
Linus Torvalds committed
928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954
			}
		}

		/* reentrancy:  if we drop the schedule lock, someone might
		 * have modified this list.  normally it's just prepending
		 * entries (which we'd ignore), but paranoia won't hurt.
		 */
		*last = ed->ed_next;
		ed->ed_next = NULL;
		modified = 0;

		/* unlink urbs as requested, but rescan the list after
		 * we call a completion since it might have unlinked
		 * another (earlier) urb
		 *
		 * When we get here, the HC doesn't see this ed.  But it
		 * must not be rescheduled until all completed URBs have
		 * been given back to the driver.
		 */
rescan_this:
		completed = 0;
		prev = &ed->hwHeadP;
		list_for_each_safe (entry, tmp, &ed->td_list) {
			struct td	*td;
			struct urb	*urb;
			urb_priv_t	*urb_priv;
			__hc32		savebits;
955
			u32		tdINFO;
Linus Torvalds's avatar
Linus Torvalds committed
956 957 958 959 960

			td = list_entry (entry, struct td, td_list);
			urb = td->urb;
			urb_priv = td->urb->hcpriv;

Alan Stern's avatar
Alan Stern committed
961
			if (!urb->unlinked) {
Linus Torvalds's avatar
Linus Torvalds committed
962 963 964 965 966 967 968 969
				prev = &td->hwNextTD;
				continue;
			}

			/* patch pointer hc uses */
			savebits = *prev & ~cpu_to_hc32 (ohci, TD_MASK);
			*prev = td->hwNextTD | savebits;

970 971 972 973 974 975 976 977 978 979 980
			/* If this was unlinked, the TD may not have been
			 * retired ... so manually save the data toggle.
			 * The controller ignores the value we save for
			 * control and ISO endpoints.
			 */
			tdINFO = hc32_to_cpup(ohci, &td->hwINFO);
			if ((tdINFO & TD_T) == TD_T_DATA0)
				ed->hwHeadP &= ~cpu_to_hc32(ohci, ED_C);
			else if ((tdINFO & TD_T) == TD_T_DATA1)
				ed->hwHeadP |= cpu_to_hc32(ohci, ED_C);

Linus Torvalds's avatar
Linus Torvalds committed
981 982 983 984 985 986 987
			/* HC may have partly processed this TD */
			td_done (ohci, urb, td);
			urb_priv->td_cnt++;

			/* if URB is done, clean up */
			if (urb_priv->td_cnt == urb_priv->length) {
				modified = completed = 1;
988
				finish_urb(ohci, urb, 0);
Linus Torvalds's avatar
Linus Torvalds committed
989 990 991 992 993 994 995
			}
		}
		if (completed && !list_empty (&ed->td_list))
			goto rescan_this;

		/* ED's now officially unlinked, hc doesn't see */
		ed->state = ED_IDLE;
996 997
		if (quirk_zfmicro(ohci) && ed->type == PIPE_INTERRUPT)
			ohci->eds_scheduled--;
Linus Torvalds's avatar
Linus Torvalds committed
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010
		ed->hwHeadP &= ~cpu_to_hc32(ohci, ED_H);
		ed->hwNextED = 0;
		wmb ();
		ed->hwINFO &= ~cpu_to_hc32 (ohci, ED_SKIP | ED_DEQUEUE);

		/* but if there's work queued, reschedule */
		if (!list_empty (&ed->td_list)) {
			if (HC_IS_RUNNING(ohci_to_hcd(ohci)->state))
				ed_schedule (ohci, ed);
		}

		if (modified)
			goto rescan_all;
1011
	}
Linus Torvalds's avatar
Linus Torvalds committed
1012

1013
	/* maybe reenable control and bulk lists */
Linus Torvalds's avatar
Linus Torvalds committed
1014 1015 1016 1017 1018 1019 1020
	if (HC_IS_RUNNING(ohci_to_hcd(ohci)->state)
			&& ohci_to_hcd(ohci)->state != HC_STATE_QUIESCING
			&& !ohci->ed_rm_list) {
		u32	command = 0, control = 0;

		if (ohci->ed_controltail) {
			command |= OHCI_CLF;
1021
			if (quirk_zfmicro(ohci))
1022
				mdelay(1);
Linus Torvalds's avatar
Linus Torvalds committed
1023 1024 1025 1026 1027 1028 1029 1030
			if (!(ohci->hc_control & OHCI_CTRL_CLE)) {
				control |= OHCI_CTRL_CLE;
				ohci_writel (ohci, 0,
					&ohci->regs->ed_controlcurrent);
			}
		}
		if (ohci->ed_bulktail) {
			command |= OHCI_BLF;
1031
			if (quirk_zfmicro(ohci))
1032
				mdelay(1);
Linus Torvalds's avatar
Linus Torvalds committed
1033 1034 1035 1036 1037 1038
			if (!(ohci->hc_control & OHCI_CTRL_BLE)) {
				control |= OHCI_CTRL_BLE;
				ohci_writel (ohci, 0,
					&ohci->regs->ed_bulkcurrent);
			}
		}
1039

Linus Torvalds's avatar
Linus Torvalds committed
1040 1041 1042
		/* CLE/BLE to enable, CLF/BLF to (maybe) kickstart */
		if (control) {
			ohci->hc_control |= control;
1043
			if (quirk_zfmicro(ohci))
1044
				mdelay(1);
1045 1046 1047
			ohci_writel (ohci, ohci->hc_control,
					&ohci->regs->control);
		}
1048
		if (command) {
1049
			if (quirk_zfmicro(ohci))
1050
				mdelay(1);
1051 1052
			ohci_writel (ohci, command, &ohci->regs->cmdstatus);
		}
1053
	}
Linus Torvalds's avatar
Linus Torvalds committed
1054 1055 1056 1057 1058 1059
}



/*-------------------------------------------------------------------------*/

1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
/*
 * Used to take back a TD from the host controller. This would normally be
 * called from within dl_done_list, however it may be called directly if the
 * HC no longer sees the TD and it has not appeared on the donelist (after
 * two frames).  This bug has been observed on ZF Micro systems.
 */
static void takeback_td(struct ohci_hcd *ohci, struct td *td)
{
	struct urb	*urb = td->urb;
	urb_priv_t	*urb_priv = urb->hcpriv;
	struct ed	*ed = td->ed;
1071
	int		status;
1072 1073

	/* update URB's length and status from TD */
1074
	status = td_done(ohci, urb, td);
1075 1076 1077 1078
	urb_priv->td_cnt++;

	/* If all this urb's TDs are done, call complete() */
	if (urb_priv->td_cnt == urb_priv->length)
1079
		finish_urb(ohci, urb, status);
1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106

	/* clean schedule:  unlink EDs that are no longer busy */
	if (list_empty(&ed->td_list)) {
		if (ed->state == ED_OPER)
			start_ed_unlink(ohci, ed);

	/* ... reenabling halted EDs only after fault cleanup */
	} else if ((ed->hwINFO & cpu_to_hc32(ohci, ED_SKIP | ED_DEQUEUE))
			== cpu_to_hc32(ohci, ED_SKIP)) {
		td = list_entry(ed->td_list.next, struct td, td_list);
		if (!(td->hwINFO & cpu_to_hc32(ohci, TD_DONE))) {
			ed->hwINFO &= ~cpu_to_hc32(ohci, ED_SKIP);
			/* ... hc may need waking-up */
			switch (ed->type) {
			case PIPE_CONTROL:
				ohci_writel(ohci, OHCI_CLF,
						&ohci->regs->cmdstatus);
				break;
			case PIPE_BULK:
				ohci_writel(ohci, OHCI_BLF,
						&ohci->regs->cmdstatus);
				break;
			}
		}
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
1107 1108 1109 1110
/*
 * Process normal completions (error or success) and clean the schedules.
 *
 * This is the main path for handing urbs back to drivers.  The only other
1111 1112 1113 1114
 * normal path is finish_unlinks(), which unlinks URBs using ed_rm_list,
 * instead of scanning the (re-reversed) donelist as this does.  There's
 * an abnormal path too, handling a quirk in some Compaq silicon:  URBs
 * with TDs that appear to be orphaned are directly reclaimed.
Linus Torvalds's avatar
Linus Torvalds committed
1115 1116
 */
static void
1117
dl_done_list (struct ohci_hcd *ohci)
Linus Torvalds's avatar
Linus Torvalds committed
1118 1119 1120
{
	struct td	*td = dl_reverse_done_list (ohci);

1121
	while (td) {
Linus Torvalds's avatar
Linus Torvalds committed
1122
		struct td	*td_next = td->next_dl_td;
1123
		takeback_td(ohci, td);
1124 1125
		td = td_next;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1126
}