jehanne/sys/src/kern/386/pci.c

/*
 * PCI support code.
 * Needs a massive rewrite.
 */
#include "u.h"
#include "../port/lib.h"
#include "mem.h"
#include "dat.h"
#include "fns.h"

#include "io.h"

struct
{
	char	output[16384];
	int	ptr;
}PCICONS;

int
pcilog(char *fmt, ...)
{
	int n;
	va_list arg;
	char buf[PRINTSIZE];

	va_start(arg, fmt);
	n = vseprint(buf, buf+sizeof(buf), fmt, arg) - buf;
	va_end(arg);

	memmove(PCICONS.output+PCICONS.ptr, buf, n);
	PCICONS.ptr += n;
	return n;
}

enum
{					/* configuration mechanism #1 */
	PciADDR		= 0xCF8,	/* CONFIG_ADDRESS */
	PciDATA		= 0xCFC,	/* CONFIG_DATA */

	MaxFNO		= 7,
	MaxDNO		= 31,
	MaxUBN		= 255,
};

enum
{					/* command register */
	IOen		= (1<<0),
	MEMen		= (1<<1),
	MASen		= (1<<2),
	MemWrInv	= (1<<4),
	PErrEn		= (1<<6),
	SErrEn		= (1<<8),
};

#define MKPCIX(b,d,f,r)	((((b)&0xFF)<<20)|(((d)&0x1F)<<15)|(((f)&7)<<12)|((r)&0xFFF))

static Lock pcicfglock;
static Lock pcicfginitlock;
static int pcicfgmode = -1;
static int pcimaxbno = 255;
static int pcimaxdno;
static int srxusehba;
static Pcidev* pciroot;
static Pcidev* pcilist;
static Pcidev* pcitail;
static int nobios, nopcirouting;

static int pcicfgrw32(int, int, int, int);
static int pcicfgrw16(int, int, int, int);
static int pcicfgrw8(int, int, int, int);

static char* bustypes[] = {
	"CBUSI",
	"CBUSII",
	"EISA",
	"FUTURE",
	"INTERN",
	"ISA",
	"MBI",
	"MBII",
	"MCA",
	"MPI",
	"MPSA",
	"NUBUS",
	"PCI",
	"PCMCIA",
	"TC",
	"VL",
	"VME",
	"XPRESS",
};

static int
strtobus(char *s)
{
	int i;

	for(i = 0; i < nelem(bustypes); i++)
		if(cistrcmp(s, bustypes[i]) == 0)
			return i;
	return BUSUNKNOWN;
}

int
strtotbdf(char *p, char **r, int base)
{
	char buf[12], *f0[4], **f;
	int i, n, bus, t[4];

	if(strchr(p, '.') == nil)
		return strtoul(p, r, base);

	snprint(buf, sizeof buf, "%s", p);
	f = f0;
	n = gettokens(buf, f, nelem(f0), ".");
	memset(t, 0, sizeof t);
	t[0] = BusPCI;
	if((bus = strtobus(f[0])) != BUSUNKNOWN){
		t[0] = bus;
		f++;
		n--;
	}
	for(i = 0; i < n; i++)
		t[i+1] = strtoul(f[i], r, base);
	if(r != nil)
		*r = *r-buf + p;
	return MKBUS(t[0], t[1], t[2], t[3]);
}

static int
tbdffmt(Fmt* fmt)
{
	char buf[32], *p, *e;
	uint32_t type, tbdf;

	p = buf;
	e = buf+sizeof buf;
	tbdf = va_arg(fmt->args, int);
	if(tbdf == -1)
		return fmtstrcpy(fmt, "isa");
	if(fmt->flags & FmtLong){
		type = BUSTYPE(tbdf);
		if(type == 12)
			p = seprint(p, e, "pci.");
		else
			p = seprint(p, e, "%d.", type);
	}
	seprint(p, e, "%d.%d.%d",
		BUSBNO(tbdf), BUSDNO(tbdf), BUSFNO(tbdf));
	return fmtstrcpy(fmt, buf);
}

uint32_t
pcibarsize(Pcidev *p, int rno)
{
	uint32_t v, size;

	v = pcicfgrw32(p->tbdf, rno, 0, 1);
	pcicfgrw32(p->tbdf, rno, 0xFFFFFFF0, 0);
	size = pcicfgrw32(p->tbdf, rno, 0, 1);
	if(v & 1)
		size |= 0xFFFF0000;
	pcicfgrw32(p->tbdf, rno, v, 0);

	return -(size & ~0x0F);
}

static int
pcisizcmp(void *a, void *b)
{
	Pcisiz *aa, *bb;

	aa = a;
	bb = b;
	return aa->siz - bb->siz;
}

static uint32_t
pcimask(uint32_t v)
{
	uint32_t mask;

	mask = BI2BY*sizeof(v);
	for(mask = 1<<(mask-1); mask != 0; mask >>= 1) {
		if(mask & v)
			break;
	}

	mask--;
	if((v & mask) == 0)
		return v;

	v |= mask;
	return v+1;
}

static void
pcibusmap(Pcidev *root, uint32_t *pmema, uint32_t *pioa, int wrreg)
{
	Pcidev *p;
	int ntb, i, size, rno, hole;
	uint32_t v, mema, ioa, sioa, smema, base, limit;
	Pcisiz *table, *tptr, *mtb, *itb;
	extern void qsort(void*, long, long, int (*)(void*, void*));

	if(!nobios)
		return;

	ioa = *pioa;
	mema = *pmema;

	DBG("pcibusmap wr=%d %T mem=%#lux io=%#lux\n",
		wrreg, root->tbdf, mema, ioa);

	ntb = 0;
	for(p = root; p != nil; p = p->link)
		ntb++;

	ntb *= (PciCIS-PciBAR0)/4;
	table = malloc(2*ntb*sizeof(Pcisiz));
	itb = table;
	mtb = table+ntb;

	/*
	 * Build a table of sizes
	 */
	for(p = root; p != nil; p = p->link) {
		if(p->ccrb == 0x06) {
			if(p->ccru != 0x04 || p->bridge == nil) {
				DBG("pci: ignored bridge %T\n", p->tbdf);
				continue;
			}

			sioa = ioa;
			smema = mema;
			pcibusmap(p->bridge, &smema, &sioa, 0);

			hole = pcimask(smema-mema);
			if(hole < (1<<20))
				hole = 1<<20;
			p->mema.size = hole;

			hole = pcimask(sioa-ioa);
			if(hole < (1<<12))
				hole = 1<<12;

			p->ioa.size = hole;

			itb->dev = p;
			itb->bar = -1;
			itb->siz = p->ioa.size;
			itb++;

			mtb->dev = p;
			mtb->bar = -1;
			mtb->siz = p->mema.size;
			mtb++;
			continue;
		}

		for(i = 0; i <= 5; i++) {
			rno = PciBAR0 + i*4;
			v = pcicfgrw32(p->tbdf, rno, 0, 1);
			size = pcibarsize(p, rno);
			if(size == 0)
				continue;

			if(v & 1) {
				itb->dev = p;
				itb->bar = i;
				itb->siz = size;
				itb++;
			}
			else {
				mtb->dev = p;
				mtb->bar = i;
				mtb->siz = size;
				mtb++;
				if(v & 4)
					i++;	/* skip high word of 64-bit register */
			}

			p->mem[i].size = size;
		}
	}

	/*
	 * Sort both tables IO smallest first, Memory largest
	 */
	qsort(table, itb-table, sizeof(Pcisiz), pcisizcmp);
	tptr = table+ntb;
	qsort(tptr, mtb-tptr, sizeof(Pcisiz), pcisizcmp);

	/*
	 * Allocate IO address space on this bus
	 */
	for(tptr = table; tptr < itb; tptr++) {
		hole = tptr->siz;
		if(tptr->bar == -1)
			hole = 1<<12;
		ioa = (ioa+hole-1) & ~(hole-1);

		p = tptr->dev;
		if(tptr->bar == -1)
			p->ioa.bar = ioa;
		else {
			p->pcr |= IOen;
			p->mem[tptr->bar].bar = ioa|1;
			if(wrreg)
				pcicfgrw32(p->tbdf, PciBAR0+(tptr->bar*4), ioa|1, 0);
		}

		ioa += tptr->siz;
	}

	/*
	 * Allocate Memory address space on this bus
	 */
	for(tptr = table+ntb; tptr < mtb; tptr++) {
		hole = tptr->siz;
		if(tptr->bar == -1)
			hole = 1<<20;
		mema = (mema+hole-1) & ~(hole-1);

		p = tptr->dev;
		if(tptr->bar == -1)
			p->mema.bar = mema;
		else {
			p->pcr |= MEMen;
			p->mem[tptr->bar].bar = mema;
			if(wrreg)
				pcicfgrw32(p->tbdf, PciBAR0+(tptr->bar*4), mema, 0);
		}
		mema += tptr->siz;
	}

	*pmema = mema;
	*pioa = ioa;
	free(table);

	if(wrreg == 0)
		return;

	/*
	 * Finally set all the bridge addresses & registers
	 */
	for(p = root; p != nil; p = p->link) {
		if(p->bridge == nil) {
			pcicfgrw8(p->tbdf, PciLTR, 64, 0);

			p->pcr |= MASen;
			pcicfgrw16(p->tbdf, PciPCR, p->pcr, 0);
			continue;
		}

		base = p->ioa.bar;
		limit = base+p->ioa.size-1;
		v = pcicfgrw32(p->tbdf, PciIBR, 0, 1);
		v = (v&0xFFFF0000)|(limit & 0xF000)|((base & 0xF000)>>8);
		pcicfgrw32(p->tbdf, PciIBR, v, 0);
		v = (limit & 0xFFFF0000)|(base>>16);
		pcicfgrw32(p->tbdf, PciIUBR, v, 0);

		base = p->mema.bar;
		limit = base+p->mema.size-1;
		v = (limit & 0xFFF00000)|((base & 0xFFF00000)>>16);
		pcicfgrw32(p->tbdf, PciMBR, v, 0);

		/*
		 * Disable memory prefetch
		 */
		pcicfgrw32(p->tbdf, PciPMBR, 0x0000FFFF, 0);
		pcicfgrw8(p->tbdf, PciLTR, 64, 0);

		/*
		 * Enable the bridge
		 */
		p->pcr |= IOen|MEMen|MASen;
		pcicfgrw32(p->tbdf, PciPCR, 0xFFFF0000|p->pcr , 0);

		sioa = p->ioa.bar;
		smema = p->mema.bar;
		pcibusmap(p->bridge, &smema, &sioa, 1);
	}
}

static void
pxh6700(Pcidev *p)
{
	int l;

	/*
	 * errata #3, "signal integrity issues when driving secondary bus
	 * in PCI or PCI-X mode 1 ... (causes) parity errors and system hangs."
	 *
	 * Solution is:
	 *
	 * 1.  Write `1's to Function 0/2 (F0 only for PXH-V) Register
	 * offset 224h Bits [29:17] prior to any PCI bus accesses to
	 * secondary interface.
	 * 2.  Wait at least 1.5 micro seconds (us) before any secondary
	 * bus accesses after setting the register.
	 * 3.  Warm (PCI-E reset) and Cold reset (PWROK reset, Front Panel
	 * reset) will clear these register bits requiring reprogramming.
	 * PXH Secondary Bus Reset will not clear the register.
	 */
	l = pcicfgrw32(p->tbdf, 0x224, 0, 1);
	l |= 0x3FFE0000;
	pcicfgrw32(p->tbdf, 0x224, l, 0);
	microdelay(2);

//	print("6700pxh %uX.%uX: pmode=%d pfreq=%d\n", p->did, p->rid, (l>>14)&3, (l>>9)&3);
}

static void
bridgecfg(Pcidev *p)
{
	int l;

	switch (p->vid<<16 | p->did) {
	case 0x80860329:
	case 0x8086032a:
		pxh6700(p);
//	case 0x80863500:
//	case 0x80863501:
//	case 0x80863502:
//	case 0x80863503:
		/* enable IOxAPIC space */
		l = pcicfgrw16(p->tbdf, 0x40, 0, 1);
		l &= ~(1<<13);
		pcicfgrw16(p->tbdf, 0x40, l, 0);
	}
}

static int
pcilscan(int bno, Pcidev** list)
{
	Pcidev *p, *head, *tail;
	int dno, fno, i, hdt, l, maxfno, maxubn, rno, sbn, tbdf, ubn;

	maxubn = bno;
	head = nil;
	tail = nil;
	for(dno = 0; dno <= pcimaxdno; dno++){
		maxfno = 0;
		for(fno = 0; fno <= maxfno; fno++){
			/*
			 * For this possible device, form the
			 * bus+device+function triplet needed to address it
			 * and try to read the vendor and device ID.
			 * If successful, allocate a device struct and
			 * start to fill it in with some useful information
			 * from the device's configuration space.
			 */
			tbdf = MKBUS(BusPCI, bno, dno, fno);
			l = pcicfgrw32(tbdf, PciVID, 0, 1);
			if(l == 0xFFFFFFFF || l == 0)
				continue;
			p = malloc(sizeof(*p));
			p->tbdf = tbdf;
			p->vid = l;
			p->did = l>>16;

			l = pcicfgrw32(tbdf, PciSVID, 0, 1);
			if(l != 0 && l != 0xFFFFFFFF) {
				p->svid = l;
				p->sdid = l>>16;
			}
			if(srxusehba &&
			    p->svid == 0x1b52 && p->did == 1){
				/* coraid-modified pci device id: use substitute id */
				p->did = p->sdid;
				p->sdid = 1;
			}

			if(pcilist != nil)
				pcitail->list = p;
			else
				pcilist = p;
			pcitail = p;

			p->pcr = pcicfgr16(p, PciPCR);
			p->rid = pcicfgr8(p, PciRID);
			p->ccrp = pcicfgr8(p, PciCCRp);
			p->ccru = pcicfgr8(p, PciCCRu);
			p->ccrb = pcicfgr8(p, PciCCRb);
			p->cls = pcicfgr8(p, PciCLS);
			p->ltr = pcicfgr8(p, PciLTR);

			p->intl = pcicfgr8(p, PciINTL);

			/*
			 * If the device is a multi-function device adjust the
			 * loop count so all possible functions are checked.
			 */
			hdt = pcicfgr8(p, PciHDT);
			if(hdt & 0x80)
				maxfno = MaxFNO;

			/*
			 * If appropriate, read the base address registers
			 * and work out the sizes.
			 */
			switch(p->ccrb) {
			default:
			case 0x01:		/* mass storage controller */
			case 0x02:		/* network controller */
			case 0x03:		/* display controller */
			case 0x04:		/* multimedia device */
			case 0x07:		/* simple comm. controllers */
			case 0x08:		/* base system peripherals */
			case 0x09:		/* input devices */
			case 0x0A:		/* docking stations */
			case 0x0B:		/* processors */
			case 0x0C:		/* serial bus controllers */
				if((hdt & 0x7F) != 0)
					break;
				for(i = 0; i < nelem(p->mem); i++) {
					rno = PciBAR0 + 4*i;
					p->mem[i].bar = pcicfgr32(p, rno);
					p->mem[i].size = pcibarsize(p, rno);
				}
				break;

			case 0x06:		/* bridge device */
				bridgecfg(p);
				break;

			case 0x00:
			case 0x05:		/* memory controller */
				break;
			}

			if(head != nil)
				tail->link = p;
			else
				head = p;
			tail = p;
		}
	}

	*list = head;
	for(p = head; p != nil; p = p->link){
		/*
		 * Find PCI-PCI bridges and recursively descend the tree.
		 */
		if(p->ccrb != 0x06 || p->ccru != 0x04)
			continue;

		/*
		 * If the secondary or subordinate bus number is not
		 * initialised try to do what the PCI BIOS should have
		 * done and fill in the numbers as the tree is descended.
		 * On the way down the subordinate bus number is set to
		 * the maximum as it's not known how many buses are behind
		 * this one; the final value is set on the way back up.
		 */
		sbn = pcicfgr8(p, PciSBN);
		ubn = pcicfgr8(p, PciUBN);

		if(sbn == 0 || ubn == 0 || nobios) {

			if(!nobios)
				print("%T: unconfigured bridge\n", p->tbdf);
			sbn = maxubn+1;
			/*
			 * Make sure memory, I/O and master enables are
			 * off, set the primary, secondary and subordinate
			 * bus numbers and clear the secondary status before
			 * attempting to scan the secondary bus.
			 *
			 * Initialisation of the bridge should be done here.
			 */
			pcicfgw32(p, PciPCR, 0xFFFF0000);
			l = (MaxUBN<<16)|(sbn<<8)|bno;
			pcicfgw32(p, PciPBN, l);
			pcicfgw16(p, PciSPSR, 0xFFFF);
			maxubn = pcilscan(sbn, &p->bridge);
			l = (maxubn<<16)|(sbn<<8)|bno;

			pcicfgw32(p, PciPBN, l);
		}
		else {
			/*
			 * You can't go back.
			 * This shouldn't be possible, but the
			 * Iwill DK8-HTX seems to have subordinate
			 * bus numbers which get smaller on the
			 * way down. Need to look more closely at
			 * this.
			 */
			if(ubn > maxubn)
				maxubn = ubn;
			pcilscan(sbn, &p->bridge);
		}
	}

	return maxubn;
}

static uint8_t
pIIxget(Pcidev *router, uint8_t link)
{
	uint8_t pirq;

	/* link should be 0x60, 0x61, 0x62, 0x63 */
	pirq = pcicfgr8(router, link);
	return (pirq < 16)? pirq: 0;
}

static void
pIIxset(Pcidev *router, uint8_t link, uint8_t irq)
{
	pcicfgw8(router, link, irq);
}

static uint8_t
viaget(Pcidev *router, uint8_t link)
{
	uint8_t pirq;

	/* link should be 1, 2, 3, 5 */
	pirq = (link < 6)? pcicfgr8(router, 0x55 + (link>>1)): 0;

	return (link & 1)? (pirq >> 4): (pirq & 15);
}

static void
viaset(Pcidev *router, uint8_t link, uint8_t irq)
{
	uint8_t pirq;

	pirq = pcicfgr8(router, 0x55 + (link >> 1));
	pirq &= (link & 1)? 0x0f: 0xf0;
	pirq |= (link & 1)? (irq << 4): (irq & 15);
	pcicfgw8(router, 0x55 + (link>>1), pirq);
}

typedef struct Bridge Bridge;
struct Bridge
{
	uint16_t	vid;
	uint16_t	did;
	uint8_t	(*get)(Pcidev *, uint8_t);
	void	(*set)(Pcidev *, uint8_t, uint8_t);
};

static Bridge southbridges[] = {
	{ 0x8086, 0xffff, pIIxget, pIIxset },	/* Intel * */
	{ 0x1002, 0xffff, nil, nil },		/* ati (amd) */
	{ 0x1022, 0xffff, nil, nil },		/* amd */

	{ 0x10DE, 0x00D1, nil, nil },		/* NVIDIA nForce 3 */
	{ 0x10DE, 0x00E0, nil, nil },		/* NVIDIA nForce 3 250 Series */
	{ 0x10DE, 0x00E1, nil, nil },		/* NVIDIA nForce 3 250 Series */

	{ 0x1106, 0x0586, viaget, viaset },	/* Viatech 82C586 */
	{ 0x1106, 0x0596, viaget, viaset },	/* Viatech 82C596 */
	{ 0x1106, 0x0686, viaget, viaset },	/* Viatech 82C686 */
	{ 0x1106, 0x3227, viaget, viaset },	/* Viatech VT8237 */

	{ 0x1166, 0x0200, nil, nil },		/* ServerWorks ServerSet III LE */
};

typedef struct Slot Slot;
struct Slot {
	uint8_t	bus;			// Pci bus number
	uint8_t	dev;			// Pci device number
	uint8_t	maps[12];		// Avoid structs!  Link and mask.
	uint8_t	slot;			// Add-in/built-in slot
	uint8_t	reserved;
};

typedef struct Router Router;
struct Router {
	uint8_t	signature[4];		// Routing table signature
	uint8_t	version[2];		// Version number
	uint8_t	size[2];		// Total table size
	uint8_t	bus;			// Interrupt router bus number
	uint8_t	devfn;			// Router's devfunc
	uint8_t	pciirqs[2];		// Exclusive PCI irqs
	uint8_t	compat[4];		// Compatible PCI interrupt router
	uint8_t	miniport[4];		// Miniport data
	uint8_t	reserved[11];
	uint8_t	checksum;
};

static uint16_t pciirqs;			// Exclusive PCI irqs
static Bridge *southbridge;		// Which southbridge to use.

static void
pcirouting(void)
{
	Slot *e;
	Router *r;
	int size, i, fn, tbdf;
	Pcidev *sbpci, *pci;
	uint8_t *p, pin, irq, link, *map;

	/* Search for PCI interrupt routing table in BIOS */
	for(p = (uint8_t *)KADDR(0xf0000); p < (uint8_t *)KADDR(0xfffff); p += 16)
		if(p[0] == '$' && p[1] == 'P' && p[2] == 'I' && p[3] == 'R')
			break;

	if(p >= (uint8_t *)KADDR(0xfffff))
		return;

	r = (Router *)p;

	print("PCI interrupt routing table version %d.%d at %#p\n",
		r->version[0], r->version[1], r);

	tbdf = (BusPCI << 24)|(r->bus << 16)|(r->devfn << 8);
	sbpci = pcimatchtbdf(tbdf);
	if(sbpci == nil) {
		print("pcirouting: Cannot find south bridge %T\n", tbdf);
		return;
	}

	for(i = 0; i != nelem(southbridges); i++)
		if(sbpci->vid == southbridges[i].vid &&
		   (southbridges[i].did == 0xffff || sbpci->did == southbridges[i].did))
			break;

	if(i == nelem(southbridges)) {
		print("pcirouting: ignoring south bridge %T %.4ux/%.4ux\n", tbdf, sbpci->vid, sbpci->did);
		return;
	}
	southbridge = &southbridges[i];
	if(southbridge->get == nil || southbridge->set == nil)
		return;

	pciirqs = (r->pciirqs[1] << 8)|r->pciirqs[0];

	size = (r->size[1] << 8)|r->size[0];
	for(e = (Slot *)&r[1]; (uint8_t *)e < p + size; e++) {
		if(0){
			print("%.2ux/%.2ux %.2ux: ", e->bus, e->dev, e->slot);
			for (i = 0; i != 4; i++) {
				uint8_t *m = &e->maps[i * 3];
				print("[%d] %.2ux %.4ux ",
					i, m[0], (m[2] << 8)|m[1]);
			}
			print("\n");
		}

		for(fn = 0; fn <= MaxFNO; fn++) {
			tbdf = (BusPCI << 24)|(e->bus << 16)|((e->dev | fn) << 8);
			pci = pcimatchtbdf(tbdf);
			if(pci == nil)
				continue;
			pin = pcicfgr8(pci, PciINTP);
			if(pin == 0 || pin == 0xff)
				continue;

			map = &e->maps[(pin - 1) * 3];
			link = map[0];
			irq = southbridge->get(sbpci, link);
			if(irq == 0 || irq == pci->intl)
				continue;
			if(pci->intl != 0 && pci->intl != 0xFF) {
				print("pcirouting: BIOS workaround: %T at pin %d link %d irq %d -> %d\n",
					  tbdf, pin, link, irq, pci->intl);
				southbridge->set(sbpci, link, pci->intl);
				continue;
			}
			print("pcirouting: %T at pin %d link %d irq %d\n", tbdf, pin, link, irq);
			pcicfgw8(pci, PciINTL, irq);
			pci->intl = irq;
		}
	}
}

static void pcireservemem(void);

static void
pcicfginit(void)
{
	char *p;
	Pcidev **list;
	uint32_t mema, ioa;
	int bno, n;

	if(pcicfgmode != -1)
		return;

	lock(&pcicfginitlock);
	if(pcicfgmode != -1){
		unlock(&pcicfginitlock);
		return;
	}

	if(getconf("*srusehba"))
		srxusehba = 1;
	if(getconf("*nobios"))
		nobios = 1;
	if(getconf("*nopcirouting"))
		nopcirouting = 1;

	/*
	 * Assume Configuration Mechanism One. Method Two was deprecated
	 * a long time ago and was only for backwards compaibility with the
	 * Intel Saturn and Mercury chip sets.
	 */
	pcicfgmode = 1;
	pcimaxdno = MaxDNO;

	fmtinstall('T', tbdffmt);

	if(p = getconf("*pcimaxbno")){
		n = strtoul(p, 0, 0);
		if(n < pcimaxbno)
			pcimaxbno = n;
	}
	if(p = getconf("*pcimaxdno")){
		n = strtoul(p, 0, 0);
		if(n < pcimaxdno)
			pcimaxdno = n;
	}

	list = &pciroot;
	for(bno = 0; bno <= pcimaxbno; bno++) {
		int sbno = bno;
		bno = pcilscan(bno, list);

		while(*list != nil)
			list = &(*list)->link;

		if (sbno == 0) {
			Pcidev *pci;

			/*
			  * If we have found a PCI-to-Cardbus bridge, make sure
			  * it has no valid mappings anymore.
			  */
			pci = pciroot;
			while (pci) {
				if (pci->ccrb == 6 && pci->ccru == 7) {
					uint16_t bcr;

					/* reset the cardbus */
					bcr = pcicfgr16(pci, PciBCR);
					pcicfgw16(pci, PciBCR, 0x40 | bcr);
					delay(50);
				}
				pci = pci->link;
			}
		}
	}

	if(pciroot == nil)
		goto out;

	if(nobios) {
		/*
		 * Work out how big the top bus is
		 */
		mema = 0;
		ioa = 0;
		pcibusmap(pciroot, &mema, &ioa, 0);

		DBG("Sizes: mem=%8.8lux size=%8.8lux io=%8.8lux\n",
			mema, pcimask(mema), ioa);

		/*
		 * Align the windows and map it
		 */
		ioa = 0xD000;
		mema = 0xFEA00000;

		pcilog("Mask sizes: mem=%lux io=%lux\n", mema, ioa);

		pcibusmap(pciroot, &mema, &ioa, 1);
		DBG("Sizes2: mem=%lux io=%lux\n", mema, ioa);

		pcireservemem();
		unlock(&pcicfginitlock);
		return;
	}

	if(!nopcirouting)
		pcirouting();

out:
	pcireservemem();
	unlock(&pcicfginitlock);

	if(getconf("*pcihinv"))
		pcihinv(nil);
}

static void
pcireservemem(void)
{
	int i;
	Pcidev *p;

	/*
	 * mark all the physical address space claimed by pci devices
	 * as in use, so that it's not given out elsewhere.
	 * beware the type and size of 'bar'.
	 */
	for(p=pciroot; p != nil; p=p->list)
		for(i=0; i<nelem(p->mem); i++)
			if(p->mem[i].bar && (p->mem[i].bar&1) == 0)
				memreserve(p->mem[i].bar&~(uintmem)0x0F, p->mem[i].size);
}

static int
pcicfgrw8(int tbdf, int rno, int data, int read)
{
	int o, type, x;

	if(pcicfgmode == -1)
		pcicfginit();

	if(BUSBNO(tbdf))
		type = 0x01;
	else
		type = 0x00;
	x = -1;
	if(BUSDNO(tbdf) > pcimaxdno)
		return x;

	ilock(&pcicfglock);
	o = rno & 0x03;
	rno &= ~0x03;
	outl(PciADDR, 0x80000000|BUSBDF(tbdf)|rno|type);
	if(read)
		x = inb(PciDATA+o);
	else
		outb(PciDATA+o, data);
	outl(PciADDR, 0);
	iunlock(&pcicfglock);

	return x;
}

int
pcicfgr8(Pcidev* pcidev, int rno)
{
	return pcicfgrw8(pcidev->tbdf, rno, 0, 1);
}

void
pcicfgw8(Pcidev* pcidev, int rno, int data)
{
	pcicfgrw8(pcidev->tbdf, rno, data, 0);
}

static int
pcicfgrw16(int tbdf, int rno, int data, int read)
{
	int o, type, x;

	if(pcicfgmode == -1)
		pcicfginit();

	if(BUSBNO(tbdf))
		type = 0x01;
	else
		type = 0x00;
	x = -1;
	if(BUSDNO(tbdf) > pcimaxdno)
		return x;

	ilock(&pcicfglock);
	o = rno & 0x02;
	rno &= ~0x03;
	outl(PciADDR, 0x80000000|BUSBDF(tbdf)|rno|type);
	if(read)
		x = ins(PciDATA+o);
	else
		outs(PciDATA+o, data);
	outl(PciADDR, 0);
	iunlock(&pcicfglock);

	return x;
}

int
pcicfgr16(Pcidev* pcidev, int rno)
{
	return pcicfgrw16(pcidev->tbdf, rno, 0, 1);
}

void
pcicfgw16(Pcidev* pcidev, int rno, int data)
{
	pcicfgrw16(pcidev->tbdf, rno, data, 0);
}

static int
pcicfgrw32(int tbdf, int rno, int data, int read)
{
	int type, x;

	if(pcicfgmode == -1)
		pcicfginit();

	if(BUSBNO(tbdf))
		type = 0x01;
	else
		type = 0x00;
	x = -1;
	if(BUSDNO(tbdf) > pcimaxdno)
		return x;

	ilock(&pcicfglock);
	rno &= ~0x03;
	outl(PciADDR, 0x80000000|BUSBDF(tbdf)|rno|type);
	if(read)
		x = inl(PciDATA);
	else
		outl(PciDATA, data);
	outl(PciADDR, 0);
	iunlock(&pcicfglock);

	return x;
}

uint32_t
pcicfgr32(Pcidev* pcidev, int rno)
{
	return pcicfgrw32(pcidev->tbdf, rno, 0, 1);
}

void
pcicfgw32(Pcidev* pcidev, int rno, int data)
{
	pcicfgrw32(pcidev->tbdf, rno, data, 0);
}

Pcidev*
pcimatch(Pcidev* prev, int vid, int did)
{
	if(pcicfgmode == -1)
		pcicfginit();

	if(prev == nil)
		prev = pcilist;
	else
		prev = prev->list;

	while(prev != nil){
		if((vid == 0 || prev->vid == vid)
		&& (did == 0 || prev->did == did))
			break;
		prev = prev->list;
	}
	return prev;
}

Pcidev*
pcimatchtbdf(int tbdf)
{
	Pcidev *p;

	if(pcicfgmode == -1)
		pcicfginit();

	for(p = pcilist; p != nil; p = p->list) {
		if(p->tbdf == tbdf)
			break;
	}
	return p;
}

uint8_t
pciipin(Pcidev *pci, uint8_t pin)
{
	uint8_t intl;

	if (pci == nil)
		pci = pcilist;

	for(; pci != nil; pci = pci->list){
		if(pcicfgr8(pci, PciINTP) == pin && pci->intl != 0 && pci->intl != 0xff)
			return pci->intl;

		if(pci->bridge && (intl = pciipin(pci->bridge, pin)) != 0)
			return intl;
	}
	return 0;
}

static void
pcilhinv(Pcidev* p)
{
	int i;
	Pcidev *t;

	if(p == nil) {
		putstrn(PCICONS.output, PCICONS.ptr);
		p = pciroot;
		print("tbdf:	 type vid  did intl memory\n");
	}
	for(t = p; t != nil; t = t->link) {
		print("%T: %.2ux %.2ux %.2ux %.4ux/%.4ux %3d  ",
			t->tbdf, t->ccrb, t->ccru, t->ccrp, t->vid, t->did, t->intl);

		for(i = 0; i < nelem(p->mem); i++) {
			if(t->mem[i].size == 0)
				continue;
			print("%d:%.8lux %d ", i,
				t->mem[i].bar, t->mem[i].size);
		}
		if(t->ioa.bar || t->ioa.size)
			print("ioa:%.8lux %d ", t->ioa.bar, t->ioa.size);
		if(t->mema.bar || t->mema.size)
			print("mema:%.8lux %d ", t->mema.bar, t->mema.size);
		if(t->bridge != nil)
			print("->%d", BUSBNO(t->bridge->tbdf));
		print("\n");
	}
	while(p != nil) {
		if(p->bridge != nil)
			pcilhinv(p->bridge);
		p = p->link;
	}
}

void
pcihinv(Pcidev* p)
{
	if(pcicfgmode == -1)
		pcicfginit();
	lock(&pcicfginitlock);
	pcilhinv(p);
	unlock(&pcicfginitlock);
}

void
pcireset(void)
{
	Pcidev *p;

	if(pcicfgmode == -1)
		pcicfginit();

	for(p = pcilist; p != nil; p = p->list) {
		/* don't mess with the bridges */
		if(p->ccrb != 0x06)
			pciclrbme(p);
	}
}

void
pcisetioe(Pcidev* p)
{
	p->pcr |= IOen;
	pcicfgw16(p, PciPCR, p->pcr);
}

void
pciclrioe(Pcidev* p)
{
	p->pcr &= ~IOen;
	pcicfgw16(p, PciPCR, p->pcr);
}

void
pcisetbme(Pcidev* p)
{
	p->pcr |= MASen;
	pcicfgw16(p, PciPCR, p->pcr);
}

void
pciclrbme(Pcidev* p)
{
	p->pcr &= ~MASen;
	pcicfgw16(p, PciPCR, p->pcr);
}

void
pcisetmwi(Pcidev* p)
{
	p->pcr |= MemWrInv;
	pcicfgw16(p, PciPCR, p->pcr);
}

void
pciclrmwi(Pcidev* p)
{
	p->pcr &= ~MemWrInv;
	pcicfgw16(p, PciPCR, p->pcr);
}

int
pcicap(Pcidev *p, int cap)
{
	int i, c, off;

	/* status register bit 4 has capabilities */
	if((pcicfgr16(p, PciPSR) & 1<<4) == 0)
		return -1;	
	switch(pcicfgr8(p, PciHDT) & 0x7f){
	default:
		return -1;
	case 0:				/* etc */
	case 1:				/* pci to pci bridge */
		off = 0x34;
		break;
	case 2:				/* cardbus bridge */
		off = 0x14;
		break;
	}
	for(i = 48; i--;){
		off = pcicfgr8(p, off);
		if(off < 0x40 || (off & 3))
			break;
		off &= ~3;
		c = pcicfgr8(p, off);
		if(c == 0xff)
			break;
		if(c == cap)
			return off;
		off++;
	}
	return -1;
}

static int
pcigetpmrb(Pcidev* p)
{
	if(p->pmrb != 0)
		return p->pmrb;
	return p->pmrb = pcicap(p, PciCapPMG);
}

enum {
	Pmgcap	= 2,		/* capabilities; 2 bytes*/
	Pmgctl	= 4,		/* ctl/status; 2 bytes */
	Pmgbrg	= 6,		/* bridge support */
	Pmgdata	= 7,
};

int
pcigetpms(Pcidev* p)
{
	int ptr;

	if((ptr = pcigetpmrb(p)) == -1)
		return -1;
	return pcicfgr16(p, ptr+Pmgctl) & 0x0003;
}

int
pcisetpms(Pcidev* p, int state)
{
	int pmc, pmcsr, ptr;

	if((ptr = pcigetpmrb(p)) == -1)
		return -1;

	pmc = pcicfgr16(p, ptr+Pmgcap);
	pmcsr = pcicfgr16(p, ptr+Pmgctl);

	switch(state){
	default:
		return -1;
	case 0:
		break;
	case 1:
		if(!(pmc & 0x0200))
			return -1;
		break;
	case 2:
		if(!(pmc & 0x0400))
			return -1;
		break;
	case 3:
		break;
	}
	pcicfgw16(p, ptr+4, (pmcsr & ~3)  | state);
	return pmcsr & 3;
}

void*
pcixcfgaddr(Pcidev *p, int rno)
{
	uint32_t tbdf;
	uintmem pa;

	tbdf = p->tbdf;
	if(p->xcfg == nil){
		pa = pcixcfgspace(BUSBNO(tbdf));
		if(pa == 0)
			return nil;
		pa += MKPCIX(BUSBNO(tbdf), BUSDNO(tbdf), BUSFNO(tbdf), 0);
		p->xcfg = vmap(pa, 4096);
		if(p->xcfg == nil)
			return nil;
	}
	return (uint8_t*)p->xcfg + rno;
}