diff options
author | attilio <attilio@FreeBSD.org> | 2013-03-07 23:43:03 +0000 |
---|---|---|
committer | attilio <attilio@FreeBSD.org> | 2013-03-07 23:43:03 +0000 |
commit | 640e058da3cee2710fd66a67abbb548aad74104e (patch) | |
tree | d7c35ea5ace438f4006e9f03068543a6f001ad74 /sys | |
parent | 1be810ec7308568f15799ceb7f6ad7162994537c (diff) | |
parent | 8f87944624f4ab56ac387d7b3d251630641df1f6 (diff) | |
download | FreeBSD-src-640e058da3cee2710fd66a67abbb548aad74104e.zip FreeBSD-src-640e058da3cee2710fd66a67abbb548aad74104e.tar.gz |
MFC
Diffstat (limited to 'sys')
102 files changed, 12424 insertions, 839 deletions
diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC index 3988195..577cf4b 100644 --- a/sys/amd64/conf/GENERIC +++ b/sys/amd64/conf/GENERIC @@ -138,7 +138,10 @@ device sa # Sequential Access (tape etc) device cd # CD device pass # Passthrough device (direct ATA/SCSI access) device ses # Enclosure Services (SES and SAF-TE) -#device ctl # CAM Target Layer +device ctl # CAM Target Layer +options CTL_DISABLE # Disable CTL by default to save memory. + # Re-enable with kern.cam.ctl.disable=0 in + # /boot/loader.conf # RAID controllers interfaced to the SCSI subsystem device amr # AMI MegaRAID diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES index a4371f7..7a41464 100644 --- a/sys/amd64/conf/NOTES +++ b/sys/amd64/conf/NOTES @@ -442,6 +442,13 @@ options SAFE_RNDTEST # enable rndtest support # # VirtIO support +# +# The virtio entry provides a generic bus for use by the device drivers. +# It must be combined with an interface that communicates with the host. +# Multiple such interfaces defined by the VirtIO specification. FreeBSD +# only has support for PCI. Therefore, virtio_pci must be statically +# compiled in or loaded as a module for the device drivers to function. +# device virtio # Generic VirtIO bus (required) device virtio_pci # VirtIO PCI Interface device vtnet # VirtIO Ethernet device diff --git a/sys/arm/arm/swtch.S b/sys/arm/arm/swtch.S index 4c422e8..4257557 100644 --- a/sys/arm/arm/swtch.S +++ b/sys/arm/arm/swtch.S @@ -211,10 +211,12 @@ ENTRY(cpu_throw) GET_PCPU(r6) str r7, [r6, #PC_CURPCB] + add sp, sp, #4; ldmfd sp!, {r4-r7, pc} ENTRY(cpu_switch) stmfd sp!, {r4-r7, lr} + sub sp, sp, #4; mov r6, r2 /* Save the mutex */ .Lswitch_resume: @@ -488,6 +490,7 @@ ENTRY(cpu_switch) * Pull the registers that got pushed when either savectx() or * cpu_switch() was called and return. */ + add sp, sp, #4; ldmfd sp!, {r4-r7, pc} #ifdef DIAGNOSTIC .Lswitch_bogons: @@ -501,6 +504,7 @@ ENTRY(cpu_switch) #endif ENTRY(savectx) stmfd sp!, {r4-r7, lr} + sub sp, sp, #4 /* * r0 = pcb */ @@ -528,6 +532,7 @@ ENTRY(savectx) bl _C_LABEL(vfp_store) 1: #endif /* ARM_VFP_SUPPORT */ + add sp, sp, #4; ldmfd sp!, {r4-r7, pc} ENTRY(fork_trampoline) diff --git a/sys/arm/arm/vm_machdep.c b/sys/arm/arm/vm_machdep.c index 97b8b76..29a213f 100644 --- a/sys/arm/arm/vm_machdep.c +++ b/sys/arm/arm/vm_machdep.c @@ -73,6 +73,12 @@ __FBSDID("$FreeBSD$"); #include <machine/md_var.h> +/* + * struct switchframe must be a multiple of 8 for correct stack alignment + */ +CTASSERT(sizeof(struct switchframe) == 24); +CTASSERT(sizeof(struct trapframe) == 76); + #ifndef NSFBUFS #define NSFBUFS (512 + maxusers * 16) #endif @@ -131,8 +137,8 @@ cpu_fork(register struct thread *td1, register struct proc *p2, pcb2->un_32.pcb32_sp = td2->td_kstack + USPACE_SVC_STACK_TOP - sizeof(*pcb2); pmap_activate(td2); - td2->td_frame = tf = - (struct trapframe *)pcb2->un_32.pcb32_sp - 1; + td2->td_frame = tf = (struct trapframe *)STACKALIGN( + pcb2->un_32.pcb32_sp - sizeof(struct trapframe)); *tf = *td1->td_frame; sf = (struct switchframe *)tf - 1; sf->sf_r4 = (u_int)fork_return; @@ -142,6 +148,8 @@ cpu_fork(register struct thread *td1, register struct proc *p2, tf->tf_r0 = 0; tf->tf_r1 = 0; pcb2->un_32.pcb32_sp = (u_int)sf; + KASSERT((pcb2->un_32.pcb32_sp & 7) == 0, + ("cpu_fork: Incorrect stack alignment")); /* Setup to release spin count in fork_exit(). */ td2->td_md.md_spinlock_count = 1; @@ -345,6 +353,8 @@ cpu_set_upcall(struct thread *td, struct thread *td0) tf->tf_r0 = 0; td->td_pcb->un_32.pcb32_sp = (u_int)sf; td->td_pcb->un_32.pcb32_und_sp = td->td_kstack + USPACE_UNDEF_STACK_TOP; + KASSERT((td->td_pcb->un_32.pcb32_sp & 7) == 0, + ("cpu_set_upcall: Incorrect stack alignment")); /* Setup to release spin count in fork_exit(). */ td->td_md.md_spinlock_count = 1; @@ -438,6 +448,8 @@ cpu_set_fork_handler(struct thread *td, void (*func)(void *), void *arg) sf->sf_r4 = (u_int)func; sf->sf_r5 = (u_int)arg; td->td_pcb->un_32.pcb32_sp = (u_int)sf; + KASSERT((td->td_pcb->un_32.pcb32_sp & 7) == 0, + ("cpu_set_fork_handler: Incorrect stack alignment")); } /* diff --git a/sys/arm/broadcom/bcm2835/bcm2835_dma.c b/sys/arm/broadcom/bcm2835/bcm2835_dma.c index 5e1c9dc..429ba7c 100644 --- a/sys/arm/broadcom/bcm2835/bcm2835_dma.c +++ b/sys/arm/broadcom/bcm2835/bcm2835_dma.c @@ -199,6 +199,7 @@ bcm_dma_reset(device_t dev, int ch) /* Reset control block */ cb = sc->sc_dma_ch[ch].cb; bzero(cb, sizeof(cb)); + cb->info = INFO_WAIT_RESP; } static int @@ -615,6 +616,7 @@ bcm_dma_intr(void *arg) debug & DEBUG_ERROR_MASK, ch->ch); bus_write_4(sc->sc_mem, BCM_DMA_DEBUG(ch->ch), debug & DEBUG_ERROR_MASK); + bcm_dma_reset(sc->sc_dev, ch->ch); } if (cs & CS_INT) { diff --git a/sys/arm/include/frame.h b/sys/arm/include/frame.h index a24eccc..09ba55f 100644 --- a/sys/arm/include/frame.h +++ b/sys/arm/include/frame.h @@ -138,10 +138,14 @@ typedef struct irqframe { } irqframe_t; /* - * Switch frame + * Switch frame. + * + * It is important this is a multiple of 8 bytes so the stack is correctly + * aligned when we create new threads. */ struct switchframe { + u_int pad; /* Used to pad the struct to a multiple of 8-bytes */ u_int sf_r4; u_int sf_r5; u_int sf_r6; diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c index 768332e..1884c7b 100644 --- a/sys/cam/ctl/ctl.c +++ b/sys/cam/ctl/ctl.c @@ -78,6 +78,8 @@ __FBSDID("$FreeBSD$"); #include <cam/ctl/ctl_scsi_all.h> #include <cam/ctl/ctl_error.h> +#include "opt_ctl.h" + struct ctl_softc *control_softc = NULL; /* @@ -317,7 +319,11 @@ static int persis_offset; static uint8_t ctl_pause_rtr; static int ctl_is_single; static int index_to_aps_page; +#ifdef CTL_DISABLE +int ctl_disable = 1; +#else int ctl_disable = 0; +#endif SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer"); SYSCTL_INT(_kern_cam_ctl, OID_AUTO, disable, CTLFLAG_RDTUN, &ctl_disable, 0, diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/bpobj.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/bpobj.c index 1920da4..3e55663 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/bpobj.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/bpobj.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012 by Delphix. All rights reserved. + * Copyright (c) 2013 by Delphix. All rights reserved. */ #include <sys/bpobj.h> @@ -414,6 +414,12 @@ bpobj_enqueue_subobj(bpobj_t *bpo, uint64_t subobj, dmu_tx_t *tx) VERIFY3U(0, ==, dmu_buf_hold(bpo->bpo_os, subsubobjs, 0, FTAG, &subdb, 0)); + /* + * Make sure that we are not asking dmu_write() + * to write more data than we have in our buffer. + */ + VERIFY3U(subdb->db_size, >=, + numsubsub * sizeof (subobj)); dmu_write(bpo->bpo_os, bpo->bpo_phys->bpo_subobjs, bpo->bpo_phys->bpo_num_subobjs * sizeof (subobj), numsubsub * sizeof (subobj), subdb->db_data, tx); diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c index fc606d5..60c5c7f 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c @@ -1711,7 +1711,7 @@ dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi) doi->doi_checksum = dn->dn_checksum; doi->doi_compress = dn->dn_compress; doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9; - doi->doi_max_offset = (dnp->dn_maxblkid + 1) * dn->dn_datablksz; + doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz; doi->doi_fill_count = 0; for (int i = 0; i < dnp->dn_nblkptr; i++) doi->doi_fill_count += dnp->dn_blkptr[i].blk_fill; diff --git a/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c b/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c index d0db808..65991af 100644 --- a/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c +++ b/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c @@ -1034,6 +1034,7 @@ fasttrap_pid_probe(struct reg *rp) #endif PROC_LOCK(p); + _PHOLD(p); pid = p->p_pid; #if defined(sun) pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock; @@ -1059,6 +1060,7 @@ fasttrap_pid_probe(struct reg *rp) #if defined(sun) mutex_exit(pid_mtx); #endif + _PRELE(p); PROC_UNLOCK(p); return (-1); } @@ -1732,7 +1734,6 @@ fasttrap_pid_probe(struct reg *rp) ASSERT(i <= sizeof (scratch)); - #if defined(sun) if (fasttrap_copyout(scratch, (char *)addr, i)) { #else @@ -1794,7 +1795,11 @@ done: } rp->r_rip = new_pc; - set_regs(curthread, rp); + + PROC_LOCK(p); + proc_write_regs(curthread, rp); + _PRELE(p); + PROC_UNLOCK(p); return (0); } diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 index 131a1e7..5e659b9 100644 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -393,11 +393,11 @@ dev/virtio/virtio.c optional virtio dev/virtio/virtqueue.c optional virtio dev/virtio/virtio_bus_if.m optional virtio dev/virtio/virtio_if.m optional virtio -dev/virtio/pci/virtio_pci.c optional virtio_pci virtio pci -dev/virtio/network/if_vtnet.c optional vtnet virtio -dev/virtio/block/virtio_blk.c optional virtio_blk virtio -dev/virtio/balloon/virtio_balloon.c optional virtio_balloon virtio -dev/virtio/scsi/virtio_scsi.c optional virtio_scsi virtio scbus +dev/virtio/pci/virtio_pci.c optional virtio_pci +dev/virtio/network/if_vtnet.c optional vtnet +dev/virtio/block/virtio_blk.c optional virtio_blk +dev/virtio/balloon/virtio_balloon.c optional virtio_balloon +dev/virtio/scsi/virtio_scsi.c optional virtio_scsi isa/syscons_isa.c optional sc isa/vga_isa.c optional vga kern/kern_clocksource.c standard diff --git a/sys/conf/files.i386 b/sys/conf/files.i386 index fd536af..f4c595a 100644 --- a/sys/conf/files.i386 +++ b/sys/conf/files.i386 @@ -374,11 +374,11 @@ dev/virtio/virtio.c optional virtio dev/virtio/virtqueue.c optional virtio dev/virtio/virtio_bus_if.m optional virtio dev/virtio/virtio_if.m optional virtio -dev/virtio/pci/virtio_pci.c optional virtio_pci virtio pci -dev/virtio/network/if_vtnet.c optional vtnet virtio -dev/virtio/block/virtio_blk.c optional virtio_blk virtio -dev/virtio/balloon/virtio_balloon.c optional virtio_balloon virtio -dev/virtio/scsi/virtio_scsi.c optional virtio_scsi virtio scbus +dev/virtio/pci/virtio_pci.c optional virtio_pci +dev/virtio/network/if_vtnet.c optional vtnet +dev/virtio/block/virtio_blk.c optional virtio_blk +dev/virtio/balloon/virtio_balloon.c optional virtio_balloon +dev/virtio/scsi/virtio_scsi.c optional virtio_scsi i386/acpica/acpi_machdep.c optional acpi acpi_wakecode.o optional acpi \ dependency "$S/i386/acpica/acpi_wakecode.S assym.s" \ diff --git a/sys/conf/options b/sys/conf/options index 75d0c97..ec9271d 100644 --- a/sys/conf/options +++ b/sys/conf/options @@ -329,6 +329,9 @@ SCSI_PT_DEFAULT_TIMEOUT opt_pt.h # Options used only in cam/scsi/scsi_ses.c SES_ENABLE_PASSTHROUGH opt_ses.h +# Options used only in cam/ctl +CTL_DISABLE opt_ctl.h + # Options used in dev/sym/ (Symbios SCSI driver). SYM_SETUP_LP_PROBE_MAP opt_sym.h #-Low Priority Probe Map (bits) # Allows the ncr to take precedence diff --git a/sys/contrib/altq/altq/altq_hfsc.c b/sys/contrib/altq/altq/altq_hfsc.c index bced145..f436bd5 100644 --- a/sys/contrib/altq/altq/altq_hfsc.c +++ b/sys/contrib/altq/altq/altq_hfsc.c @@ -104,14 +104,10 @@ static void update_ed(struct hfsc_class *, int); static void update_d(struct hfsc_class *, int); static void init_vf(struct hfsc_class *, int); static void update_vf(struct hfsc_class *, int, u_int64_t); -static ellist_t *ellist_alloc(void); -static void ellist_destroy(ellist_t *); static void ellist_insert(struct hfsc_class *); static void ellist_remove(struct hfsc_class *); static void ellist_update(struct hfsc_class *); -struct hfsc_class *ellist_get_mindl(ellist_t *, u_int64_t); -static actlist_t *actlist_alloc(void); -static void actlist_destroy(actlist_t *); +struct hfsc_class *hfsc_get_mindl(struct hfsc_if *, u_int64_t); static void actlist_insert(struct hfsc_class *); static void actlist_remove(struct hfsc_class *); static void actlist_update(struct hfsc_class *); @@ -204,12 +200,7 @@ hfsc_add_altq(struct pf_altq *a) if (hif == NULL) return (ENOMEM); - hif->hif_eligible = ellist_alloc(); - if (hif->hif_eligible == NULL) { - free(hif, M_DEVBUF); - return (ENOMEM); - } - + TAILQ_INIT(&hif->hif_eligible); hif->hif_ifq = &ifp->if_snd; /* keep the state in pf_altq */ @@ -230,8 +221,6 @@ hfsc_remove_altq(struct pf_altq *a) (void)hfsc_clear_interface(hif); (void)hfsc_class_destroy(hif->hif_rootclass); - ellist_destroy(hif->hif_eligible); - free(hif, M_DEVBUF); return (0); @@ -408,9 +397,7 @@ hfsc_class_create(struct hfsc_if *hif, struct service_curve *rsc, if (cl->cl_q == NULL) goto err_ret; - cl->cl_actc = actlist_alloc(); - if (cl->cl_actc == NULL) - goto err_ret; + TAILQ_INIT(&cl->cl_actc); if (qlimit == 0) qlimit = 50; /* use default */ @@ -544,8 +531,6 @@ hfsc_class_create(struct hfsc_if *hif, struct service_curve *rsc, return (cl); err_ret: - if (cl->cl_actc != NULL) - actlist_destroy(cl->cl_actc); if (cl->cl_red != NULL) { #ifdef ALTQ_RIO if (q_is_rio(cl->cl_q)) @@ -620,8 +605,6 @@ hfsc_class_destroy(struct hfsc_class *cl) IFQ_UNLOCK(cl->cl_hif->hif_ifq); splx(s); - actlist_destroy(cl->cl_actc); - if (cl->cl_red != NULL) { #ifdef ALTQ_RIO if (q_is_rio(cl->cl_q)) @@ -774,7 +757,7 @@ hfsc_dequeue(struct ifaltq *ifq, int op) * find the class with the minimum deadline among * the eligible classes. */ - if ((cl = ellist_get_mindl(hif->hif_eligible, cur_time)) + if ((cl = hfsc_get_mindl(hif, cur_time)) != NULL) { realtime = 1; } else { @@ -994,7 +977,7 @@ init_vf(struct hfsc_class *cl, int len) go_active = 0; if (go_active) { - max_cl = actlist_last(cl->cl_parent->cl_actc); + max_cl = TAILQ_LAST(&cl->cl_parent->cl_actc, acthead); if (max_cl != NULL) { /* * set vt to the average of the min and max @@ -1159,12 +1142,12 @@ update_cfmin(struct hfsc_class *cl) struct hfsc_class *p; u_int64_t cfmin; - if (TAILQ_EMPTY(cl->cl_actc)) { + if (TAILQ_EMPTY(&cl->cl_actc)) { cl->cl_cfmin = 0; return; } cfmin = HT_INFINITY; - TAILQ_FOREACH(p, cl->cl_actc, cl_actlist) { + TAILQ_FOREACH(p, &cl->cl_actc, cl_actlist) { if (p->cl_f == 0) { cl->cl_cfmin = 0; return; @@ -1184,22 +1167,6 @@ update_cfmin(struct hfsc_class *cl) * there is one eligible list per interface. */ -static ellist_t * -ellist_alloc(void) -{ - ellist_t *head; - - head = malloc(sizeof(ellist_t), M_DEVBUF, M_WAITOK); - TAILQ_INIT(head); - return (head); -} - -static void -ellist_destroy(ellist_t *head) -{ - free(head, M_DEVBUF); -} - static void ellist_insert(struct hfsc_class *cl) { @@ -1207,13 +1174,13 @@ ellist_insert(struct hfsc_class *cl) struct hfsc_class *p; /* check the last entry first */ - if ((p = TAILQ_LAST(hif->hif_eligible, _eligible)) == NULL || + if ((p = TAILQ_LAST(&hif->hif_eligible, elighead)) == NULL || p->cl_e <= cl->cl_e) { - TAILQ_INSERT_TAIL(hif->hif_eligible, cl, cl_ellist); + TAILQ_INSERT_TAIL(&hif->hif_eligible, cl, cl_ellist); return; } - TAILQ_FOREACH(p, hif->hif_eligible, cl_ellist) { + TAILQ_FOREACH(p, &hif->hif_eligible, cl_ellist) { if (cl->cl_e < p->cl_e) { TAILQ_INSERT_BEFORE(p, cl, cl_ellist); return; @@ -1227,7 +1194,7 @@ ellist_remove(struct hfsc_class *cl) { struct hfsc_if *hif = cl->cl_hif; - TAILQ_REMOVE(hif->hif_eligible, cl, cl_ellist); + TAILQ_REMOVE(&hif->hif_eligible, cl, cl_ellist); } static void @@ -1245,11 +1212,11 @@ ellist_update(struct hfsc_class *cl) return; /* check the last entry */ - last = TAILQ_LAST(hif->hif_eligible, _eligible); + last = TAILQ_LAST(&hif->hif_eligible, elighead); ASSERT(last != NULL); if (last->cl_e <= cl->cl_e) { - TAILQ_REMOVE(hif->hif_eligible, cl, cl_ellist); - TAILQ_INSERT_TAIL(hif->hif_eligible, cl, cl_ellist); + TAILQ_REMOVE(&hif->hif_eligible, cl, cl_ellist); + TAILQ_INSERT_TAIL(&hif->hif_eligible, cl, cl_ellist); return; } @@ -1259,7 +1226,7 @@ ellist_update(struct hfsc_class *cl) */ while ((p = TAILQ_NEXT(p, cl_ellist)) != NULL) { if (cl->cl_e < p->cl_e) { - TAILQ_REMOVE(hif->hif_eligible, cl, cl_ellist); + TAILQ_REMOVE(&hif->hif_eligible, cl, cl_ellist); TAILQ_INSERT_BEFORE(p, cl, cl_ellist); return; } @@ -1269,11 +1236,11 @@ ellist_update(struct hfsc_class *cl) /* find the class with the minimum deadline among the eligible classes */ struct hfsc_class * -ellist_get_mindl(ellist_t *head, u_int64_t cur_time) +hfsc_get_mindl(struct hfsc_if *hif, u_int64_t cur_time) { struct hfsc_class *p, *cl = NULL; - TAILQ_FOREACH(p, head, cl_ellist) { + TAILQ_FOREACH(p, &hif->hif_eligible, cl_ellist) { if (p->cl_e > cur_time) break; if (cl == NULL || p->cl_d < cl->cl_d) @@ -1287,34 +1254,20 @@ ellist_get_mindl(ellist_t *head, u_int64_t cur_time) * by their virtual time. * each intermediate class has one active children list. */ -static actlist_t * -actlist_alloc(void) -{ - actlist_t *head; - - head = malloc(sizeof(actlist_t), M_DEVBUF, M_WAITOK); - TAILQ_INIT(head); - return (head); -} static void -actlist_destroy(actlist_t *head) -{ - free(head, M_DEVBUF); -} -static void actlist_insert(struct hfsc_class *cl) { struct hfsc_class *p; /* check the last entry first */ - if ((p = TAILQ_LAST(cl->cl_parent->cl_actc, _active)) == NULL + if ((p = TAILQ_LAST(&cl->cl_parent->cl_actc, acthead)) == NULL || p->cl_vt <= cl->cl_vt) { - TAILQ_INSERT_TAIL(cl->cl_parent->cl_actc, cl, cl_actlist); + TAILQ_INSERT_TAIL(&cl->cl_parent->cl_actc, cl, cl_actlist); return; } - TAILQ_FOREACH(p, cl->cl_parent->cl_actc, cl_actlist) { + TAILQ_FOREACH(p, &cl->cl_parent->cl_actc, cl_actlist) { if (cl->cl_vt < p->cl_vt) { TAILQ_INSERT_BEFORE(p, cl, cl_actlist); return; @@ -1326,7 +1279,7 @@ actlist_insert(struct hfsc_class *cl) static void actlist_remove(struct hfsc_class *cl) { - TAILQ_REMOVE(cl->cl_parent->cl_actc, cl, cl_actlist); + TAILQ_REMOVE(&cl->cl_parent->cl_actc, cl, cl_actlist); } static void @@ -1344,11 +1297,11 @@ actlist_update(struct hfsc_class *cl) return; /* check the last entry */ - last = TAILQ_LAST(cl->cl_parent->cl_actc, _active); + last = TAILQ_LAST(&cl->cl_parent->cl_actc, acthead); ASSERT(last != NULL); if (last->cl_vt <= cl->cl_vt) { - TAILQ_REMOVE(cl->cl_parent->cl_actc, cl, cl_actlist); - TAILQ_INSERT_TAIL(cl->cl_parent->cl_actc, cl, cl_actlist); + TAILQ_REMOVE(&cl->cl_parent->cl_actc, cl, cl_actlist); + TAILQ_INSERT_TAIL(&cl->cl_parent->cl_actc, cl, cl_actlist); return; } @@ -1358,7 +1311,7 @@ actlist_update(struct hfsc_class *cl) */ while ((p = TAILQ_NEXT(p, cl_actlist)) != NULL) { if (cl->cl_vt < p->cl_vt) { - TAILQ_REMOVE(cl->cl_parent->cl_actc, cl, cl_actlist); + TAILQ_REMOVE(&cl->cl_parent->cl_actc, cl, cl_actlist); TAILQ_INSERT_BEFORE(p, cl, cl_actlist); return; } @@ -1371,7 +1324,7 @@ actlist_firstfit(struct hfsc_class *cl, u_int64_t cur_time) { struct hfsc_class *p; - TAILQ_FOREACH(p, cl->cl_actc, cl_actlist) { + TAILQ_FOREACH(p, &cl->cl_actc, cl_actlist) { if (p->cl_f <= cur_time) return (p); } diff --git a/sys/contrib/altq/altq/altq_hfsc.h b/sys/contrib/altq/altq/altq_hfsc.h index 91ba3d1..d04b378 100644 --- a/sys/contrib/altq/altq/altq_hfsc.h +++ b/sys/contrib/altq/altq/altq_hfsc.h @@ -218,16 +218,6 @@ struct runtime_sc { u_int64_t ism2; /* scaled inverse-slope of the 2nd segment */ }; -/* for TAILQ based ellist and actlist implementation */ -struct hfsc_class; -typedef TAILQ_HEAD(_eligible, hfsc_class) ellist_t; -typedef TAILQ_ENTRY(hfsc_class) elentry_t; -typedef TAILQ_HEAD(_active, hfsc_class) actlist_t; -typedef TAILQ_ENTRY(hfsc_class) actentry_t; -#define ellist_first(s) TAILQ_FIRST(s) -#define actlist_first(s) TAILQ_FIRST(s) -#define actlist_last(s) TAILQ_LAST(s, _active) - struct hfsc_class { u_int cl_id; /* class id (just for debug) */ u_int32_t cl_handle; /* class handle */ @@ -277,10 +267,10 @@ struct hfsc_class { u_int cl_vtperiod; /* vt period sequence no */ u_int cl_parentperiod; /* parent's vt period seqno */ int cl_nactive; /* number of active children */ - actlist_t *cl_actc; /* active children list */ - actentry_t cl_actlist; /* active children list entry */ - elentry_t cl_ellist; /* eligible list entry */ + TAILQ_HEAD(acthead, hfsc_class) cl_actc; /* active children list */ + TAILQ_ENTRY(hfsc_class) cl_actlist; /* active children list entry */ + TAILQ_ENTRY(hfsc_class) cl_ellist; /* eligible list entry */ struct { struct pktcntr xmit_cnt; @@ -304,7 +294,7 @@ struct hfsc_if { u_int hif_packets; /* # of packets in the tree */ u_int hif_classid; /* class id sequence number */ - ellist_t *hif_eligible; /* eligible list */ + TAILQ_HEAD(elighead, hfsc_class) hif_eligible; /* eligible list */ #ifdef ALTQ3_CLFIER_COMPAT struct acc_classifier hif_classifier; diff --git a/sys/dev/drm2/drmP.h b/sys/dev/drm2/drmP.h index 9b721a8..32c710a 100644 --- a/sys/dev/drm2/drmP.h +++ b/sys/dev/drm2/drmP.h @@ -228,6 +228,7 @@ typedef void irqreturn_t; #define IRQ_NONE /* nothing */ #define unlikely(x) __builtin_expect(!!(x), 0) +#define likely(x) __builtin_expect(!!(x), 1) #define container_of(ptr, type, member) ({ \ __typeof( ((type *)0)->member ) *__mptr = (ptr); \ (type *)( (char *)__mptr - offsetof(type,member) );}) @@ -905,6 +906,7 @@ struct drm_device { struct drm_minor *control; /**< Control node for card */ struct drm_minor *primary; /**< render type primary screen head */ + void *drm_ttm_bo; struct unrhdr *drw_unrhdr; /* RB tree of drawable infos */ RB_HEAD(drawable_tree, bsd_drm_drawable_info) drw_head; @@ -1301,10 +1303,14 @@ void drm_gem_release(struct drm_device *dev, struct drm_file *file_priv); int drm_gem_create_mmap_offset(struct drm_gem_object *obj); void drm_gem_free_mmap_offset(struct drm_gem_object *obj); -int drm_gem_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size, - struct vm_object **obj_res, int nprot); +int drm_gem_mmap_single(struct drm_device *dev, vm_ooffset_t *offset, + vm_size_t size, struct vm_object **obj_res, int nprot); void drm_gem_pager_dtr(void *obj); +struct ttm_bo_device; +int ttm_bo_mmap_single(struct ttm_bo_device *bdev, vm_ooffset_t *offset, + vm_size_t size, struct vm_object **obj_res, int nprot); + void drm_device_lock_mtx(struct drm_device *dev); void drm_device_unlock_mtx(struct drm_device *dev); int drm_device_sleep_mtx(struct drm_device *dev, void *chan, int flags, diff --git a/sys/dev/drm2/drm_drv.c b/sys/dev/drm2/drm_drv.c index 483aff5..c45bda1 100644 --- a/sys/dev/drm2/drm_drv.c +++ b/sys/dev/drm2/drm_drv.c @@ -58,6 +58,8 @@ static int drm_load(struct drm_device *dev); static void drm_unload(struct drm_device *dev); static drm_pci_id_list_t *drm_find_description(int vendor, int device, drm_pci_id_list_t *idlist); +static int drm_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, + vm_size_t size, struct vm_object **obj_res, int nprot); static int drm_modevent(module_t mod, int type, void *data) @@ -187,7 +189,7 @@ static struct cdevsw drm_cdevsw = { .d_ioctl = drm_ioctl, .d_poll = drm_poll, .d_mmap = drm_mmap, - .d_mmap_single = drm_gem_mmap_single, + .d_mmap_single = drm_mmap_single, .d_name = "drm", .d_flags = D_TRACKCLOSE }; @@ -955,6 +957,23 @@ drm_add_busid_modesetting(struct drm_device *dev, struct sysctl_ctx_list *ctx, return (0); } +static int +drm_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size, + struct vm_object **obj_res, int nprot) +{ + struct drm_device *dev; + + dev = drm_get_device_from_kdev(kdev); + if ((dev->driver->driver_features & DRIVER_GEM) != 0) { + return (drm_gem_mmap_single(dev, offset, size, obj_res, nprot)); + } else if (dev->drm_ttm_bo != NULL) { + return (ttm_bo_mmap_single(dev->drm_ttm_bo, offset, size, + obj_res, nprot)); + } else { + return (ENODEV); + } +} + #if DRM_LINUX #include <sys/sysproto.h> diff --git a/sys/dev/drm2/drm_gem.c b/sys/dev/drm2/drm_gem.c index f2c3e08..a792839 100644 --- a/sys/dev/drm2/drm_gem.c +++ b/sys/dev/drm2/drm_gem.c @@ -441,16 +441,12 @@ drm_gem_free_mmap_offset(struct drm_gem_object *obj) } int -drm_gem_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size, +drm_gem_mmap_single(struct drm_device *dev, vm_ooffset_t *offset, vm_size_t size, struct vm_object **obj_res, int nprot) { - struct drm_device *dev; struct drm_gem_object *gem_obj; struct vm_object *vm_obj; - dev = drm_get_device_from_kdev(kdev); - if ((dev->driver->driver_features & DRIVER_GEM) == 0) - return (ENODEV); DRM_LOCK(dev); gem_obj = drm_gem_object_from_offset(dev, *offset); if (gem_obj == NULL) { diff --git a/sys/dev/drm2/drm_global.c b/sys/dev/drm2/drm_global.c new file mode 100644 index 0000000..992d061 --- /dev/null +++ b/sys/dev/drm2/drm_global.c @@ -0,0 +1,110 @@ +/************************************************************************** + * + * Copyright 2008-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <dev/drm2/drmP.h> +#include <dev/drm2/drm_global.h> + +MALLOC_DEFINE(M_DRM_GLOBAL, "drm_global", "DRM Global Items"); + +struct drm_global_item { + struct sx mutex; + void *object; + int refcount; +}; + +static struct drm_global_item glob[DRM_GLOBAL_NUM]; + +void drm_global_init(void) +{ + int i; + + for (i = 0; i < DRM_GLOBAL_NUM; ++i) { + struct drm_global_item *item = &glob[i]; + sx_init(&item->mutex, "drmgi"); + item->object = NULL; + item->refcount = 0; + } +} + +void drm_global_release(void) +{ + int i; + for (i = 0; i < DRM_GLOBAL_NUM; ++i) { + struct drm_global_item *item = &glob[i]; + MPASS(item->object == NULL); + MPASS(item->refcount == 0); + sx_destroy(&item->mutex); + } +} + +int drm_global_item_ref(struct drm_global_reference *ref) +{ + int ret; + struct drm_global_item *item = &glob[ref->global_type]; + void *object; + + sx_xlock(&item->mutex); + if (item->refcount == 0) { + item->object = malloc(ref->size, M_DRM_GLOBAL, + M_WAITOK | M_ZERO); + + ref->object = item->object; + ret = ref->init(ref); + if (unlikely(ret != 0)) + goto out_err; + + } + ++item->refcount; + ref->object = item->object; + object = item->object; + sx_xunlock(&item->mutex); + return 0; +out_err: + sx_xunlock(&item->mutex); + item->object = NULL; + return ret; +} + +void drm_global_item_unref(struct drm_global_reference *ref) +{ + struct drm_global_item *item = &glob[ref->global_type]; + + sx_xlock(&item->mutex); + MPASS(item->refcount != 0); + MPASS(ref->object == item->object); + if (--item->refcount == 0) { + ref->release(ref); + item->object = NULL; + } + sx_xunlock(&item->mutex); +} diff --git a/sys/dev/drm2/drm_global.h b/sys/dev/drm2/drm_global.h new file mode 100644 index 0000000..f45b015 --- /dev/null +++ b/sys/dev/drm2/drm_global.h @@ -0,0 +1,56 @@ +/************************************************************************** + * + * Copyright 2008-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ +/* $FreeBSD$ */ + +#ifndef _DRM_GLOBAL_H_ +#define _DRM_GLOBAL_H_ +enum drm_global_types { + DRM_GLOBAL_TTM_MEM = 0, + DRM_GLOBAL_TTM_BO, + DRM_GLOBAL_TTM_OBJECT, + DRM_GLOBAL_NUM +}; + +struct drm_global_reference { + enum drm_global_types global_type; + size_t size; + void *object; + int (*init) (struct drm_global_reference *); + void (*release) (struct drm_global_reference *); +}; + +extern void drm_global_init(void); +extern void drm_global_release(void); +extern int drm_global_item_ref(struct drm_global_reference *ref); +extern void drm_global_item_unref(struct drm_global_reference *ref); + +MALLOC_DECLARE(M_DRM_GLOBAL); + +#endif diff --git a/sys/dev/drm2/drm_mm.c b/sys/dev/drm2/drm_mm.c index cb35fc0..3617b05 100644 --- a/sys/dev/drm2/drm_mm.c +++ b/sys/dev/drm2/drm_mm.c @@ -561,3 +561,40 @@ void drm_mm_takedown(struct drm_mm * mm) KASSERT(mm->num_unused == 0, ("num_unused != 0")); } + +void drm_mm_debug_table(struct drm_mm *mm, const char *prefix) +{ + struct drm_mm_node *entry; + unsigned long total_used = 0, total_free = 0, total = 0; + unsigned long hole_start, hole_end, hole_size; + + hole_start = drm_mm_hole_node_start(&mm->head_node); + hole_end = drm_mm_hole_node_end(&mm->head_node); + hole_size = hole_end - hole_start; + if (hole_size) + printf("%s 0x%08lx-0x%08lx: %8lu: free\n", + prefix, hole_start, hole_end, + hole_size); + total_free += hole_size; + + drm_mm_for_each_node(entry, mm) { + printf("%s 0x%08lx-0x%08lx: %8lu: used\n", + prefix, entry->start, entry->start + entry->size, + entry->size); + total_used += entry->size; + + if (entry->hole_follows) { + hole_start = drm_mm_hole_node_start(entry); + hole_end = drm_mm_hole_node_end(entry); + hole_size = hole_end - hole_start; + printf("%s 0x%08lx-0x%08lx: %8lu: free\n", + prefix, hole_start, hole_end, + hole_size); + total_free += hole_size; + } + } + total = total_free + total_used; + + printf("%s total: %lu, used %lu free %lu\n", prefix, total, + total_used, total_free); +} diff --git a/sys/dev/drm2/drm_mm.h b/sys/dev/drm2/drm_mm.h index 7b173af..7150e24 100644 --- a/sys/dev/drm2/drm_mm.h +++ b/sys/dev/drm2/drm_mm.h @@ -182,4 +182,6 @@ void drm_mm_init_scan_with_range(struct drm_mm *mm, unsigned long size, int drm_mm_scan_add_block(struct drm_mm_node *node); int drm_mm_scan_remove_block(struct drm_mm_node *node); +void drm_mm_debug_table(struct drm_mm *mm, const char *prefix); + #endif diff --git a/sys/dev/drm2/ttm/ttm_agp_backend.c b/sys/dev/drm2/ttm/ttm_agp_backend.c new file mode 100644 index 0000000..48f2193 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_agp_backend.c @@ -0,0 +1,145 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + * Keith Packard. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <dev/drm2/drmP.h> +#include <dev/drm2/ttm/ttm_module.h> +#include <dev/drm2/ttm/ttm_bo_driver.h> +#include <dev/drm2/ttm/ttm_page_alloc.h> +#ifdef TTM_HAS_AGP +#include <dev/drm2/ttm/ttm_placement.h> + +struct ttm_agp_backend { + struct ttm_tt ttm; + struct agp_memory *mem; + device_t bridge; +}; + +MALLOC_DEFINE(M_TTM_AGP, "ttm_agp", "TTM AGP Backend"); + +static int ttm_agp_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + struct drm_mm_node *node = bo_mem->mm_node; + struct agp_memory *mem; + int ret, cached = (bo_mem->placement & TTM_PL_FLAG_CACHED); + unsigned i; + + mem = agp_alloc_memory(agp_be->bridge, AGP_USER_MEMORY, ttm->num_pages); + if (unlikely(mem == NULL)) + return -ENOMEM; + + mem->page_count = 0; + for (i = 0; i < ttm->num_pages; i++) { + vm_page_t page = ttm->pages[i]; + + if (!page) + page = ttm->dummy_read_page; + + mem->pages[mem->page_count++] = page; + } + agp_be->mem = mem; + + mem->is_flushed = 1; + mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY; + + ret = agp_bind_memory(mem, node->start); + if (ret) + pr_err("AGP Bind memory failed\n"); + + return ret; +} + +static int ttm_agp_unbind(struct ttm_tt *ttm) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + + if (agp_be->mem) { + if (agp_be->mem->is_bound) + return agp_unbind_memory(agp_be->mem); + agp_free_memory(agp_be->mem); + agp_be->mem = NULL; + } + return 0; +} + +static void ttm_agp_destroy(struct ttm_tt *ttm) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + + if (agp_be->mem) + ttm_agp_unbind(ttm); + ttm_tt_fini(ttm); + free(agp_be, M_TTM_AGP); +} + +static struct ttm_backend_func ttm_agp_func = { + .bind = ttm_agp_bind, + .unbind = ttm_agp_unbind, + .destroy = ttm_agp_destroy, +}; + +struct ttm_tt *ttm_agp_tt_create(struct ttm_bo_device *bdev, + device_t bridge, + unsigned long size, uint32_t page_flags, + vm_page_t dummy_read_page) +{ + struct ttm_agp_backend *agp_be; + + agp_be = malloc(sizeof(*agp_be), M_TTM_AGP, M_WAITOK | M_ZERO); + + agp_be->mem = NULL; + agp_be->bridge = bridge; + agp_be->ttm.func = &ttm_agp_func; + + if (ttm_tt_init(&agp_be->ttm, bdev, size, page_flags, dummy_read_page)) { + return NULL; + } + + return &agp_be->ttm; +} + +int ttm_agp_tt_populate(struct ttm_tt *ttm) +{ + if (ttm->state != tt_unpopulated) + return 0; + + return ttm_pool_populate(ttm); +} + +void ttm_agp_tt_unpopulate(struct ttm_tt *ttm) +{ + ttm_pool_unpopulate(ttm); +} + +#endif diff --git a/sys/dev/drm2/ttm/ttm_bo.c b/sys/dev/drm2/ttm/ttm_bo.c new file mode 100644 index 0000000..12e5131 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_bo.c @@ -0,0 +1,1820 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <dev/drm2/drmP.h> +#include <dev/drm2/ttm/ttm_module.h> +#include <dev/drm2/ttm/ttm_bo_driver.h> +#include <dev/drm2/ttm/ttm_placement.h> + +#define TTM_ASSERT_LOCKED(param) +#define TTM_DEBUG(fmt, arg...) +#define TTM_BO_HASH_ORDER 13 + +static int ttm_bo_setup_vm(struct ttm_buffer_object *bo); +static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); +static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob); + +MALLOC_DEFINE(M_TTM_BO, "ttm_bo", "TTM Buffer Objects"); + +static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type) +{ + int i; + + for (i = 0; i <= TTM_PL_PRIV5; i++) + if (flags & (1 << i)) { + *mem_type = i; + return 0; + } + return -EINVAL; +} + +static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) +{ + struct ttm_mem_type_manager *man = &bdev->man[mem_type]; + + printf(" has_type: %d\n", man->has_type); + printf(" use_type: %d\n", man->use_type); + printf(" flags: 0x%08X\n", man->flags); + printf(" gpu_offset: 0x%08lX\n", man->gpu_offset); + printf(" size: %ju\n", (uintmax_t)man->size); + printf(" available_caching: 0x%08X\n", man->available_caching); + printf(" default_caching: 0x%08X\n", man->default_caching); + if (mem_type != TTM_PL_SYSTEM) + (*man->func->debug)(man, TTM_PFX); +} + +static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, + struct ttm_placement *placement) +{ + int i, ret, mem_type; + + printf("No space for %p (%lu pages, %luK, %luM)\n", + bo, bo->mem.num_pages, bo->mem.size >> 10, + bo->mem.size >> 20); + for (i = 0; i < placement->num_placement; i++) { + ret = ttm_mem_type_from_flags(placement->placement[i], + &mem_type); + if (ret) + return; + printf(" placement[%d]=0x%08X (%d)\n", + i, placement->placement[i], mem_type); + ttm_mem_type_debug(bo->bdev, mem_type); + } +} + +#if 0 +static ssize_t ttm_bo_global_show(struct ttm_bo_global *glob, + char *buffer) +{ + + return snprintf(buffer, PAGE_SIZE, "%lu\n", + (unsigned long) atomic_read(&glob->bo_count)); +} +#endif + +static inline uint32_t ttm_bo_type_flags(unsigned type) +{ + return 1 << (type); +} + +static void ttm_bo_release_list(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + size_t acc_size = bo->acc_size; + + MPASS(atomic_read(&bo->list_kref) == 0); + MPASS(atomic_read(&bo->kref) == 0); + MPASS(atomic_read(&bo->cpu_writers) == 0); + MPASS(bo->sync_obj == NULL); + MPASS(bo->mem.mm_node == NULL); + MPASS(list_empty(&bo->lru)); + MPASS(list_empty(&bo->ddestroy)); + + if (bo->ttm) + ttm_tt_destroy(bo->ttm); + atomic_dec(&bo->glob->bo_count); + if (bo->destroy) + bo->destroy(bo); + else { + free(bo, M_TTM_BO); + } + ttm_mem_global_free(bdev->glob->mem_glob, acc_size); +} + +int +ttm_bo_wait_unreserved_locked(struct ttm_buffer_object *bo, bool interruptible) +{ + const char *wmsg; + int flags, ret; + + ret = 0; + if (interruptible) { + flags = PCATCH; + wmsg = "ttbowi"; + } else { + flags = 0; + wmsg = "ttbowu"; + } + while (!ttm_bo_is_reserved(bo)) { + ret = -msleep(bo, &bo->glob->lru_lock, flags, wmsg, 0); + if (ret != 0) + break; + } + return (ret); +} + +void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_mem_type_manager *man; + + MPASS(ttm_bo_is_reserved(bo)); + + if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { + + MPASS(list_empty(&bo->lru)); + + man = &bdev->man[bo->mem.mem_type]; + list_add_tail(&bo->lru, &man->lru); + refcount_acquire(&bo->list_kref); + + if (bo->ttm != NULL) { + list_add_tail(&bo->swap, &bo->glob->swap_lru); + refcount_acquire(&bo->list_kref); + } + } +} + +int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) +{ + int put_count = 0; + + if (!list_empty(&bo->swap)) { + list_del_init(&bo->swap); + ++put_count; + } + if (!list_empty(&bo->lru)) { + list_del_init(&bo->lru); + ++put_count; + } + + /* + * TODO: Add a driver hook to delete from + * driver-specific LRU's here. + */ + + return put_count; +} + +int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait, bool use_sequence, uint32_t sequence) +{ + int ret; + + while (unlikely(atomic_read(&bo->reserved) != 0)) { + /** + * Deadlock avoidance for multi-bo reserving. + */ + if (use_sequence && bo->seq_valid) { + /** + * We've already reserved this one. + */ + if (unlikely(sequence == bo->val_seq)) + return -EDEADLK; + /** + * Already reserved by a thread that will not back + * off for us. We need to back off. + */ + if (unlikely(sequence - bo->val_seq < (1 << 31))) + return -EAGAIN; + } + + if (no_wait) + return -EBUSY; + + ret = ttm_bo_wait_unreserved_locked(bo, interruptible); + if (unlikely(ret)) + return ret; + } + + atomic_set(&bo->reserved, 1); + if (use_sequence) { + /** + * Wake up waiters that may need to recheck for deadlock, + * if we decreased the sequence number. + */ + if (unlikely((bo->val_seq - sequence < (1 << 31)) + || !bo->seq_valid)) + wakeup(bo); + + bo->val_seq = sequence; + bo->seq_valid = true; + } else { + bo->seq_valid = false; + } + + return 0; +} + +void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, + bool never_free) +{ + u_int old; + + old = atomic_fetchadd_int(&bo->list_kref, -count); + if (old <= count) { + if (never_free) + panic("ttm_bo_ref_buf"); + ttm_bo_release_list(bo); + } +} + +int ttm_bo_reserve(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait, bool use_sequence, uint32_t sequence) +{ + struct ttm_bo_global *glob = bo->glob; + int put_count = 0; + int ret; + + mtx_lock(&glob->lru_lock); + ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence, + sequence); + if (likely(ret == 0)) + put_count = ttm_bo_del_from_lru(bo); + mtx_unlock(&glob->lru_lock); + + ttm_bo_list_ref_sub(bo, put_count, true); + + return ret; +} + +void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo) +{ + ttm_bo_add_to_lru(bo); + atomic_set(&bo->reserved, 0); + wakeup(bo); +} + +void ttm_bo_unreserve(struct ttm_buffer_object *bo) +{ + struct ttm_bo_global *glob = bo->glob; + + mtx_lock(&glob->lru_lock); + ttm_bo_unreserve_locked(bo); + mtx_unlock(&glob->lru_lock); +} + +/* + * Call bo->mutex locked. + */ +static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_bo_global *glob = bo->glob; + int ret = 0; + uint32_t page_flags = 0; + + TTM_ASSERT_LOCKED(&bo->mutex); + bo->ttm = NULL; + + if (bdev->need_dma32) + page_flags |= TTM_PAGE_FLAG_DMA32; + + switch (bo->type) { + case ttm_bo_type_device: + if (zero_alloc) + page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; + case ttm_bo_type_kernel: + bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, + page_flags, glob->dummy_read_page); + if (unlikely(bo->ttm == NULL)) + ret = -ENOMEM; + break; + case ttm_bo_type_sg: + bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, + page_flags | TTM_PAGE_FLAG_SG, + glob->dummy_read_page); + if (unlikely(bo->ttm == NULL)) { + ret = -ENOMEM; + break; + } + bo->ttm->sg = bo->sg; + break; + default: + printf("[TTM] Illegal buffer object type\n"); + ret = -EINVAL; + break; + } + + return ret; +} + +static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, + struct ttm_mem_reg *mem, + bool evict, bool interruptible, + bool no_wait_gpu) +{ + struct ttm_bo_device *bdev = bo->bdev; + bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); + bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); + struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; + struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; + int ret = 0; + + if (old_is_pci || new_is_pci || + ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { + ret = ttm_mem_io_lock(old_man, true); + if (unlikely(ret != 0)) + goto out_err; + ttm_bo_unmap_virtual_locked(bo); + ttm_mem_io_unlock(old_man); + } + + /* + * Create and bind a ttm if required. + */ + + if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { + if (bo->ttm == NULL) { + bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); + ret = ttm_bo_add_ttm(bo, zero); + if (ret) + goto out_err; + } + + ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); + if (ret) + goto out_err; + + if (mem->mem_type != TTM_PL_SYSTEM) { + ret = ttm_tt_bind(bo->ttm, mem); + if (ret) + goto out_err; + } + + if (bo->mem.mem_type == TTM_PL_SYSTEM) { + if (bdev->driver->move_notify) + bdev->driver->move_notify(bo, mem); + bo->mem = *mem; + mem->mm_node = NULL; + goto moved; + } + } + + if (bdev->driver->move_notify) + bdev->driver->move_notify(bo, mem); + + if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && + !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) + ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); + else if (bdev->driver->move) + ret = bdev->driver->move(bo, evict, interruptible, + no_wait_gpu, mem); + else + ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); + + if (ret) { + if (bdev->driver->move_notify) { + struct ttm_mem_reg tmp_mem = *mem; + *mem = bo->mem; + bo->mem = tmp_mem; + bdev->driver->move_notify(bo, mem); + bo->mem = *mem; + } + + goto out_err; + } + +moved: + if (bo->evicted) { + ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); + if (ret) + printf("[TTM] Can not flush read caches\n"); + bo->evicted = false; + } + + if (bo->mem.mm_node) { + bo->offset = (bo->mem.start << PAGE_SHIFT) + + bdev->man[bo->mem.mem_type].gpu_offset; + bo->cur_placement = bo->mem.placement; + } else + bo->offset = 0; + + return 0; + +out_err: + new_man = &bdev->man[bo->mem.mem_type]; + if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { + ttm_tt_unbind(bo->ttm); + ttm_tt_destroy(bo->ttm); + bo->ttm = NULL; + } + + return ret; +} + +/** + * Call bo::reserved. + * Will release GPU memory type usage on destruction. + * This is the place to put in driver specific hooks to release + * driver private resources. + * Will release the bo::reserved lock. + */ + +static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) +{ + if (bo->bdev->driver->move_notify) + bo->bdev->driver->move_notify(bo, NULL); + + if (bo->ttm) { + ttm_tt_unbind(bo->ttm); + ttm_tt_destroy(bo->ttm); + bo->ttm = NULL; + } + ttm_bo_mem_put(bo, &bo->mem); + + atomic_set(&bo->reserved, 0); + wakeup(&bo); + + /* + * Since the final reference to this bo may not be dropped by + * the current task we have to put a memory barrier here to make + * sure the changes done in this function are always visible. + * + * This function only needs protection against the final kref_put. + */ + mb(); +} + +static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_bo_global *glob = bo->glob; + struct ttm_bo_driver *driver = bdev->driver; + void *sync_obj = NULL; + int put_count; + int ret; + + mtx_lock(&glob->lru_lock); + ret = ttm_bo_reserve_locked(bo, false, true, false, 0); + + mtx_lock(&bdev->fence_lock); + (void) ttm_bo_wait(bo, false, false, true); + if (!ret && !bo->sync_obj) { + mtx_unlock(&bdev->fence_lock); + put_count = ttm_bo_del_from_lru(bo); + + mtx_unlock(&glob->lru_lock); + ttm_bo_cleanup_memtype_use(bo); + + ttm_bo_list_ref_sub(bo, put_count, true); + + return; + } + if (bo->sync_obj) + sync_obj = driver->sync_obj_ref(bo->sync_obj); + mtx_unlock(&bdev->fence_lock); + + if (!ret) { + atomic_set(&bo->reserved, 0); + wakeup(bo); + } + + refcount_acquire(&bo->list_kref); + list_add_tail(&bo->ddestroy, &bdev->ddestroy); + mtx_unlock(&glob->lru_lock); + + if (sync_obj) { + driver->sync_obj_flush(sync_obj); + driver->sync_obj_unref(&sync_obj); + } + taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, + ((hz / 100) < 1) ? 1 : hz / 100); +} + +/** + * function ttm_bo_cleanup_refs_and_unlock + * If bo idle, remove from delayed- and lru lists, and unref. + * If not idle, do nothing. + * + * Must be called with lru_lock and reservation held, this function + * will drop both before returning. + * + * @interruptible Any sleeps should occur interruptibly. + * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. + */ + +static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait_gpu) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_bo_driver *driver = bdev->driver; + struct ttm_bo_global *glob = bo->glob; + int put_count; + int ret; + + mtx_lock(&bdev->fence_lock); + ret = ttm_bo_wait(bo, false, false, true); + + if (ret && !no_wait_gpu) { + void *sync_obj; + + /* + * Take a reference to the fence and unreserve, + * at this point the buffer should be dead, so + * no new sync objects can be attached. + */ + sync_obj = driver->sync_obj_ref(bo->sync_obj); + mtx_unlock(&bdev->fence_lock); + + atomic_set(&bo->reserved, 0); + wakeup(bo); + mtx_unlock(&glob->lru_lock); + + ret = driver->sync_obj_wait(sync_obj, false, interruptible); + driver->sync_obj_unref(&sync_obj); + if (ret) + return ret; + + /* + * remove sync_obj with ttm_bo_wait, the wait should be + * finished, and no new wait object should have been added. + */ + mtx_lock(&bdev->fence_lock); + ret = ttm_bo_wait(bo, false, false, true); + mtx_unlock(&bdev->fence_lock); + if (ret) + return ret; + + mtx_lock(&glob->lru_lock); + ret = ttm_bo_reserve_locked(bo, false, true, false, 0); + + /* + * We raced, and lost, someone else holds the reservation now, + * and is probably busy in ttm_bo_cleanup_memtype_use. + * + * Even if it's not the case, because we finished waiting any + * delayed destruction would succeed, so just return success + * here. + */ + if (ret) { + mtx_unlock(&glob->lru_lock); + return 0; + } + } else + mtx_unlock(&bdev->fence_lock); + + if (ret || unlikely(list_empty(&bo->ddestroy))) { + atomic_set(&bo->reserved, 0); + wakeup(bo); + mtx_unlock(&glob->lru_lock); + return ret; + } + + put_count = ttm_bo_del_from_lru(bo); + list_del_init(&bo->ddestroy); + ++put_count; + + mtx_unlock(&glob->lru_lock); + ttm_bo_cleanup_memtype_use(bo); + + ttm_bo_list_ref_sub(bo, put_count, true); + + return 0; +} + +/** + * Traverse the delayed list, and call ttm_bo_cleanup_refs on all + * encountered buffers. + */ + +static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) +{ + struct ttm_bo_global *glob = bdev->glob; + struct ttm_buffer_object *entry = NULL; + int ret = 0; + + mtx_lock(&glob->lru_lock); + if (list_empty(&bdev->ddestroy)) + goto out_unlock; + + entry = list_first_entry(&bdev->ddestroy, + struct ttm_buffer_object, ddestroy); + refcount_acquire(&entry->list_kref); + + for (;;) { + struct ttm_buffer_object *nentry = NULL; + + if (entry->ddestroy.next != &bdev->ddestroy) { + nentry = list_first_entry(&entry->ddestroy, + struct ttm_buffer_object, ddestroy); + refcount_acquire(&nentry->list_kref); + } + + ret = ttm_bo_reserve_locked(entry, false, !remove_all, false, 0); + if (!ret) + ret = ttm_bo_cleanup_refs_and_unlock(entry, false, + !remove_all); + else + mtx_unlock(&glob->lru_lock); + + if (refcount_release(&entry->list_kref)) + ttm_bo_release_list(entry); + entry = nentry; + + if (ret || !entry) + goto out; + + mtx_lock(&glob->lru_lock); + if (list_empty(&entry->ddestroy)) + break; + } + +out_unlock: + mtx_unlock(&glob->lru_lock); +out: + if (entry && refcount_release(&entry->list_kref)) + ttm_bo_release_list(entry); + return ret; +} + +static void ttm_bo_delayed_workqueue(void *arg, int pending __unused) +{ + struct ttm_bo_device *bdev = arg; + + if (ttm_bo_delayed_delete(bdev, false)) { + taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, + ((hz / 100) < 1) ? 1 : hz / 100); + } +} + +static void ttm_bo_release(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; + + rw_wlock(&bdev->vm_lock); + if (likely(bo->vm_node != NULL)) { + RB_REMOVE(ttm_bo_device_buffer_objects, + &bdev->addr_space_rb, bo); + drm_mm_put_block(bo->vm_node); + bo->vm_node = NULL; + } + rw_wunlock(&bdev->vm_lock); + ttm_mem_io_lock(man, false); + ttm_mem_io_free_vm(bo); + ttm_mem_io_unlock(man); + ttm_bo_cleanup_refs_or_queue(bo); + if (refcount_release(&bo->list_kref)) + ttm_bo_release_list(bo); +} + +void ttm_bo_unref(struct ttm_buffer_object **p_bo) +{ + struct ttm_buffer_object *bo = *p_bo; + + *p_bo = NULL; + if (refcount_release(&bo->kref)) + ttm_bo_release(bo); +} + +int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) +{ + int pending; + + taskqueue_cancel_timeout(taskqueue_thread, &bdev->wq, &pending); + if (pending) + taskqueue_drain_timeout(taskqueue_thread, &bdev->wq); + return (pending); +} + +void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) +{ + if (resched) { + taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, + ((hz / 100) < 1) ? 1 : hz / 100); + } +} + +static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, + bool no_wait_gpu) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_mem_reg evict_mem; + struct ttm_placement placement; + int ret = 0; + + mtx_lock(&bdev->fence_lock); + ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); + mtx_unlock(&bdev->fence_lock); + + if (unlikely(ret != 0)) { + if (ret != -ERESTART) { + printf("[TTM] Failed to expire sync object before buffer eviction\n"); + } + goto out; + } + + MPASS(ttm_bo_is_reserved(bo)); + + evict_mem = bo->mem; + evict_mem.mm_node = NULL; + evict_mem.bus.io_reserved_vm = false; + evict_mem.bus.io_reserved_count = 0; + + placement.fpfn = 0; + placement.lpfn = 0; + placement.num_placement = 0; + placement.num_busy_placement = 0; + bdev->driver->evict_flags(bo, &placement); + ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, + no_wait_gpu); + if (ret) { + if (ret != -ERESTART) { + printf("[TTM] Failed to find memory space for buffer 0x%p eviction\n", + bo); + ttm_bo_mem_space_debug(bo, &placement); + } + goto out; + } + + ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, + no_wait_gpu); + if (ret) { + if (ret != -ERESTART) + printf("[TTM] Buffer eviction failed\n"); + ttm_bo_mem_put(bo, &evict_mem); + goto out; + } + bo->evicted = true; +out: + return ret; +} + +static int ttm_mem_evict_first(struct ttm_bo_device *bdev, + uint32_t mem_type, + bool interruptible, + bool no_wait_gpu) +{ + struct ttm_bo_global *glob = bdev->glob; + struct ttm_mem_type_manager *man = &bdev->man[mem_type]; + struct ttm_buffer_object *bo; + int ret = -EBUSY, put_count; + + mtx_lock(&glob->lru_lock); + list_for_each_entry(bo, &man->lru, lru) { + ret = ttm_bo_reserve_locked(bo, false, true, false, 0); + if (!ret) + break; + } + + if (ret) { + mtx_unlock(&glob->lru_lock); + return ret; + } + + refcount_acquire(&bo->list_kref); + + if (!list_empty(&bo->ddestroy)) { + ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, + no_wait_gpu); + if (refcount_release(&bo->list_kref)) + ttm_bo_release_list(bo); + return ret; + } + + put_count = ttm_bo_del_from_lru(bo); + mtx_unlock(&glob->lru_lock); + + MPASS(ret == 0); + + ttm_bo_list_ref_sub(bo, put_count, true); + + ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); + ttm_bo_unreserve(bo); + + if (refcount_release(&bo->list_kref)) + ttm_bo_release_list(bo); + return ret; +} + +void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) +{ + struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; + + if (mem->mm_node) + (*man->func->put_node)(man, mem); +} + +/** + * Repeatedly evict memory from the LRU for @mem_type until we create enough + * space, or we've evicted everything and there isn't enough space. + */ +static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, + uint32_t mem_type, + struct ttm_placement *placement, + struct ttm_mem_reg *mem, + bool interruptible, + bool no_wait_gpu) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_mem_type_manager *man = &bdev->man[mem_type]; + int ret; + + do { + ret = (*man->func->get_node)(man, bo, placement, mem); + if (unlikely(ret != 0)) + return ret; + if (mem->mm_node) + break; + ret = ttm_mem_evict_first(bdev, mem_type, + interruptible, no_wait_gpu); + if (unlikely(ret != 0)) + return ret; + } while (1); + if (mem->mm_node == NULL) + return -ENOMEM; + mem->mem_type = mem_type; + return 0; +} + +static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, + uint32_t cur_placement, + uint32_t proposed_placement) +{ + uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; + uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; + + /** + * Keep current caching if possible. + */ + + if ((cur_placement & caching) != 0) + result |= (cur_placement & caching); + else if ((man->default_caching & caching) != 0) + result |= man->default_caching; + else if ((TTM_PL_FLAG_CACHED & caching) != 0) + result |= TTM_PL_FLAG_CACHED; + else if ((TTM_PL_FLAG_WC & caching) != 0) + result |= TTM_PL_FLAG_WC; + else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) + result |= TTM_PL_FLAG_UNCACHED; + + return result; +} + +static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, + uint32_t mem_type, + uint32_t proposed_placement, + uint32_t *masked_placement) +{ + uint32_t cur_flags = ttm_bo_type_flags(mem_type); + + if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) + return false; + + if ((proposed_placement & man->available_caching) == 0) + return false; + + cur_flags |= (proposed_placement & man->available_caching); + + *masked_placement = cur_flags; + return true; +} + +/** + * Creates space for memory region @mem according to its type. + * + * This function first searches for free space in compatible memory types in + * the priority order defined by the driver. If free space isn't found, then + * ttm_bo_mem_force_space is attempted in priority order to evict and find + * space. + */ +int ttm_bo_mem_space(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_mem_reg *mem, + bool interruptible, + bool no_wait_gpu) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_mem_type_manager *man; + uint32_t mem_type = TTM_PL_SYSTEM; + uint32_t cur_flags = 0; + bool type_found = false; + bool type_ok = false; + bool has_erestartsys = false; + int i, ret; + + mem->mm_node = NULL; + for (i = 0; i < placement->num_placement; ++i) { + ret = ttm_mem_type_from_flags(placement->placement[i], + &mem_type); + if (ret) + return ret; + man = &bdev->man[mem_type]; + + type_ok = ttm_bo_mt_compatible(man, + mem_type, + placement->placement[i], + &cur_flags); + + if (!type_ok) + continue; + + cur_flags = ttm_bo_select_caching(man, bo->mem.placement, + cur_flags); + /* + * Use the access and other non-mapping-related flag bits from + * the memory placement flags to the current flags + */ + ttm_flag_masked(&cur_flags, placement->placement[i], + ~TTM_PL_MASK_MEMTYPE); + + if (mem_type == TTM_PL_SYSTEM) + break; + + if (man->has_type && man->use_type) { + type_found = true; + ret = (*man->func->get_node)(man, bo, placement, mem); + if (unlikely(ret)) + return ret; + } + if (mem->mm_node) + break; + } + + if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { + mem->mem_type = mem_type; + mem->placement = cur_flags; + return 0; + } + + if (!type_found) + return -EINVAL; + + for (i = 0; i < placement->num_busy_placement; ++i) { + ret = ttm_mem_type_from_flags(placement->busy_placement[i], + &mem_type); + if (ret) + return ret; + man = &bdev->man[mem_type]; + if (!man->has_type) + continue; + if (!ttm_bo_mt_compatible(man, + mem_type, + placement->busy_placement[i], + &cur_flags)) + continue; + + cur_flags = ttm_bo_select_caching(man, bo->mem.placement, + cur_flags); + /* + * Use the access and other non-mapping-related flag bits from + * the memory placement flags to the current flags + */ + ttm_flag_masked(&cur_flags, placement->busy_placement[i], + ~TTM_PL_MASK_MEMTYPE); + + + if (mem_type == TTM_PL_SYSTEM) { + mem->mem_type = mem_type; + mem->placement = cur_flags; + mem->mm_node = NULL; + return 0; + } + + ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, + interruptible, no_wait_gpu); + if (ret == 0 && mem->mm_node) { + mem->placement = cur_flags; + return 0; + } + if (ret == -ERESTART) + has_erestartsys = true; + } + ret = (has_erestartsys) ? -ERESTART : -ENOMEM; + return ret; +} + +static +int ttm_bo_move_buffer(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + bool interruptible, + bool no_wait_gpu) +{ + int ret = 0; + struct ttm_mem_reg mem; + struct ttm_bo_device *bdev = bo->bdev; + + MPASS(ttm_bo_is_reserved(bo)); + + /* + * FIXME: It's possible to pipeline buffer moves. + * Have the driver move function wait for idle when necessary, + * instead of doing it here. + */ + mtx_lock(&bdev->fence_lock); + ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); + mtx_unlock(&bdev->fence_lock); + if (ret) + return ret; + mem.num_pages = bo->num_pages; + mem.size = mem.num_pages << PAGE_SHIFT; + mem.page_alignment = bo->mem.page_alignment; + mem.bus.io_reserved_vm = false; + mem.bus.io_reserved_count = 0; + /* + * Determine where to move the buffer. + */ + ret = ttm_bo_mem_space(bo, placement, &mem, + interruptible, no_wait_gpu); + if (ret) + goto out_unlock; + ret = ttm_bo_handle_move_mem(bo, &mem, false, + interruptible, no_wait_gpu); +out_unlock: + if (ret && mem.mm_node) + ttm_bo_mem_put(bo, &mem); + return ret; +} + +static int ttm_bo_mem_compat(struct ttm_placement *placement, + struct ttm_mem_reg *mem) +{ + int i; + + if (mem->mm_node && placement->lpfn != 0 && + (mem->start < placement->fpfn || + mem->start + mem->num_pages > placement->lpfn)) + return -1; + + for (i = 0; i < placement->num_placement; i++) { + if ((placement->placement[i] & mem->placement & + TTM_PL_MASK_CACHING) && + (placement->placement[i] & mem->placement & + TTM_PL_MASK_MEM)) + return i; + } + return -1; +} + +int ttm_bo_validate(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + bool interruptible, + bool no_wait_gpu) +{ + int ret; + + MPASS(ttm_bo_is_reserved(bo)); + /* Check that range is valid */ + if (placement->lpfn || placement->fpfn) + if (placement->fpfn > placement->lpfn || + (placement->lpfn - placement->fpfn) < bo->num_pages) + return -EINVAL; + /* + * Check whether we need to move buffer. + */ + ret = ttm_bo_mem_compat(placement, &bo->mem); + if (ret < 0) { + ret = ttm_bo_move_buffer(bo, placement, interruptible, + no_wait_gpu); + if (ret) + return ret; + } else { + /* + * Use the access and other non-mapping-related flag bits from + * the compatible memory placement flags to the active flags + */ + ttm_flag_masked(&bo->mem.placement, placement->placement[ret], + ~TTM_PL_MASK_MEMTYPE); + } + /* + * We might need to add a TTM. + */ + if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { + ret = ttm_bo_add_ttm(bo, true); + if (ret) + return ret; + } + return 0; +} + +int ttm_bo_check_placement(struct ttm_buffer_object *bo, + struct ttm_placement *placement) +{ + MPASS(!((placement->fpfn || placement->lpfn) && + (bo->mem.num_pages > (placement->lpfn - placement->fpfn)))); + + return 0; +} + +int ttm_bo_init(struct ttm_bo_device *bdev, + struct ttm_buffer_object *bo, + unsigned long size, + enum ttm_bo_type type, + struct ttm_placement *placement, + uint32_t page_alignment, + bool interruptible, + struct vm_object *persistent_swap_storage, + size_t acc_size, + struct sg_table *sg, + void (*destroy) (struct ttm_buffer_object *)) +{ + int ret = 0; + unsigned long num_pages; + struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; + + ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); + if (ret) { + printf("[TTM] Out of kernel memory\n"); + if (destroy) + (*destroy)(bo); + else + free(bo, M_TTM_BO); + return -ENOMEM; + } + + num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; + if (num_pages == 0) { + printf("[TTM] Illegal buffer object size\n"); + if (destroy) + (*destroy)(bo); + else + free(bo, M_TTM_BO); + ttm_mem_global_free(mem_glob, acc_size); + return -EINVAL; + } + bo->destroy = destroy; + + refcount_init(&bo->kref, 1); + refcount_init(&bo->list_kref, 1); + atomic_set(&bo->cpu_writers, 0); + atomic_set(&bo->reserved, 1); + INIT_LIST_HEAD(&bo->lru); + INIT_LIST_HEAD(&bo->ddestroy); + INIT_LIST_HEAD(&bo->swap); + INIT_LIST_HEAD(&bo->io_reserve_lru); + bo->bdev = bdev; + bo->glob = bdev->glob; + bo->type = type; + bo->num_pages = num_pages; + bo->mem.size = num_pages << PAGE_SHIFT; + bo->mem.mem_type = TTM_PL_SYSTEM; + bo->mem.num_pages = bo->num_pages; + bo->mem.mm_node = NULL; + bo->mem.page_alignment = page_alignment; + bo->mem.bus.io_reserved_vm = false; + bo->mem.bus.io_reserved_count = 0; + bo->priv_flags = 0; + bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); + bo->seq_valid = false; + bo->persistent_swap_storage = persistent_swap_storage; + bo->acc_size = acc_size; + bo->sg = sg; + atomic_inc(&bo->glob->bo_count); + + ret = ttm_bo_check_placement(bo, placement); + if (unlikely(ret != 0)) + goto out_err; + + /* + * For ttm_bo_type_device buffers, allocate + * address space from the device. + */ + if (bo->type == ttm_bo_type_device || + bo->type == ttm_bo_type_sg) { + ret = ttm_bo_setup_vm(bo); + if (ret) + goto out_err; + } + + ret = ttm_bo_validate(bo, placement, interruptible, false); + if (ret) + goto out_err; + + ttm_bo_unreserve(bo); + return 0; + +out_err: + ttm_bo_unreserve(bo); + ttm_bo_unref(&bo); + + return ret; +} + +size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, + unsigned long bo_size, + unsigned struct_size) +{ + unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; + size_t size = 0; + + size += ttm_round_pot(struct_size); + size += PAGE_ALIGN(npages * sizeof(void *)); + size += ttm_round_pot(sizeof(struct ttm_tt)); + return size; +} + +size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, + unsigned long bo_size, + unsigned struct_size) +{ + unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; + size_t size = 0; + + size += ttm_round_pot(struct_size); + size += PAGE_ALIGN(npages * sizeof(void *)); + size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); + size += ttm_round_pot(sizeof(struct ttm_dma_tt)); + return size; +} + +int ttm_bo_create(struct ttm_bo_device *bdev, + unsigned long size, + enum ttm_bo_type type, + struct ttm_placement *placement, + uint32_t page_alignment, + bool interruptible, + struct vm_object *persistent_swap_storage, + struct ttm_buffer_object **p_bo) +{ + struct ttm_buffer_object *bo; + size_t acc_size; + int ret; + + bo = malloc(sizeof(*bo), M_TTM_BO, M_WAITOK | M_ZERO); + acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); + ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, + interruptible, persistent_swap_storage, acc_size, + NULL, NULL); + if (likely(ret == 0)) + *p_bo = bo; + + return ret; +} + +static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, + unsigned mem_type, bool allow_errors) +{ + struct ttm_mem_type_manager *man = &bdev->man[mem_type]; + struct ttm_bo_global *glob = bdev->glob; + int ret; + + /* + * Can't use standard list traversal since we're unlocking. + */ + + mtx_lock(&glob->lru_lock); + while (!list_empty(&man->lru)) { + mtx_unlock(&glob->lru_lock); + ret = ttm_mem_evict_first(bdev, mem_type, false, false); + if (ret) { + if (allow_errors) { + return ret; + } else { + printf("[TTM] Cleanup eviction failed\n"); + } + } + mtx_lock(&glob->lru_lock); + } + mtx_unlock(&glob->lru_lock); + return 0; +} + +int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) +{ + struct ttm_mem_type_manager *man; + int ret = -EINVAL; + + if (mem_type >= TTM_NUM_MEM_TYPES) { + printf("[TTM] Illegal memory type %d\n", mem_type); + return ret; + } + man = &bdev->man[mem_type]; + + if (!man->has_type) { + printf("[TTM] Trying to take down uninitialized memory manager type %u\n", + mem_type); + return ret; + } + + man->use_type = false; + man->has_type = false; + + ret = 0; + if (mem_type > 0) { + ttm_bo_force_list_clean(bdev, mem_type, false); + + ret = (*man->func->takedown)(man); + } + + return ret; +} + +int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) +{ + struct ttm_mem_type_manager *man = &bdev->man[mem_type]; + + if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { + printf("[TTM] Illegal memory manager memory type %u\n", mem_type); + return -EINVAL; + } + + if (!man->has_type) { + printf("[TTM] Memory type %u has not been initialized\n", mem_type); + return 0; + } + + return ttm_bo_force_list_clean(bdev, mem_type, true); +} + +int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, + unsigned long p_size) +{ + int ret = -EINVAL; + struct ttm_mem_type_manager *man; + + MPASS(type < TTM_NUM_MEM_TYPES); + man = &bdev->man[type]; + MPASS(!man->has_type); + man->io_reserve_fastpath = true; + man->use_io_reserve_lru = false; + sx_init(&man->io_reserve_mutex, "ttmman"); + INIT_LIST_HEAD(&man->io_reserve_lru); + + ret = bdev->driver->init_mem_type(bdev, type, man); + if (ret) + return ret; + man->bdev = bdev; + + ret = 0; + if (type != TTM_PL_SYSTEM) { + ret = (*man->func->init)(man, p_size); + if (ret) + return ret; + } + man->has_type = true; + man->use_type = true; + man->size = p_size; + + INIT_LIST_HEAD(&man->lru); + + return 0; +} + +static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob) +{ + + ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); + vm_page_free(glob->dummy_read_page); + free(glob, M_DRM_GLOBAL); +} + +void ttm_bo_global_release(struct drm_global_reference *ref) +{ + struct ttm_bo_global *glob = ref->object; + + if (refcount_release(&glob->kobj_ref)) + ttm_bo_global_kobj_release(glob); +} + +int ttm_bo_global_init(struct drm_global_reference *ref) +{ + struct ttm_bo_global_ref *bo_ref = + container_of(ref, struct ttm_bo_global_ref, ref); + struct ttm_bo_global *glob = ref->object; + int ret; + + sx_init(&glob->device_list_mutex, "ttmdlm"); + mtx_init(&glob->lru_lock, "ttmlru", NULL, MTX_DEF); + glob->mem_glob = bo_ref->mem_glob; + glob->dummy_read_page = vm_page_alloc_contig(NULL, 0, + VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ, + 1, 0, VM_MAX_ADDRESS, PAGE_SIZE, 0, VM_MEMATTR_UNCACHEABLE); + + if (unlikely(glob->dummy_read_page == NULL)) { + ret = -ENOMEM; + goto out_no_drp; + } + + INIT_LIST_HEAD(&glob->swap_lru); + INIT_LIST_HEAD(&glob->device_list); + + ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); + ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); + if (unlikely(ret != 0)) { + printf("[TTM] Could not register buffer object swapout\n"); + goto out_no_shrink; + } + + atomic_set(&glob->bo_count, 0); + + refcount_init(&glob->kobj_ref, 1); + return (0); + +out_no_shrink: + vm_page_free(glob->dummy_read_page); +out_no_drp: + free(glob, M_DRM_GLOBAL); + return ret; +} + +int ttm_bo_device_release(struct ttm_bo_device *bdev) +{ + int ret = 0; + unsigned i = TTM_NUM_MEM_TYPES; + struct ttm_mem_type_manager *man; + struct ttm_bo_global *glob = bdev->glob; + + while (i--) { + man = &bdev->man[i]; + if (man->has_type) { + man->use_type = false; + if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { + ret = -EBUSY; + printf("[TTM] DRM memory manager type %d is not clean\n", + i); + } + man->has_type = false; + } + } + + sx_xlock(&glob->device_list_mutex); + list_del(&bdev->device_list); + sx_xunlock(&glob->device_list_mutex); + + if (taskqueue_cancel_timeout(taskqueue_thread, &bdev->wq, NULL)) + taskqueue_drain_timeout(taskqueue_thread, &bdev->wq); + + while (ttm_bo_delayed_delete(bdev, true)) + ; + + mtx_lock(&glob->lru_lock); + if (list_empty(&bdev->ddestroy)) + TTM_DEBUG("Delayed destroy list was clean\n"); + + if (list_empty(&bdev->man[0].lru)) + TTM_DEBUG("Swap list was clean\n"); + mtx_unlock(&glob->lru_lock); + + MPASS(drm_mm_clean(&bdev->addr_space_mm)); + rw_wlock(&bdev->vm_lock); + drm_mm_takedown(&bdev->addr_space_mm); + rw_wunlock(&bdev->vm_lock); + + return ret; +} + +int ttm_bo_device_init(struct ttm_bo_device *bdev, + struct ttm_bo_global *glob, + struct ttm_bo_driver *driver, + uint64_t file_page_offset, + bool need_dma32) +{ + int ret = -EINVAL; + + rw_init(&bdev->vm_lock, "ttmvml"); + bdev->driver = driver; + + memset(bdev->man, 0, sizeof(bdev->man)); + + /* + * Initialize the system memory buffer type. + * Other types need to be driver / IOCTL initialized. + */ + ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); + if (unlikely(ret != 0)) + goto out_no_sys; + + RB_INIT(&bdev->addr_space_rb); + ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); + if (unlikely(ret != 0)) + goto out_no_addr_mm; + + TIMEOUT_TASK_INIT(taskqueue_thread, &bdev->wq, 0, + ttm_bo_delayed_workqueue, bdev); + INIT_LIST_HEAD(&bdev->ddestroy); + bdev->dev_mapping = NULL; + bdev->glob = glob; + bdev->need_dma32 = need_dma32; + bdev->val_seq = 0; + mtx_init(&bdev->fence_lock, "ttmfence", NULL, MTX_DEF); + sx_xlock(&glob->device_list_mutex); + list_add_tail(&bdev->device_list, &glob->device_list); + sx_xunlock(&glob->device_list_mutex); + + return 0; +out_no_addr_mm: + ttm_bo_clean_mm(bdev, 0); +out_no_sys: + return ret; +} + +/* + * buffer object vm functions. + */ + +bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) +{ + struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; + + if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { + if (mem->mem_type == TTM_PL_SYSTEM) + return false; + + if (man->flags & TTM_MEMTYPE_FLAG_CMA) + return false; + + if (mem->placement & TTM_PL_FLAG_CACHED) + return false; + } + return true; +} + +void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + /* off_t offset = (off_t)bo->addr_space_offset;XXXKIB */ + /* off_t holelen = ((off_t)bo->mem.num_pages) << PAGE_SHIFT;XXXKIB */ + + if (!bdev->dev_mapping) + return; + /* unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); XXXKIB */ + ttm_mem_io_free_vm(bo); +} + +void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; + + ttm_mem_io_lock(man, false); + ttm_bo_unmap_virtual_locked(bo); + ttm_mem_io_unlock(man); +} + +static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + + /* The caller acquired bdev->vm_lock. */ + RB_INSERT(ttm_bo_device_buffer_objects, &bdev->addr_space_rb, bo); +} + +/** + * ttm_bo_setup_vm: + * + * @bo: the buffer to allocate address space for + * + * Allocate address space in the drm device so that applications + * can mmap the buffer and access the contents. This only + * applies to ttm_bo_type_device objects as others are not + * placed in the drm device address space. + */ + +static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + int ret; + +retry_pre_get: + ret = drm_mm_pre_get(&bdev->addr_space_mm); + if (unlikely(ret != 0)) + return ret; + + rw_wlock(&bdev->vm_lock); + bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm, + bo->mem.num_pages, 0, 0); + + if (unlikely(bo->vm_node == NULL)) { + ret = -ENOMEM; + goto out_unlock; + } + + bo->vm_node = drm_mm_get_block_atomic(bo->vm_node, + bo->mem.num_pages, 0); + + if (unlikely(bo->vm_node == NULL)) { + rw_wunlock(&bdev->vm_lock); + goto retry_pre_get; + } + + ttm_bo_vm_insert_rb(bo); + rw_wunlock(&bdev->vm_lock); + bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT; + + return 0; +out_unlock: + rw_wunlock(&bdev->vm_lock); + return ret; +} + +int ttm_bo_wait(struct ttm_buffer_object *bo, + bool lazy, bool interruptible, bool no_wait) +{ + struct ttm_bo_driver *driver = bo->bdev->driver; + struct ttm_bo_device *bdev = bo->bdev; + void *sync_obj; + int ret = 0; + + if (likely(bo->sync_obj == NULL)) + return 0; + + while (bo->sync_obj) { + + if (driver->sync_obj_signaled(bo->sync_obj)) { + void *tmp_obj = bo->sync_obj; + bo->sync_obj = NULL; + clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); + mtx_unlock(&bdev->fence_lock); + driver->sync_obj_unref(&tmp_obj); + mtx_lock(&bdev->fence_lock); + continue; + } + + if (no_wait) + return -EBUSY; + + sync_obj = driver->sync_obj_ref(bo->sync_obj); + mtx_unlock(&bdev->fence_lock); + ret = driver->sync_obj_wait(sync_obj, + lazy, interruptible); + if (unlikely(ret != 0)) { + driver->sync_obj_unref(&sync_obj); + mtx_lock(&bdev->fence_lock); + return ret; + } + mtx_lock(&bdev->fence_lock); + if (likely(bo->sync_obj == sync_obj)) { + void *tmp_obj = bo->sync_obj; + bo->sync_obj = NULL; + clear_bit(TTM_BO_PRIV_FLAG_MOVING, + &bo->priv_flags); + mtx_unlock(&bdev->fence_lock); + driver->sync_obj_unref(&sync_obj); + driver->sync_obj_unref(&tmp_obj); + mtx_lock(&bdev->fence_lock); + } else { + mtx_unlock(&bdev->fence_lock); + driver->sync_obj_unref(&sync_obj); + mtx_lock(&bdev->fence_lock); + } + } + return 0; +} + +int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) +{ + struct ttm_bo_device *bdev = bo->bdev; + int ret = 0; + + /* + * Using ttm_bo_reserve makes sure the lru lists are updated. + */ + + ret = ttm_bo_reserve(bo, true, no_wait, false, 0); + if (unlikely(ret != 0)) + return ret; + mtx_lock(&bdev->fence_lock); + ret = ttm_bo_wait(bo, false, true, no_wait); + mtx_unlock(&bdev->fence_lock); + if (likely(ret == 0)) + atomic_inc(&bo->cpu_writers); + ttm_bo_unreserve(bo); + return ret; +} + +void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) +{ + atomic_dec(&bo->cpu_writers); +} + +/** + * A buffer object shrink method that tries to swap out the first + * buffer object on the bo_global::swap_lru list. + */ + +static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) +{ + struct ttm_bo_global *glob = + container_of(shrink, struct ttm_bo_global, shrink); + struct ttm_buffer_object *bo; + int ret = -EBUSY; + int put_count; + uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); + + mtx_lock(&glob->lru_lock); + list_for_each_entry(bo, &glob->swap_lru, swap) { + ret = ttm_bo_reserve_locked(bo, false, true, false, 0); + if (!ret) + break; + } + + if (ret) { + mtx_unlock(&glob->lru_lock); + return ret; + } + + refcount_acquire(&bo->list_kref); + + if (!list_empty(&bo->ddestroy)) { + ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); + if (refcount_release(&bo->list_kref)) + ttm_bo_release_list(bo); + return ret; + } + + put_count = ttm_bo_del_from_lru(bo); + mtx_unlock(&glob->lru_lock); + + ttm_bo_list_ref_sub(bo, put_count, true); + + /** + * Wait for GPU, then move to system cached. + */ + + mtx_lock(&bo->bdev->fence_lock); + ret = ttm_bo_wait(bo, false, false, false); + mtx_unlock(&bo->bdev->fence_lock); + + if (unlikely(ret != 0)) + goto out; + + if ((bo->mem.placement & swap_placement) != swap_placement) { + struct ttm_mem_reg evict_mem; + + evict_mem = bo->mem; + evict_mem.mm_node = NULL; + evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; + evict_mem.mem_type = TTM_PL_SYSTEM; + + ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, + false, false); + if (unlikely(ret != 0)) + goto out; + } + + ttm_bo_unmap_virtual(bo); + + /** + * Swap out. Buffer will be swapped in again as soon as + * anyone tries to access a ttm page. + */ + + if (bo->bdev->driver->swap_notify) + bo->bdev->driver->swap_notify(bo); + + ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); +out: + + /** + * + * Unreserve without putting on LRU to avoid swapping out an + * already swapped buffer. + */ + + atomic_set(&bo->reserved, 0); + wakeup(bo); + if (refcount_release(&bo->list_kref)) + ttm_bo_release_list(bo); + return ret; +} + +void ttm_bo_swapout_all(struct ttm_bo_device *bdev) +{ + while (ttm_bo_swapout(&bdev->glob->shrink) == 0) + ; +} diff --git a/sys/dev/drm2/ttm/ttm_bo_api.h b/sys/dev/drm2/ttm/ttm_bo_api.h new file mode 100644 index 0000000..4b16ebd --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_bo_api.h @@ -0,0 +1,740 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ +/* $FreeBSD$ */ + +#ifndef _TTM_BO_API_H_ +#define _TTM_BO_API_H_ + +#include <dev/drm2/drmP.h> + +struct ttm_bo_device; + +struct drm_mm_node; + + +/** + * struct ttm_placement + * + * @fpfn: first valid page frame number to put the object + * @lpfn: last valid page frame number to put the object + * @num_placement: number of preferred placements + * @placement: preferred placements + * @num_busy_placement: number of preferred placements when need to evict buffer + * @busy_placement: preferred placements when need to evict buffer + * + * Structure indicating the placement you request for an object. + */ +struct ttm_placement { + unsigned fpfn; + unsigned lpfn; + unsigned num_placement; + const uint32_t *placement; + unsigned num_busy_placement; + const uint32_t *busy_placement; +}; + +/** + * struct ttm_bus_placement + * + * @addr: mapped virtual address + * @base: bus base address + * @is_iomem: is this io memory ? + * @size: size in byte + * @offset: offset from the base address + * @io_reserved_vm: The VM system has a refcount in @io_reserved_count + * @io_reserved_count: Refcounting the numbers of callers to ttm_mem_io_reserve + * + * Structure indicating the bus placement of an object. + */ +struct ttm_bus_placement { + void *addr; + unsigned long base; + unsigned long size; + unsigned long offset; + bool is_iomem; + bool io_reserved_vm; + uint64_t io_reserved_count; +}; + + +/** + * struct ttm_mem_reg + * + * @mm_node: Memory manager node. + * @size: Requested size of memory region. + * @num_pages: Actual size of memory region in pages. + * @page_alignment: Page alignment. + * @placement: Placement flags. + * @bus: Placement on io bus accessible to the CPU + * + * Structure indicating the placement and space resources used by a + * buffer object. + */ + +struct ttm_mem_reg { + void *mm_node; + unsigned long start; + unsigned long size; + unsigned long num_pages; + uint32_t page_alignment; + uint32_t mem_type; + uint32_t placement; + struct ttm_bus_placement bus; +}; + +/** + * enum ttm_bo_type + * + * @ttm_bo_type_device: These are 'normal' buffers that can + * be mmapped by user space. Each of these bos occupy a slot in the + * device address space, that can be used for normal vm operations. + * + * @ttm_bo_type_kernel: These buffers are like ttm_bo_type_device buffers, + * but they cannot be accessed from user-space. For kernel-only use. + * + * @ttm_bo_type_sg: Buffer made from dmabuf sg table shared with another + * driver. + */ + +enum ttm_bo_type { + ttm_bo_type_device, + ttm_bo_type_kernel, + ttm_bo_type_sg +}; + +struct ttm_tt; + +/** + * struct ttm_buffer_object + * + * @bdev: Pointer to the buffer object device structure. + * @type: The bo type. + * @destroy: Destruction function. If NULL, kfree is used. + * @num_pages: Actual number of pages. + * @addr_space_offset: Address space offset. + * @acc_size: Accounted size for this object. + * @kref: Reference count of this buffer object. When this refcount reaches + * zero, the object is put on the delayed delete list. + * @list_kref: List reference count of this buffer object. This member is + * used to avoid destruction while the buffer object is still on a list. + * Lru lists may keep one refcount, the delayed delete list, and kref != 0 + * keeps one refcount. When this refcount reaches zero, + * the object is destroyed. + * @event_queue: Queue for processes waiting on buffer object status change. + * @mem: structure describing current placement. + * @persistent_swap_storage: Usually the swap storage is deleted for buffers + * pinned in physical memory. If this behaviour is not desired, this member + * holds a pointer to a persistent shmem object. + * @ttm: TTM structure holding system pages. + * @evicted: Whether the object was evicted without user-space knowing. + * @cpu_writes: For synchronization. Number of cpu writers. + * @lru: List head for the lru list. + * @ddestroy: List head for the delayed destroy list. + * @swap: List head for swap LRU list. + * @val_seq: Sequence of the validation holding the @reserved lock. + * Used to avoid starvation when many processes compete to validate the + * buffer. This member is protected by the bo_device::lru_lock. + * @seq_valid: The value of @val_seq is valid. This value is protected by + * the bo_device::lru_lock. + * @reserved: Deadlock-free lock used for synchronization state transitions. + * @sync_obj: Pointer to a synchronization object. + * @priv_flags: Flags describing buffer object internal state. + * @vm_rb: Rb node for the vm rb tree. + * @vm_node: Address space manager node. + * @offset: The current GPU offset, which can have different meanings + * depending on the memory type. For SYSTEM type memory, it should be 0. + * @cur_placement: Hint of current placement. + * + * Base class for TTM buffer object, that deals with data placement and CPU + * mappings. GPU mappings are really up to the driver, but for simpler GPUs + * the driver can usually use the placement offset @offset directly as the + * GPU virtual address. For drivers implementing multiple + * GPU memory manager contexts, the driver should manage the address space + * in these contexts separately and use these objects to get the correct + * placement and caching for these GPU maps. This makes it possible to use + * these objects for even quite elaborate memory management schemes. + * The destroy member, the API visibility of this object makes it possible + * to derive driver specific types. + */ + +struct ttm_buffer_object { + /** + * Members constant at init. + */ + + struct ttm_bo_global *glob; + struct ttm_bo_device *bdev; + enum ttm_bo_type type; + void (*destroy) (struct ttm_buffer_object *); + unsigned long num_pages; + uint64_t addr_space_offset; + size_t acc_size; + + /** + * Members not needing protection. + */ + + u_int kref; + u_int list_kref; + /* wait_queue_head_t event_queue; */ + + /** + * Members protected by the bo::reserved lock. + */ + + struct ttm_mem_reg mem; + struct vm_object *persistent_swap_storage; + struct ttm_tt *ttm; + bool evicted; + + /** + * Members protected by the bo::reserved lock only when written to. + */ + + atomic_t cpu_writers; + + /** + * Members protected by the bdev::lru_lock. + */ + + struct list_head lru; + struct list_head ddestroy; + struct list_head swap; + struct list_head io_reserve_lru; + uint32_t val_seq; + bool seq_valid; + + /** + * Members protected by the bdev::lru_lock + * only when written to. + */ + + atomic_t reserved; + + /** + * Members protected by struct buffer_object_device::fence_lock + * In addition, setting sync_obj to anything else + * than NULL requires bo::reserved to be held. This allows for + * checking NULL while reserved but not holding the mentioned lock. + */ + + void *sync_obj; + unsigned long priv_flags; + + /** + * Members protected by the bdev::vm_lock + */ + + RB_ENTRY(ttm_buffer_object) vm_rb; + struct drm_mm_node *vm_node; + + + /** + * Special members that are protected by the reserve lock + * and the bo::lock when written to. Can be read with + * either of these locks held. + */ + + unsigned long offset; + uint32_t cur_placement; + + struct sg_table *sg; +}; + +/** + * struct ttm_bo_kmap_obj + * + * @virtual: The current kernel virtual address. + * @page: The page when kmap'ing a single page. + * @bo_kmap_type: Type of bo_kmap. + * + * Object describing a kernel mapping. Since a TTM bo may be located + * in various memory types with various caching policies, the + * mapping can either be an ioremap, a vmap, a kmap or part of a + * premapped region. + */ + +#define TTM_BO_MAP_IOMEM_MASK 0x80 +struct ttm_bo_kmap_obj { + void *virtual; + struct vm_page *page; + struct sf_buf *sf; + int num_pages; + unsigned long size; + enum { + ttm_bo_map_iomap = 1 | TTM_BO_MAP_IOMEM_MASK, + ttm_bo_map_vmap = 2, + ttm_bo_map_kmap = 3, + ttm_bo_map_premapped = 4 | TTM_BO_MAP_IOMEM_MASK, + } bo_kmap_type; + struct ttm_buffer_object *bo; +}; + +/** + * ttm_bo_reference - reference a struct ttm_buffer_object + * + * @bo: The buffer object. + * + * Returns a refcounted pointer to a buffer object. + */ + +static inline struct ttm_buffer_object * +ttm_bo_reference(struct ttm_buffer_object *bo) +{ + refcount_acquire(&bo->kref); + return bo; +} + +/** + * ttm_bo_wait - wait for buffer idle. + * + * @bo: The buffer object. + * @interruptible: Use interruptible wait. + * @no_wait: Return immediately if buffer is busy. + * + * This function must be called with the bo::mutex held, and makes + * sure any previous rendering to the buffer is completed. + * Note: It might be necessary to block validations before the + * wait by reserving the buffer. + * Returns -EBUSY if no_wait is true and the buffer is busy. + * Returns -ERESTARTSYS if interrupted by a signal. + */ +extern int ttm_bo_wait(struct ttm_buffer_object *bo, bool lazy, + bool interruptible, bool no_wait); +/** + * ttm_bo_validate + * + * @bo: The buffer object. + * @placement: Proposed placement for the buffer object. + * @interruptible: Sleep interruptible if sleeping. + * @no_wait_gpu: Return immediately if the GPU is busy. + * + * Changes placement and caching policy of the buffer object + * according proposed placement. + * Returns + * -EINVAL on invalid proposed placement. + * -ENOMEM on out-of-memory condition. + * -EBUSY if no_wait is true and buffer busy. + * -ERESTARTSYS if interrupted by a signal. + */ +extern int ttm_bo_validate(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + bool interruptible, + bool no_wait_gpu); + +/** + * ttm_bo_unref + * + * @bo: The buffer object. + * + * Unreference and clear a pointer to a buffer object. + */ +extern void ttm_bo_unref(struct ttm_buffer_object **bo); + + +/** + * ttm_bo_list_ref_sub + * + * @bo: The buffer object. + * @count: The number of references with which to decrease @bo::list_kref; + * @never_free: The refcount should not reach zero with this operation. + * + * Release @count lru list references to this buffer object. + */ +extern void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, + bool never_free); + +/** + * ttm_bo_add_to_lru + * + * @bo: The buffer object. + * + * Add this bo to the relevant mem type lru and, if it's backed by + * system pages (ttms) to the swap list. + * This function must be called with struct ttm_bo_global::lru_lock held, and + * is typically called immediately prior to unreserving a bo. + */ +extern void ttm_bo_add_to_lru(struct ttm_buffer_object *bo); + +/** + * ttm_bo_del_from_lru + * + * @bo: The buffer object. + * + * Remove this bo from all lru lists used to lookup and reserve an object. + * This function must be called with struct ttm_bo_global::lru_lock held, + * and is usually called just immediately after the bo has been reserved to + * avoid recursive reservation from lru lists. + */ +extern int ttm_bo_del_from_lru(struct ttm_buffer_object *bo); + + +/** + * ttm_bo_lock_delayed_workqueue + * + * Prevent the delayed workqueue from running. + * Returns + * True if the workqueue was queued at the time + */ +extern int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev); + +/** + * ttm_bo_unlock_delayed_workqueue + * + * Allows the delayed workqueue to run. + */ +extern void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, + int resched); + +/** + * ttm_bo_synccpu_write_grab + * + * @bo: The buffer object: + * @no_wait: Return immediately if buffer is busy. + * + * Synchronizes a buffer object for CPU RW access. This means + * command submission that affects the buffer will return -EBUSY + * until ttm_bo_synccpu_write_release is called. + * + * Returns + * -EBUSY if the buffer is busy and no_wait is true. + * -ERESTARTSYS if interrupted by a signal. + */ +extern int +ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait); + +/** + * ttm_bo_synccpu_write_release: + * + * @bo : The buffer object. + * + * Releases a synccpu lock. + */ +extern void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo); + +/** + * ttm_bo_acc_size + * + * @bdev: Pointer to a ttm_bo_device struct. + * @bo_size: size of the buffer object in byte. + * @struct_size: size of the structure holding buffer object datas + * + * Returns size to account for a buffer object + */ +size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, + unsigned long bo_size, + unsigned struct_size); +size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, + unsigned long bo_size, + unsigned struct_size); + +/** + * ttm_bo_init + * + * @bdev: Pointer to a ttm_bo_device struct. + * @bo: Pointer to a ttm_buffer_object to be initialized. + * @size: Requested size of buffer object. + * @type: Requested type of buffer object. + * @flags: Initial placement flags. + * @page_alignment: Data alignment in pages. + * @interruptible: If needing to sleep to wait for GPU resources, + * sleep interruptible. + * @persistent_swap_storage: Usually the swap storage is deleted for buffers + * pinned in physical memory. If this behaviour is not desired, this member + * holds a pointer to a persistent shmem object. Typically, this would + * point to the shmem object backing a GEM object if TTM is used to back a + * GEM user interface. + * @acc_size: Accounted size for this object. + * @destroy: Destroy function. Use NULL for kfree(). + * + * This function initializes a pre-allocated struct ttm_buffer_object. + * As this object may be part of a larger structure, this function, + * together with the @destroy function, + * enables driver-specific objects derived from a ttm_buffer_object. + * On successful return, the object kref and list_kref are set to 1. + * If a failure occurs, the function will call the @destroy function, or + * kfree() if @destroy is NULL. Thus, after a failure, dereferencing @bo is + * illegal and will likely cause memory corruption. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid placement flags. + * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. + */ + +extern int ttm_bo_init(struct ttm_bo_device *bdev, + struct ttm_buffer_object *bo, + unsigned long size, + enum ttm_bo_type type, + struct ttm_placement *placement, + uint32_t page_alignment, + bool interrubtible, + struct vm_object *persistent_swap_storage, + size_t acc_size, + struct sg_table *sg, + void (*destroy) (struct ttm_buffer_object *)); + +/** + * ttm_bo_synccpu_object_init + * + * @bdev: Pointer to a ttm_bo_device struct. + * @bo: Pointer to a ttm_buffer_object to be initialized. + * @size: Requested size of buffer object. + * @type: Requested type of buffer object. + * @flags: Initial placement flags. + * @page_alignment: Data alignment in pages. + * @interruptible: If needing to sleep while waiting for GPU resources, + * sleep interruptible. + * @persistent_swap_storage: Usually the swap storage is deleted for buffers + * pinned in physical memory. If this behaviour is not desired, this member + * holds a pointer to a persistent shmem object. Typically, this would + * point to the shmem object backing a GEM object if TTM is used to back a + * GEM user interface. + * @p_bo: On successful completion *p_bo points to the created object. + * + * This function allocates a ttm_buffer_object, and then calls ttm_bo_init + * on that object. The destroy function is set to kfree(). + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid placement flags. + * -ERESTARTSYS: Interrupted by signal while waiting for resources. + */ + +extern int ttm_bo_create(struct ttm_bo_device *bdev, + unsigned long size, + enum ttm_bo_type type, + struct ttm_placement *placement, + uint32_t page_alignment, + bool interruptible, + struct vm_object *persistent_swap_storage, + struct ttm_buffer_object **p_bo); + +/** + * ttm_bo_check_placement + * + * @bo: the buffer object. + * @placement: placements + * + * Performs minimal validity checking on an intended change of + * placement flags. + * Returns + * -EINVAL: Intended change is invalid or not allowed. + */ +extern int ttm_bo_check_placement(struct ttm_buffer_object *bo, + struct ttm_placement *placement); + +/** + * ttm_bo_init_mm + * + * @bdev: Pointer to a ttm_bo_device struct. + * @mem_type: The memory type. + * @p_size: size managed area in pages. + * + * Initialize a manager for a given memory type. + * Note: if part of driver firstopen, it must be protected from a + * potentially racing lastclose. + * Returns: + * -EINVAL: invalid size or memory type. + * -ENOMEM: Not enough memory. + * May also return driver-specified errors. + */ + +extern int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, + unsigned long p_size); +/** + * ttm_bo_clean_mm + * + * @bdev: Pointer to a ttm_bo_device struct. + * @mem_type: The memory type. + * + * Take down a manager for a given memory type after first walking + * the LRU list to evict any buffers left alive. + * + * Normally, this function is part of lastclose() or unload(), and at that + * point there shouldn't be any buffers left created by user-space, since + * there should've been removed by the file descriptor release() method. + * However, before this function is run, make sure to signal all sync objects, + * and verify that the delayed delete queue is empty. The driver must also + * make sure that there are no NO_EVICT buffers present in this memory type + * when the call is made. + * + * If this function is part of a VT switch, the caller must make sure that + * there are no appications currently validating buffers before this + * function is called. The caller can do that by first taking the + * struct ttm_bo_device::ttm_lock in write mode. + * + * Returns: + * -EINVAL: invalid or uninitialized memory type. + * -EBUSY: There are still buffers left in this memory type. + */ + +extern int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type); + +/** + * ttm_bo_evict_mm + * + * @bdev: Pointer to a ttm_bo_device struct. + * @mem_type: The memory type. + * + * Evicts all buffers on the lru list of the memory type. + * This is normally part of a VT switch or an + * out-of-memory-space-due-to-fragmentation handler. + * The caller must make sure that there are no other processes + * currently validating buffers, and can do that by taking the + * struct ttm_bo_device::ttm_lock in write mode. + * + * Returns: + * -EINVAL: Invalid or uninitialized memory type. + * -ERESTARTSYS: The call was interrupted by a signal while waiting to + * evict a buffer. + */ + +extern int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type); + +/** + * ttm_kmap_obj_virtual + * + * @map: A struct ttm_bo_kmap_obj returned from ttm_bo_kmap. + * @is_iomem: Pointer to an integer that on return indicates 1 if the + * virtual map is io memory, 0 if normal memory. + * + * Returns the virtual address of a buffer object area mapped by ttm_bo_kmap. + * If *is_iomem is 1 on return, the virtual address points to an io memory area, + * that should strictly be accessed by the iowriteXX() and similar functions. + */ + +static inline void *ttm_kmap_obj_virtual(struct ttm_bo_kmap_obj *map, + bool *is_iomem) +{ + *is_iomem = !!(map->bo_kmap_type & TTM_BO_MAP_IOMEM_MASK); + return map->virtual; +} + +/** + * ttm_bo_kmap + * + * @bo: The buffer object. + * @start_page: The first page to map. + * @num_pages: Number of pages to map. + * @map: pointer to a struct ttm_bo_kmap_obj representing the map. + * + * Sets up a kernel virtual mapping, using ioremap, vmap or kmap to the + * data in the buffer object. The ttm_kmap_obj_virtual function can then be + * used to obtain a virtual address to the data. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid range. + */ + +extern int ttm_bo_kmap(struct ttm_buffer_object *bo, unsigned long start_page, + unsigned long num_pages, struct ttm_bo_kmap_obj *map); + +/** + * ttm_bo_kunmap + * + * @map: Object describing the map to unmap. + * + * Unmaps a kernel map set up by ttm_bo_kmap. + */ + +extern void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map); + +/** + * ttm_fbdev_mmap - mmap fbdev memory backed by a ttm buffer object. + * + * @vma: vma as input from the fbdev mmap method. + * @bo: The bo backing the address space. The address space will + * have the same size as the bo, and start at offset 0. + * + * This function is intended to be called by the fbdev mmap method + * if the fbdev address space is to be backed by a bo. + */ + +/* XXXKIB +extern int ttm_fbdev_mmap(struct vm_area_struct *vma, + struct ttm_buffer_object *bo); +*/ +/** + * ttm_bo_mmap - mmap out of the ttm device address space. + * + * @filp: filp as input from the mmap method. + * @vma: vma as input from the mmap method. + * @bdev: Pointer to the ttm_bo_device with the address space manager. + * + * This function is intended to be called by the device mmap method. + * if the device address space is to be backed by the bo manager. + */ +/* XXXKIB +extern int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, + struct ttm_bo_device *bdev); +*/ +/** + * ttm_bo_io + * + * @bdev: Pointer to the struct ttm_bo_device. + * @filp: Pointer to the struct file attempting to read / write. + * @wbuf: User-space pointer to address of buffer to write. NULL on read. + * @rbuf: User-space pointer to address of buffer to read into. + * Null on write. + * @count: Number of bytes to read / write. + * @f_pos: Pointer to current file position. + * @write: 1 for read, 0 for write. + * + * This function implements read / write into ttm buffer objects, and is + * intended to + * be called from the fops::read and fops::write method. + * Returns: + * See man (2) write, man(2) read. In particular, + * the function may return -ERESTARTSYS if + * interrupted by a signal. + */ + +extern ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp, + const char *wbuf, char *rbuf, + size_t count, off_t *f_pos, bool write); + +extern void ttm_bo_swapout_all(struct ttm_bo_device *bdev); + +/** + * ttm_bo_is_reserved - return an indication if a ttm buffer object is reserved + * + * @bo: The buffer object to check. + * + * This function returns an indication if a bo is reserved or not, and should + * only be used to print an error when it is not from incorrect api usage, since + * there's no guarantee that it is the caller that is holding the reservation. + */ +static inline bool ttm_bo_is_reserved(struct ttm_buffer_object *bo) +{ + return atomic_read(&bo->reserved); +} + +#endif diff --git a/sys/dev/drm2/ttm/ttm_bo_driver.h b/sys/dev/drm2/ttm/ttm_bo_driver.h new file mode 100644 index 0000000..3f08976 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_bo_driver.h @@ -0,0 +1,1018 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 Vmware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ +/* $FreeBSD$ */ + +#ifndef _TTM_BO_DRIVER_H_ +#define _TTM_BO_DRIVER_H_ + +#include <dev/drm2/drmP.h> +#include <dev/drm2/ttm/ttm_bo_api.h> +#include <dev/drm2/ttm/ttm_memory.h> +#include <dev/drm2/ttm/ttm_module.h> +#include <dev/drm2/drm_global.h> +#include <sys/rwlock.h> +#include <sys/tree.h> + +struct ttm_backend_func { + /** + * struct ttm_backend_func member bind + * + * @ttm: Pointer to a struct ttm_tt. + * @bo_mem: Pointer to a struct ttm_mem_reg describing the + * memory type and location for binding. + * + * Bind the backend pages into the aperture in the location + * indicated by @bo_mem. This function should be able to handle + * differences between aperture and system page sizes. + */ + int (*bind) (struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem); + + /** + * struct ttm_backend_func member unbind + * + * @ttm: Pointer to a struct ttm_tt. + * + * Unbind previously bound backend pages. This function should be + * able to handle differences between aperture and system page sizes. + */ + int (*unbind) (struct ttm_tt *ttm); + + /** + * struct ttm_backend_func member destroy + * + * @ttm: Pointer to a struct ttm_tt. + * + * Destroy the backend. This will be call back from ttm_tt_destroy so + * don't call ttm_tt_destroy from the callback or infinite loop. + */ + void (*destroy) (struct ttm_tt *ttm); +}; + +#define TTM_PAGE_FLAG_WRITE (1 << 3) +#define TTM_PAGE_FLAG_SWAPPED (1 << 4) +#define TTM_PAGE_FLAG_PERSISTENT_SWAP (1 << 5) +#define TTM_PAGE_FLAG_ZERO_ALLOC (1 << 6) +#define TTM_PAGE_FLAG_DMA32 (1 << 7) +#define TTM_PAGE_FLAG_SG (1 << 8) + +enum ttm_caching_state { + tt_uncached, + tt_wc, + tt_cached +}; + +/** + * struct ttm_tt + * + * @bdev: Pointer to a struct ttm_bo_device. + * @func: Pointer to a struct ttm_backend_func that describes + * the backend methods. + * @dummy_read_page: Page to map where the ttm_tt page array contains a NULL + * pointer. + * @pages: Array of pages backing the data. + * @num_pages: Number of pages in the page array. + * @bdev: Pointer to the current struct ttm_bo_device. + * @be: Pointer to the ttm backend. + * @swap_storage: Pointer to shmem struct file for swap storage. + * @caching_state: The current caching state of the pages. + * @state: The current binding state of the pages. + * + * This is a structure holding the pages, caching- and aperture binding + * status for a buffer object that isn't backed by fixed (VRAM / AGP) + * memory. + */ + +struct ttm_tt { + struct ttm_bo_device *bdev; + struct ttm_backend_func *func; + struct vm_page *dummy_read_page; + struct vm_page **pages; + uint32_t page_flags; + unsigned long num_pages; + struct sg_table *sg; /* for SG objects via dma-buf */ + struct ttm_bo_global *glob; + struct vm_object *swap_storage; + enum ttm_caching_state caching_state; + enum { + tt_bound, + tt_unbound, + tt_unpopulated, + } state; +}; + +/** + * struct ttm_dma_tt + * + * @ttm: Base ttm_tt struct. + * @dma_address: The DMA (bus) addresses of the pages + * @pages_list: used by some page allocation backend + * + * This is a structure holding the pages, caching- and aperture binding + * status for a buffer object that isn't backed by fixed (VRAM / AGP) + * memory. + */ +struct ttm_dma_tt { + struct ttm_tt ttm; + dma_addr_t *dma_address; + struct list_head pages_list; +}; + +#define TTM_MEMTYPE_FLAG_FIXED (1 << 0) /* Fixed (on-card) PCI memory */ +#define TTM_MEMTYPE_FLAG_MAPPABLE (1 << 1) /* Memory mappable */ +#define TTM_MEMTYPE_FLAG_CMA (1 << 3) /* Can't map aperture */ + +struct ttm_mem_type_manager; + +struct ttm_mem_type_manager_func { + /** + * struct ttm_mem_type_manager member init + * + * @man: Pointer to a memory type manager. + * @p_size: Implementation dependent, but typically the size of the + * range to be managed in pages. + * + * Called to initialize a private range manager. The function is + * expected to initialize the man::priv member. + * Returns 0 on success, negative error code on failure. + */ + int (*init)(struct ttm_mem_type_manager *man, unsigned long p_size); + + /** + * struct ttm_mem_type_manager member takedown + * + * @man: Pointer to a memory type manager. + * + * Called to undo the setup done in init. All allocated resources + * should be freed. + */ + int (*takedown)(struct ttm_mem_type_manager *man); + + /** + * struct ttm_mem_type_manager member get_node + * + * @man: Pointer to a memory type manager. + * @bo: Pointer to the buffer object we're allocating space for. + * @placement: Placement details. + * @mem: Pointer to a struct ttm_mem_reg to be filled in. + * + * This function should allocate space in the memory type managed + * by @man. Placement details if + * applicable are given by @placement. If successful, + * @mem::mm_node should be set to a non-null value, and + * @mem::start should be set to a value identifying the beginning + * of the range allocated, and the function should return zero. + * If the memory region accommodate the buffer object, @mem::mm_node + * should be set to NULL, and the function should return 0. + * If a system error occurred, preventing the request to be fulfilled, + * the function should return a negative error code. + * + * Note that @mem::mm_node will only be dereferenced by + * struct ttm_mem_type_manager functions and optionally by the driver, + * which has knowledge of the underlying type. + * + * This function may not be called from within atomic context, so + * an implementation can and must use either a mutex or a spinlock to + * protect any data structures managing the space. + */ + int (*get_node)(struct ttm_mem_type_manager *man, + struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_mem_reg *mem); + + /** + * struct ttm_mem_type_manager member put_node + * + * @man: Pointer to a memory type manager. + * @mem: Pointer to a struct ttm_mem_reg to be filled in. + * + * This function frees memory type resources previously allocated + * and that are identified by @mem::mm_node and @mem::start. May not + * be called from within atomic context. + */ + void (*put_node)(struct ttm_mem_type_manager *man, + struct ttm_mem_reg *mem); + + /** + * struct ttm_mem_type_manager member debug + * + * @man: Pointer to a memory type manager. + * @prefix: Prefix to be used in printout to identify the caller. + * + * This function is called to print out the state of the memory + * type manager to aid debugging of out-of-memory conditions. + * It may not be called from within atomic context. + */ + void (*debug)(struct ttm_mem_type_manager *man, const char *prefix); +}; + +/** + * struct ttm_mem_type_manager + * + * @has_type: The memory type has been initialized. + * @use_type: The memory type is enabled. + * @flags: TTM_MEMTYPE_XX flags identifying the traits of the memory + * managed by this memory type. + * @gpu_offset: If used, the GPU offset of the first managed page of + * fixed memory or the first managed location in an aperture. + * @size: Size of the managed region. + * @available_caching: A mask of available caching types, TTM_PL_FLAG_XX, + * as defined in ttm_placement_common.h + * @default_caching: The default caching policy used for a buffer object + * placed in this memory type if the user doesn't provide one. + * @func: structure pointer implementing the range manager. See above + * @priv: Driver private closure for @func. + * @io_reserve_mutex: Mutex optionally protecting shared io_reserve structures + * @use_io_reserve_lru: Use an lru list to try to unreserve io_mem_regions + * reserved by the TTM vm system. + * @io_reserve_lru: Optional lru list for unreserving io mem regions. + * @io_reserve_fastpath: Only use bdev::driver::io_mem_reserve to obtain + * static information. bdev::driver::io_mem_free is never used. + * @lru: The lru list for this memory type. + * + * This structure is used to identify and manage memory types for a device. + * It's set up by the ttm_bo_driver::init_mem_type method. + */ + + + +struct ttm_mem_type_manager { + struct ttm_bo_device *bdev; + + /* + * No protection. Constant from start. + */ + + bool has_type; + bool use_type; + uint32_t flags; + unsigned long gpu_offset; + uint64_t size; + uint32_t available_caching; + uint32_t default_caching; + const struct ttm_mem_type_manager_func *func; + void *priv; + struct sx io_reserve_mutex; + bool use_io_reserve_lru; + bool io_reserve_fastpath; + + /* + * Protected by @io_reserve_mutex: + */ + + struct list_head io_reserve_lru; + + /* + * Protected by the global->lru_lock. + */ + + struct list_head lru; +}; + +/** + * struct ttm_bo_driver + * + * @create_ttm_backend_entry: Callback to create a struct ttm_backend. + * @invalidate_caches: Callback to invalidate read caches when a buffer object + * has been evicted. + * @init_mem_type: Callback to initialize a struct ttm_mem_type_manager + * structure. + * @evict_flags: Callback to obtain placement flags when a buffer is evicted. + * @move: Callback for a driver to hook in accelerated functions to + * move a buffer. + * If set to NULL, a potentially slow memcpy() move is used. + * @sync_obj_signaled: See ttm_fence_api.h + * @sync_obj_wait: See ttm_fence_api.h + * @sync_obj_flush: See ttm_fence_api.h + * @sync_obj_unref: See ttm_fence_api.h + * @sync_obj_ref: See ttm_fence_api.h + */ + +struct ttm_bo_driver { + /** + * ttm_tt_create + * + * @bdev: pointer to a struct ttm_bo_device: + * @size: Size of the data needed backing. + * @page_flags: Page flags as identified by TTM_PAGE_FLAG_XX flags. + * @dummy_read_page: See struct ttm_bo_device. + * + * Create a struct ttm_tt to back data with system memory pages. + * No pages are actually allocated. + * Returns: + * NULL: Out of memory. + */ + struct ttm_tt *(*ttm_tt_create)(struct ttm_bo_device *bdev, + unsigned long size, + uint32_t page_flags, + struct vm_page *dummy_read_page); + + /** + * ttm_tt_populate + * + * @ttm: The struct ttm_tt to contain the backing pages. + * + * Allocate all backing pages + * Returns: + * -ENOMEM: Out of memory. + */ + int (*ttm_tt_populate)(struct ttm_tt *ttm); + + /** + * ttm_tt_unpopulate + * + * @ttm: The struct ttm_tt to contain the backing pages. + * + * Free all backing page + */ + void (*ttm_tt_unpopulate)(struct ttm_tt *ttm); + + /** + * struct ttm_bo_driver member invalidate_caches + * + * @bdev: the buffer object device. + * @flags: new placement of the rebound buffer object. + * + * A previosly evicted buffer has been rebound in a + * potentially new location. Tell the driver that it might + * consider invalidating read (texture) caches on the next command + * submission as a consequence. + */ + + int (*invalidate_caches) (struct ttm_bo_device *bdev, uint32_t flags); + int (*init_mem_type) (struct ttm_bo_device *bdev, uint32_t type, + struct ttm_mem_type_manager *man); + /** + * struct ttm_bo_driver member evict_flags: + * + * @bo: the buffer object to be evicted + * + * Return the bo flags for a buffer which is not mapped to the hardware. + * These will be placed in proposed_flags so that when the move is + * finished, they'll end up in bo->mem.flags + */ + + void(*evict_flags) (struct ttm_buffer_object *bo, + struct ttm_placement *placement); + /** + * struct ttm_bo_driver member move: + * + * @bo: the buffer to move + * @evict: whether this motion is evicting the buffer from + * the graphics address space + * @interruptible: Use interruptible sleeps if possible when sleeping. + * @no_wait: whether this should give up and return -EBUSY + * if this move would require sleeping + * @new_mem: the new memory region receiving the buffer + * + * Move a buffer between two memory regions. + */ + int (*move) (struct ttm_buffer_object *bo, + bool evict, bool interruptible, + bool no_wait_gpu, + struct ttm_mem_reg *new_mem); + + /** + * struct ttm_bo_driver_member verify_access + * + * @bo: Pointer to a buffer object. + * @filp: Pointer to a struct file trying to access the object. + * FreeBSD: use devfs_get_cdevpriv etc. + * + * Called from the map / write / read methods to verify that the + * caller is permitted to access the buffer object. + * This member may be set to NULL, which will refuse this kind of + * access for all buffer objects. + * This function should return 0 if access is granted, -EPERM otherwise. + */ + int (*verify_access) (struct ttm_buffer_object *bo); + + /** + * In case a driver writer dislikes the TTM fence objects, + * the driver writer can replace those with sync objects of + * his / her own. If it turns out that no driver writer is + * using these. I suggest we remove these hooks and plug in + * fences directly. The bo driver needs the following functionality: + * See the corresponding functions in the fence object API + * documentation. + */ + + bool (*sync_obj_signaled) (void *sync_obj); + int (*sync_obj_wait) (void *sync_obj, + bool lazy, bool interruptible); + int (*sync_obj_flush) (void *sync_obj); + void (*sync_obj_unref) (void **sync_obj); + void *(*sync_obj_ref) (void *sync_obj); + + /* hook to notify driver about a driver move so it + * can do tiling things */ + void (*move_notify)(struct ttm_buffer_object *bo, + struct ttm_mem_reg *new_mem); + /* notify the driver we are taking a fault on this BO + * and have reserved it */ + int (*fault_reserve_notify)(struct ttm_buffer_object *bo); + + /** + * notify the driver that we're about to swap out this bo + */ + void (*swap_notify) (struct ttm_buffer_object *bo); + + /** + * Driver callback on when mapping io memory (for bo_move_memcpy + * for instance). TTM will take care to call io_mem_free whenever + * the mapping is not use anymore. io_mem_reserve & io_mem_free + * are balanced. + */ + int (*io_mem_reserve)(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem); + void (*io_mem_free)(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem); +}; + +/** + * struct ttm_bo_global_ref - Argument to initialize a struct ttm_bo_global. + */ + +struct ttm_bo_global_ref { + struct drm_global_reference ref; + struct ttm_mem_global *mem_glob; +}; + +/** + * struct ttm_bo_global - Buffer object driver global data. + * + * @mem_glob: Pointer to a struct ttm_mem_global object for accounting. + * @dummy_read_page: Pointer to a dummy page used for mapping requests + * of unpopulated pages. + * @shrink: A shrink callback object used for buffer object swap. + * @device_list_mutex: Mutex protecting the device list. + * This mutex is held while traversing the device list for pm options. + * @lru_lock: Spinlock protecting the bo subsystem lru lists. + * @device_list: List of buffer object devices. + * @swap_lru: Lru list of buffer objects used for swapping. + */ + +struct ttm_bo_global { + u_int kobj_ref; + + /** + * Constant after init. + */ + + struct ttm_mem_global *mem_glob; + struct vm_page *dummy_read_page; + struct ttm_mem_shrink shrink; + struct sx device_list_mutex; + struct mtx lru_lock; + + /** + * Protected by device_list_mutex. + */ + struct list_head device_list; + + /** + * Protected by the lru_lock. + */ + struct list_head swap_lru; + + /** + * Internal protection. + */ + atomic_t bo_count; +}; + + +#define TTM_NUM_MEM_TYPES 8 + +#define TTM_BO_PRIV_FLAG_MOVING 0 /* Buffer object is moving and needs + idling before CPU mapping */ +#define TTM_BO_PRIV_FLAG_MAX 1 +/** + * struct ttm_bo_device - Buffer object driver device-specific data. + * + * @driver: Pointer to a struct ttm_bo_driver struct setup by the driver. + * @man: An array of mem_type_managers. + * @fence_lock: Protects the synchronizing members on *all* bos belonging + * to this device. + * @addr_space_mm: Range manager for the device address space. + * lru_lock: Spinlock that protects the buffer+device lru lists and + * ddestroy lists. + * @val_seq: Current validation sequence. + * @dev_mapping: A pointer to the struct address_space representing the + * device address space. + * @wq: Work queue structure for the delayed delete workqueue. + * + */ + +struct ttm_bo_device { + + /* + * Constant after bo device init / atomic. + */ + struct list_head device_list; + struct ttm_bo_global *glob; + struct ttm_bo_driver *driver; + struct rwlock vm_lock; + struct ttm_mem_type_manager man[TTM_NUM_MEM_TYPES]; + struct mtx fence_lock; + /* + * Protected by the vm lock. + */ + RB_HEAD(ttm_bo_device_buffer_objects, ttm_buffer_object) addr_space_rb; + struct drm_mm addr_space_mm; + + /* + * Protected by the global:lru lock. + */ + struct list_head ddestroy; + uint32_t val_seq; + + /* + * Protected by load / firstopen / lastclose /unload sync. + */ + + struct address_space *dev_mapping; + + /* + * Internal protection. + */ + + struct timeout_task wq; + + bool need_dma32; +}; + +/** + * ttm_flag_masked + * + * @old: Pointer to the result and original value. + * @new: New value of bits. + * @mask: Mask of bits to change. + * + * Convenience function to change a number of bits identified by a mask. + */ + +static inline uint32_t +ttm_flag_masked(uint32_t *old, uint32_t new, uint32_t mask) +{ + *old ^= (*old ^ new) & mask; + return *old; +} + +/** + * ttm_tt_init + * + * @ttm: The struct ttm_tt. + * @bdev: pointer to a struct ttm_bo_device: + * @size: Size of the data needed backing. + * @page_flags: Page flags as identified by TTM_PAGE_FLAG_XX flags. + * @dummy_read_page: See struct ttm_bo_device. + * + * Create a struct ttm_tt to back data with system memory pages. + * No pages are actually allocated. + * Returns: + * NULL: Out of memory. + */ +extern int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, + unsigned long size, uint32_t page_flags, + struct vm_page *dummy_read_page); +extern int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, + unsigned long size, uint32_t page_flags, + struct vm_page *dummy_read_page); + +/** + * ttm_tt_fini + * + * @ttm: the ttm_tt structure. + * + * Free memory of ttm_tt structure + */ +extern void ttm_tt_fini(struct ttm_tt *ttm); +extern void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma); + +/** + * ttm_ttm_bind: + * + * @ttm: The struct ttm_tt containing backing pages. + * @bo_mem: The struct ttm_mem_reg identifying the binding location. + * + * Bind the pages of @ttm to an aperture location identified by @bo_mem + */ +extern int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem); + +/** + * ttm_ttm_destroy: + * + * @ttm: The struct ttm_tt. + * + * Unbind, unpopulate and destroy common struct ttm_tt. + */ +extern void ttm_tt_destroy(struct ttm_tt *ttm); + +/** + * ttm_ttm_unbind: + * + * @ttm: The struct ttm_tt. + * + * Unbind a struct ttm_tt. + */ +extern void ttm_tt_unbind(struct ttm_tt *ttm); + +/** + * ttm_tt_swapin: + * + * @ttm: The struct ttm_tt. + * + * Swap in a previously swap out ttm_tt. + */ +extern int ttm_tt_swapin(struct ttm_tt *ttm); + +/** + * ttm_tt_cache_flush: + * + * @pages: An array of pointers to struct page:s to flush. + * @num_pages: Number of pages to flush. + * + * Flush the data of the indicated pages from the cpu caches. + * This is used when changing caching attributes of the pages from + * cache-coherent. + */ +extern void ttm_tt_cache_flush(struct vm_page *pages[], unsigned long num_pages); + +/** + * ttm_tt_set_placement_caching: + * + * @ttm A struct ttm_tt the backing pages of which will change caching policy. + * @placement: Flag indicating the desired caching policy. + * + * This function will change caching policy of any default kernel mappings of + * the pages backing @ttm. If changing from cached to uncached or + * write-combined, + * all CPU caches will first be flushed to make sure the data of the pages + * hit RAM. This function may be very costly as it involves global TLB + * and cache flushes and potential page splitting / combining. + */ +extern int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement); +extern int ttm_tt_swapout(struct ttm_tt *ttm, + struct vm_object *persistent_swap_storage); + +/* + * ttm_bo.c + */ + +/** + * ttm_mem_reg_is_pci + * + * @bdev: Pointer to a struct ttm_bo_device. + * @mem: A valid struct ttm_mem_reg. + * + * Returns true if the memory described by @mem is PCI memory, + * false otherwise. + */ +extern bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem); + +/** + * ttm_bo_mem_space + * + * @bo: Pointer to a struct ttm_buffer_object. the data of which + * we want to allocate space for. + * @proposed_placement: Proposed new placement for the buffer object. + * @mem: A struct ttm_mem_reg. + * @interruptible: Sleep interruptible when sliping. + * @no_wait_gpu: Return immediately if the GPU is busy. + * + * Allocate memory space for the buffer object pointed to by @bo, using + * the placement flags in @mem, potentially evicting other idle buffer objects. + * This function may sleep while waiting for space to become available. + * Returns: + * -EBUSY: No space available (only if no_wait == 1). + * -ENOMEM: Could not allocate memory for the buffer object, either due to + * fragmentation or concurrent allocators. + * -ERESTARTSYS: An interruptible sleep was interrupted by a signal. + */ +extern int ttm_bo_mem_space(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_mem_reg *mem, + bool interruptible, + bool no_wait_gpu); + +extern void ttm_bo_mem_put(struct ttm_buffer_object *bo, + struct ttm_mem_reg *mem); +extern void ttm_bo_mem_put_locked(struct ttm_buffer_object *bo, + struct ttm_mem_reg *mem); + +extern void ttm_bo_global_release(struct drm_global_reference *ref); +extern int ttm_bo_global_init(struct drm_global_reference *ref); + +extern int ttm_bo_device_release(struct ttm_bo_device *bdev); + +/** + * ttm_bo_device_init + * + * @bdev: A pointer to a struct ttm_bo_device to initialize. + * @glob: A pointer to an initialized struct ttm_bo_global. + * @driver: A pointer to a struct ttm_bo_driver set up by the caller. + * @file_page_offset: Offset into the device address space that is available + * for buffer data. This ensures compatibility with other users of the + * address space. + * + * Initializes a struct ttm_bo_device: + * Returns: + * !0: Failure. + */ +extern int ttm_bo_device_init(struct ttm_bo_device *bdev, + struct ttm_bo_global *glob, + struct ttm_bo_driver *driver, + uint64_t file_page_offset, bool need_dma32); + +/** + * ttm_bo_unmap_virtual + * + * @bo: tear down the virtual mappings for this BO + */ +extern void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo); + +/** + * ttm_bo_unmap_virtual + * + * @bo: tear down the virtual mappings for this BO + * + * The caller must take ttm_mem_io_lock before calling this function. + */ +extern void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo); + +extern int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo); +extern void ttm_mem_io_free_vm(struct ttm_buffer_object *bo); +extern int ttm_mem_io_lock(struct ttm_mem_type_manager *man, + bool interruptible); +extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man); + + +/** + * ttm_bo_reserve: + * + * @bo: A pointer to a struct ttm_buffer_object. + * @interruptible: Sleep interruptible if waiting. + * @no_wait: Don't sleep while trying to reserve, rather return -EBUSY. + * @use_sequence: If @bo is already reserved, Only sleep waiting for + * it to become unreserved if @sequence < (@bo)->sequence. + * + * Locks a buffer object for validation. (Or prevents other processes from + * locking it for validation) and removes it from lru lists, while taking + * a number of measures to prevent deadlocks. + * + * Deadlocks may occur when two processes try to reserve multiple buffers in + * different order, either by will or as a result of a buffer being evicted + * to make room for a buffer already reserved. (Buffers are reserved before + * they are evicted). The following algorithm prevents such deadlocks from + * occurring: + * 1) Buffers are reserved with the lru spinlock held. Upon successful + * reservation they are removed from the lru list. This stops a reserved buffer + * from being evicted. However the lru spinlock is released between the time + * a buffer is selected for eviction and the time it is reserved. + * Therefore a check is made when a buffer is reserved for eviction, that it + * is still the first buffer in the lru list, before it is removed from the + * list. @check_lru == 1 forces this check. If it fails, the function returns + * -EINVAL, and the caller should then choose a new buffer to evict and repeat + * the procedure. + * 2) Processes attempting to reserve multiple buffers other than for eviction, + * (typically execbuf), should first obtain a unique 32-bit + * validation sequence number, + * and call this function with @use_sequence == 1 and @sequence == the unique + * sequence number. If upon call of this function, the buffer object is already + * reserved, the validation sequence is checked against the validation + * sequence of the process currently reserving the buffer, + * and if the current validation sequence is greater than that of the process + * holding the reservation, the function returns -EAGAIN. Otherwise it sleeps + * waiting for the buffer to become unreserved, after which it retries + * reserving. + * The caller should, when receiving an -EAGAIN error + * release all its buffer reservations, wait for @bo to become unreserved, and + * then rerun the validation with the same validation sequence. This procedure + * will always guarantee that the process with the lowest validation sequence + * will eventually succeed, preventing both deadlocks and starvation. + * + * Returns: + * -EAGAIN: The reservation may cause a deadlock. + * Release all buffer reservations, wait for @bo to become unreserved and + * try again. (only if use_sequence == 1). + * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by + * a signal. Release all buffer reservations and return to user-space. + * -EBUSY: The function needed to sleep, but @no_wait was true + * -EDEADLK: Bo already reserved using @sequence. This error code will only + * be returned if @use_sequence is set to true. + */ +extern int ttm_bo_reserve(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait, bool use_sequence, uint32_t sequence); + + +/** + * ttm_bo_reserve_locked: + * + * @bo: A pointer to a struct ttm_buffer_object. + * @interruptible: Sleep interruptible if waiting. + * @no_wait: Don't sleep while trying to reserve, rather return -EBUSY. + * @use_sequence: If @bo is already reserved, Only sleep waiting for + * it to become unreserved if @sequence < (@bo)->sequence. + * + * Must be called with struct ttm_bo_global::lru_lock held, + * and will not remove reserved buffers from the lru lists. + * The function may release the LRU spinlock if it needs to sleep. + * Otherwise identical to ttm_bo_reserve. + * + * Returns: + * -EAGAIN: The reservation may cause a deadlock. + * Release all buffer reservations, wait for @bo to become unreserved and + * try again. (only if use_sequence == 1). + * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by + * a signal. Release all buffer reservations and return to user-space. + * -EBUSY: The function needed to sleep, but @no_wait was true + * -EDEADLK: Bo already reserved using @sequence. This error code will only + * be returned if @use_sequence is set to true. + */ +extern int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, + bool interruptible, + bool no_wait, bool use_sequence, + uint32_t sequence); + +/** + * ttm_bo_unreserve + * + * @bo: A pointer to a struct ttm_buffer_object. + * + * Unreserve a previous reservation of @bo. + */ +extern void ttm_bo_unreserve(struct ttm_buffer_object *bo); + +/** + * ttm_bo_unreserve_locked + * + * @bo: A pointer to a struct ttm_buffer_object. + * + * Unreserve a previous reservation of @bo. + * Needs to be called with struct ttm_bo_global::lru_lock held. + */ +extern void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo); + +/** + * ttm_bo_wait_unreserved + * + * @bo: A pointer to a struct ttm_buffer_object. + * + * Wait for a struct ttm_buffer_object to become unreserved. + * This is typically used in the execbuf code to relax cpu-usage when + * a potential deadlock condition backoff. + */ +extern int ttm_bo_wait_unreserved_locked(struct ttm_buffer_object *bo, + bool interruptible); + +/* + * ttm_bo_util.c + */ + +/** + * ttm_bo_move_ttm + * + * @bo: A pointer to a struct ttm_buffer_object. + * @evict: 1: This is an eviction. Don't try to pipeline. + * @no_wait_gpu: Return immediately if the GPU is busy. + * @new_mem: struct ttm_mem_reg indicating where to move. + * + * Optimized move function for a buffer object with both old and + * new placement backed by a TTM. The function will, if successful, + * free any old aperture space, and set (@new_mem)->mm_node to NULL, + * and update the (@bo)->mem placement flags. If unsuccessful, the old + * data remains untouched, and it's up to the caller to free the + * memory space indicated by @new_mem. + * Returns: + * !0: Failure. + */ + +extern int ttm_bo_move_ttm(struct ttm_buffer_object *bo, + bool evict, bool no_wait_gpu, + struct ttm_mem_reg *new_mem); + +/** + * ttm_bo_move_memcpy + * + * @bo: A pointer to a struct ttm_buffer_object. + * @evict: 1: This is an eviction. Don't try to pipeline. + * @no_wait_gpu: Return immediately if the GPU is busy. + * @new_mem: struct ttm_mem_reg indicating where to move. + * + * Fallback move function for a mappable buffer object in mappable memory. + * The function will, if successful, + * free any old aperture space, and set (@new_mem)->mm_node to NULL, + * and update the (@bo)->mem placement flags. If unsuccessful, the old + * data remains untouched, and it's up to the caller to free the + * memory space indicated by @new_mem. + * Returns: + * !0: Failure. + */ + +extern int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, + bool evict, bool no_wait_gpu, + struct ttm_mem_reg *new_mem); + +/** + * ttm_bo_free_old_node + * + * @bo: A pointer to a struct ttm_buffer_object. + * + * Utility function to free an old placement after a successful move. + */ +extern void ttm_bo_free_old_node(struct ttm_buffer_object *bo); + +/** + * ttm_bo_move_accel_cleanup. + * + * @bo: A pointer to a struct ttm_buffer_object. + * @sync_obj: A sync object that signals when moving is complete. + * @evict: This is an evict move. Don't return until the buffer is idle. + * @no_wait_gpu: Return immediately if the GPU is busy. + * @new_mem: struct ttm_mem_reg indicating where to move. + * + * Accelerated move function to be called when an accelerated move + * has been scheduled. The function will create a new temporary buffer object + * representing the old placement, and put the sync object on both buffer + * objects. After that the newly created buffer object is unref'd to be + * destroyed when the move is complete. This will help pipeline + * buffer moves. + */ + +extern int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, + void *sync_obj, + bool evict, bool no_wait_gpu, + struct ttm_mem_reg *new_mem); +/** + * ttm_io_prot + * + * @c_state: Caching state. + * @tmp: Page protection flag for a normal, cached mapping. + * + * Utility function that returns the pgprot_t that should be used for + * setting up a PTE with the caching model indicated by @c_state. + */ +extern vm_memattr_t ttm_io_prot(uint32_t caching_flags); + +extern const struct ttm_mem_type_manager_func ttm_bo_manager_func; + +#if (defined(CONFIG_AGP) || (defined(CONFIG_AGP_MODULE) && defined(MODULE))) +#define TTM_HAS_AGP +#include <linux/agp_backend.h> + +/** + * ttm_agp_tt_create + * + * @bdev: Pointer to a struct ttm_bo_device. + * @bridge: The agp bridge this device is sitting on. + * @size: Size of the data needed backing. + * @page_flags: Page flags as identified by TTM_PAGE_FLAG_XX flags. + * @dummy_read_page: See struct ttm_bo_device. + * + * + * Create a TTM backend that uses the indicated AGP bridge as an aperture + * for TT memory. This function uses the linux agpgart interface to + * bind and unbind memory backing a ttm_tt. + */ +extern struct ttm_tt *ttm_agp_tt_create(struct ttm_bo_device *bdev, + struct agp_bridge_data *bridge, + unsigned long size, uint32_t page_flags, + struct vm_page *dummy_read_page); +int ttm_agp_tt_populate(struct ttm_tt *ttm); +void ttm_agp_tt_unpopulate(struct ttm_tt *ttm); +#endif + +int ttm_bo_cmp_rb_tree_items(struct ttm_buffer_object *a, + struct ttm_buffer_object *b); + +RB_PROTOTYPE(ttm_bo_device_buffer_objects, ttm_buffer_object, vm_rb, + ttm_bo_cmp_rb_tree_items); + +#endif diff --git a/sys/dev/drm2/ttm/ttm_bo_manager.c b/sys/dev/drm2/ttm/ttm_bo_manager.c new file mode 100644 index 0000000..4686511 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_bo_manager.c @@ -0,0 +1,157 @@ +/************************************************************************** + * + * Copyright (c) 2007-2010 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <dev/drm2/drmP.h> +#include <dev/drm2/ttm/ttm_module.h> +#include <dev/drm2/ttm/ttm_bo_driver.h> +#include <dev/drm2/ttm/ttm_placement.h> +#include <dev/drm2/drm_mm.h> + +/** + * Currently we use a spinlock for the lock, but a mutex *may* be + * more appropriate to reduce scheduling latency if the range manager + * ends up with very fragmented allocation patterns. + */ + +struct ttm_range_manager { + struct drm_mm mm; + struct mtx lock; +}; + +MALLOC_DEFINE(M_TTM_RMAN, "ttm_rman", "TTM Range Manager"); + +static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man, + struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_mem_reg *mem) +{ + struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; + struct drm_mm *mm = &rman->mm; + struct drm_mm_node *node = NULL; + unsigned long lpfn; + int ret; + + lpfn = placement->lpfn; + if (!lpfn) + lpfn = man->size; + do { + ret = drm_mm_pre_get(mm); + if (unlikely(ret)) + return ret; + + mtx_lock(&rman->lock); + node = drm_mm_search_free_in_range(mm, + mem->num_pages, mem->page_alignment, + placement->fpfn, lpfn, 1); + if (unlikely(node == NULL)) { + mtx_unlock(&rman->lock); + return 0; + } + node = drm_mm_get_block_atomic_range(node, mem->num_pages, + mem->page_alignment, + placement->fpfn, + lpfn); + mtx_unlock(&rman->lock); + } while (node == NULL); + + mem->mm_node = node; + mem->start = node->start; + return 0; +} + +static void ttm_bo_man_put_node(struct ttm_mem_type_manager *man, + struct ttm_mem_reg *mem) +{ + struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; + + if (mem->mm_node) { + mtx_lock(&rman->lock); + drm_mm_put_block(mem->mm_node); + mtx_unlock(&rman->lock); + mem->mm_node = NULL; + } +} + +static int ttm_bo_man_init(struct ttm_mem_type_manager *man, + unsigned long p_size) +{ + struct ttm_range_manager *rman; + int ret; + + rman = malloc(sizeof(*rman), M_TTM_RMAN, M_ZERO | M_WAITOK); + ret = drm_mm_init(&rman->mm, 0, p_size); + if (ret) { + free(rman, M_TTM_RMAN); + return ret; + } + + mtx_init(&rman->lock, "ttmrman", NULL, MTX_DEF); + man->priv = rman; + return 0; +} + +static int ttm_bo_man_takedown(struct ttm_mem_type_manager *man) +{ + struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; + struct drm_mm *mm = &rman->mm; + + mtx_lock(&rman->lock); + if (drm_mm_clean(mm)) { + drm_mm_takedown(mm); + mtx_unlock(&rman->lock); + mtx_destroy(&rman->lock); + free(rman, M_TTM_RMAN); + man->priv = NULL; + return 0; + } + mtx_unlock(&rman->lock); + return -EBUSY; +} + +static void ttm_bo_man_debug(struct ttm_mem_type_manager *man, + const char *prefix) +{ + struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; + + mtx_lock(&rman->lock); + drm_mm_debug_table(&rman->mm, prefix); + mtx_unlock(&rman->lock); +} + +const struct ttm_mem_type_manager_func ttm_bo_manager_func = { + ttm_bo_man_init, + ttm_bo_man_takedown, + ttm_bo_man_get_node, + ttm_bo_man_put_node, + ttm_bo_man_debug +}; diff --git a/sys/dev/drm2/ttm/ttm_bo_util.c b/sys/dev/drm2/ttm/ttm_bo_util.c new file mode 100644 index 0000000..139d134 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_bo_util.c @@ -0,0 +1,658 @@ +/************************************************************************** + * + * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <dev/drm2/drmP.h> +#include <dev/drm2/ttm/ttm_bo_driver.h> +#include <dev/drm2/ttm/ttm_placement.h> +#include <sys/sf_buf.h> + +void ttm_bo_free_old_node(struct ttm_buffer_object *bo) +{ + ttm_bo_mem_put(bo, &bo->mem); +} + +int ttm_bo_move_ttm(struct ttm_buffer_object *bo, + bool evict, + bool no_wait_gpu, struct ttm_mem_reg *new_mem) +{ + struct ttm_tt *ttm = bo->ttm; + struct ttm_mem_reg *old_mem = &bo->mem; + int ret; + + if (old_mem->mem_type != TTM_PL_SYSTEM) { + ttm_tt_unbind(ttm); + ttm_bo_free_old_node(bo); + ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, + TTM_PL_MASK_MEM); + old_mem->mem_type = TTM_PL_SYSTEM; + } + + ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); + if (unlikely(ret != 0)) + return ret; + + if (new_mem->mem_type != TTM_PL_SYSTEM) { + ret = ttm_tt_bind(ttm, new_mem); + if (unlikely(ret != 0)) + return ret; + } + + *old_mem = *new_mem; + new_mem->mm_node = NULL; + + return 0; +} + +int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) +{ + if (likely(man->io_reserve_fastpath)) + return 0; + + if (interruptible) { + if (sx_xlock_sig(&man->io_reserve_mutex)) + return (-EINTR); + else + return (0); + } + + sx_xlock(&man->io_reserve_mutex); + return 0; +} + +void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) +{ + if (likely(man->io_reserve_fastpath)) + return; + + sx_xunlock(&man->io_reserve_mutex); +} + +static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) +{ + struct ttm_buffer_object *bo; + + if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru)) + return -EAGAIN; + + bo = list_first_entry(&man->io_reserve_lru, + struct ttm_buffer_object, + io_reserve_lru); + list_del_init(&bo->io_reserve_lru); + ttm_bo_unmap_virtual_locked(bo); + + return 0; +} + +static int ttm_mem_io_reserve(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem) +{ + struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; + int ret = 0; + + if (!bdev->driver->io_mem_reserve) + return 0; + if (likely(man->io_reserve_fastpath)) + return bdev->driver->io_mem_reserve(bdev, mem); + + if (bdev->driver->io_mem_reserve && + mem->bus.io_reserved_count++ == 0) { +retry: + ret = bdev->driver->io_mem_reserve(bdev, mem); + if (ret == -EAGAIN) { + ret = ttm_mem_io_evict(man); + if (ret == 0) + goto retry; + } + } + return ret; +} + +static void ttm_mem_io_free(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem) +{ + struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; + + if (likely(man->io_reserve_fastpath)) + return; + + if (bdev->driver->io_mem_reserve && + --mem->bus.io_reserved_count == 0 && + bdev->driver->io_mem_free) + bdev->driver->io_mem_free(bdev, mem); + +} + +int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) +{ + struct ttm_mem_reg *mem = &bo->mem; + int ret; + + if (!mem->bus.io_reserved_vm) { + struct ttm_mem_type_manager *man = + &bo->bdev->man[mem->mem_type]; + + ret = ttm_mem_io_reserve(bo->bdev, mem); + if (unlikely(ret != 0)) + return ret; + mem->bus.io_reserved_vm = true; + if (man->use_io_reserve_lru) + list_add_tail(&bo->io_reserve_lru, + &man->io_reserve_lru); + } + return 0; +} + +void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) +{ + struct ttm_mem_reg *mem = &bo->mem; + + if (mem->bus.io_reserved_vm) { + mem->bus.io_reserved_vm = false; + list_del_init(&bo->io_reserve_lru); + ttm_mem_io_free(bo->bdev, mem); + } +} + +static +int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, + void **virtual) +{ + struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; + int ret; + void *addr; + + *virtual = NULL; + (void) ttm_mem_io_lock(man, false); + ret = ttm_mem_io_reserve(bdev, mem); + ttm_mem_io_unlock(man); + if (ret || !mem->bus.is_iomem) + return ret; + + if (mem->bus.addr) { + addr = mem->bus.addr; + } else { + addr = pmap_mapdev_attr(mem->bus.base + mem->bus.offset, + mem->bus.size, (mem->placement & TTM_PL_FLAG_WC) ? + VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE); + if (!addr) { + (void) ttm_mem_io_lock(man, false); + ttm_mem_io_free(bdev, mem); + ttm_mem_io_unlock(man); + return -ENOMEM; + } + } + *virtual = addr; + return 0; +} + +static +void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, + void *virtual) +{ + struct ttm_mem_type_manager *man; + + man = &bdev->man[mem->mem_type]; + + if (virtual && mem->bus.addr == NULL) + pmap_unmapdev((vm_offset_t)virtual, mem->bus.size); + (void) ttm_mem_io_lock(man, false); + ttm_mem_io_free(bdev, mem); + ttm_mem_io_unlock(man); +} + +static int ttm_copy_io_page(void *dst, void *src, unsigned long page) +{ + uint32_t *dstP = + (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); + uint32_t *srcP = + (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); + + int i; + for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) + /* iowrite32(ioread32(srcP++), dstP++); */ + *dstP++ = *srcP++; + return 0; +} + +static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, + unsigned long page, + vm_memattr_t prot) +{ + vm_page_t d = ttm->pages[page]; + void *dst; + + if (!d) + return -ENOMEM; + + src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); + + /* XXXKIB can't sleep ? */ + dst = pmap_mapdev_attr(VM_PAGE_TO_PHYS(d), PAGE_SIZE, prot); + if (!dst) + return -ENOMEM; + + memcpy(dst, src, PAGE_SIZE); + + pmap_unmapdev((vm_offset_t)dst, PAGE_SIZE); + + return 0; +} + +static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, + unsigned long page, + vm_memattr_t prot) +{ + vm_page_t s = ttm->pages[page]; + void *src; + + if (!s) + return -ENOMEM; + + dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); + src = pmap_mapdev_attr(VM_PAGE_TO_PHYS(s), PAGE_SIZE, prot); + if (!src) + return -ENOMEM; + + memcpy(dst, src, PAGE_SIZE); + + pmap_unmapdev((vm_offset_t)src, PAGE_SIZE); + + return 0; +} + +int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, + bool evict, bool no_wait_gpu, + struct ttm_mem_reg *new_mem) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; + struct ttm_tt *ttm = bo->ttm; + struct ttm_mem_reg *old_mem = &bo->mem; + struct ttm_mem_reg old_copy = *old_mem; + void *old_iomap; + void *new_iomap; + int ret; + unsigned long i; + unsigned long page; + unsigned long add = 0; + int dir; + + ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap); + if (ret) + return ret; + ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap); + if (ret) + goto out; + + if (old_iomap == NULL && new_iomap == NULL) + goto out2; + if (old_iomap == NULL && ttm == NULL) + goto out2; + + if (ttm->state == tt_unpopulated) { + ret = ttm->bdev->driver->ttm_tt_populate(ttm); + if (ret) + goto out1; + } + + add = 0; + dir = 1; + + if ((old_mem->mem_type == new_mem->mem_type) && + (new_mem->start < old_mem->start + old_mem->size)) { + dir = -1; + add = new_mem->num_pages - 1; + } + + for (i = 0; i < new_mem->num_pages; ++i) { + page = i * dir + add; + if (old_iomap == NULL) { + vm_memattr_t prot = ttm_io_prot(old_mem->placement); + ret = ttm_copy_ttm_io_page(ttm, new_iomap, page, + prot); + } else if (new_iomap == NULL) { + vm_memattr_t prot = ttm_io_prot(new_mem->placement); + ret = ttm_copy_io_ttm_page(ttm, old_iomap, page, + prot); + } else + ret = ttm_copy_io_page(new_iomap, old_iomap, page); + if (ret) + goto out1; + } + mb(); +out2: + old_copy = *old_mem; + *old_mem = *new_mem; + new_mem->mm_node = NULL; + + if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) { + ttm_tt_unbind(ttm); + ttm_tt_destroy(ttm); + bo->ttm = NULL; + } + +out1: + ttm_mem_reg_iounmap(bdev, old_mem, new_iomap); +out: + ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); + ttm_bo_mem_put(bo, &old_copy); + return ret; +} + +MALLOC_DEFINE(M_TTM_TRANSF_OBJ, "ttm_transf_obj", "TTM Transfer Objects"); + +static void ttm_transfered_destroy(struct ttm_buffer_object *bo) +{ + free(bo, M_TTM_TRANSF_OBJ); +} + +/** + * ttm_buffer_object_transfer + * + * @bo: A pointer to a struct ttm_buffer_object. + * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, + * holding the data of @bo with the old placement. + * + * This is a utility function that may be called after an accelerated move + * has been scheduled. A new buffer object is created as a placeholder for + * the old data while it's being copied. When that buffer object is idle, + * it can be destroyed, releasing the space of the old placement. + * Returns: + * !0: Failure. + */ + +static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, + struct ttm_buffer_object **new_obj) +{ + struct ttm_buffer_object *fbo; + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_bo_driver *driver = bdev->driver; + + fbo = malloc(sizeof(*fbo), M_TTM_TRANSF_OBJ, M_ZERO | M_WAITOK); + *fbo = *bo; + + /** + * Fix up members that we shouldn't copy directly: + * TODO: Explicit member copy would probably be better here. + */ + + INIT_LIST_HEAD(&fbo->ddestroy); + INIT_LIST_HEAD(&fbo->lru); + INIT_LIST_HEAD(&fbo->swap); + INIT_LIST_HEAD(&fbo->io_reserve_lru); + fbo->vm_node = NULL; + atomic_set(&fbo->cpu_writers, 0); + + fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); + refcount_init(&fbo->list_kref, 1); + refcount_init(&fbo->kref, 1); + fbo->destroy = &ttm_transfered_destroy; + fbo->acc_size = 0; + + *new_obj = fbo; + return 0; +} + +vm_memattr_t +ttm_io_prot(uint32_t caching_flags) +{ +#if defined(__i386__) || defined(__amd64__) + if (caching_flags & TTM_PL_FLAG_WC) + return (VM_MEMATTR_WRITE_COMBINING); + else + /* + * We do not support i386, look at the linux source + * for the reason of the comment. + */ + return (VM_MEMATTR_UNCACHEABLE); +#else +#error Port me +#endif +} + +static int ttm_bo_ioremap(struct ttm_buffer_object *bo, + unsigned long offset, + unsigned long size, + struct ttm_bo_kmap_obj *map) +{ + struct ttm_mem_reg *mem = &bo->mem; + + if (bo->mem.bus.addr) { + map->bo_kmap_type = ttm_bo_map_premapped; + map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); + } else { + map->bo_kmap_type = ttm_bo_map_iomap; + map->virtual = pmap_mapdev_attr(bo->mem.bus.base + + bo->mem.bus.offset + offset, size, + (mem->placement & TTM_PL_FLAG_WC) ? + VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE); + map->size = size; + } + return (!map->virtual) ? -ENOMEM : 0; +} + +static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, + unsigned long start_page, + unsigned long num_pages, + struct ttm_bo_kmap_obj *map) +{ + struct ttm_mem_reg *mem = &bo->mem; + vm_memattr_t prot; + struct ttm_tt *ttm = bo->ttm; + int i, ret; + + MPASS(ttm != NULL); + + if (ttm->state == tt_unpopulated) { + ret = ttm->bdev->driver->ttm_tt_populate(ttm); + if (ret) + return ret; + } + + if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { + /* + * We're mapping a single page, and the desired + * page protection is consistent with the bo. + */ + + map->bo_kmap_type = ttm_bo_map_kmap; + map->page = ttm->pages[start_page]; + map->sf = sf_buf_alloc(map->page, 0); + map->virtual = (void *)sf_buf_kva(map->sf); + } else { + /* + * We need to use vmap to get the desired page protection + * or to make the buffer object look contiguous. + */ + prot = (mem->placement & TTM_PL_FLAG_CACHED) ? + VM_MEMATTR_WRITE_COMBINING : + ttm_io_prot(mem->placement); + map->bo_kmap_type = ttm_bo_map_vmap; + map->num_pages = num_pages; + map->virtual = (void *)kmem_alloc_nofault(kernel_map, + num_pages * PAGE_SIZE); + if (map->virtual != NULL) { + for (i = 0; i < num_pages; i++) { + /* XXXKIB hack */ + pmap_page_set_memattr(ttm->pages[start_page + + i], prot); + } + pmap_qenter((vm_offset_t)map->virtual, + &ttm->pages[start_page], num_pages); + } + } + return (!map->virtual) ? -ENOMEM : 0; +} + +int ttm_bo_kmap(struct ttm_buffer_object *bo, + unsigned long start_page, unsigned long num_pages, + struct ttm_bo_kmap_obj *map) +{ + struct ttm_mem_type_manager *man = + &bo->bdev->man[bo->mem.mem_type]; + unsigned long offset, size; + int ret; + + MPASS(list_empty(&bo->swap)); + map->virtual = NULL; + map->bo = bo; + if (num_pages > bo->num_pages) + return -EINVAL; + if (start_page > bo->num_pages) + return -EINVAL; +#if 0 + if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) + return -EPERM; +#endif + (void) ttm_mem_io_lock(man, false); + ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); + ttm_mem_io_unlock(man); + if (ret) + return ret; + if (!bo->mem.bus.is_iomem) { + return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); + } else { + offset = start_page << PAGE_SHIFT; + size = num_pages << PAGE_SHIFT; + return ttm_bo_ioremap(bo, offset, size, map); + } +} + +void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) +{ + struct ttm_buffer_object *bo = map->bo; + struct ttm_mem_type_manager *man = + &bo->bdev->man[bo->mem.mem_type]; + + if (!map->virtual) + return; + switch (map->bo_kmap_type) { + case ttm_bo_map_iomap: + pmap_unmapdev((vm_offset_t)map->virtual, map->size); + break; + case ttm_bo_map_vmap: + pmap_qremove((vm_offset_t)(map->virtual), map->num_pages); + kmem_free(kernel_map, (vm_offset_t)map->virtual, + map->num_pages * PAGE_SIZE); + break; + case ttm_bo_map_kmap: + sf_buf_free(map->sf); + break; + case ttm_bo_map_premapped: + break; + default: + MPASS(0); + } + (void) ttm_mem_io_lock(man, false); + ttm_mem_io_free(map->bo->bdev, &map->bo->mem); + ttm_mem_io_unlock(man); + map->virtual = NULL; + map->page = NULL; + map->sf = NULL; +} + +int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, + void *sync_obj, + bool evict, + bool no_wait_gpu, + struct ttm_mem_reg *new_mem) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_bo_driver *driver = bdev->driver; + struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; + struct ttm_mem_reg *old_mem = &bo->mem; + int ret; + struct ttm_buffer_object *ghost_obj; + void *tmp_obj = NULL; + + mtx_lock(&bdev->fence_lock); + if (bo->sync_obj) { + tmp_obj = bo->sync_obj; + bo->sync_obj = NULL; + } + bo->sync_obj = driver->sync_obj_ref(sync_obj); + if (evict) { + ret = ttm_bo_wait(bo, false, false, false); + mtx_unlock(&bdev->fence_lock); + if (tmp_obj) + driver->sync_obj_unref(&tmp_obj); + if (ret) + return ret; + + if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && + (bo->ttm != NULL)) { + ttm_tt_unbind(bo->ttm); + ttm_tt_destroy(bo->ttm); + bo->ttm = NULL; + } + ttm_bo_free_old_node(bo); + } else { + /** + * This should help pipeline ordinary buffer moves. + * + * Hang old buffer memory on a new buffer object, + * and leave it to be released when the GPU + * operation has completed. + */ + + set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); + + /* ttm_buffer_object_transfer accesses bo->sync_obj */ + ret = ttm_buffer_object_transfer(bo, &ghost_obj); + mtx_unlock(&bdev->fence_lock); + if (tmp_obj) + driver->sync_obj_unref(&tmp_obj); + + if (ret) + return ret; + + /** + * If we're not moving to fixed memory, the TTM object + * needs to stay alive. Otherwhise hang it on the ghost + * bo to be unbound and destroyed. + */ + + if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) + ghost_obj->ttm = NULL; + else + bo->ttm = NULL; + + ttm_bo_unreserve(ghost_obj); + ttm_bo_unref(&ghost_obj); + } + + *old_mem = *new_mem; + new_mem->mm_node = NULL; + + return 0; +} diff --git a/sys/dev/drm2/ttm/ttm_bo_vm.c b/sys/dev/drm2/ttm/ttm_bo_vm.c new file mode 100644 index 0000000..03e2f2b --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_bo_vm.c @@ -0,0 +1,492 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ +/* + * Copyright (c) 2013 The FreeBSD Foundation + * All rights reserved. + * + * Portions of this software were developed by Konstantin Belousov + * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "opt_vm.h" + +#include <dev/drm2/drmP.h> +#include <dev/drm2/ttm/ttm_module.h> +#include <dev/drm2/ttm/ttm_bo_driver.h> +#include <dev/drm2/ttm/ttm_placement.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> + +#define TTM_BO_VM_NUM_PREFAULT 16 + +RB_GENERATE(ttm_bo_device_buffer_objects, ttm_buffer_object, vm_rb, + ttm_bo_cmp_rb_tree_items); + +int +ttm_bo_cmp_rb_tree_items(struct ttm_buffer_object *a, + struct ttm_buffer_object *b) +{ + + if (a->vm_node->start < b->vm_node->start) { + return (-1); + } else if (a->vm_node->start > b->vm_node->start) { + return (1); + } else { + return (0); + } +} + +static struct ttm_buffer_object *ttm_bo_vm_lookup_rb(struct ttm_bo_device *bdev, + unsigned long page_start, + unsigned long num_pages) +{ + unsigned long cur_offset; + struct ttm_buffer_object *bo; + struct ttm_buffer_object *best_bo = NULL; + + RB_FOREACH(bo, ttm_bo_device_buffer_objects, &bdev->addr_space_rb) { + cur_offset = bo->vm_node->start; + if (page_start >= cur_offset) { + best_bo = bo; + if (page_start == cur_offset) + break; + } + } + + if (unlikely(best_bo == NULL)) + return NULL; + + if (unlikely((best_bo->vm_node->start + best_bo->num_pages) < + (page_start + num_pages))) + return NULL; + + return best_bo; +} + +static int +ttm_bo_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset, + int prot, vm_page_t *mres) +{ + + struct ttm_buffer_object *bo = vm_obj->handle; + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_tt *ttm = NULL; + vm_page_t m, oldm; + int ret; + int retval = VM_PAGER_OK; + struct ttm_mem_type_manager *man = + &bdev->man[bo->mem.mem_type]; + + vm_object_pip_add(vm_obj, 1); + oldm = *mres; + if (oldm != NULL) { + vm_page_lock(oldm); + vm_page_remove(oldm); + vm_page_unlock(oldm); + *mres = NULL; + } else + oldm = NULL; +retry: + VM_OBJECT_UNLOCK(vm_obj); + m = NULL; + +reserve: + mtx_lock(&bo->glob->lru_lock); + ret = ttm_bo_reserve_locked(bo, false, false, false, 0); + mtx_unlock(&bo->glob->lru_lock); + if (unlikely(ret != 0)) { + if (ret == -EBUSY) { + kern_yield(0); + goto reserve; + } + } + + if (bdev->driver->fault_reserve_notify) { + ret = bdev->driver->fault_reserve_notify(bo); + switch (ret) { + case 0: + break; + case -EBUSY: + case -ERESTART: + case -EINTR: + kern_yield(0); + goto reserve; + default: + retval = VM_PAGER_ERROR; + goto out_unlock; + } + } + + /* + * Wait for buffer data in transit, due to a pipelined + * move. + */ + + mtx_lock(&bdev->fence_lock); + if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) { + ret = ttm_bo_wait(bo, false, true, false); + mtx_unlock(&bdev->fence_lock); + if (unlikely(ret != 0)) { + retval = VM_PAGER_ERROR; + goto out_unlock; + } + } else + mtx_unlock(&bdev->fence_lock); + + ret = ttm_mem_io_lock(man, true); + if (unlikely(ret != 0)) { + retval = VM_PAGER_ERROR; + goto out_unlock; + } + ret = ttm_mem_io_reserve_vm(bo); + if (unlikely(ret != 0)) { + retval = VM_PAGER_ERROR; + goto out_io_unlock; + } + + /* + * Strictly, we're not allowed to modify vma->vm_page_prot here, + * since the mmap_sem is only held in read mode. However, we + * modify only the caching bits of vma->vm_page_prot and + * consider those bits protected by + * the bo->mutex, as we should be the only writers. + * There shouldn't really be any readers of these bits except + * within vm_insert_mixed()? fork? + * + * TODO: Add a list of vmas to the bo, and change the + * vma->vm_page_prot when the object changes caching policy, with + * the correct locks held. + */ + if (!bo->mem.bus.is_iomem) { + /* Allocate all page at once, most common usage */ + ttm = bo->ttm; + if (ttm->bdev->driver->ttm_tt_populate(ttm)) { + retval = VM_PAGER_ERROR; + goto out_io_unlock; + } + } + + if (bo->mem.bus.is_iomem) { + m = vm_phys_fictitious_to_vm_page(bo->mem.bus.base + + bo->mem.bus.offset + offset); + pmap_page_set_memattr(m, ttm_io_prot(bo->mem.placement)); + } else { + ttm = bo->ttm; + m = ttm->pages[OFF_TO_IDX(offset)]; + if (unlikely(!m)) { + retval = VM_PAGER_ERROR; + goto out_io_unlock; + } + pmap_page_set_memattr(m, + (bo->mem.placement & TTM_PL_FLAG_CACHED) ? + VM_MEMATTR_WRITE_BACK : ttm_io_prot(bo->mem.placement)); + } + + VM_OBJECT_LOCK(vm_obj); + if ((m->flags & VPO_BUSY) != 0) { + vm_page_sleep(m, "ttmpbs"); + ttm_mem_io_unlock(man); + ttm_bo_unreserve(bo); + goto retry; + } + m->valid = VM_PAGE_BITS_ALL; + *mres = m; + vm_page_lock(m); + vm_page_insert(m, vm_obj, OFF_TO_IDX(offset)); + vm_page_unlock(m); + vm_page_busy(m); + + if (oldm != NULL) { + vm_page_lock(oldm); + vm_page_free(oldm); + vm_page_unlock(oldm); + } + +out_io_unlock1: + ttm_mem_io_unlock(man); +out_unlock1: + ttm_bo_unreserve(bo); + vm_object_pip_wakeup(vm_obj); + return (retval); + +out_io_unlock: + VM_OBJECT_LOCK(vm_obj); + goto out_io_unlock1; + +out_unlock: + VM_OBJECT_LOCK(vm_obj); + goto out_unlock1; +} + +static int +ttm_bo_vm_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, + vm_ooffset_t foff, struct ucred *cred, u_short *color) +{ + struct ttm_buffer_object *bo = handle; + + *color = 0; + (void)ttm_bo_reference(bo); + return (0); +} + +static void +ttm_bo_vm_dtor(void *handle) +{ + struct ttm_buffer_object *bo = handle; + + ttm_bo_unref(&bo); +} + +static struct cdev_pager_ops ttm_pager_ops = { + .cdev_pg_fault = ttm_bo_vm_fault, + .cdev_pg_ctor = ttm_bo_vm_ctor, + .cdev_pg_dtor = ttm_bo_vm_dtor +}; + +int +ttm_bo_mmap_single(struct ttm_bo_device *bdev, vm_ooffset_t *offset, vm_size_t size, + struct vm_object **obj_res, int nprot) +{ + struct ttm_bo_driver *driver; + struct ttm_buffer_object *bo; + struct vm_object *vm_obj; + int ret; + + rw_wlock(&bdev->vm_lock); + bo = ttm_bo_vm_lookup_rb(bdev, OFF_TO_IDX(*offset), OFF_TO_IDX(size)); + if (likely(bo != NULL)) + refcount_acquire(&bo->kref); + rw_wunlock(&bdev->vm_lock); + + if (unlikely(bo == NULL)) { + printf("[TTM] Could not find buffer object to map\n"); + return (EINVAL); + } + + driver = bo->bdev->driver; + if (unlikely(!driver->verify_access)) { + ret = EPERM; + goto out_unref; + } + ret = -driver->verify_access(bo); + if (unlikely(ret != 0)) + goto out_unref; + + vm_obj = cdev_pager_allocate(bo, OBJT_MGTDEVICE, &ttm_pager_ops, + size, nprot, 0, curthread->td_ucred); + if (vm_obj == NULL) { + ret = EINVAL; + goto out_unref; + } + /* + * Note: We're transferring the bo reference to vm_obj->handle here. + */ + *offset = 0; + *obj_res = vm_obj; + return 0; +out_unref: + ttm_bo_unref(&bo); + return ret; +} + +#if 0 +int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) +{ + if (vma->vm_pgoff != 0) + return -EACCES; + + vma->vm_ops = &ttm_bo_vm_ops; + vma->vm_private_data = ttm_bo_reference(bo); + vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND; + return 0; +} + +ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp, + const char __user *wbuf, char __user *rbuf, size_t count, + loff_t *f_pos, bool write) +{ + struct ttm_buffer_object *bo; + struct ttm_bo_driver *driver; + struct ttm_bo_kmap_obj map; + unsigned long dev_offset = (*f_pos >> PAGE_SHIFT); + unsigned long kmap_offset; + unsigned long kmap_end; + unsigned long kmap_num; + size_t io_size; + unsigned int page_offset; + char *virtual; + int ret; + bool no_wait = false; + bool dummy; + + read_lock(&bdev->vm_lock); + bo = ttm_bo_vm_lookup_rb(bdev, dev_offset, 1); + if (likely(bo != NULL)) + ttm_bo_reference(bo); + read_unlock(&bdev->vm_lock); + + if (unlikely(bo == NULL)) + return -EFAULT; + + driver = bo->bdev->driver; + if (unlikely(!driver->verify_access)) { + ret = -EPERM; + goto out_unref; + } + + ret = driver->verify_access(bo, filp); + if (unlikely(ret != 0)) + goto out_unref; + + kmap_offset = dev_offset - bo->vm_node->start; + if (unlikely(kmap_offset >= bo->num_pages)) { + ret = -EFBIG; + goto out_unref; + } + + page_offset = *f_pos & ~PAGE_MASK; + io_size = bo->num_pages - kmap_offset; + io_size = (io_size << PAGE_SHIFT) - page_offset; + if (count < io_size) + io_size = count; + + kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; + kmap_num = kmap_end - kmap_offset + 1; + + ret = ttm_bo_reserve(bo, true, no_wait, false, 0); + + switch (ret) { + case 0: + break; + case -EBUSY: + ret = -EAGAIN; + goto out_unref; + default: + goto out_unref; + } + + ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); + if (unlikely(ret != 0)) { + ttm_bo_unreserve(bo); + goto out_unref; + } + + virtual = ttm_kmap_obj_virtual(&map, &dummy); + virtual += page_offset; + + if (write) + ret = copy_from_user(virtual, wbuf, io_size); + else + ret = copy_to_user(rbuf, virtual, io_size); + + ttm_bo_kunmap(&map); + ttm_bo_unreserve(bo); + ttm_bo_unref(&bo); + + if (unlikely(ret != 0)) + return -EFBIG; + + *f_pos += io_size; + + return io_size; +out_unref: + ttm_bo_unref(&bo); + return ret; +} + +ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *bo, const char __user *wbuf, + char __user *rbuf, size_t count, loff_t *f_pos, + bool write) +{ + struct ttm_bo_kmap_obj map; + unsigned long kmap_offset; + unsigned long kmap_end; + unsigned long kmap_num; + size_t io_size; + unsigned int page_offset; + char *virtual; + int ret; + bool no_wait = false; + bool dummy; + + kmap_offset = (*f_pos >> PAGE_SHIFT); + if (unlikely(kmap_offset >= bo->num_pages)) + return -EFBIG; + + page_offset = *f_pos & ~PAGE_MASK; + io_size = bo->num_pages - kmap_offset; + io_size = (io_size << PAGE_SHIFT) - page_offset; + if (count < io_size) + io_size = count; + + kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; + kmap_num = kmap_end - kmap_offset + 1; + + ret = ttm_bo_reserve(bo, true, no_wait, false, 0); + + switch (ret) { + case 0: + break; + case -EBUSY: + return -EAGAIN; + default: + return ret; + } + + ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); + if (unlikely(ret != 0)) { + ttm_bo_unreserve(bo); + return ret; + } + + virtual = ttm_kmap_obj_virtual(&map, &dummy); + virtual += page_offset; + + if (write) + ret = copy_from_user(virtual, wbuf, io_size); + else + ret = copy_to_user(rbuf, virtual, io_size); + + ttm_bo_kunmap(&map); + ttm_bo_unreserve(bo); + ttm_bo_unref(&bo); + + if (unlikely(ret != 0)) + return ret; + + *f_pos += io_size; + + return io_size; +} +#endif diff --git a/sys/dev/drm2/ttm/ttm_execbuf_util.c b/sys/dev/drm2/ttm/ttm_execbuf_util.c new file mode 100644 index 0000000..ce5cf15 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_execbuf_util.c @@ -0,0 +1,230 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <dev/drm2/drmP.h> +#include <dev/drm2/ttm/ttm_execbuf_util.h> +#include <dev/drm2/ttm/ttm_bo_driver.h> +#include <dev/drm2/ttm/ttm_placement.h> + +static void ttm_eu_backoff_reservation_locked(struct list_head *list) +{ + struct ttm_validate_buffer *entry; + + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + if (!entry->reserved) + continue; + + if (entry->removed) { + ttm_bo_add_to_lru(bo); + entry->removed = false; + + } + entry->reserved = false; + atomic_set(&bo->reserved, 0); + wakeup(bo); + } +} + +static void ttm_eu_del_from_lru_locked(struct list_head *list) +{ + struct ttm_validate_buffer *entry; + + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + if (!entry->reserved) + continue; + + if (!entry->removed) { + entry->put_count = ttm_bo_del_from_lru(bo); + entry->removed = true; + } + } +} + +static void ttm_eu_list_ref_sub(struct list_head *list) +{ + struct ttm_validate_buffer *entry; + + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + + if (entry->put_count) { + ttm_bo_list_ref_sub(bo, entry->put_count, true); + entry->put_count = 0; + } + } +} + +static int ttm_eu_wait_unreserved_locked(struct list_head *list, + struct ttm_buffer_object *bo) +{ + int ret; + + ttm_eu_del_from_lru_locked(list); + ret = ttm_bo_wait_unreserved_locked(bo, true); + if (unlikely(ret != 0)) + ttm_eu_backoff_reservation_locked(list); + return ret; +} + + +void ttm_eu_backoff_reservation(struct list_head *list) +{ + struct ttm_validate_buffer *entry; + struct ttm_bo_global *glob; + + if (list_empty(list)) + return; + + entry = list_first_entry(list, struct ttm_validate_buffer, head); + glob = entry->bo->glob; + mtx_lock(&glob->lru_lock); + ttm_eu_backoff_reservation_locked(list); + mtx_unlock(&glob->lru_lock); +} + +/* + * Reserve buffers for validation. + * + * If a buffer in the list is marked for CPU access, we back off and + * wait for that buffer to become free for GPU access. + * + * If a buffer is reserved for another validation, the validator with + * the highest validation sequence backs off and waits for that buffer + * to become unreserved. This prevents deadlocks when validating multiple + * buffers in different orders. + */ + +int ttm_eu_reserve_buffers(struct list_head *list) +{ + struct ttm_bo_global *glob; + struct ttm_validate_buffer *entry; + int ret; + uint32_t val_seq; + + if (list_empty(list)) + return 0; + + list_for_each_entry(entry, list, head) { + entry->reserved = false; + entry->put_count = 0; + entry->removed = false; + } + + entry = list_first_entry(list, struct ttm_validate_buffer, head); + glob = entry->bo->glob; + + mtx_lock(&glob->lru_lock); +retry_locked: + val_seq = entry->bo->bdev->val_seq++; + + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + +retry_this_bo: + ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq); + switch (ret) { + case 0: + break; + case -EBUSY: + ret = ttm_eu_wait_unreserved_locked(list, bo); + if (unlikely(ret != 0)) { + mtx_unlock(&glob->lru_lock); + ttm_eu_list_ref_sub(list); + return ret; + } + goto retry_this_bo; + case -EAGAIN: + ttm_eu_backoff_reservation_locked(list); + ttm_eu_list_ref_sub(list); + ret = ttm_bo_wait_unreserved_locked(bo, true); + if (unlikely(ret != 0)) { + mtx_unlock(&glob->lru_lock); + return ret; + } + goto retry_locked; + default: + ttm_eu_backoff_reservation_locked(list); + mtx_unlock(&glob->lru_lock); + ttm_eu_list_ref_sub(list); + return ret; + } + + entry->reserved = true; + if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { + ttm_eu_backoff_reservation_locked(list); + mtx_unlock(&glob->lru_lock); + ttm_eu_list_ref_sub(list); + return -EBUSY; + } + } + + ttm_eu_del_from_lru_locked(list); + mtx_unlock(&glob->lru_lock); + ttm_eu_list_ref_sub(list); + + return 0; +} + +void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj) +{ + struct ttm_validate_buffer *entry; + struct ttm_buffer_object *bo; + struct ttm_bo_global *glob; + struct ttm_bo_device *bdev; + struct ttm_bo_driver *driver; + + if (list_empty(list)) + return; + + bo = list_first_entry(list, struct ttm_validate_buffer, head)->bo; + bdev = bo->bdev; + driver = bdev->driver; + glob = bo->glob; + + mtx_lock(&glob->lru_lock); + mtx_lock(&bdev->fence_lock); + + list_for_each_entry(entry, list, head) { + bo = entry->bo; + entry->old_sync_obj = bo->sync_obj; + bo->sync_obj = driver->sync_obj_ref(sync_obj); + ttm_bo_unreserve_locked(bo); + entry->reserved = false; + } + mtx_unlock(&bdev->fence_lock); + mtx_unlock(&glob->lru_lock); + + list_for_each_entry(entry, list, head) { + if (entry->old_sync_obj) + driver->sync_obj_unref(&entry->old_sync_obj); + } +} diff --git a/sys/dev/drm2/ttm/ttm_execbuf_util.h b/sys/dev/drm2/ttm/ttm_execbuf_util.h new file mode 100644 index 0000000..68b0eba --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_execbuf_util.h @@ -0,0 +1,109 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ +/* $FreeBSD$ */ + +#ifndef _TTM_EXECBUF_UTIL_H_ +#define _TTM_EXECBUF_UTIL_H_ + +#include <dev/drm2/ttm/ttm_bo_api.h> + +/** + * struct ttm_validate_buffer + * + * @head: list head for thread-private list. + * @bo: refcounted buffer object pointer. + * @reserved: Indicates whether @bo has been reserved for validation. + * @removed: Indicates whether @bo has been removed from lru lists. + * @put_count: Number of outstanding references on bo::list_kref. + * @old_sync_obj: Pointer to a sync object about to be unreferenced + */ + +struct ttm_validate_buffer { + struct list_head head; + struct ttm_buffer_object *bo; + bool reserved; + bool removed; + int put_count; + void *old_sync_obj; +}; + +/** + * function ttm_eu_backoff_reservation + * + * @list: thread private list of ttm_validate_buffer structs. + * + * Undoes all buffer validation reservations for bos pointed to by + * the list entries. + */ + +extern void ttm_eu_backoff_reservation(struct list_head *list); + +/** + * function ttm_eu_reserve_buffers + * + * @list: thread private list of ttm_validate_buffer structs. + * + * Tries to reserve bos pointed to by the list entries for validation. + * If the function returns 0, all buffers are marked as "unfenced", + * taken off the lru lists and are not synced for write CPU usage. + * + * If the function detects a deadlock due to multiple threads trying to + * reserve the same buffers in reverse order, all threads except one will + * back off and retry. This function may sleep while waiting for + * CPU write reservations to be cleared, and for other threads to + * unreserve their buffers. + * + * This function may return -ERESTART or -EAGAIN if the calling process + * receives a signal while waiting. In that case, no buffers on the list + * will be reserved upon return. + * + * Buffers reserved by this function should be unreserved by + * a call to either ttm_eu_backoff_reservation() or + * ttm_eu_fence_buffer_objects() when command submission is complete or + * has failed. + */ + +extern int ttm_eu_reserve_buffers(struct list_head *list); + +/** + * function ttm_eu_fence_buffer_objects. + * + * @list: thread private list of ttm_validate_buffer structs. + * @sync_obj: The new sync object for the buffers. + * + * This function should be called when command submission is complete, and + * it will add a new sync object to bos pointed to by entries on @list. + * It also unreserves all buffers, putting them on lru lists. + * + */ + +extern void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj); + +#endif diff --git a/sys/dev/drm2/ttm/ttm_lock.c b/sys/dev/drm2/ttm/ttm_lock.c new file mode 100644 index 0000000..8ec3041 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_lock.c @@ -0,0 +1,340 @@ +/************************************************************************** + * + * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ +/* + * Copyright (c) 2013 The FreeBSD Foundation + * All rights reserved. + * + * Portions of this software were developed by Konstantin Belousov + * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <dev/drm2/ttm/ttm_lock.h> +#include <dev/drm2/ttm/ttm_module.h> + +#define TTM_WRITE_LOCK_PENDING (1 << 0) +#define TTM_VT_LOCK_PENDING (1 << 1) +#define TTM_SUSPEND_LOCK_PENDING (1 << 2) +#define TTM_VT_LOCK (1 << 3) +#define TTM_SUSPEND_LOCK (1 << 4) + +void ttm_lock_init(struct ttm_lock *lock) +{ + mtx_init(&lock->lock, "ttmlk", NULL, MTX_DEF); + lock->rw = 0; + lock->flags = 0; + lock->kill_takers = false; + lock->signal = SIGKILL; +} + +static void +ttm_lock_send_sig(int signo) +{ + struct proc *p; + + p = curproc; /* XXXKIB curthread ? */ + PROC_LOCK(p); + kern_psignal(p, signo); + PROC_UNLOCK(p); +} + +void ttm_read_unlock(struct ttm_lock *lock) +{ + mtx_lock(&lock->lock); + if (--lock->rw == 0) + wakeup(lock); + mtx_unlock(&lock->lock); +} + +static bool __ttm_read_lock(struct ttm_lock *lock) +{ + bool locked = false; + + if (unlikely(lock->kill_takers)) { + ttm_lock_send_sig(lock->signal); + return false; + } + if (lock->rw >= 0 && lock->flags == 0) { + ++lock->rw; + locked = true; + } + return locked; +} + +int +ttm_read_lock(struct ttm_lock *lock, bool interruptible) +{ + const char *wmsg; + int flags, ret; + + ret = 0; + if (interruptible) { + flags = PCATCH; + wmsg = "ttmri"; + } else { + flags = 0; + wmsg = "ttmr"; + } + mtx_lock(&lock->lock); + while (!__ttm_read_lock(lock)) { + ret = msleep(lock, &lock->lock, flags, wmsg, 0); + if (ret != 0) + break; + } + return (-ret); +} + +static bool __ttm_read_trylock(struct ttm_lock *lock, bool *locked) +{ + bool block = true; + + *locked = false; + + if (unlikely(lock->kill_takers)) { + ttm_lock_send_sig(lock->signal); + return false; + } + if (lock->rw >= 0 && lock->flags == 0) { + ++lock->rw; + block = false; + *locked = true; + } else if (lock->flags == 0) { + block = false; + } + + return !block; +} + +int ttm_read_trylock(struct ttm_lock *lock, bool interruptible) +{ + const char *wmsg; + int flags, ret; + bool locked; + + ret = 0; + if (interruptible) { + flags = PCATCH; + wmsg = "ttmrti"; + } else { + flags = 0; + wmsg = "ttmrt"; + } + mtx_lock(&lock->lock); + while (!__ttm_read_trylock(lock, &locked)) { + ret = msleep(lock, &lock->lock, flags, wmsg, 0); + if (ret != 0) + break; + } + MPASS(!locked || ret == 0); + mtx_unlock(&lock->lock); + + return (locked) ? 0 : -EBUSY; +} + +void ttm_write_unlock(struct ttm_lock *lock) +{ + mtx_lock(&lock->lock); + lock->rw = 0; + wakeup(lock); + mtx_unlock(&lock->lock); +} + +static bool __ttm_write_lock(struct ttm_lock *lock) +{ + bool locked = false; + + if (unlikely(lock->kill_takers)) { + ttm_lock_send_sig(lock->signal); + return false; + } + if (lock->rw == 0 && ((lock->flags & ~TTM_WRITE_LOCK_PENDING) == 0)) { + lock->rw = -1; + lock->flags &= ~TTM_WRITE_LOCK_PENDING; + locked = true; + } else { + lock->flags |= TTM_WRITE_LOCK_PENDING; + } + return locked; +} + +int +ttm_write_lock(struct ttm_lock *lock, bool interruptible) +{ + const char *wmsg; + int flags, ret; + + ret = 0; + if (interruptible) { + flags = PCATCH; + wmsg = "ttmwi"; + } else { + flags = 0; + wmsg = "ttmw"; + } + mtx_lock(&lock->lock); + /* XXXKIB: linux uses __ttm_read_lock for uninterruptible sleeps */ + while (!__ttm_write_lock(lock)) { + ret = msleep(lock, &lock->lock, flags, wmsg, 0); + if (interruptible && ret != 0) { + lock->flags &= ~TTM_WRITE_LOCK_PENDING; + wakeup(lock); + break; + } + } + mtx_unlock(&lock->lock); + + return (-ret); +} + +void ttm_write_lock_downgrade(struct ttm_lock *lock) +{ + mtx_lock(&lock->lock); + lock->rw = 1; + wakeup(lock); + mtx_unlock(&lock->lock); +} + +static int __ttm_vt_unlock(struct ttm_lock *lock) +{ + int ret = 0; + + mtx_lock(&lock->lock); + if (unlikely(!(lock->flags & TTM_VT_LOCK))) + ret = -EINVAL; + lock->flags &= ~TTM_VT_LOCK; + wakeup(lock); + mtx_unlock(&lock->lock); + + return ret; +} + +static void ttm_vt_lock_remove(struct ttm_base_object **p_base) +{ + struct ttm_base_object *base = *p_base; + struct ttm_lock *lock = container_of(base, struct ttm_lock, base); + int ret; + + *p_base = NULL; + ret = __ttm_vt_unlock(lock); + MPASS(ret == 0); +} + +static bool __ttm_vt_lock(struct ttm_lock *lock) +{ + bool locked = false; + + if (lock->rw == 0) { + lock->flags &= ~TTM_VT_LOCK_PENDING; + lock->flags |= TTM_VT_LOCK; + locked = true; + } else { + lock->flags |= TTM_VT_LOCK_PENDING; + } + return locked; +} + +int ttm_vt_lock(struct ttm_lock *lock, + bool interruptible, + struct ttm_object_file *tfile) +{ + const char *wmsg; + int flags, ret; + + ret = 0; + if (interruptible) { + flags = PCATCH; + wmsg = "ttmwi"; + } else { + flags = 0; + wmsg = "ttmw"; + } + mtx_lock(&lock->lock); + while (!__ttm_vt_lock(lock)) { + ret = msleep(lock, &lock->lock, flags, wmsg, 0); + if (interruptible && ret != 0) { + lock->flags &= ~TTM_VT_LOCK_PENDING; + wakeup(lock); + break; + } + } + + /* + * Add a base-object, the destructor of which will + * make sure the lock is released if the client dies + * while holding it. + */ + + ret = ttm_base_object_init(tfile, &lock->base, false, + ttm_lock_type, &ttm_vt_lock_remove, NULL); + if (ret) + (void)__ttm_vt_unlock(lock); + else + lock->vt_holder = tfile; + + return (-ret); +} + +int ttm_vt_unlock(struct ttm_lock *lock) +{ + return ttm_ref_object_base_unref(lock->vt_holder, + lock->base.hash.key, TTM_REF_USAGE); +} + +void ttm_suspend_unlock(struct ttm_lock *lock) +{ + mtx_lock(&lock->lock); + lock->flags &= ~TTM_SUSPEND_LOCK; + wakeup(lock); + mtx_unlock(&lock->lock); +} + +static bool __ttm_suspend_lock(struct ttm_lock *lock) +{ + bool locked = false; + + if (lock->rw == 0) { + lock->flags &= ~TTM_SUSPEND_LOCK_PENDING; + lock->flags |= TTM_SUSPEND_LOCK; + locked = true; + } else { + lock->flags |= TTM_SUSPEND_LOCK_PENDING; + } + return locked; +} + +void ttm_suspend_lock(struct ttm_lock *lock) +{ + mtx_lock(&lock->lock); + while (!__ttm_suspend_lock(lock)) + msleep(lock, &lock->lock, 0, "ttms", 0); + mtx_unlock(&lock->lock); +} diff --git a/sys/dev/drm2/ttm/ttm_lock.h b/sys/dev/drm2/ttm/ttm_lock.h new file mode 100644 index 0000000..6d45457 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_lock.h @@ -0,0 +1,228 @@ +/************************************************************************** + * + * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ +/* $FreeBSD$ */ + +/** @file ttm_lock.h + * This file implements a simple replacement for the buffer manager use + * of the DRM heavyweight hardware lock. + * The lock is a read-write lock. Taking it in read mode and write mode + * is relatively fast, and intended for in-kernel use only. + * + * The vt mode is used only when there is a need to block all + * user-space processes from validating buffers. + * It's allowed to leave kernel space with the vt lock held. + * If a user-space process dies while having the vt-lock, + * it will be released during the file descriptor release. The vt lock + * excludes write lock and read lock. + * + * The suspend mode is used to lock out all TTM users when preparing for + * and executing suspend operations. + * + */ + +#ifndef _TTM_LOCK_H_ +#define _TTM_LOCK_H_ + +#include <dev/drm2/drmP.h> +#include <dev/drm2/drm.h> +#include <dev/drm2/ttm/ttm_object.h> + +/** + * struct ttm_lock + * + * @base: ttm base object used solely to release the lock if the client + * holding the lock dies. + * @queue: Queue for processes waiting for lock change-of-status. + * @lock: Spinlock protecting some lock members. + * @rw: Read-write lock counter. Protected by @lock. + * @flags: Lock state. Protected by @lock. + * @kill_takers: Boolean whether to kill takers of the lock. + * @signal: Signal to send when kill_takers is true. + */ + +struct ttm_lock { + struct ttm_base_object base; + struct mtx lock; + int32_t rw; + uint32_t flags; + bool kill_takers; + int signal; + struct ttm_object_file *vt_holder; +}; + + +/** + * ttm_lock_init + * + * @lock: Pointer to a struct ttm_lock + * Initializes the lock. + */ +extern void ttm_lock_init(struct ttm_lock *lock); + +/** + * ttm_read_unlock + * + * @lock: Pointer to a struct ttm_lock + * + * Releases a read lock. + */ +extern void ttm_read_unlock(struct ttm_lock *lock); + +/** + * ttm_read_lock + * + * @lock: Pointer to a struct ttm_lock + * @interruptible: Interruptible sleeping while waiting for a lock. + * + * Takes the lock in read mode. + * Returns: + * -ERESTARTSYS If interrupted by a signal and interruptible is true. + */ +extern int ttm_read_lock(struct ttm_lock *lock, bool interruptible); + +/** + * ttm_read_trylock + * + * @lock: Pointer to a struct ttm_lock + * @interruptible: Interruptible sleeping while waiting for a lock. + * + * Tries to take the lock in read mode. If the lock is already held + * in write mode, the function will return -EBUSY. If the lock is held + * in vt or suspend mode, the function will sleep until these modes + * are unlocked. + * + * Returns: + * -EBUSY The lock was already held in write mode. + * -ERESTARTSYS If interrupted by a signal and interruptible is true. + */ +extern int ttm_read_trylock(struct ttm_lock *lock, bool interruptible); + +/** + * ttm_lock_downgrade + * + * @lock: Pointer to a struct ttm_lock + * + * Downgrades a write lock to a read lock. + */ +extern void ttm_lock_downgrade(struct ttm_lock *lock); + +/** + * ttm_suspend_lock + * + * @lock: Pointer to a struct ttm_lock + * + * Takes the lock in suspend mode. Excludes read and write mode. + */ +extern void ttm_suspend_lock(struct ttm_lock *lock); + +/** + * ttm_suspend_unlock + * + * @lock: Pointer to a struct ttm_lock + * + * Releases a suspend lock + */ +extern void ttm_suspend_unlock(struct ttm_lock *lock); + +/** + * ttm_vt_lock + * + * @lock: Pointer to a struct ttm_lock + * @interruptible: Interruptible sleeping while waiting for a lock. + * @tfile: Pointer to a struct ttm_object_file to register the lock with. + * + * Takes the lock in vt mode. + * Returns: + * -ERESTARTSYS If interrupted by a signal and interruptible is true. + * -ENOMEM: Out of memory when locking. + */ +extern int ttm_vt_lock(struct ttm_lock *lock, bool interruptible, + struct ttm_object_file *tfile); + +/** + * ttm_vt_unlock + * + * @lock: Pointer to a struct ttm_lock + * + * Releases a vt lock. + * Returns: + * -EINVAL If the lock was not held. + */ +extern int ttm_vt_unlock(struct ttm_lock *lock); + +/** + * ttm_write_unlock + * + * @lock: Pointer to a struct ttm_lock + * + * Releases a write lock. + */ +extern void ttm_write_unlock(struct ttm_lock *lock); + +/** + * ttm_write_lock + * + * @lock: Pointer to a struct ttm_lock + * @interruptible: Interruptible sleeping while waiting for a lock. + * + * Takes the lock in write mode. + * Returns: + * -ERESTARTSYS If interrupted by a signal and interruptible is true. + */ +extern int ttm_write_lock(struct ttm_lock *lock, bool interruptible); + +void ttm_write_lock_downgrade(struct ttm_lock *lock); + +/** + * ttm_lock_set_kill + * + * @lock: Pointer to a struct ttm_lock + * @val: Boolean whether to kill processes taking the lock. + * @signal: Signal to send to the process taking the lock. + * + * The kill-when-taking-lock functionality is used to kill processes that keep + * on using the TTM functionality when its resources has been taken down, for + * example when the X server exits. A typical sequence would look like this: + * - X server takes lock in write mode. + * - ttm_lock_set_kill() is called with @val set to true. + * - As part of X server exit, TTM resources are taken down. + * - X server releases the lock on file release. + * - Another dri client wants to render, takes the lock and is killed. + * + */ +static inline void ttm_lock_set_kill(struct ttm_lock *lock, bool val, + int signal) +{ + lock->kill_takers = val; + if (val) + lock->signal = signal; +} + +#endif diff --git a/sys/dev/drm2/ttm/ttm_memory.c b/sys/dev/drm2/ttm/ttm_memory.c new file mode 100644 index 0000000..ee74d94 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_memory.c @@ -0,0 +1,471 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <dev/drm2/drmP.h> +#include <dev/drm2/ttm/ttm_memory.h> +#include <dev/drm2/ttm/ttm_module.h> +#include <dev/drm2/ttm/ttm_page_alloc.h> + +#define TTM_MEMORY_ALLOC_RETRIES 4 + +struct ttm_mem_zone { + u_int kobj_ref; + struct ttm_mem_global *glob; + const char *name; + uint64_t zone_mem; + uint64_t emer_mem; + uint64_t max_mem; + uint64_t swap_limit; + uint64_t used_mem; +}; + +MALLOC_DEFINE(M_TTM_ZONE, "ttm_zone", "TTM Zone"); + +static void ttm_mem_zone_kobj_release(struct ttm_mem_zone *zone) +{ + + printf("pTTM] Zone %7s: Used memory at exit: %llu kiB\n", + zone->name, (unsigned long long)zone->used_mem >> 10); + free(zone, M_TTM_ZONE); +} + +#if 0 +/* XXXKIB sysctl */ +static ssize_t ttm_mem_zone_show(struct ttm_mem_zone *zone; + struct attribute *attr, + char *buffer) +{ + uint64_t val = 0; + + mtx_lock(&zone->glob->lock); + if (attr == &ttm_mem_sys) + val = zone->zone_mem; + else if (attr == &ttm_mem_emer) + val = zone->emer_mem; + else if (attr == &ttm_mem_max) + val = zone->max_mem; + else if (attr == &ttm_mem_swap) + val = zone->swap_limit; + else if (attr == &ttm_mem_used) + val = zone->used_mem; + mtx_unlock(&zone->glob->lock); + + return snprintf(buffer, PAGE_SIZE, "%llu\n", + (unsigned long long) val >> 10); +} +#endif + +static void ttm_check_swapping(struct ttm_mem_global *glob); + +#if 0 +/* XXXKIB sysctl */ +static ssize_t ttm_mem_zone_store(struct ttm_mem_zone *zone, + struct attribute *attr, + const char *buffer, + size_t size) +{ + int chars; + unsigned long val; + uint64_t val64; + + chars = sscanf(buffer, "%lu", &val); + if (chars == 0) + return size; + + val64 = val; + val64 <<= 10; + + mtx_lock(&zone->glob->lock); + if (val64 > zone->zone_mem) + val64 = zone->zone_mem; + if (attr == &ttm_mem_emer) { + zone->emer_mem = val64; + if (zone->max_mem > val64) + zone->max_mem = val64; + } else if (attr == &ttm_mem_max) { + zone->max_mem = val64; + if (zone->emer_mem < val64) + zone->emer_mem = val64; + } else if (attr == &ttm_mem_swap) + zone->swap_limit = val64; + mtx_unlock(&zone->glob->lock); + + ttm_check_swapping(zone->glob); + + return size; +} +#endif + +static void ttm_mem_global_kobj_release(struct ttm_mem_global *glob) +{ + + free(glob, M_TTM_ZONE); +} + +static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, + bool from_wq, uint64_t extra) +{ + unsigned int i; + struct ttm_mem_zone *zone; + uint64_t target; + + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + + if (from_wq) + target = zone->swap_limit; + else if (priv_check(curthread, PRIV_VM_MLOCK) == 0) + target = zone->emer_mem; + else + target = zone->max_mem; + + target = (extra > target) ? 0ULL : target; + + if (zone->used_mem > target) + return true; + } + return false; +} + +/** + * At this point we only support a single shrink callback. + * Extend this if needed, perhaps using a linked list of callbacks. + * Note that this function is reentrant: + * many threads may try to swap out at any given time. + */ + +static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, + uint64_t extra) +{ + int ret; + struct ttm_mem_shrink *shrink; + + mtx_lock(&glob->lock); + if (glob->shrink == NULL) + goto out; + + while (ttm_zones_above_swap_target(glob, from_wq, extra)) { + shrink = glob->shrink; + mtx_unlock(&glob->lock); + ret = shrink->do_shrink(shrink); + mtx_lock(&glob->lock); + if (unlikely(ret != 0)) + goto out; + } +out: + mtx_unlock(&glob->lock); +} + + + +static void ttm_shrink_work(void *arg, int pending __unused) +{ + struct ttm_mem_global *glob = arg; + + ttm_shrink(glob, true, 0ULL); +} + +static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, + uint64_t mem) +{ + struct ttm_mem_zone *zone; + + zone = malloc(sizeof(*zone), M_TTM_ZONE, M_WAITOK | M_ZERO); + + zone->name = "kernel"; + zone->zone_mem = mem; + zone->max_mem = mem >> 1; + zone->emer_mem = (mem >> 1) + (mem >> 2); + zone->swap_limit = zone->max_mem - (mem >> 3); + zone->used_mem = 0; + zone->glob = glob; + glob->zone_kernel = zone; + refcount_init(&zone->kobj_ref, 1); + glob->zones[glob->num_zones++] = zone; + return 0; +} + +static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob, + uint64_t mem) +{ + struct ttm_mem_zone *zone; + + zone = malloc(sizeof(*zone), M_TTM_ZONE, M_WAITOK | M_ZERO); + + /** + * No special dma32 zone needed. + */ + + if (mem <= ((uint64_t) 1ULL << 32)) { + free(zone, M_TTM_ZONE); + return 0; + } + + /* + * Limit max dma32 memory to 4GB for now + * until we can figure out how big this + * zone really is. + */ + + mem = ((uint64_t) 1ULL << 32); + zone->name = "dma32"; + zone->zone_mem = mem; + zone->max_mem = mem >> 1; + zone->emer_mem = (mem >> 1) + (mem >> 2); + zone->swap_limit = zone->max_mem - (mem >> 3); + zone->used_mem = 0; + zone->glob = glob; + glob->zone_dma32 = zone; + refcount_init(&zone->kobj_ref, 1); + glob->zones[glob->num_zones++] = zone; + return 0; +} + +int ttm_mem_global_init(struct ttm_mem_global *glob) +{ + u_int64_t mem; + int ret; + int i; + struct ttm_mem_zone *zone; + + mtx_init(&glob->lock, "ttmgz", NULL, MTX_DEF); + glob->swap_queue = taskqueue_create("ttm_swap", M_WAITOK, + taskqueue_thread_enqueue, &glob->swap_queue); + taskqueue_start_threads(&glob->swap_queue, 1, PVM, "ttm swap"); + TASK_INIT(&glob->work, 0, ttm_shrink_work, glob); + + refcount_init(&glob->kobj_ref, 1); + + mem = physmem * PAGE_SIZE; + + ret = ttm_mem_init_kernel_zone(glob, mem); + if (unlikely(ret != 0)) + goto out_no_zone; + ret = ttm_mem_init_dma32_zone(glob, mem); + if (unlikely(ret != 0)) + goto out_no_zone; + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + printf("[TTM] Zone %7s: Available graphics memory: %llu kiB\n", + zone->name, (unsigned long long)zone->max_mem >> 10); + } + ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); + ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); + return 0; +out_no_zone: + ttm_mem_global_release(glob); + return ret; +} + +void ttm_mem_global_release(struct ttm_mem_global *glob) +{ + unsigned int i; + struct ttm_mem_zone *zone; + + /* let the page allocator first stop the shrink work. */ + ttm_page_alloc_fini(); + ttm_dma_page_alloc_fini(); + + taskqueue_drain(glob->swap_queue, &glob->work); + taskqueue_free(glob->swap_queue); + glob->swap_queue = NULL; + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + if (refcount_release(&zone->kobj_ref)) + ttm_mem_zone_kobj_release(zone); + } + if (refcount_release(&glob->kobj_ref)) + ttm_mem_global_kobj_release(glob); +} + +static void ttm_check_swapping(struct ttm_mem_global *glob) +{ + bool needs_swapping = false; + unsigned int i; + struct ttm_mem_zone *zone; + + mtx_lock(&glob->lock); + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + if (zone->used_mem > zone->swap_limit) { + needs_swapping = true; + break; + } + } + + mtx_unlock(&glob->lock); + + if (unlikely(needs_swapping)) + taskqueue_enqueue(glob->swap_queue, &glob->work); + +} + +static void ttm_mem_global_free_zone(struct ttm_mem_global *glob, + struct ttm_mem_zone *single_zone, + uint64_t amount) +{ + unsigned int i; + struct ttm_mem_zone *zone; + + mtx_lock(&glob->lock); + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + if (single_zone && zone != single_zone) + continue; + zone->used_mem -= amount; + } + mtx_unlock(&glob->lock); +} + +void ttm_mem_global_free(struct ttm_mem_global *glob, + uint64_t amount) +{ + return ttm_mem_global_free_zone(glob, NULL, amount); +} + +static int ttm_mem_global_reserve(struct ttm_mem_global *glob, + struct ttm_mem_zone *single_zone, + uint64_t amount, bool reserve) +{ + uint64_t limit; + int ret = -ENOMEM; + unsigned int i; + struct ttm_mem_zone *zone; + + mtx_lock(&glob->lock); + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + if (single_zone && zone != single_zone) + continue; + + limit = (priv_check(curthread, PRIV_VM_MLOCK) == 0) ? + zone->emer_mem : zone->max_mem; + + if (zone->used_mem > limit) + goto out_unlock; + } + + if (reserve) { + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + if (single_zone && zone != single_zone) + continue; + zone->used_mem += amount; + } + } + + ret = 0; +out_unlock: + mtx_unlock(&glob->lock); + ttm_check_swapping(glob); + + return ret; +} + + +static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, + struct ttm_mem_zone *single_zone, + uint64_t memory, + bool no_wait, bool interruptible) +{ + int count = TTM_MEMORY_ALLOC_RETRIES; + + while (unlikely(ttm_mem_global_reserve(glob, + single_zone, + memory, true) + != 0)) { + if (no_wait) + return -ENOMEM; + if (unlikely(count-- == 0)) + return -ENOMEM; + ttm_shrink(glob, false, memory + (memory >> 2) + 16); + } + + return 0; +} + +int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, + bool no_wait, bool interruptible) +{ + /** + * Normal allocations of kernel memory are registered in + * all zones. + */ + + return ttm_mem_global_alloc_zone(glob, NULL, memory, no_wait, + interruptible); +} + +#define page_to_pfn(pp) OFF_TO_IDX(VM_PAGE_TO_PHYS(pp)) + +int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, + struct vm_page *page, + bool no_wait, bool interruptible) +{ + + struct ttm_mem_zone *zone = NULL; + + /** + * Page allocations may be registed in a single zone + * only if highmem or !dma32. + */ + + if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) + zone = glob->zone_kernel; + return ttm_mem_global_alloc_zone(glob, zone, PAGE_SIZE, no_wait, + interruptible); +} + +void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct vm_page *page) +{ + struct ttm_mem_zone *zone = NULL; + + if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) + zone = glob->zone_kernel; + ttm_mem_global_free_zone(glob, zone, PAGE_SIZE); +} + + +size_t ttm_round_pot(size_t size) +{ + if ((size & (size - 1)) == 0) + return size; + else if (size > PAGE_SIZE) + return PAGE_ALIGN(size); + else { + size_t tmp_size = 4; + + while (tmp_size < size) + tmp_size <<= 1; + + return tmp_size; + } + return 0; +} diff --git a/sys/dev/drm2/ttm/ttm_memory.h b/sys/dev/drm2/ttm/ttm_memory.h new file mode 100644 index 0000000..53263c6 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_memory.h @@ -0,0 +1,149 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* $FreeBSD$ */ + +#ifndef TTM_MEMORY_H +#define TTM_MEMORY_H + +/** + * struct ttm_mem_shrink - callback to shrink TTM memory usage. + * + * @do_shrink: The callback function. + * + * Arguments to the do_shrink functions are intended to be passed using + * inheritance. That is, the argument class derives from struct ttm_mem_shrink, + * and can be accessed using container_of(). + */ + +struct ttm_mem_shrink { + int (*do_shrink) (struct ttm_mem_shrink *); +}; + +/** + * struct ttm_mem_global - Global memory accounting structure. + * + * @shrink: A single callback to shrink TTM memory usage. Extend this + * to a linked list to be able to handle multiple callbacks when needed. + * @swap_queue: A workqueue to handle shrinking in low memory situations. We + * need a separate workqueue since it will spend a lot of time waiting + * for the GPU, and this will otherwise block other workqueue tasks(?) + * At this point we use only a single-threaded workqueue. + * @work: The workqueue callback for the shrink queue. + * @lock: Lock to protect the @shrink - and the memory accounting members, + * that is, essentially the whole structure with some exceptions. + * @zones: Array of pointers to accounting zones. + * @num_zones: Number of populated entries in the @zones array. + * @zone_kernel: Pointer to the kernel zone. + * @zone_highmem: Pointer to the highmem zone if there is one. + * @zone_dma32: Pointer to the dma32 zone if there is one. + * + * Note that this structure is not per device. It should be global for all + * graphics devices. + */ + +#define TTM_MEM_MAX_ZONES 2 +struct ttm_mem_zone; +struct ttm_mem_global { + u_int kobj_ref; + struct ttm_mem_shrink *shrink; + struct taskqueue *swap_queue; + struct task work; + struct mtx lock; + struct ttm_mem_zone *zones[TTM_MEM_MAX_ZONES]; + unsigned int num_zones; + struct ttm_mem_zone *zone_kernel; + struct ttm_mem_zone *zone_dma32; +}; + +/** + * ttm_mem_init_shrink - initialize a struct ttm_mem_shrink object + * + * @shrink: The object to initialize. + * @func: The callback function. + */ + +static inline void ttm_mem_init_shrink(struct ttm_mem_shrink *shrink, + int (*func) (struct ttm_mem_shrink *)) +{ + shrink->do_shrink = func; +} + +/** + * ttm_mem_register_shrink - register a struct ttm_mem_shrink object. + * + * @glob: The struct ttm_mem_global object to register with. + * @shrink: An initialized struct ttm_mem_shrink object to register. + * + * Returns: + * -EBUSY: There's already a callback registered. (May change). + */ + +static inline int ttm_mem_register_shrink(struct ttm_mem_global *glob, + struct ttm_mem_shrink *shrink) +{ + mtx_lock(&glob->lock); + if (glob->shrink != NULL) { + mtx_unlock(&glob->lock); + return -EBUSY; + } + glob->shrink = shrink; + mtx_unlock(&glob->lock); + return 0; +} + +/** + * ttm_mem_unregister_shrink - unregister a struct ttm_mem_shrink object. + * + * @glob: The struct ttm_mem_global object to unregister from. + * @shrink: A previously registert struct ttm_mem_shrink object. + * + */ + +static inline void ttm_mem_unregister_shrink(struct ttm_mem_global *glob, + struct ttm_mem_shrink *shrink) +{ + mtx_lock(&glob->lock); + MPASS(glob->shrink == shrink); + glob->shrink = NULL; + mtx_unlock(&glob->lock); +} + +struct vm_page; + +extern int ttm_mem_global_init(struct ttm_mem_global *glob); +extern void ttm_mem_global_release(struct ttm_mem_global *glob); +extern int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, + bool no_wait, bool interruptible); +extern void ttm_mem_global_free(struct ttm_mem_global *glob, + uint64_t amount); +extern int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, + struct vm_page *page, + bool no_wait, bool interruptible); +extern void ttm_mem_global_free_page(struct ttm_mem_global *glob, + struct vm_page *page); +extern size_t ttm_round_pot(size_t size); +#endif diff --git a/sys/dev/drm2/ttm/ttm_module.h b/sys/dev/drm2/ttm/ttm_module.h new file mode 100644 index 0000000..b1ce2bc --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_module.h @@ -0,0 +1,37 @@ +/************************************************************************** + * + * Copyright 2008-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ +/* $FreeBSD$ */ + +#ifndef _TTM_MODULE_H_ +#define _TTM_MODULE_H_ + +#define TTM_PFX "[TTM] " + +#endif /* _TTM_MODULE_H_ */ diff --git a/sys/dev/drm2/ttm/ttm_object.c b/sys/dev/drm2/ttm/ttm_object.c new file mode 100644 index 0000000..fd48044 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_object.c @@ -0,0 +1,455 @@ +/************************************************************************** + * + * Copyright (c) 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ +/** @file ttm_ref_object.c + * + * Base- and reference object implementation for the various + * ttm objects. Implements reference counting, minimal security checks + * and release on file close. + */ + + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +/** + * struct ttm_object_file + * + * @tdev: Pointer to the ttm_object_device. + * + * @lock: Lock that protects the ref_list list and the + * ref_hash hash tables. + * + * @ref_list: List of ttm_ref_objects to be destroyed at + * file release. + * + * @ref_hash: Hash tables of ref objects, one per ttm_ref_type, + * for fast lookup of ref objects given a base object. + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include <dev/drm2/drmP.h> +#include <dev/drm2/drm.h> +#include <sys/rwlock.h> +#include <dev/drm2/ttm/ttm_object.h> +#include <dev/drm2/ttm/ttm_module.h> + +struct ttm_object_file { + struct ttm_object_device *tdev; + struct rwlock lock; + struct list_head ref_list; + struct drm_open_hash ref_hash[TTM_REF_NUM]; + u_int refcount; +}; + +/** + * struct ttm_object_device + * + * @object_lock: lock that protects the object_hash hash table. + * + * @object_hash: hash table for fast lookup of object global names. + * + * @object_count: Per device object count. + * + * This is the per-device data structure needed for ttm object management. + */ + +struct ttm_object_device { + struct rwlock object_lock; + struct drm_open_hash object_hash; + atomic_t object_count; + struct ttm_mem_global *mem_glob; +}; + +/** + * struct ttm_ref_object + * + * @hash: Hash entry for the per-file object reference hash. + * + * @head: List entry for the per-file list of ref-objects. + * + * @kref: Ref count. + * + * @obj: Base object this ref object is referencing. + * + * @ref_type: Type of ref object. + * + * This is similar to an idr object, but it also has a hash table entry + * that allows lookup with a pointer to the referenced object as a key. In + * that way, one can easily detect whether a base object is referenced by + * a particular ttm_object_file. It also carries a ref count to avoid creating + * multiple ref objects if a ttm_object_file references the same base + * object more than once. + */ + +struct ttm_ref_object { + struct drm_hash_item hash; + struct list_head head; + u_int kref; + enum ttm_ref_type ref_type; + struct ttm_base_object *obj; + struct ttm_object_file *tfile; +}; + +MALLOC_DEFINE(M_TTM_OBJ_FILE, "ttm_obj_file", "TTM File Objects"); + +static inline struct ttm_object_file * +ttm_object_file_ref(struct ttm_object_file *tfile) +{ + refcount_acquire(&tfile->refcount); + return tfile; +} + +static void ttm_object_file_destroy(struct ttm_object_file *tfile) +{ + + free(tfile, M_TTM_OBJ_FILE); +} + + +static inline void ttm_object_file_unref(struct ttm_object_file **p_tfile) +{ + struct ttm_object_file *tfile = *p_tfile; + + *p_tfile = NULL; + if (refcount_release(&tfile->refcount)) + ttm_object_file_destroy(tfile); +} + + +int ttm_base_object_init(struct ttm_object_file *tfile, + struct ttm_base_object *base, + bool shareable, + enum ttm_object_type object_type, + void (*rcount_release) (struct ttm_base_object **), + void (*ref_obj_release) (struct ttm_base_object *, + enum ttm_ref_type ref_type)) +{ + struct ttm_object_device *tdev = tfile->tdev; + int ret; + + base->shareable = shareable; + base->tfile = ttm_object_file_ref(tfile); + base->refcount_release = rcount_release; + base->ref_obj_release = ref_obj_release; + base->object_type = object_type; + refcount_init(&base->refcount, 1); + rw_init(&tdev->object_lock, "ttmbao"); + rw_wlock(&tdev->object_lock); + ret = drm_ht_just_insert_please(&tdev->object_hash, + &base->hash, + (unsigned long)base, 31, 0, 0); + rw_wunlock(&tdev->object_lock); + if (unlikely(ret != 0)) + goto out_err0; + + ret = ttm_ref_object_add(tfile, base, TTM_REF_USAGE, NULL); + if (unlikely(ret != 0)) + goto out_err1; + + ttm_base_object_unref(&base); + + return 0; +out_err1: + rw_wlock(&tdev->object_lock); + (void)drm_ht_remove_item(&tdev->object_hash, &base->hash); + rw_wunlock(&tdev->object_lock); +out_err0: + return ret; +} + +static void ttm_release_base(struct ttm_base_object *base) +{ + struct ttm_object_device *tdev = base->tfile->tdev; + + (void)drm_ht_remove_item(&tdev->object_hash, &base->hash); + rw_wunlock(&tdev->object_lock); + /* + * Note: We don't use synchronize_rcu() here because it's far + * too slow. It's up to the user to free the object using + * call_rcu() or ttm_base_object_kfree(). + */ + + if (base->refcount_release) { + ttm_object_file_unref(&base->tfile); + base->refcount_release(&base); + } + rw_wlock(&tdev->object_lock); +} + +void ttm_base_object_unref(struct ttm_base_object **p_base) +{ + struct ttm_base_object *base = *p_base; + struct ttm_object_device *tdev = base->tfile->tdev; + + *p_base = NULL; + + /* + * Need to take the lock here to avoid racing with + * users trying to look up the object. + */ + + rw_wlock(&tdev->object_lock); + if (refcount_release(&base->refcount)) + ttm_release_base(base); + rw_wunlock(&tdev->object_lock); +} + +struct ttm_base_object *ttm_base_object_lookup(struct ttm_object_file *tfile, + uint32_t key) +{ + struct ttm_object_device *tdev = tfile->tdev; + struct ttm_base_object *base; + struct drm_hash_item *hash; + int ret; + + rw_rlock(&tdev->object_lock); + ret = drm_ht_find_item(&tdev->object_hash, key, &hash); + + if (ret == 0) { + base = drm_hash_entry(hash, struct ttm_base_object, hash); + refcount_acquire(&base->refcount); + } + rw_runlock(&tdev->object_lock); + + if (unlikely(ret != 0)) + return NULL; + + if (tfile != base->tfile && !base->shareable) { + printf("[TTM] Attempted access of non-shareable object %p\n", + base); + ttm_base_object_unref(&base); + return NULL; + } + + return base; +} + +MALLOC_DEFINE(M_TTM_OBJ_REF, "ttm_obj_ref", "TTM Ref Objects"); + +int ttm_ref_object_add(struct ttm_object_file *tfile, + struct ttm_base_object *base, + enum ttm_ref_type ref_type, bool *existed) +{ + struct drm_open_hash *ht = &tfile->ref_hash[ref_type]; + struct ttm_ref_object *ref; + struct drm_hash_item *hash; + struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; + int ret = -EINVAL; + + if (existed != NULL) + *existed = true; + + while (ret == -EINVAL) { + rw_rlock(&tfile->lock); + ret = drm_ht_find_item(ht, base->hash.key, &hash); + + if (ret == 0) { + ref = drm_hash_entry(hash, struct ttm_ref_object, hash); + refcount_acquire(&ref->kref); + rw_runlock(&tfile->lock); + break; + } + + rw_runlock(&tfile->lock); + ret = ttm_mem_global_alloc(mem_glob, sizeof(*ref), + false, false); + if (unlikely(ret != 0)) + return ret; + ref = malloc(sizeof(*ref), M_TTM_OBJ_REF, M_WAITOK); + if (unlikely(ref == NULL)) { + ttm_mem_global_free(mem_glob, sizeof(*ref)); + return -ENOMEM; + } + + ref->hash.key = base->hash.key; + ref->obj = base; + ref->tfile = tfile; + ref->ref_type = ref_type; + refcount_init(&ref->kref, 1); + + rw_wlock(&tfile->lock); + ret = drm_ht_insert_item(ht, &ref->hash); + + if (ret == 0) { + list_add_tail(&ref->head, &tfile->ref_list); + refcount_acquire(&base->refcount); + rw_wunlock(&tfile->lock); + if (existed != NULL) + *existed = false; + break; + } + + rw_wunlock(&tfile->lock); + MPASS(ret == -EINVAL); + + ttm_mem_global_free(mem_glob, sizeof(*ref)); + free(ref, M_TTM_OBJ_REF); + } + + return ret; +} + +static void ttm_ref_object_release(struct ttm_ref_object *ref) +{ + struct ttm_base_object *base = ref->obj; + struct ttm_object_file *tfile = ref->tfile; + struct drm_open_hash *ht; + struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; + + ht = &tfile->ref_hash[ref->ref_type]; + (void)drm_ht_remove_item(ht, &ref->hash); + list_del(&ref->head); + rw_wunlock(&tfile->lock); + + if (ref->ref_type != TTM_REF_USAGE && base->ref_obj_release) + base->ref_obj_release(base, ref->ref_type); + + ttm_base_object_unref(&ref->obj); + ttm_mem_global_free(mem_glob, sizeof(*ref)); + free(ref, M_TTM_OBJ_REF); + rw_wlock(&tfile->lock); +} + +int ttm_ref_object_base_unref(struct ttm_object_file *tfile, + unsigned long key, enum ttm_ref_type ref_type) +{ + struct drm_open_hash *ht = &tfile->ref_hash[ref_type]; + struct ttm_ref_object *ref; + struct drm_hash_item *hash; + int ret; + + rw_wlock(&tfile->lock); + ret = drm_ht_find_item(ht, key, &hash); + if (unlikely(ret != 0)) { + rw_wunlock(&tfile->lock); + return -EINVAL; + } + ref = drm_hash_entry(hash, struct ttm_ref_object, hash); + if (refcount_release(&ref->kref)) + ttm_ref_object_release(ref); + rw_wunlock(&tfile->lock); + return 0; +} + +void ttm_object_file_release(struct ttm_object_file **p_tfile) +{ + struct ttm_ref_object *ref; + struct list_head *list; + unsigned int i; + struct ttm_object_file *tfile = *p_tfile; + + *p_tfile = NULL; + rw_wlock(&tfile->lock); + + /* + * Since we release the lock within the loop, we have to + * restart it from the beginning each time. + */ + + while (!list_empty(&tfile->ref_list)) { + list = tfile->ref_list.next; + ref = list_entry(list, struct ttm_ref_object, head); + ttm_ref_object_release(ref); + } + + for (i = 0; i < TTM_REF_NUM; ++i) + drm_ht_remove(&tfile->ref_hash[i]); + + rw_wunlock(&tfile->lock); + ttm_object_file_unref(&tfile); +} + +struct ttm_object_file *ttm_object_file_init(struct ttm_object_device *tdev, + unsigned int hash_order) +{ + struct ttm_object_file *tfile; + unsigned int i; + unsigned int j = 0; + int ret; + + tfile = malloc(sizeof(*tfile), M_TTM_OBJ_FILE, M_WAITOK); + rw_init(&tfile->lock, "ttmfo"); + tfile->tdev = tdev; + refcount_init(&tfile->refcount, 1); + INIT_LIST_HEAD(&tfile->ref_list); + + for (i = 0; i < TTM_REF_NUM; ++i) { + ret = drm_ht_create(&tfile->ref_hash[i], hash_order); + if (ret) { + j = i; + goto out_err; + } + } + + return tfile; +out_err: + for (i = 0; i < j; ++i) + drm_ht_remove(&tfile->ref_hash[i]); + + free(tfile, M_TTM_OBJ_FILE); + + return NULL; +} + +MALLOC_DEFINE(M_TTM_OBJ_DEV, "ttm_obj_dev", "TTM Device Objects"); + +struct ttm_object_device *ttm_object_device_init(struct ttm_mem_global + *mem_glob, + unsigned int hash_order) +{ + struct ttm_object_device *tdev; + int ret; + + tdev = malloc(sizeof(*tdev), M_TTM_OBJ_DEV, M_WAITOK); + tdev->mem_glob = mem_glob; + rw_init(&tdev->object_lock, "ttmdo"); + atomic_set(&tdev->object_count, 0); + ret = drm_ht_create(&tdev->object_hash, hash_order); + + if (ret == 0) + return tdev; + + free(tdev, M_TTM_OBJ_DEV); + return NULL; +} + +void ttm_object_device_release(struct ttm_object_device **p_tdev) +{ + struct ttm_object_device *tdev = *p_tdev; + + *p_tdev = NULL; + + rw_wlock(&tdev->object_lock); + drm_ht_remove(&tdev->object_hash); + rw_wunlock(&tdev->object_lock); + + free(tdev, M_TTM_OBJ_DEV); +} diff --git a/sys/dev/drm2/ttm/ttm_object.h b/sys/dev/drm2/ttm/ttm_object.h new file mode 100644 index 0000000..8a286f1 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_object.h @@ -0,0 +1,271 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ +/* $FreeBSD$ */ +/** @file ttm_object.h + * + * Base- and reference object implementation for the various + * ttm objects. Implements reference counting, minimal security checks + * and release on file close. + */ + +#ifndef _TTM_OBJECT_H_ +#define _TTM_OBJECT_H_ + +#include <dev/drm2/drm_hashtab.h> +#include <dev/drm2/ttm/ttm_memory.h> + +/** + * enum ttm_ref_type + * + * Describes what type of reference a ref object holds. + * + * TTM_REF_USAGE is a simple refcount on a base object. + * + * TTM_REF_SYNCCPU_READ is a SYNCCPU_READ reference on a + * buffer object. + * + * TTM_REF_SYNCCPU_WRITE is a SYNCCPU_WRITE reference on a + * buffer object. + * + */ + +enum ttm_ref_type { + TTM_REF_USAGE, + TTM_REF_SYNCCPU_READ, + TTM_REF_SYNCCPU_WRITE, + TTM_REF_NUM +}; + +/** + * enum ttm_object_type + * + * One entry per ttm object type. + * Device-specific types should use the + * ttm_driver_typex types. + */ + +enum ttm_object_type { + ttm_fence_type, + ttm_buffer_type, + ttm_lock_type, + ttm_driver_type0 = 256, + ttm_driver_type1, + ttm_driver_type2, + ttm_driver_type3, + ttm_driver_type4, + ttm_driver_type5 +}; + +struct ttm_object_file; +struct ttm_object_device; + +/** + * struct ttm_base_object + * + * @hash: hash entry for the per-device object hash. + * @type: derived type this object is base class for. + * @shareable: Other ttm_object_files can access this object. + * + * @tfile: Pointer to ttm_object_file of the creator. + * NULL if the object was not created by a user request. + * (kernel object). + * + * @refcount: Number of references to this object, not + * including the hash entry. A reference to a base object can + * only be held by a ref object. + * + * @refcount_release: A function to be called when there are + * no more references to this object. This function should + * destroy the object (or make sure destruction eventually happens), + * and when it is called, the object has + * already been taken out of the per-device hash. The parameter + * "base" should be set to NULL by the function. + * + * @ref_obj_release: A function to be called when a reference object + * with another ttm_ref_type than TTM_REF_USAGE is deleted. + * This function may, for example, release a lock held by a user-space + * process. + * + * This struct is intended to be used as a base struct for objects that + * are visible to user-space. It provides a global name, race-safe + * access and refcounting, minimal access contol and hooks for unref actions. + */ + +struct ttm_base_object { + /* struct rcu_head rhead;XXXKIB */ + struct drm_hash_item hash; + enum ttm_object_type object_type; + bool shareable; + struct ttm_object_file *tfile; + u_int refcount; + void (*refcount_release) (struct ttm_base_object **base); + void (*ref_obj_release) (struct ttm_base_object *base, + enum ttm_ref_type ref_type); +}; + +/** + * ttm_base_object_init + * + * @tfile: Pointer to a struct ttm_object_file. + * @base: The struct ttm_base_object to initialize. + * @shareable: This object is shareable with other applcations. + * (different @tfile pointers.) + * @type: The object type. + * @refcount_release: See the struct ttm_base_object description. + * @ref_obj_release: See the struct ttm_base_object description. + * + * Initializes a struct ttm_base_object. + */ + +extern int ttm_base_object_init(struct ttm_object_file *tfile, + struct ttm_base_object *base, + bool shareable, + enum ttm_object_type type, + void (*refcount_release) (struct ttm_base_object + **), + void (*ref_obj_release) (struct ttm_base_object + *, + enum ttm_ref_type + ref_type)); + +/** + * ttm_base_object_lookup + * + * @tfile: Pointer to a struct ttm_object_file. + * @key: Hash key + * + * Looks up a struct ttm_base_object with the key @key. + * Also verifies that the object is visible to the application, by + * comparing the @tfile argument and checking the object shareable flag. + */ + +extern struct ttm_base_object *ttm_base_object_lookup(struct ttm_object_file + *tfile, uint32_t key); + +/** + * ttm_base_object_unref + * + * @p_base: Pointer to a pointer referencing a struct ttm_base_object. + * + * Decrements the base object refcount and clears the pointer pointed to by + * p_base. + */ + +extern void ttm_base_object_unref(struct ttm_base_object **p_base); + +/** + * ttm_ref_object_add. + * + * @tfile: A struct ttm_object_file representing the application owning the + * ref_object. + * @base: The base object to reference. + * @ref_type: The type of reference. + * @existed: Upon completion, indicates that an identical reference object + * already existed, and the refcount was upped on that object instead. + * + * Adding a ref object to a base object is basically like referencing the + * base object, but a user-space application holds the reference. When the + * file corresponding to @tfile is closed, all its reference objects are + * deleted. A reference object can have different types depending on what + * it's intended for. It can be refcounting to prevent object destruction, + * When user-space takes a lock, it can add a ref object to that lock to + * make sure the lock is released if the application dies. A ref object + * will hold a single reference on a base object. + */ +extern int ttm_ref_object_add(struct ttm_object_file *tfile, + struct ttm_base_object *base, + enum ttm_ref_type ref_type, bool *existed); +/** + * ttm_ref_object_base_unref + * + * @key: Key representing the base object. + * @ref_type: Ref type of the ref object to be dereferenced. + * + * Unreference a ref object with type @ref_type + * on the base object identified by @key. If there are no duplicate + * references, the ref object will be destroyed and the base object + * will be unreferenced. + */ +extern int ttm_ref_object_base_unref(struct ttm_object_file *tfile, + unsigned long key, + enum ttm_ref_type ref_type); + +/** + * ttm_object_file_init - initialize a struct ttm_object file + * + * @tdev: A struct ttm_object device this file is initialized on. + * @hash_order: Order of the hash table used to hold the reference objects. + * + * This is typically called by the file_ops::open function. + */ + +extern struct ttm_object_file *ttm_object_file_init(struct ttm_object_device + *tdev, + unsigned int hash_order); + +/** + * ttm_object_file_release - release data held by a ttm_object_file + * + * @p_tfile: Pointer to pointer to the ttm_object_file object to release. + * *p_tfile will be set to NULL by this function. + * + * Releases all data associated by a ttm_object_file. + * Typically called from file_ops::release. The caller must + * ensure that there are no concurrent users of tfile. + */ + +extern void ttm_object_file_release(struct ttm_object_file **p_tfile); + +/** + * ttm_object device init - initialize a struct ttm_object_device + * + * @hash_order: Order of hash table used to hash the base objects. + * + * This function is typically called on device initialization to prepare + * data structures needed for ttm base and ref objects. + */ + +extern struct ttm_object_device *ttm_object_device_init + (struct ttm_mem_global *mem_glob, unsigned int hash_order); + +/** + * ttm_object_device_release - release data held by a ttm_object_device + * + * @p_tdev: Pointer to pointer to the ttm_object_device object to release. + * *p_tdev will be set to NULL by this function. + * + * Releases all data associated by a ttm_object_device. + * Typically called from driver::unload before the destruction of the + * device private data structure. + */ + +extern void ttm_object_device_release(struct ttm_object_device **p_tdev); + +#endif diff --git a/sys/dev/drm2/ttm/ttm_page_alloc.c b/sys/dev/drm2/ttm/ttm_page_alloc.c new file mode 100644 index 0000000..9a30a46 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_page_alloc.c @@ -0,0 +1,900 @@ +/* + * Copyright (c) Red Hat Inc. + + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sub license, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie <airlied@redhat.com> + * Jerome Glisse <jglisse@redhat.com> + * Pauli Nieminen <suokkos@gmail.com> + */ +/* + * Copyright (c) 2013 The FreeBSD Foundation + * All rights reserved. + * + * Portions of this software were developed by Konstantin Belousov + * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. + */ + +/* simple list based uncached page pool + * - Pool collects resently freed pages for reuse + * - Use page->lru to keep a free list + * - doesn't track currently in use pages + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <dev/drm2/drmP.h> +#include <dev/drm2/ttm/ttm_bo_driver.h> +#include <dev/drm2/ttm/ttm_page_alloc.h> + +#ifdef TTM_HAS_AGP +#include <asm/agp.h> +#endif + +#define VM_ALLOC_DMA32 VM_ALLOC_RESERVED1 + +#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(vm_page_t)) +#define SMALL_ALLOCATION 16 +#define FREE_ALL_PAGES (~0U) +/* times are in msecs */ +#define PAGE_FREE_INTERVAL 1000 + +/** + * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. + * + * @lock: Protects the shared pool from concurrnet access. Must be used with + * irqsave/irqrestore variants because pool allocator maybe called from + * delayed work. + * @fill_lock: Prevent concurrent calls to fill. + * @list: Pool of free uc/wc pages for fast reuse. + * @gfp_flags: Flags to pass for alloc_page. + * @npages: Number of pages in pool. + */ +struct ttm_page_pool { + struct mtx lock; + bool fill_lock; + bool dma32; + struct pglist list; + int ttm_page_alloc_flags; + unsigned npages; + char *name; + unsigned long nfrees; + unsigned long nrefills; +}; + +/** + * Limits for the pool. They are handled without locks because only place where + * they may change is in sysfs store. They won't have immediate effect anyway + * so forcing serialization to access them is pointless. + */ + +struct ttm_pool_opts { + unsigned alloc_size; + unsigned max_size; + unsigned small; +}; + +#define NUM_POOLS 4 + +/** + * struct ttm_pool_manager - Holds memory pools for fst allocation + * + * Manager is read only object for pool code so it doesn't need locking. + * + * @free_interval: minimum number of jiffies between freeing pages from pool. + * @page_alloc_inited: reference counting for pool allocation. + * @work: Work that is used to shrink the pool. Work is only run when there is + * some pages to free. + * @small_allocation: Limit in number of pages what is small allocation. + * + * @pools: All pool objects in use. + **/ +struct ttm_pool_manager { + unsigned int kobj_ref; + eventhandler_tag lowmem_handler; + struct ttm_pool_opts options; + + union { + struct ttm_page_pool u_pools[NUM_POOLS]; + struct _utag { + struct ttm_page_pool u_wc_pool; + struct ttm_page_pool u_uc_pool; + struct ttm_page_pool u_wc_pool_dma32; + struct ttm_page_pool u_uc_pool_dma32; + } _ut; + } _u; +}; + +#define pools _u.u_pools +#define wc_pool _u._ut.u_wc_pool +#define uc_pool _u._ut.u_uc_pool +#define wc_pool_dma32 _u._ut.u_wc_pool_dma32 +#define uc_pool_dma32 _u._ut.u_uc_pool_dma32 + +MALLOC_DEFINE(M_TTM_POOLMGR, "ttm_poolmgr", "TTM Pool Manager"); + +static void +ttm_vm_page_free(vm_page_t m) +{ + + KASSERT(m->object == NULL, ("ttm page %p is owned", m)); + KASSERT(m->wire_count == 1, ("ttm lost wire %p", m)); + KASSERT((m->flags & PG_FICTITIOUS) != 0, ("ttm lost fictitious %p", m)); + KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("ttm got unmanaged %p", m)); + m->flags &= ~PG_FICTITIOUS; + m->oflags |= VPO_UNMANAGED; + vm_page_unwire(m, 0); + vm_page_free(m); +} + +static vm_memattr_t +ttm_caching_state_to_vm(enum ttm_caching_state cstate) +{ + + switch (cstate) { + case tt_uncached: + return (VM_MEMATTR_UNCACHEABLE); + case tt_wc: + return (VM_MEMATTR_WRITE_COMBINING); + case tt_cached: + return (VM_MEMATTR_WRITE_BACK); + } + panic("caching state %d\n", cstate); +} + +static void ttm_pool_kobj_release(struct ttm_pool_manager *m) +{ + + free(m, M_TTM_POOLMGR); +} + +#if 0 +/* XXXKIB sysctl */ +static ssize_t ttm_pool_store(struct ttm_pool_manager *m, + struct attribute *attr, const char *buffer, size_t size) +{ + int chars; + unsigned val; + chars = sscanf(buffer, "%u", &val); + if (chars == 0) + return size; + + /* Convert kb to number of pages */ + val = val / (PAGE_SIZE >> 10); + + if (attr == &ttm_page_pool_max) + m->options.max_size = val; + else if (attr == &ttm_page_pool_small) + m->options.small = val; + else if (attr == &ttm_page_pool_alloc_size) { + if (val > NUM_PAGES_TO_ALLOC*8) { + pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + return size; + } else if (val > NUM_PAGES_TO_ALLOC) { + pr_warn("Setting allocation size to larger than %lu is not recommended\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + } + m->options.alloc_size = val; + } + + return size; +} + +static ssize_t ttm_pool_show(struct ttm_pool_manager *m, + struct attribute *attr, char *buffer) +{ + unsigned val = 0; + + if (attr == &ttm_page_pool_max) + val = m->options.max_size; + else if (attr == &ttm_page_pool_small) + val = m->options.small; + else if (attr == &ttm_page_pool_alloc_size) + val = m->options.alloc_size; + + val = val * (PAGE_SIZE >> 10); + + return snprintf(buffer, PAGE_SIZE, "%u\n", val); +} +#endif + +static struct ttm_pool_manager *_manager; + +static int set_pages_array_wb(vm_page_t *pages, int addrinarray) +{ + vm_page_t m; + int i; + + for (i = 0; i < addrinarray; i++) { + m = pages[i]; +#ifdef TTM_HAS_AGP + unmap_page_from_agp(m); +#endif + pmap_page_set_memattr(m, VM_MEMATTR_WRITE_BACK); + } + return 0; +} + +static int set_pages_array_wc(vm_page_t *pages, int addrinarray) +{ + vm_page_t m; + int i; + + for (i = 0; i < addrinarray; i++) { + m = pages[i]; +#ifdef TTM_HAS_AGP + map_page_into_agp(pages[i]); +#endif + pmap_page_set_memattr(m, VM_MEMATTR_WRITE_COMBINING); + } + return 0; +} + +static int set_pages_array_uc(vm_page_t *pages, int addrinarray) +{ + vm_page_t m; + int i; + + for (i = 0; i < addrinarray; i++) { + m = pages[i]; +#ifdef TTM_HAS_AGP + map_page_into_agp(pages[i]); +#endif + pmap_page_set_memattr(m, VM_MEMATTR_UNCACHEABLE); + } + return 0; +} + +/** + * Select the right pool or requested caching state and ttm flags. */ +static struct ttm_page_pool *ttm_get_pool(int flags, + enum ttm_caching_state cstate) +{ + int pool_index; + + if (cstate == tt_cached) + return NULL; + + if (cstate == tt_wc) + pool_index = 0x0; + else + pool_index = 0x1; + + if (flags & TTM_PAGE_FLAG_DMA32) + pool_index |= 0x2; + + return &_manager->pools[pool_index]; +} + +/* set memory back to wb and free the pages. */ +static void ttm_pages_put(vm_page_t *pages, unsigned npages) +{ + unsigned i; + + /* Our VM handles vm memattr automatically on the page free. */ + if (set_pages_array_wb(pages, npages)) + printf("[TTM] Failed to set %d pages to wb!\n", npages); + for (i = 0; i < npages; ++i) + ttm_vm_page_free(pages[i]); +} + +static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, + unsigned freed_pages) +{ + pool->npages -= freed_pages; + pool->nfrees += freed_pages; +} + +/** + * Free pages from pool. + * + * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC + * number of pages in one go. + * + * @pool: to free the pages from + * @free_all: If set to true will free all pages in pool + **/ +static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) +{ + vm_page_t p, p1; + vm_page_t *pages_to_free; + unsigned freed_pages = 0, + npages_to_free = nr_free; + + if (NUM_PAGES_TO_ALLOC < nr_free) + npages_to_free = NUM_PAGES_TO_ALLOC; + + pages_to_free = malloc(npages_to_free * sizeof(vm_page_t), + M_TEMP, M_WAITOK | M_ZERO); + +restart: + mtx_lock(&pool->lock); + + TAILQ_FOREACH_REVERSE_SAFE(p, &pool->list, pglist, pageq, p1) { + if (freed_pages >= npages_to_free) + break; + + pages_to_free[freed_pages++] = p; + /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ + if (freed_pages >= NUM_PAGES_TO_ALLOC) { + /* remove range of pages from the pool */ + TAILQ_REMOVE(&pool->list, p, pageq); + + ttm_pool_update_free_locked(pool, freed_pages); + /** + * Because changing page caching is costly + * we unlock the pool to prevent stalling. + */ + mtx_unlock(&pool->lock); + + ttm_pages_put(pages_to_free, freed_pages); + if (likely(nr_free != FREE_ALL_PAGES)) + nr_free -= freed_pages; + + if (NUM_PAGES_TO_ALLOC >= nr_free) + npages_to_free = nr_free; + else + npages_to_free = NUM_PAGES_TO_ALLOC; + + freed_pages = 0; + + /* free all so restart the processing */ + if (nr_free) + goto restart; + + /* Not allowed to fall through or break because + * following context is inside spinlock while we are + * outside here. + */ + goto out; + + } + } + + /* remove range of pages from the pool */ + if (freed_pages) { + TAILQ_REMOVE(&pool->list, p, pageq); + + ttm_pool_update_free_locked(pool, freed_pages); + nr_free -= freed_pages; + } + + mtx_unlock(&pool->lock); + + if (freed_pages) + ttm_pages_put(pages_to_free, freed_pages); +out: + free(pages_to_free, M_TEMP); + return nr_free; +} + +/* Get good estimation how many pages are free in pools */ +static int ttm_pool_get_num_unused_pages(void) +{ + unsigned i; + int total = 0; + for (i = 0; i < NUM_POOLS; ++i) + total += _manager->pools[i].npages; + + return total; +} + +/** + * Callback for mm to request pool to reduce number of page held. + */ +static int ttm_pool_mm_shrink(void *arg) +{ + static unsigned int start_pool = 0; + unsigned i; + unsigned pool_offset = atomic_fetchadd_int(&start_pool, 1); + struct ttm_page_pool *pool; + int shrink_pages = 100; /* XXXKIB */ + + pool_offset = pool_offset % NUM_POOLS; + /* select start pool in round robin fashion */ + for (i = 0; i < NUM_POOLS; ++i) { + unsigned nr_free = shrink_pages; + if (shrink_pages == 0) + break; + pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; + shrink_pages = ttm_page_pool_free(pool, nr_free); + } + /* return estimated number of unused pages in pool */ + return ttm_pool_get_num_unused_pages(); +} + +static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) +{ + + manager->lowmem_handler = EVENTHANDLER_REGISTER(vm_lowmem, + ttm_pool_mm_shrink, manager, EVENTHANDLER_PRI_ANY); +} + +static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) +{ + + EVENTHANDLER_DEREGISTER(vm_lowmem, manager->lowmem_handler); +} + +static int ttm_set_pages_caching(vm_page_t *pages, + enum ttm_caching_state cstate, unsigned cpages) +{ + int r = 0; + /* Set page caching */ + switch (cstate) { + case tt_uncached: + r = set_pages_array_uc(pages, cpages); + if (r) + printf("[TTM] Failed to set %d pages to uc!\n", cpages); + break; + case tt_wc: + r = set_pages_array_wc(pages, cpages); + if (r) + printf("[TTM] Failed to set %d pages to wc!\n", cpages); + break; + default: + break; + } + return r; +} + +/** + * Free pages the pages that failed to change the caching state. If there is + * any pages that have changed their caching state already put them to the + * pool. + */ +static void ttm_handle_caching_state_failure(struct pglist *pages, + int ttm_flags, enum ttm_caching_state cstate, + vm_page_t *failed_pages, unsigned cpages) +{ + unsigned i; + /* Failed pages have to be freed */ + for (i = 0; i < cpages; ++i) { + TAILQ_REMOVE(pages, failed_pages[i], pageq); + ttm_vm_page_free(failed_pages[i]); + } +} + +/** + * Allocate new pages with correct caching. + * + * This function is reentrant if caller updates count depending on number of + * pages returned in pages array. + */ +static int ttm_alloc_new_pages(struct pglist *pages, int ttm_alloc_flags, + int ttm_flags, enum ttm_caching_state cstate, unsigned count) +{ + vm_page_t *caching_array; + vm_page_t p; + int r = 0; + unsigned i, cpages, aflags; + unsigned max_cpages = min(count, + (unsigned)(PAGE_SIZE/sizeof(vm_page_t))); + + aflags = VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ | + ((ttm_alloc_flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0 ? + VM_ALLOC_ZERO : 0); + + /* allocate array for page caching change */ + caching_array = malloc(max_cpages * sizeof(vm_page_t), M_TEMP, + M_WAITOK | M_ZERO); + + for (i = 0, cpages = 0; i < count; ++i) { + p = vm_page_alloc_contig(NULL, 0, aflags, 1, 0, + (ttm_alloc_flags & TTM_PAGE_FLAG_DMA32) ? 0xffffffff : + VM_MAX_ADDRESS, PAGE_SIZE, 0, + ttm_caching_state_to_vm(cstate)); + if (!p) { + printf("[TTM] Unable to get page %u\n", i); + + /* store already allocated pages in the pool after + * setting the caching state */ + if (cpages) { + r = ttm_set_pages_caching(caching_array, + cstate, cpages); + if (r) + ttm_handle_caching_state_failure(pages, + ttm_flags, cstate, + caching_array, cpages); + } + r = -ENOMEM; + goto out; + } + p->oflags &= ~VPO_UNMANAGED; + p->flags |= PG_FICTITIOUS; + +#ifdef CONFIG_HIGHMEM /* KIB: nop */ + /* gfp flags of highmem page should never be dma32 so we + * we should be fine in such case + */ + if (!PageHighMem(p)) +#endif + { + caching_array[cpages++] = p; + if (cpages == max_cpages) { + + r = ttm_set_pages_caching(caching_array, + cstate, cpages); + if (r) { + ttm_handle_caching_state_failure(pages, + ttm_flags, cstate, + caching_array, cpages); + goto out; + } + cpages = 0; + } + } + + TAILQ_INSERT_HEAD(pages, p, pageq); + } + + if (cpages) { + r = ttm_set_pages_caching(caching_array, cstate, cpages); + if (r) + ttm_handle_caching_state_failure(pages, + ttm_flags, cstate, + caching_array, cpages); + } +out: + free(caching_array, M_TEMP); + + return r; +} + +/** + * Fill the given pool if there aren't enough pages and the requested number of + * pages is small. + */ +static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, + int ttm_flags, enum ttm_caching_state cstate, unsigned count) +{ + vm_page_t p; + int r; + unsigned cpages = 0; + /** + * Only allow one pool fill operation at a time. + * If pool doesn't have enough pages for the allocation new pages are + * allocated from outside of pool. + */ + if (pool->fill_lock) + return; + + pool->fill_lock = true; + + /* If allocation request is small and there are not enough + * pages in a pool we fill the pool up first. */ + if (count < _manager->options.small + && count > pool->npages) { + struct pglist new_pages; + unsigned alloc_size = _manager->options.alloc_size; + + /** + * Can't change page caching if in irqsave context. We have to + * drop the pool->lock. + */ + mtx_unlock(&pool->lock); + + TAILQ_INIT(&new_pages); + r = ttm_alloc_new_pages(&new_pages, pool->ttm_page_alloc_flags, + ttm_flags, cstate, alloc_size); + mtx_lock(&pool->lock); + + if (!r) { + TAILQ_CONCAT(&pool->list, &new_pages, pageq); + ++pool->nrefills; + pool->npages += alloc_size; + } else { + printf("[TTM] Failed to fill pool (%p)\n", pool); + /* If we have any pages left put them to the pool. */ + TAILQ_FOREACH(p, &pool->list, pageq) { + ++cpages; + } + TAILQ_CONCAT(&pool->list, &new_pages, pageq); + pool->npages += cpages; + } + + } + pool->fill_lock = false; +} + +/** + * Cut 'count' number of pages from the pool and put them on the return list. + * + * @return count of pages still required to fulfill the request. + */ +static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, + struct pglist *pages, + int ttm_flags, + enum ttm_caching_state cstate, + unsigned count) +{ + vm_page_t p; + unsigned i; + + mtx_lock(&pool->lock); + ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count); + + if (count >= pool->npages) { + /* take all pages from the pool */ + TAILQ_CONCAT(pages, &pool->list, pageq); + count -= pool->npages; + pool->npages = 0; + goto out; + } + for (i = 0; i < count; i++) { + p = TAILQ_FIRST(&pool->list); + TAILQ_REMOVE(&pool->list, p, pageq); + TAILQ_INSERT_TAIL(pages, p, pageq); + } + pool->npages -= count; + count = 0; +out: + mtx_unlock(&pool->lock); + return count; +} + +/* Put all pages in pages list to correct pool to wait for reuse */ +static void ttm_put_pages(vm_page_t *pages, unsigned npages, int flags, + enum ttm_caching_state cstate) +{ + struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); + unsigned i; + + if (pool == NULL) { + /* No pool for this memory type so free the pages */ + for (i = 0; i < npages; i++) { + if (pages[i]) { + ttm_vm_page_free(pages[i]); + pages[i] = NULL; + } + } + return; + } + + mtx_lock(&pool->lock); + for (i = 0; i < npages; i++) { + if (pages[i]) { + TAILQ_INSERT_TAIL(&pool->list, pages[i], pageq); + pages[i] = NULL; + pool->npages++; + } + } + /* Check that we don't go over the pool limit */ + npages = 0; + if (pool->npages > _manager->options.max_size) { + npages = pool->npages - _manager->options.max_size; + /* free at least NUM_PAGES_TO_ALLOC number of pages + * to reduce calls to set_memory_wb */ + if (npages < NUM_PAGES_TO_ALLOC) + npages = NUM_PAGES_TO_ALLOC; + } + mtx_unlock(&pool->lock); + if (npages) + ttm_page_pool_free(pool, npages); +} + +/* + * On success pages list will hold count number of correctly + * cached pages. + */ +static int ttm_get_pages(vm_page_t *pages, unsigned npages, int flags, + enum ttm_caching_state cstate) +{ + struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); + struct pglist plist; + vm_page_t p = NULL; + int gfp_flags, aflags; + unsigned count; + int r; + + aflags = VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | + ((flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0 ? VM_ALLOC_ZERO : 0); + + /* No pool for cached pages */ + if (pool == NULL) { + for (r = 0; r < npages; ++r) { + p = vm_page_alloc_contig(NULL, 0, aflags, 1, 0, + (flags & TTM_PAGE_FLAG_DMA32) ? 0xffffffff : + VM_MAX_ADDRESS, PAGE_SIZE, + 0, ttm_caching_state_to_vm(cstate)); + if (!p) { + printf("[TTM] Unable to allocate page\n"); + return -ENOMEM; + } + p->oflags &= ~VPO_UNMANAGED; + p->flags |= PG_FICTITIOUS; + pages[r] = p; + } + return 0; + } + + /* combine zero flag to pool flags */ + gfp_flags = flags | pool->ttm_page_alloc_flags; + + /* First we take pages from the pool */ + TAILQ_INIT(&plist); + npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); + count = 0; + TAILQ_FOREACH(p, &plist, pageq) { + pages[count++] = p; + } + + /* clear the pages coming from the pool if requested */ + if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { + TAILQ_FOREACH(p, &plist, pageq) { + pmap_zero_page(p); + } + } + + /* If pool didn't have enough pages allocate new one. */ + if (npages > 0) { + /* ttm_alloc_new_pages doesn't reference pool so we can run + * multiple requests in parallel. + **/ + TAILQ_INIT(&plist); + r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, + npages); + TAILQ_FOREACH(p, &plist, pageq) { + pages[count++] = p; + } + if (r) { + /* If there is any pages in the list put them back to + * the pool. */ + printf("[TTM] Failed to allocate extra pages for large request\n"); + ttm_put_pages(pages, count, flags, cstate); + return r; + } + } + + return 0; +} + +static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, + char *name) +{ + mtx_init(&pool->lock, "ttmpool", NULL, MTX_DEF); + pool->fill_lock = false; + TAILQ_INIT(&pool->list); + pool->npages = pool->nfrees = 0; + pool->ttm_page_alloc_flags = flags; + pool->name = name; +} + +int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) +{ + + if (_manager != NULL) + printf("[TTM] manager != NULL\n"); + printf("[TTM] Initializing pool allocator\n"); + + _manager = malloc(sizeof(*_manager), M_TTM_POOLMGR, M_WAITOK | M_ZERO); + + ttm_page_pool_init_locked(&_manager->wc_pool, 0, "wc"); + ttm_page_pool_init_locked(&_manager->uc_pool, 0, "uc"); + ttm_page_pool_init_locked(&_manager->wc_pool_dma32, + TTM_PAGE_FLAG_DMA32, "wc dma"); + ttm_page_pool_init_locked(&_manager->uc_pool_dma32, + TTM_PAGE_FLAG_DMA32, "uc dma"); + + _manager->options.max_size = max_pages; + _manager->options.small = SMALL_ALLOCATION; + _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; + + refcount_init(&_manager->kobj_ref, 1); + ttm_pool_mm_shrink_init(_manager); + + return 0; +} + +void ttm_page_alloc_fini(void) +{ + int i; + + printf("[TTM] Finalizing pool allocator\n"); + ttm_pool_mm_shrink_fini(_manager); + + for (i = 0; i < NUM_POOLS; ++i) + ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES); + + if (refcount_release(&_manager->kobj_ref)) + ttm_pool_kobj_release(_manager); + _manager = NULL; +} + +int ttm_pool_populate(struct ttm_tt *ttm) +{ + struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; + unsigned i; + int ret; + + if (ttm->state != tt_unpopulated) + return 0; + + for (i = 0; i < ttm->num_pages; ++i) { + ret = ttm_get_pages(&ttm->pages[i], 1, + ttm->page_flags, + ttm->caching_state); + if (ret != 0) { + ttm_pool_unpopulate(ttm); + return -ENOMEM; + } + + ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], + false, false); + if (unlikely(ret != 0)) { + ttm_pool_unpopulate(ttm); + return -ENOMEM; + } + } + + if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { + ret = ttm_tt_swapin(ttm); + if (unlikely(ret != 0)) { + ttm_pool_unpopulate(ttm); + return ret; + } + } + + ttm->state = tt_unbound; + return 0; +} + +void ttm_pool_unpopulate(struct ttm_tt *ttm) +{ + unsigned i; + + for (i = 0; i < ttm->num_pages; ++i) { + if (ttm->pages[i]) { + ttm_mem_global_free_page(ttm->glob->mem_glob, + ttm->pages[i]); + ttm_put_pages(&ttm->pages[i], 1, + ttm->page_flags, + ttm->caching_state); + } + } + ttm->state = tt_unpopulated; +} + +#if 0 +/* XXXKIB sysctl */ +int ttm_page_alloc_debugfs(struct seq_file *m, void *data) +{ + struct ttm_page_pool *p; + unsigned i; + char *h[] = {"pool", "refills", "pages freed", "size"}; + if (!_manager) { + seq_printf(m, "No pool allocator running.\n"); + return 0; + } + seq_printf(m, "%6s %12s %13s %8s\n", + h[0], h[1], h[2], h[3]); + for (i = 0; i < NUM_POOLS; ++i) { + p = &_manager->pools[i]; + + seq_printf(m, "%6s %12ld %13ld %8d\n", + p->name, p->nrefills, + p->nfrees, p->npages); + } + return 0; +} +#endif diff --git a/sys/dev/drm2/ttm/ttm_page_alloc.h b/sys/dev/drm2/ttm/ttm_page_alloc.h new file mode 100644 index 0000000..0824e2d --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_page_alloc.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) Red Hat Inc. + + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sub license, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie <airlied@redhat.com> + * Jerome Glisse <jglisse@redhat.com> + */ +/* $FreeBSD$ */ +#ifndef TTM_PAGE_ALLOC +#define TTM_PAGE_ALLOC + +#include <dev/drm2/ttm/ttm_bo_driver.h> +#include <dev/drm2/ttm/ttm_memory.h> + +/** + * Initialize pool allocator. + */ +int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages); +/** + * Free pool allocator. + */ +void ttm_page_alloc_fini(void); + +/** + * ttm_pool_populate: + * + * @ttm: The struct ttm_tt to contain the backing pages. + * + * Add backing pages to all of @ttm + */ +extern int ttm_pool_populate(struct ttm_tt *ttm); + +/** + * ttm_pool_unpopulate: + * + * @ttm: The struct ttm_tt which to free backing pages. + * + * Free all pages of @ttm + */ +extern void ttm_pool_unpopulate(struct ttm_tt *ttm); + +/** + * Output the state of pools to debugfs file + */ +/* XXXKIB +extern int ttm_page_alloc_debugfs(struct seq_file *m, void *data); +*/ + +#ifdef CONFIG_SWIOTLB +/** + * Initialize pool allocator. + */ +int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages); + +/** + * Free pool allocator. + */ +void ttm_dma_page_alloc_fini(void); + +/** + * Output the state of pools to debugfs file + */ +extern int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data); + +extern int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev); +extern void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev); + +#else +static inline int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, + unsigned max_pages) +{ + return -ENODEV; +} + +static inline void ttm_dma_page_alloc_fini(void) { return; } + +/* XXXKIB +static inline int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) +{ + return 0; +} +*/ +#endif + +#endif diff --git a/sys/dev/drm2/ttm/ttm_page_alloc_dma.c b/sys/dev/drm2/ttm/ttm_page_alloc_dma.c new file mode 100644 index 0000000..a5c6fed --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_page_alloc_dma.c @@ -0,0 +1,1134 @@ +/* + * Copyright 2011 (c) Oracle Corp. + + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sub license, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Author: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> + */ + +/* + * A simple DMA pool losely based on dmapool.c. It has certain advantages + * over the DMA pools: + * - Pool collects resently freed pages for reuse (and hooks up to + * the shrinker). + * - Tracks currently in use pages + * - Tracks whether the page is UC, WB or cached (and reverts to WB + * when freed). + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#define pr_fmt(fmt) "[TTM] " fmt + +#include <linux/dma-mapping.h> +#include <linux/list.h> +#include <linux/seq_file.h> /* for seq_printf */ +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/highmem.h> +#include <linux/mm_types.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/atomic.h> +#include <linux/device.h> +#include <linux/kthread.h> +#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_page_alloc.h> +#ifdef TTM_HAS_AGP +#include <asm/agp.h> +#endif + +#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) +#define SMALL_ALLOCATION 4 +#define FREE_ALL_PAGES (~0U) +/* times are in msecs */ +#define IS_UNDEFINED (0) +#define IS_WC (1<<1) +#define IS_UC (1<<2) +#define IS_CACHED (1<<3) +#define IS_DMA32 (1<<4) + +enum pool_type { + POOL_IS_UNDEFINED, + POOL_IS_WC = IS_WC, + POOL_IS_UC = IS_UC, + POOL_IS_CACHED = IS_CACHED, + POOL_IS_WC_DMA32 = IS_WC | IS_DMA32, + POOL_IS_UC_DMA32 = IS_UC | IS_DMA32, + POOL_IS_CACHED_DMA32 = IS_CACHED | IS_DMA32, +}; +/* + * The pool structure. There are usually six pools: + * - generic (not restricted to DMA32): + * - write combined, uncached, cached. + * - dma32 (up to 2^32 - so up 4GB): + * - write combined, uncached, cached. + * for each 'struct device'. The 'cached' is for pages that are actively used. + * The other ones can be shrunk by the shrinker API if neccessary. + * @pools: The 'struct device->dma_pools' link. + * @type: Type of the pool + * @lock: Protects the inuse_list and free_list from concurrnet access. Must be + * used with irqsave/irqrestore variants because pool allocator maybe called + * from delayed work. + * @inuse_list: Pool of pages that are in use. The order is very important and + * it is in the order that the TTM pages that are put back are in. + * @free_list: Pool of pages that are free to be used. No order requirements. + * @dev: The device that is associated with these pools. + * @size: Size used during DMA allocation. + * @npages_free: Count of available pages for re-use. + * @npages_in_use: Count of pages that are in use. + * @nfrees: Stats when pool is shrinking. + * @nrefills: Stats when the pool is grown. + * @gfp_flags: Flags to pass for alloc_page. + * @name: Name of the pool. + * @dev_name: Name derieved from dev - similar to how dev_info works. + * Used during shutdown as the dev_info during release is unavailable. + */ +struct dma_pool { + struct list_head pools; /* The 'struct device->dma_pools link */ + enum pool_type type; + spinlock_t lock; + struct list_head inuse_list; + struct list_head free_list; + struct device *dev; + unsigned size; + unsigned npages_free; + unsigned npages_in_use; + unsigned long nfrees; /* Stats when shrunk. */ + unsigned long nrefills; /* Stats when grown. */ + gfp_t gfp_flags; + char name[13]; /* "cached dma32" */ + char dev_name[64]; /* Constructed from dev */ +}; + +/* + * The accounting page keeping track of the allocated page along with + * the DMA address. + * @page_list: The link to the 'page_list' in 'struct dma_pool'. + * @vaddr: The virtual address of the page + * @dma: The bus address of the page. If the page is not allocated + * via the DMA API, it will be -1. + */ +struct dma_page { + struct list_head page_list; + void *vaddr; + struct page *p; + dma_addr_t dma; +}; + +/* + * Limits for the pool. They are handled without locks because only place where + * they may change is in sysfs store. They won't have immediate effect anyway + * so forcing serialization to access them is pointless. + */ + +struct ttm_pool_opts { + unsigned alloc_size; + unsigned max_size; + unsigned small; +}; + +/* + * Contains the list of all of the 'struct device' and their corresponding + * DMA pools. Guarded by _mutex->lock. + * @pools: The link to 'struct ttm_pool_manager->pools' + * @dev: The 'struct device' associated with the 'pool' + * @pool: The 'struct dma_pool' associated with the 'dev' + */ +struct device_pools { + struct list_head pools; + struct device *dev; + struct dma_pool *pool; +}; + +/* + * struct ttm_pool_manager - Holds memory pools for fast allocation + * + * @lock: Lock used when adding/removing from pools + * @pools: List of 'struct device' and 'struct dma_pool' tuples. + * @options: Limits for the pool. + * @npools: Total amount of pools in existence. + * @shrinker: The structure used by [un|]register_shrinker + */ +struct ttm_pool_manager { + struct mutex lock; + struct list_head pools; + struct ttm_pool_opts options; + unsigned npools; + struct shrinker mm_shrink; + struct kobject kobj; +}; + +static struct ttm_pool_manager *_manager; + +static struct attribute ttm_page_pool_max = { + .name = "pool_max_size", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_page_pool_small = { + .name = "pool_small_allocation", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_page_pool_alloc_size = { + .name = "pool_allocation_size", + .mode = S_IRUGO | S_IWUSR +}; + +static struct attribute *ttm_pool_attrs[] = { + &ttm_page_pool_max, + &ttm_page_pool_small, + &ttm_page_pool_alloc_size, + NULL +}; + +static void ttm_pool_kobj_release(struct kobject *kobj) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + kfree(m); +} + +static ssize_t ttm_pool_store(struct kobject *kobj, struct attribute *attr, + const char *buffer, size_t size) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + int chars; + unsigned val; + chars = sscanf(buffer, "%u", &val); + if (chars == 0) + return size; + + /* Convert kb to number of pages */ + val = val / (PAGE_SIZE >> 10); + + if (attr == &ttm_page_pool_max) + m->options.max_size = val; + else if (attr == &ttm_page_pool_small) + m->options.small = val; + else if (attr == &ttm_page_pool_alloc_size) { + if (val > NUM_PAGES_TO_ALLOC*8) { + pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + return size; + } else if (val > NUM_PAGES_TO_ALLOC) { + pr_warn("Setting allocation size to larger than %lu is not recommended\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + } + m->options.alloc_size = val; + } + + return size; +} + +static ssize_t ttm_pool_show(struct kobject *kobj, struct attribute *attr, + char *buffer) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + unsigned val = 0; + + if (attr == &ttm_page_pool_max) + val = m->options.max_size; + else if (attr == &ttm_page_pool_small) + val = m->options.small; + else if (attr == &ttm_page_pool_alloc_size) + val = m->options.alloc_size; + + val = val * (PAGE_SIZE >> 10); + + return snprintf(buffer, PAGE_SIZE, "%u\n", val); +} + +static const struct sysfs_ops ttm_pool_sysfs_ops = { + .show = &ttm_pool_show, + .store = &ttm_pool_store, +}; + +static struct kobj_type ttm_pool_kobj_type = { + .release = &ttm_pool_kobj_release, + .sysfs_ops = &ttm_pool_sysfs_ops, + .default_attrs = ttm_pool_attrs, +}; + +#ifndef CONFIG_X86 +static int set_pages_array_wb(struct page **pages, int addrinarray) +{ +#ifdef TTM_HAS_AGP + int i; + + for (i = 0; i < addrinarray; i++) + unmap_page_from_agp(pages[i]); +#endif + return 0; +} + +static int set_pages_array_wc(struct page **pages, int addrinarray) +{ +#ifdef TTM_HAS_AGP + int i; + + for (i = 0; i < addrinarray; i++) + map_page_into_agp(pages[i]); +#endif + return 0; +} + +static int set_pages_array_uc(struct page **pages, int addrinarray) +{ +#ifdef TTM_HAS_AGP + int i; + + for (i = 0; i < addrinarray; i++) + map_page_into_agp(pages[i]); +#endif + return 0; +} +#endif /* for !CONFIG_X86 */ + +static int ttm_set_pages_caching(struct dma_pool *pool, + struct page **pages, unsigned cpages) +{ + int r = 0; + /* Set page caching */ + if (pool->type & IS_UC) { + r = set_pages_array_uc(pages, cpages); + if (r) + pr_err("%s: Failed to set %d pages to uc!\n", + pool->dev_name, cpages); + } + if (pool->type & IS_WC) { + r = set_pages_array_wc(pages, cpages); + if (r) + pr_err("%s: Failed to set %d pages to wc!\n", + pool->dev_name, cpages); + } + return r; +} + +static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page) +{ + dma_addr_t dma = d_page->dma; + dma_free_coherent(pool->dev, pool->size, d_page->vaddr, dma); + + kfree(d_page); + d_page = NULL; +} +static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool) +{ + struct dma_page *d_page; + + d_page = kmalloc(sizeof(struct dma_page), GFP_KERNEL); + if (!d_page) + return NULL; + + d_page->vaddr = dma_alloc_coherent(pool->dev, pool->size, + &d_page->dma, + pool->gfp_flags); + if (d_page->vaddr) + d_page->p = virt_to_page(d_page->vaddr); + else { + kfree(d_page); + d_page = NULL; + } + return d_page; +} +static enum pool_type ttm_to_type(int flags, enum ttm_caching_state cstate) +{ + enum pool_type type = IS_UNDEFINED; + + if (flags & TTM_PAGE_FLAG_DMA32) + type |= IS_DMA32; + if (cstate == tt_cached) + type |= IS_CACHED; + else if (cstate == tt_uncached) + type |= IS_UC; + else + type |= IS_WC; + + return type; +} + +static void ttm_pool_update_free_locked(struct dma_pool *pool, + unsigned freed_pages) +{ + pool->npages_free -= freed_pages; + pool->nfrees += freed_pages; + +} + +/* set memory back to wb and free the pages. */ +static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages, + struct page *pages[], unsigned npages) +{ + struct dma_page *d_page, *tmp; + + /* Don't set WB on WB page pool. */ + if (npages && !(pool->type & IS_CACHED) && + set_pages_array_wb(pages, npages)) + pr_err("%s: Failed to set %d pages to wb!\n", + pool->dev_name, npages); + + list_for_each_entry_safe(d_page, tmp, d_pages, page_list) { + list_del(&d_page->page_list); + __ttm_dma_free_page(pool, d_page); + } +} + +static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page) +{ + /* Don't set WB on WB page pool. */ + if (!(pool->type & IS_CACHED) && set_pages_array_wb(&d_page->p, 1)) + pr_err("%s: Failed to set %d pages to wb!\n", + pool->dev_name, 1); + + list_del(&d_page->page_list); + __ttm_dma_free_page(pool, d_page); +} + +/* + * Free pages from pool. + * + * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC + * number of pages in one go. + * + * @pool: to free the pages from + * @nr_free: If set to true will free all pages in pool + **/ +static unsigned ttm_dma_page_pool_free(struct dma_pool *pool, unsigned nr_free) +{ + unsigned long irq_flags; + struct dma_page *dma_p, *tmp; + struct page **pages_to_free; + struct list_head d_pages; + unsigned freed_pages = 0, + npages_to_free = nr_free; + + if (NUM_PAGES_TO_ALLOC < nr_free) + npages_to_free = NUM_PAGES_TO_ALLOC; +#if 0 + if (nr_free > 1) { + pr_debug("%s: (%s:%d) Attempting to free %d (%d) pages\n", + pool->dev_name, pool->name, current->pid, + npages_to_free, nr_free); + } +#endif + pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), + GFP_KERNEL); + + if (!pages_to_free) { + pr_err("%s: Failed to allocate memory for pool free operation\n", + pool->dev_name); + return 0; + } + INIT_LIST_HEAD(&d_pages); +restart: + spin_lock_irqsave(&pool->lock, irq_flags); + + /* We picking the oldest ones off the list */ + list_for_each_entry_safe_reverse(dma_p, tmp, &pool->free_list, + page_list) { + if (freed_pages >= npages_to_free) + break; + + /* Move the dma_page from one list to another. */ + list_move(&dma_p->page_list, &d_pages); + + pages_to_free[freed_pages++] = dma_p->p; + /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ + if (freed_pages >= NUM_PAGES_TO_ALLOC) { + + ttm_pool_update_free_locked(pool, freed_pages); + /** + * Because changing page caching is costly + * we unlock the pool to prevent stalling. + */ + spin_unlock_irqrestore(&pool->lock, irq_flags); + + ttm_dma_pages_put(pool, &d_pages, pages_to_free, + freed_pages); + + INIT_LIST_HEAD(&d_pages); + + if (likely(nr_free != FREE_ALL_PAGES)) + nr_free -= freed_pages; + + if (NUM_PAGES_TO_ALLOC >= nr_free) + npages_to_free = nr_free; + else + npages_to_free = NUM_PAGES_TO_ALLOC; + + freed_pages = 0; + + /* free all so restart the processing */ + if (nr_free) + goto restart; + + /* Not allowed to fall through or break because + * following context is inside spinlock while we are + * outside here. + */ + goto out; + + } + } + + /* remove range of pages from the pool */ + if (freed_pages) { + ttm_pool_update_free_locked(pool, freed_pages); + nr_free -= freed_pages; + } + + spin_unlock_irqrestore(&pool->lock, irq_flags); + + if (freed_pages) + ttm_dma_pages_put(pool, &d_pages, pages_to_free, freed_pages); +out: + kfree(pages_to_free); + return nr_free; +} + +static void ttm_dma_free_pool(struct device *dev, enum pool_type type) +{ + struct device_pools *p; + struct dma_pool *pool; + + if (!dev) + return; + + mutex_lock(&_manager->lock); + list_for_each_entry_reverse(p, &_manager->pools, pools) { + if (p->dev != dev) + continue; + pool = p->pool; + if (pool->type != type) + continue; + + list_del(&p->pools); + kfree(p); + _manager->npools--; + break; + } + list_for_each_entry_reverse(pool, &dev->dma_pools, pools) { + if (pool->type != type) + continue; + /* Takes a spinlock.. */ + ttm_dma_page_pool_free(pool, FREE_ALL_PAGES); + WARN_ON(((pool->npages_in_use + pool->npages_free) != 0)); + /* This code path is called after _all_ references to the + * struct device has been dropped - so nobody should be + * touching it. In case somebody is trying to _add_ we are + * guarded by the mutex. */ + list_del(&pool->pools); + kfree(pool); + break; + } + mutex_unlock(&_manager->lock); +} + +/* + * On free-ing of the 'struct device' this deconstructor is run. + * Albeit the pool might have already been freed earlier. + */ +static void ttm_dma_pool_release(struct device *dev, void *res) +{ + struct dma_pool *pool = *(struct dma_pool **)res; + + if (pool) + ttm_dma_free_pool(dev, pool->type); +} + +static int ttm_dma_pool_match(struct device *dev, void *res, void *match_data) +{ + return *(struct dma_pool **)res == match_data; +} + +static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags, + enum pool_type type) +{ + char *n[] = {"wc", "uc", "cached", " dma32", "unknown",}; + enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_UNDEFINED}; + struct device_pools *sec_pool = NULL; + struct dma_pool *pool = NULL, **ptr; + unsigned i; + int ret = -ENODEV; + char *p; + + if (!dev) + return NULL; + + ptr = devres_alloc(ttm_dma_pool_release, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return NULL; + + ret = -ENOMEM; + + pool = kmalloc_node(sizeof(struct dma_pool), GFP_KERNEL, + dev_to_node(dev)); + if (!pool) + goto err_mem; + + sec_pool = kmalloc_node(sizeof(struct device_pools), GFP_KERNEL, + dev_to_node(dev)); + if (!sec_pool) + goto err_mem; + + INIT_LIST_HEAD(&sec_pool->pools); + sec_pool->dev = dev; + sec_pool->pool = pool; + + INIT_LIST_HEAD(&pool->free_list); + INIT_LIST_HEAD(&pool->inuse_list); + INIT_LIST_HEAD(&pool->pools); + spin_lock_init(&pool->lock); + pool->dev = dev; + pool->npages_free = pool->npages_in_use = 0; + pool->nfrees = 0; + pool->gfp_flags = flags; + pool->size = PAGE_SIZE; + pool->type = type; + pool->nrefills = 0; + p = pool->name; + for (i = 0; i < 5; i++) { + if (type & t[i]) { + p += snprintf(p, sizeof(pool->name) - (p - pool->name), + "%s", n[i]); + } + } + *p = 0; + /* We copy the name for pr_ calls b/c when dma_pool_destroy is called + * - the kobj->name has already been deallocated.*/ + snprintf(pool->dev_name, sizeof(pool->dev_name), "%s %s", + dev_driver_string(dev), dev_name(dev)); + mutex_lock(&_manager->lock); + /* You can get the dma_pool from either the global: */ + list_add(&sec_pool->pools, &_manager->pools); + _manager->npools++; + /* or from 'struct device': */ + list_add(&pool->pools, &dev->dma_pools); + mutex_unlock(&_manager->lock); + + *ptr = pool; + devres_add(dev, ptr); + + return pool; +err_mem: + devres_free(ptr); + kfree(sec_pool); + kfree(pool); + return ERR_PTR(ret); +} + +static struct dma_pool *ttm_dma_find_pool(struct device *dev, + enum pool_type type) +{ + struct dma_pool *pool, *tmp, *found = NULL; + + if (type == IS_UNDEFINED) + return found; + + /* NB: We iterate on the 'struct dev' which has no spinlock, but + * it does have a kref which we have taken. The kref is taken during + * graphic driver loading - in the drm_pci_init it calls either + * pci_dev_get or pci_register_driver which both end up taking a kref + * on 'struct device'. + * + * On teardown, the graphic drivers end up quiescing the TTM (put_pages) + * and calls the dev_res deconstructors: ttm_dma_pool_release. The nice + * thing is at that point of time there are no pages associated with the + * driver so this function will not be called. + */ + list_for_each_entry_safe(pool, tmp, &dev->dma_pools, pools) { + if (pool->type != type) + continue; + found = pool; + break; + } + return found; +} + +/* + * Free pages the pages that failed to change the caching state. If there + * are pages that have changed their caching state already put them to the + * pool. + */ +static void ttm_dma_handle_caching_state_failure(struct dma_pool *pool, + struct list_head *d_pages, + struct page **failed_pages, + unsigned cpages) +{ + struct dma_page *d_page, *tmp; + struct page *p; + unsigned i = 0; + + p = failed_pages[0]; + if (!p) + return; + /* Find the failed page. */ + list_for_each_entry_safe(d_page, tmp, d_pages, page_list) { + if (d_page->p != p) + continue; + /* .. and then progress over the full list. */ + list_del(&d_page->page_list); + __ttm_dma_free_page(pool, d_page); + if (++i < cpages) + p = failed_pages[i]; + else + break; + } + +} + +/* + * Allocate 'count' pages, and put 'need' number of them on the + * 'pages' and as well on the 'dma_address' starting at 'dma_offset' offset. + * The full list of pages should also be on 'd_pages'. + * We return zero for success, and negative numbers as errors. + */ +static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool, + struct list_head *d_pages, + unsigned count) +{ + struct page **caching_array; + struct dma_page *dma_p; + struct page *p; + int r = 0; + unsigned i, cpages; + unsigned max_cpages = min(count, + (unsigned)(PAGE_SIZE/sizeof(struct page *))); + + /* allocate array for page caching change */ + caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); + + if (!caching_array) { + pr_err("%s: Unable to allocate table for new pages\n", + pool->dev_name); + return -ENOMEM; + } + + if (count > 1) { + pr_debug("%s: (%s:%d) Getting %d pages\n", + pool->dev_name, pool->name, current->pid, count); + } + + for (i = 0, cpages = 0; i < count; ++i) { + dma_p = __ttm_dma_alloc_page(pool); + if (!dma_p) { + pr_err("%s: Unable to get page %u\n", + pool->dev_name, i); + + /* store already allocated pages in the pool after + * setting the caching state */ + if (cpages) { + r = ttm_set_pages_caching(pool, caching_array, + cpages); + if (r) + ttm_dma_handle_caching_state_failure( + pool, d_pages, caching_array, + cpages); + } + r = -ENOMEM; + goto out; + } + p = dma_p->p; +#ifdef CONFIG_HIGHMEM + /* gfp flags of highmem page should never be dma32 so we + * we should be fine in such case + */ + if (!PageHighMem(p)) +#endif + { + caching_array[cpages++] = p; + if (cpages == max_cpages) { + /* Note: Cannot hold the spinlock */ + r = ttm_set_pages_caching(pool, caching_array, + cpages); + if (r) { + ttm_dma_handle_caching_state_failure( + pool, d_pages, caching_array, + cpages); + goto out; + } + cpages = 0; + } + } + list_add(&dma_p->page_list, d_pages); + } + + if (cpages) { + r = ttm_set_pages_caching(pool, caching_array, cpages); + if (r) + ttm_dma_handle_caching_state_failure(pool, d_pages, + caching_array, cpages); + } +out: + kfree(caching_array); + return r; +} + +/* + * @return count of pages still required to fulfill the request. + */ +static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool, + unsigned long *irq_flags) +{ + unsigned count = _manager->options.small; + int r = pool->npages_free; + + if (count > pool->npages_free) { + struct list_head d_pages; + + INIT_LIST_HEAD(&d_pages); + + spin_unlock_irqrestore(&pool->lock, *irq_flags); + + /* Returns how many more are neccessary to fulfill the + * request. */ + r = ttm_dma_pool_alloc_new_pages(pool, &d_pages, count); + + spin_lock_irqsave(&pool->lock, *irq_flags); + if (!r) { + /* Add the fresh to the end.. */ + list_splice(&d_pages, &pool->free_list); + ++pool->nrefills; + pool->npages_free += count; + r = count; + } else { + struct dma_page *d_page; + unsigned cpages = 0; + + pr_err("%s: Failed to fill %s pool (r:%d)!\n", + pool->dev_name, pool->name, r); + + list_for_each_entry(d_page, &d_pages, page_list) { + cpages++; + } + list_splice_tail(&d_pages, &pool->free_list); + pool->npages_free += cpages; + r = cpages; + } + } + return r; +} + +/* + * @return count of pages still required to fulfill the request. + * The populate list is actually a stack (not that is matters as TTM + * allocates one page at a time. + */ +static int ttm_dma_pool_get_pages(struct dma_pool *pool, + struct ttm_dma_tt *ttm_dma, + unsigned index) +{ + struct dma_page *d_page; + struct ttm_tt *ttm = &ttm_dma->ttm; + unsigned long irq_flags; + int count, r = -ENOMEM; + + spin_lock_irqsave(&pool->lock, irq_flags); + count = ttm_dma_page_pool_fill_locked(pool, &irq_flags); + if (count) { + d_page = list_first_entry(&pool->free_list, struct dma_page, page_list); + ttm->pages[index] = d_page->p; + ttm_dma->dma_address[index] = d_page->dma; + list_move_tail(&d_page->page_list, &ttm_dma->pages_list); + r = 0; + pool->npages_in_use += 1; + pool->npages_free -= 1; + } + spin_unlock_irqrestore(&pool->lock, irq_flags); + return r; +} + +/* + * On success pages list will hold count number of correctly + * cached pages. On failure will hold the negative return value (-ENOMEM, etc). + */ +int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; + struct dma_pool *pool; + enum pool_type type; + unsigned i; + gfp_t gfp_flags; + int ret; + + if (ttm->state != tt_unpopulated) + return 0; + + type = ttm_to_type(ttm->page_flags, ttm->caching_state); + if (ttm->page_flags & TTM_PAGE_FLAG_DMA32) + gfp_flags = GFP_USER | GFP_DMA32; + else + gfp_flags = GFP_HIGHUSER; + if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + + pool = ttm_dma_find_pool(dev, type); + if (!pool) { + pool = ttm_dma_pool_init(dev, gfp_flags, type); + if (IS_ERR_OR_NULL(pool)) { + return -ENOMEM; + } + } + + INIT_LIST_HEAD(&ttm_dma->pages_list); + for (i = 0; i < ttm->num_pages; ++i) { + ret = ttm_dma_pool_get_pages(pool, ttm_dma, i); + if (ret != 0) { + ttm_dma_unpopulate(ttm_dma, dev); + return -ENOMEM; + } + + ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], + false, false); + if (unlikely(ret != 0)) { + ttm_dma_unpopulate(ttm_dma, dev); + return -ENOMEM; + } + } + + if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { + ret = ttm_tt_swapin(ttm); + if (unlikely(ret != 0)) { + ttm_dma_unpopulate(ttm_dma, dev); + return ret; + } + } + + ttm->state = tt_unbound; + return 0; +} +EXPORT_SYMBOL_GPL(ttm_dma_populate); + +/* Get good estimation how many pages are free in pools */ +static int ttm_dma_pool_get_num_unused_pages(void) +{ + struct device_pools *p; + unsigned total = 0; + + mutex_lock(&_manager->lock); + list_for_each_entry(p, &_manager->pools, pools) + total += p->pool->npages_free; + mutex_unlock(&_manager->lock); + return total; +} + +/* Put all pages in pages list to correct pool to wait for reuse */ +void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + struct dma_pool *pool; + struct dma_page *d_page, *next; + enum pool_type type; + bool is_cached = false; + unsigned count = 0, i, npages = 0; + unsigned long irq_flags; + + type = ttm_to_type(ttm->page_flags, ttm->caching_state); + pool = ttm_dma_find_pool(dev, type); + if (!pool) + return; + + is_cached = (ttm_dma_find_pool(pool->dev, + ttm_to_type(ttm->page_flags, tt_cached)) == pool); + + /* make sure pages array match list and count number of pages */ + list_for_each_entry(d_page, &ttm_dma->pages_list, page_list) { + ttm->pages[count] = d_page->p; + count++; + } + + spin_lock_irqsave(&pool->lock, irq_flags); + pool->npages_in_use -= count; + if (is_cached) { + pool->nfrees += count; + } else { + pool->npages_free += count; + list_splice(&ttm_dma->pages_list, &pool->free_list); + npages = count; + if (pool->npages_free > _manager->options.max_size) { + npages = pool->npages_free - _manager->options.max_size; + /* free at least NUM_PAGES_TO_ALLOC number of pages + * to reduce calls to set_memory_wb */ + if (npages < NUM_PAGES_TO_ALLOC) + npages = NUM_PAGES_TO_ALLOC; + } + } + spin_unlock_irqrestore(&pool->lock, irq_flags); + + if (is_cached) { + list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, page_list) { + ttm_mem_global_free_page(ttm->glob->mem_glob, + d_page->p); + ttm_dma_page_put(pool, d_page); + } + } else { + for (i = 0; i < count; i++) { + ttm_mem_global_free_page(ttm->glob->mem_glob, + ttm->pages[i]); + } + } + + INIT_LIST_HEAD(&ttm_dma->pages_list); + for (i = 0; i < ttm->num_pages; i++) { + ttm->pages[i] = NULL; + ttm_dma->dma_address[i] = 0; + } + + /* shrink pool if necessary (only on !is_cached pools)*/ + if (npages) + ttm_dma_page_pool_free(pool, npages); + ttm->state = tt_unpopulated; +} +EXPORT_SYMBOL_GPL(ttm_dma_unpopulate); + +/** + * Callback for mm to request pool to reduce number of page held. + */ +static int ttm_dma_pool_mm_shrink(struct shrinker *shrink, + struct shrink_control *sc) +{ + static atomic_t start_pool = ATOMIC_INIT(0); + unsigned idx = 0; + unsigned pool_offset = atomic_add_return(1, &start_pool); + unsigned shrink_pages = sc->nr_to_scan; + struct device_pools *p; + + if (list_empty(&_manager->pools)) + return 0; + + mutex_lock(&_manager->lock); + pool_offset = pool_offset % _manager->npools; + list_for_each_entry(p, &_manager->pools, pools) { + unsigned nr_free; + + if (!p->dev) + continue; + if (shrink_pages == 0) + break; + /* Do it in round-robin fashion. */ + if (++idx < pool_offset) + continue; + nr_free = shrink_pages; + shrink_pages = ttm_dma_page_pool_free(p->pool, nr_free); + pr_debug("%s: (%s:%d) Asked to shrink %d, have %d more to go\n", + p->pool->dev_name, p->pool->name, current->pid, + nr_free, shrink_pages); + } + mutex_unlock(&_manager->lock); + /* return estimated number of unused pages in pool */ + return ttm_dma_pool_get_num_unused_pages(); +} + +static void ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager) +{ + manager->mm_shrink.shrink = &ttm_dma_pool_mm_shrink; + manager->mm_shrink.seeks = 1; + register_shrinker(&manager->mm_shrink); +} + +static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager) +{ + unregister_shrinker(&manager->mm_shrink); +} + +int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) +{ + int ret = -ENOMEM; + + WARN_ON(_manager); + + pr_info("Initializing DMA pool allocator\n"); + + _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); + if (!_manager) + goto err; + + mutex_init(&_manager->lock); + INIT_LIST_HEAD(&_manager->pools); + + _manager->options.max_size = max_pages; + _manager->options.small = SMALL_ALLOCATION; + _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; + + /* This takes care of auto-freeing the _manager */ + ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, + &glob->kobj, "dma_pool"); + if (unlikely(ret != 0)) { + kobject_put(&_manager->kobj); + goto err; + } + ttm_dma_pool_mm_shrink_init(_manager); + return 0; +err: + return ret; +} + +void ttm_dma_page_alloc_fini(void) +{ + struct device_pools *p, *t; + + pr_info("Finalizing DMA pool allocator\n"); + ttm_dma_pool_mm_shrink_fini(_manager); + + list_for_each_entry_safe_reverse(p, t, &_manager->pools, pools) { + dev_dbg(p->dev, "(%s:%d) Freeing.\n", p->pool->name, + current->pid); + WARN_ON(devres_destroy(p->dev, ttm_dma_pool_release, + ttm_dma_pool_match, p->pool)); + ttm_dma_free_pool(p->dev, p->pool->type); + } + kobject_put(&_manager->kobj); + _manager = NULL; +} + +int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) +{ + struct device_pools *p; + struct dma_pool *pool = NULL; + char *h[] = {"pool", "refills", "pages freed", "inuse", "available", + "name", "virt", "busaddr"}; + + if (!_manager) { + seq_printf(m, "No pool allocator running.\n"); + return 0; + } + seq_printf(m, "%13s %12s %13s %8s %8s %8s\n", + h[0], h[1], h[2], h[3], h[4], h[5]); + mutex_lock(&_manager->lock); + list_for_each_entry(p, &_manager->pools, pools) { + struct device *dev = p->dev; + if (!dev) + continue; + pool = p->pool; + seq_printf(m, "%13s %12ld %13ld %8d %8d %8s\n", + pool->name, pool->nrefills, + pool->nfrees, pool->npages_in_use, + pool->npages_free, + pool->dev_name); + } + mutex_unlock(&_manager->lock); + return 0; +} +EXPORT_SYMBOL_GPL(ttm_dma_page_alloc_debugfs); diff --git a/sys/dev/drm2/ttm/ttm_placement.h b/sys/dev/drm2/ttm/ttm_placement.h new file mode 100644 index 0000000..20e1b4c --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_placement.h @@ -0,0 +1,93 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ +/* $FreeBSD$ */ + +#ifndef _TTM_PLACEMENT_H_ +#define _TTM_PLACEMENT_H_ +/* + * Memory regions for data placement. + */ + +#define TTM_PL_SYSTEM 0 +#define TTM_PL_TT 1 +#define TTM_PL_VRAM 2 +#define TTM_PL_PRIV0 3 +#define TTM_PL_PRIV1 4 +#define TTM_PL_PRIV2 5 +#define TTM_PL_PRIV3 6 +#define TTM_PL_PRIV4 7 +#define TTM_PL_PRIV5 8 +#define TTM_PL_SWAPPED 15 + +#define TTM_PL_FLAG_SYSTEM (1 << TTM_PL_SYSTEM) +#define TTM_PL_FLAG_TT (1 << TTM_PL_TT) +#define TTM_PL_FLAG_VRAM (1 << TTM_PL_VRAM) +#define TTM_PL_FLAG_PRIV0 (1 << TTM_PL_PRIV0) +#define TTM_PL_FLAG_PRIV1 (1 << TTM_PL_PRIV1) +#define TTM_PL_FLAG_PRIV2 (1 << TTM_PL_PRIV2) +#define TTM_PL_FLAG_PRIV3 (1 << TTM_PL_PRIV3) +#define TTM_PL_FLAG_PRIV4 (1 << TTM_PL_PRIV4) +#define TTM_PL_FLAG_PRIV5 (1 << TTM_PL_PRIV5) +#define TTM_PL_FLAG_SWAPPED (1 << TTM_PL_SWAPPED) +#define TTM_PL_MASK_MEM 0x0000FFFF + +/* + * Other flags that affects data placement. + * TTM_PL_FLAG_CACHED indicates cache-coherent mappings + * if available. + * TTM_PL_FLAG_SHARED means that another application may + * reference the buffer. + * TTM_PL_FLAG_NO_EVICT means that the buffer may never + * be evicted to make room for other buffers. + */ + +#define TTM_PL_FLAG_CACHED (1 << 16) +#define TTM_PL_FLAG_UNCACHED (1 << 17) +#define TTM_PL_FLAG_WC (1 << 18) +#define TTM_PL_FLAG_SHARED (1 << 20) +#define TTM_PL_FLAG_NO_EVICT (1 << 21) + +#define TTM_PL_MASK_CACHING (TTM_PL_FLAG_CACHED | \ + TTM_PL_FLAG_UNCACHED | \ + TTM_PL_FLAG_WC) + +#define TTM_PL_MASK_MEMTYPE (TTM_PL_MASK_MEM | TTM_PL_MASK_CACHING) + +/* + * Access flags to be used for CPU- and GPU- mappings. + * The idea is that the TTM synchronization mechanism will + * allow concurrent READ access and exclusive write access. + * Currently GPU- and CPU accesses are exclusive. + */ + +#define TTM_ACCESS_READ (1 << 0) +#define TTM_ACCESS_WRITE (1 << 1) + +#endif diff --git a/sys/dev/drm2/ttm/ttm_tt.c b/sys/dev/drm2/ttm/ttm_tt.c new file mode 100644 index 0000000..82547f1 --- /dev/null +++ b/sys/dev/drm2/ttm/ttm_tt.c @@ -0,0 +1,370 @@ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ +/* + * Copyright (c) 2013 The FreeBSD Foundation + * All rights reserved. + * + * Portions of this software were developed by Konstantin Belousov + * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <dev/drm2/drmP.h> +#include <dev/drm2/ttm/ttm_module.h> +#include <dev/drm2/ttm/ttm_bo_driver.h> +#include <dev/drm2/ttm/ttm_placement.h> +#include <dev/drm2/ttm/ttm_page_alloc.h> + +MALLOC_DEFINE(M_TTM_PD, "ttm_pd", "TTM Page Directories"); + +/** + * Allocates storage for pointers to the pages that back the ttm. + */ +static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) +{ + ttm->pages = malloc(ttm->num_pages * sizeof(void *), + M_TTM_PD, M_WAITOK | M_ZERO); +} + +static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) +{ + ttm->ttm.pages = malloc(ttm->ttm.num_pages * sizeof(void *), + M_TTM_PD, M_WAITOK | M_ZERO); + ttm->dma_address = malloc(ttm->ttm.num_pages * + sizeof(*ttm->dma_address), M_TTM_PD, M_WAITOK); +} + +#if defined(__i386__) || defined(__amd64__) +static inline int ttm_tt_set_page_caching(vm_page_t p, + enum ttm_caching_state c_old, + enum ttm_caching_state c_new) +{ + + /* XXXKIB our VM does not need this. */ +#if 0 + if (c_old != tt_cached) { + /* p isn't in the default caching state, set it to + * writeback first to free its current memtype. */ + pmap_page_set_memattr(p, VM_MEMATTR_WRITE_BACK); + } +#endif + + if (c_new == tt_wc) + pmap_page_set_memattr(p, VM_MEMATTR_WRITE_COMBINING); + else if (c_new == tt_uncached) + pmap_page_set_memattr(p, VM_MEMATTR_UNCACHEABLE); + + return (0); +} +#else +static inline int ttm_tt_set_page_caching(vm_page_t p, + enum ttm_caching_state c_old, + enum ttm_caching_state c_new) +{ + return 0; +} +#endif + +/* + * Change caching policy for the linear kernel map + * for range of pages in a ttm. + */ + +static int ttm_tt_set_caching(struct ttm_tt *ttm, + enum ttm_caching_state c_state) +{ + int i, j; + vm_page_t cur_page; + int ret; + + if (ttm->caching_state == c_state) + return 0; + + if (ttm->state == tt_unpopulated) { + /* Change caching but don't populate */ + ttm->caching_state = c_state; + return 0; + } + + if (ttm->caching_state == tt_cached) + drm_clflush_pages(ttm->pages, ttm->num_pages); + + for (i = 0; i < ttm->num_pages; ++i) { + cur_page = ttm->pages[i]; + if (likely(cur_page != NULL)) { + ret = ttm_tt_set_page_caching(cur_page, + ttm->caching_state, + c_state); + if (unlikely(ret != 0)) + goto out_err; + } + } + + ttm->caching_state = c_state; + + return 0; + +out_err: + for (j = 0; j < i; ++j) { + cur_page = ttm->pages[j]; + if (cur_page != NULL) { + (void)ttm_tt_set_page_caching(cur_page, c_state, + ttm->caching_state); + } + } + + return ret; +} + +int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) +{ + enum ttm_caching_state state; + + if (placement & TTM_PL_FLAG_WC) + state = tt_wc; + else if (placement & TTM_PL_FLAG_UNCACHED) + state = tt_uncached; + else + state = tt_cached; + + return ttm_tt_set_caching(ttm, state); +} + +void ttm_tt_destroy(struct ttm_tt *ttm) +{ + if (unlikely(ttm == NULL)) + return; + + if (ttm->state == tt_bound) { + ttm_tt_unbind(ttm); + } + + if (likely(ttm->pages != NULL)) { + ttm->bdev->driver->ttm_tt_unpopulate(ttm); + } + + if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && + ttm->swap_storage) + vm_object_deallocate(ttm->swap_storage); + + ttm->swap_storage = NULL; + ttm->func->destroy(ttm); +} + +int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, + unsigned long size, uint32_t page_flags, + vm_page_t dummy_read_page) +{ + ttm->bdev = bdev; + ttm->glob = bdev->glob; + ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; + ttm->caching_state = tt_cached; + ttm->page_flags = page_flags; + ttm->dummy_read_page = dummy_read_page; + ttm->state = tt_unpopulated; + ttm->swap_storage = NULL; + + ttm_tt_alloc_page_directory(ttm); + if (!ttm->pages) { + ttm_tt_destroy(ttm); + printf("Failed allocating page table\n"); + return -ENOMEM; + } + return 0; +} + +void ttm_tt_fini(struct ttm_tt *ttm) +{ + free(ttm->pages, M_TTM_PD); + ttm->pages = NULL; +} + +int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, + unsigned long size, uint32_t page_flags, + vm_page_t dummy_read_page) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + + ttm->bdev = bdev; + ttm->glob = bdev->glob; + ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; + ttm->caching_state = tt_cached; + ttm->page_flags = page_flags; + ttm->dummy_read_page = dummy_read_page; + ttm->state = tt_unpopulated; + ttm->swap_storage = NULL; + + INIT_LIST_HEAD(&ttm_dma->pages_list); + ttm_dma_tt_alloc_page_directory(ttm_dma); + if (!ttm->pages || !ttm_dma->dma_address) { + ttm_tt_destroy(ttm); + printf("Failed allocating page table\n"); + return -ENOMEM; + } + return 0; +} + +void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + + free(ttm->pages, M_TTM_PD); + ttm->pages = NULL; + free(ttm_dma->dma_address, M_TTM_PD); + ttm_dma->dma_address = NULL; +} + +void ttm_tt_unbind(struct ttm_tt *ttm) +{ + int ret; + + if (ttm->state == tt_bound) { + ret = ttm->func->unbind(ttm); + MPASS(ret == 0); + ttm->state = tt_unbound; + } +} + +int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) +{ + int ret = 0; + + if (!ttm) + return -EINVAL; + + if (ttm->state == tt_bound) + return 0; + + ret = ttm->bdev->driver->ttm_tt_populate(ttm); + if (ret) + return ret; + + ret = ttm->func->bind(ttm, bo_mem); + if (unlikely(ret != 0)) + return ret; + + ttm->state = tt_bound; + + return 0; +} + +int ttm_tt_swapin(struct ttm_tt *ttm) +{ + vm_object_t obj; + vm_page_t from_page, to_page; + int i, ret, rv; + + obj = ttm->swap_storage; + + VM_OBJECT_LOCK(obj); + vm_object_pip_add(obj, 1); + for (i = 0; i < ttm->num_pages; ++i) { + from_page = vm_page_grab(obj, i, VM_ALLOC_RETRY); + if (from_page->valid != VM_PAGE_BITS_ALL) { + if (vm_pager_has_page(obj, i, NULL, NULL)) { + rv = vm_pager_get_pages(obj, &from_page, 1, 0); + if (rv != VM_PAGER_OK) { + vm_page_lock(from_page); + vm_page_free(from_page); + vm_page_unlock(from_page); + ret = -EIO; + goto err_ret; + } + } else + vm_page_zero_invalid(from_page, TRUE); + } + to_page = ttm->pages[i]; + if (unlikely(to_page == NULL)) { + vm_page_wakeup(from_page); + ret = -ENOMEM; + goto err_ret; + } + pmap_copy_page(from_page, to_page); + vm_page_wakeup(from_page); + } + vm_object_pip_wakeup(obj); + VM_OBJECT_UNLOCK(obj); + + if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP)) + vm_object_deallocate(obj); + ttm->swap_storage = NULL; + ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; + return (0); + +err_ret: + vm_object_pip_wakeup(obj); + VM_OBJECT_UNLOCK(obj); + return (ret); +} + +int ttm_tt_swapout(struct ttm_tt *ttm, vm_object_t persistent_swap_storage) +{ + vm_object_t obj; + vm_page_t from_page, to_page; + int i; + + MPASS(ttm->state == tt_unbound || ttm->state == tt_unpopulated); + MPASS(ttm->caching_state == tt_cached); + + if (persistent_swap_storage == NULL) { + obj = vm_pager_allocate(OBJT_SWAP, NULL, + IDX_TO_OFF(ttm->num_pages), VM_PROT_DEFAULT, 0, + curthread->td_ucred); + if (obj == NULL) { + printf("[TTM] Failed allocating swap storage\n"); + return (-ENOMEM); + } + } else + obj = persistent_swap_storage; + + VM_OBJECT_LOCK(obj); + vm_object_pip_add(obj, 1); + for (i = 0; i < ttm->num_pages; ++i) { + from_page = ttm->pages[i]; + if (unlikely(from_page == NULL)) + continue; + to_page = vm_page_grab(obj, i, VM_ALLOC_RETRY); + pmap_copy_page(from_page, to_page); + vm_page_dirty(to_page); + to_page->valid = VM_PAGE_BITS_ALL; + vm_page_wakeup(to_page); + } + vm_object_pip_wakeup(obj); + VM_OBJECT_UNLOCK(obj); + + ttm->bdev->driver->ttm_tt_unpopulate(ttm); + ttm->swap_storage = obj; + ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; + if (persistent_swap_storage != NULL) + ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP; + return (0); +} diff --git a/sys/dev/hwpmc/hwpmc_soft.c b/sys/dev/hwpmc/hwpmc_soft.c index 48e297e..56af775 100644 --- a/sys/dev/hwpmc/hwpmc_soft.c +++ b/sys/dev/hwpmc/hwpmc_soft.c @@ -45,8 +45,6 @@ __FBSDID("$FreeBSD$"); #define SOFT_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ PMC_CAP_USER | PMC_CAP_SYSTEM) -PMC_SOFT_DECLARE( , , clock, prof); - struct soft_descr { struct pmc_descr pm_descr; /* "base class" */ }; @@ -126,9 +124,10 @@ soft_allocate_pmc(int cpu, int ri, struct pmc *pm, if (ps == NULL) return (EINVAL); pmc_soft_ev_release(ps); + /* Module unload is protected by pmc SX lock. */ + if (ps->ps_alloc != NULL) + ps->ps_alloc(); - if (ev == pmc___clock_prof.ps_ev.pm_ev_code) - cpu_startprofclock(); return (0); } @@ -315,6 +314,8 @@ static int soft_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw; + enum pmc_event ev; + struct pmc_soft *ps; (void) pmc; @@ -328,8 +329,16 @@ soft_release_pmc(int cpu, int ri, struct pmc *pmc) KASSERT(phw->phw_pmc == NULL, ("[soft,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); - if (pmc->pm_event == pmc___clock_prof.ps_ev.pm_ev_code) - cpu_stopprofclock(); + ev = pmc->pm_event; + + /* Check if event is registered. */ + ps = pmc_soft_ev_acquire(ev); + KASSERT(ps != NULL, + ("[soft,%d] unregistered event %d", __LINE__, ev)); + pmc_soft_ev_release(ps); + /* Module unload is protected by pmc SX lock. */ + if (ps->ps_release != NULL) + ps->ps_release(); return (0); } diff --git a/sys/dev/ixgbe/LICENSE b/sys/dev/ixgbe/LICENSE index 0d4f1db..d446282 100644 --- a/sys/dev/ixgbe/LICENSE +++ b/sys/dev/ixgbe/LICENSE @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2011, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/sys/dev/ixgbe/ixgbe.c b/sys/dev/ixgbe/ixgbe.c index 0b77437..9977803 100644 --- a/sys/dev/ixgbe/ixgbe.c +++ b/sys/dev/ixgbe/ixgbe.c @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -47,7 +47,7 @@ int ixgbe_display_debug_stats = 0; /********************************************************************* * Driver version *********************************************************************/ -char ixgbe_driver_version[] = "2.5.0"; +char ixgbe_driver_version[] = "2.5.7 - HEAD"; /********************************************************************* * PCI Device ID Table @@ -83,7 +83,7 @@ static ixgbe_vendor_info_t ixgbe_vendor_info_array[] = {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0}, - {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0}, + {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0}, /* required last entry */ {0, 0, 0, 0, 0} @@ -216,7 +216,6 @@ static device_method_t ixgbe_methods[] = { DEVMETHOD(device_attach, ixgbe_attach), DEVMETHOD(device_detach, ixgbe_detach), DEVMETHOD(device_shutdown, ixgbe_shutdown), - DEVMETHOD_END }; @@ -596,6 +595,9 @@ ixgbe_attach(device_t dev) "PCIE, or x4 PCIE 2 slot is required.\n"); } + /* Set an initial default flow control value */ + adapter->fc = ixgbe_fc_full; + /* let hardware know driver is loaded */ ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; @@ -652,7 +654,7 @@ ixgbe_detach(device_t dev) for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { if (que->tq) { -#ifdef IXGBE_LEGACY_TX +#ifndef IXGBE_LEGACY_TX taskqueue_drain(que->tq, &txr->txq_task); #endif taskqueue_drain(que->tq, &que->que_task); @@ -1310,7 +1312,7 @@ ixgbe_init_locked(struct adapter *adapter) tmp = IXGBE_LOW_DV(frame); hw->fc.low_water[0] = IXGBE_BT2KB(tmp); - adapter->fc = hw->fc.requested_mode = ixgbe_fc_full; + hw->fc.requested_mode = adapter->fc; hw->fc.pause_time = IXGBE_FC_PAUSE; hw->fc.send_xon = TRUE; } @@ -1680,7 +1682,7 @@ ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) ifmr->ifm_active |= IFM_100_TX | IFM_FDX; break; case IXGBE_LINK_SPEED_1GB_FULL: - ifmr->ifm_active |= adapter->optics | IFM_FDX; + ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; break; case IXGBE_LINK_SPEED_10GB_FULL: ifmr->ifm_active |= adapter->optics | IFM_FDX; @@ -1932,18 +1934,6 @@ ixgbe_set_multi(struct adapter *adapter) bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * MAX_NUM_MULTICAST_ADDRESSES); - fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); - fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); - if (ifp->if_flags & IFF_PROMISC) - fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); - else if (ifp->if_flags & IFF_ALLMULTI) { - fctrl |= IXGBE_FCTRL_MPE; - fctrl &= ~IXGBE_FCTRL_UPE; - } else - fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); - - IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); - #if __FreeBSD_version < 800000 IF_ADDR_LOCK(ifp); #else @@ -1952,6 +1942,8 @@ ixgbe_set_multi(struct adapter *adapter) TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; + if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) + break; bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr), &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], IXGBE_ETH_LENGTH_OF_ADDRESS); @@ -1963,9 +1955,24 @@ ixgbe_set_multi(struct adapter *adapter) if_maddr_runlock(ifp); #endif - update_ptr = mta; - ixgbe_update_mc_addr_list(&adapter->hw, - update_ptr, mcnt, ixgbe_mc_array_itr, TRUE); + fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); + fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); + if (ifp->if_flags & IFF_PROMISC) + fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); + else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES || + ifp->if_flags & IFF_ALLMULTI) { + fctrl |= IXGBE_FCTRL_MPE; + fctrl &= ~IXGBE_FCTRL_UPE; + } else + fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); + + IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); + + if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) { + update_ptr = mta; + ixgbe_update_mc_addr_list(&adapter->hw, + update_ptr, mcnt, ixgbe_mc_array_itr, TRUE); + } return; } @@ -2172,7 +2179,7 @@ ixgbe_setup_optics(struct adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; int layer; - + layer = ixgbe_get_supported_physical_layer(hw); if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) { @@ -2651,7 +2658,7 @@ ixgbe_config_link(struct adapter *adapter) taskqueue_enqueue(adapter->tq, &adapter->mod_task); } else { if (hw->mac.ops.check_link) - err = ixgbe_check_link(hw, &autoneg, + err = ixgbe_check_link(hw, &adapter->link_speed, &adapter->link_up, FALSE); if (err) goto out; @@ -2662,8 +2669,8 @@ ixgbe_config_link(struct adapter *adapter) if (err) goto out; if (hw->mac.ops.setup_link) - err = hw->mac.ops.setup_link(hw, autoneg, - negotiate, adapter->link_up); + err = hw->mac.ops.setup_link(hw, + autoneg, adapter->link_up); } out: return; @@ -3713,6 +3720,8 @@ ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) M_PKTHDR, rxr->mbuf_sz); if (mp == NULL) goto update; + if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) + m_adj(mp, ETHER_ALIGN); } else mp = rxbuf->buf; @@ -4408,7 +4417,6 @@ ixgbe_rxeof(struct ix_queue *que) /* Make sure bad packets are discarded */ if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || (rxr->discard)) { - ifp->if_ierrors++; rxr->rx_discarded++; if (eop) rxr->discard = FALSE; @@ -4734,14 +4742,25 @@ ixgbe_enable_intr(struct adapter *adapter) /* Enable Fan Failure detection */ if (hw->device_id == IXGBE_DEV_ID_82598AT) mask |= IXGBE_EIMS_GPI_SDP1; - else { - mask |= IXGBE_EIMS_ECC; - mask |= IXGBE_EIMS_GPI_SDP0; - mask |= IXGBE_EIMS_GPI_SDP1; - mask |= IXGBE_EIMS_GPI_SDP2; + + switch (adapter->hw.mac.type) { + case ixgbe_mac_82599EB: + mask |= IXGBE_EIMS_ECC; + mask |= IXGBE_EIMS_GPI_SDP0; + mask |= IXGBE_EIMS_GPI_SDP1; + mask |= IXGBE_EIMS_GPI_SDP2; +#ifdef IXGBE_FDIR + mask |= IXGBE_EIMS_FLOW_DIR; +#endif + break; + case ixgbe_mac_X540: + mask |= IXGBE_EIMS_ECC; #ifdef IXGBE_FDIR - mask |= IXGBE_EIMS_FLOW_DIR; + mask |= IXGBE_EIMS_FLOW_DIR; #endif + /* falls through */ + default: + break; } IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); @@ -4969,7 +4988,7 @@ ixgbe_handle_msf(void *context, int pending) if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); if (hw->mac.ops.setup_link) - hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE); + hw->mac.ops.setup_link(hw, autoneg, TRUE); return; } @@ -5013,6 +5032,11 @@ ixgbe_update_stats_counters(struct adapter *adapter) adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC); adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC); + /* + ** Note: these are for the 8 possible traffic classes, + ** which in current implementation is unused, + ** therefore only 0 should read real data. + */ for (int i = 0; i < 8; i++) { u32 mp; mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); @@ -5022,13 +5046,20 @@ ixgbe_update_stats_counters(struct adapter *adapter) adapter->stats.mpc[i] += mp; /* Running comprehensive total for stats display */ total_missed_rx += adapter->stats.mpc[i]; - if (hw->mac.type == ixgbe_mac_82598EB) + if (hw->mac.type == ixgbe_mac_82598EB) { adapter->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); + adapter->stats.qbtc[i] += + IXGBE_READ_REG(hw, IXGBE_QBTC(i)); + adapter->stats.qbrc[i] += + IXGBE_READ_REG(hw, IXGBE_QBRC(i)); + adapter->stats.pxonrxc[i] += + IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); + } else + adapter->stats.pxonrxc[i] += + IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i)); adapter->stats.pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); - adapter->stats.pxonrxc[i] += - IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); adapter->stats.pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); adapter->stats.pxoffrxc[i] += @@ -5039,12 +5070,6 @@ ixgbe_update_stats_counters(struct adapter *adapter) for (int i = 0; i < 16; i++) { adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); - adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); - adapter->stats.qbrc[i] += - ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32); - adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); - adapter->stats.qbtc[i] += - ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32); adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); } adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC); @@ -5141,8 +5166,8 @@ ixgbe_update_stats_counters(struct adapter *adapter) ifp->if_collisions = 0; /* Rx Errors */ - ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs + - adapter->stats.rlec; + ifp->if_iqdrops = total_missed_rx; + ifp->if_ierrors = adapter->stats.crcerrs + adapter->stats.rlec; } /** ixgbe_sysctl_tdh_handler - Handler function @@ -5528,10 +5553,13 @@ ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS) ixgbe_disable_rx_drop(adapter); break; case ixgbe_fc_none: - default: adapter->hw.fc.requested_mode = ixgbe_fc_none; if (adapter->num_queues > 1) ixgbe_enable_rx_drop(adapter); + break; + default: + adapter->fc = last; + return (EINVAL); } /* Don't autoneg if forcing a value */ adapter->hw.fc.disable_fc_autoneg = TRUE; @@ -5560,7 +5588,7 @@ ixgbe_set_advertise(SYSCTL_HANDLER_ARGS) last = adapter->advertise; error = sysctl_handle_int(oidp, &adapter->advertise, 0, req); - if ((error) || (adapter->advertise == -1)) + if ((error) || (req->newptr == NULL)) return (error); if (adapter->advertise == last) /* no change */ @@ -5568,11 +5596,11 @@ ixgbe_set_advertise(SYSCTL_HANDLER_ARGS) if (!((hw->phy.media_type == ixgbe_media_type_copper) || (hw->phy.multispeed_fiber))) - return (error); + return (EINVAL); if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) { device_printf(dev, "Set Advertise: 100Mb on X540 only\n"); - return (error); + return (EINVAL); } if (adapter->advertise == 1) @@ -5582,11 +5610,13 @@ ixgbe_set_advertise(SYSCTL_HANDLER_ARGS) else if (adapter->advertise == 3) speed = IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL; - else /* bogus value */ - return (error); + else { /* bogus value */ + adapter->advertise = last; + return (EINVAL); + } hw->mac.autotry_restart = TRUE; - hw->mac.ops.setup_link(hw, speed, TRUE, TRUE); + hw->mac.ops.setup_link(hw, speed, TRUE); return (error); } diff --git a/sys/dev/ixgbe/ixgbe_82598.c b/sys/dev/ixgbe/ixgbe_82598.c index 9d3164c..de3185a 100644 --- a/sys/dev/ixgbe/ixgbe_82598.c +++ b/sys/dev/ixgbe/ixgbe_82598.c @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -49,18 +49,17 @@ static s32 ixgbe_check_mac_link_82598(struct ixgbe_hw *hw, bool link_up_wait_to_complete); static s32 ixgbe_setup_mac_link_82598(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool autoneg_wait_to_complete); static s32 ixgbe_setup_copper_link_82598(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool autoneg_wait_to_complete); static s32 ixgbe_reset_hw_82598(struct ixgbe_hw *hw); static s32 ixgbe_clear_vmdq_82598(struct ixgbe_hw *hw, u32 rar, u32 vmdq); static s32 ixgbe_clear_vfta_82598(struct ixgbe_hw *hw); static void ixgbe_set_rxpba_82598(struct ixgbe_hw *hw, int num_pb, u32 headroom, int strategy); - +static s32 ixgbe_read_i2c_sff8472_82598(struct ixgbe_hw *hw, u8 byte_offset, + u8 *sff8472_data); /** * ixgbe_set_pcie_completion_timeout - set pci-e completion timeout * @hw: pointer to the HW structure @@ -155,6 +154,7 @@ s32 ixgbe_init_ops_82598(struct ixgbe_hw *hw) /* SFP+ Module */ phy->ops.read_i2c_eeprom = &ixgbe_read_i2c_eeprom_82598; + phy->ops.read_i2c_sff8472 = &ixgbe_read_i2c_sff8472_82598; /* Link */ mac->ops.check_link = &ixgbe_check_mac_link_82598; @@ -712,15 +712,15 @@ out: * ixgbe_setup_mac_link_82598 - Set MAC link speed * @hw: pointer to hardware structure * @speed: new link speed - * @autoneg: TRUE if autonegotiation enabled * @autoneg_wait_to_complete: TRUE when waiting for completion is needed * * Set the link speed in the AUTOC register and restarts link. **/ static s32 ixgbe_setup_mac_link_82598(struct ixgbe_hw *hw, - ixgbe_link_speed speed, bool autoneg, + ixgbe_link_speed speed, bool autoneg_wait_to_complete) { + bool autoneg = FALSE; s32 status = IXGBE_SUCCESS; ixgbe_link_speed link_capabilities = IXGBE_LINK_SPEED_UNKNOWN; u32 curr_autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC); @@ -766,14 +766,12 @@ static s32 ixgbe_setup_mac_link_82598(struct ixgbe_hw *hw, * ixgbe_setup_copper_link_82598 - Set the PHY autoneg advertised field * @hw: pointer to hardware structure * @speed: new link speed - * @autoneg: TRUE if autonegotiation enabled * @autoneg_wait_to_complete: TRUE if waiting is needed to complete * * Sets the link speed in the AUTOC register in the MAC and restarts link. **/ static s32 ixgbe_setup_copper_link_82598(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool autoneg_wait_to_complete) { s32 status; @@ -781,7 +779,7 @@ static s32 ixgbe_setup_copper_link_82598(struct ixgbe_hw *hw, DEBUGFUNC("ixgbe_setup_copper_link_82598"); /* Setup the PHY according to input speed */ - status = hw->phy.ops.setup_link_speed(hw, speed, autoneg, + status = hw->phy.ops.setup_link_speed(hw, speed, autoneg_wait_to_complete); /* Set up MAC */ ixgbe_start_mac_link_82598(hw, autoneg_wait_to_complete); @@ -1102,15 +1100,16 @@ s32 ixgbe_write_analog_reg8_82598(struct ixgbe_hw *hw, u32 reg, u8 val) } /** - * ixgbe_read_i2c_eeprom_82598 - Reads 8 bit word over I2C interface. + * ixgbe_read_i2c_phy_82598 - Reads 8 bit word over I2C interface. * @hw: pointer to hardware structure - * @byte_offset: EEPROM byte offset to read + * @dev_addr: address to read from + * @byte_offset: byte offset to read from dev_addr * @eeprom_data: value read * * Performs 8 byte read operation to SFP module's EEPROM over I2C interface. **/ -s32 ixgbe_read_i2c_eeprom_82598(struct ixgbe_hw *hw, u8 byte_offset, - u8 *eeprom_data) +static s32 ixgbe_read_i2c_phy_82598(struct ixgbe_hw *hw, u8 dev_addr, + u8 byte_offset, u8 *eeprom_data) { s32 status = IXGBE_SUCCESS; u16 sfp_addr = 0; @@ -1118,7 +1117,7 @@ s32 ixgbe_read_i2c_eeprom_82598(struct ixgbe_hw *hw, u8 byte_offset, u16 sfp_stat = 0; u32 i; - DEBUGFUNC("ixgbe_read_i2c_eeprom_82598"); + DEBUGFUNC("ixgbe_read_i2c_phy_82598"); if (hw->phy.type == ixgbe_phy_nl) { /* @@ -1126,7 +1125,7 @@ s32 ixgbe_read_i2c_eeprom_82598(struct ixgbe_hw *hw, u8 byte_offset, * 0xC30D. These registers are used to talk to the SFP+ * module's EEPROM through the SDA/SCL (I2C) interface. */ - sfp_addr = (IXGBE_I2C_EEPROM_DEV_ADDR << 8) + byte_offset; + sfp_addr = (dev_addr << 8) + byte_offset; sfp_addr = (sfp_addr | IXGBE_I2C_EEPROM_READ_MASK); hw->phy.ops.write_reg(hw, IXGBE_MDIO_PMA_PMD_SDA_SCL_ADDR, @@ -1158,7 +1157,6 @@ s32 ixgbe_read_i2c_eeprom_82598(struct ixgbe_hw *hw, u8 byte_offset, *eeprom_data = (u8)(sfp_data >> 8); } else { status = IXGBE_ERR_PHY; - goto out; } out: @@ -1166,6 +1164,36 @@ out: } /** + * ixgbe_read_i2c_eeprom_82598 - Reads 8 bit word over I2C interface. + * @hw: pointer to hardware structure + * @byte_offset: EEPROM byte offset to read + * @eeprom_data: value read + * + * Performs 8 byte read operation to SFP module's EEPROM over I2C interface. + **/ +s32 ixgbe_read_i2c_eeprom_82598(struct ixgbe_hw *hw, u8 byte_offset, + u8 *eeprom_data) +{ + return ixgbe_read_i2c_phy_82598(hw, IXGBE_I2C_EEPROM_DEV_ADDR, + byte_offset, eeprom_data); +} + +/** + * ixgbe_read_i2c_sff8472_82598 - Reads 8 bit word over I2C interface. + * @hw: pointer to hardware structure + * @byte_offset: byte offset at address 0xA2 + * @eeprom_data: value read + * + * Performs 8 byte read operation to SFP module's SFF-8472 data over I2C + **/ +static s32 ixgbe_read_i2c_sff8472_82598(struct ixgbe_hw *hw, u8 byte_offset, + u8 *sff8472_data) +{ + return ixgbe_read_i2c_phy_82598(hw, IXGBE_I2C_EEPROM_DEV_ADDR2, + byte_offset, sff8472_data); +} + +/** * ixgbe_get_supported_physical_layer_82598 - Returns physical layer type * @hw: pointer to hardware structure * diff --git a/sys/dev/ixgbe/ixgbe_82599.c b/sys/dev/ixgbe/ixgbe_82599.c index 083f7e1..592440b 100644 --- a/sys/dev/ixgbe/ixgbe_82599.c +++ b/sys/dev/ixgbe/ixgbe_82599.c @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -40,7 +40,6 @@ static s32 ixgbe_setup_copper_link_82599(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool autoneg_wait_to_complete); static s32 ixgbe_verify_fw_version_82599(struct ixgbe_hw *hw); static s32 ixgbe_read_eeprom_82599(struct ixgbe_hw *hw, @@ -48,14 +47,37 @@ static s32 ixgbe_read_eeprom_82599(struct ixgbe_hw *hw, static s32 ixgbe_read_eeprom_buffer_82599(struct ixgbe_hw *hw, u16 offset, u16 words, u16 *data); +static bool ixgbe_mng_enabled(struct ixgbe_hw *hw) +{ + u32 fwsm, manc, factps; + + fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); + if ((fwsm & IXGBE_FWSM_MODE_MASK) != IXGBE_FWSM_FW_MODE_PT) + return FALSE; + + manc = IXGBE_READ_REG(hw, IXGBE_MANC); + if (!(manc & IXGBE_MANC_RCV_TCO_EN)) + return FALSE; + + factps = IXGBE_READ_REG(hw, IXGBE_FACTPS); + if (factps & IXGBE_FACTPS_MNGCG) + return FALSE; + + return TRUE; +} + void ixgbe_init_mac_link_ops_82599(struct ixgbe_hw *hw) { struct ixgbe_mac_info *mac = &hw->mac; DEBUGFUNC("ixgbe_init_mac_link_ops_82599"); - /* enable the laser control functions for SFP+ fiber */ - if (mac->ops.get_media_type(hw) == ixgbe_media_type_fiber) { + /* + * enable the laser control functions for SFP+ fiber + * and MNG not enabled + */ + if ((mac->ops.get_media_type(hw) == ixgbe_media_type_fiber) && + !(ixgbe_mng_enabled(hw))) { mac->ops.disable_tx_laser = &ixgbe_disable_tx_laser_multispeed_fiber; mac->ops.enable_tx_laser = @@ -135,9 +157,8 @@ init_phy_ops_out: s32 ixgbe_setup_sfp_modules_82599(struct ixgbe_hw *hw) { s32 ret_val = IXGBE_SUCCESS; - u32 reg_anlp1 = 0; - u32 i = 0; u16 list_offset, data_offset, data_value; + bool got_lock = FALSE; DEBUGFUNC("ixgbe_setup_sfp_modules_82599"); @@ -171,28 +192,39 @@ s32 ixgbe_setup_sfp_modules_82599(struct ixgbe_hw *hw) /* Delay obtaining semaphore again to allow FW access */ msec_delay(hw->eeprom.semaphore_delay); - /* Now restart DSP by setting Restart_AN and clearing LMS */ - IXGBE_WRITE_REG(hw, IXGBE_AUTOC, ((IXGBE_READ_REG(hw, - IXGBE_AUTOC) & ~IXGBE_AUTOC_LMS_MASK) | - IXGBE_AUTOC_AN_RESTART)); - - /* Wait for AN to leave state 0 */ - for (i = 0; i < 10; i++) { - msec_delay(4); - reg_anlp1 = IXGBE_READ_REG(hw, IXGBE_ANLP1); - if (reg_anlp1 & IXGBE_ANLP1_AN_STATE_MASK) - break; + /* Need SW/FW semaphore around AUTOC writes if LESM on, + * likewise reset_pipeline requires lock as it also writes + * AUTOC. + */ + if (ixgbe_verify_lesm_fw_enabled_82599(hw)) { + ret_val = hw->mac.ops.acquire_swfw_sync(hw, + IXGBE_GSSR_MAC_CSR_SM); + if (ret_val != IXGBE_SUCCESS) { + ret_val = IXGBE_ERR_SWFW_SYNC; + goto setup_sfp_out; + } + + got_lock = TRUE; + } + + /* Restart DSP and set SFI mode */ + IXGBE_WRITE_REG(hw, IXGBE_AUTOC, ((hw->mac.orig_autoc) | + IXGBE_AUTOC_LMS_10G_SERIAL)); + hw->mac.cached_autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC); + ret_val = ixgbe_reset_pipeline_82599(hw); + + if (got_lock) { + hw->mac.ops.release_swfw_sync(hw, + IXGBE_GSSR_MAC_CSR_SM); + got_lock = FALSE; } - if (!(reg_anlp1 & IXGBE_ANLP1_AN_STATE_MASK)) { + + if (ret_val) { DEBUGOUT("sfp module setup not complete\n"); ret_val = IXGBE_ERR_SFP_SETUP_NOT_COMPLETE; goto setup_sfp_out; } - /* Restart DSP by setting Restart_AN and return to SFI mode */ - IXGBE_WRITE_REG(hw, IXGBE_AUTOC, (IXGBE_READ_REG(hw, - IXGBE_AUTOC) | IXGBE_AUTOC_LMS_10G_SERIAL | - IXGBE_AUTOC_AN_RESTART)); } setup_sfp_out: @@ -216,7 +248,7 @@ s32 ixgbe_init_ops_82599(struct ixgbe_hw *hw) DEBUGFUNC("ixgbe_init_ops_82599"); - ret_val = ixgbe_init_phy_ops_generic(hw); + ixgbe_init_phy_ops_generic(hw); ret_val = ixgbe_init_ops_generic(hw); /* PHY */ @@ -289,13 +321,13 @@ s32 ixgbe_init_ops_82599(struct ixgbe_hw *hw) * ixgbe_get_link_capabilities_82599 - Determines link capabilities * @hw: pointer to hardware structure * @speed: pointer to link speed - * @negotiation: TRUE when autoneg or autotry is enabled + * @autoneg: TRUE when autoneg or autotry is enabled * * Determines the link capabilities by reading the AUTOC register. **/ s32 ixgbe_get_link_capabilities_82599(struct ixgbe_hw *hw, ixgbe_link_speed *speed, - bool *negotiation) + bool *autoneg) { s32 status = IXGBE_SUCCESS; u32 autoc = 0; @@ -309,7 +341,7 @@ s32 ixgbe_get_link_capabilities_82599(struct ixgbe_hw *hw, hw->phy.sfp_type == ixgbe_sfp_type_1g_sx_core0 || hw->phy.sfp_type == ixgbe_sfp_type_1g_sx_core1) { *speed = IXGBE_LINK_SPEED_1GB_FULL; - *negotiation = TRUE; + *autoneg = TRUE; goto out; } @@ -326,22 +358,22 @@ s32 ixgbe_get_link_capabilities_82599(struct ixgbe_hw *hw, switch (autoc & IXGBE_AUTOC_LMS_MASK) { case IXGBE_AUTOC_LMS_1G_LINK_NO_AN: *speed = IXGBE_LINK_SPEED_1GB_FULL; - *negotiation = FALSE; + *autoneg = FALSE; break; case IXGBE_AUTOC_LMS_10G_LINK_NO_AN: *speed = IXGBE_LINK_SPEED_10GB_FULL; - *negotiation = FALSE; + *autoneg = FALSE; break; case IXGBE_AUTOC_LMS_1G_AN: *speed = IXGBE_LINK_SPEED_1GB_FULL; - *negotiation = TRUE; + *autoneg = TRUE; break; case IXGBE_AUTOC_LMS_10G_SERIAL: *speed = IXGBE_LINK_SPEED_10GB_FULL; - *negotiation = FALSE; + *autoneg = FALSE; break; case IXGBE_AUTOC_LMS_KX4_KX_KR: @@ -353,7 +385,7 @@ s32 ixgbe_get_link_capabilities_82599(struct ixgbe_hw *hw, *speed |= IXGBE_LINK_SPEED_10GB_FULL; if (autoc & IXGBE_AUTOC_KX_SUPP) *speed |= IXGBE_LINK_SPEED_1GB_FULL; - *negotiation = TRUE; + *autoneg = TRUE; break; case IXGBE_AUTOC_LMS_KX4_KX_KR_SGMII: @@ -364,12 +396,12 @@ s32 ixgbe_get_link_capabilities_82599(struct ixgbe_hw *hw, *speed |= IXGBE_LINK_SPEED_10GB_FULL; if (autoc & IXGBE_AUTOC_KX_SUPP) *speed |= IXGBE_LINK_SPEED_1GB_FULL; - *negotiation = TRUE; + *autoneg = TRUE; break; case IXGBE_AUTOC_LMS_SGMII_1G_100M: *speed = IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_100_FULL; - *negotiation = FALSE; + *autoneg = FALSE; break; default: @@ -381,7 +413,7 @@ s32 ixgbe_get_link_capabilities_82599(struct ixgbe_hw *hw, if (hw->phy.multispeed_fiber) { *speed |= IXGBE_LINK_SPEED_10GB_FULL | IXGBE_LINK_SPEED_1GB_FULL; - *negotiation = TRUE; + *autoneg = TRUE; } out: @@ -424,6 +456,7 @@ enum ixgbe_media_type ixgbe_get_media_type_82599(struct ixgbe_hw *hw) case IXGBE_DEV_ID_82599_SFP_FCOE: case IXGBE_DEV_ID_82599_SFP_EM: case IXGBE_DEV_ID_82599_SFP_SF2: + case IXGBE_DEV_ID_82599_SFP_SF_QP: case IXGBE_DEV_ID_82599EN_SFP: media_type = ixgbe_media_type_fiber; break; @@ -433,6 +466,10 @@ enum ixgbe_media_type ixgbe_get_media_type_82599(struct ixgbe_hw *hw) case IXGBE_DEV_ID_82599_T3_LOM: media_type = ixgbe_media_type_copper; break; + case IXGBE_DEV_ID_82599_BYPASS: + media_type = ixgbe_media_type_fiber_fixed; + hw->phy.multispeed_fiber = TRUE; + break; default: media_type = ixgbe_media_type_unknown; break; @@ -456,17 +493,32 @@ s32 ixgbe_start_mac_link_82599(struct ixgbe_hw *hw, u32 links_reg; u32 i; s32 status = IXGBE_SUCCESS; + bool got_lock = FALSE; DEBUGFUNC("ixgbe_start_mac_link_82599"); + /* reset_pipeline requires us to hold this lock as it writes to + * AUTOC. + */ + if (ixgbe_verify_lesm_fw_enabled_82599(hw)) { + status = hw->mac.ops.acquire_swfw_sync(hw, + IXGBE_GSSR_MAC_CSR_SM); + if (status != IXGBE_SUCCESS) + goto out; + + got_lock = TRUE; + } + /* Restart link */ - autoc_reg = IXGBE_READ_REG(hw, IXGBE_AUTOC); - autoc_reg |= IXGBE_AUTOC_AN_RESTART; - IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc_reg); + ixgbe_reset_pipeline_82599(hw); + + if (got_lock) + hw->mac.ops.release_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM); /* Only poll for autoneg to complete if specified to do so */ if (autoneg_wait_to_complete) { + autoc_reg = IXGBE_READ_REG(hw, IXGBE_AUTOC); if ((autoc_reg & IXGBE_AUTOC_LMS_MASK) == IXGBE_AUTOC_LMS_KX4_KX_KR || (autoc_reg & IXGBE_AUTOC_LMS_MASK) == @@ -490,6 +542,7 @@ s32 ixgbe_start_mac_link_82599(struct ixgbe_hw *hw, /* Add delay to filter out noises during initial link setup */ msec_delay(50); +out: return status; } @@ -555,16 +608,84 @@ void ixgbe_flap_tx_laser_multispeed_fiber(struct ixgbe_hw *hw) } /** + * ixgbe_set_fiber_fixed_speed - Set module link speed for fixed fiber + * @hw: pointer to hardware structure + * @speed: link speed to set + * + * We set the module speed differently for fixed fiber. For other + * multi-speed devices we don't have an error value so here if we + * detect an error we just log it and exit. + */ +static void ixgbe_set_fiber_fixed_speed(struct ixgbe_hw *hw, + ixgbe_link_speed speed) +{ + s32 status; + u8 rs, eeprom_data; + + switch (speed) { + case IXGBE_LINK_SPEED_10GB_FULL: + /* one bit mask same as setting on */ + rs = IXGBE_SFF_SOFT_RS_SELECT_10G; + break; + case IXGBE_LINK_SPEED_1GB_FULL: + rs = IXGBE_SFF_SOFT_RS_SELECT_1G; + break; + default: + DEBUGOUT("Invalid fixed module speed\n"); + return; + } + + /* Set RS0 */ + status = hw->phy.ops.read_i2c_byte(hw, IXGBE_SFF_SFF_8472_OSCB, + IXGBE_I2C_EEPROM_DEV_ADDR2, + &eeprom_data); + if (status) { + DEBUGOUT("Failed to read Rx Rate Select RS0\n"); + goto out; + } + + eeprom_data = (eeprom_data & ~IXGBE_SFF_SOFT_RS_SELECT_MASK) & rs; + + status = hw->phy.ops.write_i2c_byte(hw, IXGBE_SFF_SFF_8472_OSCB, + IXGBE_I2C_EEPROM_DEV_ADDR2, + eeprom_data); + if (status) { + DEBUGOUT("Failed to write Rx Rate Select RS0\n"); + goto out; + } + + /* Set RS1 */ + status = hw->phy.ops.read_i2c_byte(hw, IXGBE_SFF_SFF_8472_ESCB, + IXGBE_I2C_EEPROM_DEV_ADDR2, + &eeprom_data); + if (status) { + DEBUGOUT("Failed to read Rx Rate Select RS1\n"); + goto out; + } + + eeprom_data = (eeprom_data & ~IXGBE_SFF_SOFT_RS_SELECT_MASK) & rs; + + status = hw->phy.ops.write_i2c_byte(hw, IXGBE_SFF_SFF_8472_ESCB, + IXGBE_I2C_EEPROM_DEV_ADDR2, + eeprom_data); + if (status) { + DEBUGOUT("Failed to write Rx Rate Select RS1\n"); + goto out; + } +out: + return; +} + +/** * ixgbe_setup_mac_link_multispeed_fiber - Set MAC link speed * @hw: pointer to hardware structure * @speed: new link speed - * @autoneg: TRUE if autonegotiation enabled * @autoneg_wait_to_complete: TRUE when waiting for completion is needed * * Set the link speed in the AUTOC register and restarts link. **/ s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw, - ixgbe_link_speed speed, bool autoneg, + ixgbe_link_speed speed, bool autoneg_wait_to_complete) { s32 status = IXGBE_SUCCESS; @@ -573,13 +694,12 @@ s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw, u32 speedcnt = 0; u32 esdp_reg = IXGBE_READ_REG(hw, IXGBE_ESDP); u32 i = 0; - bool link_up = FALSE; - bool negotiation; + bool autoneg, link_up = FALSE; DEBUGFUNC("ixgbe_setup_mac_link_multispeed_fiber"); /* Mask off requested but non-supported speeds */ - status = ixgbe_get_link_capabilities(hw, &link_speed, &negotiation); + status = ixgbe_get_link_capabilities(hw, &link_speed, &autoneg); if (status != IXGBE_SUCCESS) return status; @@ -602,16 +722,20 @@ s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw, goto out; /* Set the module link speed */ - esdp_reg |= (IXGBE_ESDP_SDP5_DIR | IXGBE_ESDP_SDP5); - IXGBE_WRITE_REG(hw, IXGBE_ESDP, esdp_reg); - IXGBE_WRITE_FLUSH(hw); + if (hw->phy.media_type == ixgbe_media_type_fiber_fixed) { + ixgbe_set_fiber_fixed_speed(hw, + IXGBE_LINK_SPEED_10GB_FULL); + } else { + esdp_reg |= (IXGBE_ESDP_SDP5_DIR | IXGBE_ESDP_SDP5); + IXGBE_WRITE_REG(hw, IXGBE_ESDP, esdp_reg); + IXGBE_WRITE_FLUSH(hw); + } /* Allow module to change analog characteristics (1G->10G) */ msec_delay(40); status = ixgbe_setup_mac_link_82599(hw, IXGBE_LINK_SPEED_10GB_FULL, - autoneg, autoneg_wait_to_complete); if (status != IXGBE_SUCCESS) return status; @@ -653,17 +777,21 @@ s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw, goto out; /* Set the module link speed */ - esdp_reg &= ~IXGBE_ESDP_SDP5; - esdp_reg |= IXGBE_ESDP_SDP5_DIR; - IXGBE_WRITE_REG(hw, IXGBE_ESDP, esdp_reg); - IXGBE_WRITE_FLUSH(hw); + if (hw->phy.media_type == ixgbe_media_type_fiber_fixed) { + ixgbe_set_fiber_fixed_speed(hw, + IXGBE_LINK_SPEED_1GB_FULL); + } else { + esdp_reg &= ~IXGBE_ESDP_SDP5; + esdp_reg |= IXGBE_ESDP_SDP5_DIR; + IXGBE_WRITE_REG(hw, IXGBE_ESDP, esdp_reg); + IXGBE_WRITE_FLUSH(hw); + } /* Allow module to change analog characteristics (10G->1G) */ msec_delay(40); status = ixgbe_setup_mac_link_82599(hw, IXGBE_LINK_SPEED_1GB_FULL, - autoneg, autoneg_wait_to_complete); if (status != IXGBE_SUCCESS) return status; @@ -690,7 +818,7 @@ s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw, */ if (speedcnt > 1) status = ixgbe_setup_mac_link_multispeed_fiber(hw, - highest_link_speed, autoneg, autoneg_wait_to_complete); + highest_link_speed, autoneg_wait_to_complete); out: /* Set autoneg_advertised value based on input link speed */ @@ -709,13 +837,12 @@ out: * ixgbe_setup_mac_link_smartspeed - Set MAC link speed using SmartSpeed * @hw: pointer to hardware structure * @speed: new link speed - * @autoneg: TRUE if autonegotiation enabled * @autoneg_wait_to_complete: TRUE when waiting for completion is needed * * Implements the Intel SmartSpeed algorithm. **/ s32 ixgbe_setup_mac_link_smartspeed(struct ixgbe_hw *hw, - ixgbe_link_speed speed, bool autoneg, + ixgbe_link_speed speed, bool autoneg_wait_to_complete) { s32 status = IXGBE_SUCCESS; @@ -748,7 +875,7 @@ s32 ixgbe_setup_mac_link_smartspeed(struct ixgbe_hw *hw, /* First, try to get link with full advertisement */ hw->phy.smart_speed_active = FALSE; for (j = 0; j < IXGBE_SMARTSPEED_MAX_RETRIES; j++) { - status = ixgbe_setup_mac_link_82599(hw, speed, autoneg, + status = ixgbe_setup_mac_link_82599(hw, speed, autoneg_wait_to_complete); if (status != IXGBE_SUCCESS) goto out; @@ -783,7 +910,7 @@ s32 ixgbe_setup_mac_link_smartspeed(struct ixgbe_hw *hw, /* Turn SmartSpeed on to disable KR support */ hw->phy.smart_speed_active = TRUE; - status = ixgbe_setup_mac_link_82599(hw, speed, autoneg, + status = ixgbe_setup_mac_link_82599(hw, speed, autoneg_wait_to_complete); if (status != IXGBE_SUCCESS) goto out; @@ -808,7 +935,7 @@ s32 ixgbe_setup_mac_link_smartspeed(struct ixgbe_hw *hw, /* We didn't get link. Turn SmartSpeed back off. */ hw->phy.smart_speed_active = FALSE; - status = ixgbe_setup_mac_link_82599(hw, speed, autoneg, + status = ixgbe_setup_mac_link_82599(hw, speed, autoneg_wait_to_complete); out: @@ -822,32 +949,30 @@ out: * ixgbe_setup_mac_link_82599 - Set MAC link speed * @hw: pointer to hardware structure * @speed: new link speed - * @autoneg: TRUE if autonegotiation enabled * @autoneg_wait_to_complete: TRUE when waiting for completion is needed * * Set the link speed in the AUTOC register and restarts link. **/ s32 ixgbe_setup_mac_link_82599(struct ixgbe_hw *hw, - ixgbe_link_speed speed, bool autoneg, + ixgbe_link_speed speed, bool autoneg_wait_to_complete) { + bool autoneg = FALSE; s32 status = IXGBE_SUCCESS; - u32 autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC); + u32 autoc, pma_pmd_1g, link_mode, start_autoc; u32 autoc2 = IXGBE_READ_REG(hw, IXGBE_AUTOC2); - u32 start_autoc = autoc; u32 orig_autoc = 0; - u32 link_mode = autoc & IXGBE_AUTOC_LMS_MASK; - u32 pma_pmd_1g = autoc & IXGBE_AUTOC_1G_PMA_PMD_MASK; u32 pma_pmd_10g_serial = autoc2 & IXGBE_AUTOC2_10G_SERIAL_PMA_PMD_MASK; u32 links_reg; u32 i; ixgbe_link_speed link_capabilities = IXGBE_LINK_SPEED_UNKNOWN; + bool got_lock = FALSE; DEBUGFUNC("ixgbe_setup_mac_link_82599"); /* Check to see if speed passed in is supported. */ status = ixgbe_get_link_capabilities(hw, &link_capabilities, &autoneg); - if (status != IXGBE_SUCCESS) + if (status) goto out; speed &= link_capabilities; @@ -859,9 +984,14 @@ s32 ixgbe_setup_mac_link_82599(struct ixgbe_hw *hw, /* Use stored value (EEPROM defaults) of AUTOC to find KR/KX4 support*/ if (hw->mac.orig_link_settings_stored) - orig_autoc = hw->mac.orig_autoc; + autoc = hw->mac.orig_autoc; else - orig_autoc = autoc; + autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC); + + orig_autoc = autoc; + start_autoc = hw->mac.cached_autoc; + link_mode = autoc & IXGBE_AUTOC_LMS_MASK; + pma_pmd_1g = autoc & IXGBE_AUTOC_1G_PMA_PMD_MASK; if (link_mode == IXGBE_AUTOC_LMS_KX4_KX_KR || link_mode == IXGBE_AUTOC_LMS_KX4_KX_KR_1G_AN || @@ -900,9 +1030,31 @@ s32 ixgbe_setup_mac_link_82599(struct ixgbe_hw *hw, } if (autoc != start_autoc) { + /* Need SW/FW semaphore around AUTOC writes if LESM is on, + * likewise reset_pipeline requires us to hold this lock as + * it also writes to AUTOC. + */ + if (ixgbe_verify_lesm_fw_enabled_82599(hw)) { + status = hw->mac.ops.acquire_swfw_sync(hw, + IXGBE_GSSR_MAC_CSR_SM); + if (status != IXGBE_SUCCESS) { + status = IXGBE_ERR_SWFW_SYNC; + goto out; + } + + got_lock = TRUE; + } + /* Restart link */ - autoc |= IXGBE_AUTOC_AN_RESTART; IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc); + hw->mac.cached_autoc = autoc; + ixgbe_reset_pipeline_82599(hw); + + if (got_lock) { + hw->mac.ops.release_swfw_sync(hw, + IXGBE_GSSR_MAC_CSR_SM); + got_lock = FALSE; + } /* Only poll for autoneg to complete if specified to do so */ if (autoneg_wait_to_complete) { @@ -937,14 +1089,12 @@ out: * ixgbe_setup_copper_link_82599 - Set the PHY autoneg advertised field * @hw: pointer to hardware structure * @speed: new link speed - * @autoneg: TRUE if autonegotiation enabled * @autoneg_wait_to_complete: TRUE if waiting is needed to complete * * Restarts link on PHY and MAC based on settings passed in. **/ static s32 ixgbe_setup_copper_link_82599(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool autoneg_wait_to_complete) { s32 status; @@ -952,7 +1102,7 @@ static s32 ixgbe_setup_copper_link_82599(struct ixgbe_hw *hw, DEBUGFUNC("ixgbe_setup_copper_link_82599"); /* Setup the PHY according to input speed */ - status = hw->phy.ops.setup_link_speed(hw, speed, autoneg, + status = hw->phy.ops.setup_link_speed(hw, speed, autoneg_wait_to_complete); /* Set up MAC */ ixgbe_start_mac_link_82599(hw, autoneg_wait_to_complete); @@ -1056,14 +1206,45 @@ mac_reset_top: */ autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC); autoc2 = IXGBE_READ_REG(hw, IXGBE_AUTOC2); + + /* Enable link if disabled in NVM */ + if (autoc2 & IXGBE_AUTOC2_LINK_DISABLE_MASK) { + autoc2 &= ~IXGBE_AUTOC2_LINK_DISABLE_MASK; + IXGBE_WRITE_REG(hw, IXGBE_AUTOC2, autoc2); + IXGBE_WRITE_FLUSH(hw); + } + if (hw->mac.orig_link_settings_stored == FALSE) { hw->mac.orig_autoc = autoc; hw->mac.orig_autoc2 = autoc2; + hw->mac.cached_autoc = autoc; hw->mac.orig_link_settings_stored = TRUE; } else { - if (autoc != hw->mac.orig_autoc) - IXGBE_WRITE_REG(hw, IXGBE_AUTOC, (hw->mac.orig_autoc | - IXGBE_AUTOC_AN_RESTART)); + if (autoc != hw->mac.orig_autoc) { + /* Need SW/FW semaphore around AUTOC writes if LESM is + * on, likewise reset_pipeline requires us to hold + * this lock as it also writes to AUTOC. + */ + bool got_lock = FALSE; + if (ixgbe_verify_lesm_fw_enabled_82599(hw)) { + status = hw->mac.ops.acquire_swfw_sync(hw, + IXGBE_GSSR_MAC_CSR_SM); + if (status != IXGBE_SUCCESS) { + status = IXGBE_ERR_SWFW_SYNC; + goto reset_hw_out; + } + + got_lock = TRUE; + } + + IXGBE_WRITE_REG(hw, IXGBE_AUTOC, hw->mac.orig_autoc); + hw->mac.cached_autoc = hw->mac.orig_autoc; + ixgbe_reset_pipeline_82599(hw); + + if (got_lock) + hw->mac.ops.release_swfw_sync(hw, + IXGBE_GSSR_MAC_CSR_SM); + } if ((autoc2 & IXGBE_AUTOC2_UPPER_MASK) != (hw->mac.orig_autoc2 & IXGBE_AUTOC2_UPPER_MASK)) { @@ -1168,7 +1349,7 @@ s32 ixgbe_reinit_fdir_tables_82599(struct ixgbe_hw *hw) if (IXGBE_READ_REG(hw, IXGBE_FDIRCTRL) & IXGBE_FDIRCTRL_INIT_DONE) break; - usec_delay(10); + msec_delay(1); } if (i >= IXGBE_FDIR_INIT_DONE_POLL) { DEBUGOUT("Flow Director Signature poll time exceeded!\n"); @@ -2094,7 +2275,7 @@ s32 ixgbe_enable_rx_dma_82599(struct ixgbe_hw *hw, u32 regval) * Returns IXGBE_ERR_EEPROM_VERSION if the FW is not present or * if the FW version is not supported. **/ -static s32 ixgbe_verify_fw_version_82599(struct ixgbe_hw *hw) +s32 ixgbe_verify_fw_version_82599(struct ixgbe_hw *hw) { s32 status = IXGBE_ERR_EEPROM_VERSION; u16 fw_offset, fw_ptp_cfg_offset; @@ -2243,4 +2424,55 @@ static s32 ixgbe_read_eeprom_82599(struct ixgbe_hw *hw, return ret_val; } +/** + * ixgbe_reset_pipeline_82599 - perform pipeline reset + * + * @hw: pointer to hardware structure + * + * Reset pipeline by asserting Restart_AN together with LMS change to ensure + * full pipeline reset + **/ +s32 ixgbe_reset_pipeline_82599(struct ixgbe_hw *hw) +{ + s32 ret_val; + u32 anlp1_reg = 0; + u32 i, autoc_reg, autoc2_reg; + + /* Enable link if disabled in NVM */ + autoc2_reg = IXGBE_READ_REG(hw, IXGBE_AUTOC2); + if (autoc2_reg & IXGBE_AUTOC2_LINK_DISABLE_MASK) { + autoc2_reg &= ~IXGBE_AUTOC2_LINK_DISABLE_MASK; + IXGBE_WRITE_REG(hw, IXGBE_AUTOC2, autoc2_reg); + IXGBE_WRITE_FLUSH(hw); + } + + autoc_reg = hw->mac.cached_autoc; + autoc_reg |= IXGBE_AUTOC_AN_RESTART; + /* Write AUTOC register with toggled LMS[2] bit and Restart_AN */ + IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc_reg ^ IXGBE_AUTOC_LMS_1G_AN); + /* Wait for AN to leave state 0 */ + for (i = 0; i < 10; i++) { + msec_delay(4); + anlp1_reg = IXGBE_READ_REG(hw, IXGBE_ANLP1); + if (anlp1_reg & IXGBE_ANLP1_AN_STATE_MASK) + break; + } + + if (!(anlp1_reg & IXGBE_ANLP1_AN_STATE_MASK)) { + DEBUGOUT("auto negotiation not completed\n"); + ret_val = IXGBE_ERR_RESET_FAILED; + goto reset_pipeline_out; + } + + ret_val = IXGBE_SUCCESS; + +reset_pipeline_out: + /* Write AUTOC register with original LMS field and Restart_AN */ + IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc_reg); + IXGBE_WRITE_FLUSH(hw); + + return ret_val; +} + + diff --git a/sys/dev/ixgbe/ixgbe_82599.h b/sys/dev/ixgbe/ixgbe_82599.h index dca39b7..a77d153 100644 --- a/sys/dev/ixgbe/ixgbe_82599.h +++ b/sys/dev/ixgbe/ixgbe_82599.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -42,15 +42,15 @@ void ixgbe_disable_tx_laser_multispeed_fiber(struct ixgbe_hw *hw); void ixgbe_enable_tx_laser_multispeed_fiber(struct ixgbe_hw *hw); void ixgbe_flap_tx_laser_multispeed_fiber(struct ixgbe_hw *hw); s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw, - ixgbe_link_speed speed, bool autoneg, + ixgbe_link_speed speed, bool autoneg_wait_to_complete); s32 ixgbe_setup_mac_link_smartspeed(struct ixgbe_hw *hw, - ixgbe_link_speed speed, bool autoneg, + ixgbe_link_speed speed, bool autoneg_wait_to_complete); s32 ixgbe_start_mac_link_82599(struct ixgbe_hw *hw, bool autoneg_wait_to_complete); s32 ixgbe_setup_mac_link_82599(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool autoneg_wait_to_complete); + bool autoneg_wait_to_complete); s32 ixgbe_setup_sfp_modules_82599(struct ixgbe_hw *hw); void ixgbe_init_mac_link_ops_82599(struct ixgbe_hw *hw); s32 ixgbe_reset_hw_82599(struct ixgbe_hw *hw); @@ -61,5 +61,4 @@ s32 ixgbe_identify_phy_82599(struct ixgbe_hw *hw); s32 ixgbe_init_phy_ops_82599(struct ixgbe_hw *hw); u32 ixgbe_get_supported_physical_layer_82599(struct ixgbe_hw *hw); s32 ixgbe_enable_rx_dma_82599(struct ixgbe_hw *hw, u32 regval); -bool ixgbe_verify_lesm_fw_enabled_82599(struct ixgbe_hw *hw); #endif /* _IXGBE_82599_H_ */ diff --git a/sys/dev/ixgbe/ixgbe_api.c b/sys/dev/ixgbe/ixgbe_api.c index faee9d3..61876fb 100644 --- a/sys/dev/ixgbe/ixgbe_api.c +++ b/sys/dev/ixgbe/ixgbe_api.c @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -93,53 +93,53 @@ s32 ixgbe_set_mac_type(struct ixgbe_hw *hw) DEBUGFUNC("ixgbe_set_mac_type\n"); - if (hw->vendor_id == IXGBE_INTEL_VENDOR_ID) { - switch (hw->device_id) { - case IXGBE_DEV_ID_82598: - case IXGBE_DEV_ID_82598_BX: - case IXGBE_DEV_ID_82598AF_SINGLE_PORT: - case IXGBE_DEV_ID_82598AF_DUAL_PORT: - case IXGBE_DEV_ID_82598AT: - case IXGBE_DEV_ID_82598AT2: - case IXGBE_DEV_ID_82598EB_CX4: - case IXGBE_DEV_ID_82598_CX4_DUAL_PORT: - case IXGBE_DEV_ID_82598_DA_DUAL_PORT: - case IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM: - case IXGBE_DEV_ID_82598EB_XF_LR: - case IXGBE_DEV_ID_82598EB_SFP_LOM: - hw->mac.type = ixgbe_mac_82598EB; - break; - case IXGBE_DEV_ID_82599_KX4: - case IXGBE_DEV_ID_82599_KX4_MEZZ: - case IXGBE_DEV_ID_82599_XAUI_LOM: - case IXGBE_DEV_ID_82599_COMBO_BACKPLANE: - case IXGBE_DEV_ID_82599_KR: - case IXGBE_DEV_ID_82599_SFP: - case IXGBE_DEV_ID_82599_BACKPLANE_FCOE: - case IXGBE_DEV_ID_82599_SFP_FCOE: - case IXGBE_DEV_ID_82599_SFP_EM: - case IXGBE_DEV_ID_82599_SFP_SF2: - case IXGBE_DEV_ID_82599EN_SFP: - case IXGBE_DEV_ID_82599_CX4: - case IXGBE_DEV_ID_82599_T3_LOM: - hw->mac.type = ixgbe_mac_82599EB; - break; - case IXGBE_DEV_ID_82599_VF: - hw->mac.type = ixgbe_mac_82599_vf; - break; - case IXGBE_DEV_ID_X540_VF: - hw->mac.type = ixgbe_mac_X540_vf; - break; - case IXGBE_DEV_ID_X540T: - case IXGBE_DEV_ID_X540T1: - hw->mac.type = ixgbe_mac_X540; - break; - default: - ret_val = IXGBE_ERR_DEVICE_NOT_SUPPORTED; - break; - } - } else { + switch (hw->device_id) { + case IXGBE_DEV_ID_82598: + case IXGBE_DEV_ID_82598_BX: + case IXGBE_DEV_ID_82598AF_SINGLE_PORT: + case IXGBE_DEV_ID_82598AF_DUAL_PORT: + case IXGBE_DEV_ID_82598AT: + case IXGBE_DEV_ID_82598AT2: + case IXGBE_DEV_ID_82598EB_CX4: + case IXGBE_DEV_ID_82598_CX4_DUAL_PORT: + case IXGBE_DEV_ID_82598_DA_DUAL_PORT: + case IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM: + case IXGBE_DEV_ID_82598EB_XF_LR: + case IXGBE_DEV_ID_82598EB_SFP_LOM: + hw->mac.type = ixgbe_mac_82598EB; + break; + case IXGBE_DEV_ID_82599_KX4: + case IXGBE_DEV_ID_82599_KX4_MEZZ: + case IXGBE_DEV_ID_82599_XAUI_LOM: + case IXGBE_DEV_ID_82599_COMBO_BACKPLANE: + case IXGBE_DEV_ID_82599_KR: + case IXGBE_DEV_ID_82599_SFP: + case IXGBE_DEV_ID_82599_BACKPLANE_FCOE: + case IXGBE_DEV_ID_82599_SFP_FCOE: + case IXGBE_DEV_ID_82599_SFP_EM: + case IXGBE_DEV_ID_82599_SFP_SF2: + case IXGBE_DEV_ID_82599_SFP_SF_QP: + case IXGBE_DEV_ID_82599EN_SFP: + case IXGBE_DEV_ID_82599_CX4: + case IXGBE_DEV_ID_82599_BYPASS: + case IXGBE_DEV_ID_82599_T3_LOM: + hw->mac.type = ixgbe_mac_82599EB; + break; + case IXGBE_DEV_ID_82599_VF: + case IXGBE_DEV_ID_82599_VF_HV: + hw->mac.type = ixgbe_mac_82599_vf; + break; + case IXGBE_DEV_ID_X540_VF: + case IXGBE_DEV_ID_X540_VF_HV: + hw->mac.type = ixgbe_mac_X540_vf; + break; + case IXGBE_DEV_ID_X540T: + case IXGBE_DEV_ID_X540_BYPASS: + hw->mac.type = ixgbe_mac_X540; + break; + default: ret_val = IXGBE_ERR_DEVICE_NOT_SUPPORTED; + break; } DEBUGOUT2("ixgbe_set_mac_type found mac: %d, returns: %d\n", @@ -507,16 +507,14 @@ s32 ixgbe_check_phy_link(struct ixgbe_hw *hw, ixgbe_link_speed *speed, * ixgbe_setup_phy_link_speed - Set auto advertise * @hw: pointer to hardware structure * @speed: new link speed - * @autoneg: TRUE if autonegotiation enabled * * Sets the auto advertised capabilities **/ s32 ixgbe_setup_phy_link_speed(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool autoneg_wait_to_complete) { return ixgbe_call_func(hw, hw->phy.ops.setup_link_speed, (hw, speed, - autoneg, autoneg_wait_to_complete), + autoneg_wait_to_complete), IXGBE_NOT_IMPLEMENTED); } @@ -576,17 +574,15 @@ void ixgbe_flap_tx_laser(struct ixgbe_hw *hw) * ixgbe_setup_link - Set link speed * @hw: pointer to hardware structure * @speed: new link speed - * @autoneg: TRUE if autonegotiation enabled * * Configures link settings. Restarts the link. * Performs autonegotiation if needed. **/ s32 ixgbe_setup_link(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool autoneg_wait_to_complete) { return ixgbe_call_func(hw, hw->mac.ops.setup_link, (hw, speed, - autoneg, autoneg_wait_to_complete), + autoneg_wait_to_complete), IXGBE_NOT_IMPLEMENTED); } diff --git a/sys/dev/ixgbe/ixgbe_api.h b/sys/dev/ixgbe/ixgbe_api.h index 3a9e0cf..507792f 100644 --- a/sys/dev/ixgbe/ixgbe_api.h +++ b/sys/dev/ixgbe/ixgbe_api.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -72,13 +72,12 @@ s32 ixgbe_check_phy_link(struct ixgbe_hw *hw, bool *link_up); s32 ixgbe_setup_phy_link_speed(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool autoneg_wait_to_complete); void ixgbe_disable_tx_laser(struct ixgbe_hw *hw); void ixgbe_enable_tx_laser(struct ixgbe_hw *hw); void ixgbe_flap_tx_laser(struct ixgbe_hw *hw); s32 ixgbe_setup_link(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool autoneg_wait_to_complete); + bool autoneg_wait_to_complete); s32 ixgbe_check_link(struct ixgbe_hw *hw, ixgbe_link_speed *speed, bool *link_up, bool link_up_wait_to_complete); s32 ixgbe_get_link_capabilities(struct ixgbe_hw *hw, ixgbe_link_speed *speed, @@ -159,6 +158,7 @@ void ixgbe_atr_compute_perfect_hash_82599(union ixgbe_atr_input *input, union ixgbe_atr_input *mask); u32 ixgbe_atr_compute_sig_hash_82599(union ixgbe_atr_hash_dword input, union ixgbe_atr_hash_dword common); +bool ixgbe_verify_lesm_fw_enabled_82599(struct ixgbe_hw *hw); s32 ixgbe_read_i2c_byte(struct ixgbe_hw *hw, u8 byte_offset, u8 dev_addr, u8 *data); s32 ixgbe_write_i2c_byte(struct ixgbe_hw *hw, u8 byte_offset, u8 dev_addr, diff --git a/sys/dev/ixgbe/ixgbe_common.c b/sys/dev/ixgbe/ixgbe_common.c index 51806d6..be8736f 100644 --- a/sys/dev/ixgbe/ixgbe_common.c +++ b/sys/dev/ixgbe/ixgbe_common.c @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -147,16 +147,14 @@ s32 ixgbe_init_ops_generic(struct ixgbe_hw *hw) * function check the device id to see if the associated phy supports * autoneg flow control. **/ -static s32 ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw) +s32 ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw) { DEBUGFUNC("ixgbe_device_supports_autoneg_fc"); switch (hw->device_id) { - case IXGBE_DEV_ID_X540T: - case IXGBE_DEV_ID_X540T1: - return IXGBE_SUCCESS; case IXGBE_DEV_ID_82599_T3_LOM: + case IXGBE_DEV_ID_X540T: return IXGBE_SUCCESS; default: return IXGBE_ERR_FC_NOT_SUPPORTED; @@ -174,6 +172,7 @@ static s32 ixgbe_setup_fc(struct ixgbe_hw *hw) s32 ret_val = IXGBE_SUCCESS; u32 reg = 0, reg_bp = 0; u16 reg_cu = 0; + bool got_lock = FALSE; DEBUGFUNC("ixgbe_setup_fc"); @@ -200,6 +199,7 @@ static s32 ixgbe_setup_fc(struct ixgbe_hw *hw) * we link at 10G, the 1G advertisement is harmless and vice versa. */ switch (hw->phy.media_type) { + case ixgbe_media_type_fiber_fixed: case ixgbe_media_type_fiber: case ixgbe_media_type_backplane: reg = IXGBE_READ_REG(hw, IXGBE_PCS1GANA); @@ -297,7 +297,28 @@ static s32 ixgbe_setup_fc(struct ixgbe_hw *hw) */ if (hw->phy.media_type == ixgbe_media_type_backplane) { reg_bp |= IXGBE_AUTOC_AN_RESTART; + /* Need the SW/FW semaphore around AUTOC writes if 82599 and + * LESM is on, likewise reset_pipeline requries the lock as + * it also writes AUTOC. + */ + if ((hw->mac.type == ixgbe_mac_82599EB) && + ixgbe_verify_lesm_fw_enabled_82599(hw)) { + ret_val = hw->mac.ops.acquire_swfw_sync(hw, + IXGBE_GSSR_MAC_CSR_SM); + if (ret_val != IXGBE_SUCCESS) { + ret_val = IXGBE_ERR_SWFW_SYNC; + goto out; + } + got_lock = TRUE; + } + IXGBE_WRITE_REG(hw, IXGBE_AUTOC, reg_bp); + if (hw->mac.type == ixgbe_mac_82599EB) + ixgbe_reset_pipeline_82599(hw); + + if (got_lock) + hw->mac.ops.release_swfw_sync(hw, + IXGBE_GSSR_MAC_CSR_SM); } else if ((hw->phy.media_type == ixgbe_media_type_copper) && (ixgbe_device_supports_autoneg_fc(hw) == IXGBE_SUCCESS)) { hw->phy.ops.write_reg(hw, IXGBE_MDIO_AUTO_NEG_ADVT, @@ -680,6 +701,195 @@ s32 ixgbe_read_pba_num_generic(struct ixgbe_hw *hw, u32 *pba_num) } /** + * ixgbe_read_pba_raw + * @hw: pointer to the HW structure + * @eeprom_buf: optional pointer to EEPROM image + * @eeprom_buf_size: size of EEPROM image in words + * @max_pba_block_size: PBA block size limit + * @pba: pointer to output PBA structure + * + * Reads PBA from EEPROM image when eeprom_buf is not NULL. + * Reads PBA from physical EEPROM device when eeprom_buf is NULL. + * + **/ +s32 ixgbe_read_pba_raw(struct ixgbe_hw *hw, u16 *eeprom_buf, + u32 eeprom_buf_size, u16 max_pba_block_size, + struct ixgbe_pba *pba) +{ + s32 ret_val; + u16 pba_block_size; + + if (pba == NULL) + return IXGBE_ERR_PARAM; + + if (eeprom_buf == NULL) { + ret_val = hw->eeprom.ops.read_buffer(hw, IXGBE_PBANUM0_PTR, 2, + &pba->word[0]); + if (ret_val) + return ret_val; + } else { + if (eeprom_buf_size > IXGBE_PBANUM1_PTR) { + pba->word[0] = eeprom_buf[IXGBE_PBANUM0_PTR]; + pba->word[1] = eeprom_buf[IXGBE_PBANUM1_PTR]; + } else { + return IXGBE_ERR_PARAM; + } + } + + if (pba->word[0] == IXGBE_PBANUM_PTR_GUARD) { + if (pba->pba_block == NULL) + return IXGBE_ERR_PARAM; + + ret_val = ixgbe_get_pba_block_size(hw, eeprom_buf, + eeprom_buf_size, + &pba_block_size); + if (ret_val) + return ret_val; + + if (pba_block_size > max_pba_block_size) + return IXGBE_ERR_PARAM; + + if (eeprom_buf == NULL) { + ret_val = hw->eeprom.ops.read_buffer(hw, pba->word[1], + pba_block_size, + pba->pba_block); + if (ret_val) + return ret_val; + } else { + if (eeprom_buf_size > (u32)(pba->word[1] + + pba->pba_block[0])) { + memcpy(pba->pba_block, + &eeprom_buf[pba->word[1]], + pba_block_size * sizeof(u16)); + } else { + return IXGBE_ERR_PARAM; + } + } + } + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_write_pba_raw + * @hw: pointer to the HW structure + * @eeprom_buf: optional pointer to EEPROM image + * @eeprom_buf_size: size of EEPROM image in words + * @pba: pointer to PBA structure + * + * Writes PBA to EEPROM image when eeprom_buf is not NULL. + * Writes PBA to physical EEPROM device when eeprom_buf is NULL. + * + **/ +s32 ixgbe_write_pba_raw(struct ixgbe_hw *hw, u16 *eeprom_buf, + u32 eeprom_buf_size, struct ixgbe_pba *pba) +{ + s32 ret_val; + + if (pba == NULL) + return IXGBE_ERR_PARAM; + + if (eeprom_buf == NULL) { + ret_val = hw->eeprom.ops.write_buffer(hw, IXGBE_PBANUM0_PTR, 2, + &pba->word[0]); + if (ret_val) + return ret_val; + } else { + if (eeprom_buf_size > IXGBE_PBANUM1_PTR) { + eeprom_buf[IXGBE_PBANUM0_PTR] = pba->word[0]; + eeprom_buf[IXGBE_PBANUM1_PTR] = pba->word[1]; + } else { + return IXGBE_ERR_PARAM; + } + } + + if (pba->word[0] == IXGBE_PBANUM_PTR_GUARD) { + if (pba->pba_block == NULL) + return IXGBE_ERR_PARAM; + + if (eeprom_buf == NULL) { + ret_val = hw->eeprom.ops.write_buffer(hw, pba->word[1], + pba->pba_block[0], + pba->pba_block); + if (ret_val) + return ret_val; + } else { + if (eeprom_buf_size > (u32)(pba->word[1] + + pba->pba_block[0])) { + memcpy(&eeprom_buf[pba->word[1]], + pba->pba_block, + pba->pba_block[0] * sizeof(u16)); + } else { + return IXGBE_ERR_PARAM; + } + } + } + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_get_pba_block_size + * @hw: pointer to the HW structure + * @eeprom_buf: optional pointer to EEPROM image + * @eeprom_buf_size: size of EEPROM image in words + * @pba_data_size: pointer to output variable + * + * Returns the size of the PBA block in words. Function operates on EEPROM + * image if the eeprom_buf pointer is not NULL otherwise it accesses physical + * EEPROM device. + * + **/ +s32 ixgbe_get_pba_block_size(struct ixgbe_hw *hw, u16 *eeprom_buf, + u32 eeprom_buf_size, u16 *pba_block_size) +{ + s32 ret_val; + u16 pba_word[2]; + u16 length; + + DEBUGFUNC("ixgbe_get_pba_block_size"); + + if (eeprom_buf == NULL) { + ret_val = hw->eeprom.ops.read_buffer(hw, IXGBE_PBANUM0_PTR, 2, + &pba_word[0]); + if (ret_val) + return ret_val; + } else { + if (eeprom_buf_size > IXGBE_PBANUM1_PTR) { + pba_word[0] = eeprom_buf[IXGBE_PBANUM0_PTR]; + pba_word[1] = eeprom_buf[IXGBE_PBANUM1_PTR]; + } else { + return IXGBE_ERR_PARAM; + } + } + + if (pba_word[0] == IXGBE_PBANUM_PTR_GUARD) { + if (eeprom_buf == NULL) { + ret_val = hw->eeprom.ops.read(hw, pba_word[1] + 0, + &length); + if (ret_val) + return ret_val; + } else { + if (eeprom_buf_size > pba_word[1]) + length = eeprom_buf[pba_word[1] + 0]; + else + return IXGBE_ERR_PARAM; + } + + if (length == 0xFFFF || length == 0) + return IXGBE_ERR_PBA_SECTION; + } else { + /* PBA number in legacy format, there is no PBA Block. */ + length = 0; + } + + if (pba_block_size != NULL) + *pba_block_size = length; + + return IXGBE_SUCCESS; +} + +/** * ixgbe_get_mac_addr_generic - Generic get MAC address * @hw: pointer to hardware structure * @mac_addr: Adapter MAC address @@ -1268,7 +1478,7 @@ s32 ixgbe_read_eerd_buffer_generic(struct ixgbe_hw *hw, u16 offset, } for (i = 0; i < words; i++) { - eerd = ((offset + i) << IXGBE_EEPROM_RW_ADDR_SHIFT) + + eerd = ((offset + i) << IXGBE_EEPROM_RW_ADDR_SHIFT) | IXGBE_EEPROM_RW_REG_START; IXGBE_WRITE_REG(hw, IXGBE_EERD, eerd); @@ -2719,6 +2929,7 @@ void ixgbe_fc_autoneg(struct ixgbe_hw *hw) switch (hw->phy.media_type) { /* Autoneg flow control on fiber adapters */ + case ixgbe_media_type_fiber_fixed: case ixgbe_media_type_fiber: if (speed == IXGBE_LINK_SPEED_1GB_FULL) ret_val = ixgbe_fc_autoneg_fiber(hw); @@ -2965,6 +3176,7 @@ s32 ixgbe_blink_led_start_generic(struct ixgbe_hw *hw, u32 index) bool link_up = 0; u32 autoc_reg = IXGBE_READ_REG(hw, IXGBE_AUTOC); u32 led_reg = IXGBE_READ_REG(hw, IXGBE_LEDCTL); + s32 ret_val = IXGBE_SUCCESS; DEBUGFUNC("ixgbe_blink_led_start_generic"); @@ -2975,10 +3187,29 @@ s32 ixgbe_blink_led_start_generic(struct ixgbe_hw *hw, u32 index) hw->mac.ops.check_link(hw, &speed, &link_up, FALSE); if (!link_up) { + /* Need the SW/FW semaphore around AUTOC writes if 82599 and + * LESM is on. + */ + bool got_lock = FALSE; + if ((hw->mac.type == ixgbe_mac_82599EB) && + ixgbe_verify_lesm_fw_enabled_82599(hw)) { + ret_val = hw->mac.ops.acquire_swfw_sync(hw, + IXGBE_GSSR_MAC_CSR_SM); + if (ret_val != IXGBE_SUCCESS) { + ret_val = IXGBE_ERR_SWFW_SYNC; + goto out; + } + got_lock = TRUE; + } + autoc_reg |= IXGBE_AUTOC_AN_RESTART; autoc_reg |= IXGBE_AUTOC_FLU; IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc_reg); IXGBE_WRITE_FLUSH(hw); + + if (got_lock) + hw->mac.ops.release_swfw_sync(hw, + IXGBE_GSSR_MAC_CSR_SM); msec_delay(10); } @@ -2987,7 +3218,8 @@ s32 ixgbe_blink_led_start_generic(struct ixgbe_hw *hw, u32 index) IXGBE_WRITE_REG(hw, IXGBE_LEDCTL, led_reg); IXGBE_WRITE_FLUSH(hw); - return IXGBE_SUCCESS; +out: + return ret_val; } /** @@ -2999,21 +3231,43 @@ s32 ixgbe_blink_led_stop_generic(struct ixgbe_hw *hw, u32 index) { u32 autoc_reg = IXGBE_READ_REG(hw, IXGBE_AUTOC); u32 led_reg = IXGBE_READ_REG(hw, IXGBE_LEDCTL); + s32 ret_val = IXGBE_SUCCESS; + bool got_lock = FALSE; DEBUGFUNC("ixgbe_blink_led_stop_generic"); + /* Need the SW/FW semaphore around AUTOC writes if 82599 and + * LESM is on. + */ + if ((hw->mac.type == ixgbe_mac_82599EB) && + ixgbe_verify_lesm_fw_enabled_82599(hw)) { + ret_val = hw->mac.ops.acquire_swfw_sync(hw, + IXGBE_GSSR_MAC_CSR_SM); + if (ret_val != IXGBE_SUCCESS) { + ret_val = IXGBE_ERR_SWFW_SYNC; + goto out; + } + got_lock = TRUE; + } autoc_reg &= ~IXGBE_AUTOC_FLU; autoc_reg |= IXGBE_AUTOC_AN_RESTART; IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc_reg); + if (hw->mac.type == ixgbe_mac_82599EB) + ixgbe_reset_pipeline_82599(hw); + + if (got_lock) + hw->mac.ops.release_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM); + led_reg &= ~IXGBE_LED_MODE_MASK(index); led_reg &= ~IXGBE_LED_BLINK(index); led_reg |= IXGBE_LED_LINK_ACTIVE << IXGBE_LED_MODE_SHIFT(index); IXGBE_WRITE_REG(hw, IXGBE_LEDCTL, led_reg); IXGBE_WRITE_FLUSH(hw); - return IXGBE_SUCCESS; +out: + return ret_val; } /** @@ -3882,7 +4136,7 @@ void ixgbe_enable_relaxed_ordering_gen2(struct ixgbe_hw *hw) * Calculates the checksum for some buffer on a specified length. The * checksum calculated is returned. **/ -static u8 ixgbe_calculate_checksum(u8 *buffer, u32 length) +u8 ixgbe_calculate_checksum(u8 *buffer, u32 length) { u32 i; u8 sum = 0; @@ -3908,8 +4162,8 @@ static u8 ixgbe_calculate_checksum(u8 *buffer, u32 length) * Communicates with the manageability block. On success return IXGBE_SUCCESS * else return IXGBE_ERR_HOST_INTERFACE_COMMAND. **/ -static s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, u32 *buffer, - u32 length) +s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, u32 *buffer, + u32 length) { u32 hicr, i, bi; u32 hdr_size = sizeof(struct ixgbe_hic_hdr); diff --git a/sys/dev/ixgbe/ixgbe_common.h b/sys/dev/ixgbe/ixgbe_common.h index 3bbc537..cd9a13d 100644 --- a/sys/dev/ixgbe/ixgbe_common.h +++ b/sys/dev/ixgbe/ixgbe_common.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -41,9 +41,14 @@ IXGBE_WRITE_REG(hw, reg, (u32) value); \ IXGBE_WRITE_REG(hw, reg + 4, (u32) (value >> 32)); \ } while (0) +#if !defined(NO_READ_PBA_RAW) || !defined(NO_WRITE_PBA_RAW) +struct ixgbe_pba { + u16 word[2]; + u16 *pba_block; +}; +#endif u16 ixgbe_get_pcie_msix_count_generic(struct ixgbe_hw *hw); - s32 ixgbe_init_ops_generic(struct ixgbe_hw *hw); s32 ixgbe_init_hw_generic(struct ixgbe_hw *hw); s32 ixgbe_start_hw_generic(struct ixgbe_hw *hw); @@ -52,6 +57,13 @@ s32 ixgbe_clear_hw_cntrs_generic(struct ixgbe_hw *hw); s32 ixgbe_read_pba_num_generic(struct ixgbe_hw *hw, u32 *pba_num); s32 ixgbe_read_pba_string_generic(struct ixgbe_hw *hw, u8 *pba_num, u32 pba_num_size); +s32 ixgbe_read_pba_raw(struct ixgbe_hw *hw, u16 *eeprom_buf, + u32 eeprom_buf_size, u16 max_pba_block_size, + struct ixgbe_pba *pba); +s32 ixgbe_write_pba_raw(struct ixgbe_hw *hw, u16 *eeprom_buf, + u32 eeprom_buf_size, struct ixgbe_pba *pba); +s32 ixgbe_get_pba_block_size(struct ixgbe_hw *hw, u16 *eeprom_buf, + u32 eeprom_buf_size, u16 *pba_block_size); s32 ixgbe_get_mac_addr_generic(struct ixgbe_hw *hw, u8 *mac_addr); s32 ixgbe_get_bus_info_generic(struct ixgbe_hw *hw); void ixgbe_set_lan_id_multi_port_pcie(struct ixgbe_hw *hw); @@ -96,6 +108,7 @@ s32 ixgbe_disable_sec_rx_path_generic(struct ixgbe_hw *hw); s32 ixgbe_enable_sec_rx_path_generic(struct ixgbe_hw *hw); s32 ixgbe_fc_enable_generic(struct ixgbe_hw *hw); +s32 ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw); void ixgbe_fc_autoneg(struct ixgbe_hw *hw); s32 ixgbe_validate_mac_addr(u8 *mac_addr); @@ -137,5 +150,11 @@ void ixgbe_set_rxpba_generic(struct ixgbe_hw *hw, int num_pb, u32 headroom, void ixgbe_enable_relaxed_ordering_gen2(struct ixgbe_hw *hw); s32 ixgbe_set_fw_drv_ver_generic(struct ixgbe_hw *hw, u8 maj, u8 min, u8 build, u8 ver); +u8 ixgbe_calculate_checksum(u8 *buffer, u32 length); +s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, u32 *buffer, + u32 length); void ixgbe_clear_tx_pending(struct ixgbe_hw *hw); + +extern s32 ixgbe_reset_pipeline_82599(struct ixgbe_hw *hw); + #endif /* IXGBE_COMMON */ diff --git a/sys/dev/ixgbe/ixgbe_mbx.h b/sys/dev/ixgbe/ixgbe_mbx.h index 8ad18cb..adcba9e 100644 --- a/sys/dev/ixgbe/ixgbe_mbx.h +++ b/sys/dev/ixgbe/ixgbe_mbx.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -84,8 +84,20 @@ #define IXGBE_VF_SET_MAC_ADDR 0x02 /* VF requests PF to set MAC addr */ #define IXGBE_VF_SET_MULTICAST 0x03 /* VF requests PF to set MC addr */ #define IXGBE_VF_SET_VLAN 0x04 /* VF requests PF to set VLAN */ + +/* mailbox API, version 1.0 VF requests */ #define IXGBE_VF_SET_LPE 0x05 /* VF requests PF to set VMOLR.LPE */ #define IXGBE_VF_SET_MACVLAN 0x06 /* VF requests PF for unicast filter */ +#define IXGBE_VF_API_NEGOTIATE 0x08 /* negotiate API version */ + +/* mailbox API, version 1.1 VF requests */ +#define IXGBE_VF_GET_QUEUES 0x09 /* get queue configuration */ + +/* GET_QUEUES return data indices within the mailbox */ +#define IXGBE_VF_TX_QUEUES 1 /* number of Tx queues supported */ +#define IXGBE_VF_RX_QUEUES 2 /* number of Rx queues supported */ +#define IXGBE_VF_TRANS_VLAN 3 /* Indication of port vlan */ +#define IXGBE_VF_DEF_QUEUE 4 /* Default queue offset */ /* length of permanent address message returned from PF */ #define IXGBE_VF_PERMADDR_MSG_LEN 4 diff --git a/sys/dev/ixgbe/ixgbe_osdep.h b/sys/dev/ixgbe/ixgbe_osdep.h index a07a625..afd5ab0 100644 --- a/sys/dev/ixgbe/ixgbe_osdep.h +++ b/sys/dev/ixgbe/ixgbe_osdep.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -91,6 +91,9 @@ #define CMD_MEM_WRT_INVALIDATE 0x0010 /* BIT_4 */ #define PCI_COMMAND_REGISTER PCIR_COMMAND +/* Shared code dropped this define.. */ +#define IXGBE_INTEL_VENDOR_ID 0x8086 + /* Bunch of defines for shared code bogosity */ #define UNREFERENCED_PARAMETER(_p) #define UNREFERENCED_1PARAMETER(_p) diff --git a/sys/dev/ixgbe/ixgbe_phy.c b/sys/dev/ixgbe/ixgbe_phy.c index 3f262f9..b22797f 100644 --- a/sys/dev/ixgbe/ixgbe_phy.c +++ b/sys/dev/ixgbe/ixgbe_phy.c @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -47,6 +47,8 @@ static void ixgbe_raise_i2c_clk(struct ixgbe_hw *hw, u32 *i2cctl); static void ixgbe_lower_i2c_clk(struct ixgbe_hw *hw, u32 *i2cctl); static s32 ixgbe_set_i2c_data(struct ixgbe_hw *hw, u32 *i2cctl, bool data); static bool ixgbe_get_i2c_data(u32 *i2cctl); +static s32 ixgbe_read_i2c_sff8472_generic(struct ixgbe_hw *hw, u8 byte_offset, + u8 *sff8472_data); /** * ixgbe_init_phy_ops_generic - Inits PHY function ptrs @@ -71,6 +73,7 @@ s32 ixgbe_init_phy_ops_generic(struct ixgbe_hw *hw) phy->ops.get_firmware_version = ixgbe_get_phy_firmware_version_generic; phy->ops.read_i2c_byte = &ixgbe_read_i2c_byte_generic; phy->ops.write_i2c_byte = &ixgbe_write_i2c_byte_generic; + phy->ops.read_i2c_sff8472 = &ixgbe_read_i2c_sff8472_generic; phy->ops.read_i2c_eeprom = &ixgbe_read_i2c_eeprom_generic; phy->ops.write_i2c_eeprom = &ixgbe_write_i2c_eeprom_generic; phy->ops.i2c_bus_clear = &ixgbe_i2c_bus_clear; @@ -563,14 +566,12 @@ s32 ixgbe_setup_phy_link_generic(struct ixgbe_hw *hw) * ixgbe_setup_phy_link_speed_generic - Sets the auto advertised capabilities * @hw: pointer to hardware structure * @speed: new link speed - * @autoneg: TRUE if autonegotiation enabled **/ s32 ixgbe_setup_phy_link_speed_generic(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool autoneg_wait_to_complete) { - UNREFERENCED_2PARAMETER(autoneg, autoneg_wait_to_complete); + UNREFERENCED_1PARAMETER(autoneg_wait_to_complete); DEBUGFUNC("ixgbe_setup_phy_link_speed_generic"); @@ -969,9 +970,7 @@ s32 ixgbe_identify_sfp_module_generic(struct ixgbe_hw *hw) IXGBE_SFF_IDENTIFIER, &identifier); - if (status == IXGBE_ERR_SWFW_SYNC || - status == IXGBE_ERR_I2C || - status == IXGBE_ERR_SFP_NOT_PRESENT) + if (status != IXGBE_SUCCESS) goto err_read_i2c_eeprom; /* LAN ID is needed for sfp_type determination */ @@ -985,26 +984,20 @@ s32 ixgbe_identify_sfp_module_generic(struct ixgbe_hw *hw) IXGBE_SFF_1GBE_COMP_CODES, &comp_codes_1g); - if (status == IXGBE_ERR_SWFW_SYNC || - status == IXGBE_ERR_I2C || - status == IXGBE_ERR_SFP_NOT_PRESENT) + if (status != IXGBE_SUCCESS) goto err_read_i2c_eeprom; status = hw->phy.ops.read_i2c_eeprom(hw, IXGBE_SFF_10GBE_COMP_CODES, &comp_codes_10g); - if (status == IXGBE_ERR_SWFW_SYNC || - status == IXGBE_ERR_I2C || - status == IXGBE_ERR_SFP_NOT_PRESENT) + if (status != IXGBE_SUCCESS) goto err_read_i2c_eeprom; status = hw->phy.ops.read_i2c_eeprom(hw, IXGBE_SFF_CABLE_TECHNOLOGY, &cable_tech); - if (status == IXGBE_ERR_SWFW_SYNC || - status == IXGBE_ERR_I2C || - status == IXGBE_ERR_SFP_NOT_PRESENT) + if (status != IXGBE_SUCCESS) goto err_read_i2c_eeprom; /* ID Module @@ -1102,27 +1095,21 @@ s32 ixgbe_identify_sfp_module_generic(struct ixgbe_hw *hw) IXGBE_SFF_VENDOR_OUI_BYTE0, &oui_bytes[0]); - if (status == IXGBE_ERR_SWFW_SYNC || - status == IXGBE_ERR_I2C || - status == IXGBE_ERR_SFP_NOT_PRESENT) + if (status != IXGBE_SUCCESS) goto err_read_i2c_eeprom; status = hw->phy.ops.read_i2c_eeprom(hw, IXGBE_SFF_VENDOR_OUI_BYTE1, &oui_bytes[1]); - if (status == IXGBE_ERR_SWFW_SYNC || - status == IXGBE_ERR_I2C || - status == IXGBE_ERR_SFP_NOT_PRESENT) + if (status != IXGBE_SUCCESS) goto err_read_i2c_eeprom; status = hw->phy.ops.read_i2c_eeprom(hw, IXGBE_SFF_VENDOR_OUI_BYTE2, &oui_bytes[2]); - if (status == IXGBE_ERR_SWFW_SYNC || - status == IXGBE_ERR_I2C || - status == IXGBE_ERR_SFP_NOT_PRESENT) + if (status != IXGBE_SUCCESS) goto err_read_i2c_eeprom; vendor_oui = @@ -1333,6 +1320,22 @@ s32 ixgbe_read_i2c_eeprom_generic(struct ixgbe_hw *hw, u8 byte_offset, } /** + * ixgbe_read_i2c_sff8472_generic - Reads 8 bit word over I2C interface + * @hw: pointer to hardware structure + * @byte_offset: byte offset at address 0xA2 + * @eeprom_data: value read + * + * Performs byte read operation to SFP module's SFF-8472 data over I2C + **/ +static s32 ixgbe_read_i2c_sff8472_generic(struct ixgbe_hw *hw, u8 byte_offset, + u8 *sff8472_data) +{ + return hw->phy.ops.read_i2c_byte(hw, byte_offset, + IXGBE_I2C_EEPROM_DEV_ADDR2, + sff8472_data); +} + +/** * ixgbe_write_i2c_eeprom_generic - Writes 8 bit EEPROM word over I2C interface * @hw: pointer to hardware structure * @byte_offset: EEPROM byte offset to write @@ -1425,9 +1428,9 @@ s32 ixgbe_read_i2c_byte_generic(struct ixgbe_hw *hw, u8 byte_offset, break; fail: + ixgbe_i2c_bus_clear(hw); hw->mac.ops.release_swfw_sync(hw, swfw_mask); msec_delay(100); - ixgbe_i2c_bus_clear(hw); retry++; if (retry < max_retry) DEBUGOUT("I2C byte read error - Retrying.\n"); diff --git a/sys/dev/ixgbe/ixgbe_phy.h b/sys/dev/ixgbe/ixgbe_phy.h index 017ea88..5b2e28b 100644 --- a/sys/dev/ixgbe/ixgbe_phy.h +++ b/sys/dev/ixgbe/ixgbe_phy.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -36,7 +36,9 @@ #define _IXGBE_PHY_H_ #include "ixgbe_type.h" -#define IXGBE_I2C_EEPROM_DEV_ADDR 0xA0 +#define IXGBE_I2C_EEPROM_DEV_ADDR 0xA0 +#define IXGBE_I2C_EEPROM_DEV_ADDR2 0xA2 +#define IXGBE_I2C_EEPROM_BANK_LEN 0xFF /* EEPROM byte offsets */ #define IXGBE_SFF_IDENTIFIER 0x0 @@ -48,6 +50,10 @@ #define IXGBE_SFF_10GBE_COMP_CODES 0x3 #define IXGBE_SFF_CABLE_TECHNOLOGY 0x8 #define IXGBE_SFF_CABLE_SPEC_COMP 0x3C +#define IXGBE_SFF_SFF_8472_SWAP 0x5C +#define IXGBE_SFF_SFF_8472_COMP 0x5E +#define IXGBE_SFF_SFF_8472_OSCB 0x6E +#define IXGBE_SFF_SFF_8472_ESCB 0x76 /* Bitmasks */ #define IXGBE_SFF_DA_PASSIVE_CABLE 0x4 @@ -58,6 +64,9 @@ #define IXGBE_SFF_1GBASET_CAPABLE 0x8 #define IXGBE_SFF_10GBASESR_CAPABLE 0x10 #define IXGBE_SFF_10GBASELR_CAPABLE 0x20 +#define IXGBE_SFF_SOFT_RS_SELECT_MASK 0x8 +#define IXGBE_SFF_SOFT_RS_SELECT_10G 0x8 +#define IXGBE_SFF_SOFT_RS_SELECT_1G 0x0 #define IXGBE_I2C_EEPROM_READ_MASK 0x100 #define IXGBE_I2C_EEPROM_STATUS_MASK 0x3 #define IXGBE_I2C_EEPROM_STATUS_NO_OPERATION 0x0 @@ -95,6 +104,14 @@ #define IXGBE_TN_LASI_STATUS_REG 0x9005 #define IXGBE_TN_LASI_STATUS_TEMP_ALARM 0x0008 +/* SFP+ SFF-8472 Compliance */ +#define IXGBE_SFF_SFF_8472_UNSUP 0x00 +#define IXGBE_SFF_SFF_8472_REV_9_3 0x01 +#define IXGBE_SFF_SFF_8472_REV_9_5 0x02 +#define IXGBE_SFF_SFF_8472_REV_10_2 0x03 +#define IXGBE_SFF_SFF_8472_REV_10_4 0x04 +#define IXGBE_SFF_SFF_8472_REV_11_0 0x05 + s32 ixgbe_init_phy_ops_generic(struct ixgbe_hw *hw); bool ixgbe_validate_phy_addr(struct ixgbe_hw *hw, u32 phy_addr); enum ixgbe_phy_type ixgbe_get_phy_type_from_id(u32 phy_id); @@ -108,7 +125,6 @@ s32 ixgbe_write_phy_reg_generic(struct ixgbe_hw *hw, u32 reg_addr, s32 ixgbe_setup_phy_link_generic(struct ixgbe_hw *hw); s32 ixgbe_setup_phy_link_speed_generic(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool autoneg_wait_to_complete); s32 ixgbe_get_copper_link_capabilities_generic(struct ixgbe_hw *hw, ixgbe_link_speed *speed, diff --git a/sys/dev/ixgbe/ixgbe_type.h b/sys/dev/ixgbe/ixgbe_type.h index 52944fd..344097d 100644 --- a/sys/dev/ixgbe/ixgbe_type.h +++ b/sys/dev/ixgbe/ixgbe_type.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -38,9 +38,6 @@ #include "ixgbe_osdep.h" -/* Vendor ID */ -#define IXGBE_INTEL_VENDOR_ID 0x8086 - /* Device IDs */ #define IXGBE_DEV_ID_82598 0x10B6 #define IXGBE_DEV_ID_82598_BX 0x1508 @@ -62,18 +59,24 @@ #define IXGBE_DEV_ID_82599_CX4 0x10F9 #define IXGBE_DEV_ID_82599_SFP 0x10FB #define IXGBE_SUBDEV_ID_82599_SFP 0x11A9 +#define IXGBE_SUBDEV_ID_82599_RNDC 0x1F72 #define IXGBE_SUBDEV_ID_82599_560FLR 0x17D0 +#define IXGBE_SUBDEV_ID_82599_ECNA_DP 0x0470 #define IXGBE_DEV_ID_82599_BACKPLANE_FCOE 0x152A #define IXGBE_DEV_ID_82599_SFP_FCOE 0x1529 #define IXGBE_DEV_ID_82599_SFP_EM 0x1507 #define IXGBE_DEV_ID_82599_SFP_SF2 0x154D +#define IXGBE_DEV_ID_82599_SFP_SF_QP 0x154A #define IXGBE_DEV_ID_82599EN_SFP 0x1557 #define IXGBE_DEV_ID_82599_XAUI_LOM 0x10FC #define IXGBE_DEV_ID_82599_T3_LOM 0x151C #define IXGBE_DEV_ID_82599_VF 0x10ED -#define IXGBE_DEV_ID_X540_VF 0x1515 +#define IXGBE_DEV_ID_82599_VF_HV 0x152E +#define IXGBE_DEV_ID_82599_BYPASS 0x155D #define IXGBE_DEV_ID_X540T 0x1528 -#define IXGBE_DEV_ID_X540T1 0x1560 +#define IXGBE_DEV_ID_X540_VF 0x1515 +#define IXGBE_DEV_ID_X540_VF_HV 0x1530 +#define IXGBE_DEV_ID_X540_BYPASS 0x155C /* General Registers */ #define IXGBE_CTRL 0x00000 @@ -280,6 +283,7 @@ #define IXGBE_RETA(_i) (0x05C00 + ((_i) * 4)) /* 32 of these (0-31) */ #define IXGBE_RSSRK(_i) (0x05C80 + ((_i) * 4)) /* 10 of these (0-9) */ + /* Flow Director registers */ #define IXGBE_FDIRCTRL 0x0EE00 #define IXGBE_FDIRHKEY 0x0EE68 @@ -360,11 +364,16 @@ #define IXGBE_WUPL 0x05900 #define IXGBE_WUPM 0x05A00 /* wake up pkt memory 0x5A00-0x5A7C */ + #define IXGBE_FHFT(_n) (0x09000 + (_n * 0x100)) /* Flex host filter table */ /* Ext Flexible Host Filter Table */ #define IXGBE_FHFT_EXT(_n) (0x09800 + (_n * 0x100)) +/* Four Flexible Filters are supported */ #define IXGBE_FLEXIBLE_FILTER_COUNT_MAX 4 + +/* Six Flexible Filters are supported */ +#define IXGBE_FLEXIBLE_FILTER_COUNT_MAX_6 6 #define IXGBE_EXT_FLEXIBLE_FILTER_COUNT_MAX 2 /* Each Flexible Filter is at most 128 (0x80) bytes in length */ @@ -396,10 +405,11 @@ #define IXGBE_WUFC_FLX3 0x00080000 /* Flexible Filter 3 Enable */ #define IXGBE_WUFC_FLX4 0x00100000 /* Flexible Filter 4 Enable */ #define IXGBE_WUFC_FLX5 0x00200000 /* Flexible Filter 5 Enable */ -#define IXGBE_WUFC_FLX_FILTERS 0x000F0000 /* Mask for 4 flex filters */ +#define IXGBE_WUFC_FLX_FILTERS 0x000F0000 /* Mask for 4 flex filters */ /* Mask for Ext. flex filters */ #define IXGBE_WUFC_EXT_FLX_FILTERS 0x00300000 -#define IXGBE_WUFC_ALL_FILTERS 0x003F00FF /* Mask for all wakeup filters */ +#define IXGBE_WUFC_ALL_FILTERS 0x000F00FF /* Mask all 4 flex filters */ +#define IXGBE_WUFC_ALL_FILTERS_6 0x003F00FF /* Mask all 6 flex filters */ #define IXGBE_WUFC_FLX_OFFSET 16 /* Offset to the Flexible Filters bits */ /* Wake Up Status */ @@ -420,7 +430,6 @@ #define IXGBE_WUS_FLX5 IXGBE_WUFC_FLX5 #define IXGBE_WUS_FLX_FILTERS IXGBE_WUFC_FLX_FILTERS -/* Wake Up Packet Length */ #define IXGBE_WUPL_LENGTH_MASK 0xFFFF /* DCB registers */ @@ -574,6 +583,7 @@ #define IXGBE_RTTBCNRTT 0x05150 #define IXGBE_RTTBCNRD 0x0498C + /* FCoE DMA Context Registers */ #define IXGBE_FCPTRL 0x02410 /* FC User Desc. PTR Low */ #define IXGBE_FCPTRH 0x02414 /* FC USer Desc. PTR High */ @@ -754,11 +764,14 @@ #define IXGBE_BMCIP_IPADDR_VALID 0x00000002 /* Management Bit Fields and Masks */ +#define IXGBE_MANC_RCV_TCO_EN 0x00020000 /* Rcv TCO packet enable */ #define IXGBE_MANC_EN_BMC2OS 0x10000000 /* Ena BMC2OS and OS2BMC traffic */ #define IXGBE_MANC_EN_BMC2OS_SHIFT 28 /* Firmware Semaphore Register */ #define IXGBE_FWSM_MODE_MASK 0xE +#define IXGBE_FWSM_TS_ENABLED 0x1 +#define IXGBE_FWSM_FW_MODE_PT 0x4 /* ARC Subsystem registers */ #define IXGBE_HICR 0x15F00 @@ -1014,6 +1027,7 @@ #define IXGBE_RSCCTL_MAXDESC_4 0x04 #define IXGBE_RSCCTL_MAXDESC_8 0x08 #define IXGBE_RSCCTL_MAXDESC_16 0x0C +#define IXGBE_RSCCTL_TS_DIS 0x02 /* RSCDBU Bit Masks */ #define IXGBE_RSCDBU_RSCSMALDIS_MASK 0x0000007F @@ -1026,7 +1040,7 @@ #define IXGBE_RDRXCTL_DMAIDONE 0x00000008 /* DMA init cycle done */ #define IXGBE_RDRXCTL_AGGDIS 0x00010000 /* Aggregation disable */ #define IXGBE_RDRXCTL_RSCFRSTSIZE 0x003E0000 /* RSC First packet size */ -#define IXGBE_RDRXCTL_RSCLLIDIS 0x00800000 /* Disabl RSC compl on LLI */ +#define IXGBE_RDRXCTL_RSCLLIDIS 0x00800000 /* Disable RSC compl on LLI*/ #define IXGBE_RDRXCTL_RSCACKC 0x02000000 /* must set 1 when RSC ena */ #define IXGBE_RDRXCTL_FCOE_WRFIX 0x04000000 /* must set 1 when RSC ena */ @@ -1052,6 +1066,7 @@ #define IXGBE_CTRL_RST_MASK (IXGBE_CTRL_LNK_RST | IXGBE_CTRL_RST) /* FACTPS */ +#define IXGBE_FACTPS_MNGCG 0x20000000 /* Manageblility Clock Gated */ #define IXGBE_FACTPS_LFS 0x40000000 /* LAN Function Select */ /* MHADD Bit Masks */ @@ -1590,6 +1605,7 @@ enum { #define IXGBE_ESDP_SDP7 0x00000080 /* SDP7 Data Value */ #define IXGBE_ESDP_SDP0_DIR 0x00000100 /* SDP0 IO direction */ #define IXGBE_ESDP_SDP1_DIR 0x00000200 /* SDP1 IO direction */ +#define IXGBE_ESDP_SDP2_DIR 0x00000400 /* SDP1 IO direction */ #define IXGBE_ESDP_SDP3_DIR 0x00000800 /* SDP3 IO direction */ #define IXGBE_ESDP_SDP4_DIR 0x00001000 /* SDP4 IO direction */ #define IXGBE_ESDP_SDP5_DIR 0x00002000 /* SDP5 IO direction */ @@ -1668,6 +1684,7 @@ enum { #define IXGBE_AUTOC2_10G_KR (0x0 << IXGBE_AUTOC2_10G_SERIAL_PMA_PMD_SHIFT) #define IXGBE_AUTOC2_10G_XFI (0x1 << IXGBE_AUTOC2_10G_SERIAL_PMA_PMD_SHIFT) #define IXGBE_AUTOC2_10G_SFI (0x2 << IXGBE_AUTOC2_10G_SERIAL_PMA_PMD_SHIFT) +#define IXGBE_AUTOC2_LINK_DISABLE_MASK 0x70000000 #define IXGBE_MACC_FLU 0x00000001 #define IXGBE_MACC_FSV_10G 0x00030000 @@ -1838,7 +1855,7 @@ enum { #define IXGBE_ETH_LENGTH_OF_ADDRESS 6 #define IXGBE_EEPROM_PAGE_SIZE_MAX 128 -#define IXGBE_EEPROM_RD_BUFFER_MAX_COUNT 512 /* words rd in burst */ +#define IXGBE_EEPROM_RD_BUFFER_MAX_COUNT 256 /* words rd in burst */ #define IXGBE_EEPROM_WR_BUFFER_MAX_COUNT 256 /* words wr in burst */ #ifndef IXGBE_EEPROM_GRANT_ATTEMPTS @@ -2524,7 +2541,6 @@ typedef u32 ixgbe_link_speed; IXGBE_LINK_SPEED_1GB_FULL | \ IXGBE_LINK_SPEED_10GB_FULL) - /* Physical layer type */ typedef u32 ixgbe_physical_layer; #define IXGBE_PHYSICAL_LAYER_UNKNOWN 0 @@ -2757,6 +2773,7 @@ enum ixgbe_sfp_type { enum ixgbe_media_type { ixgbe_media_type_unknown = 0, ixgbe_media_type_fiber, + ixgbe_media_type_fiber_fixed, ixgbe_media_type_copper, ixgbe_media_type_backplane, ixgbe_media_type_cx4, @@ -2975,7 +2992,7 @@ struct ixgbe_mac_operations { void (*disable_tx_laser)(struct ixgbe_hw *); void (*enable_tx_laser)(struct ixgbe_hw *); void (*flap_tx_laser)(struct ixgbe_hw *); - s32 (*setup_link)(struct ixgbe_hw *, ixgbe_link_speed, bool, bool); + s32 (*setup_link)(struct ixgbe_hw *, ixgbe_link_speed, bool); s32 (*check_link)(struct ixgbe_hw *, ixgbe_link_speed *, bool *, bool); s32 (*get_link_capabilities)(struct ixgbe_hw *, ixgbe_link_speed *, bool *); @@ -3026,12 +3043,12 @@ struct ixgbe_phy_operations { s32 (*read_reg)(struct ixgbe_hw *, u32, u32, u16 *); s32 (*write_reg)(struct ixgbe_hw *, u32, u32, u16); s32 (*setup_link)(struct ixgbe_hw *); - s32 (*setup_link_speed)(struct ixgbe_hw *, ixgbe_link_speed, bool, - bool); + s32 (*setup_link_speed)(struct ixgbe_hw *, ixgbe_link_speed, bool); s32 (*check_link)(struct ixgbe_hw *, ixgbe_link_speed *, bool *); s32 (*get_firmware_version)(struct ixgbe_hw *, u16 *); s32 (*read_i2c_byte)(struct ixgbe_hw *, u8, u8, u8 *); s32 (*write_i2c_byte)(struct ixgbe_hw *, u8, u8, u8); + s32 (*read_i2c_sff8472)(struct ixgbe_hw *, u8 , u8 *); s32 (*read_i2c_eeprom)(struct ixgbe_hw *, u8 , u8 *); s32 (*write_i2c_eeprom)(struct ixgbe_hw *, u8, u8); void (*i2c_bus_clear)(struct ixgbe_hw *); @@ -3069,7 +3086,9 @@ struct ixgbe_mac_info { u32 max_tx_queues; u32 max_rx_queues; u32 orig_autoc; + u32 cached_autoc; u8 san_mac_rar_index; + bool get_link_status; u32 orig_autoc2; u16 max_msix_vectors; bool arc_subsystem_valid; @@ -3142,6 +3161,7 @@ struct ixgbe_hw { u16 subsystem_vendor_id; u8 revision_id; bool adapter_stopped; + int api_version; bool force_full_reset; bool allow_unsupported_sfp; }; @@ -3185,6 +3205,7 @@ struct ixgbe_hw { #define IXGBE_ERR_INVALID_ARGUMENT -32 #define IXGBE_ERR_HOST_INTERFACE_COMMAND -33 #define IXGBE_ERR_OUT_OF_MEM -34 +#define IXGBE_ERR_FEATURE_NOT_SUPPORTED -36 #define IXGBE_NOT_IMPLEMENTED 0x7FFFFFFF diff --git a/sys/dev/ixgbe/ixgbe_vf.c b/sys/dev/ixgbe/ixgbe_vf.c index 9e3dd72..14834a2 100644 --- a/sys/dev/ixgbe/ixgbe_vf.c +++ b/sys/dev/ixgbe/ixgbe_vf.c @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -142,6 +142,7 @@ s32 ixgbe_reset_hw_vf(struct ixgbe_hw *hw) /* Call adapter stop to disable tx/rx and clear interrupts */ hw->mac.ops.stop_adapter(hw); + DEBUGOUT("Issuing a function level reset to MAC\n"); ctrl = IXGBE_VFREAD_REG(hw, IXGBE_VFCTRL) | IXGBE_CTRL_RST; @@ -272,6 +273,17 @@ static s32 ixgbe_mta_vector(struct ixgbe_hw *hw, u8 *mc_addr) return vector; } +static void ixgbevf_write_msg_read_ack(struct ixgbe_hw *hw, + u32 *msg, u16 size) +{ + struct ixgbe_mbx_info *mbx = &hw->mbx; + u32 retmsg[IXGBE_VFMAILBOX_SIZE]; + s32 retval = mbx->ops.write_posted(hw, msg, size, 0); + + if (!retval) + mbx->ops.read_posted(hw, retmsg, size, 0); +} + /** * ixgbe_set_rar_vf - set device MAC address * @hw: pointer to hardware structure @@ -463,11 +475,10 @@ s32 ixgbevf_set_uc_addr_vf(struct ixgbe_hw *hw, u32 index, u8 *addr) * * Set the link speed in the AUTOC register and restarts link. **/ -s32 ixgbe_setup_mac_link_vf(struct ixgbe_hw *hw, - ixgbe_link_speed speed, bool autoneg, +s32 ixgbe_setup_mac_link_vf(struct ixgbe_hw *hw, ixgbe_link_speed speed, bool autoneg_wait_to_complete) { - UNREFERENCED_4PARAMETER(hw, speed, autoneg, autoneg_wait_to_complete); + UNREFERENCED_3PARAMETER(hw, speed, autoneg_wait_to_complete); return IXGBE_SUCCESS; } @@ -483,23 +494,26 @@ s32 ixgbe_setup_mac_link_vf(struct ixgbe_hw *hw, s32 ixgbe_check_mac_link_vf(struct ixgbe_hw *hw, ixgbe_link_speed *speed, bool *link_up, bool autoneg_wait_to_complete) { + struct ixgbe_mbx_info *mbx = &hw->mbx; + struct ixgbe_mac_info *mac = &hw->mac; + s32 ret_val = IXGBE_SUCCESS; u32 links_reg; + u32 in_msg = 0; UNREFERENCED_1PARAMETER(autoneg_wait_to_complete); - if (!(hw->mbx.ops.check_for_rst(hw, 0))) { - *link_up = FALSE; - *speed = 0; - return -1; - } + /* If we were hit with a reset drop the link */ + if (!mbx->ops.check_for_rst(hw, 0) || !mbx->timeout) + mac->get_link_status = TRUE; - links_reg = IXGBE_VFREAD_REG(hw, IXGBE_VFLINKS); + if (!mac->get_link_status) + goto out; - if (links_reg & IXGBE_LINKS_UP) - *link_up = TRUE; - else - *link_up = FALSE; + /* if link status is down no point in checking to see if pf is up */ + links_reg = IXGBE_READ_REG(hw, IXGBE_VFLINKS); + if (!(links_reg & IXGBE_LINKS_UP)) + goto out; - switch (links_reg & IXGBE_LINKS_SPEED_10G_82599) { + switch (links_reg & IXGBE_LINKS_SPEED_82599) { case IXGBE_LINKS_SPEED_10G_82599: *speed = IXGBE_LINK_SPEED_10GB_FULL; break; @@ -511,6 +525,87 @@ s32 ixgbe_check_mac_link_vf(struct ixgbe_hw *hw, ixgbe_link_speed *speed, break; } + /* if the read failed it could just be a mailbox collision, best wait + * until we are called again and don't report an error + */ + if (mbx->ops.read(hw, &in_msg, 1, 0)) + goto out; + + if (!(in_msg & IXGBE_VT_MSGTYPE_CTS)) { + /* msg is not CTS and is NACK we must have lost CTS status */ + if (in_msg & IXGBE_VT_MSGTYPE_NACK) + ret_val = -1; + goto out; + } + + /* the pf is talking, if we timed out in the past we reinit */ + if (!mbx->timeout) { + ret_val = -1; + goto out; + } + + /* if we passed all the tests above then the link is up and we no + * longer need to check for link + */ + mac->get_link_status = FALSE; + +out: + *link_up = !mac->get_link_status; + return ret_val; +} + +/** + * ixgbevf_rlpml_set_vf - Set the maximum receive packet length + * @hw: pointer to the HW structure + * @max_size: value to assign to max frame size + **/ +void ixgbevf_rlpml_set_vf(struct ixgbe_hw *hw, u16 max_size) +{ + u32 msgbuf[2]; + + msgbuf[0] = IXGBE_VF_SET_LPE; + msgbuf[1] = max_size; + ixgbevf_write_msg_read_ack(hw, msgbuf, 2); +} + +/** + * ixgbevf_negotiate_api_version - Negotiate supported API version + * @hw: pointer to the HW structure + * @api: integer containing requested API version + **/ +int ixgbevf_negotiate_api_version(struct ixgbe_hw *hw, int api) +{ + int err; + u32 msg[3]; + + /* Negotiate the mailbox API version */ + msg[0] = IXGBE_VF_API_NEGOTIATE; + msg[1] = api; + msg[2] = 0; + err = hw->mbx.ops.write_posted(hw, msg, 3, 0); + + if (!err) + err = hw->mbx.ops.read_posted(hw, msg, 3, 0); + + if (!err) { + msg[0] &= ~IXGBE_VT_MSGTYPE_CTS; + + /* Store value and return 0 on success */ + if (msg[0] == (IXGBE_VF_API_NEGOTIATE | IXGBE_VT_MSGTYPE_ACK)) { + hw->api_version = api; + return 0; + } + + err = IXGBE_ERR_INVALID_ARGUMENT; + } + + return err; +} + +int ixgbevf_get_queues(struct ixgbe_hw *hw, unsigned int *num_tcs, + unsigned int *default_tc) +{ + UNREFERENCED_3PARAMETER(hw, num_tcs, default_tc); return IXGBE_SUCCESS; } diff --git a/sys/dev/ixgbe/ixgbe_vf.h b/sys/dev/ixgbe/ixgbe_vf.h index 5c77ba4..8500768 100644 --- a/sys/dev/ixgbe/ixgbe_vf.h +++ b/sys/dev/ixgbe/ixgbe_vf.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -39,6 +39,9 @@ #define IXGBE_VF_MAX_TX_QUEUES 8 #define IXGBE_VF_MAX_RX_QUEUES 8 +/* DCB define */ +#define IXGBE_VF_MAX_TRAFFIC_CLASS 8 + #define IXGBE_VFCTRL 0x00000 #define IXGBE_VFSTATUS 0x00008 #define IXGBE_VFLINKS 0x00010 @@ -117,7 +120,7 @@ u32 ixgbe_get_num_of_tx_queues_vf(struct ixgbe_hw *hw); u32 ixgbe_get_num_of_rx_queues_vf(struct ixgbe_hw *hw); s32 ixgbe_get_mac_addr_vf(struct ixgbe_hw *hw, u8 *mac_addr); s32 ixgbe_setup_mac_link_vf(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool autoneg_wait_to_complete); + bool autoneg_wait_to_complete); s32 ixgbe_check_mac_link_vf(struct ixgbe_hw *hw, ixgbe_link_speed *speed, bool *link_up, bool autoneg_wait_to_complete); s32 ixgbe_set_rar_vf(struct ixgbe_hw *hw, u32 index, u8 *addr, u32 vmdq, @@ -127,5 +130,9 @@ s32 ixgbe_update_mc_addr_list_vf(struct ixgbe_hw *hw, u8 *mc_addr_list, u32 mc_addr_count, ixgbe_mc_addr_itr, bool clear); s32 ixgbe_set_vfta_vf(struct ixgbe_hw *hw, u32 vlan, u32 vind, bool vlan_on); +void ixgbevf_rlpml_set_vf(struct ixgbe_hw *hw, u16 max_size); +int ixgbevf_negotiate_api_version(struct ixgbe_hw *hw, int api); +int ixgbevf_get_queues(struct ixgbe_hw *hw, unsigned int *num_tcs, + unsigned int *default_tc); #endif /* __IXGBE_VF_H__ */ diff --git a/sys/dev/ixgbe/ixgbe_x540.c b/sys/dev/ixgbe/ixgbe_x540.c index 0635cda..9c7d5cc 100644 --- a/sys/dev/ixgbe/ixgbe_x540.c +++ b/sys/dev/ixgbe/ixgbe_x540.c @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -116,6 +116,7 @@ s32 ixgbe_init_ops_X540(struct ixgbe_hw *hw) mac->ops.setup_rxpba = &ixgbe_set_rxpba_generic; mac->ops.check_link = &ixgbe_check_mac_link_generic; + mac->mcft_size = 128; mac->vft_size = 128; mac->num_rar_entries = 128; @@ -177,16 +178,14 @@ enum ixgbe_media_type ixgbe_get_media_type_X540(struct ixgbe_hw *hw) * ixgbe_setup_mac_link_X540 - Sets the auto advertised capabilities * @hw: pointer to hardware structure * @speed: new link speed - * @autoneg: TRUE if autonegotiation enabled * @autoneg_wait_to_complete: TRUE when waiting for completion is needed **/ s32 ixgbe_setup_mac_link_X540(struct ixgbe_hw *hw, - ixgbe_link_speed speed, bool autoneg, + ixgbe_link_speed speed, bool autoneg_wait_to_complete) { DEBUGFUNC("ixgbe_setup_mac_link_X540"); - return hw->phy.ops.setup_link_speed(hw, speed, autoneg, - autoneg_wait_to_complete); + return hw->phy.ops.setup_link_speed(hw, speed, autoneg_wait_to_complete); } /** @@ -973,3 +972,4 @@ s32 ixgbe_blink_led_stop_X540(struct ixgbe_hw *hw, u32 index) return IXGBE_SUCCESS; } + diff --git a/sys/dev/ixgbe/ixgbe_x540.h b/sys/dev/ixgbe/ixgbe_x540.h index 29cf8bb..a553874 100644 --- a/sys/dev/ixgbe/ixgbe_x540.h +++ b/sys/dev/ixgbe/ixgbe_x540.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -41,7 +41,7 @@ s32 ixgbe_get_link_capabilities_X540(struct ixgbe_hw *hw, ixgbe_link_speed *speed, bool *autoneg); enum ixgbe_media_type ixgbe_get_media_type_X540(struct ixgbe_hw *hw); s32 ixgbe_setup_mac_link_X540(struct ixgbe_hw *hw, ixgbe_link_speed speed, - bool autoneg, bool link_up_wait_to_complete); + bool link_up_wait_to_complete); s32 ixgbe_reset_hw_X540(struct ixgbe_hw *hw); s32 ixgbe_start_hw_X540(struct ixgbe_hw *hw); u32 ixgbe_get_supported_physical_layer_X540(struct ixgbe_hw *hw); diff --git a/sys/dev/ixgbe/ixv.c b/sys/dev/ixgbe/ixv.c index b3a3d64..47a707d 100644 --- a/sys/dev/ixgbe/ixv.c +++ b/sys/dev/ixgbe/ixv.c @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2001-2012, Intel Corporation + Copyright (c) 2001-2013, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -169,7 +169,6 @@ static device_method_t ixv_methods[] = { DEVMETHOD(device_attach, ixv_attach), DEVMETHOD(device_detach, ixv_detach), DEVMETHOD(device_shutdown, ixv_shutdown), - DEVMETHOD_END }; @@ -1891,7 +1890,6 @@ ixv_config_link(struct adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 autoneg, err = 0; - bool negotiate = TRUE; if (hw->mac.ops.check_link) err = hw->mac.ops.check_link(hw, &autoneg, @@ -1900,8 +1898,8 @@ ixv_config_link(struct adapter *adapter) goto out; if (hw->mac.ops.setup_link) - err = hw->mac.ops.setup_link(hw, autoneg, - negotiate, adapter->link_up); + err = hw->mac.ops.setup_link(hw, + autoneg, adapter->link_up); out: return; } diff --git a/sys/dev/nvme/nvme.c b/sys/dev/nvme/nvme.c index ed5dbdb..8e646b7 100644 --- a/sys/dev/nvme/nvme.c +++ b/sys/dev/nvme/nvme.c @@ -200,13 +200,11 @@ nvme_modevent(module_t mod, int type, void *arg) void nvme_dump_command(struct nvme_command *cmd) { - printf("opc:%x f:%x r1:%x cid:%x nsid:%x r2:%x r3:%x " - "mptr:%qx prp1:%qx prp2:%qx cdw:%x %x %x %x %x %x\n", + printf( +"opc:%x f:%x r1:%x cid:%x nsid:%x r2:%x r3:%x mptr:%jx prp1:%jx prp2:%jx cdw:%x %x %x %x %x %x\n", cmd->opc, cmd->fuse, cmd->rsvd1, cmd->cid, cmd->nsid, cmd->rsvd2, cmd->rsvd3, - (long long unsigned int)cmd->mptr, - (long long unsigned int)cmd->prp1, - (long long unsigned int)cmd->prp2, + (uintmax_t)cmd->mptr, (uintmax_t)cmd->prp1, (uintmax_t)cmd->prp2, cmd->cdw10, cmd->cdw11, cmd->cdw12, cmd->cdw13, cmd->cdw14, cmd->cdw15); } diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c index 1ddf9cc..7ed2f7e 100644 --- a/sys/dev/nvme/nvme_ctrlr.c +++ b/sys/dev/nvme/nvme_ctrlr.c @@ -131,8 +131,8 @@ nvme_ctrlr_setup_chatham(struct nvme_controller *ctrlr) ctrlr->chatham_lbas = chatham_read_4(ctrlr, 0x8068) - 0x110; ctrlr->chatham_size = ctrlr->chatham_lbas * 512; - device_printf(ctrlr->dev, "Chatham size: %lld\n", - (long long)ctrlr->chatham_size); + device_printf(ctrlr->dev, "Chatham size: %jd\n", + (intmax_t)ctrlr->chatham_size); reg1 = reg2 = reg3 = ctrlr->chatham_size - 1; diff --git a/sys/dev/oce/oce_hw.c b/sys/dev/oce/oce_hw.c index e68ba7a..274c4d1 100644 --- a/sys/dev/oce/oce_hw.c +++ b/sys/dev/oce/oce_hw.c @@ -405,11 +405,6 @@ oce_create_nw_interface(POCE_SOFTC sc) sc->if_cap_flags = capab_en_flags; - /* Enable VLAN Promisc on HW */ - rc = oce_config_vlan(sc, (uint8_t) sc->if_id, NULL, 0, 1, 1); - if (rc) - goto error; - /* set default flow control */ rc = oce_set_flow_control(sc, sc->flow_control); if (rc) @@ -477,12 +472,9 @@ oce_hw_start(POCE_SOFTC sc) return 1; if (link.logical_link_status == NTWK_LOGICAL_LINK_UP) { - sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; sc->link_status = NTWK_LOGICAL_LINK_UP; if_link_state_change(sc->ifp, LINK_STATE_UP); } else { - sc->ifp->if_drv_flags &= - ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); sc->link_status = NTWK_LOGICAL_LINK_DOWN; if_link_state_change(sc->ifp, LINK_STATE_DOWN); } diff --git a/sys/dev/oce/oce_hw.h b/sys/dev/oce/oce_hw.h index 0e79218..aac0a8a 100644 --- a/sys/dev/oce/oce_hw.h +++ b/sys/dev/oce/oce_hw.h @@ -38,6 +38,8 @@ /* $FreeBSD$ */ +/* $FreeBSD$ */ + #include <sys/types.h> #undef _BIG_ENDIAN /* TODO */ @@ -155,7 +157,10 @@ #define ASYNC_EVENT_LINK_UP 0x1 #define ASYNC_EVENT_LINK_DOWN 0x0 #define ASYNC_EVENT_GRP5 0x5 +#define ASYNC_EVENT_CODE_DEBUG 0x6 #define ASYNC_EVENT_PVID_STATE 0x3 +#define ASYNC_EVENT_DEBUG_QNQ 0x1 +#define ASYNC_EVENT_CODE_SLIPORT 0x11 #define VLAN_VID_MASK 0x0FFF /* port link_status */ @@ -707,6 +712,17 @@ struct oce_async_event_grp5_pvid_state { uint32_t code; }; +/* async event indicating outer VLAN tag in QnQ */ +struct oce_async_event_qnq { + uint8_t valid; /* Indicates if outer VLAN is valid */ + uint8_t rsvd0; + uint16_t vlan_tag; + uint32_t event_tag; + uint8_t rsvd1[4]; + uint32_t code; +} ; + + typedef union oce_mq_ext_ctx_u { uint32_t dw[6]; struct { @@ -750,6 +766,44 @@ typedef union oce_mq_ext_ctx_u { /* dw5 */ uint32_t dw8rsvd1; } v0; + struct { + #ifdef _BIG_ENDIAN + /* dw0 */ + uint32_t cq_id:16; + uint32_t num_pages:16; + /* dw1 */ + uint32_t async_evt_bitmap; + /* dw2 */ + uint32_t dw5rsvd2:12; + uint32_t ring_size:4; + uint32_t async_cq_id:16; + /* dw3 */ + uint32_t valid:1; + uint32_t dw6rsvd1:31; + /* dw4 */ + uint32_t dw7rsvd1:31; + uint32_t async_cq_valid:1; + #else + /* dw0 */ + uint32_t num_pages:16; + uint32_t cq_id:16; + /* dw1 */ + uint32_t async_evt_bitmap; + /* dw2 */ + uint32_t async_cq_id:16; + uint32_t ring_size:4; + uint32_t dw5rsvd2:12; + /* dw3 */ + uint32_t dw6rsvd1:31; + uint32_t valid:1; + /* dw4 */ + uint32_t async_cq_valid:1; + uint32_t dw7rsvd1:31; + #endif + /* dw5 */ + uint32_t dw8rsvd1; + } v1; + } oce_mq_ext_ctx_t; @@ -826,6 +880,7 @@ enum COMMON_SUBSYSTEM_OPCODES { OPCODE_COMMON_SET_BEACON_CONFIG = 69, OPCODE_COMMON_GET_BEACON_CONFIG = 70, OPCODE_COMMON_GET_PHYSICAL_LINK_CONFIG = 71, + OPCODE_COMMON_READ_TRANSRECEIVER_DATA = 73, OPCODE_COMMON_GET_OEM_ATTRIBUTES = 76, OPCODE_COMMON_GET_PORT_NAME = 77, OPCODE_COMMON_GET_CONFIG_SIGNATURE = 78, @@ -1724,6 +1779,12 @@ struct mbx_set_common_iface_rx_filter { } params; }; +struct be_set_eqd { + uint32_t eq_id; + uint32_t phase; + uint32_t dm; +}; + /* [41] OPCODE_COMMON_MODIFY_EQ_DELAY */ struct mbx_modify_common_eq_delay { struct mbx_hdr hdr; @@ -1743,6 +1804,76 @@ struct mbx_modify_common_eq_delay { } params; }; +/* [32] OPCODE_COMMON_GET_CNTL_ATTRIBUTES */ + +struct mgmt_hba_attr { + int8_t flashrom_ver_str[32]; + int8_t manufac_name[32]; + uint32_t supp_modes; + int8_t seeprom_ver_lo; + int8_t seeprom_ver_hi; + int8_t rsvd0[2]; + uint32_t ioctl_data_struct_ver; + uint32_t ep_fw_data_struct_ver; + uint8_t ncsi_ver_str[12]; + uint32_t def_ext_to; + int8_t cntl_mod_num[32]; + int8_t cntl_desc[64]; + int8_t cntl_ser_num[32]; + int8_t ip_ver_str[32]; + int8_t fw_ver_str[32]; + int8_t bios_ver_str[32]; + int8_t redboot_ver_str[32]; + int8_t drv_ver_str[32]; + int8_t fw_on_flash_ver_str[32]; + uint32_t funcs_supp; + uint16_t max_cdblen; + uint8_t asic_rev; + uint8_t gen_guid[16]; + uint8_t hba_port_count; + uint16_t default_link_down_timeout; + uint8_t iscsi_ver_min_max; + uint8_t multifunc_dev; + uint8_t cache_valid; + uint8_t hba_status; + uint8_t max_domains_supp; + uint8_t phy_port; + uint32_t fw_post_status; + uint32_t hba_mtu[8]; + uint8_t iSCSI_feat; + uint8_t asic_gen; + uint8_t future_u8[2]; + uint32_t future_u32[3]; +}; + +struct mgmt_cntl_attr { + struct mgmt_hba_attr hba_attr; + uint16_t pci_vendor_id; + uint16_t pci_device_id; + uint16_t pci_sub_vendor_id; + uint16_t pci_sub_system_id; + uint8_t pci_bus_num; + uint8_t pci_dev_num; + uint8_t pci_func_num; + uint8_t interface_type; + uint64_t unique_id; + uint8_t netfilters; + uint8_t rsvd0[3]; + uint32_t future_u32[4]; +}; + +struct mbx_common_get_cntl_attr { + struct mbx_hdr hdr; + union { + struct { + uint32_t rsvd0; + } req; + struct { + struct mgmt_cntl_attr cntl_attr_info; + } rsp; + } params; +}; + /* [59] OPCODE_ADD_COMMON_IFACE_MAC */ struct mbx_add_common_iface_mac { struct mbx_hdr hdr; @@ -1785,6 +1916,23 @@ struct ioctl_common_function_reset { struct mbx_hdr hdr; }; +/* [73] OPCODE_COMMON_READ_TRANSRECEIVER_DATA */ +struct mbx_read_common_transrecv_data { + struct mbx_hdr hdr; + union { + struct { + uint32_t page_num; + uint32_t port; + } req; + struct { + uint32_t page_num; + uint32_t port; + uint32_t page_data[32]; + } rsp; + } params; + +}; + /* [80] OPCODE_COMMON_FUNCTION_LINK_CONFIG */ struct mbx_common_func_link_cfg { struct mbx_hdr hdr; @@ -2110,7 +2258,9 @@ enum RSS_ENABLE_FLAGS { RSS_ENABLE_IPV4 = 0x1, /* (IPV4 HASH enabled ) */ RSS_ENABLE_TCP_IPV4 = 0x2, /* (TCP IPV4 Hash enabled) */ RSS_ENABLE_IPV6 = 0x4, /* (IPV6 HASH enabled) */ - RSS_ENABLE_TCP_IPV6 = 0x8 /* (TCP IPV6 HASH */ + RSS_ENABLE_TCP_IPV6 = 0x8, /* (TCP IPV6 HASH */ + RSS_ENABLE_UDP_IPV4 = 0x10, /* UDP IPV4 HASH */ + RSS_ENABLE_UDP_IPV6 = 0x20 /* UDP IPV6 HASH */ }; #define RSS_ENABLE (RSS_ENABLE_IPV4 | RSS_ENABLE_TCP_IPV4) #define RSS_DISABLE RSS_ENABLE_NONE diff --git a/sys/dev/oce/oce_if.c b/sys/dev/oce/oce_if.c index 426acd9..48cdb16 100644 --- a/sys/dev/oce/oce_if.c +++ b/sys/dev/oce/oce_if.c @@ -36,6 +36,7 @@ * Costa Mesa, CA 92626 */ + /* $FreeBSD$ */ #include "opt_inet6.h" @@ -94,7 +95,8 @@ static void setup_max_queues_want(POCE_SOFTC sc); static void update_queues_got(POCE_SOFTC sc); static void process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe); - +static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m); +static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete); /* IP specific */ #if defined(INET6) || defined(INET) @@ -266,8 +268,6 @@ oce_attach(device_t dev) rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc); if (rc) goto stats_free; -#ifdef DEV_NETMAP -#endif /* DEV_NETMAP */ return 0; @@ -485,7 +485,7 @@ oce_multiq_start(struct ifnet *ifp, struct mbuf *m) struct oce_wq *wq = NULL; int queue_index = 0; int status = 0; - + if ((m->m_flags & M_FLOWID) != 0) queue_index = m->m_pkthdr.flowid % sc->nwqs; @@ -568,6 +568,7 @@ oce_intr(void *arg, int pending) eq_arm: oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); + return; } @@ -633,6 +634,8 @@ oce_fast_isr(void *arg) taskqueue_enqueue_fast(ii->tq, &ii->task); + ii->eq->intr++; + return FILTER_HANDLED; } @@ -780,6 +783,7 @@ oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index) struct oce_nic_frag_wqe *nicfrag; int num_wqes; uint32_t reg_value; + boolean_t complete = TRUE; m = *mpp; if (!m) @@ -790,6 +794,15 @@ oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index) goto free_ret; } + if(oce_tx_asic_stall_verify(sc, m)) { + m = oce_insert_vlan_tag(sc, m, &complete); + if(!m) { + device_printf(sc->dev, "Insertion unsuccessful\n"); + return 0; + } + + } + if (m->m_pkthdr.csum_flags & CSUM_TSO) { /* consolidate packet buffers for TSO/LSO segment offload */ #if defined(INET6) || defined(INET) @@ -837,15 +850,15 @@ retry: nichdr->u0.dw[2] = 0; nichdr->u0.dw[3] = 0; - nichdr->u0.s.complete = 1; + nichdr->u0.s.complete = complete; nichdr->u0.s.event = 1; nichdr->u0.s.crc = 1; nichdr->u0.s.forward = 0; nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0; nichdr->u0.s.udpcs = - (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0; + (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0; nichdr->u0.s.tcpcs = - (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0; + (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0; nichdr->u0.s.num_wqe = num_wqes; nichdr->u0.s.total_length = m->m_pkthdr.len; if (m->m_flags & M_VLANTAG) { @@ -895,7 +908,7 @@ retry: wq->tx_stats.tx_wrbs += num_wqes; wq->tx_stats.tx_bytes += m->m_pkthdr.len; wq->tx_stats.tx_pkts++; - + bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); reg_value = (num_wqes << 16) | wq->wq_id; @@ -1081,6 +1094,9 @@ oce_start(struct ifnet *ifp) if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return; + + if (!sc->link_status) + return; do { IF_DEQUEUE(&sc->ifp->if_snd, m); @@ -1303,7 +1319,6 @@ oce_rx(struct oce_rq *rq, uint32_t rqe_idx, struct oce_nic_rx_cqe *cqe) #if defined(INET6) || defined(INET) /* Try to queue to LRO */ if (IF_LRO_ENABLED(sc) && - !(m->m_flags & M_VLANTAG) && (cqe->u0.s.ip_cksum_pass) && (cqe->u0.s.l4_cksum_pass) && (!cqe->u0.s.ip_ver) && @@ -1343,13 +1358,6 @@ oce_discard_rx_comp(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe) POCE_SOFTC sc = (POCE_SOFTC) rq->parent; int num_frags = cqe->u0.s.num_fragments; - if (IS_XE201(sc) && cqe->u0.s.error) { - /* Lancer A0 workaround - * num_frags will be 1 more than actual in case of error - */ - if (num_frags) - num_frags -= 1; - } for (i = 0; i < num_frags; i++) { if (rq->packets_out == rq->packets_in) { device_printf(sc->dev, @@ -1458,7 +1466,7 @@ oce_free_lro(POCE_SOFTC sc) tcp_lro_free(lro); } } -#endif /* INET6 || INET */ +#endif int oce_alloc_rx_bufs(struct oce_rq *rq, int count) @@ -1471,7 +1479,7 @@ oce_alloc_rx_bufs(struct oce_rq *rq, int count) struct oce_nic_rqe *rqe; pd_rxulp_db_t rxdb_reg; - + bzero(&rxdb_reg, sizeof(pd_rxulp_db_t)); for (i = 0; i < count; i++) { in = rq->packets_in + 1; if (in == OCE_RQ_PACKET_ARRAY_SIZE) @@ -1512,14 +1520,12 @@ oce_alloc_rx_bufs(struct oce_rq *rq, int count) } if (added != 0) { for (i = added / OCE_MAX_RQ_POSTS; i > 0; i--) { - DELAY(1); rxdb_reg.bits.num_posted = OCE_MAX_RQ_POSTS; rxdb_reg.bits.qid = rq->rq_id; OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); added -= OCE_MAX_RQ_POSTS; } if (added > 0) { - DELAY(1); rxdb_reg.bits.qid = rq->rq_id; rxdb_reg.bits.num_posted = added; OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); @@ -1554,13 +1560,8 @@ oce_rq_handler(void *arg) } else { rq->rx_stats.rxcp_err++; sc->ifp->if_ierrors++; - if (IS_XE201(sc)) - /* Lancer A0 no buffer workaround */ - oce_discard_rx_comp(rq, cqe); - else - /* Post L3/L4 errors to stack.*/ - oce_rx(rq, cqe->u0.s.frag_index, cqe); - + /* Post L3/L4 errors to stack.*/ + oce_rx(rq, cqe->u0.s.frag_index, cqe); } rq->rx_stats.rx_compl++; cqe->u0.dw[2] = 0; @@ -1757,18 +1758,18 @@ oce_handle_passthrough(struct ifnet *ifp, caddr_t data) uint32_t req_size; struct mbx_hdr req; OCE_DMA_MEM dma_mem; - + struct mbx_common_get_cntl_attr *fw_cmd; if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE))) return EFAULT; - + if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE))) return EINVAL; - + ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE); if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr))) return EFAULT; - + req_size = le32toh(req.u0.req.request_length); if (req_size > 65536) return EINVAL; @@ -1792,12 +1793,86 @@ oce_handle_passthrough(struct ifnet *ifp, caddr_t data) if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) rc = EFAULT; + /* + firmware is filling all the attributes for this ioctl except + the driver version..so fill it + */ + if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) { + fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr; + strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str, + COMPONENT_REVISION, strlen(COMPONENT_REVISION)); + } + dma_free: oce_dma_free(sc, &dma_mem); return rc; } +static void +oce_eqd_set_periodic(POCE_SOFTC sc) +{ + struct oce_set_eqd set_eqd[OCE_MAX_EQ]; + struct oce_aic_obj *aic; + struct oce_eq *eqo; + uint64_t now = 0, delta; + int eqd, i, num = 0; + uint32_t ips = 0; + int tps; + + for (i = 0 ; i < sc->neqs; i++) { + eqo = sc->eq[i]; + aic = &sc->aic_obj[i]; + /* When setting the static eq delay from the user space */ + if (!aic->enable) { + eqd = aic->et_eqd; + goto modify_eqd; + } + + now = ticks; + + /* Over flow check */ + if ((now < aic->ticks) || (eqo->intr < aic->intr_prev)) + goto done; + + delta = now - aic->ticks; + tps = delta/hz; + + /* Interrupt rate based on elapsed ticks */ + if(tps) + ips = (uint32_t)(eqo->intr - aic->intr_prev) / tps; + + if (ips > INTR_RATE_HWM) + eqd = aic->cur_eqd + 20; + else if (ips < INTR_RATE_LWM) + eqd = aic->cur_eqd / 2; + else + goto done; + + if (eqd < 10) + eqd = 0; + + /* Make sure that the eq delay is in the known range */ + eqd = min(eqd, aic->max_eqd); + eqd = max(eqd, aic->min_eqd); + +modify_eqd: + if (eqd != aic->cur_eqd) { + set_eqd[num].delay_multiplier = (eqd * 65)/100; + set_eqd[num].eq_id = eqo->eq_id; + aic->cur_eqd = eqd; + num++; + } +done: + aic->intr_prev = eqo->intr; + aic->ticks = now; + } + + /* Is there atleast one eq that needs to be modified? */ + if(num) + oce_mbox_eqd_modify_periodic(sc, set_eqd, num); + +} static void oce_local_timer(void *arg) @@ -1813,6 +1888,10 @@ oce_local_timer(void *arg) for (i = 0; i < sc->nwqs; i++) oce_tx_restart(sc, sc->wq[i]); + /* calculate and set the eq delay for optimal interrupt rate */ + if (IS_BE(sc)) + oce_eqd_set_periodic(sc); + callout_reset(&sc->timer, hz, oce_local_timer, sc); } @@ -1849,17 +1928,17 @@ oce_if_deactivate(POCE_SOFTC sc) /* Stop intrs and finish any bottom halves pending */ oce_hw_intr_disable(sc); - /* Since taskqueue_drain takes a Giant Lock, We should not acquire - any other lock. So unlock device lock and require after - completing taskqueue_drain. - */ - UNLOCK(&sc->dev_lock); + /* Since taskqueue_drain takes a Gaint Lock, We should not acquire + any other lock. So unlock device lock and require after + completing taskqueue_drain. + */ + UNLOCK(&sc->dev_lock); for (i = 0; i < sc->intr_count; i++) { if (sc->intrs[i].tq != NULL) { taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task); } } - LOCK(&sc->dev_lock); + LOCK(&sc->dev_lock); /* Delete RX queue in card with flush param */ oce_stop_rx(sc); @@ -1874,7 +1953,7 @@ oce_if_deactivate(POCE_SOFTC sc) /* But still we need to get MCC aync events. So enable intrs and also arm first EQ - */ + */ oce_hw_intr_enable(sc); oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE); @@ -1947,6 +2026,7 @@ oce_mq_handler(void *arg) struct oce_mq_cqe *cqe; struct oce_async_cqe_link_state *acqe; struct oce_async_event_grp5_pvid_state *gcqe; + struct oce_async_event_qnq *dbgcqe; bus_dmamap_sync(cq->ring->dma.tag, @@ -1973,6 +2053,14 @@ oce_mq_handler(void *arg) sc->pvid = 0; } + else if(evt_type == ASYNC_EVENT_CODE_DEBUG && + optype == ASYNC_EVENT_DEBUG_QNQ) { + dbgcqe = + (struct oce_async_event_qnq *)cqe; + if(dbgcqe->valid) + sc->qnqid = dbgcqe->vlan_tag; + sc->qnq_debug_event = TRUE; + } } cqe->u0.dw[3] = 0; RING_GET(cq->ring, 1); @@ -2032,3 +2120,79 @@ update_queues_got(POCE_SOFTC sc) } } +static int +oce_check_ipv6_ext_hdr(struct mbuf *m) +{ + struct ether_header *eh = mtod(m, struct ether_header *); + caddr_t m_datatemp = m->m_data; + + if (eh->ether_type == htons(ETHERTYPE_IPV6)) { + m->m_data += sizeof(struct ether_header); + struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); + + if((ip6->ip6_nxt != IPPROTO_TCP) && \ + (ip6->ip6_nxt != IPPROTO_UDP)){ + struct ip6_ext *ip6e = NULL; + m->m_data += sizeof(struct ip6_hdr); + + ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *); + if(ip6e->ip6e_len == 0xff) { + m->m_data = m_datatemp; + return TRUE; + } + } + m->m_data = m_datatemp; + } + return FALSE; +} + +static int +is_be3_a1(POCE_SOFTC sc) +{ + if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) { + return TRUE; + } + return FALSE; +} + +static struct mbuf * +oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete) +{ + uint16_t vlan_tag = 0; + + if(!M_WRITABLE(m)) + return NULL; + + /* Embed vlan tag in the packet if it is not part of it */ + if(m->m_flags & M_VLANTAG) { + vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag); + m->m_flags &= ~M_VLANTAG; + } + + /* if UMC, ignore vlan tag insertion and instead insert pvid */ + if(sc->pvid) { + if(!vlan_tag) + vlan_tag = sc->pvid; + *complete = FALSE; + } + + if(vlan_tag) { + m = ether_vlanencap(m, vlan_tag); + } + + if(sc->qnqid) { + m = ether_vlanencap(m, sc->qnqid); + *complete = FALSE; + } + return m; +} + +static int +oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m) +{ + if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \ + oce_check_ipv6_ext_hdr(m)) { + return TRUE; + } + return FALSE; +} diff --git a/sys/dev/oce/oce_if.h b/sys/dev/oce/oce_if.h index 6428f88..ee684bd 100644 --- a/sys/dev/oce/oce_if.h +++ b/sys/dev/oce/oce_if.h @@ -36,6 +36,7 @@ * Costa Mesa, CA 92626 */ + /* $FreeBSD$ */ #include <sys/param.h> @@ -87,9 +88,7 @@ #include "oce_hw.h" -/* OCE device driver module component revision informaiton */ -#define COMPONENT_REVISION "4.2.127.0" - +#define COMPONENT_REVISION "4.6.95.0" /* OCE devices supported by this driver */ #define PCI_VENDOR_EMULEX 0x10df /* Emulex */ @@ -132,7 +131,7 @@ extern int mp_ncpus; /* system's total active cpu cores */ #define OCE_RQ_BUF_SIZE 2048 #define OCE_LSO_MAX_SIZE (64 * 1024) #define LONG_TIMEOUT 30 -#define OCE_MAX_JUMBO_FRAME_SIZE 16360 +#define OCE_MAX_JUMBO_FRAME_SIZE 9018 #define OCE_MAX_MTU (OCE_MAX_JUMBO_FRAME_SIZE - \ ETHER_VLAN_ENCAP_LEN - \ ETHER_HDR_LEN) @@ -481,7 +480,27 @@ struct oce_drv_stats { } u0; }; +#define INTR_RATE_HWM 15000 +#define INTR_RATE_LWM 10000 + +#define OCE_MAX_EQD 128u +#define OCE_MIN_EQD 50u + +struct oce_set_eqd { + uint32_t eq_id; + uint32_t phase; + uint32_t delay_multiplier; +}; +struct oce_aic_obj { /* Adaptive interrupt coalescing (AIC) info */ + boolean_t enable; + uint32_t min_eqd; /* in usecs */ + uint32_t max_eqd; /* in usecs */ + uint32_t cur_eqd; /* in usecs */ + uint32_t et_eqd; /* configured value when aic is off */ + uint64_t ticks; + uint64_t intr_prev; +}; #define MAX_LOCK_DESC_LEN 32 struct oce_lock { @@ -565,6 +584,7 @@ struct oce_eq { int cq_valid; struct eq_config eq_cfg; int vector; + uint64_t intr; }; enum cq_len { @@ -827,6 +847,9 @@ typedef struct oce_softc { uint32_t flow_control; uint32_t promisc; + + struct oce_aic_obj aic_obj[OCE_MAX_EQ]; + /*Vlan Filtering related */ eventhandler_tag vlan_attach; eventhandler_tag vlan_detach; @@ -837,7 +860,9 @@ typedef struct oce_softc { struct oce_drv_stats oce_stats_info; struct callout timer; int8_t be3_native; - uint32_t pvid; + uint16_t qnq_debug_event; + uint16_t qnqid; + uint16_t pvid; } OCE_SOFTC, *POCE_SOFTC; @@ -996,6 +1021,9 @@ int oce_mbox_create_wq(struct oce_wq *wq); int oce_mbox_create_eq(struct oce_eq *eq); int oce_mbox_cq_create(struct oce_cq *cq, uint32_t ncoalesce, uint32_t is_eventable); +int oce_mbox_read_transrecv_data(POCE_SOFTC sc, uint32_t page_num); +void oce_mbox_eqd_modify_periodic(POCE_SOFTC sc, struct oce_set_eqd *set_eqd, + int num); void mbx_common_req_hdr_init(struct mbx_hdr *hdr, uint8_t dom, uint8_t port, @@ -1076,3 +1104,12 @@ static inline uint32_t oce_highbit(uint32_t x) return 0; } +#define TRANSCEIVER_DATA_NUM_ELE 64 +#define TRANSCEIVER_DATA_SIZE 256 +#define TRANSCEIVER_A0_SIZE 128 +#define TRANSCEIVER_A2_SIZE 128 +#define PAGE_NUM_A0 0xa0 +#define PAGE_NUM_A2 0xa2 +#define IS_QNQ_OR_UMC(sc) ((sc->pvid && (sc->function_mode & FNM_UMC_MODE ))\ + || (sc->qnqid && (sc->function_mode & FNM_FLEX10_MODE))) + diff --git a/sys/dev/oce/oce_mbox.c b/sys/dev/oce/oce_mbox.c index 07107a2..2db5934 100644 --- a/sys/dev/oce/oce_mbox.c +++ b/sys/dev/oce/oce_mbox.c @@ -37,10 +37,12 @@ */ + /* $FreeBSD$ */ -#include "oce_if.h" +#include "oce_if.h" +extern uint32_t sfp_vpd_dump_buffer[TRANSCEIVER_DATA_NUM_ELE]; /** * @brief Reset (firmware) common function @@ -276,12 +278,17 @@ oce_get_fw_version(POCE_SOFTC sc) DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); ret = oce_mbox_post(sc, &mbx, NULL); - if (ret) - return ret; + if (!ret) + ret = fwcmd->hdr.u0.rsp.status; + if (ret) { + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, ret); + goto error; + } bcopy(fwcmd->params.rsp.fw_ver_str, sc->fw_version, 32); - - return 0; +error: + return ret; } @@ -428,15 +435,20 @@ oce_read_mac_addr(POCE_SOFTC sc, uint32_t if_id, DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); ret = oce_mbox_post(sc, &mbx, NULL); - if (ret) - return ret; + if (!ret) + ret = fwcmd->hdr.u0.rsp.status; + if (ret) { + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, ret); + goto error; + } /* copy the mac addres in the output parameter */ mac->size_of_struct = fwcmd->params.rsp.mac.size_of_struct; bcopy(&fwcmd->params.rsp.mac.mac_addr[0], &mac->mac_addr[0], mac->size_of_struct); - - return 0; +error: + return ret; } /** @@ -466,8 +478,13 @@ oce_get_fw_config(POCE_SOFTC sc) DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); ret = oce_mbox_post(sc, &mbx, NULL); - if (ret) - return ret; + if (!ret) + ret = fwcmd->hdr.u0.rsp.status; + if (ret) { + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, ret); + goto error; + } DW_SWAP(u32ptr(fwcmd), sizeof(struct mbx_common_query_fw_config)); @@ -485,7 +502,8 @@ oce_get_fw_config(POCE_SOFTC sc) sc->max_rx_rings = fwcmd->params.rsp.ulp[1].lro_rqid_tot; } - return 0; +error: + return ret; } @@ -540,15 +558,20 @@ oce_if_create(POCE_SOFTC sc, DW_SWAP(u32ptr(&mbx), OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); - if (rc) - return rc; + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) { + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); + goto error; + } *if_id = LE_32(fwcmd->params.rsp.if_id); if (mac_addr != NULL) sc->pmac_id = LE_32(fwcmd->params.rsp.pmac_id); - - return 0; +error: + return rc; } /** @@ -581,6 +604,11 @@ oce_if_del(POCE_SOFTC sc, uint32_t if_id) DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); return rc; } @@ -628,8 +656,12 @@ oce_config_vlan(POCE_SOFTC sc, DW_SWAP(u32ptr(&mbx), (OCE_BMBX_RHDR_SZ + mbx.payload_length)); rc = oce_mbox_post(sc, &mbx, NULL); - - return rc; + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); + return 0; } @@ -667,7 +699,11 @@ oce_set_flow_control(POCE_SOFTC sc, uint32_t flow_control) DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); - + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); return rc; } @@ -726,20 +762,28 @@ oce_config_nic_rss(POCE_SOFTC sc, uint32_t if_id, uint16_t enable_rss) struct oce_mbx mbx; struct mbx_config_nic_rss *fwcmd = (struct mbx_config_nic_rss *)&mbx.payload; + int version; bzero(&mbx, sizeof(struct oce_mbx)); + if (IS_XE201(sc)) { + version = OCE_MBX_VER_V1; + fwcmd->params.req.enable_rss = RSS_ENABLE_UDP_IPV4 | + RSS_ENABLE_UDP_IPV6; + } else + version = OCE_MBX_VER_V0; + mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_NIC, NIC_CONFIG_RSS, MBX_TIMEOUT_SEC, sizeof(struct mbx_config_nic_rss), - OCE_MBX_VER_V0); + version); if (enable_rss) - fwcmd->params.req.enable_rss = (RSS_ENABLE_IPV4 | - RSS_ENABLE_TCP_IPV4 | - RSS_ENABLE_IPV6 | - RSS_ENABLE_TCP_IPV6); + fwcmd->params.req.enable_rss |= (RSS_ENABLE_IPV4 | + RSS_ENABLE_TCP_IPV4 | + RSS_ENABLE_IPV6 | + RSS_ENABLE_TCP_IPV6); fwcmd->params.req.flush = OCE_FLUSH; fwcmd->params.req.if_id = LE_32(if_id); @@ -753,9 +797,12 @@ oce_config_nic_rss(POCE_SOFTC sc, uint32_t if_id, uint16_t enable_rss) DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); - + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); } - return rc; } @@ -834,7 +881,12 @@ oce_set_common_iface_rx_filter(POCE_SOFTC sc, POCE_DMA_MEM sgl) DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); - return rc; + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); + return 0; } /** @@ -848,17 +900,19 @@ oce_get_link_status(POCE_SOFTC sc, struct link_status *link) { struct oce_mbx mbx; struct mbx_query_common_link_config *fwcmd; - int rc = 0; + int rc = 0, version; bzero(&mbx, sizeof(struct oce_mbx)); + IS_XE201(sc) ? (version = OCE_MBX_VER_V1) : (version = OCE_MBX_VER_V0); + fwcmd = (struct mbx_query_common_link_config *)&mbx.payload; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_QUERY_LINK_CONFIG, MBX_TIMEOUT_SEC, sizeof(struct mbx_query_common_link_config), - OCE_MBX_VER_V0); + version); mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_query_common_link_config); @@ -866,15 +920,18 @@ oce_get_link_status(POCE_SOFTC sc, struct link_status *link) rc = oce_mbox_post(sc, &mbx, NULL); + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; if (rc) { - device_printf(sc->dev, "Could not get link speed: %d\n", rc); - } else { - /* interpret response */ - bcopy(&fwcmd->params.rsp, link, sizeof(struct link_status)); - link->logical_link_status = LE_32(link->logical_link_status); - link->qos_link_speed = LE_16(link->qos_link_speed); + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); + goto error; } - + /* interpret response */ + bcopy(&fwcmd->params.rsp, link, sizeof(struct link_status)); + link->logical_link_status = LE_32(link->logical_link_status); + link->qos_link_speed = LE_16(link->qos_link_speed); +error: return rc; } @@ -916,11 +973,11 @@ oce_mbox_get_nic_stats_v0(POCE_SOFTC sc, POCE_DMA_MEM pstats_dma_mem) oce_dma_sync(pstats_dma_mem, BUS_DMASYNC_POSTWRITE); - if (rc) { - device_printf(sc->dev, - "Could not get nic statistics: %d\n", rc); - } - + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); return rc; } @@ -966,10 +1023,11 @@ oce_mbox_get_nic_stats(POCE_SOFTC sc, POCE_DMA_MEM pstats_dma_mem) rc = oce_mbox_post(sc, &mbx, NULL); oce_dma_sync(pstats_dma_mem, BUS_DMASYNC_POSTWRITE); - if (rc) { - device_printf(sc->dev, - "Could not get nic statistics: %d\n", rc); - } + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); return rc; } @@ -1001,7 +1059,7 @@ oce_mbox_get_pport_stats(POCE_SOFTC sc, POCE_DMA_MEM pstats_dma_mem, OCE_MBX_VER_V0); fwcmd->params.req.reset_stats = reset_stats; - fwcmd->params.req.port_number = sc->if_id; + fwcmd->params.req.port_number = sc->port_id; mbx.u0.s.embedded = 0; /* stats too large for embedded mbx rsp */ mbx.u0.s.sge_count = 1; /* using scatter gather instead */ @@ -1017,11 +1075,11 @@ oce_mbox_get_pport_stats(POCE_SOFTC sc, POCE_DMA_MEM pstats_dma_mem, rc = oce_mbox_post(sc, &mbx, NULL); oce_dma_sync(pstats_dma_mem, BUS_DMASYNC_POSTWRITE); - if (rc != 0) { - device_printf(sc->dev, - "Could not get physical port statistics: %d\n", rc); - } - + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); return rc; } @@ -1070,11 +1128,11 @@ oce_mbox_get_vport_stats(POCE_SOFTC sc, POCE_DMA_MEM pstats_dma_mem, rc = oce_mbox_post(sc, &mbx, NULL); oce_dma_sync(pstats_dma_mem, BUS_DMASYNC_POSTWRITE); - if (rc != 0) { - device_printf(sc->dev, - "Could not get physical port statistics: %d\n", rc); - } - + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); return rc; } @@ -1115,7 +1173,11 @@ oce_update_multicast(POCE_SOFTC sc, POCE_DMA_MEM pdma_mem) DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); - + if (!rc) + rc = req->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); return rc; } @@ -1176,11 +1238,15 @@ oce_mbox_macaddr_add(POCE_SOFTC sc, uint8_t *mac_addr, mbx.payload_length = sizeof(struct mbx_add_common_iface_mac); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); - if (rc) - return rc; - + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) { + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); + goto error; + } *pmac_id = fwcmd->params.rsp.pmac_id; - +error: return rc; } @@ -1210,6 +1276,11 @@ oce_mbox_macaddr_del(POCE_SOFTC sc, uint32_t if_id, uint32_t pmac_id) DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); return rc; } @@ -1242,12 +1313,17 @@ oce_mbox_check_native_mode(POCE_SOFTC sc) DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); - //if (rc != 0) This can fail in legacy mode. So skip - // FN_LEAVE(rc); - + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) { + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); + goto error; + } sc->be3_native = fwcmd->params.rsp.capability_flags & CAP_BE3_NATIVE_ERX_API; +error: return 0; } @@ -1282,6 +1358,11 @@ oce_mbox_cmd_set_loopback(POCE_SOFTC sc, uint8_t port_num, DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); return rc; @@ -1320,10 +1401,13 @@ oce_mbox_cmd_test_loopback(POCE_SOFTC sc, uint32_t port_num, DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; if (rc) - return rc; + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); - return(fwcmd->params.rsp.status); + return rc; } int @@ -1362,11 +1446,11 @@ oce_mbox_write_flashrom(POCE_SOFTC sc, uint32_t optype,uint32_t opcode, /* post the command */ rc = oce_mbox_post(sc, &mbx, NULL); - if (rc) { - device_printf(sc->dev, "Write FlashROM mbox post failed\n"); - } else { - rc = fwcmd->hdr.u0.rsp.status; - } + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); return rc; @@ -1408,12 +1492,15 @@ oce_mbox_get_flashrom_crc(POCE_SOFTC sc, uint8_t *flash_crc, /* post the command */ rc = oce_mbox_post(sc, &mbx, NULL); + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; if (rc) { - device_printf(sc->dev, "Read FlashROM CRC mbox post failed\n"); - } else { - bcopy(fwcmd->data_buffer, flash_crc, 4); - rc = fwcmd->hdr.u0.rsp.status; + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); + goto error; } + bcopy(fwcmd->data_buffer, flash_crc, 4); +error: return rc; } @@ -1440,20 +1527,22 @@ oce_mbox_get_phy_info(POCE_SOFTC sc, struct oce_phy_info *phy_info) /* now post the command */ rc = oce_mbox_post(sc, &mbx, NULL); + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; if (rc) { - device_printf(sc->dev, "Read PHY info mbox post failed\n"); - } else { - rc = fwcmd->hdr.u0.rsp.status; - phy_info->phy_type = fwcmd->params.rsp.phy_info.phy_type; - phy_info->interface_type = - fwcmd->params.rsp.phy_info.interface_type; - phy_info->auto_speeds_supported = - fwcmd->params.rsp.phy_info.auto_speeds_supported; - phy_info->fixed_speeds_supported = - fwcmd->params.rsp.phy_info.fixed_speeds_supported; - phy_info->misc_params =fwcmd->params.rsp.phy_info.misc_params; - + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); + goto error; } + phy_info->phy_type = fwcmd->params.rsp.phy_info.phy_type; + phy_info->interface_type = + fwcmd->params.rsp.phy_info.interface_type; + phy_info->auto_speeds_supported = + fwcmd->params.rsp.phy_info.auto_speeds_supported; + phy_info->fixed_speeds_supported = + fwcmd->params.rsp.phy_info.fixed_speeds_supported; + phy_info->misc_params =fwcmd->params.rsp.phy_info.misc_params; +error: return rc; } @@ -1499,14 +1588,16 @@ oce_mbox_lancer_write_flashrom(POCE_SOFTC sc, uint32_t data_size, /* post the command */ rc = oce_mbox_post(sc, &mbx, NULL); + if (!rc) + rc = fwcmd->params.rsp.status; if (rc) { - device_printf(sc->dev, - "Write Lancer FlashROM mbox post failed\n"); - } else { - *written_data = fwcmd->params.rsp.actual_write_length; - *additional_status = fwcmd->params.rsp.additional_status; - rc = fwcmd->params.rsp.status; + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); + goto error; } + *written_data = fwcmd->params.rsp.actual_write_length; + *additional_status = fwcmd->params.rsp.additional_status; +error: return rc; } @@ -1553,15 +1644,16 @@ oce_mbox_create_rq(struct oce_rq *rq) mbx.payload_length = sizeof(struct mbx_create_nic_rq); rc = oce_mbox_post(sc, &mbx, NULL); - if (rc) + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) { + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); goto error; - + } rq->rq_id = fwcmd->params.rsp.rq_id; rq->rss_cpuid = fwcmd->params.rsp.rss_cpuid; - - return 0; error: - device_printf(sc->dev, "Mbox Create RQ failed\n"); return rc; } @@ -1603,14 +1695,15 @@ oce_mbox_create_wq(struct oce_wq *wq) mbx.payload_length = sizeof(struct mbx_create_nic_wq); rc = oce_mbox_post(sc, &mbx, NULL); - if (rc) + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) { + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); goto error; - + } wq->wq_id = LE_16(fwcmd->params.rsp.wq_id); - - return 0; error: - device_printf(sc->dev, "Mbox Create WQ failed\n"); return rc; } @@ -1649,14 +1742,15 @@ oce_mbox_create_eq(struct oce_eq *eq) mbx.payload_length = sizeof(struct mbx_create_common_eq); rc = oce_mbox_post(sc, &mbx, NULL); - if (rc) + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) { + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); goto error; - + } eq->eq_id = LE_16(fwcmd->params.rsp.eq_id); - - return 0; error: - device_printf(sc->dev, "Mbox Create EQ failed\n"); return rc; } @@ -1726,14 +1820,125 @@ oce_mbox_cq_create(struct oce_cq *cq, uint32_t ncoalesce, uint32_t is_eventable) mbx.payload_length = sizeof(struct mbx_create_common_cq); rc = oce_mbox_post(sc, &mbx, NULL); - if (rc) + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) { + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); goto error; - + } cq->cq_id = LE_16(fwcmd->params.rsp.cq_id); +error: + return rc; - return 0; +} + +int +oce_mbox_read_transrecv_data(POCE_SOFTC sc, uint32_t page_num) +{ + int rc = 0; + struct oce_mbx mbx; + struct mbx_read_common_transrecv_data *fwcmd; + struct oce_mq_sge *sgl; + OCE_DMA_MEM dma; + + /* Allocate DMA mem*/ + if (oce_dma_alloc(sc, sizeof(struct mbx_read_common_transrecv_data), + &dma, 0)) + return ENOMEM; + + fwcmd = OCE_DMAPTR(&dma, struct mbx_read_common_transrecv_data); + bzero(fwcmd, sizeof(struct mbx_read_common_transrecv_data)); + + bzero(&mbx, sizeof(struct oce_mbx)); + mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, + MBX_SUBSYSTEM_COMMON, + OPCODE_COMMON_READ_TRANSRECEIVER_DATA, + MBX_TIMEOUT_SEC, + sizeof(struct mbx_read_common_transrecv_data), + OCE_MBX_VER_V0); + + /* fill rest of mbx */ + mbx.u0.s.embedded = 0; + mbx.payload_length = sizeof(struct mbx_read_common_transrecv_data); + mbx.u0.s.sge_count = 1; + sgl = &mbx.payload.u0.u1.sgl[0]; + sgl->pa_hi = htole32(upper_32_bits(dma.paddr)); + sgl->pa_lo = htole32((dma.paddr) & 0xFFFFFFFF); + sgl->length = htole32(mbx.payload_length); + DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); + + fwcmd->params.req.port = LE_32(sc->port_id); + fwcmd->params.req.page_num = LE_32(page_num); + + /* command post */ + rc = oce_mbox_post(sc, &mbx, NULL); + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) { + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); + goto error; + } + if(fwcmd->params.rsp.page_num == PAGE_NUM_A0) + { + bcopy((char *)fwcmd->params.rsp.page_data, + (char *)&sfp_vpd_dump_buffer[0], + TRANSCEIVER_A0_SIZE); + } + + if(fwcmd->params.rsp.page_num == PAGE_NUM_A2) + { + bcopy((char *)fwcmd->params.rsp.page_data, + (char *)&sfp_vpd_dump_buffer[32], + TRANSCEIVER_A2_SIZE); + } error: - device_printf(sc->dev, "Mbox Create CQ failed\n"); return rc; +} +void +oce_mbox_eqd_modify_periodic(POCE_SOFTC sc, struct oce_set_eqd *set_eqd, + int num) +{ + struct oce_mbx mbx; + struct mbx_modify_common_eq_delay *fwcmd; + int rc = 0; + int i = 0; + + bzero(&mbx, sizeof(struct oce_mbx)); + + /* Initialize MODIFY_EQ_DELAY ioctl header */ + fwcmd = (struct mbx_modify_common_eq_delay *)&mbx.payload; + mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, + MBX_SUBSYSTEM_COMMON, + OPCODE_COMMON_MODIFY_EQ_DELAY, + MBX_TIMEOUT_SEC, + sizeof(struct mbx_modify_common_eq_delay), + OCE_MBX_VER_V0); + /* fill rest of mbx */ + mbx.u0.s.embedded = 1; + mbx.payload_length = sizeof(struct mbx_modify_common_eq_delay); + DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); + + fwcmd->params.req.num_eq = num; + for (i = 0; i < num; i++) { + fwcmd->params.req.delay[i].eq_id = + htole32(set_eqd[i].eq_id); + fwcmd->params.req.delay[i].phase = 0; + fwcmd->params.req.delay[i].dm = + htole32(set_eqd[i].delay_multiplier); + } + + + /* command post */ + rc = oce_mbox_post(sc, &mbx, NULL); + + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); } + + diff --git a/sys/dev/oce/oce_queue.c b/sys/dev/oce/oce_queue.c index 580853e..82972ee 100644 --- a/sys/dev/oce/oce_queue.c +++ b/sys/dev/oce/oce_queue.c @@ -36,8 +36,11 @@ * Costa Mesa, CA 92626 */ + + /* $FreeBSD$ */ + #include "oce_if.h" /***************************************************** @@ -89,6 +92,7 @@ oce_queue_init_all(POCE_SOFTC sc) int rc = 0, i, vector; struct oce_wq *wq; struct oce_rq *rq; + struct oce_aic_obj *aic; /* alloc TX/RX queues */ for_all_wq_queues(sc, wq, i) { @@ -113,6 +117,13 @@ oce_queue_init_all(POCE_SOFTC sc) /* create all of the event queues */ for (vector = 0; vector < sc->intr_count; vector++) { + /* setup aic defaults for each event queue */ + aic = &sc->aic_obj[vector]; + aic->max_eqd = OCE_MAX_EQD; + aic->min_eqd = OCE_MIN_EQD; + aic->et_eqd = OCE_MIN_EQD; + aic->enable = TRUE; + sc->eq[vector] = oce_eq_create(sc, EQ_LEN_1024, EQE_SIZE_4, 0, vector); if (!sc->eq[vector]) @@ -657,8 +668,7 @@ oce_mq_create(POCE_SOFTC sc, struct oce_eq *eq, uint32_t q_len) oce_mq_ext_ctx_t *ctx; uint32_t num_pages; uint32_t page_size; - uint32_t version; - + int version; cq = oce_cq_create(sc, eq, CQ_LEN_256, sizeof(struct oce_mq_cqe), 1, 1, 0, 0); @@ -680,8 +690,8 @@ oce_mq_create(POCE_SOFTC sc, struct oce_eq *eq, uint32_t q_len) bzero(&mbx, sizeof(struct oce_mbx)); + IS_XE201(sc) ? (version = OCE_MBX_VER_V1) : (version = OCE_MBX_VER_V0); fwcmd = (struct mbx_create_common_mq_ex *)&mbx.payload; - version = OCE_MBX_VER_V0; mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, MBX_SUBSYSTEM_COMMON, OPCODE_COMMON_CREATE_MQ_EXT, @@ -693,21 +703,41 @@ oce_mq_create(POCE_SOFTC sc, struct oce_eq *eq, uint32_t q_len) page_size = mq->ring->num_items * mq->ring->item_size; ctx = &fwcmd->params.req.context; - ctx->v0.num_pages = num_pages; - ctx->v0.cq_id = cq->cq_id; - ctx->v0.ring_size = OCE_LOG2(q_len) + 1; - ctx->v0.valid = 1; - /* Subscribe to Link State and Group 5 Events(bits 1 and 5 set) */ - ctx->v0.async_evt_bitmap = 0xffffffff; + + if (IS_XE201(sc)) { + ctx->v1.num_pages = num_pages; + ctx->v1.ring_size = OCE_LOG2(q_len) + 1; + ctx->v1.cq_id = cq->cq_id; + ctx->v1.valid = 1; + ctx->v1.async_cq_id = cq->cq_id; + ctx->v1.async_cq_valid = 1; + /* Subscribe to Link State and Group 5 Events(bits 1 & 5 set) */ + ctx->v1.async_evt_bitmap |= LE_32(0x00000022); + ctx->v1.async_evt_bitmap |= LE_32(1 << ASYNC_EVENT_CODE_DEBUG); + ctx->v1.async_evt_bitmap |= + LE_32(1 << ASYNC_EVENT_CODE_SLIPORT); + } + else { + ctx->v0.num_pages = num_pages; + ctx->v0.cq_id = cq->cq_id; + ctx->v0.ring_size = OCE_LOG2(q_len) + 1; + ctx->v0.valid = 1; + /* Subscribe to Link State and Group5 Events(bits 1 & 5 set) */ + ctx->v0.async_evt_bitmap = 0xffffffff; + } mbx.u0.s.embedded = 1; mbx.payload_length = sizeof(struct mbx_create_common_mq_ex); DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, &mbx, NULL); - if (rc) + if (!rc) + rc = fwcmd->hdr.u0.rsp.status; + if (rc) { + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); goto error; - + } mq->mq_id = LE_16(fwcmd->params.rsp.mq_id); mq->cq = cq; eq->cq[eq->cq_valid] = cq; @@ -824,10 +854,11 @@ oce_destroy_q(POCE_SOFTC sc, struct oce_mbx *mbx, size_t req_size, DW_SWAP(u32ptr(mbx), mbx->payload_length + OCE_BMBX_RHDR_SZ); rc = oce_mbox_post(sc, mbx, NULL); - - if (rc != 0) - device_printf(sc->dev, "Failed to del q\n"); - + if (!rc) + rc = hdr->u0.rsp.status; + if (rc) + device_printf(sc->dev,"%s failed - cmd status: %d\n", + __FUNCTION__, rc); return rc; } diff --git a/sys/dev/oce/oce_sysctl.c b/sys/dev/oce/oce_sysctl.c index 93117f6..d8a88de 100644 --- a/sys/dev/oce/oce_sysctl.c +++ b/sys/dev/oce/oce_sysctl.c @@ -38,6 +38,9 @@ /* $FreeBSD$ */ +/* $FreeBSD$ */ + + #include "oce_if.h" static void copy_stats_to_sc_xe201(POCE_SOFTC sc); @@ -49,6 +52,7 @@ static int oce_sys_fwupgrade(SYSCTL_HANDLER_ARGS); static int oce_be3_flashdata(POCE_SOFTC sc, const struct firmware *fw, int num_imgs); static int oce_lancer_fwupgrade(POCE_SOFTC sc, const struct firmware *fw); +static int oce_sysctl_sfp_vpd_dump(SYSCTL_HANDLER_ARGS); static boolean_t oce_phy_flashing_required(POCE_SOFTC sc); static boolean_t oce_img_flashing_required(POCE_SOFTC sc, const char *p, int img_optype, uint32_t img_offset, @@ -61,7 +65,7 @@ static void oce_add_stats_sysctls_xe201(POCE_SOFTC sc, struct sysctl_oid *stats_node); extern char component_revision[32]; - +uint32_t sfp_vpd_dump_buffer[TRANSCEIVER_DATA_NUM_ELE]; void oce_add_sysctls(POCE_SOFTC sc) @@ -93,7 +97,8 @@ oce_add_sysctls(POCE_SOFTC sc) sizeof(oce_max_rsp_handled), "Maximum receive frames handled per interupt"); - if (sc->function_mode & FNM_FLEX10_MODE) + if ((sc->function_mode & FNM_FLEX10_MODE) || + (sc->function_mode & FNM_UMC_MODE)) SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "speed", CTLFLAG_RD, @@ -121,6 +126,19 @@ oce_add_sysctls(POCE_SOFTC sc) CTLTYPE_STRING | CTLFLAG_RW, (void *)sc, 0, oce_sys_fwupgrade, "A", "Firmware ufi file"); + /* + * Dumps Transceiver data + * "sysctl dev.oce.0.sfp_vpd_dump=0" + * "sysctl -x dev.oce.0.sfp_vpd_dump_buffer" for hex dump + * "sysctl -b dev.oce.0.sfp_vpd_dump_buffer > sfp.bin" for binary dump + */ + SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "sfp_vpd_dump", + CTLTYPE_INT | CTLFLAG_RW, (void *)sc, 0, oce_sysctl_sfp_vpd_dump, + "I", "Initiate a sfp_vpd_dump operation"); + SYSCTL_ADD_OPAQUE(ctx, child, OID_AUTO, "sfp_vpd_dump_buffer", + CTLFLAG_RD, sfp_vpd_dump_buffer, + TRANSCEIVER_DATA_SIZE, "IU", "Access sfp_vpd_dump buffer"); + stats_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "stats", CTLFLAG_RD, NULL, "Ethernet Statistics"); @@ -133,7 +151,6 @@ oce_add_sysctls(POCE_SOFTC sc) } - static uint32_t oce_loopback_test(struct oce_softc *sc, uint8_t loopback_type) { @@ -147,7 +164,6 @@ oce_loopback_test(struct oce_softc *sc, uint8_t loopback_type) return status; } - static int oce_sysctl_loopback(SYSCTL_HANDLER_ARGS) { @@ -1303,3 +1319,31 @@ oce_refresh_nic_stats(POCE_SOFTC sc) return rc; } + +static int +oce_sysctl_sfp_vpd_dump(SYSCTL_HANDLER_ARGS) +{ + int result = 0, error; + int rc = 0; + POCE_SOFTC sc = (POCE_SOFTC) arg1; + + /* sysctl default handler */ + error = sysctl_handle_int(oidp, &result, 0, req); + if (error || !req->newptr) + return (error); + + if(result == -1) { + return EINVAL; + } + bzero((char *)sfp_vpd_dump_buffer, TRANSCEIVER_DATA_SIZE); + + rc = oce_mbox_read_transrecv_data(sc, PAGE_NUM_A0); + if(rc) + return rc; + + rc = oce_mbox_read_transrecv_data(sc, PAGE_NUM_A2); + if(rc) + return rc; + + return rc; +} diff --git a/sys/dev/oce/oce_util.c b/sys/dev/oce/oce_util.c index 7b227ac..b71c02d 100644 --- a/sys/dev/oce/oce_util.c +++ b/sys/dev/oce/oce_util.c @@ -36,8 +36,10 @@ * Costa Mesa, CA 92626 */ + /* $FreeBSD$ */ + #include "oce_if.h" static void oce_dma_map_ring(void *arg, @@ -71,7 +73,8 @@ oce_dma_alloc(POCE_SOFTC sc, bus_size_t size, POCE_DMA_MEM dma, int flags) if (rc == 0) { rc = bus_dmamem_alloc(dma->tag, &dma->ptr, - BUS_DMA_NOWAIT | BUS_DMA_COHERENT, + BUS_DMA_NOWAIT | BUS_DMA_COHERENT | + BUS_DMA_ZERO, &dma->map); } diff --git a/sys/dev/random/randomdev_soft.c b/sys/dev/random/randomdev_soft.c index 420a82a..004066eb 100644 --- a/sys/dev/random/randomdev_soft.c +++ b/sys/dev/random/randomdev_soft.c @@ -242,10 +242,10 @@ random_kthread(void *arg __unused) local_count = 0; /* Process until told to stop */ + mtx_lock_spin(&harvest_mtx); for (; random_kthread_control >= 0;) { /* Cycle through all the entropy sources */ - mtx_lock_spin(&harvest_mtx); for (source = RANDOM_START; source < ENTROPYSOURCE; source++) { /* * Drain entropy source records into a thread-local @@ -270,7 +270,6 @@ random_kthread(void *arg __unused) emptyfifo.count += local_count; local_count = 0; } - mtx_unlock_spin(&harvest_mtx); KASSERT(local_count == 0, ("random_kthread: local_count %d", local_count)); @@ -283,9 +282,11 @@ random_kthread(void *arg __unused) random_kthread_control = 0; /* Work done, so don't belabour the issue */ - pause("-", hz / 10); + msleep_spin_sbt(&random_kthread_control, &harvest_mtx, + "-", SBT_1S / 10, 0, C_PREL(1)); } + mtx_unlock_spin(&harvest_mtx); random_set_wakeup_exit(&random_kthread_control); /* NOTREACHED */ diff --git a/sys/dev/sound/pci/hda/hdaa_patches.c b/sys/dev/sound/pci/hda/hdaa_patches.c index 41d51d7..ef3124a 100644 --- a/sys/dev/sound/pci/hda/hdaa_patches.c +++ b/sys/dev/sound/pci/hda/hdaa_patches.c @@ -333,6 +333,23 @@ hdac_pin_patch(struct hdaa_widget *w) patch = "as=1 seq=15"; break; } + } else if (id == HDA_CODEC_CX20590 && + (subid == LENOVO_X1_SUBVENDOR || + subid == LENOVO_X220_SUBVENDOR || + subid == LENOVO_T420_SUBVENDOR || + subid == LENOVO_T520_SUBVENDOR)) { + switch (nid) { + case 25: + patch = "as=1 seq=15"; + break; + } + } else if (id == HDA_CODEC_ALC269 && + subid == LENOVO_X1CRBN_SUBVENDOR) { + switch (nid) { + case 21: + patch = "as=1 seq=15"; + break; + } } if (patch != NULL) diff --git a/sys/dev/sound/pci/hda/hdac.h b/sys/dev/sound/pci/hda/hdac.h index 9f9d687..e82b6ae 100644 --- a/sys/dev/sound/pci/hda/hdac.h +++ b/sys/dev/sound/pci/hda/hdac.h @@ -220,7 +220,12 @@ #define LENOVO_3KN200_SUBVENDOR HDA_MODEL_CONSTRUCT(LENOVO, 0x384e) #define LENOVO_B450_SUBVENDOR HDA_MODEL_CONSTRUCT(LENOVO, 0x3a0d) #define LENOVO_TCA55_SUBVENDOR HDA_MODEL_CONSTRUCT(LENOVO, 0x1015) +#define LENOVO_X1_SUBVENDOR HDA_MODEL_CONSTRUCT(LENOVO, 0x21e8) +#define LENOVO_X1CRBN_SUBVENDOR HDA_MODEL_CONSTRUCT(LENOVO, 0x21f9) +#define LENOVO_X220_SUBVENDOR HDA_MODEL_CONSTRUCT(LENOVO, 0x21da) #define LENOVO_X300_SUBVENDOR HDA_MODEL_CONSTRUCT(LENOVO, 0x20ac) +#define LENOVO_T420_SUBVENDOR HDA_MODEL_CONSTRUCT(LENOVO, 0x21ce) +#define LENOVO_T520_SUBVENDOR HDA_MODEL_CONSTRUCT(LENOVO, 0x21cf) #define LENOVO_ALL_SUBVENDOR HDA_MODEL_CONSTRUCT(LENOVO, 0xffff) /* Samsung */ diff --git a/sys/dev/sound/pci/hda/hdacc.c b/sys/dev/sound/pci/hda/hdacc.c index 55bf4af..c9282c9 100644 --- a/sys/dev/sound/pci/hda/hdacc.c +++ b/sys/dev/sound/pci/hda/hdacc.c @@ -460,8 +460,12 @@ hdacc_attach(device_t dev) static int hdacc_detach(device_t dev) { + struct hdacc_softc *codec = device_get_softc(dev); + int error; - return (device_delete_children(dev)); + error = device_delete_children(dev); + free(codec->fgs, M_HDACC); + return (error); } static int diff --git a/sys/dev/syscons/syscons.c b/sys/dev/syscons/syscons.c index 8094ea3..b863f90 100644 --- a/sys/dev/syscons/syscons.c +++ b/sys/dev/syscons/syscons.c @@ -506,6 +506,8 @@ sc_attach_unit(int unit, int flags) sc = sc_get_softc(unit, flags & SC_KERNEL_CONSOLE); sc->config = flags; + callout_init(&sc->ctimeout, 0); + callout_init(&sc->cblink, 0); scp = sc_get_stat(sc->dev[0]); if (sc_console == NULL) /* sc_console_unit < 0 */ sc_console = scp; @@ -1831,13 +1833,11 @@ static void scrn_timer(void *arg) { #ifndef PC98 - static int kbd_interval = 0; + static time_t kbd_time_stamp = 0; #endif - struct timeval tv; sc_softc_t *sc; scr_stat *scp; - int again; - int s; + int again, rate; again = (arg != NULL); if (arg != NULL) @@ -1847,18 +1847,18 @@ scrn_timer(void *arg) else return; + /* find the vty to update */ + scp = sc->cur_scp; + /* don't do anything when we are performing some I/O operations */ - if (suspend_in_progress || sc->font_loading_in_progress) { - if (again) - timeout(scrn_timer, sc, hz / 10); - return; - } - s = spltty(); + if (suspend_in_progress || sc->font_loading_in_progress) + goto done; #ifndef PC98 if ((sc->kbd == NULL) && (sc->config & SC_AUTODETECT_KBD)) { /* try to allocate a keyboard automatically */ - if (++kbd_interval >= 25) { + if (kbd_time_stamp != time_uptime) { + kbd_time_stamp = time_uptime; sc->keyboard = sc_allocate_keyboard(sc, -1); if (sc->keyboard >= 0) { sc->kbd = kbd_get_keyboard(sc->keyboard); @@ -1867,25 +1867,20 @@ scrn_timer(void *arg) update_kbd_state(sc->cur_scp, sc->cur_scp->status, LOCK_MASK); } - kbd_interval = 0; } } #endif /* PC98 */ - /* find the vty to update */ - scp = sc->cur_scp; - /* should we stop the screen saver? */ - getmicrouptime(&tv); if (debugger > 0 || panicstr || shutdown_in_progress) sc_touch_scrn_saver(); if (run_scrn_saver) { - if (tv.tv_sec > sc->scrn_time_stamp + scrn_blank_time) + if (time_uptime > sc->scrn_time_stamp + scrn_blank_time) sc->flags |= SC_SCRN_IDLE; else sc->flags &= ~SC_SCRN_IDLE; } else { - sc->scrn_time_stamp = tv.tv_sec; + sc->scrn_time_stamp = time_uptime; sc->flags &= ~SC_SCRN_IDLE; if (scrn_blank_time > 0) run_scrn_saver = TRUE; @@ -1898,12 +1893,8 @@ scrn_timer(void *arg) /* should we just return ? */ if (sc->blink_in_progress || sc->switch_in_progress - || sc->write_in_progress) { - if (again) - timeout(scrn_timer, sc, hz / 10); - splx(s); - return; - } + || sc->write_in_progress) + goto done; /* Update the screen */ scp = sc->cur_scp; /* cur_scp may have changed... */ @@ -1917,9 +1908,19 @@ scrn_timer(void *arg) (*current_saver)(sc, TRUE); #endif - if (again) - timeout(scrn_timer, sc, hz / 25); - splx(s); +done: + if (again) { + /* + * Use reduced "refresh" rate if we are in graphics and that is not a + * graphical screen saver. In such case we just have nothing to do. + */ + if (ISGRAPHSC(scp) && !(sc->flags & SC_SCRN_BLANKED)) + rate = 2; + else + rate = 30; + callout_reset_sbt(&sc->ctimeout, SBT_1S / rate, 0, + scrn_timer, sc, C_PREL(1)); + } } static int @@ -3863,7 +3864,8 @@ blink_screen(void *arg) (*scp->rndr->draw)(scp, 0, scp->xsize*scp->ysize, scp->sc->blink_in_progress & 1); scp->sc->blink_in_progress--; - timeout(blink_screen, scp, hz / 10); + callout_reset_sbt(&scp->sc->cblink, SBT_1S / 15, 0, + blink_screen, scp, C_PREL(0)); } } diff --git a/sys/dev/syscons/syscons.h b/sys/dev/syscons/syscons.h index 23a7bba..353b67f 100644 --- a/sys/dev/syscons/syscons.h +++ b/sys/dev/syscons/syscons.h @@ -269,6 +269,8 @@ typedef struct sc_softc { #ifdef KDB int sc_altbrk; #endif + struct callout ctimeout; + struct callout cblink; } sc_softc_t; /* virtual screen */ diff --git a/sys/dev/virtio/block/virtio_blk.c b/sys/dev/virtio/block/virtio_blk.c index cb628f6..21d7703 100644 --- a/sys/dev/virtio/block/virtio_blk.c +++ b/sys/dev/virtio/block/virtio_blk.c @@ -72,6 +72,7 @@ struct vtblk_softc { #define VTBLK_FLAG_DETACH 0x0004 #define VTBLK_FLAG_SUSPEND 0x0008 #define VTBLK_FLAG_DUMPING 0x0010 +#define VTBLK_FLAG_BARRIER 0x0020 struct virtqueue *vtblk_vq; struct sglist *vtblk_sglist; @@ -81,7 +82,8 @@ struct vtblk_softc { TAILQ_HEAD(, vtblk_request) vtblk_req_free; TAILQ_HEAD(, vtblk_request) - vtblk_req_ready; + vtblk_req_ready; + struct vtblk_request *vtblk_req_ordered; struct taskqueue *vtblk_tq; struct task vtblk_intr_task; @@ -278,9 +280,10 @@ vtblk_attach(device_t dev) if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) sc->vtblk_flags |= VTBLK_FLAG_INDIRECT; - if (virtio_with_feature(dev, VIRTIO_BLK_F_RO)) sc->vtblk_flags |= VTBLK_FLAG_READONLY; + if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER)) + sc->vtblk_flags |= VTBLK_FLAG_BARRIER; /* Get local copy of config. */ virtio_read_device_config(dev, 0, &blkcfg, @@ -766,25 +769,45 @@ vtblk_bio_request(struct vtblk_softc *sc) bp->bio_cmd); } - if (bp->bio_flags & BIO_ORDERED) - req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER; - return (req); } static int vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req) { + struct virtqueue *vq; struct sglist *sg; struct bio *bp; - int readable, writable, error; + int ordered, readable, writable, error; + vq = sc->vtblk_vq; sg = sc->vtblk_sglist; bp = req->vbr_bp; + ordered = 0; writable = 0; VTBLK_LOCK_ASSERT(sc); + /* + * Wait until the ordered request completes before + * executing subsequent requests. + */ + if (sc->vtblk_req_ordered != NULL) + return (EBUSY); + + if (bp->bio_flags & BIO_ORDERED) { + if ((sc->vtblk_flags & VTBLK_FLAG_BARRIER) == 0) { + /* + * This request will be executed once all + * the in-flight requests are completed. + */ + if (!virtqueue_empty(vq)) + return (EBUSY); + ordered = 1; + } else + req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER; + } + sglist_reset(sg); sglist_append(sg, &req->vbr_hdr, sizeof(struct virtio_blk_outhdr)); @@ -802,10 +825,13 @@ vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req) writable++; sglist_append(sg, &req->vbr_ack, sizeof(uint8_t)); - readable = sg->sg_nseg - writable; - return (virtqueue_enqueue(sc->vtblk_vq, req, sg, readable, writable)); + error = virtqueue_enqueue(vq, req, sg, readable, writable); + if (error == 0 && ordered) + sc->vtblk_req_ordered = req; + + return (error); } static int @@ -1013,6 +1039,12 @@ vtblk_finish_completed(struct vtblk_softc *sc) while ((req = virtqueue_dequeue(sc->vtblk_vq, NULL)) != NULL) { bp = req->vbr_bp; + if (sc->vtblk_req_ordered != NULL) { + /* This should be the only outstanding request. */ + MPASS(sc->vtblk_req_ordered == req); + sc->vtblk_req_ordered = NULL; + } + error = vtblk_request_error(req); if (error) disk_err(bp, "hard error", -1, 1); @@ -1039,6 +1071,7 @@ vtblk_drain_vq(struct vtblk_softc *sc, int skip_done) vtblk_enqueue_request(sc, req); } + sc->vtblk_req_ordered = NULL; KASSERT(virtqueue_empty(vq), ("virtqueue not empty")); } diff --git a/sys/geom/label/g_label_ntfs.c b/sys/geom/label/g_label_ntfs.c index f00669e..1ed4a07 100644 --- a/sys/geom/label/g_label_ntfs.c +++ b/sys/geom/label/g_label_ntfs.c @@ -55,7 +55,7 @@ struct ntfs_attr { uint16_t reserved3; uint16_t a_dataoff; uint16_t a_indexed; -}; +} __packed; struct ntfs_filerec { uint32_t fr_hdrmagic; @@ -70,7 +70,7 @@ struct ntfs_filerec { uint32_t fr_allocated; uint64_t fr_mainrec; uint16_t fr_attrnum; -}; +} __packed; struct ntfs_bootfile { uint8_t reserved1[3]; @@ -89,7 +89,7 @@ struct ntfs_bootfile { uint8_t bf_mftrecsz; uint32_t bf_ibsz; uint32_t bf_volsn; -}; +} __packed; static void g_label_ntfs_taste(struct g_consumer *cp, char *label, size_t size) diff --git a/sys/geom/part/g_part.c b/sys/geom/part/g_part.c index e2ba79e..f7f6f55 100644 --- a/sys/geom/part/g_part.c +++ b/sys/geom/part/g_part.c @@ -80,6 +80,7 @@ struct g_part_alias_list { { "bios-boot", G_PART_ALIAS_BIOS_BOOT }, { "ebr", G_PART_ALIAS_EBR }, { "efi", G_PART_ALIAS_EFI }, + { "fat16", G_PART_ALIAS_MS_FAT16 }, { "fat32", G_PART_ALIAS_MS_FAT32 }, { "freebsd", G_PART_ALIAS_FREEBSD }, { "freebsd-boot", G_PART_ALIAS_FREEBSD_BOOT }, diff --git a/sys/geom/part/g_part.h b/sys/geom/part/g_part.h index 39bcc9d..05a8c57 100644 --- a/sys/geom/part/g_part.h +++ b/sys/geom/part/g_part.h @@ -68,6 +68,7 @@ enum g_part_alias { G_PART_ALIAS_NETBSD_SWAP, /* A NetBSD swap partition entry. */ G_PART_ALIAS_NETBSD_LFS, /* A NetBSD LFS partition entry. */ G_PART_ALIAS_EBR, /* A EBR partition entry. */ + G_PART_ALIAS_MS_FAT16, /* A Microsoft FAT16 partition entry. */ G_PART_ALIAS_MS_FAT32, /* A Microsoft FAT32 partition entry. */ G_PART_ALIAS_BIOS_BOOT, /* A GRUB 2 boot partition entry. */ G_PART_ALIAS_VMFS, /* A VMware VMFS partition entry */ diff --git a/sys/geom/part/g_part_mbr.c b/sys/geom/part/g_part_mbr.c index 619553c..d522d02 100644 --- a/sys/geom/part/g_part_mbr.c +++ b/sys/geom/part/g_part_mbr.c @@ -118,6 +118,7 @@ static struct g_part_mbr_alias { { DOSPTYP_386BSD, G_PART_ALIAS_FREEBSD }, { DOSPTYP_EXT, G_PART_ALIAS_EBR }, { DOSPTYP_NTFS, G_PART_ALIAS_MS_NTFS }, + { DOSPTYP_FAT16, G_PART_ALIAS_MS_FAT16 }, { DOSPTYP_FAT32, G_PART_ALIAS_MS_FAT32 }, { DOSPTYP_EXTLBA, G_PART_ALIAS_EBR }, { DOSPTYP_LDM, G_PART_ALIAS_MS_LDM_DATA }, diff --git a/sys/geom/raid/md_ddf.c b/sys/geom/raid/md_ddf.c index 7d4bd1e..c18a319 100644 --- a/sys/geom/raid/md_ddf.c +++ b/sys/geom/raid/md_ddf.c @@ -881,7 +881,10 @@ ddf_vol_meta_update(struct ddf_vol_meta *dst, struct ddf_meta *src, hdr = src->hdr; vde = &src->vdr->entry[ddf_meta_find_vd(src, GUID)]; vdc = ddf_meta_find_vdc(src, GUID); - bvd = GET8D(src, vdc->Secondary_Element_Seq); + if (GET8D(src, vdc->Secondary_Element_Count) == 1) + bvd = 0; + else + bvd = GET8D(src, vdc->Secondary_Element_Seq); size = GET16(src, hdr->Configuration_Record_Length) * src->sectorsize; if (dst->vdc == NULL || diff --git a/sys/i386/conf/GENERIC b/sys/i386/conf/GENERIC index 3a98ded..d35a780 100644 --- a/sys/i386/conf/GENERIC +++ b/sys/i386/conf/GENERIC @@ -146,7 +146,10 @@ device sa # Sequential Access (tape etc) device cd # CD device pass # Passthrough device (direct ATA/SCSI access) device ses # Enclosure Services (SES and SAF-TE) -#device ctl # CAM Target Layer +device ctl # CAM Target Layer +options CTL_DISABLE # Disable CTL by default to save memory. + # Re-enable with kern.cam.ctl.disable=0 in + # /boot/loader.conf # RAID controllers interfaced to the SCSI subsystem device amr # AMI MegaRAID diff --git a/sys/i386/conf/NOTES b/sys/i386/conf/NOTES index 95ccd4e..f8f78cc 100644 --- a/sys/i386/conf/NOTES +++ b/sys/i386/conf/NOTES @@ -773,6 +773,13 @@ device glxsb # AMD Geode LX Security Block # # VirtIO support +# +# The virtio entry provides a generic bus for use by the device drivers. +# It must be combined with an interface that communicates with the host. +# Multiple such interfaces defined by the VirtIO specification. FreeBSD +# only has support for PCI. Therefore, virtio_pci must be statically +# compiled in or loaded as a module for the device drivers to function. +# device virtio # Generic VirtIO bus (required) device virtio_pci # VirtIO PCI Interface device vtnet # VirtIO Ethernet device diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c index 4439ccb..2841f8c 100644 --- a/sys/kern/kern_clock.c +++ b/sys/kern/kern_clock.c @@ -78,7 +78,8 @@ __FBSDID("$FreeBSD$"); #include <sys/pmckern.h> PMC_SOFT_DEFINE( , , clock, hard); PMC_SOFT_DEFINE( , , clock, stat); -PMC_SOFT_DEFINE( , , clock, prof); +PMC_SOFT_DEFINE_EX( , , clock, prof, \ + cpu_startprofclock, cpu_stopprofclock); #endif #ifdef DEVICE_POLLING diff --git a/sys/kern/kern_event.c b/sys/kern/kern_event.c index 6a920ca..c3d1b86 100644 --- a/sys/kern/kern_event.c +++ b/sys/kern/kern_event.c @@ -517,39 +517,28 @@ knote_fork(struct knlist *list, int pid) * XXX: EVFILT_TIMER should perhaps live in kern_time.c beside the * interval timer support code. */ -static int -timertoticks(intptr_t data) +static __inline sbintime_t +timer2sbintime(intptr_t data) { - struct timeval tv; - int tticks; - - tv.tv_sec = data / 1000; - tv.tv_usec = (data % 1000) * 1000; - tticks = tvtohz(&tv); - return tticks; + return (SBT_1MS * data); } static void filt_timerexpire(void *knx) { - struct knote *kn = knx; struct callout *calloutp; + struct knote *kn; + kn = knx; kn->kn_data++; KNOTE_ACTIVATE(kn, 0); /* XXX - handle locking */ - /* - * timertoticks() uses tvtohz() which always adds 1 to allow - * for the time until the next clock interrupt being strictly - * less than 1 clock tick. We don't want that here since we - * want to appear to be in sync with the clock interrupt even - * when we're delayed. - */ if ((kn->kn_flags & EV_ONESHOT) != EV_ONESHOT) { calloutp = (struct callout *)kn->kn_hook; - callout_reset_curcpu(calloutp, timertoticks(kn->kn_sdata) - 1, - filt_timerexpire, kn); + callout_reset_sbt_on(calloutp, + timer2sbintime(kn->kn_sdata), 0 /* 1ms? */, + filt_timerexpire, kn, PCPU_GET(cpuid), 0); } } @@ -573,8 +562,9 @@ filt_timerattach(struct knote *kn) calloutp = malloc(sizeof(*calloutp), M_KQUEUE, M_WAITOK); callout_init(calloutp, CALLOUT_MPSAFE); kn->kn_hook = calloutp; - callout_reset_curcpu(calloutp, timertoticks(kn->kn_sdata), - filt_timerexpire, kn); + callout_reset_sbt_on(calloutp, + timer2sbintime(kn->kn_sdata), 0 /* 1ms? */, + filt_timerexpire, kn, PCPU_GET(cpuid), 0); return (0); } @@ -1319,10 +1309,9 @@ kqueue_scan(struct kqueue *kq, int maxevents, struct kevent_copyops *k_ops, const struct timespec *tsp, struct kevent *keva, struct thread *td) { struct kevent *kevp; - struct timeval atv, rtv, ttv; struct knote *kn, *marker; - int count, timeout, nkev, error, influx; - int haskqglobal, touch; + sbintime_t asbt, rsbt; + int count, error, haskqglobal, influx, nkev, touch; count = maxevents; nkev = 0; @@ -1332,24 +1321,28 @@ kqueue_scan(struct kqueue *kq, int maxevents, struct kevent_copyops *k_ops, if (maxevents == 0) goto done_nl; + rsbt = 0; if (tsp != NULL) { - TIMESPEC_TO_TIMEVAL(&atv, tsp); - if (itimerfix(&atv)) { + if (tsp->tv_sec < 0 || tsp->tv_nsec < 0 || + tsp->tv_nsec >= 1000000000) { error = EINVAL; goto done_nl; } - if (tsp->tv_sec == 0 && tsp->tv_nsec == 0) - timeout = -1; - else - timeout = atv.tv_sec > 24 * 60 * 60 ? - 24 * 60 * 60 * hz : tvtohz(&atv); - getmicrouptime(&rtv); - timevaladd(&atv, &rtv); - } else { - atv.tv_sec = 0; - atv.tv_usec = 0; - timeout = 0; - } + if (timespecisset(tsp)) { + if (tsp->tv_sec < INT32_MAX) { + rsbt = tstosbt(*tsp); + if (TIMESEL(&asbt, rsbt)) + asbt += tc_tick_sbt; + asbt += rsbt; + if (asbt < rsbt) + asbt = 0; + rsbt >>= tc_precexp; + } else + asbt = 0; + } else + asbt = -1; + } else + asbt = 0; marker = knote_alloc(1); if (marker == NULL) { error = ENOMEM; @@ -1357,28 +1350,16 @@ kqueue_scan(struct kqueue *kq, int maxevents, struct kevent_copyops *k_ops, } marker->kn_status = KN_MARKER; KQ_LOCK(kq); - goto start; retry: - if (atv.tv_sec || atv.tv_usec) { - getmicrouptime(&rtv); - if (timevalcmp(&rtv, &atv, >=)) - goto done; - ttv = atv; - timevalsub(&ttv, &rtv); - timeout = ttv.tv_sec > 24 * 60 * 60 ? - 24 * 60 * 60 * hz : tvtohz(&ttv); - } - -start: kevp = keva; if (kq->kq_count == 0) { - if (timeout < 0) { + if (asbt == -1) { error = EWOULDBLOCK; } else { kq->kq_state |= KQ_SLEEP; - error = msleep(kq, &kq->kq_lock, PSOCK | PCATCH, - "kqread", timeout); + error = msleep_sbt(kq, &kq->kq_lock, PSOCK | PCATCH, + "kqread", asbt, rsbt, C_ABSOLUTE); } if (error == 0) goto retry; diff --git a/sys/kern/kern_resource.c b/sys/kern/kern_resource.c index 590cfcb..57ee671 100644 --- a/sys/kern/kern_resource.c +++ b/sys/kern/kern_resource.c @@ -469,8 +469,7 @@ sys_rtprio(td, uap) int rtp_to_pri(struct rtprio *rtp, struct thread *td) { - u_char newpri; - u_char oldpri; + u_char newpri, oldclass, oldpri; switch (RTP_PRIO_BASE(rtp->type)) { case RTP_PRIO_REALTIME: @@ -493,11 +492,12 @@ rtp_to_pri(struct rtprio *rtp, struct thread *td) } thread_lock(td); + oldclass = td->td_pri_class; sched_class(td, rtp->type); /* XXX fix */ oldpri = td->td_user_pri; sched_user_prio(td, newpri); - if (td->td_user_pri != oldpri && (td == curthread || - td->td_priority == oldpri || td->td_user_pri <= PRI_MAX_REALTIME)) + if (td->td_user_pri != oldpri && (oldclass != RTP_PRIO_NORMAL || + td->td_pri_class != RTP_PRIO_NORMAL)) sched_prio(td, td->td_user_pri); if (TD_ON_UPILOCK(td) && oldpri != newpri) { critical_enter(); @@ -645,7 +645,8 @@ lim_cb(void *arg) } } if ((p->p_flag & P_WEXIT) == 0) - callout_reset(&p->p_limco, hz, lim_cb, p); + callout_reset_sbt(&p->p_limco, SBT_1S, 0, + lim_cb, p, C_PREL(1)); } int @@ -697,7 +698,8 @@ kern_proc_setrlimit(struct thread *td, struct proc *p, u_int which, case RLIMIT_CPU: if (limp->rlim_cur != RLIM_INFINITY && p->p_cpulimit == RLIM_INFINITY) - callout_reset(&p->p_limco, hz, lim_cb, p); + callout_reset_sbt(&p->p_limco, SBT_1S, 0, + lim_cb, p, C_PREL(1)); p->p_cpulimit = limp->rlim_cur; break; case RLIMIT_DATA: @@ -1137,7 +1139,8 @@ lim_fork(struct proc *p1, struct proc *p2) p2->p_limit = lim_hold(p1->p_limit); callout_init_mtx(&p2->p_limco, &p2->p_mtx, 0); if (p1->p_cpulimit != RLIM_INFINITY) - callout_reset(&p2->p_limco, hz, lim_cb, p2); + callout_reset_sbt(&p2->p_limco, SBT_1S, 0, + lim_cb, p2, C_PREL(1)); } void diff --git a/sys/kern/kern_time.c b/sys/kern/kern_time.c index 97c288d..4225928 100644 --- a/sys/kern/kern_time.c +++ b/sys/kern/kern_time.c @@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$"); #include <sys/resourcevar.h> #include <sys/signalvar.h> #include <sys/kernel.h> +#include <sys/sleepqueue.h> #include <sys/syscallsubr.h> #include <sys/sysctl.h> #include <sys/sysent.h> @@ -481,38 +482,45 @@ static int nanowait; int kern_nanosleep(struct thread *td, struct timespec *rqt, struct timespec *rmt) { - struct timespec ts, ts2, ts3; - struct timeval tv; + struct timespec ts; + sbintime_t sbt, sbtt, prec, tmp; + time_t over; int error; if (rqt->tv_nsec < 0 || rqt->tv_nsec >= 1000000000) return (EINVAL); if (rqt->tv_sec < 0 || (rqt->tv_sec == 0 && rqt->tv_nsec == 0)) return (0); - getnanouptime(&ts); - timespecadd(&ts, rqt); - TIMESPEC_TO_TIMEVAL(&tv, rqt); - for (;;) { - error = tsleep(&nanowait, PWAIT | PCATCH, "nanslp", - tvtohz(&tv)); - getnanouptime(&ts2); - if (error != EWOULDBLOCK) { - if (error == ERESTART) - error = EINTR; - if (rmt != NULL) { - timespecsub(&ts, &ts2); - if (ts.tv_sec < 0) - timespecclear(&ts); - *rmt = ts; - } - return (error); + ts = *rqt; + if (ts.tv_sec > INT32_MAX / 2) { + over = ts.tv_sec - INT32_MAX / 2; + ts.tv_sec -= over; + } else + over = 0; + tmp = tstosbt(ts); + prec = tmp; + prec >>= tc_precexp; + if (TIMESEL(&sbt, tmp)) + sbt += tc_tick_sbt; + sbt += tmp; + error = tsleep_sbt(&nanowait, PWAIT | PCATCH, "nanslp", sbt, prec, + C_ABSOLUTE); + if (error != EWOULDBLOCK) { + if (error == ERESTART) + error = EINTR; + TIMESEL(&sbtt, tmp); + if (rmt != NULL) { + ts = sbttots(sbt - sbtt); + ts.tv_sec += over; + if (ts.tv_sec < 0) + timespecclear(&ts); + *rmt = ts; } - if (timespeccmp(&ts2, &ts, >=)) + if (sbtt >= sbt) return (0); - ts3 = ts; - timespecsub(&ts3, &ts2); - TIMESPEC_TO_TIMEVAL(&tv, &ts3); + return (error); } + return (0); } #ifndef _SYS_SYSPROTO_H_ @@ -683,7 +691,7 @@ kern_getitimer(struct thread *td, u_int which, struct itimerval *aitv) *aitv = p->p_realtimer; PROC_UNLOCK(p); if (timevalisset(&aitv->it_value)) { - getmicrouptime(&ctv); + microuptime(&ctv); if (timevalcmp(&aitv->it_value, &ctv, <)) timevalclear(&aitv->it_value); else @@ -728,28 +736,33 @@ kern_setitimer(struct thread *td, u_int which, struct itimerval *aitv, { struct proc *p = td->td_proc; struct timeval ctv; + sbintime_t sbt, pr; if (aitv == NULL) return (kern_getitimer(td, which, oitv)); if (which > ITIMER_PROF) return (EINVAL); - if (itimerfix(&aitv->it_value)) + if (itimerfix(&aitv->it_value) || + aitv->it_value.tv_sec > INT32_MAX / 2) return (EINVAL); if (!timevalisset(&aitv->it_value)) timevalclear(&aitv->it_interval); - else if (itimerfix(&aitv->it_interval)) + else if (itimerfix(&aitv->it_interval) || + aitv->it_interval.tv_sec > INT32_MAX / 2) return (EINVAL); if (which == ITIMER_REAL) { PROC_LOCK(p); if (timevalisset(&p->p_realtimer.it_value)) callout_stop(&p->p_itcallout); - getmicrouptime(&ctv); + microuptime(&ctv); if (timevalisset(&aitv->it_value)) { - callout_reset(&p->p_itcallout, tvtohz(&aitv->it_value), - realitexpire, p); + pr = tvtosbt(aitv->it_value) >> tc_precexp; timevaladd(&aitv->it_value, &ctv); + sbt = tvtosbt(aitv->it_value); + callout_reset_sbt(&p->p_itcallout, sbt, pr, + realitexpire, p, C_ABSOLUTE); } *oitv = p->p_realtimer; p->p_realtimer = *aitv; @@ -785,7 +798,8 @@ void realitexpire(void *arg) { struct proc *p; - struct timeval ctv, ntv; + struct timeval ctv; + sbintime_t isbt; p = (struct proc *)arg; kern_psignal(p, SIGALRM); @@ -795,19 +809,17 @@ realitexpire(void *arg) wakeup(&p->p_itcallout); return; } - for (;;) { + isbt = tvtosbt(p->p_realtimer.it_interval); + if (isbt >= sbt_timethreshold) + getmicrouptime(&ctv); + else + microuptime(&ctv); + do { timevaladd(&p->p_realtimer.it_value, &p->p_realtimer.it_interval); - getmicrouptime(&ctv); - if (timevalcmp(&p->p_realtimer.it_value, &ctv, >)) { - ntv = p->p_realtimer.it_value; - timevalsub(&ntv, &ctv); - callout_reset(&p->p_itcallout, tvtohz(&ntv) - 1, - realitexpire, p); - return; - } - } - /*NOTREACHED*/ + } while (timevalcmp(&p->p_realtimer.it_value, &ctv, <=)); + callout_reset_sbt(&p->p_itcallout, tvtosbt(p->p_realtimer.it_value), + isbt >> tc_precexp, realitexpire, p, C_ABSOLUTE); } /* @@ -822,8 +834,9 @@ itimerfix(struct timeval *tv) if (tv->tv_sec < 0 || tv->tv_usec < 0 || tv->tv_usec >= 1000000) return (EINVAL); - if (tv->tv_sec == 0 && tv->tv_usec != 0 && tv->tv_usec < tick) - tv->tv_usec = tick; + if (tv->tv_sec == 0 && tv->tv_usec != 0 && + tv->tv_usec < (u_int)tick / 16) + tv->tv_usec = (u_int)tick / 16; return (0); } diff --git a/sys/kern/kern_timeout.c b/sys/kern/kern_timeout.c index 0787c01..df459cd 100644 --- a/sys/kern/kern_timeout.c +++ b/sys/kern/kern_timeout.c @@ -126,8 +126,8 @@ struct cc_exec { int ce_migration_cpu; sbintime_t ce_migration_time; #endif - boolean_t cc_cancel; - boolean_t cc_waiting; + bool cc_cancel; + bool cc_waiting; }; /* @@ -215,8 +215,8 @@ cc_cce_cleanup(struct callout_cpu *cc, int direct) cc->cc_exec_entity[direct].cc_curr = NULL; cc->cc_exec_entity[direct].cc_next = NULL; - cc->cc_exec_entity[direct].cc_cancel = FALSE; - cc->cc_exec_entity[direct].cc_waiting = FALSE; + cc->cc_exec_entity[direct].cc_cancel = false; + cc->cc_exec_entity[direct].cc_waiting = false; #ifdef SMP cc->cc_exec_entity[direct].ce_migration_cpu = CPUBLOCK; cc->cc_exec_entity[direct].ce_migration_time = 0; @@ -601,7 +601,7 @@ softclock_call_cc(struct callout *c, struct callout_cpu *cc, sbintime_t new_time; #endif #if defined(DIAGNOSTIC) || defined(CALLOUT_PROFILING) - sbintime_t bt1, bt2; + sbintime_t sbt1, sbt2; struct timespec ts2; static sbintime_t maxdt = 2 * SBT_1MS; /* 2 msec */ static timeout_t *lastfunc; @@ -621,7 +621,7 @@ softclock_call_cc(struct callout *c, struct callout_cpu *cc, else c->c_flags &= ~CALLOUT_PENDING; cc->cc_exec_entity[direct].cc_curr = c; - cc->cc_exec_entity[direct].cc_cancel = FALSE; + cc->cc_exec_entity[direct].cc_cancel = false; CC_UNLOCK(cc); if (c_lock != NULL) { class->lc_lock(c_lock, sharedlock); @@ -634,7 +634,7 @@ softclock_call_cc(struct callout *c, struct callout_cpu *cc, goto skip; } /* The callout cannot be stopped now. */ - cc->cc_exec_entity[direct].cc_cancel = TRUE; + cc->cc_exec_entity[direct].cc_cancel = true; if (c_lock == &Giant.lock_object) { #ifdef CALLOUT_PROFILING (*gcalls)++; @@ -655,7 +655,7 @@ softclock_call_cc(struct callout *c, struct callout_cpu *cc, CTR3(KTR_CALLOUT, "callout %p func %p arg %p", c, c_func, c_arg); } -#ifdef DIAGNOSTIC +#if defined(DIAGNOSTIC) || defined(CALLOUT_PROFILING) sbt1 = sbinuptime(); #endif THREAD_NO_SLEEPING(); @@ -663,17 +663,17 @@ softclock_call_cc(struct callout *c, struct callout_cpu *cc, c_func(c_arg); SDT_PROBE(callout_execute, kernel, , callout_end, c, 0, 0, 0, 0); THREAD_SLEEPING_OK(); -#ifdef DIAGNOSTIC - bt2 = sbinuptime(); - bt2 -= bt1; - if (bt2 > maxdt) { - if (lastfunc != c_func || bt2 > maxdt * 2) { - ts2 = sbttots(bt2); +#if defined(DIAGNOSTIC) || defined(CALLOUT_PROFILING) + sbt2 = sbinuptime(); + sbt2 -= sbt1; + if (sbt2 > maxdt) { + if (lastfunc != c_func || sbt2 > maxdt * 2) { + ts2 = sbttots(sbt2); printf( "Expensive timeout(9) function: %p(%p) %jd.%09ld s\n", c_func, c_arg, (intmax_t)ts2.tv_sec, ts2.tv_nsec); } - maxdt = bt2; + maxdt = sbt2; lastfunc = c_func; } #endif @@ -700,7 +700,7 @@ skip: */ c->c_flags &= ~CALLOUT_DFRMIGRATION; } - cc->cc_exec_entity[direct].cc_waiting = FALSE; + cc->cc_exec_entity[direct].cc_waiting = false; CC_UNLOCK(cc); wakeup(&cc->cc_exec_entity[direct].cc_waiting); CC_LOCK(cc); @@ -954,7 +954,7 @@ callout_reset_sbt_on(struct callout *c, sbintime_t sbt, sbintime_t precision, * can cancel the callout if it has not really started. */ if (c->c_lock != NULL && !cc->cc_exec_entity[direct].cc_cancel) - cancelled = cc->cc_exec_entity[direct].cc_cancel = TRUE; + cancelled = cc->cc_exec_entity[direct].cc_cancel = true; if (cc->cc_exec_entity[direct].cc_waiting) { /* * Someone has called callout_drain to kill this @@ -1135,7 +1135,7 @@ again: * will be packed up, just let softclock() * take care of it. */ - cc->cc_exec_entity[direct].cc_waiting = TRUE; + cc->cc_exec_entity[direct].cc_waiting = true; DROP_GIANT(); CC_UNLOCK(cc); sleepq_add( @@ -1161,7 +1161,7 @@ again: * lock, the callout will be skipped in * softclock(). */ - cc->cc_exec_entity[direct].cc_cancel = TRUE; + cc->cc_exec_entity[direct].cc_cancel = true; CTR3(KTR_CALLOUT, "cancelled %p func %p arg %p", c, c->c_func, c->c_arg); KASSERT(!cc_cce_migrating(cc, direct), diff --git a/sys/kern/subr_log.c b/sys/kern/subr_log.c index 8981358..1e61274 100644 --- a/sys/kern/subr_log.c +++ b/sys/kern/subr_log.c @@ -117,8 +117,8 @@ logopen(struct cdev *dev, int flags, int mode, struct thread *td) return (EBUSY); } log_open = 1; - callout_reset(&logsoftc.sc_callout, hz / log_wakeups_per_second, - logtimeout, NULL); + callout_reset_sbt(&logsoftc.sc_callout, + SBT_1S / log_wakeups_per_second, 0, logtimeout, NULL, C_PREL(1)); mtx_unlock(&msgbuf_lock); fsetown(td->td_proc->p_pid, &logsoftc.sc_sigio); /* signal process only */ @@ -233,22 +233,21 @@ logtimeout(void *arg) if (!log_open) return; - if (log_wakeups_per_second < 1) { - printf("syslog wakeup is less than one. Adjusting to 1.\n"); - log_wakeups_per_second = 1; - } - if (msgbuftrigger == 0) { - callout_schedule(&logsoftc.sc_callout, - hz / log_wakeups_per_second); - return; - } + if (msgbuftrigger == 0) + goto done; msgbuftrigger = 0; selwakeuppri(&logsoftc.sc_selp, LOG_RDPRI); KNOTE_LOCKED(&logsoftc.sc_selp.si_note, 0); if ((logsoftc.sc_state & LOG_ASYNC) && logsoftc.sc_sigio != NULL) pgsigio(&logsoftc.sc_sigio, SIGIO, 0); cv_broadcastpri(&log_wakeup, LOG_RDPRI); - callout_schedule(&logsoftc.sc_callout, hz / log_wakeups_per_second); +done: + if (log_wakeups_per_second < 1) { + printf("syslog wakeup is less than one. Adjusting to 1.\n"); + log_wakeups_per_second = 1; + } + callout_reset_sbt(&logsoftc.sc_callout, + SBT_1S / log_wakeups_per_second, 0, logtimeout, NULL, C_PREL(1)); } /*ARGSUSED*/ diff --git a/sys/kern/sys_generic.c b/sys/kern/sys_generic.c index 39f33f3..2d149b5 100644 --- a/sys/kern/sys_generic.c +++ b/sys/kern/sys_generic.c @@ -103,7 +103,7 @@ static int dofilewrite(struct thread *, int, struct file *, struct uio *, off_t, int); static void doselwakeup(struct selinfo *, int); static void seltdinit(struct thread *); -static int seltdwait(struct thread *, int); +static int seltdwait(struct thread *, sbintime_t, sbintime_t); static void seltdclear(struct thread *); /* @@ -950,9 +950,10 @@ kern_select(struct thread *td, int nd, fd_set *fd_in, fd_set *fd_ou, */ fd_mask s_selbits[howmany(2048, NFDBITS)]; fd_mask *ibits[3], *obits[3], *selbits, *sbp; - struct timeval atv, rtv, ttv; - int error, lf, ndu, timo; + struct timeval rtv; + sbintime_t asbt, precision, rsbt; u_int nbufbytes, ncpbytes, ncpubytes, nfdbits; + int error, lf, ndu; if (nd < 0) return (EINVAL); @@ -1042,35 +1043,36 @@ kern_select(struct thread *td, int nd, fd_set *fd_in, fd_set *fd_ou, if (nbufbytes != 0) bzero(selbits, nbufbytes / 2); + precision = 0; if (tvp != NULL) { - atv = *tvp; - if (itimerfix(&atv)) { + rtv = *tvp; + if (rtv.tv_sec < 0 || rtv.tv_usec < 0 || + rtv.tv_usec >= 1000000) { error = EINVAL; goto done; } - getmicrouptime(&rtv); - timevaladd(&atv, &rtv); - } else { - atv.tv_sec = 0; - atv.tv_usec = 0; - } - timo = 0; + if (rtv.tv_sec == 0 && rtv.tv_usec == 0) + asbt = 0; + else if (rtv.tv_sec < INT32_MAX) { + rsbt = tvtosbt(rtv); + precision = rsbt; + precision >>= tc_precexp; + if (TIMESEL(&asbt, rsbt)) + asbt += tc_tick_sbt; + asbt += rsbt; + if (asbt < rsbt) + asbt = -1; + } else + asbt = -1; + } else + asbt = -1; seltdinit(td); /* Iterate until the timeout expires or descriptors become ready. */ for (;;) { error = selscan(td, ibits, obits, nd); if (error || td->td_retval[0] != 0) break; - if (atv.tv_sec || atv.tv_usec) { - getmicrouptime(&rtv); - if (timevalcmp(&rtv, &atv, >=)) - break; - ttv = atv; - timevalsub(&ttv, &rtv); - timo = ttv.tv_sec > 24 * 60 * 60 ? - 24 * 60 * 60 * hz : tvtohz(&ttv); - } - error = seltdwait(td, timo); + error = seltdwait(td, asbt, precision); if (error) break; error = selrescan(td, ibits, obits); @@ -1278,9 +1280,9 @@ sys_poll(td, uap) { struct pollfd *bits; struct pollfd smallbits[32]; - struct timeval atv, rtv, ttv; - int error, timo; + sbintime_t asbt, precision, rsbt; u_int nfds; + int error; size_t ni; nfds = uap->nfds; @@ -1294,36 +1296,31 @@ sys_poll(td, uap) error = copyin(uap->fds, bits, ni); if (error) goto done; + precision = 0; if (uap->timeout != INFTIM) { - atv.tv_sec = uap->timeout / 1000; - atv.tv_usec = (uap->timeout % 1000) * 1000; - if (itimerfix(&atv)) { + if (uap->timeout < 0) { error = EINVAL; goto done; } - getmicrouptime(&rtv); - timevaladd(&atv, &rtv); - } else { - atv.tv_sec = 0; - atv.tv_usec = 0; - } - timo = 0; + if (uap->timeout == 0) + asbt = 0; + else { + rsbt = SBT_1MS * uap->timeout; + precision = rsbt; + precision >>= tc_precexp; + if (TIMESEL(&asbt, rsbt)) + asbt += tc_tick_sbt; + asbt += rsbt; + } + } else + asbt = -1; seltdinit(td); /* Iterate until the timeout expires or descriptors become ready. */ for (;;) { error = pollscan(td, bits, nfds); if (error || td->td_retval[0] != 0) break; - if (atv.tv_sec || atv.tv_usec) { - getmicrouptime(&rtv); - if (timevalcmp(&rtv, &atv, >=)) - break; - ttv = atv; - timevalsub(&ttv, &rtv); - timo = ttv.tv_sec > 24 * 60 * 60 ? - 24 * 60 * 60 * hz : tvtohz(&ttv); - } - error = seltdwait(td, timo); + error = seltdwait(td, asbt, precision); if (error) break; error = pollrescan(td); @@ -1667,7 +1664,7 @@ out: } static int -seltdwait(struct thread *td, int timo) +seltdwait(struct thread *td, sbintime_t sbt, sbintime_t precision) { struct seltd *stp; int error; @@ -1686,8 +1683,11 @@ seltdwait(struct thread *td, int timo) mtx_unlock(&stp->st_mtx); return (0); } - if (timo > 0) - error = cv_timedwait_sig(&stp->st_wait, &stp->st_mtx, timo); + if (sbt == 0) + error = EWOULDBLOCK; + else if (sbt != -1) + error = cv_timedwait_sig_sbt(&stp->st_wait, &stp->st_mtx, + sbt, precision, C_ABSOLUTE); else error = cv_wait_sig(&stp->st_wait, &stp->st_mtx); mtx_unlock(&stp->st_mtx); diff --git a/sys/modules/drm2/drm2/Makefile b/sys/modules/drm2/drm2/Makefile index 1bc979d..215a0ed 100644 --- a/sys/modules/drm2/drm2/Makefile +++ b/sys/modules/drm2/drm2/Makefile @@ -1,6 +1,6 @@ # $FreeBSD$ -.PATH: ${.CURDIR}/../../../dev/drm2 +.PATH: ${.CURDIR}/../../../dev/drm2 ${.CURDIR}/../../../dev/drm2/ttm KMOD = drm2 SRCS = \ drm_agpsupport.c \ @@ -18,6 +18,7 @@ SRCS = \ drm_fops.c \ drm_gem.c \ drm_gem_names.c \ + drm_global.c \ drm_hashtab.c \ drm_ioctl.c \ drm_irq.c \ @@ -31,7 +32,19 @@ SRCS = \ drm_sman.c \ drm_stub.c \ drm_sysctl.c \ - drm_vm.c + drm_vm.c \ + ttm_lock.c \ + ttm_object.c \ + ttm_tt.c \ + ttm_bo_util.c \ + ttm_bo.c \ + ttm_bo_manager.c \ + ttm_execbuf_util.c \ + ttm_memory.c \ + ttm_page_alloc.c \ + ttm_bo_vm.c +#ttm_agp_backend.c +#ttm_page_alloc_dma.c .if ${MACHINE_CPUARCH} == "amd64" SRCS += drm_ioc32.c diff --git a/sys/modules/uart/Makefile b/sys/modules/uart/Makefile index 61bcdec..d8a4869 100644 --- a/sys/modules/uart/Makefile +++ b/sys/modules/uart/Makefile @@ -26,6 +26,7 @@ SRCS= uart_bus_acpi.c ${uart_bus_ebus} uart_bus_isa.c uart_bus_pccard.c \ SRCS+= bus_if.h card_if.h device_if.h isa_if.h ${ofw_bus_if} pci_if.h \ power_if.h pccarddevs.h serdev_if.h +SRCS+= opt_platform.h MFILES= dev/pccard/card_if.m dev/pccard/power_if.m dev/pci/pci_if.m \ dev/ofw/ofw_bus_if.m dev/uart/uart_if.m isa/isa_if.m kern/bus_if.m \ diff --git a/sys/net/route.c b/sys/net/route.c index e69ce48..c0f6526 100644 --- a/sys/net/route.c +++ b/sys/net/route.c @@ -1498,7 +1498,7 @@ rtinit1(struct ifaddr *ifa, int cmd, int flags, int fibnum) if (rnh == NULL) /* this table doesn't exist but others might */ continue; - RADIX_NODE_HEAD_LOCK(rnh); + RADIX_NODE_HEAD_RLOCK(rnh); #ifdef RADIX_MPATH if (rn_mpath_capable(rnh)) { @@ -1527,7 +1527,7 @@ rtinit1(struct ifaddr *ifa, int cmd, int flags, int fibnum) (rn->rn_flags & RNF_ROOT) || RNTORT(rn)->rt_ifa != ifa || !sa_equal((struct sockaddr *)rn->rn_key, dst)); - RADIX_NODE_HEAD_UNLOCK(rnh); + RADIX_NODE_HEAD_RUNLOCK(rnh); if (error) { /* this is only an error if bad on ALL tables */ continue; diff --git a/sys/netinet/siftr.c b/sys/netinet/siftr.c index b0e111c..886be06 100644 --- a/sys/netinet/siftr.c +++ b/sys/netinet/siftr.c @@ -1314,7 +1314,7 @@ siftr_manage_ops(uint8_t action) * flow seen and freeing any malloc'd memory. * The hash consists of an array of LISTs (man 3 queue). */ - for (i = 0; i < siftr_hashmask; i++) { + for (i = 0; i <= siftr_hashmask; i++) { LIST_FOREACH_SAFE(counter, counter_hash + i, nodes, tmp_counter) { key = counter->key; diff --git a/sys/sparc64/pci/psycho.c b/sys/sparc64/pci/psycho.c index a6e1469..8717426 100644 --- a/sys/sparc64/pci/psycho.c +++ b/sys/sparc64/pci/psycho.c @@ -200,7 +200,7 @@ struct psycho_icarg { * "Sabre" is the UltraSPARC IIi onboard UPA to PCI bridge. It manages a * single PCI bus and does not have a streaming buffer. It often has an APB * (advanced PCI bridge) connected to it, which was designed specifically for - * the IIi. The APB let's the IIi handle two independednt PCI buses, and + * the IIi. The APB lets the IIi handle two independent PCI buses, and * appears as two "Simba"'s underneath the Sabre. * * "Hummingbird" is the UltraSPARC IIe onboard UPA to PCI bridge. It's diff --git a/sys/sys/diskmbr.h b/sys/sys/diskmbr.h index 747f70b..c92364d 100644 --- a/sys/sys/diskmbr.h +++ b/sys/sys/diskmbr.h @@ -45,6 +45,7 @@ #define DOSMAGIC 0xAA55 #define DOSPTYP_EXT 0x05 /* DOS extended partition */ +#define DOSPTYP_FAT16 0x06 /* FAT16 partition */ #define DOSPTYP_NTFS 0x07 /* NTFS partition */ #define DOSPTYP_FAT32 0x0b /* FAT32 partition */ #define DOSPTYP_EXTLBA 0x0f /* DOS extended partition */ diff --git a/sys/sys/param.h b/sys/sys/param.h index 26db635..ce72863 100644 --- a/sys/sys/param.h +++ b/sys/sys/param.h @@ -58,7 +58,7 @@ * in the range 5 to 9. */ #undef __FreeBSD_version -#define __FreeBSD_version 1000028 /* Master, propagated to newvers */ +#define __FreeBSD_version 1000029 /* Master, propagated to newvers */ /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD, diff --git a/sys/sys/pmckern.h b/sys/sys/pmckern.h index c623648..e3e18a6 100644 --- a/sys/sys/pmckern.h +++ b/sys/sys/pmckern.h @@ -87,9 +87,9 @@ struct pmckern_soft { * Soft PMC. */ -#define PMC_SOFT_DEFINE(prov, mod, func, name) \ +#define PMC_SOFT_DEFINE_EX(prov, mod, func, name, alloc, release) \ struct pmc_soft pmc_##prov##_##mod##_##func##_##name = \ - { 0, { #prov "_" #mod "_" #func "." #name, 0 } }; \ + { 0, alloc, release, { #prov "_" #mod "_" #func "." #name, 0 } }; \ SYSINIT(pmc_##prov##_##mod##_##func##_##name##_init, SI_SUB_KDTRACE, \ SI_ORDER_SECOND + 1, pmc_soft_ev_register, \ &pmc_##prov##_##mod##_##func##_##name ); \ @@ -97,6 +97,9 @@ struct pmckern_soft { SI_SUB_KDTRACE, SI_ORDER_SECOND + 1, pmc_soft_ev_deregister, \ &pmc_##prov##_##mod##_##func##_##name ) +#define PMC_SOFT_DEFINE(prov, mod, func, name) \ + PMC_SOFT_DEFINE_EX(prov, mod, func, name, NULL, NULL) + #define PMC_SOFT_DECLARE(prov, mod, func, name) \ extern struct pmc_soft pmc_##prov##_##mod##_##func##_##name @@ -147,6 +150,8 @@ do { \ struct pmc_soft { int ps_running; + void (*ps_alloc)(void); + void (*ps_release)(void); struct pmc_dyn_event_descr ps_ev; }; |