From 48088f84e771353c5f90424b69ea409c5d5d8f1d Mon Sep 17 00:00:00 2001 From: adrian Date: Thu, 14 Mar 2013 06:20:02 +0000 Subject: Implement "holding buffers" per TX queue rather than globally. When working on TDMA, Sam Leffler found that the MAC DMA hardware would re-read the last TX descriptor when getting ready to transmit the next one. Thus the whole ATH_BUF_BUSY came into existance - the descriptor must be left alone (very specifically the link pointer must be maintained) until the hardware has moved onto the next frame. He saw this in TDMA because the MAC would be frequently stopping during active transmit (ie, when it wasn't its turn to transmit.) Fast-forward to today. It turns out that this is a problem not with a single MAC DMA instance, but with each QCU (from 0->9). They each maintain separate descriptor pointers and will re-read the last descriptor when starting to transmit the next. So when your AP is busy transmitting from multiple TX queues, you'll (more) frequently see one QCU stopped, waiting for a higher-priority QCU to finsh transmitting, before it'll go ahead and continue. If you mess up the descriptor (ie by freeing it) then you're short of luck. Thanks to rpaulo for sticking with me whilst I diagnosed this issue that he was quite reliably triggering in his environment. This is a reimplementation; it doesn't have anything in common with the ath9k or the Qualcomm Atheros reference driver. Now - it in theory doesn't apply on the EDMA chips, as long as you push one complete frame into the FIFO at a time. But the MAC can DMA from a list of frames pushed into the hardware queue (ie, you concat 'n' frames together with link pointers, and then push the head pointer into the TXQ FIFO.) Since that's likely how I'm going to implement CABQ handling in hostap mode, it's likely that I will end up teaching the EDMA TX completion code about busy buffers, just to be "sure" this doesn't creep up. Tested - iperf ap->sta and sta->ap (with both sides running this code): * AR5416 STA * AR9160/AR9220 hostap To validate that it doesn't break the EDMA (FIFO) chips: * AR9380, AR9485, AR9462 STA Using iperf with the -S to set the TCP client side DSCP bits, mapping to different TIDs and thus different TX queues. TODO: * Make this work on the EDMA chips, if we end up pushing lists of frames to the hardware (eg how we eventually will handle cabq in hostap/ibss mode.) --- sys/dev/ath/if_ath.c | 108 +++++++++++++++++++++++++------------------- sys/dev/ath/if_ath_sysctl.c | 6 ++- sys/dev/ath/if_athvar.h | 1 + 3 files changed, 66 insertions(+), 49 deletions(-) (limited to 'sys') diff --git a/sys/dev/ath/if_ath.c b/sys/dev/ath/if_ath.c index 11ecd1a..cf69990 100644 --- a/sys/dev/ath/if_ath.c +++ b/sys/dev/ath/if_ath.c @@ -3750,39 +3750,6 @@ ath_tx_update_ratectrl(struct ath_softc *sc, struct ieee80211_node *ni, } /* - * Update the busy status of the last frame on the free list. - * When doing TDMA, the busy flag tracks whether the hardware - * currently points to this buffer or not, and thus gated DMA - * may restart by re-reading the last descriptor in this - * buffer. - * - * This should be called in the completion function once one - * of the buffers has been used. - */ -static void -ath_tx_update_busy(struct ath_softc *sc) -{ - struct ath_buf *last; - - /* - * Since the last frame may still be marked - * as ATH_BUF_BUSY, unmark it here before - * finishing the frame processing. - * Since we've completed a frame (aggregate - * or otherwise), the hardware has moved on - * and is no longer referencing the previous - * descriptor. - */ - ATH_TXBUF_LOCK_ASSERT(sc); - last = TAILQ_LAST(&sc->sc_txbuf_mgmt, ath_bufhead_s); - if (last != NULL) - last->bf_flags &= ~ATH_BUF_BUSY; - last = TAILQ_LAST(&sc->sc_txbuf, ath_bufhead_s); - if (last != NULL) - last->bf_flags &= ~ATH_BUF_BUSY; -} - -/* * Process the completion of the given buffer. * * This calls the rate control update and then the buffer completion. @@ -3901,7 +3868,6 @@ ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq, int dosched) break; } ATH_TXQ_REMOVE(txq, bf, bf_list); -#ifdef IEEE80211_SUPPORT_TDMA if (txq->axq_depth > 0) { /* * More frames follow. Mark the buffer busy @@ -3914,9 +3880,6 @@ ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq, int dosched) */ bf->bf_last->bf_flags |= ATH_BUF_BUSY; } else -#else - if (txq->axq_depth == 0) -#endif txq->axq_link = NULL; if (bf->bf_state.bfs_aggr) txq->axq_aggr_depth--; @@ -4188,6 +4151,51 @@ ath_returnbuf_head(struct ath_softc *sc, struct ath_buf *bf) } /* + * Free the holding buffer if it exists + */ +static void +ath_txq_freeholdingbuf(struct ath_softc *sc, struct ath_txq *txq) +{ + + if (txq->axq_holdingbf == NULL) + return; + + txq->axq_holdingbf->bf_flags &= ~ATH_BUF_BUSY; + ATH_TXBUF_LOCK(sc); + ath_returnbuf_tail(sc, txq->axq_holdingbf); + ATH_TXBUF_UNLOCK(sc); + txq->axq_holdingbf = NULL; +} + +/* + * Add this buffer to the holding queue, freeing the previous + * one if it exists. + */ +static void +ath_txq_addholdingbuf(struct ath_softc *sc, struct ath_buf *bf) +{ + struct ath_txq *txq; + + /* XXX assert ATH_BUF_BUSY is set */ + + /* XXX assert the tx queue is under the max number */ + if (bf->bf_state.bfs_tx_queue > HAL_NUM_TX_QUEUES) { + device_printf(sc->sc_dev, "%s: bf=%p: invalid tx queue (%d)\n", + __func__, + bf, + bf->bf_state.bfs_tx_queue); + bf->bf_flags &= ~ATH_BUF_BUSY; + ath_returnbuf_tail(sc, bf); + return; + } + + txq = &sc->sc_txq[bf->bf_state.bfs_tx_queue]; + ath_txq_freeholdingbuf(sc, txq); + + txq->axq_holdingbf = bf; +} + +/* * Return a buffer to the pool and update the 'busy' flag on the * previous 'tail' entry. * @@ -4207,8 +4215,18 @@ ath_freebuf(struct ath_softc *sc, struct ath_buf *bf) KASSERT((bf->bf_node == NULL), ("%s: bf->bf_node != NULL\n", __func__)); KASSERT((bf->bf_m == NULL), ("%s: bf->bf_m != NULL\n", __func__)); + /* + * If this buffer is busy, push it onto the holding queue + */ + if (bf->bf_flags & ATH_BUF_BUSY) { + ath_txq_addholdingbuf(sc, bf); + return; + } + + /* + * Not a busy buffer, so free normally + */ ATH_TXBUF_LOCK(sc); - ath_tx_update_busy(sc); ath_returnbuf_tail(sc, bf); ATH_TXBUF_UNLOCK(sc); } @@ -4261,15 +4279,6 @@ ath_tx_draintxq(struct ath_softc *sc, struct ath_txq *txq) * NB: this assumes output has been stopped and * we do not need to block ath_tx_proc */ - ATH_TXBUF_LOCK(sc); - bf = TAILQ_LAST(&sc->sc_txbuf, ath_bufhead_s); - if (bf != NULL) - bf->bf_flags &= ~ATH_BUF_BUSY; - bf = TAILQ_LAST(&sc->sc_txbuf_mgmt, ath_bufhead_s); - if (bf != NULL) - bf->bf_flags &= ~ATH_BUF_BUSY; - ATH_TXBUF_UNLOCK(sc); - for (ix = 0;; ix++) { ATH_TX_LOCK(sc); bf = TAILQ_FIRST(&txq->axq_q); @@ -4331,6 +4340,11 @@ ath_tx_draintxq(struct ath_softc *sc, struct ath_txq *txq) } /* + * Free the holding buffer if it exists + */ + ath_txq_freeholdingbuf(sc, txq); + + /* * Drain software queued frames which are on * active TIDs. */ diff --git a/sys/dev/ath/if_ath_sysctl.c b/sys/dev/ath/if_ath_sysctl.c index bbd1fe8..f1c794e 100644 --- a/sys/dev/ath/if_ath_sysctl.c +++ b/sys/dev/ath/if_ath_sysctl.c @@ -361,11 +361,13 @@ ath_sysctl_txagg(SYSCTL_HANDLER_ARGS) for (i = 0; i < HAL_NUM_TX_QUEUES; i++) { if (ATH_TXQ_SETUP(sc, i)) { - printf("HW TXQ %d: axq_depth=%d, axq_aggr_depth=%d, axq_fifo_depth=%d\n", + printf("HW TXQ %d: axq_depth=%d, axq_aggr_depth=%d, " + "axq_fifo_depth=%d, holdingbf=%p\n", i, sc->sc_txq[i].axq_depth, sc->sc_txq[i].axq_aggr_depth, - sc->sc_txq[i].axq_fifo_depth); + sc->sc_txq[i].axq_fifo_depth, + sc->sc_txq[i].axq_holdingbf); } } diff --git a/sys/dev/ath/if_athvar.h b/sys/dev/ath/if_athvar.h index 06cffb6..1ac6fe7 100644 --- a/sys/dev/ath/if_athvar.h +++ b/sys/dev/ath/if_athvar.h @@ -329,6 +329,7 @@ struct ath_txq { u_int axq_intrcnt; /* interrupt count */ u_int32_t *axq_link; /* link ptr in last TX desc */ TAILQ_HEAD(axq_q_s, ath_buf) axq_q; /* transmit queue */ + struct ath_buf *axq_holdingbf; /* holding TX buffer */ char axq_name[12]; /* e.g. "ath0_txq4" */ /* Per-TID traffic queue for software -> hardware TX */ -- cgit v1.1 From 20173378f6418ff9af971e15450fcc901c720fbe Mon Sep 17 00:00:00 2001 From: joel Date: Thu, 14 Mar 2013 19:36:20 +0000 Subject: Fix minor spelling error in a comment. --- sys/conf/NOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sys') diff --git a/sys/conf/NOTES b/sys/conf/NOTES index 27c3380..5a5000c 100644 --- a/sys/conf/NOTES +++ b/sys/conf/NOTES @@ -986,7 +986,7 @@ options DUMMYNET # See zero_copy(9) for more details. # XXX: The COW based send mechanism is not safe and may result in # kernel crashes. -# XXX: None of the current NIC drivers support disposeable pages. +# XXX: None of the current NIC drivers support disposable pages. options SOCKET_SEND_COW options SOCKET_RECV_PFLIP -- cgit v1.1 From 376c14488bc4e1c0b35d09dce966d5fcfae8a4c1 Mon Sep 17 00:00:00 2001 From: kib Date: Thu, 14 Mar 2013 19:48:25 +0000 Subject: Rewrite the vfs_bio_clrbuf(9) to not access the b_data for B_VMIO buffers directly, use pmap_zero_page_area(9) for each zeroing page region instead. Sponsored by: The FreeBSD Foundation Tested by: pho MFC after: 2 weeks --- sys/kern/vfs_bio.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'sys') diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 6d110ab..324efe5 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -3712,8 +3712,7 @@ vfs_bio_set_valid(struct buf *bp, int base, int size) void vfs_bio_clrbuf(struct buf *bp) { - int i, j, mask; - caddr_t sa, ea; + int i, j, mask, sa, ea, slide; if ((bp->b_flags & (B_VMIO | B_MALLOC)) != B_VMIO) { clrbuf(bp); @@ -3731,30 +3730,33 @@ vfs_bio_clrbuf(struct buf *bp) if ((bp->b_pages[0]->valid & mask) == mask) goto unlock; if ((bp->b_pages[0]->valid & mask) == 0) { - bzero(bp->b_data, bp->b_bufsize); + pmap_zero_page_area(bp->b_pages[0], 0, bp->b_bufsize); bp->b_pages[0]->valid |= mask; goto unlock; } } - ea = sa = bp->b_data; - for(i = 0; i < bp->b_npages; i++, sa = ea) { - ea = (caddr_t)trunc_page((vm_offset_t)sa + PAGE_SIZE); - ea = (caddr_t)(vm_offset_t)ulmin( - (u_long)(vm_offset_t)ea, - (u_long)(vm_offset_t)bp->b_data + bp->b_bufsize); + sa = bp->b_offset & PAGE_MASK; + slide = 0; + for (i = 0; i < bp->b_npages; i++, sa = 0) { + slide = imin(slide + PAGE_SIZE, bp->b_offset + bp->b_bufsize); + ea = slide & PAGE_MASK; + if (ea == 0) + ea = PAGE_SIZE; if (bp->b_pages[i] == bogus_page) continue; - j = ((vm_offset_t)sa & PAGE_MASK) / DEV_BSIZE; + j = sa / DEV_BSIZE; mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j; VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[i]->object); if ((bp->b_pages[i]->valid & mask) == mask) continue; if ((bp->b_pages[i]->valid & mask) == 0) - bzero(sa, ea - sa); + pmap_zero_page_area(bp->b_pages[i], sa, ea - sa); else { for (; sa < ea; sa += DEV_BSIZE, j++) { - if ((bp->b_pages[i]->valid & (1 << j)) == 0) - bzero(sa, DEV_BSIZE); + if ((bp->b_pages[i]->valid & (1 << j)) == 0) { + pmap_zero_page_area(bp->b_pages[i], + sa, DEV_BSIZE); + } } } bp->b_pages[i]->valid |= mask; -- cgit v1.1 From 51407f194b2206d9518a5190c79fd836d556d263 Mon Sep 17 00:00:00 2001 From: kib Date: Thu, 14 Mar 2013 19:50:09 +0000 Subject: Remove excessive and inconsistent initializers for the various kernel maps and submaps. MFC after: 2 weeks --- sys/vm/vm_kern.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'sys') diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c index 9f602b7..64a2ebb 100644 --- a/sys/vm/vm_kern.c +++ b/sys/vm/vm_kern.c @@ -85,11 +85,11 @@ __FBSDID("$FreeBSD$"); #include #include -vm_map_t kernel_map=0; -vm_map_t kmem_map=0; -vm_map_t exec_map=0; +vm_map_t kernel_map; +vm_map_t kmem_map; +vm_map_t exec_map; vm_map_t pipe_map; -vm_map_t buffer_map=0; +vm_map_t buffer_map; const void *zero_region; CTASSERT((ZERO_REGION_SIZE & PAGE_MASK) == 0); -- cgit v1.1 From 63efc821c3e4785928997ea88e1de93e62ce3acb Mon Sep 17 00:00:00 2001 From: kib Date: Thu, 14 Mar 2013 20:18:12 +0000 Subject: Add pmap function pmap_copy_pages(), which copies the content of the pages around, taking array of vm_page_t both for source and destination. Starting offsets and total transfer size are specified. The function implements optimal algorithm for copying using the platform-specific optimizations. For instance, on the architectures were the direct map is available, no transient mappings are created, for i386 the per-cpu ephemeral page frame is used. The code was typically borrowed from the pmap_copy_page() for the same architecture. Only i386/amd64, powerpc aim and arm/arm-v6 implementations were tested at the time of commit. High-level code, not committed yet to the tree, ensures that the use of the function is only allowed after explicit enablement. For sparc64, the existing code has known issues and a stab is added instead, to allow the kernel linking. Sponsored by: The FreeBSD Foundation Tested by: pho (i386, amd64), scottl (amd64), ian (arm and arm-v6) MFC after: 2 weeks --- sys/amd64/amd64/pmap.c | 24 ++++++++++ sys/arm/arm/pmap-v6.c | 39 ++++++++++++++++ sys/arm/arm/pmap.c | 93 ++++++++++++++++++++++++++++++++++++- sys/arm/include/pmap.h | 2 + sys/i386/i386/pmap.c | 43 +++++++++++++++++ sys/i386/xen/pmap.c | 40 ++++++++++++++++ sys/ia64/ia64/pmap.c | 24 ++++++++++ sys/mips/mips/pmap.c | 45 ++++++++++++++++++ sys/powerpc/aim/mmu_oea.c | 27 +++++++++++ sys/powerpc/aim/mmu_oea64.c | 69 +++++++++++++++++++++++++++ sys/powerpc/booke/pmap.c | 33 +++++++++++++ sys/powerpc/powerpc/mmu_if.m | 8 ++++ sys/powerpc/powerpc/pmap_dispatch.c | 10 ++++ sys/sparc64/sparc64/pmap.c | 8 ++++ sys/vm/pmap.h | 2 + 15 files changed, 466 insertions(+), 1 deletion(-) (limited to 'sys') diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index aa38013..4777c08 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -4272,6 +4272,30 @@ pmap_copy_page(vm_page_t msrc, vm_page_t mdst) pagecopy((void *)src, (void *)dst); } +void +pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], + vm_offset_t b_offset, int xfersize) +{ + void *a_cp, *b_cp; + vm_offset_t a_pg_offset, b_pg_offset; + int cnt; + + while (xfersize > 0) { + a_pg_offset = a_offset & PAGE_MASK; + cnt = min(xfersize, PAGE_SIZE - a_pg_offset); + a_cp = (char *)PHYS_TO_DMAP(ma[a_offset >> PAGE_SHIFT]-> + phys_addr) + a_pg_offset; + b_pg_offset = b_offset & PAGE_MASK; + cnt = min(cnt, PAGE_SIZE - b_pg_offset); + b_cp = (char *)PHYS_TO_DMAP(mb[b_offset >> PAGE_SHIFT]-> + phys_addr) + b_pg_offset; + bcopy(a_cp, b_cp, cnt); + a_offset += cnt; + b_offset += cnt; + xfersize -= cnt; + } +} + /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may diff --git a/sys/arm/arm/pmap-v6.c b/sys/arm/arm/pmap-v6.c index 2affa3e..0083f29 100644 --- a/sys/arm/arm/pmap-v6.c +++ b/sys/arm/arm/pmap-v6.c @@ -3313,6 +3313,45 @@ pmap_copy_page_generic(vm_paddr_t src, vm_paddr_t dst) } void +pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], + vm_offset_t b_offset, int xfersize) +{ + vm_page_t a_pg, b_pg; + vm_offset_t a_pg_offset, b_pg_offset; + int cnt; + + mtx_lock(&cmtx); + while (xfersize > 0) { + a_pg = ma[a_offset >> PAGE_SHIFT]; + a_pg_offset = a_offset & PAGE_MASK; + cnt = min(xfersize, PAGE_SIZE - a_pg_offset); + b_pg = mb[b_offset >> PAGE_SHIFT]; + b_pg_offset = b_offset & PAGE_MASK; + cnt = min(cnt, PAGE_SIZE - b_pg_offset); + *csrc_pte = L2_S_PROTO | VM_PAGE_TO_PHYS(a_pg) | + pte_l2_s_cache_mode; + pmap_set_prot(csrc_pte, VM_PROT_READ, 0); + PTE_SYNC(csrc_pte); + *cdst_pte = L2_S_PROTO | VM_PAGE_TO_PHYS(b_pg) | + pte_l2_s_cache_mode; + pmap_set_prot(cdst_pte, VM_PROT_READ | VM_PROT_WRITE, 0); + PTE_SYNC(cdst_pte); + cpu_tlb_flushD_SE(csrcp); + cpu_tlb_flushD_SE(cdstp); + cpu_cpwait(); + bcopy((char *)csrcp + a_pg_offset, (char *)cdstp + b_pg_offset, + cnt); + cpu_idcache_wbinv_range(cdstp + b_pg_offset, cnt); + pmap_l2cache_wbinv_range(cdstp + b_pg_offset, + VM_PAGE_TO_PHYS(b_pg) + b_pg_offset, cnt); + xfersize -= cnt; + a_offset += cnt; + b_offset += cnt; + } + mtx_unlock(&cmtx); +} + +void pmap_copy_page(vm_page_t src, vm_page_t dst) { diff --git a/sys/arm/arm/pmap.c b/sys/arm/arm/pmap.c index 7070cb2..c18783b 100644 --- a/sys/arm/arm/pmap.c +++ b/sys/arm/arm/pmap.c @@ -258,6 +258,9 @@ pt_entry_t pte_l1_c_proto; pt_entry_t pte_l2_s_proto; void (*pmap_copy_page_func)(vm_paddr_t, vm_paddr_t); +void (*pmap_copy_page_offs_func)(vm_paddr_t a_phys, + vm_offset_t a_offs, vm_paddr_t b_phys, vm_offset_t b_offs, + int cnt); void (*pmap_zero_page_func)(vm_paddr_t, int, int); struct msgbuf *msgbufp = 0; @@ -400,6 +403,13 @@ static vm_paddr_t pmap_kernel_l2ptp_phys; static int pv_entry_count=0, pv_entry_max=0, pv_entry_high_water=0; static struct rwlock pvh_global_lock; +void pmap_copy_page_offs_generic(vm_paddr_t a_phys, vm_offset_t a_offs, + vm_paddr_t b_phys, vm_offset_t b_offs, int cnt); +#if ARM_MMU_XSCALE == 1 +void pmap_copy_page_offs_xscale(vm_paddr_t a_phys, vm_offset_t a_offs, + vm_paddr_t b_phys, vm_offset_t b_offs, int cnt); +#endif + /* * This list exists for the benefit of pmap_map_chunk(). It keeps track * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can @@ -484,6 +494,7 @@ pmap_pte_init_generic(void) pte_l2_s_proto = L2_S_PROTO_generic; pmap_copy_page_func = pmap_copy_page_generic; + pmap_copy_page_offs_func = pmap_copy_page_offs_generic; pmap_zero_page_func = pmap_zero_page_generic; } @@ -660,6 +671,7 @@ pmap_pte_init_xscale(void) #ifdef CPU_XSCALE_CORE3 pmap_copy_page_func = pmap_copy_page_generic; + pmap_copy_page_offs_func = pmap_copy_page_offs_generic; pmap_zero_page_func = pmap_zero_page_generic; xscale_use_minidata = 0; /* Make sure it is L2-cachable */ @@ -672,6 +684,7 @@ pmap_pte_init_xscale(void) #else pmap_copy_page_func = pmap_copy_page_xscale; + pmap_copy_page_offs_func = pmap_copy_page_offs_xscale; pmap_zero_page_func = pmap_zero_page_xscale; #endif @@ -4300,6 +4313,29 @@ pmap_copy_page_generic(vm_paddr_t src, vm_paddr_t dst) cpu_l2cache_inv_range(csrcp, PAGE_SIZE); cpu_l2cache_wbinv_range(cdstp, PAGE_SIZE); } + +void +pmap_copy_page_offs_generic(vm_paddr_t a_phys, vm_offset_t a_offs, + vm_paddr_t b_phys, vm_offset_t b_offs, int cnt) +{ + + mtx_lock(&cmtx); + *csrc_pte = L2_S_PROTO | a_phys | + L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; + PTE_SYNC(csrc_pte); + *cdst_pte = L2_S_PROTO | b_phys | + L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; + PTE_SYNC(cdst_pte); + cpu_tlb_flushD_SE(csrcp); + cpu_tlb_flushD_SE(cdstp); + cpu_cpwait(); + bcopy((char *)csrcp + a_offs, (char *)cdstp + b_offs, cnt); + mtx_unlock(&cmtx); + cpu_dcache_inv_range(csrcp + a_offs, cnt); + cpu_dcache_wbinv_range(cdstp + b_offs, cnt); + cpu_l2cache_inv_range(csrcp + a_offs, cnt); + cpu_l2cache_wbinv_range(cdstp + b_offs, cnt); +} #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ #if ARM_MMU_XSCALE == 1 @@ -4344,6 +4380,28 @@ pmap_copy_page_xscale(vm_paddr_t src, vm_paddr_t dst) mtx_unlock(&cmtx); xscale_cache_clean_minidata(); } + +void +pmap_copy_page_offs_xscale(vm_paddr_t a_phys, vm_offset_t a_offs, + vm_paddr_t b_phys, vm_offset_t b_offs, int cnt) +{ + + mtx_lock(&cmtx); + *csrc_pte = L2_S_PROTO | a_phys | + L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | + L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); + PTE_SYNC(csrc_pte); + *cdst_pte = L2_S_PROTO | b_phys | + L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | + L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); + PTE_SYNC(cdst_pte); + cpu_tlb_flushD_SE(csrcp); + cpu_tlb_flushD_SE(cdstp); + cpu_cpwait(); + bcopy((char *)csrcp + a_offs, (char *)cdstp + b_offs, cnt); + mtx_unlock(&cmtx); + xscale_cache_clean_minidata(); +} #endif /* ARM_MMU_XSCALE == 1 */ void @@ -4370,8 +4428,41 @@ pmap_copy_page(vm_page_t src, vm_page_t dst) #endif } +void +pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], + vm_offset_t b_offset, int xfersize) +{ + vm_page_t a_pg, b_pg; + vm_offset_t a_pg_offset, b_pg_offset; + int cnt; +#ifdef ARM_USE_SMALL_ALLOC + vm_offset_t a_va, b_va; +#endif - + cpu_dcache_wbinv_all(); + cpu_l2cache_wbinv_all(); + while (xfersize > 0) { + a_pg = ma[a_offset >> PAGE_SHIFT]; + a_pg_offset = a_offset & PAGE_MASK; + cnt = min(xfersize, PAGE_SIZE - a_pg_offset); + b_pg = mb[b_offset >> PAGE_SHIFT]; + b_pg_offset = b_offset & PAGE_MASK; + cnt = min(cnt, PAGE_SIZE - b_pg_offset); +#ifdef ARM_USE_SMALL_ALLOC + a_va = arm_ptovirt(VM_PAGE_TO_PHYS(a_pg)) + a_pg_offset; + b_va = arm_ptovirt(VM_PAGE_TO_PHYS(b_pg)) + b_pg_offset; + bcopy((char *)a_va, (char *)b_va, cnt); + cpu_dcache_wbinv_range(b_va, cnt); + cpu_l2cache_wbinv_range(b_va, cnt); +#else + pmap_copy_page_offs_func(VM_PAGE_TO_PHYS(a_pg), a_pg_offset, + VM_PAGE_TO_PHYS(b_pg), b_pg_offset, cnt); +#endif + xfersize -= cnt; + a_offset += cnt; + b_offset += cnt; + } +} /* * this routine returns true if a physical page resides diff --git a/sys/arm/include/pmap.h b/sys/arm/include/pmap.h index 523499f..7c8d073 100644 --- a/sys/arm/include/pmap.h +++ b/sys/arm/include/pmap.h @@ -533,6 +533,8 @@ extern pt_entry_t pte_l1_c_proto; extern pt_entry_t pte_l2_s_proto; extern void (*pmap_copy_page_func)(vm_paddr_t, vm_paddr_t); +extern void (*pmap_copy_page_offs_func)(vm_paddr_t a_phys, + vm_offset_t a_offs, vm_paddr_t b_phys, vm_offset_t b_offs, int cnt); extern void (*pmap_zero_page_func)(vm_paddr_t, int, int); #if (ARM_MMU_GENERIC + ARM_MMU_V6 + ARM_MMU_V7 + ARM_MMU_SA1) != 0 || defined(CPU_XSCALE_81342) diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index a62e180..4e66dbd 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -4238,6 +4238,49 @@ pmap_copy_page(vm_page_t src, vm_page_t dst) mtx_unlock(&sysmaps->lock); } +void +pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], + vm_offset_t b_offset, int xfersize) +{ + struct sysmaps *sysmaps; + vm_page_t a_pg, b_pg; + char *a_cp, *b_cp; + vm_offset_t a_pg_offset, b_pg_offset; + int cnt; + + sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; + mtx_lock(&sysmaps->lock); + if (*sysmaps->CMAP1 != 0) + panic("pmap_copy_pages: CMAP1 busy"); + if (*sysmaps->CMAP2 != 0) + panic("pmap_copy_pages: CMAP2 busy"); + sched_pin(); + while (xfersize > 0) { + invlpg((u_int)sysmaps->CADDR1); + invlpg((u_int)sysmaps->CADDR2); + a_pg = ma[a_offset >> PAGE_SHIFT]; + a_pg_offset = a_offset & PAGE_MASK; + cnt = min(xfersize, PAGE_SIZE - a_pg_offset); + b_pg = mb[b_offset >> PAGE_SHIFT]; + b_pg_offset = b_offset & PAGE_MASK; + cnt = min(cnt, PAGE_SIZE - b_pg_offset); + *sysmaps->CMAP1 = PG_V | VM_PAGE_TO_PHYS(a_pg) | PG_A | + pmap_cache_bits(b_pg->md.pat_mode, 0); + *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(b_pg) | PG_A | + PG_M | pmap_cache_bits(b_pg->md.pat_mode, 0); + a_cp = sysmaps->CADDR1 + a_pg_offset; + b_cp = sysmaps->CADDR2 + b_pg_offset; + bcopy(a_cp, b_cp, cnt); + a_offset += cnt; + b_offset += cnt; + xfersize -= cnt; + } + *sysmaps->CMAP1 = 0; + *sysmaps->CMAP2 = 0; + sched_unpin(); + mtx_unlock(&sysmaps->lock); +} + /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may diff --git a/sys/i386/xen/pmap.c b/sys/i386/xen/pmap.c index 65e2e76..e8115b0 100644 --- a/sys/i386/xen/pmap.c +++ b/sys/i386/xen/pmap.c @@ -3447,6 +3447,46 @@ pmap_copy_page(vm_page_t src, vm_page_t dst) mtx_unlock(&sysmaps->lock); } +void +pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], + vm_offset_t b_offset, int xfersize) +{ + struct sysmaps *sysmaps; + vm_page_t a_pg, b_pg; + char *a_cp, *b_cp; + vm_offset_t a_pg_offset, b_pg_offset; + int cnt; + + sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; + mtx_lock(&sysmaps->lock); + if (*sysmaps->CMAP1 != 0) + panic("pmap_copy_pages: CMAP1 busy"); + if (*sysmaps->CMAP2 != 0) + panic("pmap_copy_pages: CMAP2 busy"); + sched_pin(); + while (xfersize > 0) { + a_pg = ma[a_offset >> PAGE_SHIFT]; + a_pg_offset = a_offset & PAGE_MASK; + cnt = min(xfersize, PAGE_SIZE - a_pg_offset); + b_pg = mb[b_offset >> PAGE_SHIFT]; + b_pg_offset = b_offset & PAGE_MASK; + cnt = min(cnt, PAGE_SIZE - b_pg_offset); + PT_SET_MA(sysmaps->CADDR1, PG_V | VM_PAGE_TO_MACH(a_pg) | PG_A); + PT_SET_MA(sysmaps->CADDR2, PG_V | PG_RW | + VM_PAGE_TO_MACH(b_pg) | PG_A | PG_M); + a_cp = sysmaps->CADDR1 + a_pg_offset; + b_cp = sysmaps->CADDR2 + b_pg_offset; + bcopy(a_cp, b_cp, cnt); + a_offset += cnt; + b_offset += cnt; + xfersize -= cnt; + } + PT_SET_MA(sysmaps->CADDR1, 0); + PT_SET_MA(sysmaps->CADDR2, 0); + sched_unpin(); + mtx_unlock(&sysmaps->lock); +} + /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index 1dff1f9..3256600 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -2014,6 +2014,30 @@ pmap_copy_page(vm_page_t msrc, vm_page_t mdst) bcopy(src, dst, PAGE_SIZE); } +void +pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], + vm_offset_t b_offset, int xfersize) +{ + void *a_cp, *b_cp; + vm_offset_t a_pg_offset, b_pg_offset; + int cnt; + + while (xfersize > 0) { + a_pg_offset = a_offset & PAGE_MASK; + cnt = min(xfersize, PAGE_SIZE - a_pg_offset); + a_cp = (char *)pmap_page_to_va(ma[a_offset >> PAGE_SHIFT]) + + a_pg_offset; + b_pg_offset = b_offset & PAGE_MASK; + cnt = min(cnt, PAGE_SIZE - b_pg_offset); + b_cp = (char *)pmap_page_to_va(mb[b_offset >> PAGE_SHIFT]) + + b_pg_offset; + bcopy(a_cp, b_cp, cnt); + a_offset += cnt; + b_offset += cnt; + xfersize -= cnt; + } +} + /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may diff --git a/sys/mips/mips/pmap.c b/sys/mips/mips/pmap.c index 9e1b812..4fe6ebe 100644 --- a/sys/mips/mips/pmap.c +++ b/sys/mips/mips/pmap.c @@ -2576,6 +2576,51 @@ pmap_copy_page(vm_page_t src, vm_page_t dst) } } +void +pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], + vm_offset_t b_offset, int xfersize) +{ + char *a_cp, *b_cp; + vm_page_t a_m, b_m; + vm_offset_t a_pg_offset, b_pg_offset; + vm_paddr_t a_phys, b_phys; + int cnt; + + while (xfersize > 0) { + a_pg_offset = a_offset & PAGE_MASK; + cnt = min(xfersize, PAGE_SIZE - a_pg_offset); + a_m = ma[a_offset >> PAGE_SHIFT]; + a_phys = VM_PAGE_TO_PHYS(a_m); + b_pg_offset = b_offset & PAGE_MASK; + cnt = min(cnt, PAGE_SIZE - b_pg_offset); + b_m = mb[b_offset >> PAGE_SHIFT]; + b_phys = VM_PAGE_TO_PHYS(b_m); + if (MIPS_DIRECT_MAPPABLE(a_phys) && + MIPS_DIRECT_MAPPABLE(b_phys)) { + pmap_flush_pvcache(a_m); + mips_dcache_wbinv_range_index( + MIPS_PHYS_TO_DIRECT(b_phys), PAGE_SIZE); + a_cp = (char *)MIPS_PHYS_TO_DIRECT(a_phys) + + a_pg_offset; + b_cp = (char *)MIPS_PHYS_TO_DIRECT(b_phys) + + b_pg_offset; + bcopy(a_cp, b_cp, cnt); + mips_dcache_wbinv_range((vm_offset_t)b_cp, cnt); + } else { + a_cp = (char *)pmap_lmem_map2(a_phys, b_phys); + b_cp = (char *)a_cp + PAGE_SIZE; + a_cp += a_pg_offset; + b_cp += b_pg_offset; + bcopy(a_cp, b_cp, cnt); + mips_dcache_wbinv_range((vm_offset_t)b_cp, cnt); + pmap_lmem_unmap(); + } + a_offset += cnt; + b_offset += cnt; + xfersize -= cnt; + } +} + /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c index 6a339ca..ddd22ae 100644 --- a/sys/powerpc/aim/mmu_oea.c +++ b/sys/powerpc/aim/mmu_oea.c @@ -275,6 +275,8 @@ void moea_change_wiring(mmu_t, pmap_t, vm_offset_t, boolean_t); void moea_clear_modify(mmu_t, vm_page_t); void moea_clear_reference(mmu_t, vm_page_t); void moea_copy_page(mmu_t, vm_page_t, vm_page_t); +void moea_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, + vm_page_t *mb, vm_offset_t b_offset, int xfersize); void moea_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t); void moea_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_page_t, vm_prot_t); @@ -320,6 +322,7 @@ static mmu_method_t moea_methods[] = { MMUMETHOD(mmu_clear_modify, moea_clear_modify), MMUMETHOD(mmu_clear_reference, moea_clear_reference), MMUMETHOD(mmu_copy_page, moea_copy_page), + MMUMETHOD(mmu_copy_pages, moea_copy_pages), MMUMETHOD(mmu_enter, moea_enter), MMUMETHOD(mmu_enter_object, moea_enter_object), MMUMETHOD(mmu_enter_quick, moea_enter_quick), @@ -1043,6 +1046,30 @@ moea_copy_page(mmu_t mmu, vm_page_t msrc, vm_page_t mdst) bcopy((void *)src, (void *)dst, PAGE_SIZE); } +void +moea_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, + vm_page_t *mb, vm_offset_t b_offset, int xfersize) +{ + void *a_cp, *b_cp; + vm_offset_t a_pg_offset, b_pg_offset; + int cnt; + + while (xfersize > 0) { + a_pg_offset = a_offset & PAGE_MASK; + cnt = min(xfersize, PAGE_SIZE - a_pg_offset); + a_cp = (char *)VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT]) + + a_pg_offset; + b_pg_offset = b_offset & PAGE_MASK; + cnt = min(cnt, PAGE_SIZE - b_pg_offset); + b_cp = (char *)VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT]) + + b_pg_offset; + bcopy(a_cp, b_cp, cnt); + a_offset += cnt; + b_offset += cnt; + xfersize -= cnt; + } +} + /* * Zero a page of physical memory by temporarily mapping it into the tlb. */ diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c index 14b88f0..90466e8a 100644 --- a/sys/powerpc/aim/mmu_oea64.c +++ b/sys/powerpc/aim/mmu_oea64.c @@ -290,6 +290,8 @@ void moea64_change_wiring(mmu_t, pmap_t, vm_offset_t, boolean_t); void moea64_clear_modify(mmu_t, vm_page_t); void moea64_clear_reference(mmu_t, vm_page_t); void moea64_copy_page(mmu_t, vm_page_t, vm_page_t); +void moea64_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, + vm_page_t *mb, vm_offset_t b_offset, int xfersize); void moea64_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t); void moea64_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_page_t, vm_prot_t); @@ -334,6 +336,7 @@ static mmu_method_t moea64_methods[] = { MMUMETHOD(mmu_clear_modify, moea64_clear_modify), MMUMETHOD(mmu_clear_reference, moea64_clear_reference), MMUMETHOD(mmu_copy_page, moea64_copy_page), + MMUMETHOD(mmu_copy_pages, moea64_copy_pages), MMUMETHOD(mmu_enter, moea64_enter), MMUMETHOD(mmu_enter_object, moea64_enter_object), MMUMETHOD(mmu_enter_quick, moea64_enter_quick), @@ -1104,6 +1107,72 @@ moea64_copy_page(mmu_t mmu, vm_page_t msrc, vm_page_t mdst) } } +static inline void +moea64_copy_pages_dmap(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, + vm_page_t *mb, vm_offset_t b_offset, int xfersize) +{ + void *a_cp, *b_cp; + vm_offset_t a_pg_offset, b_pg_offset; + int cnt; + + while (xfersize > 0) { + a_pg_offset = a_offset & PAGE_MASK; + cnt = min(xfersize, PAGE_SIZE - a_pg_offset); + a_cp = (char *)VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT]) + + a_pg_offset; + b_pg_offset = b_offset & PAGE_MASK; + cnt = min(cnt, PAGE_SIZE - b_pg_offset); + b_cp = (char *)VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT]) + + b_pg_offset; + bcopy(a_cp, b_cp, cnt); + a_offset += cnt; + b_offset += cnt; + xfersize -= cnt; + } +} + +static inline void +moea64_copy_pages_nodmap(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, + vm_page_t *mb, vm_offset_t b_offset, int xfersize) +{ + void *a_cp, *b_cp; + vm_offset_t a_pg_offset, b_pg_offset; + int cnt; + + mtx_lock(&moea64_scratchpage_mtx); + while (xfersize > 0) { + a_pg_offset = a_offset & PAGE_MASK; + cnt = min(xfersize, PAGE_SIZE - a_pg_offset); + moea64_set_scratchpage_pa(mmu, 0, + VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])); + a_cp = (char *)moea64_scratchpage_va[0] + a_pg_offset; + b_pg_offset = b_offset & PAGE_MASK; + cnt = min(cnt, PAGE_SIZE - b_pg_offset); + moea64_set_scratchpage_pa(mmu, 1, + VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])); + b_cp = (char *)moea64_scratchpage_va[1] + b_pg_offset; + bcopy(a_cp, b_cp, cnt); + a_offset += cnt; + b_offset += cnt; + xfersize -= cnt; + } + mtx_unlock(&moea64_scratchpage_mtx); +} + +void +moea64_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, + vm_page_t *mb, vm_offset_t b_offset, int xfersize) +{ + + if (hw_direct_map) { + moea64_copy_pages_dmap(mmu, ma, a_offset, mb, b_offset, + xfersize); + } else { + moea64_copy_pages_nodmap(mmu, ma, a_offset, mb, b_offset, + xfersize); + } +} + void moea64_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size) { diff --git a/sys/powerpc/booke/pmap.c b/sys/powerpc/booke/pmap.c index 343b046..b9f4838 100644 --- a/sys/powerpc/booke/pmap.c +++ b/sys/powerpc/booke/pmap.c @@ -274,6 +274,8 @@ static void mmu_booke_clear_reference(mmu_t, vm_page_t); static void mmu_booke_copy(mmu_t, pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t); static void mmu_booke_copy_page(mmu_t, vm_page_t, vm_page_t); +static void mmu_booke_copy_pages(mmu_t, vm_page_t *, + vm_offset_t, vm_page_t *, vm_offset_t, int); static void mmu_booke_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t); static void mmu_booke_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, @@ -334,6 +336,7 @@ static mmu_method_t mmu_booke_methods[] = { MMUMETHOD(mmu_clear_reference, mmu_booke_clear_reference), MMUMETHOD(mmu_copy, mmu_booke_copy), MMUMETHOD(mmu_copy_page, mmu_booke_copy_page), + MMUMETHOD(mmu_copy_pages, mmu_booke_copy_pages), MMUMETHOD(mmu_enter, mmu_booke_enter), MMUMETHOD(mmu_enter_object, mmu_booke_enter_object), MMUMETHOD(mmu_enter_quick, mmu_booke_enter_quick), @@ -2136,6 +2139,36 @@ mmu_booke_copy_page(mmu_t mmu, vm_page_t sm, vm_page_t dm) mtx_unlock(©_page_mutex); } +static inline void +mmu_booke_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, + vm_page_t *mb, vm_offset_t b_offset, int xfersize) +{ + void *a_cp, *b_cp; + vm_offset_t a_pg_offset, b_pg_offset; + int cnt; + + mtx_lock(©_page_mutex); + while (xfersize > 0) { + a_pg_offset = a_offset & PAGE_MASK; + cnt = min(xfersize, PAGE_SIZE - a_pg_offset); + mmu_booke_kenter(mmu, copy_page_src_va, + VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])); + a_cp = (char *)copy_page_src_va + a_pg_offset; + b_pg_offset = b_offset & PAGE_MASK; + cnt = min(cnt, PAGE_SIZE - b_pg_offset); + mmu_booke_kenter(mmu, copy_page_dst_va, + VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])); + b_cp = (char *)copy_page_dst_va + b_pg_offset; + bcopy(a_cp, b_cp, cnt); + mmu_booke_kremove(mmu, copy_page_dst_va); + mmu_booke_kremove(mmu, copy_page_src_va); + a_offset += cnt; + b_offset += cnt; + xfersize -= cnt; + } + mtx_unlock(©_page_mutex); +} + /* * mmu_booke_zero_page_idle zeros the specified hardware page by mapping it * into virtual memory and using bzero to clear its contents. This is intended diff --git a/sys/powerpc/powerpc/mmu_if.m b/sys/powerpc/powerpc/mmu_if.m index 8cd6e52..0382bd8 100644 --- a/sys/powerpc/powerpc/mmu_if.m +++ b/sys/powerpc/powerpc/mmu_if.m @@ -215,6 +215,14 @@ METHOD void copy_page { vm_page_t _dst; }; +METHOD void copy_pages { + mmu_t _mmu; + vm_page_t *_ma; + vm_offset_t _a_offset; + vm_page_t *_mb; + vm_offset_t _b_offset; + int _xfersize; +}; /** * @brief Create a mapping between a virtual/physical address pair in the diff --git a/sys/powerpc/powerpc/pmap_dispatch.c b/sys/powerpc/powerpc/pmap_dispatch.c index c919196..42f1a39 100644 --- a/sys/powerpc/powerpc/pmap_dispatch.c +++ b/sys/powerpc/powerpc/pmap_dispatch.c @@ -133,6 +133,16 @@ pmap_copy_page(vm_page_t src, vm_page_t dst) } void +pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], + vm_offset_t b_offset, int xfersize) +{ + + CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma, + a_offset, mb, b_offset, xfersize); + MMU_COPY_PAGES(mmu_obj, ma, a_offset, mb, b_offset, xfersize); +} + +void pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t p, vm_prot_t prot, boolean_t wired) { diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c index 8bfc454..97a085a 100644 --- a/sys/sparc64/sparc64/pmap.c +++ b/sys/sparc64/sparc64/pmap.c @@ -1918,6 +1918,14 @@ pmap_copy_page(vm_page_t msrc, vm_page_t mdst) } } +void +pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], + vm_offset_t b_offset, int xfersize) +{ + + panic("pmap_copy_pages: not implemented"); +} + /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may diff --git a/sys/vm/pmap.h b/sys/vm/pmap.h index d06c22b..c64a549 100644 --- a/sys/vm/pmap.h +++ b/sys/vm/pmap.h @@ -108,6 +108,8 @@ void pmap_clear_modify(vm_page_t m); void pmap_clear_reference(vm_page_t m); void pmap_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t); void pmap_copy_page(vm_page_t, vm_page_t); +void pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, + vm_page_t mb[], vm_offset_t b_offset, int xfersize); void pmap_enter(pmap_t, vm_offset_t, vm_prot_t, vm_page_t, vm_prot_t, boolean_t); void pmap_enter_object(pmap_t pmap, vm_offset_t start, -- cgit v1.1 From 9b0c4b125bb359cc94686ed5aafa1aaeecbcc3cf Mon Sep 17 00:00:00 2001 From: kib Date: Thu, 14 Mar 2013 20:28:26 +0000 Subject: Add currently unused flag argument to the cluster_read(), cluster_write() and cluster_wbuild() functions. The flags to be allowed are a subset of the GB_* flags for getblk(). Sponsored by: The FreeBSD Foundation Tested by: pho --- sys/fs/cd9660/cd9660_vnops.c | 2 +- sys/fs/ext2fs/ext2_balloc.c | 2 +- sys/fs/ext2fs/ext2_vnops.c | 9 +++++---- sys/fs/msdosfs/msdosfs_vnops.c | 4 ++-- sys/fs/udf/udf_vnops.c | 5 +++-- sys/kern/vfs_bio.c | 5 +++-- sys/kern/vfs_cluster.c | 24 ++++++++---------------- sys/sys/buf.h | 6 +++--- sys/ufs/ffs/ffs_balloc.c | 4 ++-- sys/ufs/ffs/ffs_vnops.c | 6 ++++-- 10 files changed, 32 insertions(+), 35 deletions(-) (limited to 'sys') diff --git a/sys/fs/cd9660/cd9660_vnops.c b/sys/fs/cd9660/cd9660_vnops.c index 21ee0fc..47d4f75 100644 --- a/sys/fs/cd9660/cd9660_vnops.c +++ b/sys/fs/cd9660/cd9660_vnops.c @@ -329,7 +329,7 @@ cd9660_read(ap) if (lblktosize(imp, rablock) < ip->i_size) error = cluster_read(vp, (off_t)ip->i_size, lbn, size, NOCRED, uio->uio_resid, - (ap->a_ioflag >> 16), &bp); + (ap->a_ioflag >> 16), 0, &bp); else error = bread(vp, lbn, size, NOCRED, &bp); } else { diff --git a/sys/fs/ext2fs/ext2_balloc.c b/sys/fs/ext2fs/ext2_balloc.c index 1c0cc0e..88ad710 100644 --- a/sys/fs/ext2fs/ext2_balloc.c +++ b/sys/fs/ext2fs/ext2_balloc.c @@ -276,7 +276,7 @@ ext2_balloc(struct inode *ip, int32_t lbn, int size, struct ucred *cred, if (seqcount && (vp->v_mount->mnt_flag & MNT_NOCLUSTERR) == 0) { error = cluster_read(vp, ip->i_size, lbn, (int)fs->e2fs_bsize, NOCRED, - MAXBSIZE, seqcount, &nbp); + MAXBSIZE, seqcount, 0, &nbp); } else { error = bread(vp, lbn, (int)fs->e2fs_bsize, NOCRED, &nbp); } diff --git a/sys/fs/ext2fs/ext2_vnops.c b/sys/fs/ext2fs/ext2_vnops.c index 1c0b7a1..77eb74b 100644 --- a/sys/fs/ext2fs/ext2_vnops.c +++ b/sys/fs/ext2fs/ext2_vnops.c @@ -1618,10 +1618,11 @@ ext2_read(struct vop_read_args *ap) if (lblktosize(fs, nextlbn) >= ip->i_size) error = bread(vp, lbn, size, NOCRED, &bp); - else if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERR) == 0) + else if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERR) == 0) { error = cluster_read(vp, ip->i_size, lbn, size, - NOCRED, blkoffset + uio->uio_resid, seqcount, &bp); - else if (seqcount > 1) { + NOCRED, blkoffset + uio->uio_resid, seqcount, + 0, &bp); + } else if (seqcount > 1) { int nextsize = blksize(fs, ip, nextlbn); error = breadn(vp, lbn, size, &nextlbn, &nextsize, 1, NOCRED, &bp); @@ -1831,7 +1832,7 @@ ext2_write(struct vop_write_args *ap) } else if (xfersize + blkoffset == fs->e2fs_fsize) { if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERW) == 0) { bp->b_flags |= B_CLUSTEROK; - cluster_write(vp, bp, ip->i_size, seqcount); + cluster_write(vp, bp, ip->i_size, seqcount, 0); } else { bawrite(bp); } diff --git a/sys/fs/msdosfs/msdosfs_vnops.c b/sys/fs/msdosfs/msdosfs_vnops.c index 8e045cb..213ae81 100644 --- a/sys/fs/msdosfs/msdosfs_vnops.c +++ b/sys/fs/msdosfs/msdosfs_vnops.c @@ -600,7 +600,7 @@ msdosfs_read(ap) error = bread(vp, lbn, blsize, NOCRED, &bp); } else if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERR) == 0) { error = cluster_read(vp, dep->de_FileSize, lbn, blsize, - NOCRED, on + uio->uio_resid, seqcount, &bp); + NOCRED, on + uio->uio_resid, seqcount, 0, &bp); } else if (seqcount > 1) { rasize = blsize; error = breadn(vp, lbn, @@ -820,7 +820,7 @@ msdosfs_write(ap) else if (n + croffset == pmp->pm_bpcluster) { if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERW) == 0) cluster_write(vp, bp, dep->de_FileSize, - seqcount); + seqcount, 0); else bawrite(bp); } else diff --git a/sys/fs/udf/udf_vnops.c b/sys/fs/udf/udf_vnops.c index b1a3b1d..abe073e 100644 --- a/sys/fs/udf/udf_vnops.c +++ b/sys/fs/udf/udf_vnops.c @@ -478,8 +478,9 @@ udf_read(struct vop_read_args *ap) rablock = lbn + 1; if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERR) == 0) { if (lblktosize(udfmp, rablock) < fsize) { - error = cluster_read(vp, fsize, lbn, size, NOCRED, - uio->uio_resid, (ap->a_ioflag >> 16), &bp); + error = cluster_read(vp, fsize, lbn, size, + NOCRED, uio->uio_resid, + (ap->a_ioflag >> 16), 0, &bp); } else { error = bread(vp, lbn, size, NOCRED, &bp); } diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 324efe5..4ac9996 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -1794,8 +1794,9 @@ vfs_bio_awrite(struct buf *bp) */ if (ncl != 1) { BUF_UNLOCK(bp); - nwritten = cluster_wbuild(vp, size, lblkno - j, ncl); - return nwritten; + nwritten = cluster_wbuild(vp, size, lblkno - j, ncl, + 0); + return (nwritten); } } bremfree(bp); diff --git a/sys/kern/vfs_cluster.c b/sys/kern/vfs_cluster.c index 70937a2..e0397d7 100644 --- a/sys/kern/vfs_cluster.c +++ b/sys/kern/vfs_cluster.c @@ -84,15 +84,9 @@ extern vm_page_t bogus_page; * cluster_read replaces bread. */ int -cluster_read(vp, filesize, lblkno, size, cred, totread, seqcount, bpp) - struct vnode *vp; - u_quad_t filesize; - daddr_t lblkno; - long size; - struct ucred *cred; - long totread; - int seqcount; - struct buf **bpp; +cluster_read(struct vnode *vp, u_quad_t filesize, daddr_t lblkno, long size, + struct ucred *cred, long totread, int seqcount, int gbflags, + struct buf **bpp) { struct buf *bp, *rbp, *reqbp; struct bufobj *bo; @@ -576,7 +570,7 @@ cluster_wbuild_wb(struct vnode *vp, long size, daddr_t start_lbn, int len) start_lbn -= len; /* FALLTHROUGH */ case 1: - r = cluster_wbuild(vp, size, start_lbn, len); + r = cluster_wbuild(vp, size, start_lbn, len, 0); /* FALLTHROUGH */ default: /* FALLTHROUGH */ @@ -596,7 +590,8 @@ cluster_wbuild_wb(struct vnode *vp, long size, daddr_t start_lbn, int len) * 4. end of a cluster - asynchronously write cluster */ void -cluster_write(struct vnode *vp, struct buf *bp, u_quad_t filesize, int seqcount) +cluster_write(struct vnode *vp, struct buf *bp, u_quad_t filesize, int seqcount, + int gbflags) { daddr_t lbn; int maxclen, cursize; @@ -742,11 +737,8 @@ cluster_write(struct vnode *vp, struct buf *bp, u_quad_t filesize, int seqcount) * the current block (if last_bp == NULL). */ int -cluster_wbuild(vp, size, start_lbn, len) - struct vnode *vp; - long size; - daddr_t start_lbn; - int len; +cluster_wbuild(struct vnode *vp, long size, daddr_t start_lbn, int len, + int gbflags) { struct buf *bp, *tbp; struct bufobj *bo; diff --git a/sys/sys/buf.h b/sys/sys/buf.h index 672ef5a..2261b56 100644 --- a/sys/sys/buf.h +++ b/sys/sys/buf.h @@ -508,9 +508,9 @@ void bufdone_finish(struct buf *); void bd_speedup(void); int cluster_read(struct vnode *, u_quad_t, daddr_t, long, - struct ucred *, long, int, struct buf **); -int cluster_wbuild(struct vnode *, long, daddr_t, int); -void cluster_write(struct vnode *, struct buf *, u_quad_t, int); + struct ucred *, long, int, int, struct buf **); +int cluster_wbuild(struct vnode *, long, daddr_t, int, int); +void cluster_write(struct vnode *, struct buf *, u_quad_t, int, int); void vfs_bio_set_valid(struct buf *, int base, int size); void vfs_bio_clrbuf(struct buf *); void vfs_busy_pages(struct buf *, int clear_modify); diff --git a/sys/ufs/ffs/ffs_balloc.c b/sys/ufs/ffs/ffs_balloc.c index 0e29be87f..8ac9d80 100644 --- a/sys/ufs/ffs/ffs_balloc.c +++ b/sys/ufs/ffs/ffs_balloc.c @@ -418,7 +418,7 @@ retry: if (seqcount && (vp->v_mount->mnt_flag & MNT_NOCLUSTERR) == 0) { error = cluster_read(vp, ip->i_size, lbn, (int)fs->fs_bsize, NOCRED, - MAXBSIZE, seqcount, &nbp); + MAXBSIZE, seqcount, 0, &nbp); } else { error = bread(vp, lbn, (int)fs->fs_bsize, NOCRED, &nbp); } @@ -966,7 +966,7 @@ retry: if (seqcount && (vp->v_mount->mnt_flag & MNT_NOCLUSTERR) == 0) { error = cluster_read(vp, ip->i_size, lbn, (int)fs->fs_bsize, NOCRED, - MAXBSIZE, seqcount, &nbp); + MAXBSIZE, seqcount, 0, &nbp); } else { error = bread(vp, lbn, (int)fs->fs_bsize, NOCRED, &nbp); } diff --git a/sys/ufs/ffs/ffs_vnops.c b/sys/ufs/ffs/ffs_vnops.c index c065187..c310ece 100644 --- a/sys/ufs/ffs/ffs_vnops.c +++ b/sys/ufs/ffs/ffs_vnops.c @@ -519,7 +519,8 @@ ffs_read(ap) * doing sequential access. */ error = cluster_read(vp, ip->i_size, lbn, - size, NOCRED, blkoffset + uio->uio_resid, seqcount, &bp); + size, NOCRED, blkoffset + uio->uio_resid, + seqcount, 0, &bp); } else if (seqcount > 1) { /* * If we are NOT allowed to cluster, then @@ -784,7 +785,8 @@ ffs_write(ap) } else if (xfersize + blkoffset == fs->fs_bsize) { if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERW) == 0) { bp->b_flags |= B_CLUSTEROK; - cluster_write(vp, bp, ip->i_size, seqcount); + cluster_write(vp, bp, ip->i_size, seqcount, + 0); } else { bawrite(bp); } -- cgit v1.1 From 7ca94eca24d0f2013eefeb498c3ae0489022ee3a Mon Sep 17 00:00:00 2001 From: kib Date: Thu, 14 Mar 2013 20:31:39 +0000 Subject: Some style fixes. Sponsored by: The FreeBSD Foundation --- sys/kern/vfs_bio.c | 12 +++++------- sys/kern/vfs_cluster.c | 2 +- sys/ufs/ffs/ffs_balloc.c | 8 ++++---- sys/vm/vnode_pager.c | 2 +- 4 files changed, 11 insertions(+), 13 deletions(-) (limited to 'sys') diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 4ac9996..d20c829 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -830,9 +830,8 @@ breada(struct vnode * vp, daddr_t * rablkno, int * rabsize, * getblk(). Also starts asynchronous I/O on read-ahead blocks. */ int -breadn_flags(struct vnode * vp, daddr_t blkno, int size, - daddr_t * rablkno, int *rabsize, int cnt, - struct ucred * cred, int flags, struct buf **bpp) +breadn_flags(struct vnode *vp, daddr_t blkno, int size, daddr_t *rablkno, + int *rabsize, int cnt, struct ucred *cred, int flags, struct buf **bpp) { struct buf *bp; int rv = 0, readwait = 0; @@ -1809,7 +1808,7 @@ vfs_bio_awrite(struct buf *bp) nwritten = bp->b_bufsize; (void) bwrite(bp); - return nwritten; + return (nwritten); } /* @@ -2631,7 +2630,7 @@ vfs_setdirty_locked_object(struct buf *bp) * prior to issuing the READ. biodone() will *not* clear B_INVAL. */ struct buf * -getblk(struct vnode * vp, daddr_t blkno, int size, int slpflag, int slptimeo, +getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo, int flags) { struct buf *bp; @@ -2709,9 +2708,8 @@ loop: } /* - * check for size inconsistancies for non-VMIO case. + * check for size inconsistencies for non-VMIO case. */ - if (bp->b_bcount != size) { if ((bp->b_flags & B_VMIO) == 0 || (size > bp->b_kvasize)) { diff --git a/sys/kern/vfs_cluster.c b/sys/kern/vfs_cluster.c index e0397d7..28aa4ff 100644 --- a/sys/kern/vfs_cluster.c +++ b/sys/kern/vfs_cluster.c @@ -563,7 +563,7 @@ cluster_wbuild_wb(struct vnode *vp, long size, daddr_t start_lbn, int len) { int r = 0; - switch(write_behind) { + switch (write_behind) { case 2: if (start_lbn < len) break; diff --git a/sys/ufs/ffs/ffs_balloc.c b/sys/ufs/ffs/ffs_balloc.c index 8ac9d80..a5e99d9 100644 --- a/sys/ufs/ffs/ffs_balloc.c +++ b/sys/ufs/ffs/ffs_balloc.c @@ -679,9 +679,9 @@ ffs_balloc_ufs2(struct vnode *vp, off_t startoffset, int size, if (osize < fs->fs_bsize && osize > 0) { UFS_LOCK(ump); error = ffs_realloccg(ip, nb, dp->di_db[nb], - ffs_blkpref_ufs2(ip, lastlbn, (int)nb, - &dp->di_db[0]), osize, (int)fs->fs_bsize, - flags, cred, &bp); + ffs_blkpref_ufs2(ip, lastlbn, (int)nb, + &dp->di_db[0]), osize, (int)fs->fs_bsize, + flags, cred, &bp); if (error) return (error); if (DOINGSOFTDEP(vp)) @@ -733,7 +733,7 @@ ffs_balloc_ufs2(struct vnode *vp, off_t startoffset, int size, UFS_LOCK(ump); error = ffs_realloccg(ip, lbn, dp->di_db[lbn], ffs_blkpref_ufs2(ip, lbn, (int)lbn, - &dp->di_db[0]), osize, nsize, flags, + &dp->di_db[0]), osize, nsize, flags, cred, &bp); if (error) return (error); diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index 5e331ee..a0340af 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -900,7 +900,7 @@ vnode_pager_generic_getpages(vp, m, bytecount, reqpage) } bp = getpbuf(&vnode_pbuf_freecnt); - kva = (vm_offset_t) bp->b_data; + kva = (vm_offset_t)bp->b_data; /* * and map the pages to be read into the kva -- cgit v1.1 From 1441d9fb6e2050e01a11c4855fd520d18c63ee41 Mon Sep 17 00:00:00 2001 From: brooks Date: Thu, 14 Mar 2013 22:16:13 +0000 Subject: FDT_DTS_FILE is expanded in a Makefile so use :R to remove the suffix rather than using echo|cut to remove everything after the first '.'. --- sys/conf/files | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'sys') diff --git a/sys/conf/files b/sys/conf/files index 627e7d6..bba5c9d 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -55,9 +55,9 @@ aic79xx_reg_print.o optional ahd pci ahd_reg_pretty_print \ # from the specified source (DTS) file: .dts -> .dtb # fdt_dtb_file optional fdt \ - compile-with "if [ -f $S/boot/fdt/dts/${FDT_DTS_FILE} ]; then dtc -O dtb -o `echo ${FDT_DTS_FILE} | cut -d. -f1`.dtb -b 0 -p 1024 $S/boot/fdt/dts/${FDT_DTS_FILE}; fi" \ + compile-with "if [ -f $S/boot/fdt/dts/${FDT_DTS_FILE} ]; then dtc -O dtb -o ${FDT_DTS_FILE:R}.dtb -b 0 -p 1024 $S/boot/fdt/dts/${FDT_DTS_FILE}; fi" \ no-obj no-implicit-rule before-depend \ - clean "`echo ${FDT_DTS_FILE} | cut -d. -f1`.dtb" + clean "${FDT_DTS_FILE:R}.dtb" fdt_static_dtb.h optional fdt fdt_dtb_static \ compile-with "sh $S/tools/fdt/make_dtbh.sh ${FDT_DTS_FILE} ." \ no-obj no-implicit-rule before-depend \ -- cgit v1.1 From 2e500238ddb670ec730cc2561f406d256e2a2aa9 Mon Sep 17 00:00:00 2001 From: pjd Date: Thu, 14 Mar 2013 23:07:01 +0000 Subject: We don't need buffer to handle BIO_DELETE, so don't check buffer size for it. This fixes handling BIO_DELETE larger than MAXPHYS. --- sys/geom/gate/g_gate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sys') diff --git a/sys/geom/gate/g_gate.c b/sys/geom/gate/g_gate.c index d1ddd40..c44993a 100644 --- a/sys/geom/gate/g_gate.c +++ b/sys/geom/gate/g_gate.c @@ -813,7 +813,7 @@ g_gate_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, struct threa } } ggio->gctl_cmd = bp->bio_cmd; - if ((bp->bio_cmd == BIO_DELETE || bp->bio_cmd == BIO_WRITE) && + if (bp->bio_cmd == BIO_WRITE && bp->bio_length > ggio->gctl_length) { mtx_unlock(&sc->sc_queue_mtx); ggio->gctl_length = bp->bio_length; -- cgit v1.1 From 8593db8ad43b6bccf4eb60017ae9ebedfe7722f6 Mon Sep 17 00:00:00 2001 From: trasz Date: Thu, 14 Mar 2013 23:20:18 +0000 Subject: Accessing td_state requires thread lock to be held. Submitted by: Rudo Tomori Reviewed by: kib --- sys/kern/kern_racct.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'sys') diff --git a/sys/kern/kern_racct.c b/sys/kern/kern_racct.c index bf8141c..d31c832 100644 --- a/sys/kern/kern_racct.c +++ b/sys/kern/kern_racct.c @@ -1033,6 +1033,7 @@ racct_proc_throttle(struct proc *p) p->p_throttled = 1; FOREACH_THREAD_IN_PROC(p, td) { + thread_lock(td); switch (td->td_state) { case TDS_RUNQ: /* @@ -1041,27 +1042,24 @@ racct_proc_throttle(struct proc *p) * TDF_NEEDRESCHED for the thread, so that once it is * running, it is taken off the cpu as soon as possible. */ - thread_lock(td); td->td_flags |= TDF_NEEDRESCHED; - thread_unlock(td); break; case TDS_RUNNING: /* * If the thread is running, we request a context * switch for it by setting the TDF_NEEDRESCHED flag. */ - thread_lock(td); td->td_flags |= TDF_NEEDRESCHED; #ifdef SMP cpuid = td->td_oncpu; if ((cpuid != NOCPU) && (td != curthread)) ipi_cpu(cpuid, IPI_AST); #endif - thread_unlock(td); break; default: break; } + thread_unlock(td); } } -- cgit v1.1 From 5b0f44c9215684a9705167191bf33f92d8ba7a36 Mon Sep 17 00:00:00 2001 From: trasz Date: Thu, 14 Mar 2013 23:25:42 +0000 Subject: When throttling a process to enforce RACCT limits, do not use neither PBDRY (which simply doesn't make any sense) nor PCATCH (which could be used by a malicious process to work around the PCPU limit). Submitted by: Rudo Tomori Reviewed by: kib --- sys/kern/subr_trap.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) (limited to 'sys') diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c index 1f24e88..11a4bc4 100644 --- a/sys/kern/subr_trap.c +++ b/sys/kern/subr_trap.c @@ -100,9 +100,6 @@ void userret(struct thread *td, struct trapframe *frame) { struct proc *p = td->td_proc; -#ifdef RACCT - int sig; -#endif CTR3(KTR_SYSC, "userret: thread %p (pid %d, %s)", td, p->p_pid, td->td_name); @@ -175,12 +172,8 @@ userret(struct thread *td, struct trapframe *frame) #endif #ifdef RACCT PROC_LOCK(p); - while (p->p_throttled == 1) { - sig = msleep(p->p_racct, &p->p_mtx, PCATCH | PBDRY, "racct", - hz); - if ((sig == EINTR) || (sig == ERESTART)) - break; - } + while (p->p_throttled == 1) + msleep(p->p_racct, &p->p_mtx, 0, "racct", 0); PROC_UNLOCK(p); #endif } -- cgit v1.1 From 7be3120d8378c9cb3f30d1f57d8c93308ccf11f6 Mon Sep 17 00:00:00 2001 From: adrian Date: Fri, 15 Mar 2013 02:52:37 +0000 Subject: Add locking around the new holdingbf code. Since this is being done during buffer free, it's a crap shoot whether the TX path lock is held or not. I tried putting the ath_freebuf() code inside the TX lock and I got all kinds of locking issues - it turns out that the buffer free path sometimes is called with the lock held and sometimes isn't. So I'll go and fix that soon. Hence for now the holdingbf buffers are protected by the TXBUF lock. --- sys/dev/ath/if_ath.c | 11 +++++++---- sys/dev/ath/if_athvar.h | 9 +++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) (limited to 'sys') diff --git a/sys/dev/ath/if_ath.c b/sys/dev/ath/if_ath.c index cf69990..7601f2f 100644 --- a/sys/dev/ath/if_ath.c +++ b/sys/dev/ath/if_ath.c @@ -4156,14 +4156,13 @@ ath_returnbuf_head(struct ath_softc *sc, struct ath_buf *bf) static void ath_txq_freeholdingbuf(struct ath_softc *sc, struct ath_txq *txq) { + ATH_TXBUF_LOCK_ASSERT(sc); if (txq->axq_holdingbf == NULL) return; txq->axq_holdingbf->bf_flags &= ~ATH_BUF_BUSY; - ATH_TXBUF_LOCK(sc); ath_returnbuf_tail(sc, txq->axq_holdingbf); - ATH_TXBUF_UNLOCK(sc); txq->axq_holdingbf = NULL; } @@ -4176,6 +4175,8 @@ ath_txq_addholdingbuf(struct ath_softc *sc, struct ath_buf *bf) { struct ath_txq *txq; + ATH_TXBUF_LOCK_ASSERT(sc); + /* XXX assert ATH_BUF_BUSY is set */ /* XXX assert the tx queue is under the max number */ @@ -4188,10 +4189,8 @@ ath_txq_addholdingbuf(struct ath_softc *sc, struct ath_buf *bf) ath_returnbuf_tail(sc, bf); return; } - txq = &sc->sc_txq[bf->bf_state.bfs_tx_queue]; ath_txq_freeholdingbuf(sc, txq); - txq->axq_holdingbf = bf; } @@ -4219,7 +4218,9 @@ ath_freebuf(struct ath_softc *sc, struct ath_buf *bf) * If this buffer is busy, push it onto the holding queue */ if (bf->bf_flags & ATH_BUF_BUSY) { + ATH_TXBUF_LOCK(sc); ath_txq_addholdingbuf(sc, bf); + ATH_TXBUF_UNLOCK(sc); return; } @@ -4342,7 +4343,9 @@ ath_tx_draintxq(struct ath_softc *sc, struct ath_txq *txq) /* * Free the holding buffer if it exists */ + ATH_TXBUF_LOCK(sc); ath_txq_freeholdingbuf(sc, txq); + ATH_TXBUF_UNLOCK(sc); /* * Drain software queued frames which are on diff --git a/sys/dev/ath/if_athvar.h b/sys/dev/ath/if_athvar.h index 1ac6fe7..d3e4aa30 100644 --- a/sys/dev/ath/if_athvar.h +++ b/sys/dev/ath/if_athvar.h @@ -329,6 +329,15 @@ struct ath_txq { u_int axq_intrcnt; /* interrupt count */ u_int32_t *axq_link; /* link ptr in last TX desc */ TAILQ_HEAD(axq_q_s, ath_buf) axq_q; /* transmit queue */ + /* + * XXX the holdingbf field is protected by the TXBUF lock + * for now, NOT the TX lock. + * + * Architecturally, it would likely be better to move + * the holdingbf field to a separate array in ath_softc + * just to highlight that it's not protected by the normal + * TX path lock. + */ struct ath_buf *axq_holdingbf; /* holding TX buffer */ char axq_name[12]; /* e.g. "ath0_txq4" */ -- cgit v1.1 From ad703924188fb242c928ffcb8b2a6c008b143991 Mon Sep 17 00:00:00 2001 From: adrian Date: Fri, 15 Mar 2013 04:43:27 +0000 Subject: Remove a now incorrect comment. This comment dates back to my initial stab at TX aggregation completion, where I didn't even bother trying to do software retries. --- sys/dev/ath/if_ath_tx.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'sys') diff --git a/sys/dev/ath/if_ath_tx.c b/sys/dev/ath/if_ath_tx.c index 33318ea..97e6967 100644 --- a/sys/dev/ath/if_ath_tx.c +++ b/sys/dev/ath/if_ath_tx.c @@ -4176,8 +4176,6 @@ ath_tx_comp_cleanup_aggr(struct ath_softc *sc, struct ath_buf *bf_first) /* * Handle completion of an set of aggregate frames. * - * XXX for now, simply complete each sub-frame. - * * Note: the completion handler is the last descriptor in the aggregate, * not the last descriptor in the first frame. */ -- cgit v1.1 From 4502cb662c82b7919c3dd6c0616a07f8bc59ca55 Mon Sep 17 00:00:00 2001 From: glebius Date: Fri, 15 Mar 2013 10:15:07 +0000 Subject: Make m_get2() never use clusters that are bigger than PAGE_SIZE. Requested by: andre, jhb Sponsored by: Nginx, Inc. --- sys/kern/uipc_mbuf.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) (limited to 'sys') diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c index e962670..e2b008a 100644 --- a/sys/kern/uipc_mbuf.c +++ b/sys/kern/uipc_mbuf.c @@ -92,7 +92,6 @@ m_get2(int size, int how, short type, int flags) { struct mb_args args; struct mbuf *m, *n; - uma_zone_t zone; args.flags = flags; args.type = type; @@ -101,24 +100,15 @@ m_get2(int size, int how, short type, int flags) return (uma_zalloc_arg(zone_mbuf, &args, how)); if (size <= MCLBYTES) return (uma_zalloc_arg(zone_pack, &args, how)); - if (size > MJUM16BYTES) + + if (size > MJUMPAGESIZE) return (NULL); m = uma_zalloc_arg(zone_mbuf, &args, how); if (m == NULL) return (NULL); -#if MJUMPAGESIZE != MCLBYTES - if (size <= MJUMPAGESIZE) - zone = zone_jumbop; - else -#endif - if (size <= MJUM9BYTES) - zone = zone_jumbo9; - else - zone = zone_jumbo16; - - n = uma_zalloc_arg(zone, m, how); + n = uma_zalloc_arg(zone_jumbop, m, how); if (n == NULL) { uma_zfree(zone_mbuf, m); return (NULL); -- cgit v1.1 From 4d24ab3136bd066fd4c31400aa74b0bbc7e5d664 Mon Sep 17 00:00:00 2001 From: glebius Date: Fri, 15 Mar 2013 10:17:24 +0000 Subject: - Use m_get2() instead of hand allocating. - No need for u_int cast here. Sponsored by: Nginx, Inc. --- sys/kern/uipc_syscalls.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'sys') diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c index 4827cab..3a2f032 100644 --- a/sys/kern/uipc_syscalls.c +++ b/sys/kern/uipc_syscalls.c @@ -1701,18 +1701,16 @@ sockargs(mp, buf, buflen, type) struct mbuf *m; int error; - if ((u_int)buflen > MLEN) { + if (buflen > MLEN) { #ifdef COMPAT_OLDSOCK - if (type == MT_SONAME && (u_int)buflen <= 112) + if (type == MT_SONAME && buflen <= 112) buflen = MLEN; /* unix domain compat. hack */ else #endif - if ((u_int)buflen > MCLBYTES) + if (buflen > MCLBYTES) return (EINVAL); } - m = m_get(M_WAITOK, type); - if ((u_int)buflen > MLEN) - MCLGET(m, M_WAITOK); + m = m_get2(buflen, M_WAITOK, type, 0); m->m_len = buflen; error = copyin(buf, mtod(m, caddr_t), (u_int)buflen); if (error) -- cgit v1.1 From f1574e6b22a8e4c942bcaa09c0a3559a9a7e82ee Mon Sep 17 00:00:00 2001 From: glebius Date: Fri, 15 Mar 2013 10:20:15 +0000 Subject: Use m_get2() + m_align() instead of hand made key_alloc_mbuf(). Code examination shows, that although key_alloc_mbuf() could return chains, the callers never use chains, so m_get2() should suffice. Sponsored by: Nginx, Inc. --- sys/netipsec/key.c | 154 +++++++++++++++++------------------------------------ 1 file changed, 50 insertions(+), 104 deletions(-) (limited to 'sys') diff --git a/sys/netipsec/key.c b/sys/netipsec/key.c index a4b7ecb..67afed2 100644 --- a/sys/netipsec/key.c +++ b/sys/netipsec/key.c @@ -547,7 +547,6 @@ static const char *key_getfqdn __P((void)); static const char *key_getuserfqdn __P((void)); #endif static void key_sa_chgstate __P((struct secasvar *, u_int8_t)); -static struct mbuf *key_alloc_mbuf __P((int)); static __inline void sa_initref(struct secasvar *sav) @@ -1634,15 +1633,11 @@ key_sp2msg(sp) tlen = key_getspreqmsglen(sp); - m = key_alloc_mbuf(tlen); - if (!m || m->m_next) { /*XXX*/ - if (m) - m_freem(m); - return NULL; - } - + m = m_get2(tlen, M_NOWAIT, MT_DATA, 0); + if (m == NULL) + return (NULL); + m_align(m, tlen); m->m_len = tlen; - m->m_next = NULL; xpl = mtod(m, struct sadb_x_policy *); bzero(xpl, tlen); @@ -1732,12 +1727,11 @@ key_gather_mbuf(m, mhp, ndeep, nitem, va_alist) mtod(n, caddr_t)); } else if (i < ndeep) { len = mhp->extlen[idx]; - n = key_alloc_mbuf(len); - if (!n || n->m_next) { /*XXX*/ - if (n) - m_freem(n); + n = m_get2(len, M_NOWAIT, MT_DATA, 0); + if (n == NULL) goto fail; - } + m_align(n, len); + n->m_len = len; m_copydata(m, mhp->extoff[idx], mhp->extlen[idx], mtod(n, caddr_t)); } else { @@ -2602,13 +2596,13 @@ key_spdexpire(sp) /* create lifetime extension (current and hard) */ len = PFKEY_ALIGN8(sizeof(*lt)) * 2; - m = key_alloc_mbuf(len); - if (!m || m->m_next) { /*XXX*/ - if (m) - m_freem(m); + m = m_get2(len, M_NOWAIT, MT_DATA, 0); + if (m == NULL) { error = ENOBUFS; goto fail; } + m_align(m, len); + m->m_len = len; bzero(mtod(m, caddr_t), len); lt = mtod(m, struct sadb_lifetime *); lt->sadb_lifetime_len = PFKEY_UNIT64(sizeof(struct sadb_lifetime)); @@ -3602,15 +3596,12 @@ key_setsadbsa(sav) int len; len = PFKEY_ALIGN8(sizeof(struct sadb_sa)); - m = key_alloc_mbuf(len); - if (!m || m->m_next) { /*XXX*/ - if (m) - m_freem(m); - return NULL; - } - + m = m_get2(len, M_NOWAIT, MT_DATA, 0); + if (m == NULL) + return (NULL); + m_align(m, len); + m->m_len = len; p = mtod(m, struct sadb_sa *); - bzero(p, len); p->sadb_sa_len = PFKEY_UNIT64(len); p->sadb_sa_exttype = SADB_EXT_SA; @@ -3636,13 +3627,11 @@ key_setsadbaddr(u_int16_t exttype, const struct sockaddr *saddr, u_int8_t prefix len = PFKEY_ALIGN8(sizeof(struct sadb_address)) + PFKEY_ALIGN8(saddr->sa_len); - m = key_alloc_mbuf(len); - if (!m || m->m_next) { /*XXX*/ - if (m) - m_freem(m); - return NULL; - } - + m = m_get2(len, M_NOWAIT, MT_DATA, 0); + if (m == NULL) + return (NULL); + m_align(m, len); + m->m_len = len; p = mtod(m, struct sadb_address *); bzero(p, len); @@ -3682,13 +3671,11 @@ key_setsadbxsa2(u_int8_t mode, u_int32_t seq, u_int32_t reqid) size_t len; len = PFKEY_ALIGN8(sizeof(struct sadb_x_sa2)); - m = key_alloc_mbuf(len); - if (!m || m->m_next) { /*XXX*/ - if (m) - m_freem(m); - return NULL; - } - + m = m_get2(len, M_NOWAIT, MT_DATA, 0); + if (m == NULL) + return (NULL); + m_align(m, len); + m->m_len = len; p = mtod(m, struct sadb_x_sa2 *); bzero(p, len); @@ -3716,13 +3703,11 @@ key_setsadbxtype(u_int16_t type) len = PFKEY_ALIGN8(sizeof(struct sadb_x_nat_t_type)); - m = key_alloc_mbuf(len); - if (!m || m->m_next) { /*XXX*/ - if (m) - m_freem(m); + m = m_get2(len, M_NOWAIT, MT_DATA, 0); + if (m == NULL) return (NULL); - } - + m_align(m, len); + m->m_len = len; p = mtod(m, struct sadb_x_nat_t_type *); bzero(p, len); @@ -3745,13 +3730,11 @@ key_setsadbxport(u_int16_t port, u_int16_t type) len = PFKEY_ALIGN8(sizeof(struct sadb_x_nat_t_port)); - m = key_alloc_mbuf(len); - if (!m || m->m_next) { /*XXX*/ - if (m) - m_freem(m); + m = m_get2(len, M_NOWAIT, MT_DATA, 0); + if (m == NULL) return (NULL); - } - + m_align(m, len); + m->m_len = len; p = mtod(m, struct sadb_x_nat_t_port *); bzero(p, len); @@ -3822,13 +3805,11 @@ key_setsadbxpolicy(u_int16_t type, u_int8_t dir, u_int32_t id) size_t len; len = PFKEY_ALIGN8(sizeof(struct sadb_x_policy)); - m = key_alloc_mbuf(len); - if (!m || m->m_next) { /*XXX*/ - if (m) - m_freem(m); - return NULL; - } - + m = m_get2(len, M_NOWAIT, MT_DATA, 0); + if (m == NULL) + return (NULL); + m_align(m, len); + m->m_len = len; p = mtod(m, struct sadb_x_policy *); bzero(p, len); @@ -6951,13 +6932,13 @@ key_expire(struct secasvar *sav) /* create lifetime extension (current and soft) */ len = PFKEY_ALIGN8(sizeof(*lt)) * 2; - m = key_alloc_mbuf(len); - if (!m || m->m_next) { /*XXX*/ - if (m) - m_freem(m); + m = m_get2(len, M_NOWAIT, MT_DATA, 0); + if (m == NULL) { error = ENOBUFS; goto fail; } + m_align(m, len); + m->m_len = len; bzero(mtod(m, caddr_t), len); lt = mtod(m, struct sadb_lifetime *); lt->sadb_lifetime_len = PFKEY_UNIT64(sizeof(struct sadb_lifetime)); @@ -7959,45 +7940,6 @@ key_sa_stir_iv(sav) key_randomfill(sav->iv, sav->ivlen); } -/* XXX too much? */ -static struct mbuf * -key_alloc_mbuf(l) - int l; -{ - struct mbuf *m = NULL, *n; - int len, t; - - len = l; - while (len > 0) { - MGET(n, M_NOWAIT, MT_DATA); - if (n && len > MLEN) - MCLGET(n, M_NOWAIT); - if (!n) { - m_freem(m); - return NULL; - } - - n->m_next = NULL; - n->m_len = 0; - n->m_len = M_TRAILINGSPACE(n); - /* use the bottom of mbuf, hoping we can prepend afterwards */ - if (n->m_len > len) { - t = (n->m_len - len) & ~(sizeof(long) - 1); - n->m_data += t; - n->m_len = len; - } - - len -= n->m_len; - - if (m) - m_cat(m, n); - else - m = n; - } - - return m; -} - /* * Take one of the kernel's security keys and convert it into a PF_KEY * structure within an mbuf, suitable for sending up to a waiting @@ -8022,9 +7964,11 @@ key_setkey(struct seckey *src, u_int16_t exttype) return NULL; len = PFKEY_ALIGN8(sizeof(struct sadb_key) + _KEYLEN(src)); - m = key_alloc_mbuf(len); + m = m_get2(len, M_NOWAIT, MT_DATA, 0); if (m == NULL) return NULL; + m_align(m, len); + m->m_len = len; p = mtod(m, struct sadb_key *); bzero(p, len); p->sadb_key_len = PFKEY_UNIT64(len); @@ -8059,9 +8003,11 @@ key_setlifetime(struct seclifetime *src, u_int16_t exttype) if (src == NULL) return NULL; - m = key_alloc_mbuf(len); + m = m_get2(len, M_NOWAIT, MT_DATA, 0); if (m == NULL) return m; + m_align(m, len); + m->m_len = len; p = mtod(m, struct sadb_lifetime *); bzero(p, len); -- cgit v1.1 From a10c1a7c72239c6be42dd7e90417b5caef91c8c7 Mon Sep 17 00:00:00 2001 From: glebius Date: Fri, 15 Mar 2013 10:21:18 +0000 Subject: Use m_get() and m_getcl() instead of compat macros. --- sys/kern/uipc_sockbuf.c | 4 ++-- sys/nfs/nfs_common.c | 4 ++-- sys/xdr/xdr_mbuf.c | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) (limited to 'sys') diff --git a/sys/kern/uipc_sockbuf.c b/sys/kern/uipc_sockbuf.c index 6325840..93c96bf 100644 --- a/sys/kern/uipc_sockbuf.c +++ b/sys/kern/uipc_sockbuf.c @@ -644,8 +644,8 @@ sbappendaddr_locked(struct sockbuf *sb, const struct sockaddr *asa, if (asa->sa_len > MLEN) return (0); #endif - MGET(m, M_NOWAIT, MT_SONAME); - if (m == 0) + m = m_get(M_NOWAIT, MT_SONAME); + if (m == NULL) return (0); m->m_len = asa->sa_len; bcopy(asa, mtod(m, caddr_t), asa->sa_len); diff --git a/sys/nfs/nfs_common.c b/sys/nfs/nfs_common.c index 83e7125..ad9e7a1 100644 --- a/sys/nfs/nfs_common.c +++ b/sys/nfs/nfs_common.c @@ -192,7 +192,7 @@ nfsm_disct(struct mbuf **mdp, caddr_t *dposp, int siz, int left, int how) } else if (siz > MHLEN) { panic("nfs S too big"); } else { - MGET(mp2, how, MT_DATA); + mp2 = m_get(how, MT_DATA); if (mp2 == NULL) return (NULL); mp2->m_len = siz; @@ -266,7 +266,7 @@ nfsm_build_xx(int s, struct mbuf **mb, caddr_t *bpos) void *ret; if (s > M_TRAILINGSPACE(*mb)) { - MGET(mb2, M_WAITOK, MT_DATA); + mb2 = m_get(M_WAITOK, MT_DATA); if (s > MLEN) panic("build > MLEN"); (*mb)->m_next = mb2; diff --git a/sys/xdr/xdr_mbuf.c b/sys/xdr/xdr_mbuf.c index b2394bc..690c2d2 100644 --- a/sys/xdr/xdr_mbuf.c +++ b/sys/xdr/xdr_mbuf.c @@ -123,7 +123,7 @@ xdrmbuf_getall(XDR *xdrs) if (m) m_adj(m, xdrs->x_handy); else - MGET(m, M_WAITOK, MT_DATA); + m = m_get(M_WAITOK, MT_DATA); return (m); } @@ -228,9 +228,10 @@ xdrmbuf_putbytes(XDR *xdrs, const char *addr, u_int len) if (xdrs->x_handy == m->m_len && M_TRAILINGSPACE(m) == 0) { if (!m->m_next) { - MGET(n, M_WAITOK, m->m_type); if (m->m_flags & M_EXT) - MCLGET(n, M_WAITOK); + n = m_getcl(M_WAITOK, m->m_type, 0); + else + n = m_get(M_WAITOK, m->m_type); m->m_next = n; } m = m->m_next; -- cgit v1.1 From 9b5b993aea1b3a3be06fefb010355b869ce49aa8 Mon Sep 17 00:00:00 2001 From: kib Date: Fri, 15 Mar 2013 11:16:12 +0000 Subject: Implement the helper function vn_io_fault_pgmove(), intended to use by the filesystem VOP_READ() and VOP_WRITE() implementations in the same way as vn_io_fault_uiomove() over the unmapped buffers. Helper provides the convenient wrapper over the pmap_copy_pages() for struct uio consumers, taking care of the TDP_UIOHELD situations. Sponsored by: The FreeBSD Foundation Tested by: pho MFC after: 2 weeks --- sys/kern/vfs_vnops.c | 39 +++++++++++++++++++++++++++++++++++++++ sys/sys/vnode.h | 2 ++ 2 files changed, 41 insertions(+) (limited to 'sys') diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c index d367340..cebc4f9 100644 --- a/sys/kern/vfs_vnops.c +++ b/sys/kern/vfs_vnops.c @@ -1122,6 +1122,45 @@ vn_io_fault_uiomove(char *data, int xfersize, struct uio *uio) return (error); } +int +vn_io_fault_pgmove(vm_page_t ma[], vm_offset_t offset, int xfersize, + struct uio *uio) +{ + struct thread *td; + vm_offset_t iov_base; + int cnt, pgadv; + + td = curthread; + if ((td->td_pflags & TDP_UIOHELD) == 0 || + uio->uio_segflg != UIO_USERSPACE) + return (uiomove_fromphys(ma, offset, xfersize, uio)); + + KASSERT(uio->uio_iovcnt == 1, ("uio_iovcnt %d", uio->uio_iovcnt)); + cnt = xfersize > uio->uio_resid ? uio->uio_resid : xfersize; + iov_base = (vm_offset_t)uio->uio_iov->iov_base; + switch (uio->uio_rw) { + case UIO_WRITE: + pmap_copy_pages(td->td_ma, iov_base & PAGE_MASK, ma, + offset, cnt); + break; + case UIO_READ: + pmap_copy_pages(ma, offset, td->td_ma, iov_base & PAGE_MASK, + cnt); + break; + } + pgadv = ((iov_base + cnt) >> PAGE_SHIFT) - (iov_base >> PAGE_SHIFT); + td->td_ma += pgadv; + KASSERT(td->td_ma_cnt >= pgadv, ("consumed pages %d %d", td->td_ma_cnt, + pgadv)); + td->td_ma_cnt -= pgadv; + uio->uio_iov->iov_base = (char *)(iov_base + cnt); + uio->uio_iov->iov_len -= cnt; + uio->uio_resid -= cnt; + uio->uio_offset += cnt; + return (0); +} + + /* * File table truncate routine. */ diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h index 0696edd..bc16acf 100644 --- a/sys/sys/vnode.h +++ b/sys/sys/vnode.h @@ -693,6 +693,8 @@ int vn_vget_ino(struct vnode *vp, ino_t ino, int lkflags, struct vnode **rvp); int vn_io_fault_uiomove(char *data, int xfersize, struct uio *uio); +int vn_io_fault_pgmove(vm_page_t ma[], vm_offset_t offset, int xfersize, + struct uio *uio); #define vn_rangelock_unlock(vp, cookie) \ rangelock_unlock(&(vp)->v_rl, (cookie), VI_MTX(vp)) -- cgit v1.1 From ace684a13202a960c7f5a15ff9d3604793bced5f Mon Sep 17 00:00:00 2001 From: glebius Date: Fri, 15 Mar 2013 12:33:23 +0000 Subject: Use m_getcl() instead of hand made allocation. Sponsored by: Nginx, Inc. --- sys/netinet6/ip6_input.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) (limited to 'sys') diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c index 45c4ff6..90a4f4e 100644 --- a/sys/netinet6/ip6_input.c +++ b/sys/netinet6/ip6_input.c @@ -497,21 +497,16 @@ ip6_input(struct mbuf *m) if (m && m->m_next != NULL && m->m_pkthdr.len < MCLBYTES) { struct mbuf *n; - MGETHDR(n, M_NOWAIT, MT_HEADER); - if (n) - M_MOVE_PKTHDR(n, m); - if (n && n->m_pkthdr.len > MHLEN) { - MCLGET(n, M_NOWAIT); - if ((n->m_flags & M_EXT) == 0) { - m_freem(n); - n = NULL; - } - } + if (m->m_pkthdr.len > MHLEN) + n = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); + else + n = m_gethdr(M_NOWAIT, MT_DATA); if (n == NULL) { m_freem(m); return; /* ENOBUFS */ } + M_MOVE_PKTHDR(n, m); m_copydata(m, 0, n->m_pkthdr.len, mtod(n, caddr_t)); n->m_len = n->m_pkthdr.len; m_freem(m); -- cgit v1.1 From 79cb402edbb474fc00a40f3fc46c9b5c2304d461 Mon Sep 17 00:00:00 2001 From: glebius Date: Fri, 15 Mar 2013 12:50:29 +0000 Subject: - Use m_getcl() instead of hand allocating. - Use m_get()/m_gethdr() instead of macros. - Remove superfluous cleaning of mbuf fields after allocation. Sponsored by: Nginx, Inc. --- sys/netinet6/icmp6.c | 33 +++++++++++++-------------------- sys/netinet6/ip6_mroute.c | 5 ++--- sys/netinet6/ip6_output.c | 11 ++++------- sys/netinet6/mld6.c | 6 +++--- 4 files changed, 22 insertions(+), 33 deletions(-) (limited to 'sys') diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c index 0cd6e5a..089dabe 100644 --- a/sys/netinet6/icmp6.c +++ b/sys/netinet6/icmp6.c @@ -581,7 +581,7 @@ icmp6_input(struct mbuf **mp, int *offp, int proto) const int maxlen = sizeof(*nip6) + sizeof(*nicmp6); int n0len; - MGETHDR(n, M_NOWAIT, n0->m_type); + n = m_gethdr(M_NOWAIT, n0->m_type); n0len = n0->m_pkthdr.len; /* save for use below */ if (n) M_MOVE_PKTHDR(n, n0); /* FIB copied. */ @@ -699,7 +699,7 @@ icmp6_input(struct mbuf **mp, int *offp, int proto) /* Give up remote */ break; } - MGETHDR(n, M_NOWAIT, m->m_type); + n = m_gethdr(M_NOWAIT, m->m_type); if (n && maxlen > MHLEN) { MCLGET(n, M_NOWAIT); if ((n->m_flags & M_EXT) == 0) { @@ -1495,7 +1495,7 @@ ni6_input(struct mbuf *m, int off) } /* allocate an mbuf to reply. */ - MGETHDR(n, M_NOWAIT, m->m_type); + n = m_gethdr(M_NOWAIT, m->m_type); if (n == NULL) { m_freem(m); return (NULL); @@ -1608,16 +1608,13 @@ ni6_nametodns(const char *name, int namelen, int old) else len = MCLBYTES; - /* because MAXHOSTNAMELEN is usually 256, we use cluster mbuf */ - MGET(m, M_NOWAIT, MT_DATA); - if (m && len > MLEN) { - MCLGET(m, M_NOWAIT); - if ((m->m_flags & M_EXT) == 0) - goto fail; - } - if (!m) + /* Because MAXHOSTNAMELEN is usually 256, we use cluster mbuf. */ + if (len > MLEN) + m = m_getcl(M_NOWAIT, MT_DATA, 0); + else + m = m_get(M_NOWAIT, MT_DATA); + if (m == NULL) goto fail; - m->m_next = NULL; if (old) { m->m_len = len; @@ -2063,7 +2060,7 @@ icmp6_rip6_input(struct mbuf **mp, int off) */ if ((m->m_flags & M_EXT) && m->m_next == NULL && m->m_len <= MHLEN) { - MGET(n, M_NOWAIT, m->m_type); + n = m_get(M_NOWAIT, m->m_type); if (n != NULL) { if (m_dup_pkthdr(n, m, M_NOWAIT)) { bcopy(m->m_data, n->m_data, @@ -2113,7 +2110,7 @@ icmp6_rip6_input(struct mbuf **mp, int off) m->m_len <= MHLEN) { struct mbuf *n; - MGET(n, M_NOWAIT, m->m_type); + n = m_get(M_NOWAIT, m->m_type); if (n != NULL) { if (m_dup_pkthdr(n, m, M_NOWAIT)) { bcopy(m->m_data, n->m_data, m->m_len); @@ -2592,14 +2589,10 @@ icmp6_redirect_output(struct mbuf *m0, struct rtentry *rt) #if IPV6_MMTU >= MCLBYTES # error assumption failed about IPV6_MMTU and MCLBYTES #endif - MGETHDR(m, M_NOWAIT, MT_HEADER); - if (m && IPV6_MMTU >= MHLEN) - MCLGET(m, M_NOWAIT); - if (!m) + m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); + if (m == NULL) goto fail; M_SETFIB(m, rt->rt_fibnum); - m->m_pkthdr.rcvif = NULL; - m->m_len = 0; maxlen = M_TRAILINGSPACE(m); maxlen = min(IPV6_MMTU, maxlen); /* just for safety */ diff --git a/sys/netinet6/ip6_mroute.c b/sys/netinet6/ip6_mroute.c index a221110..aeeb644 100644 --- a/sys/netinet6/ip6_mroute.c +++ b/sys/netinet6/ip6_mroute.c @@ -1698,11 +1698,10 @@ register_send(struct ip6_hdr *ip6, struct mif6 *mif, struct mbuf *m) #endif ++pim6stat.pim6s_snd_registers; - /* Make a copy of the packet to send to the user level process */ - MGETHDR(mm, M_NOWAIT, MT_HEADER); + /* Make a copy of the packet to send to the user level process. */ + mm = m_gethdr(M_NOWAIT, MT_DATA); if (mm == NULL) return (ENOBUFS); - mm->m_pkthdr.rcvif = NULL; mm->m_data += max_linkhdr; mm->m_len = sizeof(struct ip6_hdr); diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c index 5e8c11f..cb1feab 100644 --- a/sys/netinet6/ip6_output.c +++ b/sys/netinet6/ip6_output.c @@ -774,9 +774,7 @@ again: /* * XXX: ip6_mforward expects that rcvif is NULL * when it is called from the originating path. - * However, it is not always the case, since - * some versions of MGETHDR() does not - * initialize the field. + * However, it may not always be the case. */ m->m_pkthdr.rcvif = NULL; if (ip6_mforward(ip6, ifp, m) != 0) { @@ -1122,13 +1120,12 @@ passout: */ m0 = m; for (off = hlen; off < tlen; off += len) { - MGETHDR(m, M_NOWAIT, MT_HEADER); + m = m_gethdr(M_NOWAIT, MT_DATA); if (!m) { error = ENOBUFS; V_ip6stat.ip6s_odropped++; goto sendorfree; } - m->m_pkthdr.rcvif = NULL; m->m_flags = m0->m_flags & M_COPYFLAGS; /* incl. FIB */ *mnext = m; mnext = &m->m_nextpkt; @@ -3045,8 +3042,8 @@ ip6_splithdr(struct mbuf *m, struct ip6_exthdrs *exthdrs) ip6 = mtod(m, struct ip6_hdr *); if (m->m_len > sizeof(*ip6)) { - MGETHDR(mh, M_NOWAIT, MT_HEADER); - if (mh == 0) { + mh = m_gethdr(M_NOWAIT, MT_DATA); + if (mh == NULL) { m_freem(m); return ENOBUFS; } diff --git a/sys/netinet6/mld6.c b/sys/netinet6/mld6.c index 0d9e300..560e8d6 100644 --- a/sys/netinet6/mld6.c +++ b/sys/netinet6/mld6.c @@ -1799,13 +1799,13 @@ mld_v1_transmit_report(struct in6_multi *in6m, const int type) ia = in6ifa_ifpforlinklocal(ifp, IN6_IFF_NOTREADY|IN6_IFF_ANYCAST); /* ia may be NULL if link-local address is tentative. */ - MGETHDR(mh, M_NOWAIT, MT_HEADER); + mh = m_gethdr(M_NOWAIT, MT_DATA); if (mh == NULL) { if (ia != NULL) ifa_free(&ia->ia_ifa); return (ENOMEM); } - MGET(md, M_NOWAIT, MT_DATA); + md = m_get(M_NOWAIT, MT_DATA); if (md == NULL) { m_free(mh); if (ia != NULL) @@ -3173,7 +3173,7 @@ mld_v2_encap_report(struct ifnet *ifp, struct mbuf *m) if (ia == NULL) CTR1(KTR_MLD, "%s: warning: ia is NULL", __func__); - MGETHDR(mh, M_NOWAIT, MT_HEADER); + mh = m_gethdr(M_NOWAIT, MT_DATA); if (mh == NULL) { if (ia != NULL) ifa_free(&ia->ia_ifa); -- cgit v1.1 From 76306b34651c9b81d0d3bf6b97757c7373d3d636 Mon Sep 17 00:00:00 2001 From: glebius Date: Fri, 15 Mar 2013 12:52:59 +0000 Subject: - Use m_getcl() instead of hand allocating. - Convert panic() to KASSERT. - Remove superfluous cleaning of mbuf fields after allocation. - Add comment on possible use of m_get2() here. Sponsored by: Nginx, Inc. --- sys/net/rtsock.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'sys') diff --git a/sys/net/rtsock.c b/sys/net/rtsock.c index 9a08a76..c3781d5 100644 --- a/sys/net/rtsock.c +++ b/sys/net/rtsock.c @@ -1118,20 +1118,17 @@ rt_msg1(int type, struct rt_addrinfo *rtinfo) default: len = sizeof(struct rt_msghdr); } - if (len > MCLBYTES) - panic("rt_msg1"); - m = m_gethdr(M_NOWAIT, MT_DATA); - if (m && len > MHLEN) { - MCLGET(m, M_NOWAIT); - if ((m->m_flags & M_EXT) == 0) { - m_free(m); - m = NULL; - } - } + + /* XXXGL: can we use MJUMPAGESIZE cluster here? */ + KASSERT(len <= MCLBYTES, ("%s: message too big", __func__)); + if (len > MHLEN) + m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); + else + m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) return (m); + m->m_pkthdr.len = m->m_len = len; - m->m_pkthdr.rcvif = NULL; rtm = mtod(m, struct rt_msghdr *); bzero((caddr_t)rtm, len); for (i = 0; i < RTAX_MAX; i++) { -- cgit v1.1 From d9a8b8e5e610631cff464a76929f0ff4de7dcff2 Mon Sep 17 00:00:00 2001 From: glebius Date: Fri, 15 Mar 2013 12:53:53 +0000 Subject: - Use m_getcl() instead of hand allocating. Sponsored by: Nginx, Inc. --- sys/netinet/tcp_output.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to 'sys') diff --git a/sys/netinet/tcp_output.c b/sys/netinet/tcp_output.c index ac7ac31..e291bb3 100644 --- a/sys/netinet/tcp_output.c +++ b/sys/netinet/tcp_output.c @@ -842,23 +842,19 @@ send: TCPSTAT_INC(tcps_sndpack); TCPSTAT_ADD(tcps_sndbyte, len); } - MGETHDR(m, M_NOWAIT, MT_DATA); +#ifdef INET6 + if (MHLEN < hdrlen + max_linkhdr) + m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); + else +#endif + m = m_gethdr(M_NOWAIT, MT_DATA); + if (m == NULL) { SOCKBUF_UNLOCK(&so->so_snd); error = ENOBUFS; goto out; } -#ifdef INET6 - if (MHLEN < hdrlen + max_linkhdr) { - MCLGET(m, M_NOWAIT); - if ((m->m_flags & M_EXT) == 0) { - SOCKBUF_UNLOCK(&so->so_snd); - m_freem(m); - error = ENOBUFS; - goto out; - } - } -#endif + m->m_data += max_linkhdr; m->m_len = hdrlen; -- cgit v1.1 From b37af62b9e1d9dc462e88e964b0c1012f8144495 Mon Sep 17 00:00:00 2001 From: glebius Date: Fri, 15 Mar 2013 12:55:30 +0000 Subject: Use m_get/m_gethdr instead of compat macros. Sponsored by: Nginx, Inc. --- sys/compat/ndis/kern_ndis.c | 8 ++------ sys/net/bridgestp.c | 4 ++-- sys/net/if_gre.c | 3 +-- sys/netinet/ip_carp.c | 4 ++-- sys/netinet/ip_input.c | 2 +- sys/netinet/ip_mroute.c | 7 +++---- sys/netinet/ip_options.c | 2 +- sys/netinet/ip_output.c | 4 ++-- sys/netinet/tcp_subr.c | 2 +- sys/netpfil/pf/pf.c | 2 +- 10 files changed, 16 insertions(+), 22 deletions(-) (limited to 'sys') diff --git a/sys/compat/ndis/kern_ndis.c b/sys/compat/ndis/kern_ndis.c index e094997..71b4b80 100644 --- a/sys/compat/ndis/kern_ndis.c +++ b/sys/compat/ndis/kern_ndis.c @@ -656,13 +656,9 @@ ndis_ptom(m0, p) for (buf = priv->npp_head; buf != NULL; buf = buf->mdl_next) { if (buf == priv->npp_head) -#ifdef MT_HEADER - MGETHDR(m, M_NOWAIT, MT_HEADER); -#else - MGETHDR(m, M_NOWAIT, MT_DATA); -#endif + m = m_gethdr(M_NOWAIT, MT_DATA); else - MGET(m, M_NOWAIT, MT_DATA); + m = m_get(M_NOWAIT, MT_DATA); if (m == NULL) { m_freem(*m0); *m0 = NULL; diff --git a/sys/net/bridgestp.c b/sys/net/bridgestp.c index 43dd005..07cf219 100644 --- a/sys/net/bridgestp.c +++ b/sys/net/bridgestp.c @@ -234,7 +234,7 @@ bstp_transmit_tcn(struct bstp_state *bs, struct bstp_port *bp) if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) return; - MGETHDR(m, M_NOWAIT, MT_DATA); + m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) return; @@ -348,7 +348,7 @@ bstp_send_bpdu(struct bstp_state *bs, struct bstp_port *bp, if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) return; - MGETHDR(m, M_NOWAIT, MT_DATA); + m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) return; diff --git a/sys/net/if_gre.c b/sys/net/if_gre.c index 27e58d8..48255b0 100644 --- a/sys/net/if_gre.c +++ b/sys/net/if_gre.c @@ -384,8 +384,7 @@ gre_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, mob_h.hcrc = gre_in_cksum((u_int16_t *)&mob_h, msiz); if ((m->m_data - msiz) < m->m_pktdat) { - /* need new mbuf */ - MGETHDR(m0, M_NOWAIT, MT_DATA); + m0 = m_gethdr(M_NOWAIT, MT_DATA); if (m0 == NULL) { _IF_DROP(&ifp->if_snd); m_freem(m); diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c index ae3da96..eaff4f6 100644 --- a/sys/netinet/ip_carp.c +++ b/sys/netinet/ip_carp.c @@ -764,7 +764,7 @@ carp_send_ad_locked(struct carp_softc *sc) if (sc->sc_naddrs) { struct ip *ip; - MGETHDR(m, M_NOWAIT, MT_HEADER); + m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) { CARPSTATS_INC(carps_onomem); goto resched; @@ -832,7 +832,7 @@ carp_send_ad_locked(struct carp_softc *sc) if (sc->sc_naddrs6) { struct ip6_hdr *ip6; - MGETHDR(m, M_NOWAIT, MT_HEADER); + m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) { CARPSTATS_INC(carps_onomem); goto resched; diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c index 555b5d0..1f1122c 100644 --- a/sys/netinet/ip_input.c +++ b/sys/netinet/ip_input.c @@ -1406,7 +1406,7 @@ ip_forward(struct mbuf *m, int srcrt) * assume exclusive access to the IP header in `m', so any * data in a cluster may change before we reach icmp_error(). */ - MGETHDR(mcopy, M_NOWAIT, m->m_type); + mcopy = m_gethdr(M_NOWAIT, m->m_type); if (mcopy != NULL && !m_dup_pkthdr(mcopy, m, M_NOWAIT)) { /* * It's probably ok if the pkthdr dup fails (because diff --git a/sys/netinet/ip_mroute.c b/sys/netinet/ip_mroute.c index 6228933..db8238f 100644 --- a/sys/netinet/ip_mroute.c +++ b/sys/netinet/ip_mroute.c @@ -2083,13 +2083,12 @@ bw_upcalls_send(void) * Allocate a new mbuf, initialize it with the header and * the payload for the pending calls. */ - MGETHDR(m, M_NOWAIT, MT_DATA); + m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) { log(LOG_WARNING, "bw_upcalls_send: cannot allocate mbuf\n"); return; } - m->m_len = m->m_pkthdr.len = 0; m_copyback(m, 0, sizeof(struct igmpmsg), (caddr_t)&igmpmsg); m_copyback(m, sizeof(struct igmpmsg), len, (caddr_t)&V_bw_upcalls[0]); @@ -2430,7 +2429,7 @@ pim_register_send_upcall(struct ip *ip, struct vif *vifp, /* * Add a new mbuf with an upcall header */ - MGETHDR(mb_first, M_NOWAIT, MT_DATA); + mb_first = m_gethdr(M_NOWAIT, MT_DATA); if (mb_first == NULL) { m_freem(mb_copy); return ENOBUFS; @@ -2488,7 +2487,7 @@ pim_register_send_rp(struct ip *ip, struct vif *vifp, struct mbuf *mb_copy, /* * Add a new mbuf with the encapsulating header */ - MGETHDR(mb_first, M_NOWAIT, MT_DATA); + mb_first = m_gethdr(M_NOWAIT, MT_DATA); if (mb_first == NULL) { m_freem(mb_copy); return ENOBUFS; diff --git a/sys/netinet/ip_options.c b/sys/netinet/ip_options.c index 239c699..55cc81f 100644 --- a/sys/netinet/ip_options.c +++ b/sys/netinet/ip_options.c @@ -495,7 +495,7 @@ ip_insertoptions(struct mbuf *m, struct mbuf *opt, int *phlen) if (p->ipopt_dst.s_addr) ip->ip_dst = p->ipopt_dst; if (m->m_flags & M_EXT || m->m_data - optlen < m->m_pktdat) { - MGETHDR(n, M_NOWAIT, MT_DATA); + n = m_gethdr(M_NOWAIT, MT_DATA); if (n == NULL) { *phlen = 0; return (m); diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c index 166bef5..fd418e9 100644 --- a/sys/netinet/ip_output.c +++ b/sys/netinet/ip_output.c @@ -784,7 +784,7 @@ smart_frag_failure: struct mbuf *m; int mhlen = sizeof (struct ip); - MGETHDR(m, M_NOWAIT, MT_DATA); + m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) { error = ENOBUFS; IPSTAT_INC(ips_odropped); @@ -951,7 +951,7 @@ ip_ctloutput(struct socket *so, struct sockopt *sopt) error = EMSGSIZE; break; } - MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); + m = m_get(sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); if (m == NULL) { error = ENOBUFS; break; diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c index 3c9dd34..24e4db2 100644 --- a/sys/netinet/tcp_subr.c +++ b/sys/netinet/tcp_subr.c @@ -1860,7 +1860,7 @@ ipsec_hdrsiz_tcp(struct tcpcb *tp) if ((tp == NULL) || ((inp = tp->t_inpcb) == NULL)) return (0); - MGETHDR(m, M_NOWAIT, MT_DATA); + m = m_gethdr(M_NOWAIT, MT_DATA); if (!m) return (0); diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c index 6667d7a..168ad5c 100644 --- a/sys/netpfil/pf/pf.c +++ b/sys/netpfil/pf/pf.c @@ -2168,7 +2168,7 @@ pf_send_tcp(struct mbuf *replyto, const struct pf_rule *r, sa_family_t af, pfse = malloc(sizeof(*pfse), M_PFTEMP, M_NOWAIT); if (pfse == NULL) return; - m = m_gethdr(M_NOWAIT, MT_HEADER); + m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) { free(pfse, M_PFTEMP); return; -- cgit v1.1 From 28990b868636d278559e5e9082c3b609f85151f7 Mon Sep 17 00:00:00 2001 From: kib Date: Fri, 15 Mar 2013 12:57:30 +0000 Subject: Add my copyright for the 2012 year work, in particular vn_io_fault() and f_offset locking. Add required Foundation notice for r248319. Sponsored by: The FreeBSD Foundation MFC after: 2 weeks --- sys/kern/vfs_vnops.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'sys') diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c index cebc4f9..95151c9 100644 --- a/sys/kern/vfs_vnops.c +++ b/sys/kern/vfs_vnops.c @@ -7,6 +7,11 @@ * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * + * Copyright (c) 2012 Konstantin Belousov + * Copyright (c) 2013 The FreeBSD Foundation + * Portions of this software were developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: -- cgit v1.1 From fcc62f81475d59413520237c6004a5d311c02b16 Mon Sep 17 00:00:00 2001 From: glebius Date: Fri, 15 Mar 2013 13:10:06 +0000 Subject: We can, and should use M_WAITOK here. Sponsored by: Nginx, Inc. --- sys/netinet/igmp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sys') diff --git a/sys/netinet/igmp.c b/sys/netinet/igmp.c index e87da5e..71aa31d 100644 --- a/sys/netinet/igmp.c +++ b/sys/netinet/igmp.c @@ -523,7 +523,7 @@ igmp_ra_alloc(void) struct mbuf *m; struct ipoption *p; - MGET(m, M_NOWAIT, MT_DATA); + m = m_get(M_WAITOK, MT_DATA); p = mtod(m, struct ipoption *); p->ipopt_dst.s_addr = INADDR_ANY; p->ipopt_list[0] = IPOPT_RA; /* Router Alert Option */ -- cgit v1.1 From f07362f54e91804d5113a9973ea23cf1feab6fbf Mon Sep 17 00:00:00 2001 From: glebius Date: Fri, 15 Mar 2013 13:48:53 +0000 Subject: - Use m_getcl() instead of hand allocating. - Do not calculate constant length values at run time, CTASSERT() their sanity. - Remove superfluous cleaning of mbuf fields after allocation. - Replace compat macros with function calls. Sponsored by: Nginx, Inc. --- sys/netinet6/icmp6.c | 71 +++++++++++++++++------------------------------ sys/netinet6/ip6_input.c | 22 ++++----------- sys/netinet6/ip6_output.c | 36 ++++++++---------------- sys/netinet6/nd6_nbr.c | 26 ++++++----------- 4 files changed, 51 insertions(+), 104 deletions(-) (limited to 'sys') diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c index 089dabe..8772779 100644 --- a/sys/netinet6/icmp6.c +++ b/sys/netinet6/icmp6.c @@ -578,25 +578,18 @@ icmp6_input(struct mbuf **mp, int *offp, int proto) if ((n->m_flags & M_EXT) != 0 || n->m_len < off + sizeof(struct icmp6_hdr)) { struct mbuf *n0 = n; - const int maxlen = sizeof(*nip6) + sizeof(*nicmp6); int n0len; + CTASSERT(sizeof(*nip6) + sizeof(*nicmp6) <= MHLEN); n = m_gethdr(M_NOWAIT, n0->m_type); - n0len = n0->m_pkthdr.len; /* save for use below */ - if (n) - M_MOVE_PKTHDR(n, n0); /* FIB copied. */ - if (n && maxlen >= MHLEN) { - MCLGET(n, M_NOWAIT); - if ((n->m_flags & M_EXT) == 0) { - m_free(n); - n = NULL; - } - } if (n == NULL) { /* Give up remote */ m_freem(n0); break; } + + m_move_pkthdr(n, n0); /* FIB copied. */ + n0len = n0->m_pkthdr.len; /* save for use below */ /* * Copy IPv6 and ICMPv6 only. */ @@ -683,7 +676,7 @@ icmp6_input(struct mbuf **mp, int *offp, int proto) } else { struct prison *pr; u_char *p; - int maxlen, maxhlen, hlen; + int maxhlen, hlen; /* * XXX: this combination of flags is pointless, @@ -694,20 +687,14 @@ icmp6_input(struct mbuf **mp, int *offp, int proto) if (code != 0) goto badcode; - maxlen = sizeof(*nip6) + sizeof(*nicmp6) + 4; - if (maxlen >= MCLBYTES) { + + CTASSERT(sizeof(*nip6) + sizeof(*nicmp6) + 4 <= MHLEN); + n = m_gethdr(M_NOWAIT, m->m_type); + if (n == NULL) { /* Give up remote */ break; } - n = m_gethdr(M_NOWAIT, m->m_type); - if (n && maxlen > MHLEN) { - MCLGET(n, M_NOWAIT); - if ((n->m_flags & M_EXT) == 0) { - m_free(n); - n = NULL; - } - } - if (n && !m_dup_pkthdr(n, m, M_NOWAIT)) { + if (!m_dup_pkthdr(n, m, M_NOWAIT)) { /* * Previous code did a blind M_COPY_PKTHDR * and said "just for rcvif". If true, then @@ -718,13 +705,8 @@ icmp6_input(struct mbuf **mp, int *offp, int proto) m_free(n); n = NULL; } - if (n == NULL) { - /* Give up remote */ - break; - } - n->m_pkthdr.rcvif = NULL; - n->m_len = 0; - maxhlen = M_TRAILINGSPACE(n) - maxlen; + maxhlen = M_TRAILINGSPACE(n) - + (sizeof(*nip6) + sizeof(*nicmp6) + 4); pr = curthread->td_ucred->cr_prison; mtx_lock(&pr->pr_mtx); hlen = strlen(pr->pr_hostname); @@ -1494,26 +1476,23 @@ ni6_input(struct mbuf *m, int off) break; } - /* allocate an mbuf to reply. */ - n = m_gethdr(M_NOWAIT, m->m_type); + /* Allocate an mbuf to reply. */ + if (replylen > MCLBYTES) { + /* + * XXX: should we try to allocate more? But MCLBYTES + * is probably much larger than IPV6_MMTU... + */ + goto bad; + } + if (replylen > MHLEN) + n = m_getcl(M_NOWAIT, m->m_type, M_PKTHDR); + else + n = m_gethdr(M_NOWAIT, m->m_type); if (n == NULL) { m_freem(m); return (NULL); } - M_MOVE_PKTHDR(n, m); /* just for recvif and FIB */ - if (replylen > MHLEN) { - if (replylen > MCLBYTES) { - /* - * XXX: should we try to allocate more? But MCLBYTES - * is probably much larger than IPV6_MMTU... - */ - goto bad; - } - MCLGET(n, M_NOWAIT); - if ((n->m_flags & M_EXT) == 0) { - goto bad; - } - } + m_move_pkthdr(n, m); /* just for recvif and FIB */ n->m_pkthdr.len = n->m_len = replylen; /* copy mbuf header and IPv6 + Node Information base headers */ diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c index 90a4f4e..e037737 100644 --- a/sys/netinet6/ip6_input.c +++ b/sys/netinet6/ip6_input.c @@ -506,7 +506,7 @@ ip6_input(struct mbuf *m) return; /* ENOBUFS */ } - M_MOVE_PKTHDR(n, m); + m_move_pkthdr(n, m); m_copydata(m, 0, n->m_pkthdr.len, mtod(n, caddr_t)); n->m_len = n->m_pkthdr.len; m_freem(m); @@ -1662,22 +1662,12 @@ ip6_pullexthdr(struct mbuf *m, size_t off, int nxt) else elen = (ip6e.ip6e_len + 1) << 3; - MGET(n, M_NOWAIT, MT_DATA); - if (n && elen >= MLEN) { - MCLGET(n, M_NOWAIT); - if ((n->m_flags & M_EXT) == 0) { - m_free(n); - n = NULL; - } - } - if (!n) - return NULL; - - n->m_len = 0; - if (elen >= M_TRAILINGSPACE(n)) { - m_free(n); + if (elen > MLEN) + n = m_getcl(M_NOWAIT, MT_DATA, 0); + else + n = m_get(M_NOWAIT, MT_DATA); + if (n == NULL) return NULL; - } m_copydata(m, off, elen, mtod(n, caddr_t)); n->m_len = elen; diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c index cb1feab..e5042d2 100644 --- a/sys/netinet6/ip6_output.c +++ b/sys/netinet6/ip6_output.c @@ -1219,17 +1219,12 @@ ip6_copyexthdr(struct mbuf **mp, caddr_t hdr, int hlen) if (hlen > MCLBYTES) return (ENOBUFS); /* XXX */ - MGET(m, M_NOWAIT, MT_DATA); - if (!m) + if (hlen > MLEN) + m = m_getcl(M_NOWAIT, MT_DATA, 0); + else + m = m_get(M_NOWAIT, MT_DATA); + if (m == NULL) return (ENOBUFS); - - if (hlen > MLEN) { - MCLGET(m, M_NOWAIT); - if ((m->m_flags & M_EXT) == 0) { - m_free(m); - return (ENOBUFS); - } - } m->m_len = hlen; if (hdr) bcopy(hdr, mtod(m, caddr_t), hlen); @@ -1257,8 +1252,8 @@ ip6_insert_jumboopt(struct ip6_exthdrs *exthdrs, u_int32_t plen) * Otherwise, use it to store the options. */ if (exthdrs->ip6e_hbh == 0) { - MGET(mopt, M_NOWAIT, MT_DATA); - if (mopt == 0) + mopt = m_get(M_NOWAIT, MT_DATA); + if (mopt == NULL) return (ENOBUFS); mopt->m_len = JUMBOOPTLEN; optbuf = mtod(mopt, u_char *); @@ -1289,15 +1284,8 @@ ip6_insert_jumboopt(struct ip6_exthdrs *exthdrs, u_int32_t plen) * As a consequence, we must always prepare a cluster * at this point. */ - MGET(n, M_NOWAIT, MT_DATA); - if (n) { - MCLGET(n, M_NOWAIT); - if ((n->m_flags & M_EXT) == 0) { - m_freem(n); - n = NULL; - } - } - if (!n) + n = m_getcl(M_NOWAIT, MT_DATA, 0); + if (n == NULL) return (ENOBUFS); n->m_len = oldoptlen + JUMBOOPTLEN; bcopy(mtod(mopt, caddr_t), mtod(n, caddr_t), @@ -1366,8 +1354,8 @@ ip6_insertfraghdr(struct mbuf *m0, struct mbuf *m, int hlen, /* allocate a new mbuf for the fragment header */ struct mbuf *mfrg; - MGET(mfrg, M_NOWAIT, MT_DATA); - if (mfrg == 0) + mfrg = m_get(M_NOWAIT, MT_DATA); + if (mfrg == NULL) return (ENOBUFS); mfrg->m_len = sizeof(struct ip6_frag); *frghdrp = mtod(mfrg, struct ip6_frag *); @@ -3047,7 +3035,7 @@ ip6_splithdr(struct mbuf *m, struct ip6_exthdrs *exthdrs) m_freem(m); return ENOBUFS; } - M_MOVE_PKTHDR(mh, m); + m_move_pkthdr(mh, m); MH_ALIGN(mh, sizeof(*ip6)); m->m_len -= sizeof(*ip6); m->m_data += sizeof(*ip6); diff --git a/sys/netinet6/nd6_nbr.c b/sys/netinet6/nd6_nbr.c index 7fe75bf..532c9b8 100644 --- a/sys/netinet6/nd6_nbr.c +++ b/sys/netinet6/nd6_nbr.c @@ -419,17 +419,12 @@ nd6_ns_output(struct ifnet *ifp, const struct in6_addr *daddr6, return; } - MGETHDR(m, M_NOWAIT, MT_DATA); - if (m && max_linkhdr + maxlen >= MHLEN) { - MCLGET(m, M_NOWAIT); - if ((m->m_flags & M_EXT) == 0) { - m_free(m); - m = NULL; - } - } + if (max_linkhdr + maxlen > MHLEN) + m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); + else + m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) return; - m->m_pkthdr.rcvif = NULL; bzero(&ro, sizeof(ro)); @@ -997,17 +992,12 @@ nd6_na_output_fib(struct ifnet *ifp, const struct in6_addr *daddr6_0, return; } - MGETHDR(m, M_NOWAIT, MT_DATA); - if (m && max_linkhdr + maxlen >= MHLEN) { - MCLGET(m, M_NOWAIT); - if ((m->m_flags & M_EXT) == 0) { - m_free(m); - m = NULL; - } - } + if (max_linkhdr + maxlen > MHLEN) + m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); + else + m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) return; - m->m_pkthdr.rcvif = NULL; M_SETFIB(m, fibnum); if (IN6_IS_ADDR_MULTICAST(&daddr6)) { -- cgit v1.1 From 284e554a670b378e0cc0b7f67224d8a6a6f5af76 Mon Sep 17 00:00:00 2001 From: kib Date: Fri, 15 Mar 2013 14:01:37 +0000 Subject: Separate the copyright lines and the informational block by a blank line. Requested by: joel MFC after: 2 weeks --- sys/kern/vfs_vnops.c | 1 + 1 file changed, 1 insertion(+) (limited to 'sys') diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c index 95151c9..c8f832f 100644 --- a/sys/kern/vfs_vnops.c +++ b/sys/kern/vfs_vnops.c @@ -9,6 +9,7 @@ * * Copyright (c) 2012 Konstantin Belousov * Copyright (c) 2013 The FreeBSD Foundation + * * Portions of this software were developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * -- cgit v1.1 From 9613a0ac27804da699d33959a2f1f0618f34abc4 Mon Sep 17 00:00:00 2001 From: rstone Date: Fri, 15 Mar 2013 19:58:44 +0000 Subject: Add support for Exar XR17V358 8-port serial device to puc(4) Reviewed by: marius Sponsored by: Sandvine Inc. MFC after: 1 week --- sys/dev/puc/pucdata.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'sys') diff --git a/sys/dev/puc/pucdata.c b/sys/dev/puc/pucdata.c index 6d933e8..d82f031 100644 --- a/sys/dev/puc/pucdata.c +++ b/sys/dev/puc/pucdata.c @@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$"); static puc_config_f puc_config_amc; static puc_config_f puc_config_diva; static puc_config_f puc_config_exar; +static puc_config_f puc_config_exar_pcie; static puc_config_f puc_config_icbook; static puc_config_f puc_config_moxa; static puc_config_f puc_config_oxford_pcie; @@ -630,6 +631,14 @@ const struct puc_cfg puc_pci_devices[] = { PUC_PORT_8S, 0x10, 0, -1, }, + /* The XR17V358 uses the 125MHz PCIe clock as its reference clock. */ + { 0x13a8, 0x0358, 0xffff, 0, + "Exar XR17V358", + 125000000, + PUC_PORT_8S, 0x10, 0, -1, + .config_function = puc_config_exar_pcie + }, + { 0x13fe, 0x1600, 0x1602, 0x0002, "Advantech PCI-1602", DEFAULT_RCLK * 8, @@ -1186,6 +1195,17 @@ puc_config_exar(struct puc_softc *sc, enum puc_cfg_cmd cmd, int port, } static int +puc_config_exar_pcie(struct puc_softc *sc, enum puc_cfg_cmd cmd, int port, + intptr_t *res) +{ + if (cmd == PUC_CFG_GET_OFS) { + *res = port * 0x400; + return (0); + } + return (ENXIO); +} + +static int puc_config_icbook(struct puc_softc *sc, enum puc_cfg_cmd cmd, int port, intptr_t *res) { -- cgit v1.1 From 90b1ba3bc42c541bb65e1492274cdbdeca8d1dea Mon Sep 17 00:00:00 2001 From: adrian Date: Fri, 15 Mar 2013 20:00:08 +0000 Subject: Fix two bugs: * when pulling frames off of the TID queue, the ATH_TID_REMOVE() macro decrements the axq_depth field. So don't do it twice. * in ath_tx_comp_cleanup_aggr(), bf wasn't being reset to bf_first before walking the buffer list to complete buffers; so those buffers will leak. --- sys/dev/ath/if_ath_tx.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'sys') diff --git a/sys/dev/ath/if_ath_tx.c b/sys/dev/ath/if_ath_tx.c index 97e6967..3fe2777 100644 --- a/sys/dev/ath/if_ath_tx.c +++ b/sys/dev/ath/if_ath_tx.c @@ -3757,7 +3757,8 @@ ath_tx_tid_cleanup(struct ath_softc *sc, struct ath_node *an, int tid) if (bf->bf_state.bfs_isretried) { bf_next = TAILQ_NEXT(bf, bf_list); ATH_TID_REMOVE(atid, bf, bf_list); - atid->axq_depth--; + // Don't need this anymore; ATH_TID_REMOVE() decrements it for us + //atid->axq_depth--; if (bf->bf_state.bfs_dobaw) { ath_tx_update_baw(sc, an, atid, bf); if (! bf->bf_state.bfs_addedbaw) @@ -4140,11 +4141,10 @@ ath_tx_comp_cleanup_aggr(struct ath_softc *sc, struct ath_buf *bf_first) int tid = bf_first->bf_state.bfs_tid; struct ath_tid *atid = &an->an_tid[tid]; - bf = bf_first; - ATH_TX_LOCK(sc); /* update incomp */ + bf = bf_first; while (bf) { atid->incomp--; bf = bf->bf_next; @@ -4160,12 +4160,17 @@ ath_tx_comp_cleanup_aggr(struct ath_softc *sc, struct ath_buf *bf_first) /* Send BAR if required */ /* XXX why would we send a BAR when transitioning to non-aggregation? */ + /* + * XXX TODO: we should likely just tear down the BAR state here, + * rather than sending a BAR. + */ if (ath_tx_tid_bar_tx_ready(sc, atid)) ath_tx_tid_bar_tx(sc, atid); ATH_TX_UNLOCK(sc); /* Handle frame completion */ + bf = bf_first; while (bf) { bf_next = bf->bf_next; ath_tx_default_comp(sc, bf, 1); -- cgit v1.1 From e3bfa0687cf43740ac4bf886a8089374386b1d0d Mon Sep 17 00:00:00 2001 From: adrian Date: Fri, 15 Mar 2013 20:22:20 +0000 Subject: Why'd I keep this here? remove it entirely now. --- sys/dev/ath/if_ath_tx.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'sys') diff --git a/sys/dev/ath/if_ath_tx.c b/sys/dev/ath/if_ath_tx.c index 3fe2777..c7b0fc1 100644 --- a/sys/dev/ath/if_ath_tx.c +++ b/sys/dev/ath/if_ath_tx.c @@ -3757,8 +3757,6 @@ ath_tx_tid_cleanup(struct ath_softc *sc, struct ath_node *an, int tid) if (bf->bf_state.bfs_isretried) { bf_next = TAILQ_NEXT(bf, bf_list); ATH_TID_REMOVE(atid, bf, bf_list); - // Don't need this anymore; ATH_TID_REMOVE() decrements it for us - //atid->axq_depth--; if (bf->bf_state.bfs_dobaw) { ath_tx_update_baw(sc, an, atid, bf); if (! bf->bf_state.bfs_addedbaw) -- cgit v1.1 From 552a9fbc055835ad0b4dadbb343f5fa16f27cbdf Mon Sep 17 00:00:00 2001 From: pjd Date: Fri, 15 Mar 2013 23:00:13 +0000 Subject: Sort syscalls properly. --- sys/kern/capabilities.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sys') diff --git a/sys/kern/capabilities.conf b/sys/kern/capabilities.conf index 71c22bf..be01c9d 100644 --- a/sys/kern/capabilities.conf +++ b/sys/kern/capabilities.conf @@ -446,9 +446,9 @@ olio_listio ## Operations relative to directory capabilities. ## faccessat -fstatat fchmodat fchownat +fstatat futimesat linkat mkdirat -- cgit v1.1 From 112fb744f44f1dc3da68930c1fa793d7b9c2a963 Mon Sep 17 00:00:00 2001 From: andrew Date: Sat, 16 Mar 2013 02:48:49 +0000 Subject: Add an END macro to ARM. This is mostly used to tell gas where the bounds of the functions are when creating the EABI unwind tables. --- sys/arm/arm/bcopy_page.S | 4 ++++ sys/arm/arm/bcopyinout.S | 5 +++++ sys/arm/arm/bcopyinout_xscale.S | 4 +++- sys/arm/arm/blockio.S | 11 +++++++++-- sys/arm/arm/bus_space_asm_generic.S | 23 +++++++++++++++++++++++ sys/arm/arm/copystr.S | 3 +++ sys/arm/arm/cpufunc_asm.S | 15 ++++++++++++++- sys/arm/arm/cpufunc_asm_arm10.S | 13 ++++++++++++- sys/arm/arm/cpufunc_asm_arm11.S | 12 +++++++++++- sys/arm/arm/cpufunc_asm_arm11x6.S | 9 +++++++++ sys/arm/arm/cpufunc_asm_arm7tdmi.S | 6 ++++++ sys/arm/arm/cpufunc_asm_arm8.S | 12 ++++++++++++ sys/arm/arm/cpufunc_asm_arm9.S | 11 +++++++++++ sys/arm/arm/cpufunc_asm_armv4.S | 6 ++++++ sys/arm/arm/cpufunc_asm_armv5.S | 9 +++++++++ sys/arm/arm/cpufunc_asm_armv5_ec.S | 9 +++++++++ sys/arm/arm/cpufunc_asm_armv6.S | 10 ++++++++++ sys/arm/arm/cpufunc_asm_armv7.S | 16 ++++++++++++++++ sys/arm/arm/cpufunc_asm_fa526.S | 15 +++++++++++++++ sys/arm/arm/cpufunc_asm_ixp12x0.S | 2 ++ sys/arm/arm/cpufunc_asm_pj4b.S | 15 +++++++++++++++ sys/arm/arm/cpufunc_asm_sa1.S | 21 +++++++++++++++++++++ sys/arm/arm/cpufunc_asm_sa11x0.S | 5 ++++- sys/arm/arm/cpufunc_asm_sheeva.S | 11 +++++++++++ sys/arm/arm/cpufunc_asm_xscale.S | 27 +++++++++++++++++++++++++++ sys/arm/arm/cpufunc_asm_xscale_c3.S | 20 +++++++++++++++++++- sys/arm/arm/exception.S | 11 ++++++++--- sys/arm/arm/fiq_subr.S | 2 ++ sys/arm/arm/fusu.S | 14 ++++++++++++++ sys/arm/arm/in_cksum_arm.S | 6 +++++- sys/arm/arm/irq_dispatch.S | 1 + sys/arm/arm/locore.S | 10 ++++++++++ sys/arm/arm/setcpsr.S | 2 ++ sys/arm/arm/support.S | 8 ++++++++ sys/arm/arm/swtch.S | 6 ++++++ sys/arm/include/asm.h | 14 +++++++++++++- 36 files changed, 355 insertions(+), 13 deletions(-) (limited to 'sys') diff --git a/sys/arm/arm/bcopy_page.S b/sys/arm/arm/bcopy_page.S index 27921d4..92e38cc 100644 --- a/sys/arm/arm/bcopy_page.S +++ b/sys/arm/arm/bcopy_page.S @@ -117,6 +117,7 @@ ENTRY(bcopy_page) bne 1b RESTORE_REGS /* ...and return. */ +END(bcopy_page) /* * bzero_page(dest) @@ -178,6 +179,7 @@ ENTRY(bzero_page) bne 1b ldmfd sp!, {r4-r8, pc} +END(bzero_page) #else /* _ARM_ARCH_5E */ @@ -246,6 +248,7 @@ ENTRY(bcopy_page) bgt 1b ldmfd sp!, {r4, r5} RET +END(bcopy_page) /* * armv5e version of bzero_page @@ -273,4 +276,5 @@ ENTRY(bzero_page) subs r1, r1, #128 bne 1b RET +END(bzero_page) #endif /* _ARM_ARCH_5E */ diff --git a/sys/arm/arm/bcopyinout.S b/sys/arm/arm/bcopyinout.S index 992d0d7..68fdf20 100644 --- a/sys/arm/arm/bcopyinout.S +++ b/sys/arm/arm/bcopyinout.S @@ -312,6 +312,7 @@ ENTRY(copyin) RESTORE_REGS RET +END(copyin) /* * r0 = kernel space address @@ -538,6 +539,7 @@ ENTRY(copyout) RESTORE_REGS RET +END(copyout) #endif /* @@ -564,6 +566,7 @@ ENTRY(badaddr_read_1) mov r0, #0 /* No fault */ 1: str ip, [r2, #PCB_ONFAULT] RET +END(badaddr_read_1) /* * int badaddr_read_2(const uint16_t *src, uint16_t *dest) @@ -589,6 +592,7 @@ ENTRY(badaddr_read_2) mov r0, #0 /* No fault */ 1: str ip, [r2, #PCB_ONFAULT] RET +END(badaddr_read_2) /* * int badaddr_read_4(const uint32_t *src, uint32_t *dest) @@ -614,4 +618,5 @@ ENTRY(badaddr_read_4) mov r0, #0 /* No fault */ 1: str ip, [r2, #PCB_ONFAULT] RET +END(badaddr_read_4) diff --git a/sys/arm/arm/bcopyinout_xscale.S b/sys/arm/arm/bcopyinout_xscale.S index a2853cc..2cb98d9 100644 --- a/sys/arm/arm/bcopyinout_xscale.S +++ b/sys/arm/arm/bcopyinout_xscale.S @@ -492,7 +492,7 @@ ENTRY(copyin) ldrbt ip, [r0] strb ip, [r1] RET - +END(copyin) /* * r0 = kernel space address @@ -935,3 +935,5 @@ ENTRY(copyout) ldrb ip, [r0] strbt ip, [r1] RET +END(copyout) + diff --git a/sys/arm/arm/blockio.S b/sys/arm/arm/blockio.S index 7e750b4..d121f2c 100644 --- a/sys/arm/arm/blockio.S +++ b/sys/arm/arm/blockio.S @@ -101,6 +101,7 @@ ENTRY(read_multi_1) ldrgtb r3, [r0] strgtb r3, [r1], #1 ldmdb fp, {fp, sp, pc} +END(read_multi_1) /* * Write bytes to an I/O address from a block of memory @@ -152,6 +153,7 @@ ENTRY(write_multi_1) ldrgtb r3, [r1], #1 strgtb r3, [r0] ldmdb fp, {fp, sp, pc} +END(write_multi_1) /* * Reads short ints (16 bits) from an I/O address into a block of memory @@ -199,7 +201,7 @@ ENTRY(insw) bgt .Lfastinswloop RET - +END(insw) /* * Writes short ints (16 bits) from a block of memory to an I/O address @@ -260,6 +262,7 @@ ENTRY(outsw) bgt .Lfastoutswloop RET +END(outsw) /* * reads short ints (16 bits) from an I/O address into a block of memory @@ -318,7 +321,7 @@ ENTRY(insw16) bgt .Linsw16loop ldmfd sp!, {r4,r5,pc} /* Restore regs and go home */ - +END(insw16) /* * Writes short ints (16 bits) from a block of memory to an I/O address @@ -385,6 +388,7 @@ ENTRY(outsw16) bgt .Loutsw16loop ldmfd sp!, {r4,r5,pc} /* and go home */ +END(outsw16) /* * reads short ints (16 bits) from an I/O address into a block of memory @@ -481,6 +485,7 @@ ENTRY(inswm8) .Linswm8_l1: ldmfd sp!, {r4-r9,pc} /* And go home */ +END(inswm8) /* * write short ints (16 bits) to an I/O address from a block of memory @@ -585,3 +590,5 @@ ENTRY(outswm8) .Loutswm8_l1: ldmfd sp!, {r4-r8,pc} /* And go home */ +END(outswm8) + diff --git a/sys/arm/arm/bus_space_asm_generic.S b/sys/arm/arm/bus_space_asm_generic.S index 2492474..4aa7197 100644 --- a/sys/arm/arm/bus_space_asm_generic.S +++ b/sys/arm/arm/bus_space_asm_generic.S @@ -50,14 +50,17 @@ __FBSDID("$FreeBSD$"); ENTRY(generic_bs_r_1) ldrb r0, [r1, r2] RET +END(generic_bs_r_1) ENTRY(generic_armv4_bs_r_2) ldrh r0, [r1, r2] RET +END(generic_armv4_bs_r_2) ENTRY(generic_bs_r_4) ldr r0, [r1, r2] RET +END(generic_bs_r_4) /* * write single @@ -66,14 +69,17 @@ ENTRY(generic_bs_r_4) ENTRY(generic_bs_w_1) strb r3, [r1, r2] RET +END(generic_bs_w_1) ENTRY(generic_armv4_bs_w_2) strh r3, [r1, r2] RET +END(generic_armv4_bs_w_2) ENTRY(generic_bs_w_4) str r3, [r1, r2] RET +END(generic_bs_w_4) /* * read multiple @@ -92,6 +98,7 @@ ENTRY(generic_bs_rm_1) bne 1b RET +END(generic_bs_rm_1) ENTRY(generic_armv4_bs_rm_2) add r0, r1, r2 @@ -106,6 +113,7 @@ ENTRY(generic_armv4_bs_rm_2) bne 1b RET +END(generic_armv4_bs_rm_2) ENTRY(generic_bs_rm_4) add r0, r1, r2 @@ -120,6 +128,7 @@ ENTRY(generic_bs_rm_4) bne 1b RET +END(generic_bs_rm_4) /* * write multiple @@ -138,6 +147,7 @@ ENTRY(generic_bs_wm_1) bne 1b RET +END(generic_bs_wm_1) ENTRY(generic_armv4_bs_wm_2) add r0, r1, r2 @@ -152,6 +162,7 @@ ENTRY(generic_armv4_bs_wm_2) bne 1b RET +END(generic_armv4_bs_wm_2) ENTRY(generic_bs_wm_4) add r0, r1, r2 @@ -166,6 +177,7 @@ ENTRY(generic_bs_wm_4) bne 1b RET +END(generic_bs_wm_4) /* * read region @@ -184,6 +196,7 @@ ENTRY(generic_bs_rr_1) bne 1b RET +END(generic_bs_rr_1) ENTRY(generic_armv4_bs_rr_2) add r0, r1, r2 @@ -198,6 +211,7 @@ ENTRY(generic_armv4_bs_rr_2) bne 1b RET +END(generic_armv4_bs_rr_2) ENTRY(generic_bs_rr_4) add r0, r1, r2 @@ -212,6 +226,7 @@ ENTRY(generic_bs_rr_4) bne 1b RET +END(generic_bs_rr_4) /* * write region. @@ -230,6 +245,7 @@ ENTRY(generic_bs_wr_1) bne 1b RET +END(generic_bs_wr_1) ENTRY(generic_armv4_bs_wr_2) add r0, r1, r2 @@ -244,6 +260,7 @@ ENTRY(generic_armv4_bs_wr_2) bne 1b RET +END(generic_armv4_bs_wr_2) ENTRY(generic_bs_wr_4) add r0, r1, r2 @@ -258,6 +275,7 @@ ENTRY(generic_bs_wr_4) bne 1b RET +END(generic_bs_wr_4) /* * set region @@ -275,6 +293,7 @@ ENTRY(generic_bs_sr_1) bne 1b RET +END(generic_bs_sr_1) ENTRY(generic_armv4_bs_sr_2) add r0, r1, r2 @@ -288,6 +307,7 @@ ENTRY(generic_armv4_bs_sr_2) bne 1b RET +END(generic_armv4_bs_sr_2) ENTRY(generic_bs_sr_4) add r0, r1, r2 @@ -301,6 +321,7 @@ ENTRY(generic_bs_sr_4) bne 1b RET +END(generic_bs_sr_4) /* * copy region @@ -335,3 +356,5 @@ ENTRY(generic_armv4_bs_c_2) bne 3b RET +END(generic_armv4_bs_c_2) + diff --git a/sys/arm/arm/copystr.S b/sys/arm/arm/copystr.S index 9eb8682..83b7ec7 100644 --- a/sys/arm/arm/copystr.S +++ b/sys/arm/arm/copystr.S @@ -93,6 +93,7 @@ ENTRY(copystr) ldmfd sp!, {r4-r5} /* stack is 8 byte aligned */ RET +END(copystr) #define SAVE_REGS stmfd sp!, {r4-r6} #define RESTORE_REGS ldmfd sp!, {r4-r6} @@ -143,6 +144,7 @@ ENTRY(copyinstr) RESTORE_REGS RET +END(copyinstr) /* * r0 - kernel space address @@ -190,6 +192,7 @@ ENTRY(copyoutstr) RESTORE_REGS RET +END(copyoutstr) /* A fault occurred during the copy */ .Lcopystrfault: diff --git a/sys/arm/arm/cpufunc_asm.S b/sys/arm/arm/cpufunc_asm.S index 1709796..eeff722 100644 --- a/sys/arm/arm/cpufunc_asm.S +++ b/sys/arm/arm/cpufunc_asm.S @@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$"); ENTRY(cpufunc_nullop) RET +END(cpufunc_nullop) /* * Generic functions to read the internal coprocessor registers @@ -64,27 +65,32 @@ ENTRY(cpufunc_nullop) ENTRY(cpufunc_id) mrc p15, 0, r0, c0, c0, 0 RET +END(cpufunc_id) ENTRY(cpufunc_cpuid) mrc p15, 0, r0, c0, c0, 0 RET +END(cpufunc_cpuid) ENTRY(cpu_get_control) mrc p15, 0, r0, c1, c0, 0 RET +END(cpu_get_control) ENTRY(cpu_read_cache_config) mrc p15, 0, r0, c0, c0, 1 RET +END(cpu_read_cache_config) ENTRY(cpufunc_faultstatus) mrc p15, 0, r0, c5, c0, 0 RET +END(cpufunc_faultstatus) ENTRY(cpufunc_faultaddress) mrc p15, 0, r0, c6, c0, 0 RET - +END(cpufunc_faultaddress) /* * Generic functions to write the internal coprocessor registers @@ -101,11 +107,13 @@ ENTRY(cpufunc_faultaddress) ENTRY(cpufunc_control) mcr p15, 0, r0, c1, c0, 0 RET +END(cpufunc_control) #endif ENTRY(cpufunc_domains) mcr p15, 0, r0, c3, c0, 0 RET +END(cpufunc_domains) /* * Generic functions to read/modify/write the internal coprocessor registers @@ -131,6 +139,8 @@ ENTRY(cpufunc_control) .Lglou: .asciz "plop %p\n" .align 0 +END(cpufunc_control) + /* * other potentially useful software functions are: * clean D cache entry and flush I cache entry @@ -157,6 +167,7 @@ ENTRY(get_pc_str_offset) ldr r0, [sp] sub r0, r0, r1 ldmdb fp, {fp, sp, pc} +END(get_pc_str_offset) /* Allocate and lock a cacheline for the specified address. */ @@ -180,3 +191,5 @@ ENTRY(arm_lock_cache_line) mcr p15, 0, r1, c9, c2, 0 /* Disable data cache lock mode */ CPWAIT() RET +END(arm_lock_cache_line) + diff --git a/sys/arm/arm/cpufunc_asm_arm10.S b/sys/arm/arm/cpufunc_asm_arm10.S index 2ef999c..654219b 100644 --- a/sys/arm/arm/cpufunc_asm_arm10.S +++ b/sys/arm/arm/cpufunc_asm_arm10.S @@ -50,6 +50,7 @@ ENTRY(arm10_setttb) mcr p15, 0, r0, c8, c7, 0 /* invalidate I+D TLBs */ bx lr +END(arm10_setttb) /* * TLB functions @@ -58,11 +59,12 @@ ENTRY(arm10_tlb_flushID_SE) mcr p15, 0, r0, c8, c6, 1 /* flush D tlb single entry */ mcr p15, 0, r0, c8, c5, 1 /* flush I tlb single entry */ bx lr +END(arm10_tlb_flushID_SE) ENTRY(arm10_tlb_flushI_SE) mcr p15, 0, r0, c8, c5, 1 /* flush I tlb single entry */ bx lr - +END(arm10_tlb_flushI_SE) /* * Cache operations. For the entire cache we use the set/index @@ -90,6 +92,7 @@ ENTRY_NP(arm10_icache_sync_range) bhi .Larm10_sync_next mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ bx lr +END(arm10_icache_sync_range) ENTRY_NP(arm10_icache_sync_all) .Larm10_icache_sync_all: @@ -114,6 +117,7 @@ ENTRY_NP(arm10_icache_sync_all) bhs .Lnext_set /* Next set */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ bx lr +END(arm10_icache_sync_all) .Larm10_line_size: .word _C_LABEL(arm_pdcache_line_size) @@ -134,6 +138,7 @@ ENTRY(arm10_dcache_wb_range) bhi .Larm10_wb_next mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ bx lr +END(arm10_dcache_wb_range) ENTRY(arm10_dcache_wbinv_range) ldr ip, .Larm10_line_size @@ -151,6 +156,7 @@ ENTRY(arm10_dcache_wbinv_range) bhi .Larm10_wbinv_next mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ bx lr +END(arm10_dcache_wbinv_range) /* * Note, we must not invalidate everything. If the range is too big we @@ -172,6 +178,7 @@ ENTRY(arm10_dcache_inv_range) bhi .Larm10_inv_next mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ bx lr +END(arm10_dcache_inv_range) ENTRY(arm10_idcache_wbinv_range) ldr ip, .Larm10_line_size @@ -190,6 +197,7 @@ ENTRY(arm10_idcache_wbinv_range) bhi .Larm10_id_wbinv_next mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ bx lr +END(arm10_idcache_wbinv_range) ENTRY_NP(arm10_idcache_wbinv_all) .Larm10_idcache_wbinv_all: @@ -215,6 +223,8 @@ ENTRY(arm10_dcache_wbinv_all) bhs .Lnext_set_inv /* Next set */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ bx lr +END(arm10_idcache_wbinv_all) +END(arm10_dcache_wbinv_all) .Larm10_cache_data: .word _C_LABEL(arm10_dcache_sets_max) @@ -242,6 +252,7 @@ ENTRY(arm10_context_switch) nop nop bx lr +END(arm10_context_switch) .bss diff --git a/sys/arm/arm/cpufunc_asm_arm11.S b/sys/arm/arm/cpufunc_asm_arm11.S index b8d8f19..723afc6 100644 --- a/sys/arm/arm/cpufunc_asm_arm11.S +++ b/sys/arm/arm/cpufunc_asm_arm11.S @@ -55,6 +55,7 @@ ENTRY(arm11_setttb) mcr p15, 0, r0, c8, c7, 0 /* invalidate I+D TLBs */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ RET +END(arm11_setttb) /* * TLB functions @@ -64,12 +65,13 @@ ENTRY(arm11_tlb_flushID_SE) mcr p15, 0, r0, c8, c5, 1 /* flush I tlb single entry */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ RET +END(arm11_tlb_flushID_SE) ENTRY(arm11_tlb_flushI_SE) mcr p15, 0, r0, c8, c5, 1 /* flush I tlb single entry */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ RET - +END(arm11_tlb_flushI_SE) /* * Context switch. @@ -94,6 +96,7 @@ ENTRY(arm11_context_switch) nop nop RET +END(arm11_context_switch) /* * TLB functions @@ -102,21 +105,25 @@ ENTRY(arm11_tlb_flushID) mcr p15, 0, r0, c8, c7, 0 /* flush I+D tlb */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr +END(arm11_tlb_flushID) ENTRY(arm11_tlb_flushI) mcr p15, 0, r0, c8, c5, 0 /* flush I tlb */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr +END(arm11_tlb_flushI) ENTRY(arm11_tlb_flushD) mcr p15, 0, r0, c8, c6, 0 /* flush D tlb */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr +END(arm11_tlb_flushD) ENTRY(arm11_tlb_flushD_SE) mcr p15, 0, r0, c8, c6, 1 /* flush D tlb single entry */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr +END(arm11_tlb_flushD_SE) /* * Other functions @@ -124,8 +131,11 @@ ENTRY(arm11_tlb_flushD_SE) ENTRY(arm11_drain_writebuf) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr +END(arm11_drain_writebuf) ENTRY_NP(arm11_sleep) mov r0, #0 mcr p15, 0, r0, c7, c0, 4 /* wait for interrupt */ RET +END(arm11_sleep) + diff --git a/sys/arm/arm/cpufunc_asm_arm11x6.S b/sys/arm/arm/cpufunc_asm_arm11x6.S index e223208..6c7eb56 100644 --- a/sys/arm/arm/cpufunc_asm_arm11x6.S +++ b/sys/arm/arm/cpufunc_asm_arm11x6.S @@ -124,24 +124,29 @@ ENTRY(arm11x6_setttb) mcr p15, 0, r1, c8, c7, 0 /* invalidate I+D TLBs */ mcr p15, 0, r1, c7, c10, 4 /* drain write buffer */ RET +END(arm11x6_setttb) ENTRY_NP(arm11x6_idcache_wbinv_all) Flush_D_cache(r0) Invalidate_I_cache(r0, r1) RET +END(arm11x6_idcache_wbinv_all) ENTRY_NP(arm11x6_dcache_wbinv_all) Flush_D_cache(r0) RET +END(arm11x6_dcache_wbinv_all) ENTRY_NP(arm11x6_icache_sync_all) Flush_D_cache(r0) Invalidate_I_cache(r0, r1) RET +END(arm11x6_icache_sync_all) ENTRY_NP(arm11x6_flush_prefetchbuf) mcr p15, 0, r0, c7, c5, 4 /* Flush Prefetch Buffer */ RET +END(arm11x6_flush_prefetchbuf) ENTRY_NP(arm11x6_icache_sync_range) add r1, r1, r0 @@ -168,6 +173,7 @@ ENTRY_NP(arm11x6_icache_sync_range) mcrr p15, 0, r1, r0, c12 /* clean and invalidate D cache range */ /* XXXNH */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(arm11x6_icache_sync_range) ENTRY_NP(arm11x6_idcache_wbinv_range) add r1, r1, r0 @@ -194,6 +200,7 @@ ENTRY_NP(arm11x6_idcache_wbinv_range) mcrr p15, 0, r1, r0, c14 /* clean and invalidate D cache range */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(arm11x6_idcache_wbinv_range) /* * Preload the cache before issuing the WFI by conditionally disabling the @@ -216,3 +223,5 @@ ENTRY_NP(arm11x6_sleep) nop bne 1b RET +END(arm11x6_sleep) + diff --git a/sys/arm/arm/cpufunc_asm_arm7tdmi.S b/sys/arm/arm/cpufunc_asm_arm7tdmi.S index fed6f16..2ac2502 100644 --- a/sys/arm/arm/cpufunc_asm_arm7tdmi.S +++ b/sys/arm/arm/cpufunc_asm_arm7tdmi.S @@ -60,6 +60,7 @@ ENTRY(arm7tdmi_setttb) bl _C_LABEL(arm7tdmi_cache_flushID) mov pc, r2 +END(arm7tdmi_setttb) /* * TLB functions @@ -68,10 +69,12 @@ ENTRY(arm7tdmi_tlb_flushID) mov r0, #0 mcr p15, 0, r0, c8, c7, 0 RET +END(arm7tdmi_tlb_flushID) ENTRY(arm7tdmi_tlb_flushID_SE) mcr p15, 0, r0, c8, c7, 1 RET +END(arm7tdmi_tlb_flushID_SE) /* * Cache functions @@ -86,6 +89,7 @@ ENTRY(arm7tdmi_cache_flushID) mov r0, r0 RET +END(arm7tdmi_cache_flushID) /* * Context switch. @@ -98,3 +102,5 @@ ENTRY(arm7tdmi_cache_flushID) */ ENTRY(arm7tdmi_context_switch) b _C_LABEL(arm7tdmi_setttb) +END(arm7tdmi_context_switch) + diff --git a/sys/arm/arm/cpufunc_asm_arm8.S b/sys/arm/arm/cpufunc_asm_arm8.S index 9f23548..2cb8b11 100644 --- a/sys/arm/arm/cpufunc_asm_arm8.S +++ b/sys/arm/arm/cpufunc_asm_arm8.S @@ -58,6 +58,7 @@ ENTRY(arm8_clock_config) mcr p15, 0, r2, c15, c0, 0 /* Write clock register */ mov r0, r3 /* Return old value */ RET +END(arm8_clock_config) /* * Functions to set the MMU Translation Table Base register @@ -90,6 +91,7 @@ ENTRY(arm8_setttb) msr cpsr_all, r3 RET +END(arm8_setttb) /* * TLB functions @@ -97,10 +99,12 @@ ENTRY(arm8_setttb) ENTRY(arm8_tlb_flushID) mcr p15, 0, r0, c8, c7, 0 /* flush I+D tlb */ RET +END(arm8_tlb_flushID) ENTRY(arm8_tlb_flushID_SE) mcr p15, 0, r0, c8, c7, 1 /* flush I+D tlb single entry */ RET +END(arm8_tlb_flushID_SE) /* * Cache functions @@ -108,10 +112,12 @@ ENTRY(arm8_tlb_flushID_SE) ENTRY(arm8_cache_flushID) mcr p15, 0, r0, c7, c7, 0 /* flush I+D cache */ RET +END(arm8_cache_flushID) ENTRY(arm8_cache_flushID_E) mcr p15, 0, r0, c7, c7, 1 /* flush I+D single entry */ RET +END(arm8_cache_flushID_E) ENTRY(arm8_cache_cleanID) mov r0, #0x00000000 @@ -153,10 +159,12 @@ ENTRY(arm8_cache_cleanID) bne 1b RET +END(arm8_cache_cleanID) ENTRY(arm8_cache_cleanID_E) mcr p15, 0, r0, c7, c11, 1 /* clean I+D single entry */ RET +END(arm8_cache_cleanID_E) ENTRY(arm8_cache_purgeID) /* @@ -232,6 +240,7 @@ ENTRY(arm8_cache_purgeID) msr cpsr_all, r3 RET +END(arm8_cache_purgeID) ENTRY(arm8_cache_purgeID_E) /* @@ -253,6 +262,7 @@ ENTRY(arm8_cache_purgeID_E) mcr p15, 0, r0, c7, c7, 1 /* flush I+D single entry */ msr cpsr_all, r3 RET +END(arm8_cache_purgeID_E) /* * Context switch. @@ -282,3 +292,5 @@ ENTRY(arm8_context_switch) mov r0, r0 mov r0, r0 RET +END(arm8_context_switch) + diff --git a/sys/arm/arm/cpufunc_asm_arm9.S b/sys/arm/arm/cpufunc_asm_arm9.S index ae9fe00..dd29479 100644 --- a/sys/arm/arm/cpufunc_asm_arm9.S +++ b/sys/arm/arm/cpufunc_asm_arm9.S @@ -49,6 +49,7 @@ ENTRY(arm9_setttb) mcr p15, 0, r0, c8, c7, 0 /* invalidate I+D TLBs */ mov pc, lr +END(arm9_setttb) /* * TLB functions @@ -57,6 +58,7 @@ ENTRY(arm9_tlb_flushID_SE) mcr p15, 0, r0, c8, c6, 1 /* flush D tlb single entry */ mcr p15, 0, r0, c8, c5, 1 /* flush I tlb single entry */ mov pc, lr +END(arm9_tlb_flushID_SE) /* * Cache operations. For the entire cache we use the set/index @@ -83,6 +85,7 @@ ENTRY_NP(arm9_icache_sync_range) subs r1, r1, ip bhi .Larm9_sync_next mov pc, lr +END(arm9_icache_sync_range) ENTRY_NP(arm9_icache_sync_all) .Larm9_icache_sync_all: @@ -106,6 +109,7 @@ ENTRY_NP(arm9_icache_sync_all) subs s_max, s_max, s_inc bhs .Lnext_set /* Next set */ mov pc, lr +END(arm9_icache_sync_all) .Larm9_line_size: .word _C_LABEL(arm_pdcache_line_size) @@ -125,6 +129,7 @@ ENTRY(arm9_dcache_wb_range) subs r1, r1, ip bhi .Larm9_wb_next mov pc, lr +END(arm9_dcache_wb_range) ENTRY(arm9_dcache_wbinv_range) ldr ip, .Larm9_line_size @@ -141,6 +146,7 @@ ENTRY(arm9_dcache_wbinv_range) subs r1, r1, ip bhi .Larm9_wbinv_next mov pc, lr +END(arm9_dcache_wbinv_range) /* * Note, we must not invalidate everything. If the range is too big we @@ -161,6 +167,7 @@ ENTRY(arm9_dcache_inv_range) subs r1, r1, ip bhi .Larm9_inv_next mov pc, lr +END(arm9_dcache_inv_range) ENTRY(arm9_idcache_wbinv_range) ldr ip, .Larm9_line_size @@ -178,6 +185,7 @@ ENTRY(arm9_idcache_wbinv_range) subs r1, r1, ip bhi .Larm9_id_wbinv_next mov pc, lr +END(arm9_idcache_wbinv_range) ENTRY_NP(arm9_idcache_wbinv_all) .Larm9_idcache_wbinv_all: @@ -202,6 +210,8 @@ ENTRY(arm9_dcache_wbinv_all) subs s_max, s_max, s_inc bhs .Lnext_set_inv /* Next set */ mov pc, lr +END(arm9_idcache_wbinv_all) +END(arm9_dcache_wbinv_all) .Larm9_cache_data: .word _C_LABEL(arm9_dcache_sets_max) @@ -229,6 +239,7 @@ ENTRY(arm9_context_switch) nop nop mov pc, lr +END(arm9_context_switch) .bss diff --git a/sys/arm/arm/cpufunc_asm_armv4.S b/sys/arm/arm/cpufunc_asm_armv4.S index 1b8797d..1123e4a 100644 --- a/sys/arm/arm/cpufunc_asm_armv4.S +++ b/sys/arm/arm/cpufunc_asm_armv4.S @@ -46,18 +46,22 @@ __FBSDID("$FreeBSD$"); ENTRY(armv4_tlb_flushID) mcr p15, 0, r0, c8, c7, 0 /* flush I+D tlb */ RET +END(armv4_tlb_flushID) ENTRY(armv4_tlb_flushI) mcr p15, 0, r0, c8, c5, 0 /* flush I tlb */ RET +END(armv4_tlb_flushI) ENTRY(armv4_tlb_flushD) mcr p15, 0, r0, c8, c6, 0 /* flush D tlb */ RET +END(armv4_tlb_flushD) ENTRY(armv4_tlb_flushD_SE) mcr p15, 0, r0, c8, c6, 1 /* flush D tlb single entry */ RET +END(armv4_tlb_flushD_SE) /* * Other functions @@ -65,3 +69,5 @@ ENTRY(armv4_tlb_flushD_SE) ENTRY(armv4_drain_writebuf) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ RET +END(armv4_drain_writebuf) + diff --git a/sys/arm/arm/cpufunc_asm_armv5.S b/sys/arm/arm/cpufunc_asm_armv5.S index 2faa5f4..94e6b43 100644 --- a/sys/arm/arm/cpufunc_asm_armv5.S +++ b/sys/arm/arm/cpufunc_asm_armv5.S @@ -51,6 +51,7 @@ ENTRY(armv5_setttb) mcr p15, 0, r0, c8, c7, 0 /* invalidate I+D TLBs */ RET +END(armv5_setttb) /* * Cache operations. For the entire cache we use the set/index @@ -79,6 +80,7 @@ ENTRY_NP(armv5_icache_sync_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_icache_sync_range) ENTRY_NP(armv5_icache_sync_all) .Larmv5_icache_sync_all: @@ -105,6 +107,7 @@ ENTRY_NP(armv5_icache_sync_all) bpl 1b /* Next set */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_icache_sync_all) .Larmv5_line_size: .word _C_LABEL(arm_pdcache_line_size) @@ -126,6 +129,7 @@ ENTRY(armv5_dcache_wb_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_dcache_wb_range) ENTRY(armv5_dcache_wbinv_range) ldr ip, .Larmv5_line_size @@ -144,6 +148,7 @@ ENTRY(armv5_dcache_wbinv_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_dcache_wbinv_range) /* * Note, we must not invalidate everything. If the range is too big we @@ -166,6 +171,7 @@ ENTRY(armv5_dcache_inv_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_dcache_inv_range) ENTRY(armv5_idcache_wbinv_range) ldr ip, .Larmv5_line_size @@ -185,6 +191,7 @@ ENTRY(armv5_idcache_wbinv_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_idcache_wbinv_range) ENTRY_NP(armv5_idcache_wbinv_all) .Larmv5_idcache_wbinv_all: @@ -212,6 +219,8 @@ ENTRY(armv5_dcache_wbinv_all) bpl 1b /* Next set */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_idcache_wbinv_all) +END(armv5_dcache_wbinv_all) .Larmv5_cache_data: .word _C_LABEL(armv5_dcache_sets_max) diff --git a/sys/arm/arm/cpufunc_asm_armv5_ec.S b/sys/arm/arm/cpufunc_asm_armv5_ec.S index 4012563..a86ac80 100644 --- a/sys/arm/arm/cpufunc_asm_armv5_ec.S +++ b/sys/arm/arm/cpufunc_asm_armv5_ec.S @@ -66,6 +66,7 @@ ENTRY(armv5_ec_setttb) mcr p15, 0, r0, c8, c7, 0 /* invalidate I+D TLBs */ RET +END(armv5_ec_setttb) /* * Cache operations. For the entire cache we use the enhanced cache @@ -90,6 +91,7 @@ ENTRY_NP(armv5_ec_icache_sync_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_ec_icache_sync_range) ENTRY_NP(armv5_ec_icache_sync_all) .Larmv5_ec_icache_sync_all: @@ -107,6 +109,7 @@ ENTRY_NP(armv5_ec_icache_sync_all) bne 1b /* More to do? */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_ec_icache_sync_all) .Larmv5_ec_line_size: .word _C_LABEL(arm_pdcache_line_size) @@ -128,6 +131,7 @@ ENTRY(armv5_ec_dcache_wb_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_ec_dcache_wb_range) ENTRY(armv5_ec_dcache_wbinv_range) ldr ip, .Larmv5_ec_line_size @@ -146,6 +150,7 @@ ENTRY(armv5_ec_dcache_wbinv_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_ec_dcache_wbinv_range) /* * Note, we must not invalidate everything. If the range is too big we @@ -168,6 +173,7 @@ ENTRY(armv5_ec_dcache_inv_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_ec_dcache_inv_range) ENTRY(armv5_ec_idcache_wbinv_range) ldr ip, .Larmv5_ec_line_size @@ -187,6 +193,7 @@ ENTRY(armv5_ec_idcache_wbinv_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_ec_idcache_wbinv_range) ENTRY_NP(armv5_ec_idcache_wbinv_all) .Larmv5_ec_idcache_wbinv_all: @@ -197,6 +204,7 @@ ENTRY_NP(armv5_ec_idcache_wbinv_all) */ mcr p15, 0, r0, c7, c5, 0 /* Invalidate ICache */ /* Fall through to purge Dcache. */ +END(armv5_ec_idcache_wbinv_all) ENTRY(armv5_ec_dcache_wbinv_all) .Larmv5_ec_dcache_wbinv_all: @@ -204,4 +212,5 @@ ENTRY(armv5_ec_dcache_wbinv_all) bne 1b /* More to do? */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv5_ec_dcache_wbinv_all) diff --git a/sys/arm/arm/cpufunc_asm_armv6.S b/sys/arm/arm/cpufunc_asm_armv6.S index f735754..b8a2d9c 100644 --- a/sys/arm/arm/cpufunc_asm_armv6.S +++ b/sys/arm/arm/cpufunc_asm_armv6.S @@ -59,6 +59,7 @@ ENTRY(armv6_setttb) mcr p15, 0, r0, c8, c7, 0 /* invalidate I+D TLBs */ RET +END(armv6_setttb) /* * Cache operations. @@ -72,6 +73,7 @@ ENTRY_NP(armv6_icache_sync_range) mcrr p15, 0, r1, r0, c12 /* clean D cache range */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv6_icache_sync_range) /* LINTSTUB: void armv6_icache_sync_all(void); */ ENTRY_NP(armv6_icache_sync_all) @@ -84,6 +86,7 @@ ENTRY_NP(armv6_icache_sync_all) mcr p15, 0, r0, c7, c10, 0 /* Clean D cache */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv6_icache_sync_all) /* LINTSTUB: void armv6_dcache_wb_range(vaddr_t, vsize_t); */ ENTRY(armv6_dcache_wb_range) @@ -92,6 +95,7 @@ ENTRY(armv6_dcache_wb_range) mcrr p15, 0, r1, r0, c12 /* clean D cache range */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv6_dcache_wb_range) /* LINTSTUB: void armv6_dcache_wbinv_range(vaddr_t, vsize_t); */ ENTRY(armv6_dcache_wbinv_range) @@ -100,6 +104,7 @@ ENTRY(armv6_dcache_wbinv_range) mcrr p15, 0, r1, r0, c14 /* clean and invaliate D cache range */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv6_dcache_wbinv_range) /* * Note, we must not invalidate everything. If the range is too big we @@ -113,6 +118,7 @@ ENTRY(armv6_dcache_inv_range) mcrr p15, 0, r1, r0, c6 /* invaliate D cache range */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv6_dcache_inv_range) /* LINTSTUB: void armv6_idcache_wbinv_range(vaddr_t, vsize_t); */ ENTRY(armv6_idcache_wbinv_range) @@ -122,6 +128,7 @@ ENTRY(armv6_idcache_wbinv_range) mcrr p15, 0, r1, r0, c14 /* clean & invaliate D cache range */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv6_idcache_wbinv_range) /* LINTSTUB: void armv6_idcache_wbinv_all(void); */ ENTRY_NP(armv6_idcache_wbinv_all) @@ -138,3 +145,6 @@ ENTRY(armv6_dcache_wbinv_all) mcr p15, 0, r0, c7, c14, 0 /* clean & invalidate D cache */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv6_idcache_wbinv_all) +END(armv6_dcache_wbinv_all) + diff --git a/sys/arm/arm/cpufunc_asm_armv7.S b/sys/arm/arm/cpufunc_asm_armv7.S index 58f295c..2b4be85 100644 --- a/sys/arm/arm/cpufunc_asm_armv7.S +++ b/sys/arm/arm/cpufunc_asm_armv7.S @@ -78,6 +78,7 @@ ENTRY(armv7_setttb) dsb isb RET +END(armv7_setttb) ENTRY(armv7_tlb_flushID) dsb @@ -91,6 +92,7 @@ ENTRY(armv7_tlb_flushID) dsb isb mov pc, lr +END(armv7_tlb_flushID) ENTRY(armv7_tlb_flushID_SE) ldr r1, .Lpage_mask @@ -105,6 +107,7 @@ ENTRY(armv7_tlb_flushID_SE) dsb isb mov pc, lr +END(armv7_tlb_flushID_SE) /* Based on algorithm from ARM Architecture Reference Manual */ ENTRY(armv7_dcache_wbinv_all) @@ -157,6 +160,7 @@ Finished: dsb ldmia sp!, {r4, r5, r6, r7, r8, r9} RET +END(armv7_dcache_wbinv_all) ENTRY(armv7_idcache_wbinv_all) stmdb sp!, {lr} @@ -170,6 +174,7 @@ ENTRY(armv7_idcache_wbinv_all) isb ldmia sp!, {lr} RET +END(armv7_idcache_wbinv_all) /* XXX Temporary set it to 32 for MV cores, however this value should be * get from Cache Type register @@ -190,6 +195,7 @@ ENTRY(armv7_dcache_wb_range) bhi .Larmv7_wb_next dsb /* data synchronization barrier */ RET +END(armv7_dcache_wb_range) ENTRY(armv7_dcache_wbinv_range) ldr ip, .Larmv7_line_size @@ -204,6 +210,7 @@ ENTRY(armv7_dcache_wbinv_range) bhi .Larmv7_wbinv_next dsb /* data synchronization barrier */ RET +END(armv7_dcache_wbinv_range) /* * Note, we must not invalidate everything. If the range is too big we @@ -222,6 +229,7 @@ ENTRY(armv7_dcache_inv_range) bhi .Larmv7_inv_next dsb /* data synchronization barrier */ RET +END(armv7_dcache_inv_range) ENTRY(armv7_idcache_wbinv_range) ldr ip, .Larmv7_line_size @@ -238,6 +246,7 @@ ENTRY(armv7_idcache_wbinv_range) isb /* instruction synchronization barrier */ dsb /* data synchronization barrier */ RET +END(armv7_idcache_wbinv_range) ENTRY_NP(armv7_icache_sync_range) ldr ip, .Larmv7_line_size @@ -250,11 +259,13 @@ ENTRY_NP(armv7_icache_sync_range) isb /* instruction synchronization barrier */ dsb /* data synchronization barrier */ RET +END(armv7_icache_sync_range) ENTRY(armv7_cpu_sleep) dsb /* data synchronization barrier */ wfi /* wait for interrupt */ RET +END(armv7_cpu_sleep) ENTRY(armv7_context_switch) dsb @@ -269,16 +280,19 @@ ENTRY(armv7_context_switch) dsb isb RET +END(armv7_context_switch) ENTRY(armv7_drain_writebuf) dsb RET +END(armv7_drain_writebuf) ENTRY(armv7_sev) dsb sev nop RET +END(armv7_sev) ENTRY(armv7_auxctrl) mrc p15, 0, r2, c1, c0, 1 @@ -289,3 +303,5 @@ ENTRY(armv7_auxctrl) mcrne p15, 0, r3, c1, c0, 1 mov r0, r2 RET +END(armv7_auxctrl) + diff --git a/sys/arm/arm/cpufunc_asm_fa526.S b/sys/arm/arm/cpufunc_asm_fa526.S index d53d29a..55c2f37 100644 --- a/sys/arm/arm/cpufunc_asm_fa526.S +++ b/sys/arm/arm/cpufunc_asm_fa526.S @@ -54,6 +54,7 @@ ENTRY(fa526_setttb) mov r0, r0 mov r0, r0 mov pc, lr +END(fa526_setttb) /* * TLB functions @@ -61,6 +62,7 @@ ENTRY(fa526_setttb) ENTRY(fa526_tlb_flushID_SE) mcr p15, 0, r0, c8, c7, 1 /* flush Utlb single entry */ mov pc, lr +END(fa526_tlb_flushID_SE) /* * TLB functions @@ -68,6 +70,7 @@ ENTRY(fa526_tlb_flushID_SE) ENTRY(fa526_tlb_flushI_SE) mcr p15, 0, r0, c8, c5, 1 /* flush Itlb single entry */ mov pc, lr +END(fa526_tlb_flushI_SE) ENTRY(fa526_cpu_sleep) mov r0, #0 @@ -75,11 +78,13 @@ ENTRY(fa526_cpu_sleep) nop*/ mcr p15, 0, r0, c7, c0, 4 /* Wait for interrupt*/ mov pc, lr +END(fa526_cpu_sleep) ENTRY(fa526_flush_prefetchbuf) mov r0, #0 mcr p15, 0, r0, c7, c5, 4 /* Pre-fetch flush */ mov pc, lr +END(fa526_flush_prefetchbuf) /* * Cache functions @@ -90,17 +95,20 @@ ENTRY(fa526_idcache_wbinv_all) mcr p15, 0, r0, c7, c5, 0 /* invalidate I$ */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr +END(fa526_idcache_wbinv_all) ENTRY(fa526_icache_sync_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 /* invalidate I$ */ mov pc, lr +END(fa526_icache_sync_all) ENTRY(fa526_dcache_wbinv_all) mov r0, #0 mcr p15, 0, r0, c7, c14, 0 /* clean and invalidate D$ */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr +END(fa526_dcache_wbinv_all) /* * Soft functions @@ -120,6 +128,7 @@ ENTRY(fa526_dcache_wbinv_range) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr +END(fa526_dcache_wbinv_range) ENTRY(fa526_dcache_wb_range) cmp r1, #0x4000 @@ -140,6 +149,7 @@ ENTRY(fa526_dcache_wb_range) 3: mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr +END(fa526_dcache_wb_range) ENTRY(fa526_dcache_inv_range) and r2, r0, #(CACHELINE_SIZE - 1) @@ -152,6 +162,7 @@ ENTRY(fa526_dcache_inv_range) bhi 1b mov pc, lr +END(fa526_dcache_inv_range) ENTRY(fa526_idcache_wbinv_range) cmp r1, #0x4000 @@ -169,6 +180,7 @@ ENTRY(fa526_idcache_wbinv_range) 2: mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr +END(fa526_idcache_wbinv_range) ENTRY(fa526_icache_sync_range) cmp r1, #0x4000 @@ -186,11 +198,13 @@ ENTRY(fa526_icache_sync_range) 2: mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr +END(fa526_icache_sync_range) ENTRY(fa526_flush_brnchtgt_E) mov r0, #0 mcr p15, 0, r0, c7, c5, 6 /* invalidate BTB cache */ mov pc, lr +END(fa526_flush_brnchtgt_E) ENTRY(fa526_context_switch) /* @@ -210,4 +224,5 @@ ENTRY(fa526_context_switch) mov r0, r0 mov r0, r0 mov pc, lr +END(fa526_context_switch) diff --git a/sys/arm/arm/cpufunc_asm_ixp12x0.S b/sys/arm/arm/cpufunc_asm_ixp12x0.S index efc5950..481cf0d 100644 --- a/sys/arm/arm/cpufunc_asm_ixp12x0.S +++ b/sys/arm/arm/cpufunc_asm_ixp12x0.S @@ -61,10 +61,12 @@ ENTRY(ixp12x0_context_switch) mov r0, r0 mov r0, r0 RET +END(ixp12x0_context_switch) ENTRY(ixp12x0_drain_readbuf) mcr p15, 0, r0, c9, c0, 0 /* drain read buffer */ RET +END(ixp12x0_drain_readbuf) /* * Information for the IXP12X0 cache clean/purge functions: diff --git a/sys/arm/arm/cpufunc_asm_pj4b.S b/sys/arm/arm/cpufunc_asm_pj4b.S index f6890d9..2e325f3 100644 --- a/sys/arm/arm/cpufunc_asm_pj4b.S +++ b/sys/arm/arm/cpufunc_asm_pj4b.S @@ -46,6 +46,7 @@ ENTRY(pj4b_setttb) mcr p15, 0, r0, c2, c0, 0 /* load new TTB */ mcr p15, 0, r0, c8, c7, 0 /* invalidate I+D TLBs */ RET +END(pj4b_setttb) ENTRY_NP(armv6_icache_sync_all) /* @@ -58,6 +59,7 @@ ENTRY_NP(armv6_icache_sync_all) mcr p15, 0, r0, c7, c10, 0 /* Clean (don't invalidate) DCache */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv6_icache_sync_all) ENTRY(pj4b_icache_sync_range) sub r1, r1, #1 @@ -66,6 +68,7 @@ ENTRY(pj4b_icache_sync_range) mcrr p15, 0, r1, r0, c12 /* clean DC range */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(pj4b_icache_sync_range) ENTRY(pj4b_dcache_inv_range) ldr ip, .Lpj4b_cache_line_size @@ -84,6 +87,7 @@ ENTRY(pj4b_dcache_inv_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(pj4b_dcache_inv_range) ENTRY(armv6_idcache_wbinv_all) mov r0, #0 @@ -91,12 +95,14 @@ ENTRY(armv6_idcache_wbinv_all) mcr p15, 0, r0, c7, c14, 0 /* clean and invalidate DCache */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv6_idcache_wbinv_all) ENTRY(armv6_dcache_wbinv_all) mov r0, #0 mcr p15, 0, r0, c7, c14, 0 /* clean and invalidate DCache */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(armv6_dcache_wbinv_all) ENTRY(pj4b_idcache_wbinv_range) ldr ip, .Lpj4b_cache_line_size @@ -121,6 +127,7 @@ ENTRY(pj4b_idcache_wbinv_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(pj4b_idcache_wbinv_range) ENTRY(pj4b_dcache_wbinv_range) ldr ip, .Lpj4b_cache_line_size @@ -144,6 +151,7 @@ ENTRY(pj4b_dcache_wbinv_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(pj4b_dcache_wbinv_range) ENTRY(pj4b_dcache_wb_range) ldr ip, .Lpj4b_cache_line_size @@ -167,22 +175,27 @@ ENTRY(pj4b_dcache_wb_range) bpl 1b mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(pj4b_dcache_wb_range) ENTRY(pj4b_drain_readbuf) mcr p15, 0, r0, c7, c5, 4 /* flush prefetch buffers */ RET +END(pj4b_drain_readbuf) ENTRY(pj4b_flush_brnchtgt_all) mcr p15, 0, r0, c7, c5, 6 /* flush entrie branch target cache */ RET +END(pj4b_flush_brnchtgt_all) ENTRY(pj4b_flush_brnchtgt_va) mcr p15, 0, r0, c7, c5, 7 /* flush branch target cache by VA */ RET +END(pj4b_flush_brnchtgt_va) ENTRY(get_core_id) mrc p15, 0, r0, c0, c0, 5 RET +END(get_core_id) ENTRY(pj4b_config) /* Set Auxiliary Debug Modes Control 2 register */ @@ -200,3 +213,5 @@ ENTRY(pj4b_config) mcr p15, 0, r0, c1, c0, 1 #endif RET +END(pj4b_config) + diff --git a/sys/arm/arm/cpufunc_asm_sa1.S b/sys/arm/arm/cpufunc_asm_sa1.S index 0bdd6e7..99cd4f1 100644 --- a/sys/arm/arm/cpufunc_asm_sa1.S +++ b/sys/arm/arm/cpufunc_asm_sa1.S @@ -85,6 +85,8 @@ ENTRY(sa1_setttb) str r2, [r3] #endif RET +END(getttb) +END(sa1_setttb) /* * TLB functions @@ -93,6 +95,7 @@ ENTRY(sa1_tlb_flushID_SE) mcr p15, 0, r0, c8, c6, 1 /* flush D tlb single entry */ mcr p15, 0, r0, c8, c5, 0 /* flush I tlb */ RET +END(sa1_tlb_flushID_SE) /* * Cache functions @@ -100,22 +103,27 @@ ENTRY(sa1_tlb_flushID_SE) ENTRY(sa1_cache_flushID) mcr p15, 0, r0, c7, c7, 0 /* flush I+D cache */ RET +END(sa1_cache_flushID) ENTRY(sa1_cache_flushI) mcr p15, 0, r0, c7, c5, 0 /* flush I cache */ RET +END(sa1_cache_flushI) ENTRY(sa1_cache_flushD) mcr p15, 0, r0, c7, c6, 0 /* flush D cache */ RET +END(sa1_cache_flushD) ENTRY(sa1_cache_flushD_SE) mcr p15, 0, r0, c7, c6, 1 /* flush D cache single entry */ RET +END(sa1_cache_flushD_SE) ENTRY(sa1_cache_cleanD_E) mcr p15, 0, r0, c7, c10, 1 /* clean D cache entry */ RET +END(sa1_cache_cleanD_E) /* * Information for the SA-1 cache clean/purge functions: @@ -196,6 +204,11 @@ ENTRY(sa1_cache_cleanD) SA1_CACHE_CLEAN_EPILOGUE RET +END(sa1_cache_syncI) +END(sa1_cache_purgeID) +END(sa1_cache_cleanID) +END(sa1_cache_purgeD) +END(sa1_cache_cleanD) ENTRY(sa1_cache_purgeID_E) mcr p15, 0, r0, c7, c10, 1 /* clean dcache entry */ @@ -203,12 +216,14 @@ ENTRY(sa1_cache_purgeID_E) mcr p15, 0, r0, c7, c5, 0 /* flush I cache */ mcr p15, 0, r0, c7, c6, 1 /* flush D cache single entry */ RET +END(sa1_cache_purgeID_E) ENTRY(sa1_cache_purgeD_E) mcr p15, 0, r0, c7, c10, 1 /* clean dcache entry */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mcr p15, 0, r0, c7, c6, 1 /* flush D cache single entry */ RET +END(sa1_cache_purgeD_E) /* * Soft functions @@ -231,6 +246,8 @@ ENTRY(sa1_cache_cleanD_rng) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ RET +END(sa1_cache_cleanID_rng) +END(sa1_cache_cleanD_rng) ENTRY(sa1_cache_purgeID_rng) cmp r1, #0x4000 @@ -249,6 +266,7 @@ ENTRY(sa1_cache_purgeID_rng) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mcr p15, 0, r0, c7, c5, 0 /* flush I cache */ RET +END(sa1_cache_purgeID_rng) ENTRY(sa1_cache_purgeD_rng) cmp r1, #0x4000 @@ -266,6 +284,7 @@ ENTRY(sa1_cache_purgeD_rng) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ RET +END(sa1_cache_purgeD_rng) ENTRY(sa1_cache_syncI_rng) cmp r1, #0x4000 @@ -284,6 +303,7 @@ ENTRY(sa1_cache_syncI_rng) mcr p15, 0, r0, c7, c5, 0 /* flush I cache */ RET +END(sa1_cache_syncI_rng) /* * Context switch. @@ -313,4 +333,5 @@ ENTRY(sa110_context_switch) mov r0, r0 mov r0, r0 RET +END(sa110_context_switch) #endif diff --git a/sys/arm/arm/cpufunc_asm_sa11x0.S b/sys/arm/arm/cpufunc_asm_sa11x0.S index ca167c8..17efc8f 100644 --- a/sys/arm/arm/cpufunc_asm_sa11x0.S +++ b/sys/arm/arm/cpufunc_asm_sa11x0.S @@ -95,7 +95,7 @@ ENTRY(sa11x0_cpu_sleep) /* Restore interrupts (which will cause them to be serviced). */ msr cpsr_all, r3 RET - +END(sa11x0_cpu_sleep) /* * This function is the same as sa110_context_switch for now, the plan @@ -119,7 +119,10 @@ ENTRY(sa11x0_context_switch) mov r0, r0 mov r0, r0 RET +END(sa11x0_context_switch) ENTRY(sa11x0_drain_readbuf) mcr p15, 0, r0, c9, c0, 0 /* drain read buffer */ RET +END(sa11x0_drain_readbuf) + diff --git a/sys/arm/arm/cpufunc_asm_sheeva.S b/sys/arm/arm/cpufunc_asm_sheeva.S index d185547..796f63e 100644 --- a/sys/arm/arm/cpufunc_asm_sheeva.S +++ b/sys/arm/arm/cpufunc_asm_sheeva.S @@ -62,6 +62,7 @@ ENTRY(sheeva_setttb) mcr p15, 0, r0, c8, c7, 0 /* invalidate I+D TLBs */ RET +END(sheeva_setttb) ENTRY(sheeva_dcache_wbinv_range) str lr, [sp, #-4]! @@ -104,6 +105,7 @@ ENTRY(sheeva_dcache_wbinv_range) mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ ldr lr, [sp], #4 RET +END(sheeva_dcache_wbinv_range) ENTRY(sheeva_idcache_wbinv_range) str lr, [sp, #-4]! @@ -155,6 +157,7 @@ ENTRY(sheeva_idcache_wbinv_range) mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ ldr lr, [sp], #4 RET +END(sheeva_idcache_wbinv_range) ENTRY(sheeva_dcache_inv_range) str lr, [sp, #-4]! @@ -197,6 +200,7 @@ ENTRY(sheeva_dcache_inv_range) mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ ldr lr, [sp], #4 RET +END(sheeva_dcache_inv_range) ENTRY(sheeva_dcache_wb_range) str lr, [sp, #-4]! @@ -239,6 +243,7 @@ ENTRY(sheeva_dcache_wb_range) mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ ldr lr, [sp], #4 RET +END(sheeva_dcache_wb_range) ENTRY(sheeva_l2cache_wbinv_range) str lr, [sp, #-4]! @@ -283,6 +288,7 @@ ENTRY(sheeva_l2cache_wbinv_range) mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ ldr lr, [sp], #4 RET +END(sheeva_l2cache_wbinv_range) ENTRY(sheeva_l2cache_inv_range) str lr, [sp, #-4]! @@ -325,6 +331,7 @@ ENTRY(sheeva_l2cache_inv_range) mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ ldr lr, [sp], #4 RET +END(sheeva_l2cache_inv_range) ENTRY(sheeva_l2cache_wb_range) str lr, [sp, #-4]! @@ -367,6 +374,7 @@ ENTRY(sheeva_l2cache_wb_range) mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ ldr lr, [sp], #4 RET +END(sheeva_l2cache_wb_range) ENTRY(sheeva_l2cache_wbinv_all) mov r0, #0 @@ -374,6 +382,7 @@ ENTRY(sheeva_l2cache_wbinv_all) mcr p15, 1, r0, c15, c11, 0 /* Invalidate L2 */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ RET +END(sheeva_l2cache_wbinv_all) /* This function modifies register value as follows: * @@ -392,10 +401,12 @@ ENTRY(sheeva_control_ext) mcrne p15, 1, r2, c15, c1, 0 /* Write new control register */ mov r0, r3 /* Return old value */ RET +END(sheeva_control_ext) ENTRY(sheeva_cpu_sleep) mov r0, #0 mcr p15, 0, r0, c7, c10, 4 /* Drain write buffer */ mcr p15, 0, r0, c7, c0, 4 /* Wait for interrupt */ mov pc, lr +END(sheeva_cpu_sleep) diff --git a/sys/arm/arm/cpufunc_asm_xscale.S b/sys/arm/arm/cpufunc_asm_xscale.S index 3601b9a..56008dc 100644 --- a/sys/arm/arm/cpufunc_asm_xscale.S +++ b/sys/arm/arm/cpufunc_asm_xscale.S @@ -106,6 +106,7 @@ __FBSDID("$FreeBSD$"); ENTRY(xscale_cpwait) CPWAIT_AND_RETURN(r0) +END(xscale_cpwait) /* * We need a separate cpu_control() entry point, since we have to @@ -123,6 +124,7 @@ ENTRY(xscale_control) mov r0, r3 /* Return old value */ CPWAIT_AND_RETURN(r1) +END(xscale_control) /* * Functions to set the MMU Translation Table Base register @@ -167,6 +169,7 @@ ENTRY(xscale_setttb) str r2, [r3] #endif RET +END(xscale_setttb) /* * TLB functions @@ -176,6 +179,7 @@ ENTRY(xscale_tlb_flushID_SE) mcr p15, 0, r0, c8, c6, 1 /* flush D tlb single entry */ mcr p15, 0, r0, c8, c5, 1 /* flush I tlb single entry */ CPWAIT_AND_RETURN(r0) +END(xscale_tlb_flushID_SE) /* * Cache functions @@ -183,18 +187,22 @@ ENTRY(xscale_tlb_flushID_SE) ENTRY(xscale_cache_flushID) mcr p15, 0, r0, c7, c7, 0 /* flush I+D cache */ CPWAIT_AND_RETURN(r0) +END(xscale_cache_flushID) ENTRY(xscale_cache_flushI) mcr p15, 0, r0, c7, c5, 0 /* flush I cache */ CPWAIT_AND_RETURN(r0) +END(xscale_cache_flushI) ENTRY(xscale_cache_flushD) mcr p15, 0, r0, c7, c6, 0 /* flush D cache */ CPWAIT_AND_RETURN(r0) +END(xscale_cache_flushD) ENTRY(xscale_cache_flushI_SE) mcr p15, 0, r0, c7, c5, 1 /* flush I cache single entry */ CPWAIT_AND_RETURN(r0) +END(xscale_cache_flushI_SE) ENTRY(xscale_cache_flushD_SE) /* @@ -205,10 +213,12 @@ ENTRY(xscale_cache_flushD_SE) mcr p15, 0, r0, c7, c10, 1 mcr p15, 0, r0, c7, c6, 1 /* flush D cache single entry */ CPWAIT_AND_RETURN(r0) +END(xscale_cache_flushD_SE) ENTRY(xscale_cache_cleanD_E) mcr p15, 0, r0, c7, c10, 1 /* clean D cache entry */ CPWAIT_AND_RETURN(r0) +END(xscale_cache_cleanD_E) /* * Information for the XScale cache clean/purge functions: @@ -316,6 +326,11 @@ ENTRY(xscale_cache_cleanD) XSCALE_CACHE_CLEAN_EPILOGUE RET +END(xscale_cache_syncI) +END(xscale_cache_purgeID) +END(xscale_cache_cleanID) +END(xscale_cache_purgeD) +END(xscale_cache_cleanD) /* * Clean the mini-data cache. @@ -335,6 +350,7 @@ ENTRY(xscale_cache_clean_minidata) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ CPWAIT_AND_RETURN(r1) +END(xscale_cache_clean_minidata) ENTRY(xscale_cache_purgeID_E) mcr p15, 0, r0, c7, c10, 1 /* clean D cache entry */ @@ -343,6 +359,7 @@ ENTRY(xscale_cache_purgeID_E) mcr p15, 0, r0, c7, c5, 1 /* flush I cache single entry */ mcr p15, 0, r0, c7, c6, 1 /* flush D cache single entry */ CPWAIT_AND_RETURN(r1) +END(xscale_cache_purgeID_E) ENTRY(xscale_cache_purgeD_E) mcr p15, 0, r0, c7, c10, 1 /* clean D cache entry */ @@ -350,6 +367,7 @@ ENTRY(xscale_cache_purgeD_E) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mcr p15, 0, r0, c7, c6, 1 /* flush D cache single entry */ CPWAIT_AND_RETURN(r1) +END(xscale_cache_purgeD_E) /* * Soft functions @@ -375,6 +393,8 @@ ENTRY(xscale_cache_cleanD_rng) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ CPWAIT_AND_RETURN(r0) +END(xscale_cache_cleanID_rng) +END(xscale_cache_cleanD_rng) ENTRY(xscale_cache_purgeID_rng) cmp r1, #0x4000 @@ -396,6 +416,7 @@ ENTRY(xscale_cache_purgeID_rng) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ CPWAIT_AND_RETURN(r0) +END(xscale_cache_purgeID_rng) ENTRY(xscale_cache_purgeD_rng) cmp r1, #0x4000 @@ -416,6 +437,7 @@ ENTRY(xscale_cache_purgeD_rng) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ CPWAIT_AND_RETURN(r0) +END(xscale_cache_purgeD_rng) ENTRY(xscale_cache_syncI_rng) cmp r1, #0x4000 @@ -436,6 +458,7 @@ ENTRY(xscale_cache_syncI_rng) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ CPWAIT_AND_RETURN(r0) +END(xscale_cache_syncI_rng) ENTRY(xscale_cache_flushD_rng) and r2, r0, #0x1f @@ -450,6 +473,7 @@ ENTRY(xscale_cache_flushD_rng) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ CPWAIT_AND_RETURN(r0) +END(xscale_cache_flushD_rng) /* * Context switch. @@ -475,6 +499,7 @@ ENTRY(xscale_context_switch) mcr p15, 0, r0, c8, c7, 0 /* flush the I+D tlb */ CPWAIT_AND_RETURN(r0) +END(xscale_context_switch) /* * xscale_cpu_sleep @@ -493,3 +518,5 @@ ENTRY(xscale_cpu_sleep) 1: RET +END(xscale_cpu_sleep) + diff --git a/sys/arm/arm/cpufunc_asm_xscale_c3.S b/sys/arm/arm/cpufunc_asm_xscale_c3.S index 9a003d0..a0494d5 100644 --- a/sys/arm/arm/cpufunc_asm_xscale_c3.S +++ b/sys/arm/arm/cpufunc_asm_xscale_c3.S @@ -168,6 +168,11 @@ ENTRY(xscalec3_cache_cleanD) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ RET +END(xscalec3_cache_syncI) +END(xscalec3_cache_purgeID) +END(xscalec3_cache_cleanID) +END(xscalec3_cache_purgeD) +END(xscalec3_cache_cleanD) ENTRY(xscalec3_cache_purgeID_rng) @@ -189,6 +194,7 @@ ENTRY(xscalec3_cache_purgeID_rng) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ CPWAIT_AND_RETURN(r0) +END(xscalec3_cache_purgeID_rng) ENTRY(xscalec3_cache_syncI_rng) cmp r1, #0x4000 @@ -209,6 +215,7 @@ ENTRY(xscalec3_cache_syncI_rng) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ CPWAIT_AND_RETURN(r0) +END(xscalec3_cache_syncI_rng) ENTRY(xscalec3_cache_purgeD_rng) @@ -228,6 +235,8 @@ ENTRY(xscalec3_cache_purgeD_rng) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ CPWAIT_AND_RETURN(r0) +END(xscalec3_cache_purgeD_rng) + ENTRY(xscalec3_cache_cleanID_rng) ENTRY(xscalec3_cache_cleanD_rng) @@ -248,7 +257,8 @@ ENTRY(xscalec3_cache_cleanD_rng) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ CPWAIT_AND_RETURN(r0) - +END(xscalec3_cache_cleanID_rng) +END(xscalec3_cache_cleanD_rng) ENTRY(xscalec3_l2cache_purge) /* Clean-up the L2 cache */ @@ -271,6 +281,7 @@ ENTRY(xscalec3_l2cache_purge) CPWAIT(r0) mcr p15, 0, r0, c7, c10, 5 /* Data memory barrier */ RET +END(xscalec3_l2cache_purge) ENTRY(xscalec3_l2cache_clean_rng) mcr p15, 0, r0, c7, c10, 5 /* Data memory barrier */ @@ -291,6 +302,7 @@ ENTRY(xscalec3_l2cache_clean_rng) mcr p15, 0, r0, c7, c10, 5 CPWAIT_AND_RETURN(r0) +END(xscalec3_l2cache_clean_rng) ENTRY(xscalec3_l2cache_purge_rng) @@ -310,6 +322,7 @@ ENTRY(xscalec3_l2cache_purge_rng) mcr p15, 0, r0, c7, c10, 5 CPWAIT_AND_RETURN(r0) +END(xscalec3_l2cache_purge_rng) ENTRY(xscalec3_l2cache_flush_rng) mcr p15, 0, r0, c7, c10, 5 /* Data memory barrier */ @@ -325,6 +338,8 @@ ENTRY(xscalec3_l2cache_flush_rng) mcr p15, 0, r0, c7, c10, 4 @ data write barrier mcr p15, 0, r0, c7, c10, 5 CPWAIT_AND_RETURN(r0) +END(xscalec3_l2cache_flush_rng) + /* * Functions to set the MMU Translation Table Base register * @@ -368,6 +383,7 @@ ENTRY(xscalec3_setttb) str r2, [r3] #endif RET +END(xscalec3_setttb) /* * Context switch. @@ -395,3 +411,5 @@ ENTRY(xscalec3_context_switch) mcr p15, 0, r0, c8, c7, 0 /* flush the I+D tlb */ CPWAIT_AND_RETURN(r0) +END(xscalec3_context_switch) + diff --git a/sys/arm/arm/exception.S b/sys/arm/arm/exception.S index b2ce47b..2ff0840 100644 --- a/sys/arm/arm/exception.S +++ b/sys/arm/arm/exception.S @@ -70,6 +70,7 @@ ASENTRY_NP(reset_entry) Lreset_panicmsg: .asciz "Reset vector called, LR = 0x%08x" .balign 4 +END(reset_entry) /* * swi_entry @@ -77,8 +78,7 @@ Lreset_panicmsg: * Handler for the Software Interrupt exception. */ ASENTRY_NP(swi_entry) - .fnstart - .cantunwind /* Don't unwind past here */ + STOP_UNWINDING /* Don't unwind past here */ PUSHFRAME @@ -91,7 +91,7 @@ ASENTRY_NP(swi_entry) DO_AST PULLFRAME movs pc, lr /* Exit */ - .fnend +END(swi_entry) /* * prefetch_abort_entry: @@ -128,6 +128,7 @@ abortprefetch: abortprefetchmsg: .asciz "abortprefetch" .align 0 +END(prefetch_abort_entry) /* * data_abort_entry: @@ -163,6 +164,7 @@ abortdata: abortdatamsg: .asciz "abortdata" .align 0 +END(data_abort_entry) /* * address_exception_entry: @@ -183,6 +185,7 @@ ASENTRY_NP(address_exception_entry) Laddress_exception_msg: .asciz "Address Exception CPSR=0x%08x SPSR=0x%08x LR=0x%08x\n" .balign 4 +END(address_exception_entry) /* * General exception exit handler @@ -224,6 +227,7 @@ ASENTRY_NP(undefined_entry) Lundefined_handler_indirection: .word Lundefined_handler_indirection_data +END(undefined_entry) /* * assembly bounce code for calling the kernel @@ -254,3 +258,4 @@ Lundefined_handler_indirection_data: .global _C_LABEL(undefined_handler_address) _C_LABEL(undefined_handler_address): .word _C_LABEL(undefinedinstruction_bounce) +END(undefinedinstruction_bounce) diff --git a/sys/arm/arm/fiq_subr.S b/sys/arm/arm/fiq_subr.S index 4cde665..7f510b2 100644 --- a/sys/arm/arm/fiq_subr.S +++ b/sys/arm/arm/fiq_subr.S @@ -74,6 +74,7 @@ ENTRY(fiq_getregs) BACK_TO_SVC_MODE RET +END(fiq_getregs) /* * fiq_setregs: @@ -88,6 +89,7 @@ ENTRY(fiq_setregs) BACK_TO_SVC_MODE RET +END(fiq_setregs) /* * fiq_nullhandler: diff --git a/sys/arm/arm/fusu.S b/sys/arm/arm/fusu.S index edf1a63..443ca21 100644 --- a/sys/arm/arm/fusu.S +++ b/sys/arm/arm/fusu.S @@ -76,6 +76,8 @@ ENTRY(casuword) mov r1, #0x00000000 str r1, [r3, #PCB_ONFAULT] RET +END(casuword32) +END(casuword) /* * Handle faults from casuword. Clean up and return -1. @@ -87,6 +89,7 @@ ENTRY(casuword) mvn r0, #0x00000000 ldmfd sp!, {r4, r5} RET + /* * fuword(caddr_t uaddr); * Fetch an int from the user's address space. @@ -111,6 +114,8 @@ ENTRY(fuword) str r1, [r2, #PCB_ONFAULT] mov r0, r3 RET +END(fuword32) +END(fuword) /* * fusword(caddr_t uaddr); @@ -139,6 +144,7 @@ ENTRY(fusword) mov r1, #0x00000000 str r1, [r2, #PCB_ONFAULT] RET +END(fusword) /* * fuswintr(caddr_t uaddr); @@ -175,6 +181,7 @@ ENTRY(fuswintr) mov r1, #0x00000000 str r1, [r2, #PCB_ONFAULT] RET +END(fuswintr) Lblock_userspace_access: .word _C_LABEL(block_userspace_access) @@ -209,6 +216,7 @@ ENTRY(fubyte) str r1, [r2, #PCB_ONFAULT] mov r0, r3 RET +END(fubyte) /* * Handle faults from [fs]u*(). Clean up and return -1. @@ -272,6 +280,8 @@ ENTRY(suword) mov r0, #0x00000000 str r0, [r2, #PCB_ONFAULT] RET +END(suword32) +END(suword) /* * suswintr(caddr_t uaddr, short x); @@ -309,6 +319,7 @@ ENTRY(suswintr) mov r0, #0x00000000 str r0, [r2, #PCB_ONFAULT] RET +END(suswintr) /* * susword(caddr_t uaddr, short x); @@ -339,6 +350,7 @@ ENTRY(susword) mov r0, #0x00000000 str r0, [r2, #PCB_ONFAULT] RET +END(susword) /* * subyte(caddr_t uaddr, char x); @@ -362,3 +374,5 @@ ENTRY(subyte) mov r0, #0x00000000 str r0, [r2, #PCB_ONFAULT] RET +END(subyte) + diff --git a/sys/arm/arm/in_cksum_arm.S b/sys/arm/arm/in_cksum_arm.S index 3646c64..6305caf 100644 --- a/sys/arm/arm/in_cksum_arm.S +++ b/sys/arm/arm/in_cksum_arm.S @@ -90,13 +90,15 @@ ENTRY(in_cksum) and r0, r0, r1 eor r0, r0, r1 ldmfd sp!, {r4-r11,pc} - +END(in_cksum) ENTRY(do_cksum) stmfd sp!, {r4-r7, lr} bl L_cksumdata mov r0, r2 ldmfd sp!, {r4-r7, pc} +END(do_cksum) + /* * The main in*_cksum() workhorse... * @@ -337,3 +339,5 @@ ASENTRY_NP(L_cksumdata) adds r2, r2, r3 adc r2, r2, #0x00 RET +END(L_cksumdata) + diff --git a/sys/arm/arm/irq_dispatch.S b/sys/arm/arm/irq_dispatch.S index 6e510dd..823091d 100644 --- a/sys/arm/arm/irq_dispatch.S +++ b/sys/arm/arm/irq_dispatch.S @@ -97,6 +97,7 @@ ASENTRY_NP(irq_entry) DO_AST PULLFRAMEFROMSVCANDEXIT movs pc, lr /* Exit */ +END(irq_entry) .data .align 0 diff --git a/sys/arm/arm/locore.S b/sys/arm/arm/locore.S index 37e88fe..51fd5c1 100644 --- a/sys/arm/arm/locore.S +++ b/sys/arm/arm/locore.S @@ -242,6 +242,9 @@ Lstartup_pagetable: Lstartup_pagetable_secondary: .word temp_pagetable #endif +END(btext) +END(_start) + mmu_init_table: /* fill all table VA==PA */ /* map SDRAM VA==PA, WT cacheable */ @@ -324,6 +327,7 @@ ASENTRY_NP(mptramp) Lpmureg: .word 0xd0022124 +END(mptramp) ASENTRY_NP(mpentry) @@ -408,6 +412,7 @@ mpvirt_done: .Lmpreturned: .asciz "main() returned" .align 0 +END(mpentry) #endif ENTRY_NP(cpu_halt) @@ -461,6 +466,7 @@ ENTRY_NP(cpu_halt) */ .Lcpu_reset_needs_v4_MMU_disable: .word _C_LABEL(cpu_reset_needs_v4_MMU_disable) +END(cpu_halt) /* @@ -470,11 +476,13 @@ ENTRY(setjmp) stmia r0, {r4-r14} mov r0, #0x00000000 RET +END(setjmp) ENTRY(longjmp) ldmia r0, {r4-r14} mov r0, #0x00000001 RET +END(longjmp) .data .global _C_LABEL(esym) @@ -482,6 +490,7 @@ _C_LABEL(esym): .word _C_LABEL(end) ENTRY_NP(abort) b _C_LABEL(abort) +END(abort) ENTRY_NP(sigcode) mov r0, sp @@ -517,4 +526,5 @@ ENTRY_NP(sigcode) .global szsigcode szsigcode: .long esigcode-sigcode +END(sigcode) /* End of locore.S */ diff --git a/sys/arm/arm/setcpsr.S b/sys/arm/arm/setcpsr.S index 4597d53..ac86ba3 100644 --- a/sys/arm/arm/setcpsr.S +++ b/sys/arm/arm/setcpsr.S @@ -66,6 +66,7 @@ ENTRY_NP(SetCPSR) mov r0, r3 /* Return the old CPSR */ RET +END(SetCPSR) /* Gets the CPSR register @@ -77,4 +78,5 @@ ENTRY_NP(GetCPSR) mrs r0, cpsr /* Get the CPSR */ RET +END(GetCPSR) diff --git a/sys/arm/arm/support.S b/sys/arm/arm/support.S index d4c6fb4..0c117a9 100644 --- a/sys/arm/arm/support.S +++ b/sys/arm/arm/support.S @@ -277,6 +277,8 @@ do_memset: strgeb r3, [ip], #0x01 /* Set another byte */ strgtb r3, [ip] /* and a third */ RET /* Exit */ +END(bzero) +END(memset) ENTRY(bcmp) mov ip, r0 @@ -386,6 +388,7 @@ ENTRY(bcmp) RETne /* Return if mismatch on #4 */ sub r0, r3, r2 /* r0 = b1#5 - b2#5 */ RET +END(bcmp) ENTRY(bcopy) /* switch the source and destination registers */ @@ -929,6 +932,8 @@ ENTRY(memmove) .Lmemmove_bsrcul1l4: add r1, r1, #1 b .Lmemmove_bl4 +END(bcopy) +END(memmove) #if !defined(_ARM_ARCH_5E) ENTRY(memcpy) @@ -1164,6 +1169,8 @@ ENTRY(memcpy) .Lmemcpy_srcul3l4: sub r1, r1, #1 b .Lmemcpy_l4 +END(memcpy) + #else /* LINTSTUB: Func: void *memcpy(void *dst, const void *src, size_t len) */ ENTRY(memcpy) @@ -2932,6 +2939,7 @@ ENTRY(memcpy) strh r2, [r0, #0x09] strb r1, [r0, #0x0b] RET +END(memcpy) #endif /* _ARM_ARCH_5E */ #ifdef GPROF diff --git a/sys/arm/arm/swtch.S b/sys/arm/arm/swtch.S index 4257557..f10b8f9 100644 --- a/sys/arm/arm/swtch.S +++ b/sys/arm/arm/swtch.S @@ -213,6 +213,7 @@ ENTRY(cpu_throw) add sp, sp, #4; ldmfd sp!, {r4-r7, pc} +END(cpu_throw) ENTRY(cpu_switch) stmfd sp!, {r4-r7, lr} @@ -502,6 +503,8 @@ ENTRY(cpu_switch) .Lswitch_panic_str: .asciz "cpu_switch: sched_qs empty with non-zero sched_whichqs!\n" #endif +END(cpu_switch) + ENTRY(savectx) stmfd sp!, {r4-r7, lr} sub sp, sp, #4 @@ -534,6 +537,7 @@ ENTRY(savectx) #endif /* ARM_VFP_SUPPORT */ add sp, sp, #4; ldmfd sp!, {r4-r7, pc} +END(savectx) ENTRY(fork_trampoline) mov r1, r5 @@ -551,3 +555,5 @@ ENTRY(fork_trampoline) movs pc, lr /* Exit */ AST_LOCALS +END(fork_trampoline) + diff --git a/sys/arm/include/asm.h b/sys/arm/include/asm.h index 3ae25b8..81f67a3 100644 --- a/sys/arm/include/asm.h +++ b/sys/arm/include/asm.h @@ -66,6 +66,16 @@ # define _ALIGN_TEXT .align 0 #endif +#ifdef __ARM_EABI__ +#define STOP_UNWINDING .cantunwind +#define _FNSTART .fnstart +#define _FNEND .fnend +#else +#define STOP_UNWINDING +#define _FNSTART +#define _FNEND +#endif + /* * gas/arm uses @ as a single comment character and thus cannot be used here * Instead it recognised the # instead of an @ symbols in .type directives @@ -76,7 +86,9 @@ #define _ASM_TYPE_OBJECT #object #define GLOBAL(X) .globl x #define _ENTRY(x) \ - .text; _ALIGN_TEXT; .globl x; .type x,_ASM_TYPE_FUNCTION; x: + .text; _ALIGN_TEXT; .globl x; .type x,_ASM_TYPE_FUNCTION; x: _FNSTART + +#define END(x) .size x, . - x; _FNEND #ifdef GPROF # define _PROF_PROLOGUE \ -- cgit v1.1 From a27e52f9c0c2593dcbcd77d40785bf5ff391b527 Mon Sep 17 00:00:00 2001 From: andrew Date: Sat, 16 Mar 2013 03:15:24 +0000 Subject: Adjust the indentation of the trampoline compilation to make the commands easier to follow. --- sys/conf/Makefile.arm | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) (limited to 'sys') diff --git a/sys/conf/Makefile.arm b/sys/conf/Makefile.arm index 8e3b82e..06a5d98 100644 --- a/sys/conf/Makefile.arm +++ b/sys/conf/Makefile.arm @@ -82,44 +82,43 @@ ${KERNEL_KO}.tramp: ${KERNEL_KO} $S/$M/$M/inckern.S $S/$M/$M/elf_trampoline.c echo "#define KERNNAME \"${KERNEL_KO}.tmp\"" >opt_kernname.h sed s/${KERNVIRTADDR}/${KERNPHYSADDR}/ ldscript.$M > ldscript.$M.tramp sed s/" + SIZEOF_HEADERS"// ldscript.$M.tramp > \ - ldscript.$M.tramp.noheader + ldscript.$M.tramp.noheader echo "#include " >tmphack.S echo "ENTRY(_start)" >>tmphack.S echo "bl _startC" >>tmphack.S ${OBJCOPY} --strip-symbol '$$d' --strip-symbol '$$a' \ - -g --strip-symbol '$$t' ${FULLKERNEL} ${KERNEL_KO}.tmp + -g --strip-symbol '$$t' ${FULLKERNEL} ${KERNEL_KO}.tmp eval $$(stat -s ${KERNEL_KO}.tmp) && \ - echo "#define KERNSIZE $$st_size" >>opt_kernname.h + echo "#define KERNSIZE $$st_size" >>opt_kernname.h ${CC} -O -nostdlib -I. -I$S -Xlinker -T -Xlinker ldscript.$M.tramp \ - tmphack.S $S/$M/$M/elf_trampoline.c $S/$M/$M/inckern.S \ - ${FILES_CPU_FUNC} -o ${KERNEL_KO}.tramp + tmphack.S $S/$M/$M/elf_trampoline.c $S/$M/$M/inckern.S \ + ${FILES_CPU_FUNC} -o ${KERNEL_KO}.tramp ${CC} -O -nostdlib -I. -I$S -Xlinker -T -Xlinker \ - ldscript.$M.tramp.noheader \ - tmphack.S $S/$M/$M/elf_trampoline.c $S/$M/$M/inckern.S \ + ldscript.$M.tramp.noheader \ + tmphack.S $S/$M/$M/elf_trampoline.c $S/$M/$M/inckern.S \ ${FILES_CPU_FUNC} -o ${KERNEL_KO}.tramp.noheader - ${OBJCOPY} -S -O binary ${KERNEL_KO}.tramp.noheader \ - ${KERNEL_KO}.tramp.bin + ${OBJCOPY} -S -O binary ${KERNEL_KO}.tramp.noheader \ + ${KERNEL_KO}.tramp.bin ${OBJCOPY} ${STRIP_FLAGS} ${KERNEL_KO}.tmp - echo "#define KERNNAME \"${KERNEL_KO}.tmp.gz\"" \ - >opt_kernname.h + echo "#define KERNNAME \"${KERNEL_KO}.tmp.gz\"" >opt_kernname.h eval $$(stat -s ${KERNEL_KO}.tmp) && \ - echo "#define KERNSIZE $$st_size" >>opt_kernname.h + echo "#define KERNSIZE $$st_size" >>opt_kernname.h gzip -f9 ${KERNEL_KO}.tmp eval $$(stat -s ${KERNEL_KO}.tmp.gz) && \ - echo "#define KERNCOMPSIZE $$st_size" >>opt_kernname.h + echo "#define KERNCOMPSIZE $$st_size" >>opt_kernname.h ${CC} -O2 -ffreestanding -DKZIP -I. -I$S -c $S/kern/inflate.c -o \ inflate-tramp.o ${CC} -O -nostdlib -I. -I$S -Xlinker -T -Xlinker ldscript.$M.tramp \ - -DKZIP tmphack.S $S/$M/$M/elf_trampoline.c inflate-tramp.o \ - $S/$M/$M/inckern.S ${FILES_CPU_FUNC} -o ${KERNEL_KO}.gz.tramp + -DKZIP tmphack.S $S/$M/$M/elf_trampoline.c inflate-tramp.o \ + $S/$M/$M/inckern.S ${FILES_CPU_FUNC} -o ${KERNEL_KO}.gz.tramp ${CC} -O -nostdlib -I. -I$S -Xlinker -T -Xlinker \ - ldscript.$M.tramp.noheader \ - -DKZIP tmphack.S $S/$M/$M/elf_trampoline.c inflate-tramp.o \ - $S/$M/$M/inckern.S ${FILES_CPU_FUNC} -o ${KERNEL_KO}.tramp.noheader + ldscript.$M.tramp.noheader \ + -DKZIP tmphack.S $S/$M/$M/elf_trampoline.c inflate-tramp.o \ + $S/$M/$M/inckern.S ${FILES_CPU_FUNC} -o ${KERNEL_KO}.tramp.noheader ${OBJCOPY} -S -O binary ${KERNEL_KO}.tramp.noheader \ ${KERNEL_KO}.gz.tramp.bin rm ${KERNEL_KO}.tmp.gz ${KERNEL_KO}.tramp.noheader opt_kernname.h \ - inflate-tramp.o tmphack.S + inflate-tramp.o tmphack.S MKMODULESENV+= MACHINE=${MACHINE} -- cgit v1.1 From 547c9f6633a015a09f4cbfa963c4b0643f702ca1 Mon Sep 17 00:00:00 2001 From: andrew Date: Sat, 16 Mar 2013 03:21:25 +0000 Subject: Fix the indentation for a few commands that were missed or incorrectly indented in r248362. --- sys/conf/Makefile.arm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'sys') diff --git a/sys/conf/Makefile.arm b/sys/conf/Makefile.arm index 06a5d98..20e7b71 100644 --- a/sys/conf/Makefile.arm +++ b/sys/conf/Makefile.arm @@ -96,8 +96,8 @@ ${KERNEL_KO}.tramp: ${KERNEL_KO} $S/$M/$M/inckern.S $S/$M/$M/elf_trampoline.c ${CC} -O -nostdlib -I. -I$S -Xlinker -T -Xlinker \ ldscript.$M.tramp.noheader \ tmphack.S $S/$M/$M/elf_trampoline.c $S/$M/$M/inckern.S \ - ${FILES_CPU_FUNC} -o ${KERNEL_KO}.tramp.noheader - ${OBJCOPY} -S -O binary ${KERNEL_KO}.tramp.noheader \ + ${FILES_CPU_FUNC} -o ${KERNEL_KO}.tramp.noheader + ${OBJCOPY} -S -O binary ${KERNEL_KO}.tramp.noheader \ ${KERNEL_KO}.tramp.bin ${OBJCOPY} ${STRIP_FLAGS} ${KERNEL_KO}.tmp echo "#define KERNNAME \"${KERNEL_KO}.tmp.gz\"" >opt_kernname.h @@ -116,7 +116,7 @@ ${KERNEL_KO}.tramp: ${KERNEL_KO} $S/$M/$M/inckern.S $S/$M/$M/elf_trampoline.c -DKZIP tmphack.S $S/$M/$M/elf_trampoline.c inflate-tramp.o \ $S/$M/$M/inckern.S ${FILES_CPU_FUNC} -o ${KERNEL_KO}.tramp.noheader ${OBJCOPY} -S -O binary ${KERNEL_KO}.tramp.noheader \ - ${KERNEL_KO}.gz.tramp.bin + ${KERNEL_KO}.gz.tramp.bin rm ${KERNEL_KO}.tmp.gz ${KERNEL_KO}.tramp.noheader opt_kernname.h \ inflate-tramp.o tmphack.S -- cgit v1.1 From 14fe99e96981427528fbea82d97c4b81a73ff8b6 Mon Sep 17 00:00:00 2001 From: andrew Date: Sat, 16 Mar 2013 03:50:27 +0000 Subject: Implement the required but unused __aeabi_unwind_cpp_* functions in the trampoline kernel. --- sys/arm/arm/elf_trampoline.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'sys') diff --git a/sys/arm/arm/elf_trampoline.c b/sys/arm/arm/elf_trampoline.c index 31e8bc5..fe03adf 100644 --- a/sys/arm/arm/elf_trampoline.c +++ b/sys/arm/arm/elf_trampoline.c @@ -701,3 +701,18 @@ __start(void) do_call(dst, kernel, dst + (unsigned int)(&func_end) - (unsigned int)(&load_kernel) + 800, sp); } + +#ifdef __ARM_EABI__ +/* We need to provide these functions but never call them */ +void __aeabi_unwind_cpp_pr0(void); +void __aeabi_unwind_cpp_pr1(void); +void __aeabi_unwind_cpp_pr2(void); + +__strong_reference(__aeabi_unwind_cpp_pr0, __aeabi_unwind_cpp_pr1); +__strong_reference(__aeabi_unwind_cpp_pr0, __aeabi_unwind_cpp_pr2); +void +__aeabi_unwind_cpp_pr0(void) +{ +} +#endif + -- cgit v1.1 From 2b4b00f4f6e62b0d54e096333cfffb05d33ccd30 Mon Sep 17 00:00:00 2001 From: andrew Date: Sat, 16 Mar 2013 03:57:46 +0000 Subject: The compiler argument -mno-apcs-frame has no meaning when using EABI as we will use aapcs frames, not apcs frames. --- sys/conf/Makefile.arm | 2 ++ 1 file changed, 2 insertions(+) (limited to 'sys') diff --git a/sys/conf/Makefile.arm b/sys/conf/Makefile.arm index 20e7b71..10939db 100644 --- a/sys/conf/Makefile.arm +++ b/sys/conf/Makefile.arm @@ -44,7 +44,9 @@ CFLAGS += -mno-thumb-interwork .endif .if empty(DDB_ENABLED) +.if !defined(WITH_ARM_EABI) CFLAGS += -mno-apcs-frame +.endif .elif defined(WITH_ARM_EABI) CFLAGS += -funwind-tables .if ${COMPILER_TYPE} == "clang" -- cgit v1.1 From 147f0b89f3dc8b52074ba05825f0e444127c50f2 Mon Sep 17 00:00:00 2001 From: andrew Date: Sat, 16 Mar 2013 04:06:49 +0000 Subject: Move the __aeabi_unwind_cpp_pr{0,1,2} functions to libkern so they can be referenced in a non-debug kernel. --- sys/arm/arm/db_trace.c | 23 ----------------- sys/conf/files.arm | 1 + sys/libkern/arm/aeabi_unwind.c | 58 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 23 deletions(-) create mode 100644 sys/libkern/arm/aeabi_unwind.c (limited to 'sys') diff --git a/sys/arm/arm/db_trace.c b/sys/arm/arm/db_trace.c index f921cfd..90ac0b1 100644 --- a/sys/arm/arm/db_trace.c +++ b/sys/arm/arm/db_trace.c @@ -126,29 +126,6 @@ struct unwind_state { uint16_t update_mask; }; -/* We need to provide these but never use them */ -void __aeabi_unwind_cpp_pr0(void); -void __aeabi_unwind_cpp_pr1(void); -void __aeabi_unwind_cpp_pr2(void); - -void -__aeabi_unwind_cpp_pr0(void) -{ - panic("__aeabi_unwind_cpp_pr0"); -} - -void -__aeabi_unwind_cpp_pr1(void) -{ - panic("__aeabi_unwind_cpp_pr1"); -} - -void -__aeabi_unwind_cpp_pr2(void) -{ - panic("__aeabi_unwind_cpp_pr2"); -} - /* Expand a 31-bit signed value to a 32-bit signed value */ static __inline int32_t db_expand_prel31(uint32_t prel31) diff --git a/sys/conf/files.arm b/sys/conf/files.arm index 33ec0fb..6e439b4 100644 --- a/sys/conf/files.arm +++ b/sys/conf/files.arm @@ -72,6 +72,7 @@ font.h optional sc \ clean "font.h ${SC_DFLT_FONT}-8x14 ${SC_DFLT_FONT}-8x16 ${SC_DFLT_FONT}-8x8" kern/subr_busdma_bufalloc.c standard kern/subr_dummy_vdso_tc.c standard +libkern/arm/aeabi_unwind.c standard libkern/arm/divsi3.S standard libkern/arm/ffs.S standard libkern/arm/ldivmod.S standard diff --git a/sys/libkern/arm/aeabi_unwind.c b/sys/libkern/arm/aeabi_unwind.c new file mode 100644 index 0000000..098e6e6 --- /dev/null +++ b/sys/libkern/arm/aeabi_unwind.c @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2013 Andrew Turner + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#ifdef __ARM_EABI__ +/* We need to provide these functions never call them */ +void __aeabi_unwind_cpp_pr0(void); +void __aeabi_unwind_cpp_pr1(void); +void __aeabi_unwind_cpp_pr2(void); + +void +__aeabi_unwind_cpp_pr0(void) +{ + panic("__aeabi_unwind_cpp_pr0"); +} + +void +__aeabi_unwind_cpp_pr1(void) +{ + panic("__aeabi_unwind_cpp_pr1"); +} + +void +__aeabi_unwind_cpp_pr2(void) +{ + panic("__aeabi_unwind_cpp_pr2"); +} +#endif + -- cgit v1.1 From faaba46948fb2a2f0c76f4ef57dc6d7384fd55fe Mon Sep 17 00:00:00 2001 From: andrew Date: Sat, 16 Mar 2013 04:08:01 +0000 Subject: Add END to ARM libkern assembly functions --- sys/libkern/arm/divsi3.S | 13 +++++++++++++ sys/libkern/arm/ffs.S | 2 ++ sys/libkern/arm/ldivmod.S | 2 ++ sys/libkern/arm/memcpy.S | 3 ++- 4 files changed, 19 insertions(+), 1 deletion(-) (limited to 'sys') diff --git a/sys/libkern/arm/divsi3.S b/sys/libkern/arm/divsi3.S index 700ae37..302f179 100644 --- a/sys/libkern/arm/divsi3.S +++ b/sys/libkern/arm/divsi3.S @@ -29,6 +29,7 @@ ENTRY_NP(__umodsi3) add sp, sp, #4 /* unalign stack */ mov r0, r1 ldmfd sp!, {pc} +END(__umodsi3) ENTRY_NP(__modsi3) stmfd sp!, {lr} @@ -48,6 +49,7 @@ ENTRY_NP(__modsi3) mvn r0, #0 #endif RET +END(__modsi3) #ifdef __ARM_EABI__ ENTRY_NP(__aeabi_uidiv) @@ -74,6 +76,11 @@ ENTRY_NP(__udivsi3) mov r0, r1 mov r1, #0 RET +#ifdef __ARM_EABI__ +END(__aeabi_uidiv) +END(__aeabi_uidivmod) +#endif +END(__udivsi3) #ifdef __ARM_EABI__ ENTRY_NP(__aeabi_idiv) @@ -393,3 +400,9 @@ ENTRY_NP(__divsi3) addhs r3, r3, r2 mov r0, r3 RET +#ifdef __ARM_EABI__ +END(__aeabi_idiv) +END(__aeabi_idivmod) +#endif +END(__divsi3) + diff --git a/sys/libkern/arm/ffs.S b/sys/libkern/arm/ffs.S index ba0af49..a43f2b6 100644 --- a/sys/libkern/arm/ffs.S +++ b/sys/libkern/arm/ffs.S @@ -82,3 +82,5 @@ ENTRY(ffs) rsbne r0, r0, #32 RET #endif +END(ffs) + diff --git a/sys/libkern/arm/ldivmod.S b/sys/libkern/arm/ldivmod.S index a88db54..26a3944 100644 --- a/sys/libkern/arm/ldivmod.S +++ b/sys/libkern/arm/ldivmod.S @@ -53,6 +53,7 @@ ENTRY_NP(__aeabi_ldivmod) add sp, sp, #8 /* Move sp to the remainder value */ ldmfd sp!, {r2, r3} /* Load the remainder */ RET +END(__aeabi_ldivmod) ENTRY_NP(__aeabi_uldivmod) sub sp, sp, #8 /* Space for the remainder */ @@ -62,6 +63,7 @@ ENTRY_NP(__aeabi_uldivmod) add sp, sp, #8 /* Move sp to the remainder value */ ldmfd sp!, {r2, r3} /* Load the remainder */ RET +END(__aeabi_uldivmod) #endif diff --git a/sys/libkern/arm/memcpy.S b/sys/libkern/arm/memcpy.S index b2997db..9fca8f6 100644 --- a/sys/libkern/arm/memcpy.S +++ b/sys/libkern/arm/memcpy.S @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012 Andrew Turner + * Copyright (C) 2013 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -32,6 +32,7 @@ __FBSDID("$FreeBSD$"); ENTRY_NP(__aeabi_memcpy) b memcpy +END(__aeabi_memcpy) #endif -- cgit v1.1 From 9d8d25cc83c979fb0b6573f16488716d02f01aa4 Mon Sep 17 00:00:00 2001 From: glebius Date: Sat, 16 Mar 2013 08:55:21 +0000 Subject: Contrary to what the deleted comment said, the m_move_pkthdr() will not smash the M_EXT and data pointer, so it is safe to pass an mbuf with external storage procuded by m_getcl() to m_move_pkthdr(). Reviewed by: andre Sponsored by: Nginx, Inc. --- sys/kern/uipc_mbuf.c | 45 ++++++++++----------------------------------- 1 file changed, 10 insertions(+), 35 deletions(-) (limited to 'sys') diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c index e2b008a..2107730 100644 --- a/sys/kern/uipc_mbuf.c +++ b/sys/kern/uipc_mbuf.c @@ -1968,43 +1968,18 @@ m_unshare(struct mbuf *m0, int how) } /* - * Allocate new space to hold the copy... - */ - /* XXX why can M_PKTHDR be set past the first mbuf? */ - if (mprev == NULL && (m->m_flags & M_PKTHDR)) { - /* - * NB: if a packet header is present we must - * allocate the mbuf separately from any cluster - * because M_MOVE_PKTHDR will smash the data - * pointer and drop the M_EXT marker. - */ - MGETHDR(n, how, m->m_type); - if (n == NULL) { - m_freem(m0); - return (NULL); - } - M_MOVE_PKTHDR(n, m); - MCLGET(n, how); - if ((n->m_flags & M_EXT) == 0) { - m_free(n); - m_freem(m0); - return (NULL); - } - } else { - n = m_getcl(how, m->m_type, m->m_flags); - if (n == NULL) { - m_freem(m0); - return (NULL); - } - } - /* - * ... and copy the data. We deal with jumbo mbufs - * (i.e. m_len > MCLBYTES) by splitting them into - * clusters. We could just malloc a buffer and make - * it external but too many device drivers don't know - * how to break up the non-contiguous memory when + * Allocate new space to hold the copy and copy the data. + * We deal with jumbo mbufs (i.e. m_len > MCLBYTES) by + * splitting them into clusters. We could just malloc a + * buffer and make it external but too many device drivers + * don't know how to break up the non-contiguous memory when * doing DMA. */ + n = m_getcl(how, m->m_type, m->m_flags); + if (n == NULL) { + m_freem(m0); + return (NULL); + } len = m->m_len; off = 0; mfirst = n; -- cgit v1.1 From 9edd6e11740d739eabb503b17a6aeb26168b8f71 Mon Sep 17 00:00:00 2001 From: glebius Date: Sat, 16 Mar 2013 08:57:36 +0000 Subject: - Replace compat macros with function calls. - Remove superfluous cleaning of m_len after allocating. Sponsored by: Nginx, Inc. --- sys/kern/uipc_mbuf.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) (limited to 'sys') diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c index 2107730..621a3eb 100644 --- a/sys/kern/uipc_mbuf.c +++ b/sys/kern/uipc_mbuf.c @@ -530,8 +530,8 @@ m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how) #if 0 /* * The mbuf allocator only initializes the pkthdr - * when the mbuf is allocated with MGETHDR. Many users - * (e.g. m_copy*, m_prepend) use MGET and then + * when the mbuf is allocated with m_gethdr(). Many users + * (e.g. m_copy*, m_prepend) use m_get() and then * smash the pkthdr as needed causing these * assertions to trip. For now just disable them. */ @@ -563,15 +563,15 @@ m_prepend(struct mbuf *m, int len, int how) struct mbuf *mn; if (m->m_flags & M_PKTHDR) - MGETHDR(mn, how, m->m_type); + mn = m_gethdr(how, m->m_type); else - MGET(mn, how, m->m_type); + mn = m_get(how, m->m_type); if (mn == NULL) { m_freem(m); return (NULL); } if (m->m_flags & M_PKTHDR) - M_MOVE_PKTHDR(mn, m); + m_move_pkthdr(mn, m); mn->m_next = m; m = mn; if(m->m_flags & M_PKTHDR) { @@ -621,9 +621,9 @@ m_copym(struct mbuf *m, int off0, int len, int wait) break; } if (copyhdr) - MGETHDR(n, wait, m->m_type); + n = m_gethdr(wait, m->m_type); else - MGET(n, wait, m->m_type); + n = m_get(wait, m->m_type); *np = n; if (n == NULL) goto nospace; @@ -822,7 +822,7 @@ m_copypacket(struct mbuf *m, int how) struct mbuf *top, *n, *o; MBUF_CHECKSLEEP(how); - MGET(n, how, m->m_type); + n = m_get(how, m->m_type); top = n; if (n == NULL) goto nospace; @@ -840,7 +840,7 @@ m_copypacket(struct mbuf *m, int how) m = m->m_next; while (m) { - MGET(o, how, m->m_type); + o = m_get(how, m->m_type); if (o == NULL) goto nospace; @@ -1096,12 +1096,11 @@ m_pullup(struct mbuf *n, int len) } else { if (len > MHLEN) goto bad; - MGET(m, M_NOWAIT, n->m_type); + m = m_get(M_NOWAIT, n->m_type); if (m == NULL) goto bad; - m->m_len = 0; if (n->m_flags & M_PKTHDR) - M_MOVE_PKTHDR(m, n); + m_move_pkthdr(m, n); } space = &m->m_dat[MLEN] - (m->m_data + m->m_len); do { @@ -1144,12 +1143,11 @@ m_copyup(struct mbuf *n, int len, int dstoff) if (len > (MHLEN - dstoff)) goto bad; - MGET(m, M_NOWAIT, n->m_type); + m = m_get(M_NOWAIT, n->m_type); if (m == NULL) goto bad; - m->m_len = 0; if (n->m_flags & M_PKTHDR) - M_MOVE_PKTHDR(m, n); + m_move_pkthdr(m, n); m->m_data += dstoff; space = &m->m_dat[MLEN] - (m->m_data + m->m_len); do { @@ -1200,7 +1198,7 @@ m_split(struct mbuf *m0, int len0, int wait) return (NULL); remain = m->m_len - len; if (m0->m_flags & M_PKTHDR) { - MGETHDR(n, wait, m0->m_type); + n = m_gethdr(wait, m0->m_type); if (n == NULL) return (NULL); n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; @@ -1226,7 +1224,7 @@ m_split(struct mbuf *m0, int len0, int wait) m->m_next = NULL; return (n); } else { - MGET(n, wait, m->m_type); + n = m_get(wait, m->m_type); if (n == NULL) return (NULL); M_ALIGN(n, remain); -- cgit v1.1 From f4837a2e6313ecc34b76db756f9488926534b3d9 Mon Sep 17 00:00:00 2001 From: glebius Date: Sat, 16 Mar 2013 08:58:28 +0000 Subject: - Replace compat macros with function calls. --- sys/netinet/igmp.c | 2 +- sys/netinet/ip_options.c | 2 +- sys/netinet/tcp_output.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'sys') diff --git a/sys/netinet/igmp.c b/sys/netinet/igmp.c index 71aa31d..fd7eb3c 100644 --- a/sys/netinet/igmp.c +++ b/sys/netinet/igmp.c @@ -2203,7 +2203,7 @@ igmp_v1v2_queue_report(struct in_multi *inm, const int type) ifp = inm->inm_ifp; - MGETHDR(m, M_NOWAIT, MT_DATA); + m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) return (ENOMEM); MH_ALIGN(m, sizeof(struct ip) + sizeof(struct igmp)); diff --git a/sys/netinet/ip_options.c b/sys/netinet/ip_options.c index 55cc81f..8911992 100644 --- a/sys/netinet/ip_options.c +++ b/sys/netinet/ip_options.c @@ -500,7 +500,7 @@ ip_insertoptions(struct mbuf *m, struct mbuf *opt, int *phlen) *phlen = 0; return (m); } - M_MOVE_PKTHDR(n, m); + m_move_pkthdr(n, m); n->m_pkthdr.rcvif = NULL; n->m_pkthdr.len += optlen; m->m_len -= sizeof(struct ip); diff --git a/sys/netinet/tcp_output.c b/sys/netinet/tcp_output.c index e291bb3..bcfed8a 100644 --- a/sys/netinet/tcp_output.c +++ b/sys/netinet/tcp_output.c @@ -898,7 +898,7 @@ send: else TCPSTAT_INC(tcps_sndwinup); - MGETHDR(m, M_NOWAIT, MT_DATA); + m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) { error = ENOBUFS; goto out; -- cgit v1.1 From 44d915cf60f5e9b16ab6ce306bfe24ac750a1396 Mon Sep 17 00:00:00 2001 From: joel Date: Sat, 16 Mar 2013 17:57:00 +0000 Subject: Hide version string under verbose. Approved by: mav --- sys/dev/sound/pcm/sndstat.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'sys') diff --git a/sys/dev/sound/pcm/sndstat.c b/sys/dev/sound/pcm/sndstat.c index 7171065..c11eaa3 100644 --- a/sys/dev/sound/pcm/sndstat.c +++ b/sys/dev/sound/pcm/sndstat.c @@ -345,8 +345,12 @@ sndstat_prepare(struct sbuf *s) struct snddev_info *d; int i, j; - sbuf_printf(s, "FreeBSD Audio Driver (newpcm: %ubit %d/%s)\n", - (u_int)sizeof(intpcm32_t) << 3, SND_DRV_VERSION, MACHINE_ARCH); + if (snd_verbose > 0) { + sbuf_printf(s, "FreeBSD Audio Driver (%ubit %d/%s)\n", + (u_int)sizeof(intpcm32_t) << 3, SND_DRV_VERSION, + MACHINE_ARCH); + } + if (SLIST_EMPTY(&sndstat_devlist)) { sbuf_printf(s, "No devices installed.\n"); sbuf_finish(s); -- cgit v1.1 From 27049a86aeaef295edddbef380b1974edd35f006 Mon Sep 17 00:00:00 2001 From: pjd Date: Sat, 16 Mar 2013 22:36:24 +0000 Subject: Style: Remove redundant space. --- sys/kern/vfs_syscalls.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sys') diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c index fc78235..f237386 100644 --- a/sys/kern/vfs_syscalls.c +++ b/sys/kern/vfs_syscalls.c @@ -2854,7 +2854,7 @@ kern_fchmodat(struct thread *td, int fd, char *path, enum uio_seg pathseg, AUDIT_ARG_MODE(mode); follow = (flag & AT_SYMLINK_NOFOLLOW) ? NOFOLLOW : FOLLOW; - NDINIT_ATRIGHTS(&nd, LOOKUP, follow | AUDITVNODE1, pathseg, path, fd, + NDINIT_ATRIGHTS(&nd, LOOKUP, follow | AUDITVNODE1, pathseg, path, fd, CAP_FCHMOD, td); if ((error = namei(&nd)) != 0) return (error); -- cgit v1.1 From a03e9d6f4ccb1f2a2645c9eb5d8c9106557a8bc1 Mon Sep 17 00:00:00 2001 From: pjd Date: Sat, 16 Mar 2013 22:37:30 +0000 Subject: Style: Whitespace fixes. --- sys/kern/vfs_syscalls.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'sys') diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c index f237386..401c21c 100644 --- a/sys/kern/vfs_syscalls.c +++ b/sys/kern/vfs_syscalls.c @@ -445,7 +445,7 @@ sys_getfsstat(td, uap) /* * If (bufsize > 0 && bufseg == UIO_SYSSPACE) - * The caller is responsible for freeing memory which will be allocated + * The caller is responsible for freeing memory which will be allocated * in '*buf'. */ int @@ -2843,7 +2843,6 @@ sys_lchmod(td, uap) uap->mode, AT_SYMLINK_NOFOLLOW)); } - int kern_fchmodat(struct thread *td, int fd, char *path, enum uio_seg pathseg, mode_t mode, int flag) @@ -4679,7 +4678,7 @@ kern_posix_fadvise(struct thread *td, int fd, off_t offset, off_t len, new = fa; fp->f_advice = NULL; } else if (offset <= fa->fa_start && - end >= fa->fa_start) + end >= fa->fa_start) fa->fa_start = end + 1; else if (offset <= fa->fa_end && end >= fa->fa_end) fa->fa_end = offset - 1; -- cgit v1.1 From 8089ec6659b314c029180ab7e322a27747d4d337 Mon Sep 17 00:00:00 2001 From: neel Date: Sat, 16 Mar 2013 22:40:20 +0000 Subject: Allow vmm stats to be specific to the underlying hardware assist technology. This can be done by using the new macros VMM_STAT_INTEL() and VMM_STAT_AMD(). Statistic counters that are common across the two are defined using VMM_STAT(). Suggested by: Anish Gupta Discussed with: grehan Obtained from: NetApp --- sys/amd64/vmm/intel/vmx.c | 7 +++---- sys/amd64/vmm/vmm.c | 6 +++--- sys/amd64/vmm/vmm_stat.c | 13 +++++++++++++ sys/amd64/vmm/vmm_stat.h | 27 ++++++++++++++++++++++++--- 4 files changed, 43 insertions(+), 10 deletions(-) (limited to 'sys') diff --git a/sys/amd64/vmm/intel/vmx.c b/sys/amd64/vmm/intel/vmx.c index 287ac8c..8db79ce 100644 --- a/sys/amd64/vmm/intel/vmx.c +++ b/sys/amd64/vmm/intel/vmx.c @@ -153,10 +153,7 @@ static int cap_unrestricted_guest; static int cap_monitor_trap; /* statistics */ -static VMM_STAT_DEFINE(VCPU_MIGRATIONS, "vcpu migration across host cpus"); -static VMM_STAT_DEFINE(VMEXIT_EXTINT, "vm exits due to external interrupt"); -static VMM_STAT_DEFINE(VMEXIT_HLT_IGNORED, "number of times hlt was ignored"); -static VMM_STAT_DEFINE(VMEXIT_HLT, "number of times hlt was intercepted"); +static VMM_STAT_INTEL(VMEXIT_HLT_IGNORED, "number of times hlt was ignored"); #ifdef KTR static const char * @@ -1216,6 +1213,8 @@ vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) qual = vmexit->u.vmx.exit_qualification; vmexit->exitcode = VM_EXITCODE_BOGUS; + vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1); + switch (vmexit->u.vmx.exit_reason) { case EXIT_REASON_CR_ACCESS: handled = vmx_emulate_cr_access(vmx, vcpu, qual); diff --git a/sys/amd64/vmm/vmm.c b/sys/amd64/vmm/vmm.c index 85d277e..174d479 100644 --- a/sys/amd64/vmm/vmm.c +++ b/sys/amd64/vmm/vmm.c @@ -139,7 +139,7 @@ static MALLOC_DEFINE(M_VM, "vm", "vm"); CTASSERT(VMM_MSR_NUM <= 64); /* msr_mask can keep track of up to 64 msrs */ /* statistics */ -static VMM_STAT_DEFINE(VCPU_TOTAL_RUNTIME, "vcpu total runtime"); +static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime"); static void vcpu_cleanup(struct vcpu *vcpu) @@ -612,7 +612,7 @@ save_guest_fpustate(struct vcpu *vcpu) fpu_start_emulating(); } -static VMM_STAT_DEFINE(VCPU_IDLE_TICKS, "number of ticks vcpu was idle"); +static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle"); int vm_run(struct vm *vm, struct vm_run *vmrun) @@ -717,7 +717,7 @@ vm_inject_event(struct vm *vm, int vcpuid, int type, return (VMINJECT(vm->cookie, vcpuid, type, vector, code, code_valid)); } -static VMM_STAT_DEFINE(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu"); +static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu"); int vm_inject_nmi(struct vm *vm, int vcpuid) diff --git a/sys/amd64/vmm/vmm_stat.c b/sys/amd64/vmm/vmm_stat.c index ae60979..ae156ee 100644 --- a/sys/amd64/vmm/vmm_stat.c +++ b/sys/amd64/vmm/vmm_stat.c @@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include "vmm_util.h" #include "vmm_stat.h" static int vstnum; @@ -52,6 +53,12 @@ vmm_stat_init(void *arg) if (vst->desc == NULL) return; + if (vst->scope == VMM_STAT_SCOPE_INTEL && !vmm_is_intel()) + return; + + if (vst->scope == VMM_STAT_SCOPE_AMD && !vmm_is_amd()) + return; + if (vstnum >= MAX_VMM_STAT_TYPES) { printf("Cannot accomodate vmm stat type \"%s\"!\n", vst->desc); return; @@ -102,3 +109,9 @@ vmm_stat_desc(int index) else return (NULL); } + +/* global statistics */ +VMM_STAT(VCPU_MIGRATIONS, "vcpu migration across host cpus"); +VMM_STAT(VMEXIT_COUNT, "total number of vm exits"); +VMM_STAT(VMEXIT_EXTINT, "vm exits due to external interrupt"); +VMM_STAT(VMEXIT_HLT, "number of times hlt was intercepted"); diff --git a/sys/amd64/vmm/vmm_stat.h b/sys/amd64/vmm/vmm_stat.h index 7c075a6..a1c0967 100644 --- a/sys/amd64/vmm/vmm_stat.h +++ b/sys/amd64/vmm/vmm_stat.h @@ -36,19 +36,36 @@ struct vm; #define MAX_VMM_STAT_TYPES 64 /* arbitrary */ +enum vmm_stat_scope { + VMM_STAT_SCOPE_ANY, + VMM_STAT_SCOPE_INTEL, /* Intel VMX specific statistic */ + VMM_STAT_SCOPE_AMD, /* AMD SVM specific statistic */ +}; + struct vmm_stat_type { - const char *desc; /* description of statistic */ int index; /* position in the stats buffer */ + const char *desc; /* description of statistic */ + enum vmm_stat_scope scope; }; void vmm_stat_init(void *arg); -#define VMM_STAT_DEFINE(type, desc) \ +#define VMM_STAT_DEFINE(type, desc, scope) \ struct vmm_stat_type type[1] = { \ - { desc, -1 } \ + { -1, desc, scope } \ }; \ SYSINIT(type##_stat, SI_SUB_KLD, SI_ORDER_ANY, vmm_stat_init, type) +#define VMM_STAT_DECLARE(type) \ + extern struct vmm_stat_type type[1] + +#define VMM_STAT(type, desc) \ + VMM_STAT_DEFINE(type, desc, VMM_STAT_SCOPE_ANY) +#define VMM_STAT_INTEL(type, desc) \ + VMM_STAT_DEFINE(type, desc, VMM_STAT_SCOPE_INTEL) +#define VMM_STAT_AMD(type, desc) \ + VMM_STAT_DEFINE(type, desc, VMM_STAT_SCOPE_AMD) + void *vmm_stat_alloc(void); void vmm_stat_free(void *vp); @@ -68,4 +85,8 @@ vmm_stat_incr(struct vm *vm, int vcpu, struct vmm_stat_type *vst, uint64_t x) #endif } +VMM_STAT_DECLARE(VCPU_MIGRATIONS); +VMM_STAT_DECLARE(VMEXIT_COUNT); +VMM_STAT_DECLARE(VMEXIT_EXTINT); +VMM_STAT_DECLARE(VMEXIT_HLT); #endif -- cgit v1.1 From 4bbf2e81a6a3b076d53a9eb2d67eec6b39027517 Mon Sep 17 00:00:00 2001 From: neel Date: Sat, 16 Mar 2013 22:53:05 +0000 Subject: Fix the '-Wtautological-compare' warning emitted by clang for comparing the unsigned enum type with a negative value. Obtained from: NetApp --- sys/amd64/vmm/vmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sys') diff --git a/sys/amd64/vmm/vmm.c b/sys/amd64/vmm/vmm.c index 174d479..1de4470 100644 --- a/sys/amd64/vmm/vmm.c +++ b/sys/amd64/vmm/vmm.c @@ -937,7 +937,7 @@ vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state) if (vcpuid < 0 || vcpuid >= VM_MAXCPU) return (EINVAL); - if (state < 0 || state >= X2APIC_STATE_LAST) + if (state >= X2APIC_STATE_LAST) return (EINVAL); vm->vcpu[vcpuid].x2apic_state = state; -- cgit v1.1 From e271e575514044f71031184c0e56a57acbb7e6cd Mon Sep 17 00:00:00 2001 From: andrew Date: Sat, 16 Mar 2013 23:11:55 +0000 Subject: Add __aeabi_memset to libkern, implemented using memset, as clang may generate calls to it. --- sys/conf/files.arm | 1 + sys/libkern/arm/memset.S | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 sys/libkern/arm/memset.S (limited to 'sys') diff --git a/sys/conf/files.arm b/sys/conf/files.arm index 6e439b4..f9e0be8 100644 --- a/sys/conf/files.arm +++ b/sys/conf/files.arm @@ -78,6 +78,7 @@ libkern/arm/ffs.S standard libkern/arm/ldivmod.S standard libkern/arm/ldivmod_helper.c standard libkern/arm/memcpy.S standard +libkern/arm/memset.S standard libkern/arm/muldi3.c standard libkern/ashldi3.c standard libkern/ashrdi3.c standard diff --git a/sys/libkern/arm/memset.S b/sys/libkern/arm/memset.S new file mode 100644 index 0000000..57d2507 --- /dev/null +++ b/sys/libkern/arm/memset.S @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2013 Andrew Turner + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#ifdef __ARM_EABI__ + +/* + * This implements + * void __aeabi_memset(void *dest, size_t len, int c) + * by calling: + * void *memset(dest, c, len) + * + * The arguments are in r0-r2, r3 can be used as a scratch register. + */ +ENTRY_NP(__aeabi_memset) + mov r3, r2 + mov r2, r1 + mov r1, r3 + b memset +END(__aeabi_memset) + +#endif + -- cgit v1.1 From acae942b0530c6e423e597a93933d0dc4d59185a Mon Sep 17 00:00:00 2001 From: pjd Date: Sat, 16 Mar 2013 23:19:13 +0000 Subject: Require CAP_SEEK if both O_APPEND and O_TRUNC flags are absent. In other words we don't require CAP_SEEK if either O_APPEND or O_TRUNC flag is given, because O_APPEND doesn't allow to overwrite existing data and O_TRUNC requires CAP_FTRUNCATE already. Sponsored by: The FreeBSD Foundation --- sys/kern/vfs_syscalls.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sys') diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c index 401c21c..1c361bd 100644 --- a/sys/kern/vfs_syscalls.c +++ b/sys/kern/vfs_syscalls.c @@ -971,7 +971,7 @@ flags_to_rights(int flags) /* FALLTHROUGH */ case O_WRONLY: rights |= CAP_WRITE; - if (!(flags & O_APPEND)) + if (!(flags & (O_APPEND | O_TRUNC))) rights |= CAP_SEEK; break; } -- cgit v1.1 From 9501bcd7501beb9d3154c39d899b469bf5a2381f Mon Sep 17 00:00:00 2001 From: andrew Date: Sun, 17 Mar 2013 00:56:17 +0000 Subject: The -mno-apcs-frame argument is unavaliable on clang, also ignore it there. --- sys/conf/Makefile.arm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sys') diff --git a/sys/conf/Makefile.arm b/sys/conf/Makefile.arm index 10939db..8348e2a 100644 --- a/sys/conf/Makefile.arm +++ b/sys/conf/Makefile.arm @@ -44,7 +44,7 @@ CFLAGS += -mno-thumb-interwork .endif .if empty(DDB_ENABLED) -.if !defined(WITH_ARM_EABI) +.if !defined(WITH_ARM_EABI) && ${COMPILER_TYPE} != "clang" CFLAGS += -mno-apcs-frame .endif .elif defined(WITH_ARM_EABI) -- cgit v1.1 From b2acea1ddc304f859bc444697275a81703847dda Mon Sep 17 00:00:00 2001 From: ian Date: Sun, 17 Mar 2013 03:04:43 +0000 Subject: Add a macro that gets the physical address of a memory mapped device register from a bus space resource. Note that this macro is just for ARM, and is intended to have a short lifespan. The DMA engines in some SoCs need the physical address of a memory-mapped device register as one of the arguments for the transfer. Several scattered ad-hoc solutions have been converted to use this macro, which now also serves to mark the places where a more complete fix needs to be applied (after that fix has been designed). --- sys/arm/broadcom/bcm2835/bcm2835_sdhci.c | 6 ++---- sys/arm/include/bus.h | 8 ++++++++ sys/arm/ti/cpsw/if_cpsw.c | 2 +- sys/arm/ti/ti_mmchs.c | 5 ++--- 4 files changed, 13 insertions(+), 8 deletions(-) (limited to 'sys') diff --git a/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c b/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c index 3512954..b329e30 100644 --- a/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c +++ b/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c @@ -173,7 +173,6 @@ bcm_sdhci_attach(device_t dev) int default_freq; void *buffer; vm_paddr_t buffer_phys; - void *va; sc->sc_dev = dev; sc->sc_req = NULL; @@ -284,9 +283,8 @@ bcm_sdhci_attach(device_t dev) sc->sc_dma_buffer = buffer; sc->sc_dma_buffer_phys = buffer_phys; - va = (void*)rman_get_start(sc->sc_mem_res); - sc->sc_sdhci_buffer_phys = - pmap_kextract((vm_offset_t)va) + SDHCI_BUFFER; + sc->sc_sdhci_buffer_phys = BUS_SPACE_PHYSADDR(sc->sc_mem_res, + SDHCI_BUFFER); bus_generic_probe(dev); bus_generic_attach(dev); diff --git a/sys/arm/include/bus.h b/sys/arm/include/bus.h index cabf1f7..ce8f5ad 100644 --- a/sys/arm/include/bus.h +++ b/sys/arm/include/bus.h @@ -725,4 +725,12 @@ bs_c_8_proto(f); #include +/* + * Get the physical address of a bus space memory-mapped resource. + * Doing this as a macro is a temporary solution until a more robust fix is + * designed. It also serves to mark the locations needing that fix. + */ +#define BUS_SPACE_PHYSADDR(res, offs) \ + (vtophys(rman_get_start(res)+(offs))) + #endif /* _MACHINE_BUS_H_ */ diff --git a/sys/arm/ti/cpsw/if_cpsw.c b/sys/arm/ti/cpsw/if_cpsw.c index 93702df..4b01a74 100644 --- a/sys/arm/ti/cpsw/if_cpsw.c +++ b/sys/arm/ti/cpsw/if_cpsw.c @@ -327,7 +327,7 @@ cpsw_debugf(const char *fmt, ...) #define cpsw_cpdma_bd_offset(i) (CPSW_CPPI_RAM_OFFSET + ((i)*16)) #define cpsw_cpdma_bd_paddr(sc, slot) \ - (slot->bd_offset + vtophys(rman_get_start(sc->res[0]))) + BUS_SPACE_PHYSADDR(sc->res[0], slot->bd_offset) #define cpsw_cpdma_read_bd(sc, slot, val) \ bus_read_region_4(sc->res[0], slot->bd_offset, (uint32_t *) val, 4) #define cpsw_cpdma_write_bd(sc, slot, val) \ diff --git a/sys/arm/ti/ti_mmchs.c b/sys/arm/ti/ti_mmchs.c index cf9dc21..76ecc58 100644 --- a/sys/arm/ti/ti_mmchs.c +++ b/sys/arm/ti/ti_mmchs.c @@ -1584,7 +1584,6 @@ static int ti_mmchs_activate(device_t dev) { struct ti_mmchs_softc *sc = device_get_softc(dev); - unsigned long addr; int rid; int err; @@ -1630,8 +1629,8 @@ ti_mmchs_activate(device_t dev) panic("Unknown OMAP device\n"); /* Get the physical address of the MMC data register, needed for DMA */ - addr = vtophys(rman_get_start(sc->sc_mem_res)); - sc->sc_data_reg_paddr = addr + sc->sc_reg_off + MMCHS_DATA; + sc->sc_data_reg_paddr = BUS_SPACE_PHYSADDR(sc->sc_mem_res, + sc->sc_reg_off + MMCHS_DATA); /* Set the initial power state to off */ sc->sc_cur_power_mode = power_off; -- cgit v1.1 From 279dc85cc8705cbac9b6cd73851a7068a3247274 Mon Sep 17 00:00:00 2001 From: andrew Date: Sun, 17 Mar 2013 06:33:49 +0000 Subject: In the uart module build ofw_bus_if.h on arm along with sparc64 as LINT fails when built locally without it. --- sys/modules/uart/Makefile | 3 +++ 1 file changed, 3 insertions(+) (limited to 'sys') diff --git a/sys/modules/uart/Makefile b/sys/modules/uart/Makefile index d8a4869..06b005b 100644 --- a/sys/modules/uart/Makefile +++ b/sys/modules/uart/Makefile @@ -4,6 +4,9 @@ .if ${MACHINE_CPUARCH} == "sparc64" uart_bus_ebus= uart_bus_ebus.c +.endif + +.if ${MACHINE_CPUARCH} == "arm" || ${MACHINE_CPUARCH} == "sparc64" ofw_bus_if= ofw_bus_if.h .endif -- cgit v1.1 From afdfb863d6e2a2abc194a74c429d06da714a78f0 Mon Sep 17 00:00:00 2001 From: rpaulo Date: Sun, 17 Mar 2013 07:28:17 +0000 Subject: Fix a typo in a comment. --- sys/dev/acpica/acpi_powerres.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sys') diff --git a/sys/dev/acpica/acpi_powerres.c b/sys/dev/acpica/acpi_powerres.c index ba08286..aa343a6 100644 --- a/sys/dev/acpica/acpi_powerres.c +++ b/sys/dev/acpica/acpi_powerres.c @@ -47,7 +47,7 @@ __FBSDID("$FreeBSD$"); * resource, and only deactivate it when there are no powered devices. * * Note that this only manages resources for known devices. There is an - * ugly case where we may turn of power to a device which is in use because + * ugly case where we may turn off power to a device which is in use because * we don't know that it depends on a given resource. We should perhaps * try to be smarter about this, but a more complete solution would involve * scanning all of the ACPI namespace to find devices we're not currently -- cgit v1.1 From 09c3ea7c844d5dbcddd89629f703b84c753cdd17 Mon Sep 17 00:00:00 2001 From: glebius Date: Sun, 17 Mar 2013 07:37:10 +0000 Subject: In m_megapullup() instead of reserving some space at the end of packet, m_align() it, reserving space to prepend data. Reviewed by: mav --- sys/netinet/libalias/alias.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) (limited to 'sys') diff --git a/sys/netinet/libalias/alias.c b/sys/netinet/libalias/alias.c index 33876c4..8f23699 100644 --- a/sys/netinet/libalias/alias.c +++ b/sys/netinet/libalias/alias.c @@ -1749,26 +1749,22 @@ LibAliasUnLoadAllModule(void) struct mbuf * m_megapullup(struct mbuf *m, int len) { struct mbuf *mcl; - + if (len > m->m_pkthdr.len) goto bad; - - /* Do not reallocate packet if it is sequentional, - * writable and has some extra space for expansion. - * XXX: Constant 100bytes is completely empirical. */ -#define RESERVE 100 - if (m->m_next == NULL && M_WRITABLE(m) && M_TRAILINGSPACE(m) >= RESERVE) + + if (m->m_next == NULL && M_WRITABLE(m)) return (m); - mcl = m_get2(len + RESERVE, M_NOWAIT, MT_DATA, M_PKTHDR); + mcl = m_get2(len, M_NOWAIT, MT_DATA, M_PKTHDR); if (mcl == NULL) goto bad; - + m_align(mcl, len); m_move_pkthdr(mcl, m); m_copydata(m, 0, len, mtod(mcl, caddr_t)); mcl->m_len = mcl->m_pkthdr.len = len; m_freem(m); - + return (mcl); bad: m_freem(m); -- cgit v1.1 From 7bcebf010f85ebc9c6893f5aef56eb0a8cd73529 Mon Sep 17 00:00:00 2001 From: glebius Date: Sun, 17 Mar 2013 07:39:45 +0000 Subject: Add MEXT_ALIGN() macro, similar to M_ALIGN() and MH_ALIGN(), but for mbufs with external buffer. --- sys/sys/mbuf.h | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'sys') diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h index a4b0f71..ec10bc6 100644 --- a/sys/sys/mbuf.h +++ b/sys/sys/mbuf.h @@ -195,7 +195,7 @@ struct mbuf { #define M_FIRSTFRAG 0x00001000 /* packet is first fragment */ #define M_LASTFRAG 0x00002000 /* packet is last fragment */ #define M_SKIP_FIREWALL 0x00004000 /* skip firewall processing */ -#define M_FREELIST 0x00008000 /* mbuf is on the free list */ + /* 0x00008000 free */ #define M_VLANTAG 0x00010000 /* ether_vtag is valid */ #define M_PROMISC 0x00020000 /* packet was not for us */ #define M_NOFREE 0x00040000 /* do not free mbuf, embedded in cluster */ @@ -708,6 +708,18 @@ m_last(struct mbuf *m) } while (0) /* + * As above, for mbuf with external storage. + */ +#define MEXT_ALIGN(m, len) do { \ + KASSERT((m)->m_flags & M_EXT, \ + ("%s: MEXT_ALIGN not an M_EXT mbuf", __func__)); \ + KASSERT((m)->m_data == (m)->m_ext.ext_buf, \ + ("%s: MEXT_ALIGN not a virgin mbuf", __func__)); \ + (m)->m_data += ((m)->m_ext.ext_size - (len)) & \ + ~(sizeof(long) - 1); \ +} while (0) + +/* * Compute the amount of space available before the current start of data in * an mbuf. * -- cgit v1.1 From 338f6e587c1344da33c4c32894447d1d4257e330 Mon Sep 17 00:00:00 2001 From: glebius Date: Sun, 17 Mar 2013 07:41:14 +0000 Subject: In m_align() add assertions that mbuf is virgin, similar to assertions in M_ALIGN(), MH_ALIGN, MEXT_ALIGN() macros. --- sys/kern/uipc_mbuf.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'sys') diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c index 621a3eb..ea4fd80 100644 --- a/sys/kern/uipc_mbuf.c +++ b/sys/kern/uipc_mbuf.c @@ -395,7 +395,7 @@ m_demote(struct mbuf *m0, int all) m_freem(m->m_nextpkt); m->m_nextpkt = NULL; } - m->m_flags = m->m_flags & (M_EXT|M_RDONLY|M_FREELIST|M_NOFREE); + m->m_flags = m->m_flags & (M_EXT|M_RDONLY|M_NOFREE); } } @@ -1877,14 +1877,22 @@ m_mbuftouio(struct uio *uio, struct mbuf *m, int len) void m_align(struct mbuf *m, int len) { +#ifdef INVARIANTS + const char *msg = "%s: not a virgin mbuf"; +#endif int adjust; - if (m->m_flags & M_EXT) + if (m->m_flags & M_EXT) { + KASSERT(m->m_data == m->m_ext.ext_buf, (msg, __func__)); adjust = m->m_ext.ext_size - len; - else if (m->m_flags & M_PKTHDR) + } else if (m->m_flags & M_PKTHDR) { + KASSERT(m->m_data == m->m_pktdat, (msg, __func__)); adjust = MHLEN - len; - else + } else { + KASSERT(m->m_data == m->m_dat, (msg, __func__)); adjust = MLEN - len; + } + m->m_data += adjust &~ (sizeof(long)-1); } -- cgit v1.1 From 1b20f7cc1854e8167f6e8ae75c8636919e7b176e Mon Sep 17 00:00:00 2001 From: kib Date: Sun, 17 Mar 2013 15:11:37 +0000 Subject: Remove negative name cache entry pointing to the target name, which could be instantiated while tdvp was unlocked. Reported by: Rick Miller Tested by: pho MFC after: 1 week --- sys/fs/tmpfs/tmpfs_vnops.c | 1 + sys/ufs/ufs/ufs_vnops.c | 1 + 2 files changed, 2 insertions(+) (limited to 'sys') diff --git a/sys/fs/tmpfs/tmpfs_vnops.c b/sys/fs/tmpfs/tmpfs_vnops.c index 54c95ff..156fd43 100644 --- a/sys/fs/tmpfs/tmpfs_vnops.c +++ b/sys/fs/tmpfs/tmpfs_vnops.c @@ -1297,6 +1297,7 @@ tmpfs_rename(struct vop_rename_args *v) cache_purge(fvp); if (tvp != NULL) cache_purge(tvp); + cache_purge_negative(tdvp); error = 0; diff --git a/sys/ufs/ufs/ufs_vnops.c b/sys/ufs/ufs/ufs_vnops.c index f69b1f9..574c59c 100644 --- a/sys/ufs/ufs/ufs_vnops.c +++ b/sys/ufs/ufs/ufs_vnops.c @@ -1554,6 +1554,7 @@ relock: cache_purge(fvp); if (tvp) cache_purge(tvp); + cache_purge_negative(tdvp); unlockout: vput(fdvp); -- cgit v1.1 From b36b09239d80ebc5806b3520c7528e155edee7d1 Mon Sep 17 00:00:00 2001 From: mm Date: Sun, 17 Mar 2013 15:53:27 +0000 Subject: Fix typo in sysctl description Reported by: Jeremy Chadwick MFC after: 3 days --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sys') diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c index dc64682..a8916d2 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c +++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c @@ -271,7 +271,7 @@ uint64_t zfs_deadman_synctime = 1000ULL; TUNABLE_QUAD("vfs.zfs.deadman_synctime", &zfs_deadman_synctime); SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, deadman_synctime, CTLFLAG_RDTUN, &zfs_deadman_synctime, 0, - "Stalled ZFS I/O expiration time in units of vfs.zfs.txg_synctime_ms"); + "Stalled ZFS I/O expiration time in units of vfs.zfs.txg.synctime_ms"); /* * Default value of -1 for zfs_deadman_enabled is resolved in -- cgit v1.1 From 2a3e1335af62a2ac1465820fcb08d501a3db1935 Mon Sep 17 00:00:00 2001 From: ian Date: Sun, 17 Mar 2013 16:31:09 +0000 Subject: Eliminate an intermediate buffer and some memcpy() operations, and do DMA directly to/from the buffers passed in from higher layer drivers. Reviewed by: gonzo --- sys/arm/broadcom/bcm2835/bcm2835_sdhci.c | 110 ++++++++++++------------------- 1 file changed, 43 insertions(+), 67 deletions(-) (limited to 'sys') diff --git a/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c b/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c index b329e30..f4a2de6 100644 --- a/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c +++ b/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c @@ -122,9 +122,7 @@ struct bcm_sdhci_softc { int sc_dma_ch; bus_dma_tag_t sc_dma_tag; bus_dmamap_t sc_dma_map; - void *sc_dma_buffer; - vm_paddr_t sc_dma_buffer_phys; - vm_paddr_t sc_sdhci_buffer_phys;; + vm_paddr_t sc_sdhci_buffer_phys; }; static int bcm_sdhci_probe(device_t); @@ -171,8 +169,6 @@ bcm_sdhci_attach(device_t dev) phandle_t node; pcell_t cell; int default_freq; - void *buffer; - vm_paddr_t buffer_phys; sc->sc_dev = dev; sc->sc_req = NULL; @@ -209,7 +205,7 @@ bcm_sdhci_attach(device_t dev) goto fail; } - if (bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_MISC | INTR_MPSAFE, + if (bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE, NULL, bcm_sdhci_intr, sc, &sc->sc_intrhand)) { bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res); @@ -242,7 +238,7 @@ bcm_sdhci_attach(device_t dev) bcm_dma_setup_intr(sc->sc_dma_ch, bcm_sdhci_dma_intr, sc); - /* Allocate DMA buffers */ + /* Allocate bus_dma resources. */ err = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, @@ -255,34 +251,12 @@ bcm_sdhci_attach(device_t dev) goto fail; } - err = bus_dmamem_alloc(sc->sc_dma_tag, &buffer, - BUS_DMA_WAITOK | BUS_DMA_COHERENT| BUS_DMA_ZERO, - &sc->sc_dma_map); - - if (err) { - device_printf(dev, "cannot allocate DMA memory\n"); - goto fail; - } - - err = bus_dmamap_load(sc->sc_dma_tag, sc->sc_dma_map, buffer, - BCM_SDHCI_BUFFER_SIZE, bcm_dmamap_cb, &buffer_phys, - BUS_DMA_WAITOK); + err = bus_dmamap_create(sc->sc_dma_tag, 0, &sc->sc_dma_map); if (err) { - device_printf(dev, "cannot load DMA memory\n"); - goto fail; - } - - /* - * Sanity check: two least bits of address should be zero - */ - if ((uintptr_t)buffer & 3) { - device_printf(dev, - "DMA address is not word-aligned\n"); + device_printf(dev, "bus_dmamap_create failed\n"); goto fail; } - sc->sc_dma_buffer = buffer; - sc->sc_dma_buffer_phys = buffer_phys; sc->sc_sdhci_buffer_phys = BUS_SPACE_PHYSADDR(sc->sc_mem_res, SDHCI_BUFFER); @@ -445,27 +419,23 @@ bcm_sdhci_dma_intr(int ch, void *arg) struct bcm_sdhci_softc *sc = (struct bcm_sdhci_softc *)arg; struct sdhci_slot *slot = &sc->sc_slot; uint32_t reg, mask; - void *buffer; + bus_addr_t pmem; + vm_paddr_t pdst, psrc; size_t len; - int left; + int left, sync_op; mtx_lock(&slot->mtx); - /* copy DMA buffer to VA if READ */ len = bcm_dma_length(sc->sc_dma_ch); if (slot->curcmd->data->flags & MMC_DATA_READ) { - bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map, - BUS_DMASYNC_POSTREAD); - + sync_op = BUS_DMASYNC_POSTREAD; mask = SDHCI_INT_DATA_AVAIL; - /* all dma data in single or contiguous page */ - buffer = (uint8_t*)(slot->curcmd->data->data) + slot->offset; - memcpy(buffer, sc->sc_dma_buffer, len); } else { - bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map, - BUS_DMASYNC_POSTWRITE); + sync_op = BUS_DMASYNC_POSTWRITE; mask = SDHCI_INT_SPACE_AVAIL; } + bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map, sync_op); + bus_dmamap_unload(sc->sc_dma_tag, sc->sc_dma_map); slot->offset += len; sc->sc_dma_inuse = 0; @@ -499,27 +469,22 @@ bcm_sdhci_dma_intr(int ch, void *arg) SDHCI_INT_STATUS, mask); /* continue next DMA transfer */ + bus_dmamap_load(sc->sc_dma_tag, sc->sc_dma_map, + (uint8_t *)slot->curcmd->data->data + + slot->offset, left, bcm_dmamap_cb, &pmem, 0); if (slot->curcmd->data->flags & MMC_DATA_READ) { - bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map, - BUS_DMASYNC_PREREAD); - - /* DMA start */ - if (bcm_dma_start(sc->sc_dma_ch, - sc->sc_sdhci_buffer_phys, - sc->sc_dma_buffer_phys, left) != 0) - device_printf(sc->sc_dev, "failed DMA start\n"); + psrc = sc->sc_sdhci_buffer_phys; + pdst = pmem; + sync_op = BUS_DMASYNC_PREREAD; } else { - buffer = (char*)slot->curcmd->data->data + slot->offset; - memcpy(sc->sc_dma_buffer, buffer, left); - - bus_dmamap_sync(sc->sc_dma_tag, - sc->sc_dma_map, BUS_DMASYNC_PREWRITE); - - /* DMA start */ - if (bcm_dma_start(sc->sc_dma_ch, - sc->sc_dma_buffer_phys, - sc->sc_sdhci_buffer_phys, left) != 0) - device_printf(sc->sc_dev, "failed DMA start\n"); + psrc = pmem; + pdst = sc->sc_sdhci_buffer_phys; + sync_op = BUS_DMASYNC_PREWRITE; + } + bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map, sync_op); + if (bcm_dma_start(sc->sc_dma_ch, psrc, pdst, left)) { + /* XXX stop xfer, other error recovery? */ + device_printf(sc->sc_dev, "failed DMA start\n"); } } else { /* wait for next data by INT */ @@ -540,6 +505,7 @@ bcm_sdhci_read_dma(struct sdhci_slot *slot) { struct bcm_sdhci_softc *sc = device_get_softc(slot->bus); size_t left; + bus_addr_t paddr; if (sc->sc_dma_inuse) { device_printf(sc->sc_dev, "DMA in use\n"); @@ -560,12 +526,16 @@ bcm_sdhci_read_dma(struct sdhci_slot *slot) BCM_DMA_INC_ADDR, (left & 0xf) ? BCM_DMA_32BIT : BCM_DMA_128BIT); + bus_dmamap_load(sc->sc_dma_tag, sc->sc_dma_map, + (uint8_t *)slot->curcmd->data->data + slot->offset, left, + bcm_dmamap_cb, &paddr, 0); + bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map, BUS_DMASYNC_PREREAD); /* DMA start */ if (bcm_dma_start(sc->sc_dma_ch, sc->sc_sdhci_buffer_phys, - sc->sc_dma_buffer_phys, left) != 0) + paddr, left) != 0) device_printf(sc->sc_dev, "failed DMA start\n"); } @@ -573,8 +543,8 @@ static void bcm_sdhci_write_dma(struct sdhci_slot *slot) { struct bcm_sdhci_softc *sc = device_get_softc(slot->bus); - char *buffer; size_t left; + bus_addr_t paddr; if (sc->sc_dma_inuse) { device_printf(sc->sc_dev, "DMA in use\n"); @@ -589,8 +559,9 @@ bcm_sdhci_write_dma(struct sdhci_slot *slot) KASSERT((left & 3) == 0, ("%s: len = %d, not word-aligned", __func__, left)); - buffer = (char*)slot->curcmd->data->data + slot->offset; - memcpy(sc->sc_dma_buffer, buffer, left); + bus_dmamap_load(sc->sc_dma_tag, sc->sc_dma_map, + (uint8_t *)slot->curcmd->data->data + slot->offset, left, + bcm_dmamap_cb, &paddr, 0); bcm_dma_setup_src(sc->sc_dma_ch, BCM_DMA_DREQ_NONE, BCM_DMA_INC_ADDR, @@ -602,7 +573,7 @@ bcm_sdhci_write_dma(struct sdhci_slot *slot) BUS_DMASYNC_PREWRITE); /* DMA start */ - if (bcm_dma_start(sc->sc_dma_ch, sc->sc_dma_buffer_phys, + if (bcm_dma_start(sc->sc_dma_ch, paddr, sc->sc_sdhci_buffer_phys, left) != 0) device_printf(sc->sc_dev, "failed DMA start\n"); } @@ -612,11 +583,16 @@ bcm_sdhci_will_handle_transfer(device_t dev, struct sdhci_slot *slot) { size_t left; - /* Do not use DMA for transfers less then block size */ + /* + * Do not use DMA for transfers less than block size or with a length + * that is not a multiple of four. + */ left = min(BCM_DMA_BLOCK_SIZE, slot->curcmd->data->len - slot->offset); if (left < BCM_DMA_BLOCK_SIZE) return (0); + if (left & 0x03) + return (0); return (1); } -- cgit v1.1