Update to DRM CVS as of 2006-04-09. The most notable new feature is the updated

Radeon memmap code, which with a new DDX driver and DRI drivers should fix
long-term stability issues with Radeons.  Also adds support for r200's
ATI_fragment_shader, r300 texrect support and texture caching fixes, i915
vblank support and bugfixes, and new PCI IDs.

1 files changed, 180 insertions, 96 deletions

diff --git a/sys/dev/drm/radeon_cp.c b/sys/dev/drm/radeon_cp.c
index cb78288..4c8526d 100644
--- a/sys/dev/drm/radeon_cp.c
+++ b/sys/dev/drm/radeon_cp.c
@@ -947,7 +947,7 @@ static void radeon_cp_load_microcode(drm_radeon_private_t * dev_priv)
 
 	RADEON_WRITE(RADEON_CP_ME_RAM_ADDR, 0);
 
-	if (dev_priv->microcode_version==UCODE_R200) {
+	if (dev_priv->microcode_version == UCODE_R200) {
 		DRM_INFO("Loading R200 Microcode\n");
 		for (i = 0; i < 256; i++) {
 			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
@@ -955,13 +955,13 @@ static void radeon_cp_load_microcode(drm_radeon_private_t * dev_priv)
 			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 				     R200_cp_microcode[i][0]);
 		}
-	} else if (dev_priv->microcode_version==UCODE_R300) {
+	} else if (dev_priv->microcode_version == UCODE_R300) {
 		DRM_INFO("Loading R300 Microcode\n");
-		for ( i = 0 ; i < 256 ; i++ ) {
-			RADEON_WRITE( RADEON_CP_ME_RAM_DATAH,
-				      R300_cp_microcode[i][1] );
-			RADEON_WRITE( RADEON_CP_ME_RAM_DATAL,
-				      R300_cp_microcode[i][0] );
+		for (i = 0; i < 256; i++) {
+			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
+				     R300_cp_microcode[i][1]);
+			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
+				     R300_cp_microcode[i][0]);
 		}
 	} else {
 		for (i = 0; i < 256; i++) {
@@ -1121,26 +1121,33 @@ static void radeon_cp_init_ring_buffer(drm_device_t * dev,
 {
 	u32 ring_start, cur_read_ptr;
 	u32 tmp;
-
-	/* Initialize the memory controller */
-	RADEON_WRITE(RADEON_MC_FB_LOCATION,
-		     ((dev_priv->gart_vm_start - 1) & 0xffff0000)
-		     | (dev_priv->fb_location >> 16));
+	
+	/* Initialize the memory controller. With new memory map, the fb location
+	 * is not changed, it should have been properly initialized already. Part
+	 * of the problem is that the code below is bogus, assuming the GART is
+	 * always appended to the fb which is not necessarily the case
+	 */
+	if (!dev_priv->new_memmap)
+		RADEON_WRITE(RADEON_MC_FB_LOCATION,
+			     ((dev_priv->gart_vm_start - 1) & 0xffff0000)
+			     | (dev_priv->fb_location >> 16));
 
 #if __OS_HAS_AGP
 	if (dev_priv->flags & CHIP_IS_AGP) {
+		RADEON_WRITE(RADEON_AGP_BASE, (unsigned int)dev->agp->base);
 		RADEON_WRITE(RADEON_MC_AGP_LOCATION,
 			     (((dev_priv->gart_vm_start - 1 +
 				dev_priv->gart_size) & 0xffff0000) |
 			      (dev_priv->gart_vm_start >> 16)));
 
 		ring_start = (dev_priv->cp_ring->offset
-			      - dev->agp->base + dev_priv->gart_vm_start);
+			      - dev->agp->base
+			      + dev_priv->gart_vm_start);
 	} else
 #endif
 		ring_start = (dev_priv->cp_ring->offset
-			      - (unsigned long)dev->sg->virtual + dev_priv->gart_vm_start);
-
+			      - (unsigned long)dev->sg->virtual
+			      + dev_priv->gart_vm_start);
 
 	RADEON_WRITE(RADEON_CP_RB_BASE, ring_start);
 
@@ -1155,8 +1162,6 @@ static void radeon_cp_init_ring_buffer(drm_device_t * dev,
 
 #if __OS_HAS_AGP
 	if (dev_priv->flags & CHIP_IS_AGP) {
-		/* set RADEON_AGP_BASE here instead of relying on X from user space */
-		RADEON_WRITE(RADEON_AGP_BASE, (unsigned int)dev->agp->base);
 		RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
 			     dev_priv->ring_rptr->offset
 			     - dev->agp->base + dev_priv->gart_vm_start);
@@ -1166,7 +1171,8 @@ static void radeon_cp_init_ring_buffer(drm_device_t * dev,
 		drm_sg_mem_t *entry = dev->sg;
 		unsigned long tmp_ofs, page_ofs;
 
-		tmp_ofs = dev_priv->ring_rptr->offset - (unsigned long)dev->sg->virtual;
+		tmp_ofs = dev_priv->ring_rptr->offset -
+				(unsigned long)dev->sg->virtual;
 		page_ofs = tmp_ofs >> PAGE_SHIFT;
 
 		RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR, entry->busaddr[page_ofs]);
@@ -1175,6 +1181,17 @@ static void radeon_cp_init_ring_buffer(drm_device_t * dev,
 			  entry->handle + tmp_ofs);
 	}
 
+	/* Set ring buffer size */
+#ifdef __BIG_ENDIAN
+	RADEON_WRITE(RADEON_CP_RB_CNTL,
+		     dev_priv->ring.size_l2qw | RADEON_BUF_SWAP_32BIT);
+#else
+	RADEON_WRITE(RADEON_CP_RB_CNTL, dev_priv->ring.size_l2qw);
+#endif
+
+	/* Start with assuming that writeback doesn't work */
+	dev_priv->writeback_works = 0;
+
 	/* Initialize the scratch register pointer.  This will cause
 	 * the scratch register values to be written out to memory
 	 * whenever they are updated.
@@ -1191,28 +1208,9 @@ static void radeon_cp_init_ring_buffer(drm_device_t * dev,
 
 	RADEON_WRITE(RADEON_SCRATCH_UMSK, 0x7);
 
-	/* Writeback doesn't seem to work everywhere, test it first */
-	DRM_WRITE32(dev_priv->ring_rptr, RADEON_SCRATCHOFF(1), 0);
-	RADEON_WRITE(RADEON_SCRATCH_REG1, 0xdeadbeef);
-
-	for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
-		if (DRM_READ32(dev_priv->ring_rptr, RADEON_SCRATCHOFF(1)) ==
-		    0xdeadbeef)
-			break;
-		DRM_UDELAY(1);
-	}
-
-	if (tmp < dev_priv->usec_timeout) {
-		dev_priv->writeback_works = 1;
-		DRM_DEBUG("writeback test succeeded, tmp=%d\n", tmp);
-	} else {
-		dev_priv->writeback_works = 0;
-		DRM_DEBUG("writeback test failed\n");
-	}
-	if (radeon_no_wb == 1) {
-		dev_priv->writeback_works = 0;
-		DRM_DEBUG("writeback forced off\n");
-	}
+	/* Turn on bus mastering */
+	tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RADEON_BUS_MASTER_DIS;
+	RADEON_WRITE(RADEON_BUS_CNTL, tmp);
 
 	dev_priv->sarea_priv->last_frame = dev_priv->scratch[0] = 0;
 	RADEON_WRITE(RADEON_LAST_FRAME_REG, dev_priv->sarea_priv->last_frame);
@@ -1224,26 +1222,45 @@ static void radeon_cp_init_ring_buffer(drm_device_t * dev,
 	dev_priv->sarea_priv->last_clear = dev_priv->scratch[2] = 0;
 	RADEON_WRITE(RADEON_LAST_CLEAR_REG, dev_priv->sarea_priv->last_clear);
 
-	/* Set ring buffer size */
-#ifdef __BIG_ENDIAN
-	RADEON_WRITE(RADEON_CP_RB_CNTL,
-		     dev_priv->ring.size_l2qw | RADEON_BUF_SWAP_32BIT);
-#else
-	RADEON_WRITE(RADEON_CP_RB_CNTL, dev_priv->ring.size_l2qw);
-#endif
-
 	radeon_do_wait_for_idle(dev_priv);
 
-	/* Turn on bus mastering */
-	tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RADEON_BUS_MASTER_DIS;
-	RADEON_WRITE(RADEON_BUS_CNTL, tmp);
-
 	/* Sync everything up */
 	RADEON_WRITE(RADEON_ISYNC_CNTL,
 		     (RADEON_ISYNC_ANY2D_IDLE3D |
 		      RADEON_ISYNC_ANY3D_IDLE2D |
 		      RADEON_ISYNC_WAIT_IDLEGUI |
 		      RADEON_ISYNC_CPSCRATCH_IDLEGUI));
+
+}
+
+static void radeon_test_writeback(drm_radeon_private_t * dev_priv)
+{
+	u32 tmp;
+
+	/* Writeback doesn't seem to work everywhere, test it here and possibly
+	 * enable it if it appears to work
+	 */
+	DRM_WRITE32(dev_priv->ring_rptr, RADEON_SCRATCHOFF(1), 0);
+	RADEON_WRITE(RADEON_SCRATCH_REG1, 0xdeadbeef);
+
+	for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
+		if (DRM_READ32(dev_priv->ring_rptr, RADEON_SCRATCHOFF(1)) ==
+		    0xdeadbeef)
+			break;
+		DRM_UDELAY(1);
+	}
+
+	if (tmp < dev_priv->usec_timeout) {
+		dev_priv->writeback_works = 1;
+		DRM_INFO("writeback test succeeded in %d usecs\n", tmp);
+	} else {
+		dev_priv->writeback_works = 0;
+		DRM_INFO("writeback test failed\n");
+	}
+	if (radeon_no_wb == 1) {
+		dev_priv->writeback_works = 0;
+		DRM_INFO("writeback forced off\n");
+	}
 }
 
 /* Enable or disable PCI-E GART on the chip */
@@ -1253,19 +1270,26 @@ static void radeon_set_pciegart(drm_radeon_private_t * dev_priv, int on)
 	if (on) {
 
 		DRM_DEBUG("programming pcie %08X %08lX %08X\n",
-			  dev_priv->gart_vm_start, (long)dev_priv->gart_info.bus_addr,
+			  dev_priv->gart_vm_start,
+			  (long)dev_priv->gart_info.bus_addr,
 			  dev_priv->gart_size);
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO, dev_priv->gart_vm_start);
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_BASE, dev_priv->gart_info.bus_addr);
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_START_LO, dev_priv->gart_vm_start);
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_END_LO, dev_priv->gart_vm_start
-			     + dev_priv->gart_size - 1);
+		RADEON_WRITE_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO,
+				  dev_priv->gart_vm_start);
+		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_BASE,
+				  dev_priv->gart_info.bus_addr);
+		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_START_LO,
+				  dev_priv->gart_vm_start);
+		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_END_LO,
+				  dev_priv->gart_vm_start +
+				  dev_priv->gart_size - 1);
 
 		RADEON_WRITE(RADEON_MC_AGP_LOCATION, 0xffffffc0);	/* ?? */
 
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL, RADEON_PCIE_TX_GART_EN);
+		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
+				  RADEON_PCIE_TX_GART_EN);
 	} else {
-		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL, (tmp & ~RADEON_PCIE_TX_GART_EN) | RADEON_PCIE_TX_GART_INVALIDATE_TLB);
+		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
+				  tmp & ~RADEON_PCIE_TX_GART_EN);
 	}
 }
 
@@ -1274,8 +1298,7 @@ static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
 {
 	u32 tmp;
 
-	if (dev_priv->flags & CHIP_IS_PCIE)
-	{
+	if (dev_priv->flags & CHIP_IS_PCIE) {
 		radeon_set_pciegart(dev_priv, on);
 		return;
 	}
@@ -1309,8 +1332,17 @@ static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
 static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 {
 	drm_radeon_private_t *dev_priv = dev->dev_private;
+
 	DRM_DEBUG("\n");
 
+	/* if we require new memory map but we don't have it fail */
+	if ((dev_priv->flags & CHIP_NEW_MEMMAP) && !dev_priv->new_memmap)
+	{
+		DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
+		radeon_do_cleanup_cp(dev);
+		return DRM_ERR(EINVAL);
+	}
+
 	if (init->is_pci && (dev_priv->flags & CHIP_IS_AGP))
 	{
 		DRM_DEBUG("Forcing AGP card to PCI mode\n");
@@ -1333,14 +1365,13 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 
 	switch(init->func) {
 	case RADEON_INIT_R200_CP:
-		dev_priv->microcode_version=UCODE_R200;
+		dev_priv->microcode_version = UCODE_R200;
 		break;
 	case RADEON_INIT_R300_CP:
-		dev_priv->microcode_version=UCODE_R300;
+		dev_priv->microcode_version = UCODE_R300;
 		break;
 	default:
-		dev_priv->microcode_version=UCODE_R100;
-		break;
+		dev_priv->microcode_version = UCODE_R100;
 	}
 
 	dev_priv->do_boxes = 0;
@@ -1390,8 +1421,8 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 	 */
 	dev_priv->depth_clear.rb3d_cntl = (RADEON_PLANE_MASK_ENABLE |
 					   (dev_priv->color_fmt << 10) |
-					   (dev_priv->microcode_version == UCODE_R100 ?
-						RADEON_ZBLOCK16 : 0));
+					   (dev_priv->microcode_version ==
+					    UCODE_R100 ? RADEON_ZBLOCK16 : 0));
 
 	dev_priv->depth_clear.rb3d_zstencilcntl =
 	    (dev_priv->depth_fmt |
@@ -1491,6 +1522,9 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 
 	dev_priv->fb_location = (RADEON_READ(RADEON_MC_FB_LOCATION)
 				 & 0xffff) << 16;
+	dev_priv->fb_size = 
+		((RADEON_READ(RADEON_MC_FB_LOCATION) & 0xffff0000u) + 0x10000)
+		- dev_priv->fb_location;
 
 	dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
 					((dev_priv->front_offset
@@ -1505,8 +1539,46 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 					  + dev_priv->fb_location) >> 10));
 
 	dev_priv->gart_size = init->gart_size;
-	dev_priv->gart_vm_start = dev_priv->fb_location
-	    + RADEON_READ(RADEON_CONFIG_APER_SIZE);
+
+	/* New let's set the memory map ... */
+	if (dev_priv->new_memmap) {
+		u32 base = 0;
+
+		DRM_INFO("Setting GART location based on new memory map\n");
+
+		/* If using AGP, try to locate the AGP aperture at the same
+		 * location in the card and on the bus, though we have to
+		 * align it down.
+		 */
+#if __OS_HAS_AGP
+		if (dev_priv->flags & CHIP_IS_AGP) {
+			base = dev->agp->base;
+			/* Check if valid */
+			if ((base + dev_priv->gart_size) > dev_priv->fb_location &&
+			    base < (dev_priv->fb_location + dev_priv->fb_size)) {
+				DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
+					 dev->agp->base);
+				base = 0;
+			}
+		}
+#endif
+		/* If not or if AGP is at 0 (Macs), try to put it elsewhere */
+		if (base == 0) {
+			base = dev_priv->fb_location + dev_priv->fb_size;
+			if (((base + dev_priv->gart_size) & 0xfffffffful)
+			    < base)
+				base = dev_priv->fb_location
+					- dev_priv->gart_size;
+		}		
+		dev_priv->gart_vm_start = base & 0xffc00000u;
+		if (dev_priv->gart_vm_start != base)
+			DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
+				 base, dev_priv->gart_vm_start);
+	} else {
+		DRM_INFO("Setting GART location based on old memory map\n");
+		dev_priv->gart_vm_start = dev_priv->fb_location +
+			RADEON_READ(RADEON_CONFIG_APER_SIZE);
+	}
 
 #if __OS_HAS_AGP
 	if (dev_priv->flags & CHIP_IS_AGP)
@@ -1516,8 +1588,8 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 	else
 #endif
 		dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
-						 - (unsigned long)dev->sg->virtual
-						 + dev_priv->gart_vm_start);
+					- (unsigned long)dev->sg->virtual
+					+ dev_priv->gart_vm_start);
 
 	DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
 	DRM_DEBUG("dev_priv->gart_vm_start 0x%x\n", dev_priv->gart_vm_start);
@@ -1543,24 +1615,33 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 	{
 		/* if we have an offset set from userspace */
 		if (dev_priv->pcigart_offset) {
-			dev_priv->gart_info.bus_addr = dev_priv->pcigart_offset + dev_priv->fb_location;
-			dev_priv->gart_info.mapping.offset = dev_priv->gart_info.bus_addr;
-			dev_priv->gart_info.mapping.size = RADEON_PCIGART_TABLE_SIZE;
-			drm_core_ioremap(&dev_priv->gart_info.mapping, dev);
-			dev_priv->gart_info.addr = dev_priv->gart_info.mapping.handle;
+			dev_priv->gart_info.bus_addr =
+			    dev_priv->pcigart_offset + dev_priv->fb_location;
+			dev_priv->gart_info.mapping.offset =
+			    dev_priv->gart_info.bus_addr;
+			dev_priv->gart_info.mapping.size =
+			    RADEON_PCIGART_TABLE_SIZE;
 
-			dev_priv->gart_info.is_pcie = !!(dev_priv->flags & CHIP_IS_PCIE);
-			dev_priv->gart_info.gart_table_location = DRM_ATI_GART_FB;
-			
-			DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n", dev_priv->gart_info.addr, dev_priv->pcigart_offset);
-		}
-		else {
-			dev_priv->gart_info.gart_table_location = DRM_ATI_GART_MAIN;
+			drm_core_ioremap(&dev_priv->gart_info.mapping, dev);
+			dev_priv->gart_info.addr =
+			    dev_priv->gart_info.mapping.handle;
+
+			dev_priv->gart_info.is_pcie =
+			    !!(dev_priv->flags & CHIP_IS_PCIE);
+			dev_priv->gart_info.gart_table_location =
+			    DRM_ATI_GART_FB;
+
+			DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n",
+				  dev_priv->gart_info.addr,
+				  dev_priv->pcigart_offset);
+		} else {
+			dev_priv->gart_info.gart_table_location =
+			    DRM_ATI_GART_MAIN;
 			dev_priv->gart_info.addr = NULL;
 			dev_priv->gart_info.bus_addr = 0;
-			if (dev_priv->flags & CHIP_IS_PCIE)
-			{
-				DRM_ERROR("Cannot use PCI Express without GART in FB memory\n");
+			if (dev_priv->flags & CHIP_IS_PCIE) {
+				DRM_ERROR
+				    ("Cannot use PCI Express without GART in FB memory\n");
 				radeon_do_cleanup_cp(dev);
 				return DRM_ERR(EINVAL);
 			}
@@ -1582,6 +1663,7 @@ static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
 	dev_priv->last_buf = 0;
 
 	radeon_do_engine_reset(dev);
+	radeon_test_writeback(dev_priv);
 
 	return 0;
 }
@@ -1684,9 +1766,9 @@ int radeon_cp_init(DRM_IOCTL_ARGS)
 	DRM_COPY_FROM_USER_IOCTL(init, (drm_radeon_init_t __user *) data,
 				 sizeof(init));
 
-	if(init.func == RADEON_INIT_R300_CP)
+	if (init.func == RADEON_INIT_R300_CP)
 		r300_init_reg_flags();
-	
+
 	switch (init.func) {
 	case RADEON_INIT_CP:
 	case RADEON_INIT_R200_CP:
@@ -1775,7 +1857,6 @@ void radeon_do_release(drm_device_t * dev)
 	int i, ret;
 
 	if (dev_priv) {
-
 		if (dev_priv->cp_running) {
 			/* Stop the cp */
 			while ((ret = radeon_do_cp_idle(dev_priv)) != 0) {
@@ -1799,11 +1880,13 @@ void radeon_do_release(drm_device_t * dev)
 		if (dev_priv->mmio)	/* remove this after permanent addmaps */
 			RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
 
-		if (dev_priv->mmio) {/* remove all surfaces */
+		if (dev_priv->mmio) {	/* remove all surfaces */
 			for (i = 0; i < RADEON_MAX_SURFACES; i++) {
-				RADEON_WRITE(RADEON_SURFACE0_INFO + 16*i, 0);
-				RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND + 16*i, 0);
-				RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND + 16*i, 0);
+				RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * i, 0);
+				RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND +
+					     16 * i, 0);
+				RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND +
+					     16 * i, 0);
 			}
 		}
 
@@ -2107,11 +2190,13 @@ int radeon_driver_load(struct drm_device *dev, unsigned long flags)
 	case CHIP_RV200:
 	case CHIP_R200:
 	case CHIP_R300:
+	case CHIP_R350:
 	case CHIP_R420:
+	case CHIP_RV410:
 		dev_priv->flags |= CHIP_HAS_HIERZ;
 		break;
 	default:
-	/* all other chips have no hierarchical z buffer */
+		/* all other chips have no hierarchical z buffer */
 		break;
 	}
 
@@ -2123,7 +2208,6 @@ int radeon_driver_load(struct drm_device *dev, unsigned long flags)
 
 	DRM_DEBUG("%s card detected\n",
 		  ((dev_priv->flags & CHIP_IS_AGP) ? "AGP" : (((dev_priv->flags & CHIP_IS_PCIE) ? "PCIE" : "PCI"))));
-
 	return ret;
 }