diff options
Diffstat (limited to 'arch/um/drivers/ubd_kern.c')
-rw-r--r-- | arch/um/drivers/ubd_kern.c | 351 |
1 files changed, 195 insertions, 156 deletions
diff --git a/arch/um/drivers/ubd_kern.c b/arch/um/drivers/ubd_kern.c index 8bd9204..70509dd 100644 --- a/arch/um/drivers/ubd_kern.c +++ b/arch/um/drivers/ubd_kern.c @@ -39,7 +39,6 @@ #include "asm/irq.h" #include "asm/types.h" #include "asm/tlbflush.h" -#include "user_util.h" #include "mem_user.h" #include "kern_util.h" #include "kern.h" @@ -90,7 +89,7 @@ static inline int ubd_test_bit(__u64 bit, unsigned char *data) bits = sizeof(data[0]) * 8; n = bit / bits; off = bit % bits; - return((data[n] & (1 << off)) != 0); + return (data[n] & (1 << off)) != 0; } static inline void ubd_set_bit(__u64 bit, unsigned char *data) @@ -147,10 +146,13 @@ struct cow { unsigned long *bitmap; unsigned long bitmap_len; int bitmap_offset; - int data_offset; + int data_offset; }; +#define MAX_SG 64 + struct ubd { + struct list_head restart; /* name (and fd, below) of the file opened for writing, either the * backing or the cow file. */ char *file; @@ -165,15 +167,17 @@ struct ubd { struct platform_device pdev; struct request_queue *queue; spinlock_t lock; - int active; + struct scatterlist sg[MAX_SG]; + struct request *request; + int start_sg, end_sg; }; #define DEFAULT_COW { \ .file = NULL, \ - .fd = -1, \ - .bitmap = NULL, \ + .fd = -1, \ + .bitmap = NULL, \ .bitmap_offset = 0, \ - .data_offset = 0, \ + .data_offset = 0, \ } #define DEFAULT_UBD { \ @@ -183,11 +187,13 @@ struct ubd { .size = -1, \ .boot_openflags = OPEN_FLAGS, \ .openflags = OPEN_FLAGS, \ - .no_cow = 0, \ + .no_cow = 0, \ .shared = 0, \ - .cow = DEFAULT_COW, \ + .cow = DEFAULT_COW, \ .lock = SPIN_LOCK_UNLOCKED, \ - .active = 0, \ + .request = NULL, \ + .start_sg = 0, \ + .end_sg = 0, \ } /* Protected by ubd_lock */ @@ -243,7 +249,7 @@ static void make_ide_entries(char *dev_name) static int fake_ide_setup(char *str) { fake_ide = 1; - return(1); + return 1; } __setup("fake_ide", fake_ide_setup); @@ -261,7 +267,7 @@ static int parse_unit(char **ptr) if(isdigit(*str)) { n = simple_strtoul(str, &end, 0); if(end == str) - return(-1); + return -1; *ptr = end; } else if (('a' <= *str) && (*str <= 'z')) { @@ -269,7 +275,7 @@ static int parse_unit(char **ptr) str++; *ptr = str; } - return(n); + return n; } /* If *index_out == -1 at exit, the passed option was a general one; @@ -436,7 +442,7 @@ static int udb_setup(char *str) { printk("udb%s specified on command line is almost certainly a ubd -> " "udb TYPO\n", str); - return(1); + return 1; } __setup("udb", udb_setup); @@ -467,66 +473,75 @@ static void do_ubd_request(request_queue_t * q); /* Only changed by ubd_init, which is an initcall. */ int thread_fd = -1; -/* call ubd_finish if you need to serialize */ -static void __ubd_finish(struct request *req, int error) +static void ubd_end_request(struct request *req, int bytes, int uptodate) { - int nsect; - - if(error){ - end_request(req, 0); - return; + if (!end_that_request_first(req, uptodate, bytes >> 9)) { + struct ubd *dev = req->rq_disk->private_data; + unsigned long flags; + + add_disk_randomness(req->rq_disk); + spin_lock_irqsave(&dev->lock, flags); + end_that_request_last(req, uptodate); + spin_unlock_irqrestore(&dev->lock, flags); } - nsect = req->current_nr_sectors; - req->sector += nsect; - req->buffer += nsect << 9; - req->errors = 0; - req->nr_sectors -= nsect; - req->current_nr_sectors = 0; - end_request(req, 1); } /* Callable only from interrupt context - otherwise you need to do * spin_lock_irq()/spin_lock_irqsave() */ -static inline void ubd_finish(struct request *req, int error) +static inline void ubd_finish(struct request *req, int bytes) { - struct ubd *dev = req->rq_disk->private_data; - - spin_lock(&dev->lock); - __ubd_finish(req, error); - spin_unlock(&dev->lock); + if(bytes < 0){ + ubd_end_request(req, 0, 0); + return; + } + ubd_end_request(req, bytes, 1); } +static LIST_HEAD(restart); + /* XXX - move this inside ubd_intr. */ /* Called without dev->lock held, and only in interrupt context. */ static void ubd_handler(void) { - struct io_thread_req req; + struct io_thread_req *req; struct request *rq; - struct ubd *dev; + struct ubd *ubd; + struct list_head *list, *next_ele; + unsigned long flags; int n; - n = os_read_file(thread_fd, &req, sizeof(req)); - if(n != sizeof(req)){ - printk(KERN_ERR "Pid %d - spurious interrupt in ubd_handler, " - "err = %d\n", os_getpid(), -n); - return; - } - - rq = req.req; - dev = rq->rq_disk->private_data; - dev->active = 0; + while(1){ + n = os_read_file(thread_fd, &req, + sizeof(struct io_thread_req *)); + if(n != sizeof(req)){ + if(n == -EAGAIN) + break; + printk(KERN_ERR "spurious interrupt in ubd_handler, " + "err = %d\n", -n); + return; + } - ubd_finish(rq, req.error); + rq = req->req; + rq->nr_sectors -= req->length >> 9; + if(rq->nr_sectors == 0) + ubd_finish(rq, rq->hard_nr_sectors << 9); + kfree(req); + } reactivate_fd(thread_fd, UBD_IRQ); - spin_lock(&dev->lock); - do_ubd_request(dev->queue); - spin_unlock(&dev->lock); + + list_for_each_safe(list, next_ele, &restart){ + ubd = container_of(list, struct ubd, restart); + list_del_init(&ubd->restart); + spin_lock_irqsave(&ubd->lock, flags); + do_ubd_request(ubd->queue); + spin_unlock_irqrestore(&ubd->lock, flags); + } } static irqreturn_t ubd_intr(int irq, void *dev) { ubd_handler(); - return(IRQ_HANDLED); + return IRQ_HANDLED; } /* Only changed by ubd_init, which is an initcall. */ @@ -545,7 +560,7 @@ static inline int ubd_file_size(struct ubd *ubd_dev, __u64 *size_out) char *file; file = ubd_dev->cow.file ? ubd_dev->cow.file : ubd_dev->file; - return(os_file_size(file, size_out)); + return os_file_size(file, size_out); } static void ubd_close_dev(struct ubd *ubd_dev) @@ -617,10 +632,18 @@ static int ubd_open_dev(struct ubd *ubd_dev) if(err < 0) goto error; ubd_dev->cow.fd = err; } - return(0); + return 0; error: os_close_file(ubd_dev->fd); - return(err); + return err; +} + +static void ubd_device_release(struct device *dev) +{ + struct ubd *ubd_dev = dev->driver_data; + + blk_cleanup_queue(ubd_dev->queue); + *ubd_dev = ((struct ubd) DEFAULT_UBD); } static int ubd_disk_register(int major, u64 size, int unit, @@ -630,7 +653,7 @@ static int ubd_disk_register(int major, u64 size, int unit, disk = alloc_disk(1 << UBD_SHIFT); if(disk == NULL) - return(-ENOMEM); + return -ENOMEM; disk->major = major; disk->first_minor = unit << UBD_SHIFT; @@ -645,6 +668,8 @@ static int ubd_disk_register(int major, u64 size, int unit, if (major == MAJOR_NR) { ubd_devs[unit].pdev.id = unit; ubd_devs[unit].pdev.name = DRIVER_NAME; + ubd_devs[unit].pdev.dev.release = ubd_device_release; + ubd_devs[unit].pdev.dev.driver_data = &ubd_devs[unit]; platform_device_register(&ubd_devs[unit].pdev); disk->driverfs_dev = &ubd_devs[unit].pdev.dev; } @@ -675,6 +700,8 @@ static int ubd_add(int n, char **error_out) ubd_dev->size = ROUND_BLOCK(ubd_dev->size); + INIT_LIST_HEAD(&ubd_dev->restart); + err = -ENOMEM; ubd_dev->queue = blk_init_queue(do_ubd_request, &ubd_dev->lock); if (ubd_dev->queue == NULL) { @@ -683,6 +710,7 @@ static int ubd_add(int n, char **error_out) } ubd_dev->queue->queuedata = ubd_dev; + blk_queue_max_hw_segments(ubd_dev->queue, MAX_SG); err = ubd_disk_register(MAJOR_NR, ubd_dev->size, n, &ubd_gendisk[n]); if(err){ *error_out = "Failed to register device"; @@ -730,14 +758,14 @@ static int ubd_config(char *str, char **error_out) goto err_free; } - mutex_lock(&ubd_lock); + mutex_lock(&ubd_lock); ret = ubd_add(n, error_out); if (ret) ubd_devs[n].file = NULL; - mutex_unlock(&ubd_lock); + mutex_unlock(&ubd_lock); out: - return ret; + return ret; err_free: kfree(str); @@ -752,7 +780,7 @@ static int ubd_get_config(char *name, char *str, int size, char **error_out) n = parse_unit(&name); if((n >= MAX_DEV) || (n < 0)){ *error_out = "ubd_get_config : device number out of range"; - return(-1); + return -1; } ubd_dev = &ubd_devs[n]; @@ -773,29 +801,27 @@ static int ubd_get_config(char *name, char *str, int size, char **error_out) out: mutex_unlock(&ubd_lock); - return(len); + return len; } static int ubd_id(char **str, int *start_out, int *end_out) { - int n; + int n; n = parse_unit(str); - *start_out = 0; - *end_out = MAX_DEV - 1; - return n; + *start_out = 0; + *end_out = MAX_DEV - 1; + return n; } static int ubd_remove(int n, char **error_out) { + struct gendisk *disk = ubd_gendisk[n]; struct ubd *ubd_dev; int err = -ENODEV; mutex_lock(&ubd_lock); - if(ubd_gendisk[n] == NULL) - goto out; - ubd_dev = &ubd_devs[n]; if(ubd_dev->file == NULL) @@ -806,9 +832,11 @@ static int ubd_remove(int n, char **error_out) if(ubd_dev->count > 0) goto out; - del_gendisk(ubd_gendisk[n]); - put_disk(ubd_gendisk[n]); ubd_gendisk[n] = NULL; + if(disk != NULL){ + del_gendisk(disk); + put_disk(disk); + } if(fake_gendisk[n] != NULL){ del_gendisk(fake_gendisk[n]); @@ -816,10 +844,8 @@ static int ubd_remove(int n, char **error_out) fake_gendisk[n] = NULL; } - blk_cleanup_queue(ubd_dev->queue); - platform_device_unregister(&ubd_dev->pdev); - *ubd_dev = ((struct ubd) DEFAULT_UBD); err = 0; + platform_device_unregister(&ubd_dev->pdev); out: mutex_unlock(&ubd_lock); return err; @@ -832,7 +858,7 @@ static struct mc_device ubd_mc = { .list = LIST_HEAD_INIT(ubd_mc.list), .name = "ubd", .config = ubd_config, - .get_config = ubd_get_config, + .get_config = ubd_get_config, .id = ubd_id, .remove = ubd_remove, }; @@ -854,7 +880,7 @@ static int __init ubd0_init(void) ubd_dev->file = "root_fs"; mutex_unlock(&ubd_lock); - return(0); + return 0; } __initcall(ubd0_init); @@ -882,14 +908,14 @@ static int __init ubd_init(void) return -1; } platform_driver_register(&ubd_driver); - mutex_lock(&ubd_lock); + mutex_lock(&ubd_lock); for (i = 0; i < MAX_DEV; i++){ err = ubd_add(i, &error); if(err) printk(KERN_ERR "Failed to initialize ubd device %d :" "%s\n", i, error); } - mutex_unlock(&ubd_lock); + mutex_unlock(&ubd_lock); return 0; } @@ -913,7 +939,7 @@ static int __init ubd_driver_init(void){ "ubd : Failed to start I/O thread (errno = %d) - " "falling back to synchronous I/O\n", -io_pid); io_pid = -1; - return(0); + return 0; } err = um_request_irq(UBD_IRQ, thread_fd, IRQ_READ, ubd_intr, IRQF_DISABLED, "ubd", ubd_devs); @@ -948,7 +974,7 @@ static int ubd_open(struct inode *inode, struct file *filp) err = -EROFS; }*/ out: - return(err); + return err; } static int ubd_release(struct inode * inode, struct file * file) @@ -958,7 +984,7 @@ static int ubd_release(struct inode * inode, struct file * file) if(--ubd_dev->count == 0) ubd_close_dev(ubd_dev); - return(0); + return 0; } static void cowify_bitmap(__u64 io_offset, int length, unsigned long *cow_mask, @@ -1014,7 +1040,7 @@ static void cowify_req(struct io_thread_req *req, unsigned long *bitmap, if(ubd_test_bit(sector + i, (unsigned char *) bitmap)) ubd_set_bit(i, (unsigned char *) &req->sector_mask); - } + } } else cowify_bitmap(req->offset, req->length, &req->sector_mask, &req->cow_offset, bitmap, bitmap_offset, @@ -1022,26 +1048,16 @@ static void cowify_req(struct io_thread_req *req, unsigned long *bitmap, } /* Called with dev->lock held */ -static int prepare_request(struct request *req, struct io_thread_req *io_req) +static void prepare_request(struct request *req, struct io_thread_req *io_req, + unsigned long long offset, int page_offset, + int len, struct page *page) { struct gendisk *disk = req->rq_disk; struct ubd *ubd_dev = disk->private_data; - __u64 offset; - int len; - - /* This should be impossible now */ - if((rq_data_dir(req) == WRITE) && !ubd_dev->openflags.w){ - printk("Write attempted on readonly ubd device %s\n", - disk->disk_name); - end_request(req, 0); - return(1); - } - - offset = ((__u64) req->sector) << 9; - len = req->current_nr_sectors << 9; io_req->req = req; - io_req->fds[0] = (ubd_dev->cow.file != NULL) ? ubd_dev->cow.fd : ubd_dev->fd; + io_req->fds[0] = (ubd_dev->cow.file != NULL) ? ubd_dev->cow.fd : + ubd_dev->fd; io_req->fds[1] = ubd_dev->fd; io_req->cow_offset = -1; io_req->offset = offset; @@ -1052,45 +1068,66 @@ static int prepare_request(struct request *req, struct io_thread_req *io_req) io_req->op = (rq_data_dir(req) == READ) ? UBD_READ : UBD_WRITE; io_req->offsets[0] = 0; io_req->offsets[1] = ubd_dev->cow.data_offset; - io_req->buffer = req->buffer; + io_req->buffer = page_address(page) + page_offset; io_req->sectorsize = 1 << 9; if(ubd_dev->cow.file != NULL) - cowify_req(io_req, ubd_dev->cow.bitmap, ubd_dev->cow.bitmap_offset, - ubd_dev->cow.bitmap_len); + cowify_req(io_req, ubd_dev->cow.bitmap, + ubd_dev->cow.bitmap_offset, ubd_dev->cow.bitmap_len); - return(0); } /* Called with dev->lock held */ static void do_ubd_request(request_queue_t *q) { - struct io_thread_req io_req; + struct io_thread_req *io_req; struct request *req; - int err, n; - - if(thread_fd == -1){ - while((req = elv_next_request(q)) != NULL){ - err = prepare_request(req, &io_req); - if(!err){ - do_io(&io_req); - __ubd_finish(req, io_req.error); - } - } - } - else { + int n; + + while(1){ struct ubd *dev = q->queuedata; - if(dev->active || (req = elv_next_request(q)) == NULL) - return; - err = prepare_request(req, &io_req); - if(!err){ - dev->active = 1; - n = os_write_file(thread_fd, (char *) &io_req, - sizeof(io_req)); - if(n != sizeof(io_req)) - printk("write to io thread failed, " - "errno = %d\n", -n); + if(dev->end_sg == 0){ + struct request *req = elv_next_request(q); + if(req == NULL) + return; + + dev->request = req; + blkdev_dequeue_request(req); + dev->start_sg = 0; + dev->end_sg = blk_rq_map_sg(q, req, dev->sg); } + + req = dev->request; + while(dev->start_sg < dev->end_sg){ + struct scatterlist *sg = &dev->sg[dev->start_sg]; + + io_req = kmalloc(sizeof(struct io_thread_req), + GFP_ATOMIC); + if(io_req == NULL){ + if(list_empty(&dev->restart)) + list_add(&dev->restart, &restart); + return; + } + prepare_request(req, io_req, + (unsigned long long) req->sector << 9, + sg->offset, sg->length, sg->page); + + n = os_write_file(thread_fd, &io_req, + sizeof(struct io_thread_req *)); + if(n != sizeof(struct io_thread_req *)){ + if(n != -EAGAIN) + printk("write to io thread failed, " + "errno = %d\n", -n); + else if(list_empty(&dev->restart)) + list_add(&dev->restart, &restart); + return; + } + + req->sector += sg->length >> 9; + dev->start_sg++; + } + dev->end_sg = 0; + dev->request = NULL; } } @@ -1120,21 +1157,21 @@ static int ubd_ioctl(struct inode * inode, struct file * file, ubd_id.cyls = ubd_dev->size / (128 * 32 * 512); if(copy_to_user((char __user *) arg, (char *) &ubd_id, sizeof(ubd_id))) - return(-EFAULT); - return(0); + return -EFAULT; + return 0; case CDROMVOLREAD: if(copy_from_user(&volume, (char __user *) arg, sizeof(volume))) - return(-EFAULT); + return -EFAULT; volume.channel0 = 255; volume.channel1 = 255; volume.channel2 = 255; volume.channel3 = 255; if(copy_to_user((char __user *) arg, &volume, sizeof(volume))) - return(-EFAULT); - return(0); + return -EFAULT; + return 0; } - return(-EINVAL); + return -EINVAL; } static int path_requires_switch(char *from_cmdline, char *from_cow, char *cow) @@ -1176,29 +1213,29 @@ static int backing_file_mismatch(char *file, __u64 size, time_t mtime) if(err < 0){ printk("Failed to get modification time of backing file " "\"%s\", err = %d\n", file, -err); - return(err); + return err; } err = os_file_size(file, &actual); if(err < 0){ printk("Failed to get size of backing file \"%s\", " "err = %d\n", file, -err); - return(err); + return err; } - if(actual != size){ + if(actual != size){ /*__u64 can be a long on AMD64 and with %lu GCC complains; so * the typecast.*/ printk("Size mismatch (%llu vs %llu) of COW header vs backing " "file\n", (unsigned long long) size, actual); - return(-EINVAL); + return -EINVAL; } if(modtime != mtime){ printk("mtime mismatch (%ld vs %ld) of COW header vs backing " "file\n", mtime, modtime); - return(-EINVAL); + return -EINVAL; } - return(0); + return 0; } int read_cow_bitmap(int fd, void *buf, int offset, int len) @@ -1207,13 +1244,13 @@ int read_cow_bitmap(int fd, void *buf, int offset, int len) err = os_seek_file(fd, offset); if(err < 0) - return(err); + return err; err = os_read_file(fd, buf, len); if(err < 0) - return(err); + return err; - return(0); + return 0; } int open_ubd_file(char *file, struct openflags *openflags, int shared, @@ -1231,14 +1268,14 @@ int open_ubd_file(char *file, struct openflags *openflags, int shared, if (fd < 0) { if ((fd == -ENOENT) && (create_cow_out != NULL)) *create_cow_out = 1; - if (!openflags->w || - ((fd != -EROFS) && (fd != -EACCES))) + if (!openflags->w || + ((fd != -EROFS) && (fd != -EACCES))) return fd; openflags->w = 0; fd = os_open_file(file, *openflags, mode); if (fd < 0) return fd; - } + } if(shared) printk("Not locking \"%s\" on the host\n", file); @@ -1252,7 +1289,7 @@ int open_ubd_file(char *file, struct openflags *openflags, int shared, /* Successful return case! */ if(backing_file_out == NULL) - return(fd); + return fd; err = read_cow_header(file_reader, &fd, &version, &backing_file, &mtime, &size, §orsize, &align, bitmap_offset_out); @@ -1262,7 +1299,7 @@ int open_ubd_file(char *file, struct openflags *openflags, int shared, goto out_close; } if(err) - return(fd); + return fd; asked_switch = path_requires_switch(*backing_file_out, backing_file, file); @@ -1285,7 +1322,7 @@ int open_ubd_file(char *file, struct openflags *openflags, int shared, cow_sizes(version, size, sectorsize, align, *bitmap_offset_out, bitmap_len_out, data_offset_out); - return fd; + return fd; out_close: os_close_file(fd); return err; @@ -1310,10 +1347,10 @@ int create_cow_file(char *cow_file, char *backing_file, struct openflags flags, bitmap_offset_out, bitmap_len_out, data_offset_out); if(!err) - return(fd); + return fd; os_close_file(fd); out: - return(err); + return err; } static int update_bitmap(struct io_thread_req *req) @@ -1321,23 +1358,23 @@ static int update_bitmap(struct io_thread_req *req) int n; if(req->cow_offset == -1) - return(0); + return 0; n = os_seek_file(req->fds[1], req->cow_offset); if(n < 0){ printk("do_io - bitmap lseek failed : err = %d\n", -n); - return(1); + return 1; } n = os_write_file(req->fds[1], &req->bitmap_words, - sizeof(req->bitmap_words)); + sizeof(req->bitmap_words)); if(n != sizeof(req->bitmap_words)){ printk("do_io - bitmap update failed, err = %d fd = %d\n", -n, req->fds[1]); - return(1); + return 1; } - return(0); + return 0; } void do_io(struct io_thread_req *req) @@ -1409,13 +1446,14 @@ static int io_count = 0; int io_thread(void *arg) { - struct io_thread_req req; + struct io_thread_req *req; int n; ignore_sigwinch_sig(); while(1){ - n = os_read_file(kernel_fd, &req, sizeof(req)); - if(n != sizeof(req)){ + n = os_read_file(kernel_fd, &req, + sizeof(struct io_thread_req *)); + if(n != sizeof(struct io_thread_req *)){ if(n < 0) printk("io_thread - read failed, fd = %d, " "err = %d\n", kernel_fd, -n); @@ -1426,9 +1464,10 @@ int io_thread(void *arg) continue; } io_count++; - do_io(&req); - n = os_write_file(kernel_fd, &req, sizeof(req)); - if(n != sizeof(req)) + do_io(req); + n = os_write_file(kernel_fd, &req, + sizeof(struct io_thread_req *)); + if(n != sizeof(struct io_thread_req *)) printk("io_thread - write failed, fd = %d, err = %d\n", kernel_fd, -n); } |