genext2fs: support large files and filesystems without using large amounts of memory

update_to_1.95.patch was generated by making a diff bewteen the 1.4.1 release
and the latest 1.9.5 version in the cvs repo:
http://genext2fs.cvs.sourceforge.net/viewvc/genext2fs/genext2fs/genext2fs.c?revision=1.95

The patches 0001-0019 come from mailing list of genext2fs-devel
http://sourceforge.net/mailarchive/forum.php?forum_name=genext2fs-devel&max_rows=100&style=flat&viewmonth=201106

(From OE-Core rev: 8f17e499cf91191727c8767e839738cb39c21655)

Signed-off-by: Dexuan Cui <dexuan.cui@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 272 insertions, 0 deletions

diff --git a/meta/recipes-devtools/genext2fs/genext2fs-1.4.1/0006-Improve-the-efficiency-of-extend_blk.patch b/meta/recipes-devtools/genext2fs/genext2fs-1.4.1/0006-Improve-the-efficiency-of-extend_blk.patch
new file mode 100644
index 0000000..494e3f9
--- /dev/null
+++ b/meta/recipes-devtools/genext2fs/genext2fs-1.4.1/0006-Improve-the-efficiency-of-extend_blk.patch
@@ -0,0 +1,272 @@
+Upstream-Status: inappropriate
+
+From ea6cbe2880e02026667b9007db9a742be7dcd52b Mon Sep 17 00:00:00 2001
+From: Corey Minyard <cminyard@mvista.com>
+Date: Sun, 5 Jun 2011 10:26:11 -0500
+Subject: [PATCH 06/19] Improve the efficiency of extend_blk
+
+When doing multiple extensions operations on the same inode, extend_blk()
+would search to the end every time.  In the current structure, that means
+when creating the reserved blocks, it could parse over the block list
+many thousands of time on a large filesystem.  For an 800MB filesystem,
+that took 2/3rds of the time.
+
+So create a structure for holding the inode position and use it to
+know where the end is.  This neatens things up a bit, too, more
+clearly showing when a truncate or extend is occuring.  In future
+changes, this will also make it efficient for supporting very large
+files that cannot be fully allocated in memory.
+---
+ genext2fs.c |  147 ++++++++++++++++++++++++++++++++++++++++-------------------
+ 1 files changed, 100 insertions(+), 47 deletions(-)
+
+diff --git a/genext2fs.c b/genext2fs.c
+index 46c9605..e45e520 100644
+--- a/genext2fs.c
++++ b/genext2fs.c
+@@ -436,6 +436,17 @@ swab32(uint32 val)
+ 			((val<<8)&0xFF0000) | (val<<24));
+ }
+ 
++static inline int
++is_blk_empty(uint8 *b)
++{
++	uint32 i;
++	uint32 *v = (uint32 *) b;
++
++	for(i = 0; i < BLOCKSIZE / 4; i++)
++		if (*v++)
++			return 0;
++	return 1;
++}
+ 
+ // on-disk structures
+ // this trick makes me declare things only once
+@@ -1165,7 +1176,6 @@ init_bw(blockwalker *bw)
+ //				  used after being freed, so once you start
+ //				  freeing blocks don't stop until the end of
+ //				  the file. moreover, i_blocks isn't updated.
+-//				  in fact, don't do that, just use extend_blk
+ // if hole!=0, create a hole in the file
+ static uint32
+ walk_bw(filesystem *fs, uint32 nod, blockwalker *bw, int32 *create, uint32 hole)
+@@ -1422,52 +1432,80 @@ walk_bw(filesystem *fs, uint32 nod, blockwalker *bw, int32 *create, uint32 hole)
+ 	return bk;
+ }
+ 
+-// add blocks to an inode (file/dir/etc...)
+-static void
+-extend_blk(filesystem *fs, uint32 nod, block b, int amount)
++typedef struct
+ {
+-	int create = amount;
+-	blockwalker bw, lbw;
+-	uint32 bk;
++	blockwalker bw;
++	uint32 nod;
+ 	nod_info *ni;
+ 	inode *inod;
++} inode_pos;
++#define INODE_POS_TRUNCATE 0
++#define INODE_POS_EXTEND 1
++
++// Call this to set up an ipos structure for future use with
++// extend_inode_blk to append blocks to the given inode.  If
++// op is INODE_POS_TRUNCATE, the inode is truncated to zero size.
++// If op is INODE_POS_EXTEND, the position is moved to the end
++// of the inode's data blocks.
++// Call inode_pos_finish when done with the inode_pos structure.
++static void
++inode_pos_init(filesystem *fs, inode_pos *ipos, uint32 nod, int op,
++	       blockwalker *endbw)
++{
++	blockwalker lbw;
+ 
+-	inod = get_nod(fs, nod, &ni);
+-	init_bw(&bw);
+-	if(amount < 0)
+-	{
+-		uint32 i;
+-		for(i = 0; i < inod->i_blocks / INOBLK + amount; i++)
+-			walk_bw(fs, nod, &bw, 0, 0);
+-		while(walk_bw(fs, nod, &bw, &create, 0) != WALK_END)
++	init_bw(&ipos->bw);
++	ipos->nod = nod;
++	ipos->inod = get_nod(fs, nod, &ipos->ni);
++	if (op == INODE_POS_TRUNCATE) {
++		int32 create = -1;
++		while(walk_bw(fs, nod, &ipos->bw, &create, 0) != WALK_END)
+ 			/*nop*/;
+-		inod->i_blocks += amount * INOBLK;
++		ipos->inod->i_blocks = 0;
+ 	}
+-	else
++
++	if (endbw)
++		ipos->bw = *endbw;
++	else {
++		/* Seek to the end */
++		init_bw(&ipos->bw);
++		lbw = ipos->bw;
++		while(walk_bw(fs, nod, &ipos->bw, 0, 0) != WALK_END)
++			lbw = ipos->bw;
++		ipos->bw = lbw;
++	}
++}
++
++// Clean up the inode_pos structure.
++static void
++inode_pos_finish(filesystem *fs, inode_pos *ipos)
++{
++	put_nod(ipos->ni);
++}
++
++// add blocks to an inode (file/dir/etc...) at the given position.
++// This will only work when appending to the end of an inode.
++static void
++extend_inode_blk(filesystem *fs, inode_pos *ipos, block b, int amount)
++{
++	uint32 bk;
++	uint32 pos;
++
++	if (amount < 0)
++		error_msg_and_die("extend_inode_blk: Got negative amount");
++
++	for (pos = 0; amount; pos += BLOCKSIZE)
+ 	{
+-		lbw = bw;
+-		while((bk = walk_bw(fs, nod, &bw, 0, 0)) != WALK_END)
+-			lbw = bw;
+-		bw = lbw;
+-		while(create)
+-		{
+-			int i, copyb = 0;
+-			if(!(fs->sb->s_reserved[200] & OP_HOLES))
+-				copyb = 1;
+-			else
+-				for(i = 0; i < BLOCKSIZE / 4; i++)
+-					if(((int32*)(b + BLOCKSIZE * (amount - create)))[i])
+-					{
+-						copyb = 1;
+-						break;
+-					}
+-			if((bk = walk_bw(fs, nod, &bw, &create, !copyb)) == WALK_END)
+-				break;
+-			if(copyb) {
+-				blk_info *bi;
+-				memcpy(get_blk(fs, bk, &bi), b + BLOCKSIZE * (amount - create - 1), BLOCKSIZE);
+-				put_blk(bi);
+-			}
++		int hole = ((fs->sb->s_reserved[200] & OP_HOLES) && is_blk_empty(b + pos));
++
++		bk = walk_bw(fs, ipos->nod, &ipos->bw, &amount, hole);
++		if (bk == WALK_END)
++			error_msg_and_die("extend_inode_blk: extend failed");
++		if (!hole) {
++			blk_info *bi;
++			uint8 *block = get_blk(fs, bk, &bi);
++			memcpy(block, b + pos, BLOCKSIZE);
++			put_blk(bi);
+ 		}
+ 	}
+ }
+@@ -1484,6 +1522,7 @@ add2dir(filesystem *fs, uint32 dnod, uint32 nod, const char* name)
+ 	inode *node;
+ 	inode *pnode;
+ 	nod_info *dni, *ni;
++	inode_pos ipos;
+ 
+ 	pnode = get_nod(fs, dnod, &dni);
+ 	if((pnode->i_mode & FM_IFMT) != FM_IFDIR)
+@@ -1534,7 +1573,11 @@ add2dir(filesystem *fs, uint32 dnod, uint32 nod, const char* name)
+ 	node->i_links_count++;
+ 	put_nod(ni);
+ 	next_dir(&dw); // Force the data into the buffer
+-	extend_blk(fs, dnod, dir_data(&dw), 1);
++
++	inode_pos_init(fs, &ipos, dnod, INODE_POS_EXTEND, &lbw);
++	extend_inode_blk(fs, &ipos, dir_data(&dw), 1);
++	inode_pos_finish(fs, &ipos);
++
+ 	put_dir(&dw);
+ 	pnode->i_size += BLOCKSIZE;
+ out:
+@@ -1654,8 +1697,9 @@ mklink_fs(filesystem *fs, uint32 parent_nod, const char *name, size_t size, uint
+ 	uint32 nod = mknod_fs(fs, parent_nod, name, FM_IFLNK | FM_IRWXU | FM_IRWXG | FM_IRWXO, uid, gid, 0, 0, ctime, mtime);
+ 	nod_info *ni;
+ 	inode *node = get_nod(fs, nod, &ni);
++	inode_pos ipos;
+ 
+-	extend_blk(fs, nod, 0, - (int)node->i_blocks / INOBLK);
++	inode_pos_init(fs, &ipos, nod, INODE_POS_TRUNCATE, NULL);
+ 	node->i_size = size;
+ 	if(size <= 4 * (EXT2_TIND_BLOCK+1))
+ 	{
+@@ -1663,7 +1707,8 @@ mklink_fs(filesystem *fs, uint32 parent_nod, const char *name, size_t size, uint
+ 		put_nod(ni);
+ 		return nod;
+ 	}
+-	extend_blk(fs, nod, b, rndup(size, BLOCKSIZE) / BLOCKSIZE);
++	extend_inode_blk(fs, &ipos, b, rndup(size, BLOCKSIZE) / BLOCKSIZE);
++	inode_pos_finish(fs, &ipos);
+ 	put_nod(ni);
+ 	return nod;
+ }
+@@ -1676,8 +1721,9 @@ mkfile_fs(filesystem *fs, uint32 parent_nod, const char *name, uint32 mode, size
+ 	uint32 nod = mknod_fs(fs, parent_nod, name, mode|FM_IFREG, uid, gid, 0, 0, ctime, mtime);
+ 	nod_info *ni;
+ 	inode *node = get_nod(fs, nod, &ni);
++	inode_pos ipos;
+ 
+-	extend_blk(fs, nod, 0, - (int)node->i_blocks / INOBLK);
++	inode_pos_init(fs, &ipos, nod, INODE_POS_TRUNCATE, NULL);
+ 	node->i_size = size;
+ 	if (size) {
+ 		if(!(b = (uint8*)calloc(rndup(size, BLOCKSIZE), 1)))
+@@ -1685,9 +1731,11 @@ mkfile_fs(filesystem *fs, uint32 parent_nod, const char *name, uint32 mode, size
+ 		if(f)
+ 			if (fread(b, size, 1, f) != 1) // FIXME: ugly. use mmap() ...
+ 				error_msg_and_die("fread failed");
+-		extend_blk(fs, nod, b, rndup(size, BLOCKSIZE) / BLOCKSIZE);
++		extend_inode_blk(fs, &ipos, b,
++				 rndup(size, BLOCKSIZE) / BLOCKSIZE);
+ 		free(b);
+ 	}
++	inode_pos_finish(fs, &ipos);
+ 	put_nod(ni);
+ 	return nod;
+ }
+@@ -2210,6 +2258,7 @@ init_fs(int nbblocks, int nbinodes, int nbresrvd, int holes, uint32 fs_timestamp
+ 	inode *itab0;
+ 	blk_info *bi;
+ 	nod_info *ni;
++	inode_pos ipos;
+ 	
+ 	if(nbresrvd < 0)
+ 		error_msg_and_die("reserved blocks value is invalid. Note: options have changed, see --help or the man page.");
+@@ -2337,7 +2386,9 @@ init_fs(int nbblocks, int nbinodes, int nbresrvd, int holes, uint32 fs_timestamp
+ 	new_dir(fs, EXT2_ROOT_INO, ".", 1, &dw);
+ 	shrink_dir(&dw, EXT2_ROOT_INO, "..", 2);
+ 	next_dir(&dw); // Force the data into the buffer
+-	extend_blk(fs, EXT2_ROOT_INO, dir_data(&dw), 1);
++	inode_pos_init(fs, &ipos, EXT2_ROOT_INO, INODE_POS_EXTEND, NULL);
++	extend_inode_blk(fs, &ipos, dir_data(&dw), 1);
++	inode_pos_finish(fs, &ipos);
+ 	put_dir(&dw);
+ 
+ 	// make lost+found directory and reserve blocks
+@@ -2355,8 +2406,10 @@ init_fs(int nbblocks, int nbinodes, int nbresrvd, int holes, uint32 fs_timestamp
+ 		 */
+ 		if (fs->sb->s_r_blocks_count > fs->sb->s_blocks_count * MAX_RESERVED_BLOCKS )
+ 			fs->sb->s_r_blocks_count = fs->sb->s_blocks_count * MAX_RESERVED_BLOCKS;
++		inode_pos_init(fs, &ipos, nod, INODE_POS_EXTEND, NULL);
+ 		for(i = 1; i < fs->sb->s_r_blocks_count; i++)
+-			extend_blk(fs, nod, b, 1);
++			extend_inode_blk(fs, &ipos, b, 1);
++		inode_pos_finish(fs, &ipos);
+ 		free_workblk(b);
+ 		node = get_nod(fs, nod, &ni);
+ 		node->i_size = fs->sb->s_r_blocks_count * BLOCKSIZE;
+-- 
+1.7.4.1
+