From 3b2148e2997b381131bff3e262e5d3b4754fed50 Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 18 Feb 2016 08:44:16 +0000 Subject: Reorganize the handling all-zeroes terminal block in sparse mode The intent of the previous code in that case was to force an explicit write, but the implementation was incorrect, and as a result the write was never performed. This new implementation instead uses ftruncate(2) to extend the file with a trailing hole. Also introduce regression tests for these cases. PR: 189284 (original PR whose fix introduced this bug) PR: 207092 Differential Revision: D5248 Reviewed by: sobomax,kib MFC after: 2 weeks --- bin/dd/Makefile | 13 ++++++++++++- bin/dd/dd.c | 36 +++++++++++++++++++++--------------- bin/dd/dd.h | 1 + bin/dd/gen.c | 13 ++++++++++--- bin/dd/ref.obs_zeroes | 3 +++ 5 files changed, 47 insertions(+), 19 deletions(-) create mode 100644 bin/dd/ref.obs_zeroes (limited to 'bin') diff --git a/bin/dd/Makefile b/bin/dd/Makefile index 5f07dbc..6b17d29 100644 --- a/bin/dd/Makefile +++ b/bin/dd/Makefile @@ -24,7 +24,18 @@ test: ${PROG} gen LC_ALL=en_US.US-ASCII hexdump -C | \ diff -I FreeBSD - ${.CURDIR}/ref.${conv} .endfor - @rm -f gen + @${ECHO} "testing sparse file (obs zeroes)" + @./gen 189284 | ./dd ibs=16 obs=8 conv=sparse of=obs_zeroes 2> /dev/null + @hexdump -C obs_zeroes | diff -I FreeBSD - ${.CURDIR}/ref.obs_zeroes + + @${ECHO} "testing spase file (all zeroes)" + @./dd if=/dev/zero of=1M_zeroes bs=1048576 count=1 2> /dev/null + @./dd if=1M_zeroes of=1M_zeroes.1 bs=1048576 conv=sparse 2> /dev/null + @./dd if=1M_zeroes of=1M_zeroes.2 bs=1048576 2> /dev/null + @diff 1M_zeroes 1M_zeroes.1 + @diff 1M_zeroes 1M_zeroes.2 + + @rm -f gen 1M_zeroes* obs_zeroes .if ${MK_TESTS} != "no" SUBDIR+= tests diff --git a/bin/dd/dd.c b/bin/dd/dd.c index 8ae11a7..4c31a5e 100644 --- a/bin/dd/dd.c +++ b/bin/dd/dd.c @@ -77,7 +77,6 @@ STAT st; /* statistics */ void (*cfunc)(void); /* conversion function */ uintmax_t cpy_cnt; /* # of blocks to copy */ static off_t pending = 0; /* pending seek if sparse */ -static off_t last_sp = 0; /* size of last added sparse block */ u_int ddflags = 0; /* conversion options */ size_t cbsz; /* conversion block size */ uintmax_t files_cnt = 1; /* # of files to copy */ @@ -409,6 +408,15 @@ dd_close(void) } if (out.dbcnt || pending) dd_out(1); + + /* + * If the file ends with a hole, ftruncate it to extend its size + * up to the end of the hole (without having to write any data). + */ + if (out.seek_offset > 0 && (out.flags & ISTRUNC)) { + if (ftruncate(out.fd, out.seek_offset) == -1) + err(1, "truncating %s", out.name); + } } void @@ -457,29 +465,27 @@ dd_out(int force) } if (sparse && !force) { pending += cnt; - last_sp = cnt; nw = cnt; } else { if (pending != 0) { - /* If forced to write, and we have no - * data left, we need to write the last - * sparse block explicitly. + /* + * Seek past hole. Note that we need to record the + * reached offset, because we might have no more data + * to write, in which case we'll need to call + * ftruncate to extend the file size. */ - if (force && cnt == 0) { - pending -= last_sp; - assert(outp == out.db); - memset(outp, 0, cnt); - } - if (lseek(out.fd, pending, SEEK_CUR) == - -1) + out.seek_offset = lseek(out.fd, pending, SEEK_CUR); + if (out.seek_offset == -1) err(2, "%s: seek error creating sparse file", out.name); - pending = last_sp = 0; + pending = 0; } - if (cnt) + if (cnt) { nw = write(out.fd, outp, cnt); - else + out.seek_offset = 0; + } else { return; + } } if (nw <= 0) { diff --git a/bin/dd/dd.h b/bin/dd/dd.h index a8b45e5..196f804 100644 --- a/bin/dd/dd.h +++ b/bin/dd/dd.h @@ -54,6 +54,7 @@ typedef struct { const char *name; /* name */ int fd; /* file descriptor */ off_t offset; /* # of blocks to skip */ + off_t seek_offset; /* offset of last seek past output hole */ } IO; typedef struct { diff --git a/bin/dd/gen.c b/bin/dd/gen.c index 9c7571a..d53d8fb 100644 --- a/bin/dd/gen.c +++ b/bin/dd/gen.c @@ -5,13 +5,20 @@ */ #include +#include int -main(int argc __unused, char **argv __unused) +main(int argc, char **argv) { int i; - for (i = 0; i < 256; i++) - putchar(i); + if (argc > 1 && !strcmp(argv[1], "189284")) { + fputs("ABCDEFGH", stdout); + for (i = 0; i < 8; i++) + putchar(0); + } else { + for (i = 0; i < 256; i++) + putchar(i); + } return (0); } diff --git a/bin/dd/ref.obs_zeroes b/bin/dd/ref.obs_zeroes new file mode 100644 index 0000000..473ff7c --- /dev/null +++ b/bin/dd/ref.obs_zeroes @@ -0,0 +1,3 @@ +$FreeBSD$ +00000000 41 42 43 44 45 46 47 48 00 00 00 00 00 00 00 00 |ABCDEFGH........| +00000010 -- cgit v1.1 From f4563d34505e86a964ca266dc23dad8f024c0273 Mon Sep 17 00:00:00 2001 From: jilles Date: Fri, 19 Feb 2016 16:56:07 +0000 Subject: sh: Add tests for comments in sh -c. --- bin/sh/tests/parser/Makefile | 2 ++ bin/sh/tests/parser/comment1.0 | 3 +++ bin/sh/tests/parser/comment2.42 | 4 ++++ 3 files changed, 9 insertions(+) create mode 100644 bin/sh/tests/parser/comment1.0 create mode 100644 bin/sh/tests/parser/comment2.42 (limited to 'bin') diff --git a/bin/sh/tests/parser/Makefile b/bin/sh/tests/parser/Makefile index 0d2ca0f..78a0734 100644 --- a/bin/sh/tests/parser/Makefile +++ b/bin/sh/tests/parser/Makefile @@ -25,6 +25,8 @@ FILES+= alias15.0 alias15.0.stdout FILES+= and-pipe-not.0 FILES+= case1.0 FILES+= case2.0 +FILES+= comment1.0 +FILES+= comment2.42 FILES+= dollar-quote1.0 FILES+= dollar-quote2.0 FILES+= dollar-quote3.0 diff --git a/bin/sh/tests/parser/comment1.0 b/bin/sh/tests/parser/comment1.0 new file mode 100644 index 0000000..21e7ade --- /dev/null +++ b/bin/sh/tests/parser/comment1.0 @@ -0,0 +1,3 @@ +# $FreeBSD$ + +${SH} -c '#' diff --git a/bin/sh/tests/parser/comment2.42 b/bin/sh/tests/parser/comment2.42 new file mode 100644 index 0000000..196b733 --- /dev/null +++ b/bin/sh/tests/parser/comment2.42 @@ -0,0 +1,4 @@ +# $FreeBSD$ + +${SH} -c '# +exit 42' -- cgit v1.1 From 5e2165399aecc8f21bb606161c2ce029768989e1 Mon Sep 17 00:00:00 2001 From: jilles Date: Fri, 19 Feb 2016 21:53:12 +0000 Subject: sh: Rework code to remove '\0' from shell input. This fixes bugs where '\0' was not removed correctly and speeds up the parser. --- bin/sh/input.c | 52 ++++++++++++++++++++------------------------ bin/sh/tests/parser/Makefile | 1 + bin/sh/tests/parser/nul1.0 | 12 ++++++++++ 3 files changed, 37 insertions(+), 28 deletions(-) create mode 100644 bin/sh/tests/parser/nul1.0 (limited to 'bin') diff --git a/bin/sh/input.c b/bin/sh/input.c index 412f144..5921e08 100644 --- a/bin/sh/input.c +++ b/bin/sh/input.c @@ -195,8 +195,7 @@ retry: int preadbuffer(void) { - char *p, *q; - int more; + char *p, *q, *r, *end; char savec; while (parsefile->strpush) { @@ -224,34 +223,31 @@ again: } } - q = p = parsefile->buf + (parsenextc - parsefile->buf); - - /* delete nul characters */ - for (more = 1; more;) { - switch (*p) { - case '\0': - p++; /* Skip nul */ - goto check; - - case '\n': - parsenleft = q - parsenextc; - more = 0; /* Stop processing here */ - break; - - default: - break; - } - - *q++ = *p++; -check: - if (--parselleft <= 0) { - parsenleft = q - parsenextc - 1; - if (parsenleft < 0) - goto again; - *q = '\0'; - more = 0; + p = parsefile->buf + (parsenextc - parsefile->buf); + end = p + parselleft; + *end = '\0'; + q = strchrnul(p, '\n'); + if (q != end && *q == '\0') { + /* delete nul characters */ + for (r = q; q != end; q++) { + if (*q != '\0') + *r++ = *q; } + parselleft -= end - r; + if (parselleft == 0) + goto again; + end = p + parselleft; + *end = '\0'; + q = strchrnul(p, '\n'); + } + if (q == end) { + parsenleft = parselleft; + parselleft = 0; + } else /* *q == '\n' */ { + parsenleft = q - parsenextc + 1; + parselleft -= parsenleft; } + parsenleft--; savec = *q; *q = '\0'; diff --git a/bin/sh/tests/parser/Makefile b/bin/sh/tests/parser/Makefile index 78a0734..eb1e6f5 100644 --- a/bin/sh/tests/parser/Makefile +++ b/bin/sh/tests/parser/Makefile @@ -73,6 +73,7 @@ FILES+= line-cont10.0 FILES+= line-cont11.0 FILES+= no-space1.0 FILES+= no-space2.0 +FILES+= nul1.0 FILES+= only-redir1.0 FILES+= only-redir2.0 FILES+= only-redir3.0 diff --git a/bin/sh/tests/parser/nul1.0 b/bin/sh/tests/parser/nul1.0 new file mode 100644 index 0000000..49c5ab1 --- /dev/null +++ b/bin/sh/tests/parser/nul1.0 @@ -0,0 +1,12 @@ +# $FreeBSD$ +# Although POSIX does not specify the effect of NUL bytes in scripts, +# we ignore them. + +{ + printf 'v=%03000d\0%02000d' 7 2 + dd if=/dev/zero bs=1000 count=1 status=none + printf '1 w=%03000d%02000d1\0\n' 7 2 + printf '\0l\0v\0=\0$\0{\0#\0v\0}\n' + printf '\0l\0w\0=\0\0$\0{\0#\0w}\0\0\0\n' + printf '[ "$lv.$lw.$v" = "5001.5001.$w" ]\n' +} | ${SH} -- cgit v1.1 From c5bee1a6aa9165429481650fdd1a309113ee83bd Mon Sep 17 00:00:00 2001 From: trasz Date: Sun, 21 Feb 2016 14:36:50 +0000 Subject: Make the "invalid numeric value" error message actually displayable (was a dead code before). Submitted by: bde@ (earlier version) Reviewed by: bde@ MFC after: 1 month Sponsored by: The FreeBSD Foundation --- bin/dd/args.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'bin') diff --git a/bin/dd/args.c b/bin/dd/args.c index db8d445..4607d67 100644 --- a/bin/dd/args.c +++ b/bin/dd/args.c @@ -422,11 +422,10 @@ get_num(const char *val) errno = 0; num = strtoumax(val, &expr, 0); - if (errno != 0) /* Overflow or underflow. */ - err(1, "%s", oper); - if (expr == val) /* No valid digits. */ - errx(1, "%s: illegal numeric value", oper); + errx(1, "%s: invalid numeric value", oper); + if (errno != 0) + err(1, "%s", oper); mult = postfix_to_mult(*expr); @@ -472,11 +471,10 @@ get_off_t(const char *val) errno = 0; num = strtoimax(val, &expr, 0); - if (errno != 0) /* Overflow or underflow. */ - err(1, "%s", oper); - if (expr == val) /* No valid digits. */ - errx(1, "%s: illegal numeric value", oper); + errx(1, "%s: invalid numeric value", oper); + if (errno != 0) + err(1, "%s", oper); mult = postfix_to_mult(*expr); -- cgit v1.1 From 7ac5bf14ff211d70f6a8b3b484b0490416424090 Mon Sep 17 00:00:00 2001 From: jilles Date: Sun, 21 Feb 2016 16:48:37 +0000 Subject: sh: Remove unnecessary flushouts while reading script. Output is flushed when a builtin is done or immediately after writing it (error messages, set -v output, prompts). --- bin/sh/input.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'bin') diff --git a/bin/sh/input.c b/bin/sh/input.c index 5921e08..27b835a 100644 --- a/bin/sh/input.c +++ b/bin/sh/input.c @@ -212,8 +212,6 @@ preadbuffer(void) } if (parsenleft == EOF_NLEFT || parsefile->buf == NULL) return PEOF; - flushout(&output); - flushout(&errout); again: if (parselleft <= 0) { -- cgit v1.1 From f523bd6c78181527b21a37f4dc5c51e0517bf523 Mon Sep 17 00:00:00 2001 From: jilles Date: Sun, 21 Feb 2016 18:54:17 +0000 Subject: sh: Optimize setprompt(0). Avoid doing work to print an empty prompt (such as when reading scripts). --- bin/sh/parser.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'bin') diff --git a/bin/sh/parser.c b/bin/sh/parser.c index 53d7923..d324d88 100644 --- a/bin/sh/parser.c +++ b/bin/sh/parser.c @@ -1930,6 +1930,8 @@ static void setprompt(int which) { whichprompt = which; + if (which == 0) + return; #ifndef NO_HISTORY if (!el) -- cgit v1.1 From 100946a0d0f01db31afd0e3a8796cac6b30f9b80 Mon Sep 17 00:00:00 2001 From: jilles Date: Sun, 21 Feb 2016 20:58:24 +0000 Subject: sh: Don't hash alias name when there are no aliases. --- bin/sh/alias.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'bin') diff --git a/bin/sh/alias.c b/bin/sh/alias.c index a77ce99..a35513b 100644 --- a/bin/sh/alias.c +++ b/bin/sh/alias.c @@ -144,9 +144,11 @@ rmaliases(void) struct alias * lookupalias(const char *name, int check) { - struct alias *ap = *hashalias(name); + struct alias *ap; - for (; ap; ap = ap->next) { + if (aliases == 0) + return (NULL); + for (ap = *hashalias(name); ap; ap = ap->next) { if (equal(name, ap->name)) { if (check && (ap->flag & ALIASINUSE)) return (NULL); -- cgit v1.1