sh: Change interaction of command substitution and here documents.

If a command substitution contains a newline token, this no longer starts
here documents of outer commands. This way, we follow POSIX's idea of the
command substitution being a separate script more closely. It also matches
other shells better and is consistent with newline characters in quotes not
starting here documents.

The extension tested in parser/heredoc3.0 ($(cat <<EOF)\ntext\nEOF\n)
continues to be supported.

In particular, this change allows things like
  cat <<EOF && echo `pwd`
(a `` command substitution after a here document)
which formerly silently used an empty file as the here document, because the
EOF of the inner command "pwd" also forced an empty here document.

2 files changed, 100 insertions, 0 deletions

diff --git a/tools/regression/bin/sh/parser/heredoc4.0 b/tools/regression/bin/sh/parser/heredoc4.0
new file mode 100644
index 0000000..fa3af5f
--- /dev/null
+++ b/tools/regression/bin/sh/parser/heredoc4.0
@@ -0,0 +1,44 @@
+# $FreeBSD$
+
+failures=0
+
+check() {
+	if ! eval "[ $* ]"; then
+		echo "Failed: $*"
+		: $((failures += 1))
+	fi
+}
+
+f() {
+	cat <<EOF && echo `echo bar`
+foo
+EOF
+}
+check '"`f`" = "foo
+bar"'
+
+f() {
+	cat <<EOF && echo $(echo bar)
+foo
+EOF
+}
+check '"$(f)" = "foo
+bar"'
+
+f() {
+	echo `echo bar` && cat <<EOF
+foo
+EOF
+}
+check '"`f`" = "bar
+foo"'
+
+f() {
+	echo $(echo bar) && cat <<EOF
+foo
+EOF
+}
+check '"$(f)" = "bar
+foo"'
+
+exit $((failures != 0))
diff --git a/tools/regression/bin/sh/parser/heredoc5.0 b/tools/regression/bin/sh/parser/heredoc5.0
new file mode 100644
index 0000000..84b0eb2
--- /dev/null
+++ b/tools/regression/bin/sh/parser/heredoc5.0
@@ -0,0 +1,56 @@
+# $FreeBSD$
+
+failures=0
+
+check() {
+	if ! eval "[ $* ]"; then
+		echo "Failed: $*"
+		: $((failures += 1))
+	fi
+}
+
+f() {
+	cat <<EOF && echo `cat <<EOF
+bar
+EOF
+`
+foo
+EOF
+}
+check '"`f`" = "foo
+bar"'
+
+f() {
+	cat <<EOF && echo $(cat <<EOF
+bar
+EOF
+)
+foo
+EOF
+}
+check '"$(f)" = "foo
+bar"'
+
+f() {
+	echo `cat <<EOF
+bar
+EOF
+` && cat <<EOF
+foo
+EOF
+}
+check '"`f`" = "bar
+foo"'
+
+f() {
+	echo $(cat <<EOF
+bar
+EOF
+) && cat <<EOF
+foo
+EOF
+}
+check '"$(f)" = "bar
+foo"'
+
+exit $((failures != 0))