Changes have been made to support a concept of VAR_FIXIT_TTY being

standard or serial.  This change needs to be done to the entire system that
depends on this.  This way we don't have some code using OnVTY checks
and other doing
        strcmp(variable_get(VAR_FIXIT_TTY), "standard") == 0
checks.  Also we need to set VAR_FIXIT_TTY to "serial" if we come up on
a serial console.

Also fixed a dialog problem in that dialog was used when dialog was
disabled causing some troubles such as not letting the cursor keys
work when exiting the fixit mode on media (ie. not the fixit shell but
for example fixit on a floppy).

Submitted by:	Doug Ambrisko <ambrisko@whistle.com>
PR:		22352

2 files changed, 43 insertions, 26 deletions

diff --git a/release/sysinstall/install.c b/release/sysinstall/install.c
index 48ce0c7..630e7f8 100644
--- a/release/sysinstall/install.c
+++ b/release/sysinstall/install.c
@@ -400,23 +400,24 @@ fixit_common(void)
 	msgConfirm("Couldn't symlink the /etc/ files!  I'm not sure I like this..");
     if (!file_readable(TERMCAP_FILE))
 	create_termcap();
-    if (!OnVTY)
+    if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0) 
 	systemSuspendDialog();	/* must be before the fork() */
     if (!(child = fork())) {
-	int i, fd, fdstop;
+	int i, fd;
 	struct termios foo;
 	extern int login_tty(int);
 
 	ioctl(0, TIOCNOTTY, NULL);
-	fdstop = strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0 ? 3 : 0;
-	for (i = getdtablesize(); i >= fdstop; --i)
+	for (i = getdtablesize(); i >= 0; --i)
 	    close(i);
-	if (strcmp(variable_get(VAR_FIXIT_TTY), "standard") == 0) {
+
+	if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0) 
+	    fd = open("/dev/console", O_RDWR);
+	else
 	    fd = open("/dev/ttyv3", O_RDWR);
-	    ioctl(0, TIOCSCTTY, &fd);
-	    dup2(0, 1);
-	    dup2(0, 2);
-	}
+	ioctl(0, TIOCSCTTY, &fd);
+	dup2(0, 1);
+	dup2(0, 2);
 	DebugFD = 2;
 	if (login_tty(fd) == -1)
 	    msgDebug("fixit: I can't set the controlling terminal.\n");
@@ -431,6 +432,13 @@ fixit_common(void)
 	    msgDebug("fixit shell: Unable to get terminal attributes!\n");
 	setenv("PATH", "/bin:/sbin:/usr/bin:/usr/sbin:/stand:"
 	       "/mnt2/stand:/mnt2/bin:/mnt2/sbin:/mnt2/usr/bin:/mnt2/usr/sbin", 1);
+	if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0) {
+	    printf("Waiting for fixit shell to exit.\n"
+		"When you are done, type ``exit'' to exit\n"
+		"the fixit shell and be returned here.\n\n");
+	    fflush(stdout);
+	}
+
 	/* use the .profile from the fixit medium */
 	setenv("HOME", "/mnt2", 1);
 	chdir("/mnt2");
@@ -439,17 +447,15 @@ fixit_common(void)
 	_exit(1);;
     }
     else {
-	dialog_clear_norefresh();
 	if (strcmp(variable_get(VAR_FIXIT_TTY), "standard") == 0) {
+	    dialog_clear_norefresh();
 	    msgNotify("Waiting for fixit shell to exit.  Go to VTY4 now by\n"
 		"typing ALT-F4.  When you are done, type ``exit'' to exit\n"
-		"the fixit shell and be returned here.");
-	} else {
-	    msgNotify("Waiting for fixit shell to exit.\n"
-		"When you are done, type ``exit'' to exit\n"
-		"the fixit shell and be returned here.");
+		"the fixit shell and be returned here\n.");
 	}
 	(void)waitpid(child, &waitstatus, 0);
+	if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0)
+	    systemResumeDialog();
     }
     dialog_clear();
 }
@@ -1049,7 +1055,10 @@ installVarDefaults(dialogMenuItem *self)
     variable_set2(VAR_BROWSER_BINARY,		"/usr/local/bin/lynx", 0);
     variable_set2(VAR_FTP_STATE,		"passive", 0);
     variable_set2(VAR_NFS_SECURE,		"NO", -1);
-    variable_set2(VAR_FIXIT_TTY,		"standard", 0);
+    if (OnVTY)
+	    variable_set2(VAR_FIXIT_TTY,		"standard", 0);
+    else
+	    variable_set2(VAR_FIXIT_TTY,		"serial", 0);
     variable_set2(VAR_PKG_TMPDIR,		"/usr/tmp", 0);
     variable_set2(VAR_MEDIA_TIMEOUT,		itoa(MEDIA_TIMEOUT), 0);
     if (getpid() != 1)
diff --git a/release/sysinstall/system.c b/release/sysinstall/system.c
index 1423405..e2a36eb 100644
--- a/release/sysinstall/system.c
+++ b/release/sysinstall/system.c
@@ -128,9 +128,10 @@ systemInitialize(int argc, char **argv)
 	setsid();
 	close(0);
 	fd = open("/dev/ttyv0", O_RDWR);
-	if (fd == -1)
+	if (fd == -1) {
 	    fd = open("/dev/console", O_RDWR);	/* fallback */
-	else
+	    variable_set2(VAR_FIXIT_TTY, "serial", 0); /* give fixit a hint */
+	} else
 	    OnVTY = TRUE;
 	/*
 	 * To make _sure_ we're on a VTY and don't have /dev/console switched
@@ -143,6 +144,9 @@ systemInitialize(int argc, char **argv)
 	    if ((fd2 = open("/dev/console", O_RDWR)) != -1) {
 		if (ioctl(fd2, CONS_CURSORTYPE, &type) == -1) {
 		    OnVTY = FALSE;
+		    variable_set2(VAR_FIXIT_TTY, "serial", 0); /* Tell Fixit
+								  the console
+								  type */
 		    close(fd); close(fd2);
 		    open("/dev/console", O_RDWR);
 		}
@@ -445,7 +449,7 @@ systemCreateHoloshell(void)
 	    (void) waitpid(ehs_pid, &pstat, WNOHANG);
 	}
 
-	if (!OnVTY)
+	if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0) 
 	    systemSuspendDialog();	/* must be before the fork() */
 	if ((ehs_pid = fork()) == 0) {
 	    int i, fd;
@@ -455,10 +459,10 @@ systemCreateHoloshell(void)
 	    ioctl(0, TIOCNOTTY, NULL);
 	    for (i = getdtablesize(); i >= 0; --i)
 		close(i);
-	    if (OnVTY)
-	        fd = open("/dev/ttyv3", O_RDWR);
-	    else
+	    if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0) 
 	        fd = open("/dev/console", O_RDWR);
+	    else
+	        fd = open("/dev/ttyv3", O_RDWR);
 	    ioctl(0, TIOCSCTTY, &fd);
 	    dup2(0, 1);
 	    dup2(0, 2);
@@ -473,7 +477,7 @@ systemCreateHoloshell(void)
 	    }
 	    else
 		msgDebug("Doctor: I'm unable to get the terminal attributes!\n");
-	    if (!OnVTY){
+	    if (strcmp(variable_get(VAR_FIXIT_TTY), "serial") == 0) {
 	        printf("Type ``exit'' in this fixit shell to resume sysinstall.\n\n");
 		fflush(stdout);
 	    }
@@ -482,15 +486,19 @@ systemCreateHoloshell(void)
 	    exit(1);
 	}
 	else {
-	    if (OnVTY) {
+	    if (strcmp(variable_get(VAR_FIXIT_TTY), "standard") == 0) {
 	        WINDOW *w = savescr();
 
 	        msgNotify("Starting an emergency holographic shell on VTY4");
 	        sleep(2);
 	        restorescr(w);
 	    }
-	    if (!OnVTY){
-	        (void)waitpid(ehs_pid, &waitstatus, 0);
+	    else {
+	        (void)waitpid(ehs_pid, &waitstatus, 0); /* we only wait for
+							   shell to finish 
+							   it serial mode
+							   since there is no
+							   virtual console */
 	        systemResumeDialog();
 	    }
 	}