o Fix http proxy code for various extentions and deal with FTP archive

  format changes.

o Make anonymous ftp setup more analy retentive.

PR:		16070
Submitted by:	Philipp Mergenthaler <un1i@rz.uni-karlsruhe.de>

5 files changed, 145 insertions, 110 deletions

diff --git a/release/sysinstall/anonFTP.c b/release/sysinstall/anonFTP.c
index 83795cd..75ffab3 100644
--- a/release/sysinstall/anonFTP.c
+++ b/release/sysinstall/anonFTP.c
@@ -284,9 +284,8 @@ configAnonFTP(dialogMenuItem *self)
 	
 	if (DITEM_STATUS(createFtpUser()) == DITEM_SUCCESS) {
 	    msgNotify("Copying password information for anon FTP.");
-	    vsystem("cp /etc/pwd.db %s/etc && chmod 444 %s/etc/pwd.db", tconf.homedir, tconf.homedir);
-	    vsystem("cp /etc/passwd %s/etc && chmod 444 %s/etc/passwd", tconf.homedir, tconf.homedir);
-	    vsystem("cp /etc/group %s/etc && chmod 444 %s/etc/group", tconf.homedir, tconf.homedir);
+	    vsystem("awk -F: '{if ($3 < 10 || $1 == "ftp") print $0}' /etc/passwd > %s/etc/passwd && chmod 444 %s/etc/passwd", tconf.homedir, tconf.homedir);
+	    vsystem("awk -F: '{if ($3 < 100) print $0}' /etc/group > %s/etc/group && chmod 444 %s/etc/group", tconf.homedir, tconf.homedir);
 	    vsystem("chown -R root.%s %s/pub", tconf.group, tconf.homedir);
 	}
 	else {
diff --git a/release/sysinstall/dist.c b/release/sysinstall/dist.c
index 6e4abe2..ccae4ce 100644
--- a/release/sysinstall/dist.c
+++ b/release/sysinstall/dist.c
@@ -576,14 +576,15 @@ check_for_interrupt(void)
 static Boolean
 distExtract(char *parent, Distribution *me)
 {
-    int i, status, total, intr;
+    int i,j, status, total, intr;
     int cpid, zpid, fd2, chunk, numchunks;
-    char *path, *dist, buf[BUFSIZ];
+    char *path, *dist, buf[300000];
     const char *tmp;
     FILE *fp;
     WINDOW *w = savescr();
     struct timeval start, stop;
     struct sigaction old, new;
+    properties dist_attr;
 
     status = TRUE;
     if (isDebug())
@@ -646,8 +647,6 @@ distExtract(char *parent, Distribution *me)
 	    }
 	}
 	else if (fp > 0) {
-	    properties dist_attr;
-
 	    if (isDebug())
 		msgDebug("Parsing attributes file for distribution %s\n", dist);
 
@@ -664,7 +663,6 @@ distExtract(char *parent, Distribution *me)
 		    numchunks = strtol(tmp, 0, 0);
 	    }
 	    fclose(fp);
-	    properties_free(dist_attr);
 	    if (!numchunks)
 		continue;
 	}
@@ -720,12 +718,19 @@ distExtract(char *parent, Distribution *me)
 	/* And go for all the chunks */
 	dialog_clear_norefresh();
 	for (chunk = 0; chunk < numchunks; chunk++) {
-	    int n, retval, last_msg;
+	    int n, retval, last_msg, chunksize, realsize;
 	    char prompt[80];
 
 	    last_msg = 0;
 
 	getchunk:
+	    snprintf(buf, sizeof buf, "cksum.%c%c",  (chunk / 26) + 'a', (chunk % 26) + 'a');
+	    tmp = property_find(dist_attr, buf);
+	    chunksize = 0;
+	    if (tmp) {
+		tmp=index(tmp, ' ');
+		chunksize = strtol(tmp, 0, 0);
+	    }
 	    snprintf(buf, sizeof buf, "%s/%s.%c%c", path, dist, (chunk / 26) + 'a', (chunk % 26) + 'a');
 	    if (isDebug())
 		msgDebug("trying for piece %d of %d: %s\n", chunk + 1, numchunks, buf);
@@ -747,10 +752,11 @@ distExtract(char *parent, Distribution *me)
 	    snprintf(prompt, sizeof prompt, "Extracting %s into %s directory...", dist, root_bias(me[i].my_dir));
 	    dialog_gauge("Progress", prompt, 8, 15, 6, 50, (int)((float)(chunk + 1) / numchunks * 100));
 
+	    realsize = 0;
 	    while (1) {
 		int seconds;
 
-		n = fread(buf, 1, BUFSIZ, fp);
+		n = fread(buf + realsize, 1, BUFSIZ, fp);
 		if (check_for_interrupt()) {
 		    msgConfirm("Media read error:  User interrupt.");
 		    fclose(fp);
@@ -759,6 +765,7 @@ distExtract(char *parent, Distribution *me)
 		else if (n <= 0)
 		    break;
 		total += n;
+		realsize += n;
 
 		/* Print statistics about how we're doing */
 		(void) gettimeofday(&stop, (struct timezone *)0);
@@ -775,19 +782,36 @@ distExtract(char *parent, Distribution *me)
 		    msgInfo("%10d bytes read from %s dist, chunk %2d of %2d @ %.1f KB/sec.",
 			    total, dist, chunk + 1, numchunks, (total / seconds) / 1024.0);
 		}
-		retval = write(fd2, buf, n);
-		if (retval != n) {
+	    }
+	    fclose(fp);
+            
+	    if (!chunksize || (realsize == chunksize)) {
+		/* No substitution necessary */
+		retval = write(fd2, buf, realsize);
+		if (retval != realsize) {
 		    fclose(fp);
 		    dialog_clear_norefresh();
-		    msgConfirm("Write failure on transfer! (wrote %d bytes of %d bytes)", retval, n);
+		    msgConfirm("Write failure on transfer! (wrote %d bytes of %d bytes)", retval, realsize);
 		    goto punt;
 		}
+	    } else {
+		for(j = 0; j < realsize; j++) {
+		    /* On finding CRLF, skip the CR; don't exceed end of buffer. */
+		    if ((buf[j] != 0x0d) || (j == total - 1) || (buf[j + 1] != 0x0a)) {
+			retval = write(fd2, buf + j, 1);
+			if (retval != 1) {
+			    fclose(fp);
+			    dialog_clear_norefresh();
+			    msgConfirm("Write failure on transfer! (wrote %d bytes of %d bytes)", j, chunksize);
+			    goto punt;
+			}
+		    }
+		}
 	    }
-	    fclose(fp);
 	}
 	close(fd2);
 	status = mediaExtractDistEnd(zpid, cpid);
-        goto done;
+	goto done;
 
     punt:
 	close(fd2);
@@ -817,6 +841,7 @@ distExtract(char *parent, Distribution *me)
 	else
 	    continue;
     }
+    properties_free(dist_attr);
     sigaction(SIGINT, &old, NULL);	/* Restore signal handler */
     restorescr(w);
     return status;
diff --git a/release/sysinstall/http.c b/release/sysinstall/http.c
index 0fcf159..14220bb 100644
--- a/release/sysinstall/http.c
+++ b/release/sysinstall/http.c
@@ -11,38 +11,31 @@ Boolean
 mediaInitHTTP(Device *dev)
 {
 /* 
- * Some proxies think that files with the extension ".ai" are postscript
- * files and use "ascii mode" instead of "binary mode" for ftp.
- * The FTP server then translates all LF to CRLF.
- * I don't know how to handle this elegantly...
- * Squid uses ascii mode, ftpget uses binary mode and both tell us:
- * "Content-Type: application/postscript"
- *
- * Probably the safest way would be to get the file, look at its checksum
- * and, if it doesn't match, replace all CRLF by LF and check again.
+ * Some proxies fetch files with certain extensions in "ascii mode" instead
+ * of "binary mode" for FTP. The FTP server then translates all LF to CRLF.
  *
  * You can force Squid to use binary mode by appending ";type=i" to the URL,
- * which is what I do here.
- *
+ * which is what I do here. For other proxies, the LF->CRLF substitution
+ * is reverted in distExtract().
  */
 
     extern int h_errno;
     int rv,s;
-    bool el;                    /* end of header line */
-    char *cp, buf[PATH_MAX], req[1000];
+    bool el;		    /* end of header line */
+    char *cp, buf[PATH_MAX], req[BUFSIZ];
     struct sockaddr_in peer;
     struct hostent *peer_in;
 
     s=socket(PF_INET, SOCK_STREAM, 6);    /* tcp */
     if (s == -1) {
-      msgConfirm("Network error");
-      return FALSE;
+	msgConfirm("Network error");
+	return FALSE;
     }
 
     peer_in=gethostbyname(variable_get(VAR_HTTP_HOST));
     if (peer_in == NULL) {
-      msgConfirm("%s",hstrerror(h_errno));
-      return FALSE;
+	msgConfirm("%s",hstrerror(h_errno));
+	return FALSE;
     }
 
     peer.sin_len=peer_in->h_length;
@@ -52,9 +45,9 @@ mediaInitHTTP(Device *dev)
 
     rv=connect(s,(struct sockaddr *)&peer,sizeof(peer));
     if (rv == -1) {
-      msgConfirm("Couldn't connect to proxy %s:%s",
-                  variable_get(VAR_HTTP_HOST),variable_get(VAR_FTP_PORT));
-      return FALSE;
+	msgConfirm("Couldn't connect to proxy %s:%s",
+		    variable_get(VAR_HTTP_HOST),variable_get(VAR_HTTP_PORT));
+	return FALSE;
     }
 
     sprintf(req,"GET / HTTP/1.0\r\n\r\n");
@@ -69,28 +62,28 @@ mediaInitHTTP(Device *dev)
     rv=read(s,cp,1);
     variable_set2(VAR_HTTP_FTP_MODE,"",0);
     while (rv>0) {
-      if ((*cp == '\012') && el) { 
-        /* reached end of a header line */
-        if (!strncmp(buf,"Server: ",8)) {
-          if (!strncmp(buf,"Server: Squid",13)) {
-            variable_set2(VAR_HTTP_FTP_MODE,";type=i",1);
-          } else {
-            variable_set2(VAR_HTTP_FTP_MODE,"",1);
-          }
-        }
-        /* ignore other headers */
-        /* check for "\015\012" at beginning of line, i.e. end of headers */
-        if ((cp-buf) == 1)
-          break;
-        cp=buf;
-        rv=read(s,cp,1);
-      } else {
-        el=FALSE;
-        if (*cp == '\015')
-          el=TRUE;
-        cp++;
-        rv=read(s,cp,1);
-      }
+	if ((*cp == '\012') && el) { 
+	    /* reached end of a header line */
+	    if (!strncmp(buf,"Server: ",8)) {
+		if (!strncmp(buf,"Server: Squid",13)) {
+		    variable_set2(VAR_HTTP_FTP_MODE,";type=i",0);
+		} else {
+		    variable_set2(VAR_HTTP_FTP_MODE,"",0);
+		}
+	    }
+	    /* ignore other headers */
+	    /* check for "\015\012" at beginning of line, i.e. end of headers */
+	    if ((cp-buf) == 1)
+		break;
+	    cp=buf;
+	    rv=read(s,cp,1);
+	} else {
+	    el=FALSE;
+	    if (*cp == '\015')
+		el=TRUE;
+	    cp++;
+	    rv=read(s,cp,1);
+	}
     }
     close(s);
     return TRUE;
@@ -103,14 +96,14 @@ mediaGetHTTP(Device *dev, char *file, Boolean probe)
     FILE *fp;
     int rv,s;
     bool el;			/* end of header line */
-    char *cp, buf[PATH_MAX], req[1000];
+    char *cp, buf[PATH_MAX], req[BUFSIZ];
     struct sockaddr_in peer;
     struct hostent *peer_in;
 
     s=socket(PF_INET, SOCK_STREAM, 6);    /* tcp */
     if (s == -1) {
-      msgConfirm("Network error");
-      return NULL;
+	msgConfirm("Network error");
+	return NULL;
     }
       
     peer_in=gethostbyname(variable_get(VAR_HTTP_HOST));
@@ -121,16 +114,18 @@ mediaGetHTTP(Device *dev, char *file, Boolean probe)
 
     rv=connect(s,(struct sockaddr *)&peer,sizeof(peer));
     if (rv == -1) {
-      msgConfirm("Couldn't connect to proxy %s:%s",
-                  variable_get(VAR_HTTP_HOST),variable_get(VAR_FTP_PORT));
-      return NULL;
+	msgConfirm("Couldn't connect to proxy %s:%s",
+		    variable_get(VAR_HTTP_HOST),variable_get(VAR_FTP_PORT));
+	return NULL;
+    }
+						   
+    sprintf(req,"GET %s/%s/%s%s HTTP/1.0\r\n\r\n",
+	    variable_get(VAR_FTP_PATH), variable_get(VAR_RELNAME),
+	    file, variable_get(VAR_HTTP_FTP_MODE));
+
+    if (isDebug()) {
+	msgDebug("sending http request: %s",req);
     }
-                                                   
-    sprintf(req,"GET ftp://%s:%s%s%s/%s%s HTTP/1.0\r\n\r\n",
-            variable_get(VAR_FTP_HOST), variable_get(VAR_FTP_PORT),
-            "/pub/FreeBSD/", variable_get(VAR_RELNAME),
-            file,variable_get(VAR_HTTP_FTP_MODE));
-    msgDebug("sending http request: %s",req);
     write(s,req,strlen(req));
 
 /*
@@ -142,41 +137,43 @@ mediaGetHTTP(Device *dev, char *file, Boolean probe)
     el=FALSE;
     rv=read(s,cp,1);
     while (rv>0) {
-      if ((*cp == '\012') && el) {
-        /* reached end of a header line */
-        if (!strncmp(buf,"HTTP",4)) {
-          rv=strtol((char *)(buf+9),0,0);
-          *(cp-1)='\0';		/* chop the CRLF off */
-          if (rv >= 500) {
-            msgConfirm("Server error %s, you could try an other server",buf);
-            return NULL;
-          } else if (rv == 404) {
-            msgConfirm("%s was not found, maybe directory or release-version are wrong?",req);
-            return NULL;
-          } else if (rv >= 400) {
-            msgConfirm("Client error %s, you could try an other server",buf);
-            return NULL;
-          } else if (rv >= 300) {
-            msgConfirm("Error %s,",buf);
-            return NULL;
-          } else if (rv != 200) {
-            msgConfirm("Error %s when trying to fetch %s",buf,req);
-            return NULL;
-          }
-        }
-        /* ignore other headers */
-        /* check for "\015\012" at beginning of line, i.e. end of headers */
-        if ((cp-buf) == 1) 
-          break;
-        cp=buf;
-        rv=read(s,cp,1);
-      } else {
-        el=FALSE;
-        if (*cp == '\015')
-          el=TRUE;
-        cp++;
-        rv=read(s,cp,1);
-      }
+	if ((*cp == '\012') && el) {
+  	    /* reached end of a header line */
+  	    if (!strncmp(buf,"HTTP",4)) {
+		rv=strtol((char *)(buf+9),0,0);
+		*(cp-1)='\0';		/* chop the CRLF off */
+		if (probe && (rv != 200)) {
+		    return NULL;
+		} else if (rv >= 500) {
+		    msgConfirm("Server error %s when sending %s, you could try an other server",buf, req);
+		    return NULL;
+		} else if (rv == 404) {
+		    msgConfirm("%s was not found, maybe directory or release-version are wrong?",req);
+		    return NULL;
+		} else if (rv >= 400) {
+		    msgConfirm("Client error %s, you could try an other server",buf);
+		    return NULL;
+		} else if (rv >= 300) {
+		    msgConfirm("Error %s,",buf);
+		    return NULL;
+		} else if (rv != 200) {
+		    msgConfirm("Error %s when sending %s, you could try an other server",buf, req);
+		    return NULL;
+		}
+	    }
+	    /* ignore other headers */
+	    /* check for "\015\012" at beginning of line, i.e. end of headers */
+	    if ((cp-buf) == 1) 
+		break;
+	    cp=buf;
+	    rv=read(s,cp,1);
+	} else {
+	    el=FALSE;
+	    if (*cp == '\015')
+		el=TRUE;
+	    cp++;
+	    rv=read(s,cp,1);
+	}
     }
     fp=fdopen(s,"r");
     return fp;
diff --git a/release/sysinstall/media.c b/release/sysinstall/media.c
index 960e669..806cb4c 100644
--- a/release/sysinstall/media.c
+++ b/release/sysinstall/media.c
@@ -445,7 +445,6 @@ int mediaSetHTTP(dialogMenuItem *self)
     if (DITEM_STATUS(result) != DITEM_SUCCESS)
 	return result;
  
-    variable_set2(VAR_HTTP_PATH, "", 0);
     cp = variable_get_value(VAR_HTTP_PATH,
 	"Please enter the address of the HTTP proxy in this format:\n"
 	" hostname:port (the ':port' is optional, default is 3128)",0);
@@ -461,11 +460,11 @@ int mediaSetHTTP(dialogMenuItem *self)
 
     variable_set2(VAR_HTTP_HOST, hostname, 0);
     variable_set2(VAR_HTTP_PORT, itoa(HttpPort), 0);
-    msgDebug("VAR_HTTP_HOST, _PORT: %s:%s",variable_get(VAR_HTTP_HOST),
-                                           variable_get(VAR_HTTP_PORT));
-
-    msgDebug("VAR_FTP_HOST, _PORT: %s:%s", variable_get(VAR_FTP_HOST),
-                                           variable_get(VAR_FTP_PORT));
+    if (isDebug()) {
+      msgDebug("VAR_FTP_PATH : %s",variable_get(VAR_FTP_PATH));
+      msgDebug("VAR_HTTP_HOST, _PORT: %s:%s",variable_get(VAR_HTTP_HOST),
+                                             variable_get(VAR_HTTP_PORT));
+    }
 
     /* mediaDevice has been set by mediaSetFTP(), overwrite partly: */
     mediaDevice->type = DEVICE_TYPE_HTTP;
diff --git a/release/sysinstall/sysinstall.8 b/release/sysinstall/sysinstall.8
index 530bcf8..33552fa 100644
--- a/release/sysinstall/sysinstall.8
+++ b/release/sysinstall/sysinstall.8
@@ -675,6 +675,9 @@ The first primary partition found is used (e.g. C:).
 Select a tape device as the installation media.
 .Pp
 \fBVariables:\fR None
+
+.\" XXX
+
 .It mediaSetFTP
 Select an FTP site as the installation media.
 .Pp
@@ -719,6 +722,18 @@ using "passive" FTP transfer mode.
 .Pp
 \fBVariables:\fR Same as for
 .Ar mediaSetFTP .
+.It mediaSetHTTP
+Alias for
+.Ar mediaSetFTP
+using an HTTP proxy.
+.Pp
+\fBVariables:\fR See
+.Ar mediaSetFTP ,
+plus
+.Bl -tag -width indent
+.It _httpPath
+The proxy to use (host:port) (non-optional).
+.El
 .It mediaSetUFS
 Select an existing UFS partition (mounted with the label editor) as
 the installation media.