diff options
Diffstat (limited to 'usr.sbin/portsnap')
-rw-r--r-- | usr.sbin/portsnap/Makefile | 5 | ||||
-rw-r--r-- | usr.sbin/portsnap/Makefile.inc | 5 | ||||
-rw-r--r-- | usr.sbin/portsnap/make_index/Makefile | 9 | ||||
-rw-r--r-- | usr.sbin/portsnap/make_index/make_index.c | 491 | ||||
-rw-r--r-- | usr.sbin/portsnap/phttpget/Makefile | 9 | ||||
-rw-r--r-- | usr.sbin/portsnap/phttpget/phttpget.c | 598 | ||||
-rw-r--r-- | usr.sbin/portsnap/portsnap/Makefile | 6 | ||||
-rw-r--r-- | usr.sbin/portsnap/portsnap/portsnap.8 | 208 | ||||
-rw-r--r-- | usr.sbin/portsnap/portsnap/portsnap.sh | 901 |
9 files changed, 2232 insertions, 0 deletions
diff --git a/usr.sbin/portsnap/Makefile b/usr.sbin/portsnap/Makefile new file mode 100644 index 0000000..0f77bcb --- /dev/null +++ b/usr.sbin/portsnap/Makefile @@ -0,0 +1,5 @@ +# $FreeBSD$ + +SUBDIR= portsnap make_index phttpget + +.include <bsd.subdir.mk> diff --git a/usr.sbin/portsnap/Makefile.inc b/usr.sbin/portsnap/Makefile.inc new file mode 100644 index 0000000..5f6527e --- /dev/null +++ b/usr.sbin/portsnap/Makefile.inc @@ -0,0 +1,5 @@ +# $FreeBSD$ + +LIBEXECDIR?= /usr/libexec + +.include "../Makefile.inc" diff --git a/usr.sbin/portsnap/make_index/Makefile b/usr.sbin/portsnap/make_index/Makefile new file mode 100644 index 0000000..65b1a8e --- /dev/null +++ b/usr.sbin/portsnap/make_index/Makefile @@ -0,0 +1,9 @@ +# $FreeBSD$ + +PROG= make_index +NO_MAN= +WARNS?= 6 + +BINDIR= ${LIBEXECDIR} + +.include <bsd.prog.mk> diff --git a/usr.sbin/portsnap/make_index/make_index.c b/usr.sbin/portsnap/make_index/make_index.c new file mode 100644 index 0000000..6139fad --- /dev/null +++ b/usr.sbin/portsnap/make_index/make_index.c @@ -0,0 +1,491 @@ +/*- + * Copyright 2005 Colin Percival + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <err.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +struct port; + +typedef union { + char * name; + struct port * p; +} DEP; + +typedef struct port { + char * pkgname; + char * portdir; + char * prefix; + char * comment; + char * pkgdescr; + char * maintainer; + char * categories; + size_t n_edep; + DEP * edep; + size_t n_pdep; + DEP * pdep; + size_t n_fdep; + DEP * fdep; + size_t n_bdep; + DEP * bdep; + size_t n_rdep; + DEP * rdep; + char * www; + int recursed; +} PORT; + +static void usage(void); +static char * strdup2(const char *str); +static DEP * makelist(char * str, size_t * n); +static PORT * portify(char * line); +static int portcompare(char * a, char * b); +static void heapifyports(PORT **pp, size_t size, size_t pos); +static PORT * findport(PORT ** pp, size_t st, size_t en, char * name); +static void translateport(PORT ** pp, size_t pplen, PORT * p); +static DEP * recurse_one(DEP * d, size_t * nd); +static void recurse(PORT * p); +static void heapifypkgs(DEP * d, size_t size, size_t pos); +static void sortpkgs(DEP * d, size_t nd); +static void printport(PORT * p); + +static void +usage(void) +{ + + fprintf(stderr, "usage: make_index file\n"); + exit(1); + /* NOTREACHED */ +} + +static char * +strdup2(const char *str) +{ + char * r; + + r = strdup(str); + if (r == NULL) + err(1, "strdup"); + return r; +} + +/* Take a space-separated list and return an array of (char *) */ +static DEP * +makelist(char * str, size_t * n) +{ + DEP * d; + size_t i; + + /* No depends at all? */ + if (str[0] == 0) { + *n = 0; + return NULL; + } + + /* Count the number of fields */ + *n = 1; + for (i = 0; str[i] != 0; i++) + if (str[i] == ' ') + (*n)++; + + /* Allocate and fill an array */ + d = malloc(*n * sizeof(DEP)); + for (i = 0; i < *n; i++) { + d[i].name = strdup2(strsep(&str, " ")); + + /* Strip trailing slashes */ + if (d[i].name[strlen(d[i].name) - 1] == '/') + d[i].name[strlen(d[i].name) - 1] = 0; + } + + return d; +} + +/* Take a port's describe line and split it into fields */ +static PORT * +portify(char * line) +{ + PORT * p; + size_t i, n; + + /* Verify that line has the right number of fields */ + for (n = i = 0; line[i] != 0; i++) + if (line[i] == '|') + n++; + if (n != 12) + errx(1, "Port describe line is corrupt:\n%s\n", line); + + p = malloc(sizeof(PORT)); + if (p == NULL) + err(1, "malloc(PORT)"); + + p->pkgname = strdup2(strsep(&line, "|")); + p->portdir = strdup2(strsep(&line, "|")); + p->prefix = strdup2(strsep(&line, "|")); + p->comment = strdup2(strsep(&line, "|")); + p->pkgdescr = strdup2(strsep(&line, "|")); + p->maintainer = strdup2(strsep(&line, "|")); + p->categories = strdup2(strsep(&line, "|")); + p->edep = makelist(strsep(&line, "|"), &p->n_edep); + p->pdep = makelist(strsep(&line, "|"), &p->n_pdep); + p->fdep = makelist(strsep(&line, "|"), &p->n_fdep); + p->bdep = makelist(strsep(&line, "|"), &p->n_bdep); + p->rdep = makelist(strsep(&line, "|"), &p->n_rdep); + p->www = strdup2(strsep(&line, "|")); + + p->recursed = 0; + + /* + * line will now be equal to NULL -- we counted the field + * separators at the top of the function. + */ + + return p; +} + +/* Returns -1, 0, or 1 based on a comparison of the portdir strings */ +static int +portcompare(char * a, char * b) +{ + size_t i; + + /* Find first non-matching position */ + for (i = 0; ; i++) { + if (a[i] != b[i]) + break; + if (a[i] == 0) /* End of strings */ + return 0; + } + + /* One string is a prefix of the other */ + if (a[i] == 0) + return -1; + if (b[i] == 0) + return 1; + + /* One string has a category which is a prefix of the other */ + if (a[i] == '/') + return -1; + if (b[i] == '/') + return 1; + + /* The two strings are simply different */ + if (a[i] < b[i]) + return -1; + else + return 1; +} + +/* Heapify (PORT *) number pos in a pseudo-heap pp[0]..pp[size - 1] */ +static void +heapifyports(PORT **pp, size_t size, size_t pos) +{ + size_t i = pos; + PORT * tmp; + +top: + /* Find the largest value out of {pos, 2*pos+1, 2*pos+2} */ + if ((2 * pos + 1 < size) && + (portcompare(pp[i]->portdir, pp[2 * pos + 1]->portdir) < 0)) + i = 2 * pos + 1; + if ((2 * pos + 2 < size) && + (portcompare(pp[i]->portdir, pp[2 * pos + 2]->portdir) < 0)) + i = 2 * pos + 2; + + /* If necessary, swap elements and iterate down the tree. */ + if (i != pos) { + tmp = pp[pos]; + pp[pos] = pp[i]; + pp[i] = tmp; + pos = i; + goto top; + } +} + +/* Translate a port directory name into a (PORT *), and free the name */ +static PORT * +findport(PORT ** pp, size_t st, size_t en, char * name) +{ + size_t mid; + int r; + + if (st == en) + errx(1, "Unresolved dependency: %s", name); + + mid = (st + en) / 2; + r = portcompare(pp[mid]->portdir, name); + + if (r == 0) { + free(name); + return pp[mid]; + } else if (r < 0) + return findport(pp, mid + 1, en, name); + else + return findport(pp, st, mid, name); +} + +/* Translate all depends from names into PORT *s */ +static void +translateport(PORT ** pp, size_t pplen, PORT * p) +{ + size_t i; + + for (i = 0; i < p->n_edep; i++) + p->edep[i].p = findport(pp, 0, pplen, p->edep[i].name); + for (i = 0; i < p->n_pdep; i++) + p->pdep[i].p = findport(pp, 0, pplen, p->pdep[i].name); + for (i = 0; i < p->n_fdep; i++) + p->fdep[i].p = findport(pp, 0, pplen, p->fdep[i].name); + for (i = 0; i < p->n_bdep; i++) + p->bdep[i].p = findport(pp, 0, pplen, p->bdep[i].name); + for (i = 0; i < p->n_rdep; i++) + p->rdep[i].p = findport(pp, 0, pplen, p->rdep[i].name); +} + +/* Recurse on one specific depends list */ +static DEP * +recurse_one(DEP * d, size_t * nd) +{ + size_t i, j, k, n, N; + + N = n = *nd; + for (i = 0; i < n; i++) { + recurse(d[i].p); + for (j = 0; j < d[i].p->n_rdep; j++) { + for (k = 0; k < N; k++) { + if (d[i].p->rdep[j].p == d[k].p) + break; + } + if (k == N) { + N++; + if (N >= *nd) { + *nd += *nd; + d = realloc(d, *nd * sizeof(DEP)); + if (d == NULL) + err(1, "realloc(d)"); + } + d[k].p = d[i].p->rdep[j].p; + } + } + } + *nd = N; + + return d; +} + +/* Recurse on the depends lists */ +static void +recurse(PORT * p) +{ + if (p->recursed != 0) + return; + p->recursed = 1; + + p->edep = recurse_one(p->edep, &p->n_edep); + p->pdep = recurse_one(p->pdep, &p->n_pdep); + p->fdep = recurse_one(p->fdep, &p->n_fdep); + p->bdep = recurse_one(p->bdep, &p->n_bdep); + p->rdep = recurse_one(p->rdep, &p->n_rdep); +} + +/* Heapify an element in a package list */ +static void +heapifypkgs(DEP * d, size_t size, size_t pos) +{ + size_t i = pos; + PORT * tmp; + +top: + /* Find the largest value out of {pos, 2*pos+1, 2*pos+2} */ + if ((2 * pos + 1 < size) && + (strcmp(d[i].p->pkgname, d[2 * pos + 1].p->pkgname) < 0)) + i = 2 * pos + 1; + if ((2 * pos + 2 < size) && + (strcmp(d[i].p->pkgname, d[2 * pos + 2].p->pkgname) < 0)) + i = 2 * pos + 2; + + /* If necessary, swap elements and iterate down the tree. */ + if (i != pos) { + tmp = d[pos].p; + d[pos].p = d[i].p; + d[i].p = tmp; + pos = i; + goto top; + } +} + +/* Sort a list of dependent packages in alphabetical order */ +static void +sortpkgs(DEP * d, size_t nd) +{ + size_t i; + PORT * tmp; + + if (nd == 0) + return; + + for (i = nd; i > 0; i--) + heapifypkgs(d, nd, i - 1); /* Build a heap */ + for (i = nd - 1; i > 0; i--) { + tmp = d[0].p; /* Extract elements */ + d[0].p = d[i].p; + d[i].p = tmp; + heapifypkgs(d, i, 0); /* And re-heapify */ + } +} + +/* Output an index line for the given port. */ +static void +printport(PORT * p) +{ + size_t i; + + sortpkgs(p->edep, p->n_edep); + sortpkgs(p->pdep, p->n_pdep); + sortpkgs(p->fdep, p->n_fdep); + sortpkgs(p->bdep, p->n_bdep); + sortpkgs(p->rdep, p->n_rdep); + + printf("%s|%s|%s|%s|%s|%s|%s|", + p->pkgname, p->portdir, p->prefix, p->comment, p->pkgdescr, + p->maintainer, p->categories); + for (i = 0; i < p->n_bdep; i++) + printf("%s%s", i ? " " : "", p->bdep[i].p->pkgname); + printf("|"); + for (i = 0; i < p->n_rdep; i++) + printf("%s%s", i ? " " : "", p->rdep[i].p->pkgname); + printf("|"); + printf("%s|", p->www); + for (i = 0; i < p->n_edep; i++) + printf("%s%s", i ? " " : "", p->edep[i].p->pkgname); + printf("|"); + for (i = 0; i < p->n_pdep; i++) + printf("%s%s", i ? " " : "", p->pdep[i].p->pkgname); + printf("|"); + for (i = 0; i < p->n_fdep; i++) + printf("%s%s", i ? " " : "", p->fdep[i].p->pkgname); + printf("\n"); +} + +/* + * Algorithm: + * 1. Suck in all the data, splitting into fields. + * 2. Sort the ports according to port directory. + * 3. Using a binary search, translate each dependency from a + * port directory name into a pointer to a port. + * 4. Recursively follow dependencies, expanding the lists of + * pointers as needed (using realloc). + * 5. Iterate through the ports, printing them out (remembering + * to list the dependent ports in alphabetical order). + */ + +int +main(int argc, char *argv[]) +{ + FILE * f; + char * line; + size_t linelen; + PORT ** pp; /* Array of pointers to PORTs */ + PORT * tmp; + size_t pplen; /* Allocated size of array */ + size_t i; + + if (argc != 2) + usage(); + if ((f = fopen(argv[1], "r")) == NULL) + err(1, "fopen(%s)", argv[1]); + + pplen = 1024; + if ((pp = malloc(pplen * sizeof(PORT *))) == NULL) + err(1, "malloc(pp)"); + + /* + * 1. Suck in all the data, splitting into fields. + */ + for(i = 0; (line = fgetln(f, &linelen)) != NULL; i++) { + if (line[linelen - 1] != '\n') + errx(1, "Unterminated line encountered"); + line[linelen - 1] = 0; + + /* Enlarge array if needed */ + if (i >= pplen) { + pplen *= 2; + if ((pp = realloc(pp, pplen * sizeof(PORT *))) == NULL) + err(1, "realloc(pp)"); + } + + pp[i] = portify(line); + } + /* Reallocate to the correct size */ + pplen = i; + if ((pp = realloc(pp, pplen * sizeof(PORT *))) == NULL) + err(1, "realloc(pp)"); + + /* Make sure we actually reached the EOF */ + if (!feof(f)) + err(1, "fgetln(%s)", argv[1]); + /* Close the describes file */ + if (fclose(f) != 0) + err(1, "fclose(%s)", argv[1]); + + /* + * 2. Sort the ports according to port directory. + */ + for (i = pplen; i > 0; i--) + heapifyports(pp, pplen, i - 1); /* Build a heap */ + for (i = pplen - 1; i > 0; i--) { + tmp = pp[0]; /* Extract elements */ + pp[0] = pp[i]; + pp[i] = tmp; + heapifyports(pp, i, 0); /* And re-heapify */ + } + + /* + * 3. Using a binary search, translate each dependency from a + * port directory name into a pointer to a port. + */ + for (i = 0; i < pplen; i++) + translateport(pp, pplen, pp[i]); + + /* + * 4. Recursively follow dependencies, expanding the lists of + * pointers as needed (using realloc). + */ + for (i = 0; i < pplen; i++) + recurse(pp[i]); + + /* + * 5. Iterate through the ports, printing them out (remembering + * to list the dependent ports in alphabetical order). + */ + for (i = 0; i < pplen; i++) + printport(pp[i]); + + return 0; +} diff --git a/usr.sbin/portsnap/phttpget/Makefile b/usr.sbin/portsnap/phttpget/Makefile new file mode 100644 index 0000000..154ff15 --- /dev/null +++ b/usr.sbin/portsnap/phttpget/Makefile @@ -0,0 +1,9 @@ +# $FreeBSD$ + +PROG= phttpget +NO_MAN= +WARNS?= 6 + +BINDIR= ${LIBEXECDIR} + +.include <bsd.prog.mk> diff --git a/usr.sbin/portsnap/phttpget/phttpget.c b/usr.sbin/portsnap/phttpget/phttpget.c new file mode 100644 index 0000000..a75c6ea --- /dev/null +++ b/usr.sbin/portsnap/phttpget/phttpget.c @@ -0,0 +1,598 @@ +/*- + * Copyright 2005 Colin Percival + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/types.h> +#include <sys/time.h> +#include <sys/socket.h> + +#include <ctype.h> +#include <err.h> +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <netdb.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sysexits.h> +#include <unistd.h> + +static const char * env_HTTP_PROXY; +static const char * env_HTTP_USER_AGENT; +static const char * proxyport; + +static struct timeval timo = { 15, 0}; + +static void +usage(void) +{ + + fprintf(stderr, "usage: phttpget server [file ...]\n"); + exit(EX_USAGE); +} + +static void +readenv(void) +{ + char * p; + + env_HTTP_PROXY = getenv("HTTP_PROXY"); + if (env_HTTP_PROXY) { + if (strncmp(env_HTTP_PROXY, "http://", 7) == 0) + env_HTTP_PROXY += 7; + p = strchr(env_HTTP_PROXY, ':'); + if (p != NULL) { + *p = 0; + proxyport = p + 1; + } else + proxyport = "3128"; + } + + env_HTTP_USER_AGENT = getenv("HTTP_USER_AGENT"); + if (env_HTTP_USER_AGENT == NULL) + env_HTTP_USER_AGENT = "phttpget/0.1"; +} + +static int +makerequest(char ** buf, char * path, char * server, int connclose) +{ + int buflen; + + buflen = asprintf(buf, + "GET %s%s/%s HTTP/1.1\r\n" + "Host: %s\r\n" + "User-Agent: %s\r\n" + "%s" + "\r\n", + env_HTTP_PROXY ? "http://" : "", + env_HTTP_PROXY ? server : "", + path, server, env_HTTP_USER_AGENT, + connclose ? "Connection: Close\r\n" : ""); + if (buflen == -1) + err(1, "asprintf"); + return(buflen); +} + +static int +readln(int sd, char * resbuf, int * resbuflen, int * resbufpos) +{ + ssize_t len; + + while (strnstr(resbuf + *resbufpos, "\r\n", + *resbuflen - *resbufpos) == NULL) { + /* Move buffered data to the start of the buffer */ + if (*resbufpos != 0) { + memmove(resbuf, resbuf + *resbufpos, + *resbuflen - *resbufpos); + *resbuflen -= *resbufpos; + *resbufpos = 0; + } + + /* If the buffer is full, complain */ + if (*resbuflen == BUFSIZ) + return -1; + + /* Read more data into the buffer */ + len = recv(sd, resbuf + *resbuflen, BUFSIZ - *resbuflen, 0); + if ((len == -1) && (errno != EINTR)) + return -1; + + if (len != -1) + *resbuflen += len; + } + + return 0; +} + +static int +copybytes(int sd, int fd, off_t copylen, char * resbuf, int * resbuflen, + int * resbufpos) +{ + ssize_t len; + + while (copylen) { + /* Write data from resbuf to fd */ + len = *resbuflen - *resbufpos; + if (copylen < len) + len = copylen; + if (len > 0) { + if (fd != -1) + len = write(fd, resbuf + *resbufpos, len); + if (len == -1) + err(1, "write"); + *resbufpos += len; + copylen -= len; + continue; + } + + /* Read more data into buffer */ + len = recv(sd, resbuf, BUFSIZ, 0); + if (len == -1) { + if (errno == EINTR) + continue; + return -1; + } else if (len == 0) { + return -2; + } else { + *resbuflen = len; + *resbufpos = 0; + } + } + + return 0; +} + +int +main(int argc, char *argv[]) +{ + struct addrinfo hints; /* Hints to getaddrinfo */ + struct addrinfo *res; /* Pointer to server address being used */ + struct addrinfo *res0; /* Pointer to server addresses */ + char * resbuf = NULL; /* Response buffer */ + int resbufpos = 0; /* Response buffer position */ + int resbuflen = 0; /* Response buffer length */ + char * eolp; /* Pointer to "\r\n" within resbuf */ + char * hln0; /* Pointer to start of header line */ + char * hln; /* Pointer within header line */ + char * servername; /* Name of server */ + char * fname = NULL; /* Name of downloaded file */ + char * reqbuf = NULL; /* Request buffer */ + int reqbufpos = 0; /* Request buffer position */ + int reqbuflen = 0; /* Request buffer length */ + ssize_t len; /* Length sent or received */ + int nreq = 0; /* Number of next request to send */ + int nres = 0; /* Number of next reply to receive */ + int pipelined = 0; /* != 0 if connection in pipelined mode. */ + int sd = -1; /* Socket descriptor */ + int sdflags = 0; /* Flags on the socket sd */ + int fd = -1; /* Descriptor for download target file */ + int error; /* Error code */ + int statuscode; /* HTTP Status code */ + off_t contentlength; /* Value from Content-Length header */ + int chunked; /* != if transfer-encoding is chunked */ + off_t clen; /* Chunk length */ + int firstreq = 0; /* # of first request for this connection */ + + /* Check that the arguments are sensible */ + if (argc < 2) + usage(); + + /* Read important environment variables */ + readenv(); + + /* Get server name and adjust arg[cv] to point at file names */ + servername = argv[1]; + argv += 2; + argc -= 2; + + /* Allocate response buffer */ + resbuf = malloc(BUFSIZ); + if (resbuf == NULL) + err(1, "malloc"); + + /* Look up server */ + memset(&hints, 0, sizeof(hints)); + hints.ai_family = PF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + error = getaddrinfo(env_HTTP_PROXY ? env_HTTP_PROXY : servername, + env_HTTP_PROXY ? proxyport : "http", &hints, &res0); + if (error) + errx(1, "%s: %s", + env_HTTP_PROXY ? env_HTTP_PROXY : servername, + gai_strerror(error)); + if (res0 == NULL) + errx(1, "could not look up %s", servername); + res = res0; + + /* Do the fetching */ + while (nres < argc) { + /* Make sure we have a connected socket */ + for (; sd == -1; res = res->ai_next) { + /* No addresses left to try :-( */ + if (res == NULL) + errx(1, "Could not connect to %s", servername); + + /* Create a socket... */ + sd = socket(res->ai_family, res->ai_socktype, + res->ai_protocol); + if (sd == -1) + continue; + + /* ... set 15-second timeouts ... */ + setsockopt(sd, SOL_SOCKET, SO_SNDTIMEO, + (void *)&timo, (socklen_t)sizeof(timo)); + setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, + (void *)&timo, (socklen_t)sizeof(timo)); + + /* ... and connect to the server. */ + if(connect(sd, res->ai_addr, res->ai_addrlen)) { + close(sd); + sd = -1; + continue; + } + + firstreq = nres; + } + + /* + * If in pipelined HTTP mode, put socket into non-blocking + * mode, since we're probably going to want to try to send + * several HTTP requests. + */ + if (pipelined) { + sdflags = fcntl(sd, F_GETFL); + if (fcntl(sd, F_SETFL, sdflags | O_NONBLOCK) == -1) + err(1, "fcntl"); + } + + /* Construct requests and/or send them without blocking */ + while ((nreq < argc) && ((reqbuf == NULL) || pipelined)) { + /* If not in the middle of a request, make one */ + if (reqbuf == NULL) { + reqbuflen = makerequest(&reqbuf, argv[nreq], + servername, (nreq == argc - 1)); + reqbufpos = 0; + } + + /* If in pipelined mode, try to send the request */ + if (pipelined) { + while (reqbufpos < reqbuflen) { + len = send(sd, reqbuf + reqbufpos, + reqbuflen - reqbufpos, 0); + if (len == -1) + break; + reqbufpos += len; + } + if (reqbufpos < reqbuflen) { + if (errno != EAGAIN) + goto conndied; + break; + } else { + free(reqbuf); + reqbuf = NULL; + nreq++; + } + } + } + + /* Put connection back into blocking mode */ + if (pipelined) { + if (fcntl(sd, F_SETFL, sdflags) == -1) + err(1, "fcntl"); + } + + /* Do we need to blocking-send a request? */ + if (nres == nreq) { + while (reqbufpos < reqbuflen) { + len = send(sd, reqbuf + reqbufpos, + reqbuflen - reqbufpos, 0); + if (len == -1) + goto conndied; + reqbufpos += len; + } + free(reqbuf); + reqbuf = NULL; + nreq++; + } + + /* Scan through the response processing headers. */ + statuscode = 0; + contentlength = -1; + chunked = 0; + do { + /* Get a header line */ + error = readln(sd, resbuf, &resbuflen, &resbufpos); + if (error) + goto conndied; + hln0 = hln = resbuf + resbufpos; + eolp = strnstr(hln, "\r\n", resbuflen - resbufpos); + resbufpos = (eolp - resbuf) + 2; + *eolp = '\0'; + + /* Make sure it doesn't contain a NUL character */ + if (strchr(hln, '\0') != eolp) + goto conndied; + + if (statuscode == 0) { + /* The first line MUST be HTTP/1.x xxx ... */ + if ((strncmp(hln, "HTTP/1.", 7) != 0) || + ! isdigit(hln[7])) + goto conndied; + + /* + * If the minor version number isn't zero, + * then we can assume that pipelining our + * requests is OK -- as long as we don't + * see a "Connection: close" line later + * and we either have a Content-Length or + * Transfer-Encoding: chunked header to + * tell us the length. + */ + if (hln[7] != '0') + pipelined = 1; + + /* Skip over the minor version number */ + hln = strchr(hln + 7, ' '); + if (hln == NULL) + goto conndied; + else + hln++; + + /* Read the status code */ + while (isdigit(*hln)) { + statuscode = statuscode * 10 + + *hln - '0'; + hln++; + } + + if (statuscode < 100 || statuscode > 599) + goto conndied; + + /* Ignore the rest of the line */ + continue; + } + + /* Check for "Connection: close" header */ + if (strncmp(hln, "Connection:", 11) == 0) { + hln += 11; + if (strstr(hln, "close") != NULL) + pipelined = 0; + + /* Next header... */ + continue; + } + + /* Check for "Content-Length:" header */ + if (strncmp(hln, "Content-Length:", 15) == 0) { + hln += 15; + contentlength = 0; + + /* Find the start of the length */ + while (!isdigit(*hln) && (*hln != '\0')) + hln++; + + /* Compute the length */ + while (isdigit(*hln)) { + if (contentlength > INT_MAX / 10) { + /* Nasty people... */ + goto conndied; + } + contentlength = contentlength * 10 + + *hln - '0'; + hln++; + } + + /* Next header... */ + continue; + } + + /* Check for "Transfer-Encoding: chunked" header */ + if (strncmp(hln, "Transfer-Encoding:", 18) == 0) { + hln += 18; + if (strstr(hln, "chunked") != NULL) + chunked = 1; + + /* Next header... */ + continue; + } + + /* We blithely ignore any other header lines */ + + /* No more header lines */ + if (strlen(hln) == 0) { + /* + * If the status code was 1xx, then there will + * be a real header later. Servers may emit + * 1xx header blocks at will, but since we + * don't expect one, we should just ignore it. + */ + if (100 <= statuscode && statuscode <= 199) { + statuscode = 0; + continue; + } + + /* End of header; message body follows */ + break; + } + } while (1); + + /* No message body for 204 or 304 */ + if (statuscode == 204 || statuscode == 304) { + nres++; + continue; + } + + /* + * There should be a message body coming, but we only want + * to send it to a file if the status code is 200 + */ + if (statuscode == 200) { + /* Generate a file name for the download */ + fname = strrchr(argv[nres], '/'); + if (fname == NULL) + fname = argv[nres]; + else + fname++; + if (strlen(fname) == 0) + errx(1, "Cannot obtain file name from %s\n", + argv[nres]); + + fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644); + if (fd == -1) + errx(1, "open(%s)", fname); + }; + + /* Read the message and send data to fd if appropriate */ + if (chunked) { + /* Handle a chunked-encoded entity */ + + /* Read chunks */ + do { + error = readln(sd, resbuf, &resbuflen, + &resbufpos); + if (error) + goto conndied; + hln = resbuf + resbufpos; + eolp = strstr(hln, "\r\n"); + resbufpos = (eolp - resbuf) + 2; + + clen = 0; + while (isxdigit(*hln)) { + if (clen > INT_MAX / 16) { + /* Nasty people... */ + goto conndied; + } + if (isdigit(*hln)) + clen = clen * 16 + *hln - '0'; + else + clen = clen * 16 + 10 + + tolower(*hln) - 'a'; + hln++; + } + + error = copybytes(sd, fd, clen, resbuf, + &resbuflen, &resbufpos); + if (error) { + goto conndied; + } + } while (clen != 0); + + /* Read trailer and final CRLF */ + do { + error = readln(sd, resbuf, &resbuflen, + &resbufpos); + if (error) + goto conndied; + hln = resbuf + resbufpos; + eolp = strstr(hln, "\r\n"); + resbufpos = (eolp - resbuf) + 2; + } while (hln != eolp); + } else if (contentlength != -1) { + error = copybytes(sd, fd, contentlength, resbuf, + &resbuflen, &resbufpos); + if (error) + goto conndied; + } else { + /* + * Not chunked, and no content length header. + * Read everything until the server closes the + * socket. + */ + error = copybytes(sd, fd, INT_MAX, resbuf, + &resbuflen, &resbufpos); + if (error == -1) + goto conndied; + pipelined = 0; + } + + if (fd != -1) { + close(fd); + fd = -1; + } + + fprintf(stderr, "http://%s/%s: %d ", servername, argv[nres], + statuscode); + if (statuscode == 200) + fprintf(stderr, "OK\n"); + else if (statuscode < 300) + fprintf(stderr, "Successful (ignored)\n"); + else if (statuscode < 400) + fprintf(stderr, "Redirection (ignored)\n"); + else + fprintf(stderr, "Error (ignored)\n"); + + /* We've finished this file! */ + nres++; + + /* + * If necessary, clean up this connection so that we + * can start a new one. + */ + if (pipelined == 0) + goto cleanupconn; + continue; + +conndied: + /* + * Something went wrong -- our connection died, the server + * sent us garbage, etc. If this happened on the first + * request we sent over this connection, give up. Otherwise, + * close this connection, open a new one, and reissue the + * request. + */ + if (nres == firstreq) + errx(1, "Connection failure"); + +cleanupconn: + /* + * Clean up our connection and keep on going + */ + shutdown(sd, SHUT_RDWR); + close(sd); + sd = -1; + if (fd != -1) { + close(fd); + fd = -1; + } + if (reqbuf != NULL) { + free(reqbuf); + reqbuf = NULL; + } + nreq = nres; + res = res0; + pipelined = 0; + resbufpos = resbuflen = 0; + continue; + } + + free(resbuf); + freeaddrinfo(res0); + + return 0; +} diff --git a/usr.sbin/portsnap/portsnap/Makefile b/usr.sbin/portsnap/portsnap/Makefile new file mode 100644 index 0000000..53ae4d7 --- /dev/null +++ b/usr.sbin/portsnap/portsnap/Makefile @@ -0,0 +1,6 @@ +# $FreeBSD$ + +SCRIPTS=portsnap.sh +MAN8= portsnap.8 + +.include <bsd.prog.mk> diff --git a/usr.sbin/portsnap/portsnap/portsnap.8 b/usr.sbin/portsnap/portsnap/portsnap.8 new file mode 100644 index 0000000..9e1aa0f --- /dev/null +++ b/usr.sbin/portsnap/portsnap/portsnap.8 @@ -0,0 +1,208 @@ +\.\"- +.\" Copyright 2004-2005 Colin Percival +.\" All rights reserved +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted providing that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +.\" POSSIBILITY OF SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd January 30, 2005 +.Dt PORTSNAP 8 +.Os FreeBSD +.Sh NAME +.Nm portsnap +.Nd fetch and extract compressed snapshots of the ports tree +.Sh SYNOPSIS +.Nm +.Op Fl I +.Op Fl d Ar workdir +.Op Fl f Ar conffile +.Op Fl k Ar KEY +.Op Fl p Ar portsdir +.Op Fl s Ar server +.Cm command +.Op Ar path +.Sh DESCRIPTION +The +.Nm +tool is used to fetch and update compressed snapshots +of the FreeBSD ports tree, and extract and update an +uncompressed ports tree. +.Sh OPTIONS +The following options are supported: +.Bl -tag -width "-f conffile" +.It Fl d Ar workdir +Store working files (e.g. downloaded updates) in +.Ar workdir . +(default: +.Pa /var/db/portsnap , +or as given in the configuration file.) +.It Fl f Ar conffile +Read the configuration from from +.Ar conffile . +(default: +.Pa /etc/portsnap.conf ) +.It Fl I +For the +.Cm update +command, update INDEX files, but not the rest of the ports tree. +.It Fl k Ar KEY +Expect a public key with given SHA256 hash. +(default: read value from configuration file.) +.It Fl p Ar portsdir +When extracting or updating an uncompressed snapshot, +operate on the directory +.Ar portsdir . +(default: +.Pa /usr/ports/ , +or as given in the configuration file.) +.It Fl s Ar server +Fetch files from the specified server or server pool. +(default: portsnap.FreeBSD.org , or as given in the +configuration file.) +.It path +For +.Cm extract +command only, operate only on parts of the ports tree starting with +.Ar path . +(e.g. +.Nm +.cm extract +.Ar sysutils/port +would extract sysutils/portsman, sysutils/portsnap, +sysutils/portupgrade, etc.) +.El +.Sh COMMANDS +The +.Cm command +can be any one of the following: +.Pp +.Bl -tag -width "-f conffile" +.It fetch +Fetch a compressed snapshot of the ports tree, or update +the existing snapshot. +This command should only be used interactively; for +non-interactive use, you should use the +.Cm cron +command. +.It cron +Sleep a random amount of time, then operate as if the +.Cm fetch +command was specified. +As the name suggests, this command is designed for running +from +.Xr cron 8 ; +the random delay serves to minimize the probability that +a large number of machines will simultaneously attempt to +fetch updates. +.It extract +Extract a ports tree, replacing existing files and directories. +NOTE: This will remove anything occupying the location where +files or directories are being extracted; in particular, any +changes made locally to the ports tree (for example, adding new +patches) will be silently obliterated. +.Pp +Only run this command to initialize your portsnap-maintained +ports tree for the first time, if you wish to start over with +a clean, completely unmodified tree, or if you wish to extract +a specific part of the tree (using the +.Ar path +option). +.It update +Update a ports tree extracted using the +.Cm extract +command. +You must run this command to apply changes to your ports tree +after downloading updates via the +.Cm fetch +or +.Cm cron +commands. +Again, note that in the parts of the ports tree which are being +updated, any local changes or additions will be removed. +.El +.Sh TIPS +.Bl -bullet +.It +If your clock is set to local time, adding the line +.Pp +.Dl 0 3 * * * root /usr/sbin/portsnap cron +.Pp +to /etc/crontab is a good way to make sure you always have +an up-to-date snapshot of the ports tree available which +can quickly be extracted into +.Pa /usr/ports . +If your clock is set to UTC, please pick a random time other +than 3AM, to avoid overly imposing an uneven load on the +server(s) hosting the snapshots. +.It +Running +.Nm +.Cm update +from +.Xr cron 8 +is a bad idea -- if you're ever installing or updating a +port at the time the cron job runs, you'll probably end up +in a mess when +.Cm +updates or removes files which are being used by the port +build. +However, running +.Nm +.Fl I +.Cm update +is probably safe, and can be used together with +.Xr portversion 1 +to identify installed software which is out of date. +.It +If you wish to use +.Nm +to keep a large number of machines up to date, you may wish +to set up a caching HTTP proxy. Since +.Nm +uses +.Xr fetch 1 +to download updates, setting the +.Ev HTTP_PROXY +environment variable will direct it to fetch updates from +the given proxy. +This is much more efficient than +.Em mirroring +the files on the portsnap server, since the vast majority +of files are not needed by any particular client. +.El +.Sh FILES +.Bl -tag -width "/etc/portsnap.conf" +.It /etc/portsnap.conf +Default location of the portsnap configuration file. +.It /var/db/portsnap +Default location where compressed snapshots are stored. +.It /usr/ports +Default location where the ports tree is extracted. +.El +.Sh SEE ALSO +.Xr fetch 1 +.Xr fetch 3 +.Xr portsnap.conf 5 +.Xr sha256 8 +.Sh AUTHORS +.An Colin Percival Aq cperciva@FreeBSD.org diff --git a/usr.sbin/portsnap/portsnap/portsnap.sh b/usr.sbin/portsnap/portsnap/portsnap.sh new file mode 100644 index 0000000..d73965b --- /dev/null +++ b/usr.sbin/portsnap/portsnap/portsnap.sh @@ -0,0 +1,901 @@ +#!/bin/sh + +#- +# Copyright 2004-2005 Colin Percival +# All rights reserved +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted providing that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# $FreeBSD$ + +#### Usage function -- called from command-line handling code. + +# Usage instructions. Options not listed: +# --debug -- don't filter output from utilities +# --no-stats -- don't show progress statistics while fetching files +usage() { + cat <<EOF +usage: `basename $0` [options] command [path] + +Options: + -d workdir -- Store working files in workdir + (default: /var/db/portsnap/) + -f conffile -- Read configuration options from conffile + (default: /etc/portsnap.conf) + -I -- Update INDEX only. (update command only) + -k KEY -- Trust an RSA key with SHA256 hash of KEY + -p portsdir -- Location of uncompressed ports tree + (default: /usr/ports/) + -s server -- Server from which to fetch updates. + (default: portsnap.FreeBSD.org) + path -- Extract only parts of the tree starting with the given + string. (extract command only) +Commands: + fetch -- Fetch a compressed snapshot of the ports tree, + or update an existing snapshot. + cron -- Sleep rand(3600) seconds, and then fetch updates. + extract -- Extract snapshot of ports tree, replacing existing + files and directories. + update -- Update ports tree to match current snapshot, replacing + files and directories which have changed. +EOF + exit 0 +} + +#### Parameter handling functions. + +# Initialize parameters to null, just in case they're +# set in the environment. +init_params() { + KEYPRINT="" + EXTRACTPATH="" + WORKDIR="" + PORTSDIR="" + CONFFILE="" + COMMAND="" + QUIETREDIR="" + QUIETFLAG="" + STATSREDIR="" + NDEBUG="" + DDSTATS="" + INDEXONLY="" + SERVERNAME="" +} + +# Parse the command line +parse_cmdline() { + while [ $# -gt 0 ]; do + case "$1" in + -d) + if [ $# -eq 1 ]; then usage; fi + if [ ! -z "${WORKDIR}" ]; then usage; fi + shift; WORKDIR="$1" + ;; + --debug) + QUIETREDIR="/dev/stderr" + STATSREDIR="/dev/stderr" + QUIETFLAG=" " + NDEBUG=" " + DDSTATS=".." + ;; + -f) + if [ $# -eq 1 ]; then usage; fi + if [ ! -z "${CONFFILE}" ]; then usage; fi + shift; CONFFILE="$1" + ;; + -h | --help | help) + usage + ;; + -I) + INDEXONLY="YES" + ;; + -k) + if [ $# -eq 1 ]; then usage; fi + if [ ! -z "${KEYPRINT}" ]; then usage; fi + shift; KEYPRINT="$1" + ;; + --no-stats) + if [ -z "${STATSREDIR}" ]; then + STATSREDIR="/dev/null" + DDSTATS=".. " + fi + ;; + -p) + if [ $# -eq 1 ]; then usage; fi + if [ ! -z "${PORTSDIR}" ]; then usage; fi + shift; PORTSDIR="$1" + ;; + -s) + if [ $# -eq 1 ]; then usage; fi + if [ ! -z "${SERVERNAME}" ]; then usage; fi + shift; SERVERNAME="$1" + ;; + cron | extract | fetch | update) + if [ ! -z "${COMMAND}" ]; then usage; fi + COMMAND="$1" + ;; + *) + if [ $# -gt 1 ]; then usage; fi + if [ "${COMMAND}" = "extract" ]; then usage; fi + EXTRACTPATH="$1" + ;; + esac + shift + done + + if [ -z "${COMMAND}" ]; then + usage + fi +} + +# If CONFFILE was specified at the command-line, make +# sure that it exists and is readable. +sanity_conffile() { + if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then + echo -n "File does not exist " + echo -n "or is not readable: " + echo ${CONFFILE} + exit 1 + fi +} + +# If a configuration file hasn't been specified, use +# the default value (/etc/portsnap.conf) +default_conffile() { + if [ -z "${CONFFILE}" ]; then + CONFFILE="/etc/portsnap.conf" + fi +} + +# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration +# file if they haven't already been set. If the configuration +# file doesn't exist, do nothing. +parse_conffile() { + if [ -r "${CONFFILE}" ]; then + for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do + eval _=\$${X} + if [ -z "${_}" ]; then + eval ${X}=`grep "^${X}=" "${CONFFILE}" | + cut -f 2- -d '=' | tail -1` + fi + done + fi +} + +# If parameters have not been set, use default values +default_params() { + _QUIETREDIR="/dev/null" + _QUIETFLAG="-q" + _STATSREDIR="/dev/stdout" + _WORKDIR="/var/db/portsnap" + _PORTSDIR="/usr/ports" + _NDEBUG="-n" + for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR NDEBUG; do + eval _=\$${X} + eval __=\$_${X} + if [ -z "${_}" ]; then + eval ${X}=${__} + fi + done +} + +# Perform sanity checks and set some final parameters +# in preparation for fetching files. Also chdir into +# the working directory. +fetch_check_params() { + export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" + + _SERVERNAME_z=\ +"SERVERNAME must be given via command line or configuration file." + _KEYPRINT_z="Key must be given via -k option or configuration file." + _KEYPRINT_bad="Invalid key fingerprint: " + _WORKDIR_bad="Directory does not exist or is not writable: " + + if [ -z "${SERVERNAME}" ]; then + echo -n "`basename $0`: " + echo "${_SERVERNAME_z}" + exit 1 + fi + if [ -z "${KEYPRINT}" ]; then + echo -n "`basename $0`: " + echo "${_KEYPRINT_z}" + exit 1 + fi + if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then + echo -n "`basename $0`: " + echo -n "${_KEYPRINT_bad}" + echo ${KEYPRINT} + exit 1 + fi + if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then + echo -n "`basename $0`: " + echo -n "${_WORKDIR_bad}" + echo ${WORKDIR} + exit 1 + fi + cd ${WORKDIR} || exit 1 + + BSPATCH=/usr/bin/bspatch + SHA256=/sbin/sha256 + PHTTPGET=/usr/libexec/phttpget +} + +# Perform sanity checks and set some final parameters +# in preparation for extracting or updating ${PORTSDIR} +extract_check_params() { + _WORKDIR_bad="Directory does not exist: " + _PORTSDIR_bad="Directory does not exist or is not writable: " + + if ! [ -d "${WORKDIR}" ]; then + echo -n "`basename $0`: " + echo -n "${_WORKDIR_bad}" + echo ${WORKDIR} + exit 1 + fi + if ! [ -d "${PORTSDIR}" -a -w "${PORTSDIR}" ]; then + echo -n "`basename $0`: " + echo -n "${_PORTSDIR_bad}" + echo ${PORTSDIR} + exit 1 + fi + + if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ + -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then + echo "No snapshot available. Try running" + echo "# `basename $0` fetch" + exit 1 + fi + + MKINDEX=/usr/libexec/make_index +} + +# Perform sanity checks and set some final parameters +# in preparation for updating ${PORTSDIR} +update_check_params() { + extract_check_params + + if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then + echo "${PORTSDIR} was not created by portsnap." + echo -n "You must run '`basename $0` extract' before " + echo "running '`basename $0` update'." + exit 1 + fi + +} + +#### Core functionality -- the actual work gets done here + +# Use an SRV query to pick a server. If the SRV query doesn't provide +# a useful answer, use the server name specified by the user. +# Put another way... look up _http._tcp.${SERVERNAME} and pick a server +# from that; or if no servers are returned, use ${SERVERNAME}. +# This allows a user to specify "portsnap.freebsd.org" (in which case +# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" +# (in which case portsnap will use that particular server, since there +# won't be an SRV entry for that name). +# +# We don't implement the recommendations from RFC 2782 completely, since +# we are only looking to pick a single server -- the recommendations are +# targetted at applications which obtain a list of servers and then try +# each in turn, but we are instead just going to pick one server and let +# the user re-run portsnap if a broken server was selected. +# +# We also ignore the Port field, since we are always going to use port 80. +fetch_pick_server() { + echo -n "Looking up ${SERVERNAME} mirrors..." + +# Issue the SRV query and pull out the Priority, Weight, and Target fields. + host -t srv "_http._tcp.${SERVERNAME}" | + grep -E "^_http._tcp.${SERVERNAME} has SRV record" | + cut -f 5,6,8 -d ' ' > serverlist + +# If no records, give up -- we'll just use the server name we were given. + if [ `wc -l < serverlist` -eq 0 ]; then + echo " none found." + return + fi + +# Find the highest priority level (lowest numeric value). + SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` + +# Add up the weights of the response lines at that priority level. + SRV_WSUM=0; + while read X; do + case "$X" in + ${SRV_PRIORITY}\ *) + SRV_W=`echo $X | cut -f 2 -d ' '` + SRV_WSUM=$(($SRV_WSUM + $SRV_W)) + ;; + esac + done < serverlist + +# If all the weights are 0, pretend that they are all 1 instead. + if [ ${SRV_WSUM} -eq 0 ]; then + SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` + SRV_W_ADD=1 + else + SRV_W_ADD=0 + fi + +# Pick a random value between 1 and the sum of the weights + SRV_RND=`jot -r 1 1 ${SRV_WSUM}` + +# Read through the list of mirrors and set SERVERNAME + while read X; do + case "$X" in + ${SRV_PRIORITY}\ *) + SRV_W=`echo $X | cut -f 2 -d ' '` + SRV_W=$(($SRV_W + $SRV_W_ADD)) + if [ $SRV_RND -le $SRV_W ]; then + SERVERNAME=`echo $X | cut -f 3 -d ' '` + break + else + SRV_RND=$(($SRV_RND - $SRV_W)) + fi + ;; + esac + done < serverlist + + echo " using ${SERVERNAME}" +} + +# Check that we have a public key with an appropriate hash, or +# fetch the key if it doesn't exist. +fetch_key() { + if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then + return + fi + + echo -n "Fetching public key... " + rm -f pub.ssl + fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ + 2>${QUIETREDIR} || true + if ! [ -r pub.ssl ]; then + echo "failed." + return 1 + fi + if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then + echo "key has incorrect hash." + rm -f pub.ssl + return 1 + fi + echo "done." +} + +# Fetch a snapshot tag +fetch_tag() { + rm -f snapshot.ssl tag.new + + echo ${NDEBUG} "Fetching snapshot tag... " + fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl + 2>${QUIETREDIR} || true + if ! [ -r $1.ssl ]; then + echo "failed." + return 1 + fi + + openssl rsautl -pubin -inkey pub.ssl -verify \ + < $1.ssl > tag.new 2>${QUIETREDIR} || true + rm $1.ssl + + if ! [ `wc -l < tag.new` = 1 ] || + ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then + echo "invalid snapshot tag." + return 1 + fi + + echo "done." + + SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` + SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` +} + +# Sanity-check the date on a snapshot tag +fetch_snapshot_tagsanity() { + if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then + echo "Snapshot appears to be more than a year old!" + echo "(Is the system clock correct?)" + echo "Cowarly refusing to proceed any further." + return 1 + fi + if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then + echo -n "Snapshot appears to have been created more than " + echo "one day into the future!" + echo "(Is the system clock correct?)" + echo "Cowardly refusing to proceed any further." + return 1 + fi +} + +# Sanity-check the date on a snapshot update tag +fetch_update_tagsanity() { + fetch_snapshot_tagsanity || return 1 + + if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then + echo -n "Latest snapshot on server is " + echo "older than what we already have!" + echo -n "Cowardly refusing to downgrade from " + date -r ${OLDSNAPSHOTDATE} + echo -n "to `date -r ${SNAPSHOTDATE}`." + return 1 + fi +} + +# Compare old and new tags; return 1 if update is unnecessary +fetch_update_neededp() { + if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then + echo -n "Latest snapshot on server matches " + echo "what we already have." + echo "No updates needed." + rm tag.new + return 1 + fi + if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then + echo -n "Ports tree hasn't changed since " + echo "last snapshot." + echo "No updates needed." + rm tag.new + return 1 + fi + + return 0 +} + +# Fetch snapshot metadata file +fetch_metadata() { + rm -f ${SNAPSHOTHASH} tINDEX.new + + echo ${NDEBUG} "Fetching snapshot metadata... " + fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} + 2>${QUIETREDIR} || return + if [ `${SHA256} -q ${SNAPSHOTHASH}` != ${SNAPSHOTHASH} ]; then + echo "snapshot metadata corrupt." + return 1 + fi + mv ${SNAPSHOTHASH} tINDEX.new + echo "done." +} + +# Warn user about bogus metadata +fetch_metadata_freakout() { + echo + echo "Portsnap metadata is correctly signed, but contains" + echo "at least one line which appears bogus." + echo "Cowardly refusing to proceed any further." +} + +# Sanity-check a snapshot metadata file +fetch_metadata_sanity() { + if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then + fetch_metadata_freakout + return 1 + fi + if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then + echo + echo "Portsnap metadata appears bogus." + echo "Cowardly refusing to proceed any further." + return 1 + fi +} + +# Take a list of ${oldhash}|${newhash} and output a list of needed patches +fetch_make_patchlist() { + grep -vE "^([0-9a-f]{64})\|\1$" | + while read LINE; do + X=`echo ${LINE} | cut -f 1 -d '|'` + Y=`echo ${LINE} | cut -f 2 -d '|'` + if [ -f "files/${Y}.gz" ]; then continue; fi + if [ ! -f "files/${X}.gz" ]; then continue; fi + echo "${LINE}" + done +} + +# Print user-friendly progress statistics +fetch_progress() { + LNC=0 + while read x; do + LNC=$(($LNC + 1)) + if [ $(($LNC % 10)) = 0 ]; then + echo -n $FN2 + elif [ $(($LNC % 2)) = 0 ]; then + echo -n . + fi + done + echo -n " " +} + +# Sanity-check an index file +fetch_index_sanity() { + if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || + fgrep -q "./" INDEX.new; then + fetch_metadata_freakout + return 1 + fi +} + +# Verify a list of files +fetch_snapshot_verify() { + while read F; do + if [ `gunzip -c snap/${F} | ${SHA256} -q` != ${F} ]; then + echo "snapshot corrupt." + return 1 + fi + done + return 0 +} + +# Fetch a snapshot tarball, extract, and verify. +fetch_snapshot() { + fetch_tag snapshot || return 1 + fetch_snapshot_tagsanity || return 1 + fetch_metadata || return 1 + fetch_metadata_sanity || return 1 + + rm -f ${SNAPSHOTHASH}.tgz + rm -rf snap/ + +# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will +# probably take a while, so the progrees reports that fetch(1) generates +# will be useful for keeping the users' attention from drifting. + echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" + fetch http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 + + echo -n "Extracting snapshot... " + tar -xzf ${SNAPSHOTHASH}.tgz snap/ || return 1 + rm ${SNAPSHOTHASH}.tgz + echo "done." + + echo -n "Verifying snapshot integrity... " +# Verify the metadata files + cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 +# Extract the index + rm -f INDEX.new + gunzip -c snap/`look INDEX tINDEX.new | + cut -f 2 -d '|'`.gz > INDEX.new + fetch_index_sanity || return 1 +# Verify the snapshot contents + cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 + echo "done." + +# Move files into their proper locations + rm -f tag INDEX tINDEX + rm -rf files + mv tag.new tag + mv tINDEX.new tINDEX + mv INDEX.new INDEX + mv snap/ files/ + + return 0 +} + +# Update a compressed snapshot +fetch_update() { + rm -f patchlist diff OLD NEW filelist INDEX.new + + OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` + OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` + + fetch_tag latest || return 1 + fetch_update_tagsanity || return 1 + fetch_update_neededp || return 0 + fetch_metadata || return 1 + fetch_metadata_sanity || return 1 + + echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " + echo "to `date -r ${SNAPSHOTDATE}`." + +# Generate a list of wanted metadata patches + join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | + fetch_make_patchlist > patchlist + +# Attempt to fetch metadata patches + echo -n "Fetching `wc -l < patchlist | tr -d ' '` " + echo ${NDEBUG} "metadata patches.${DDSTATS}" + tr '|' '-' < patchlist | + lam -s "tp/" - -s ".gz" | + xargs ${PHTTPGET} ${SERVERNAME} \ + 2>${STATSREDIR} | fetch_progress + echo "done." + +# Attempt to apply metadata patches + echo -n "Applying metadata patches... " + while read LINE; do + X=`echo ${LINE} | cut -f 1 -d '|'` + Y=`echo ${LINE} | cut -f 2 -d '|'` + if [ ! -f "${X}-${Y}.gz" ]; then continue; fi + gunzip -c < ${X}-${Y}.gz > diff + gunzip -c < files/${X}.gz > OLD + cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp + grep '^\+' diff | cut -c 2- | + sort -k 1,1 -t '|' -m - ptmp > NEW + if [ `${SHA256} -q NEW` = ${Y} ]; then + mv NEW files/${Y} + gzip -n files/${Y} + fi + rm -f diff OLD NEW ${X}-${Y}.gz ptmp + done < patchlist 2>${QUIETREDIR} + echo "done." + +# Update metadata without patches + join -t '|' -v 2 tINDEX tINDEX.new | + cut -f 2 -d '|' /dev/stdin patchlist | + while read Y; do + if [ ! -f "files/${Y}.gz" ]; then + echo ${Y}; + fi + done > filelist + echo -n "Fetching `wc -l < filelist | tr -d ' '` " + echo ${NDEBUG} "metadata files... " + lam -s "f/" - -s ".gz" < filelist | + xargs ${PHTTPGET} ${SERVERNAME} \ + 2>${QUIETREDIR} + + while read Y; do + if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then + mv ${Y}.gz files/${Y}.gz + else + echo "metadata is corrupt." + return 1 + fi + done < filelist + echo "done." + +# Extract the index + gunzip -c files/`look INDEX tINDEX.new | + cut -f 2 -d '|'`.gz > INDEX.new + fetch_index_sanity || return 1 + +# Generate a list of wanted ports patches + join -t '|' -o 1.2,2.2 INDEX INDEX.new | + fetch_make_patchlist > patchlist + +# Attempt to fetch ports patches + echo -n "Fetching `wc -l < patchlist | tr -d ' '` " + echo ${NDEBUG} "patches.${DDSTATS}" + tr '|' '-' < patchlist | lam -s "bp/" - | + xargs ${PHTTPGET} ${SERVERNAME} \ + 2>${STATSREDIR} | fetch_progress + echo "done." + +# Attempt to apply ports patches + echo -n "Applying patches... " + while read LINE; do + X=`echo ${LINE} | cut -f 1 -d '|'` + Y=`echo ${LINE} | cut -f 2 -d '|'` + if [ ! -f "${X}-${Y}" ]; then continue; fi + gunzip -c < files/${X}.gz > OLD + ${BSPATCH} OLD NEW ${X}-${Y} + if [ `${SHA256} -q NEW` = ${Y} ]; then + mv NEW files/${Y} + gzip -n files/${Y} + fi + rm -f diff OLD NEW ${X}-${Y} + done < patchlist 2>${QUIETREDIR} + echo "done." + +# Update ports without patches + join -t '|' -v 2 INDEX INDEX.new | + cut -f 2 -d '|' /dev/stdin patchlist | + while read Y; do + if [ ! -f "files/${Y}.gz" ]; then + echo ${Y}; + fi + done > filelist + echo -n "Fetching `wc -l < filelist | tr -d ' '` " + echo ${NDEBUG} "new ports or files... " + lam -s "f/" - -s ".gz" < filelist | + xargs ${PHTTPGET} ${SERVERNAME} \ + 2>${QUIETREDIR} + + while read Y; do + if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then + mv ${Y}.gz files/${Y}.gz + else + echo "snapshot is corrupt." + return 1 + fi + done < filelist + echo "done." + +# Remove files which are no longer needed + cut -f 2 -d '|' tINDEX INDEX | sort > oldfiles + cut -f 2 -d '|' tINDEX.new INDEX.new | sort | comm -13 - oldfiles | + lam -s "files/" - -s ".gz" | xargs rm -f + rm patchlist filelist oldfiles + +# We're done! + mv INDEX.new INDEX + mv tINDEX.new tINDEX + mv tag.new tag + + return 0 +} + +# Do the actual work involved in "fetch" / "cron". +fetch_run() { + fetch_pick_server + + fetch_key || return 1 + + if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then + fetch_snapshot || return 1 + fi + fetch_update || return 1 +} + +# Build a ports INDEX file +extract_make_index() { + gunzip -c "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | + cut -f 2 -d '|'`.gz" | ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 +} + +# Create INDEX, INDEX-5, INDEX-6 +extract_indices() { + echo -n "Building new INDEX files... " + extract_make_index DESCRIBE.4 INDEX || return 1 + extract_make_index DESCRIBE.5 INDEX-5 || return 1 + extract_make_index DESCRIBE.6 INDEX-6 || return 1 + echo "done." +} + +# Create .portsnap.INDEX +extract_metadata() { + sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX +} + +# Do the actual work involved in "extract" +extract_run() { + grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX | while read LINE; do + FILE=`echo ${LINE} | cut -f 1 -d '|'` + HASH=`echo ${LINE} | cut -f 2 -d '|'` + echo ${PORTSDIR}/${FILE} + if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then + echo "files/${HASH}.gz not found -- snapshot corrupt." + return 1 + fi + case ${FILE} in + */) + rm -rf ${PORTSDIR}/${FILE} + mkdir -p ${PORTSDIR}/${FILE} + tar -xzf ${WORKDIR}/files/${HASH}.gz \ + -C ${PORTSDIR}/${FILE} + ;; + *) + rm -f ${PORTSDIR}/${FILE} + tar -xzf ${WORKDIR}/files/${HASH}.gz \ + -C ${PORTSDIR} ${FILE} + ;; + esac + done + if [ ! -z "${EXTRACTPATH}" ]; then + return 0; + fi + + extract_metadata + extract_indices +} + +# Do the actual work involved in "update" +update_run() { + if ! [ -z "${INDEXONLY}" ]; then + extract_indices >/dev/null || return 1 + return 0 + fi + + echo -n "Removing old files and directories... " + sort ${WORKDIR}/INDEX | comm -23 ${PORTSDIR}/.portsnap.INDEX - | + cut -f 1 -d '|' | lam -s "${PORTSDIR}/" - | xargs rm -rf + echo "done." + +# Install new files + echo "Extracting new files:" + sort ${WORKDIR}/INDEX | comm -13 ${PORTSDIR}/.portsnap.INDEX - | + while read LINE; do + FILE=`echo ${LINE} | cut -f 1 -d '|'` + HASH=`echo ${LINE} | cut -f 2 -d '|'` + echo ${PORTSDIR}/${FILE} + if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then + echo "files/${HASH}.gz not found -- snapshot corrupt." + return 1 + fi + case ${FILE} in + */) + mkdir -p ${PORTSDIR}/${FILE} + tar -xzf ${WORKDIR}/files/${HASH}.gz \ + -C ${PORTSDIR}/${FILE} + ;; + *) + tar -xzf ${WORKDIR}/files/${HASH}.gz \ + -C ${PORTSDIR} ${FILE} + ;; + esac + done + + extract_metadata + extract_indices +} + +#### Main functions -- call parameter-handling and core functions + +# Using the command line, configuration file, and defaults, +# set all the parameters which are needed later. +get_params() { + init_params + parse_cmdline $@ + sanity_conffile + default_conffile + parse_conffile + default_params +} + +# Fetch command. Make sure that we're being called +# interactively, then run fetch_check_params and fetch_run +cmd_fetch() { + if [ ! -t 0 ]; then + echo -n "`basename $0` fetch should not " + echo "be run non-interactively." + echo "Run `basename $0` cron instead." + exit 1 + fi + fetch_check_params + fetch_run || exit 1 +} + +# Cron command. Make sure the parameters are sensible; wait +# rand(3600) seconds; then fetch updates. While fetching updates, +# send output to a temporary file; only print that file if the +# fetching failed. +cmd_cron() { + fetch_check_params + sleep `jot -r 1 0 3600` + + TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 + if ! fetch_run >> ${TMPFILE}; then + cat ${TMPFILE} + rm ${TMPFILE} + exit 1 + fi + + rm ${TMPFILE} +} + +# Extract command. Make sure the parameters are sensible, +# then extract the ports tree (or part thereof). +cmd_extract() { + extract_check_params + extract_run || exit 1 +} + +# Update command. Make sure the parameters are sensible, +# then update the ports tree. +cmd_update() { + update_check_params + update_run || exit 1 +} + +#### Entry point + +# Make sure we find utilities from the base system +export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} + +get_params $@ +cmd_${COMMAND} |