Huge cleanup of random(4) code.

* GENERAL
- Update copyright.
- Make kernel options for RANDOM_YARROW and RANDOM_DUMMY. Set
  neither to ON, which means we want Fortuna
- If there is no 'device random' in the kernel, there will be NO
  random(4) device in the kernel, and the KERN_ARND sysctl will
  return nothing. With RANDOM_DUMMY there will be a random(4) that
  always blocks.
- Repair kern.arandom (KERN_ARND sysctl). The old version went
  through arc4random(9) and was a bit weird.
- Adjust arc4random stirring a bit - the existing code looks a little
  suspect.
- Fix the nasty pre- and post-read overloading by providing explictit
  functions to do these tasks.
- Redo read_random(9) so as to duplicate random(4)'s read internals.
  This makes it a first-class citizen rather than a hack.
- Move stuff out of locked regions when it does not need to be
  there.
- Trim RANDOM_DEBUG printfs. Some are excess to requirement, some
  behind boot verbose.
- Use SYSINIT to sequence the startup.
- Fix init/deinit sysctl stuff.
- Make relevant sysctls also tunables.
- Add different harvesting "styles" to allow for different requirements
  (direct, queue, fast).
- Add harvesting of FFS atime events. This needs to be checked for
  weighing down the FS code.
- Add harvesting of slab allocator events. This needs to be checked for
  weighing down the allocator code.
- Fix the random(9) manpage.
- Loadable modules are not present for now. These will be re-engineered
  when the dust settles.
- Use macros for locks.
- Fix comments.

* src/share/man/...
- Update the man pages.

* src/etc/...
- The startup/shutdown work is done in D2924.

* src/UPDATING
- Add UPDATING announcement.

* src/sys/dev/random/build.sh
- Add copyright.
- Add libz for unit tests.

* src/sys/dev/random/dummy.c
- Remove; no longer needed. Functionality incorporated into randomdev.*.

* live_entropy_sources.c live_entropy_sources.h
- Remove; content moved.
- move content to randomdev.[ch] and optimise.

* src/sys/dev/random/random_adaptors.c src/sys/dev/random/random_adaptors.h
- Remove; plugability is no longer used. Compile-time algorithm
  selection is the way to go.

* src/sys/dev/random/random_harvestq.c src/sys/dev/random/random_harvestq.h
- Add early (re)boot-time randomness caching.

* src/sys/dev/random/randomdev_soft.c src/sys/dev/random/randomdev_soft.h
- Remove; no longer needed.

* src/sys/dev/random/uint128.h
- Provide a fake uint128_t; if a real one ever arrived, we can use
  that instead. All that is needed here is N=0, N++, N==0, and some
  localised trickery is used to manufacture a 128-bit 0ULLL.

* src/sys/dev/random/unit_test.c src/sys/dev/random/unit_test.h
- Improve unit tests; previously the testing human needed clairvoyance;
  now the test will do a basic check of compressibility. Clairvoyant
  talent is still a good idea.
- This is still a long way off a proper unit test.

* src/sys/dev/random/fortuna.c src/sys/dev/random/fortuna.h
- Improve messy union to just uint128_t.
- Remove unneeded 'static struct fortuna_start_cache'.
- Tighten up up arithmetic.
- Provide a method to allow eternal junk to be introduced; harden
  it against blatant by compress/hashing.
- Assert that locks are held correctly.
- Fix the nasty pre- and post-read overloading by providing explictit
  functions to do these tasks.
- Turn into self-sufficient module (no longer requires randomdev_soft.[ch])

* src/sys/dev/random/yarrow.c src/sys/dev/random/yarrow.h
- Improve messy union to just uint128_t.
- Remove unneeded 'staic struct start_cache'.
- Tighten up up arithmetic.
- Provide a method to allow eternal junk to be introduced; harden
  it against blatant by compress/hashing.
- Assert that locks are held correctly.
- Fix the nasty pre- and post-read overloading by providing explictit
  functions to do these tasks.
- Turn into self-sufficient module (no longer requires randomdev_soft.[ch])
- Fix some magic numbers elsewhere used as FAST and SLOW.

Differential Revision: https://reviews.freebsd.org/D2025
Reviewed by: vsevolod,delphij,rwatson,trasz,jmg
Approved by: so (delphij)

1 files changed, 100 insertions, 19 deletions

diff --git a/sys/dev/random/unit_test.c b/sys/dev/random/unit_test.c
index 32b3363..80b32cf 100644
--- a/sys/dev/random/unit_test.c
+++ b/sys/dev/random/unit_test.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2000-2013 Mark R V Murray
+ * Copyright (c) 2000-2015 Mark R V Murray
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,6 +37,7 @@ cc -g -O0 -pthread -DRANDOM_<alg> -DRANDOM_DEBUG -I../.. -lstdthreads -Wall \
 	../../crypto/rijndael/rijndael-api-fst.c \
 	../../crypto/rijndael/rijndael-alg-fst.c \
 	../../crypto/sha2/sha2.c \
+        -lz \
 	-o unit_test
 ./unit_test
 
@@ -49,6 +50,7 @@ Where <alg> is YARROW or FORTUNA.
 #include <stdlib.h>
 #include <threads.h>
 #include <unistd.h>
+#include <zlib.h>
 
 #include "unit_test.h"
 
@@ -59,10 +61,77 @@ Where <alg> is YARROW or FORTUNA.
 #include "dev/random/fortuna.h"
 #endif
 
-#define NUM_THREADS	  3
+#define	NUM_THREADS	  3
+#define	DEBUG
 
 static volatile int stopseeding = 0;
 
+static __inline void
+check_err(int err, const char *func)
+{
+	if (err != Z_OK) {
+		fprintf(stderr, "Compress error in %s: %d\n", func, err);
+		exit(0);
+	}
+}
+
+void *
+myalloc(void *q, unsigned n, unsigned m)
+{
+	q = Z_NULL;
+	return (calloc(n, m));
+}
+
+void myfree(void *q, void *p)
+{
+	q = Z_NULL;
+	free(p);
+}
+
+size_t
+block_deflate(uint8_t *uncompr, uint8_t *compr, const size_t len)
+{
+	z_stream c_stream;
+	int err;
+
+	if (len == 0)
+		return (0);
+
+	c_stream.zalloc = myalloc;
+	c_stream.zfree = myfree;
+	c_stream.opaque = NULL;
+
+	err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION);
+	check_err(err, "deflateInit");
+
+	c_stream.next_in  = uncompr;
+	c_stream.next_out = compr;
+	c_stream.avail_in = len;
+	c_stream.avail_out = len*2u +512u;
+
+	while (c_stream.total_in != len && c_stream.total_out < (len*2u + 512u)) {
+		err = deflate(&c_stream, Z_NO_FLUSH);
+#ifdef DEBUG
+		printf("deflate: len = %zd  total_in = %lu  total_out = %lu\n", len, c_stream.total_in, c_stream.total_out);
+#endif
+		check_err(err, "deflate(..., Z_NO_FLUSH)");
+	}
+
+	for (;;) {
+		err = deflate(&c_stream, Z_FINISH);
+#ifdef DEBUG
+		printf("deflate: len = %zd  total_in = %lu  total_out = %lu\n", len, c_stream.total_in, c_stream.total_out);
+#endif
+		if (err == Z_STREAM_END) break;
+		check_err(err, "deflate(..., Z_STREAM_END)");
+	}
+
+	err = deflateEnd(&c_stream);
+	check_err(err, "deflateEnd");
+
+	return ((size_t)c_stream.total_out);
+}
+
 void
 random_adaptor_unblock(void)
 {
@@ -128,6 +197,7 @@ WriteCSPRNG(void *threadid)
 		if (i % 1000 == 0)
 			printf("Thread write 1 - %d\n", i);
 		if (buf != NULL) {
+			printf("Thread 1 writing.\n");
 #ifdef RANDOM_YARROW
 			random_yarrow_write(buf, i);
 #endif
@@ -149,9 +219,12 @@ WriteCSPRNG(void *threadid)
 static int
 ReadCSPRNG(void *threadid)
 {
-	size_t tid;
-	uint8_t *buf;
+	size_t tid, zsize;
+	uint8_t *buf, *zbuf;
 	int i;
+#ifdef DEBUG
+	int j;
+#endif
 
 	tid = (size_t)threadid;
 	printf("Thread #%zd starts\n", tid);
@@ -164,42 +237,50 @@ ReadCSPRNG(void *threadid)
 #endif
 	{
 #ifdef RANDOM_YARROW
-		random_yarrow_read(NULL, 0);
-		random_yarrow_read(NULL, 1);
+		random_yarrow_pre_read();
+		random_yarrow_post_read();
 #endif
 #ifdef RANDOM_FORTUNA
-		random_fortuna_read(NULL, 0);
-		random_fortuna_read(NULL, 1);
+		random_fortuna_pre_read();
+		random_fortuna_post_read();
 #endif
 		usleep(100);
 	}
 
 	for (i = 0; i < 100000; i++) {
 		buf = malloc(i);
+		zbuf = malloc(2*i + 1024);
 		if (i % 1000 == 0)
-			printf("Thread read %zd - %d %p\n", tid, i, buf);
-		if (buf != NULL) {
+			printf("Thread read %zd - %d\n", tid, i);
+		if (buf != NULL && zbuf != NULL) {
 #ifdef RANDOM_YARROW
-			random_yarrow_read(NULL, 0);
+			random_yarrow_pre_read();
 			random_yarrow_read(buf, i);
-			random_yarrow_read(NULL, 1);
+			random_yarrow_post_read();
 #endif
 #ifdef RANDOM_FORTUNA
-			random_fortuna_read(NULL, 0);
+			random_fortuna_pre_read();
 			random_fortuna_read(buf, i);
-			random_fortuna_read(NULL, 1);
+			random_fortuna_post_read();
 #endif
-#if 0
-			{
-			int j;
-
+			zsize = block_deflate(buf, zbuf, i);
+			if (zsize < i)
+				printf("ERROR!! Compressible RNG output!\n");
+#ifdef DEBUG
+			printf("RNG output:\n");
 			for (j = 0; j < i; j++) {
 				printf(" %02X", buf[j]);
 				if (j % 32 == 31 || j == i - 1)
 					printf("\n");
 			}
+			printf("Compressed output:\n");
+			for (j = 0; j < zsize; j++) {
+				printf(" %02X", zbuf[j]);
+				if (j % 32 == 31 || j == zsize - 1)
+					printf("\n");
 			}
 #endif
+			free(zbuf);
 			free(buf);
 		}
 		usleep(100);
@@ -228,7 +309,7 @@ main(int argc, char *argv[])
 
 	for (t = 0; t < NUM_THREADS; t++) {
 		printf("In main: creating thread %ld\n", t);
-		rc = thrd_create(&threads[t], (t == 0 ? RunHarvester : (t == 1 ? WriteCSPRNG : ReadCSPRNG)), t);
+		rc = thrd_create(&threads[t], (t == 0 ? RunHarvester : (t == 1 ? WriteCSPRNG : ReadCSPRNG)), NULL);
 		if (rc != thrd_success) {
 			printf("ERROR; return code from thrd_create() is %d\n", rc);
 			exit(-1);