Merge github-cathugger:cathugger/mkp224o

.gitignore

@@ -1,5 +1,7 @@
 # output files
 mkp224o
+calcest
+test_*
 *.o
 
 # generated onions
@@ -7,6 +9,7 @@ mkp224o
 
 # garbage
 configure
+configure~
 GNUmakefile
 GNUmakefile.in.bak
 config.status

GNUmakefile.in

@@ -57,6 +57,7 @@ TEST_ED25519_OBJ= \
 
 ALL_O= $(sort \
 	$(MAIN_OBJ) \
+	$(UTIL_CALCEST_OBJ) \
 	$(TEST_BASE64_OBJ) \
 	$(TEST_BASE32_OBJ) \
 	$(TEST_BASE16_OBJ) \

calcest.c

@@ -15,19 +15,20 @@
  */
 
 const double probs[] = { 0.5, 0.8, 0.9, 0.95, 0.99 };
-const int charcounts[] = { 2, 3, 4, 5, 6, 7 };
+const int charcounts[] = { 2, 3, 4, 5, 6, 7, 8, 9, 10 };
 
-int main(void)
+int main(int argc,char **argv)
 {
+
 	printf("   |");
 	for (size_t i = 0; i < sizeof(probs)/sizeof(probs[0]); ++i) {
-		printf(" %11d%% |",(int)((probs[i]*100)+0.5));
+		printf(" %15d%% |",(int)((probs[i]*100)+0.5));
 	}
 	printf("\n");
 
 	printf("---+");
 	for (size_t i = 0; i < sizeof(probs)/sizeof(probs[0]); ++i) {
-		printf("--------------+");
+		printf("------------------+");
 	}
 	printf("\n");
 
@@ -35,7 +36,7 @@ int main(void)
 		printf("%2d |",charcounts[i]);
 		for (size_t j = 0; j < sizeof(probs)/sizeof(probs[0]); ++j) {
 			double t = log2(1 - probs[j]) / log2(1 - (1 / pow(32,charcounts[i])));
-			printf(" %12.0f |",t);
+			printf(" %16.0f |",t);
 		}
 		printf("\n");
 	}

configure.ac

@@ -112,7 +112,7 @@ AC_ARG_ENABLE([donna-sse2],
 )
 
 # default
-AS_IF([test "x$ed25519impl" == "x"],[ed25519impl="donna"])
+AS_IF([test "x$ed25519impl" = "x"],[ed25519impl="donna"])
 
 if test "$ed25519impl" = "donna-sse2"
 then

ioutil.c

@@ -1,7 +1,10 @@
 #include <stdint.h>
 #include <string.h>
+#include <stdlib.h>
 #include "types.h"
 #include "ioutil.h"
+#include "vec.h"
+#include <stdio.h>
 
 #ifndef _WIN32
 
@@ -31,7 +34,7 @@ FH createfile(const char *path,int secret)
 	int fd;
 	do {
 		fd = open(path,O_WRONLY | O_CREAT | O_TRUNC,secret ? 0600 : 0666);
-		if (fd == -1) {
+		if (fd < 0) {
 			if (errno == EINTR)
 				continue;
 			return -1;
@@ -45,7 +48,7 @@ int closefile(FH fd)
 	int cret;
 	do {
 		cret = close(fd);
-		if (cret == -1) {
+		if (cret < 0) {
 			if (errno == EINTR)
 				continue;
 			return -1;
@@ -59,6 +62,122 @@ int createdir(const char *path,int secret)
 	return mkdir(path,secret ? 0700 : 0777);
 }
 
+static int syncwritefile(const char *filename,const char *tmpname,int secret,const u8 *data,size_t datalen)
+{
+	FH f = createfile(tmpname,secret);
+	if (f == FH_invalid)
+		return -1;
+
+	if (writeall(f,data,datalen) < 0) {
+		goto failclose;
+	}
+
+	int sret;
+	do {
+		sret = fsync(f);
+		if (sret < 0) {
+			if (errno == EINTR)
+				continue;
+
+			goto failclose;
+		}
+	} while (0);
+
+	if (closefile(f) < 0) {
+		goto failrm;
+	}
+
+	if (rename(tmpname,filename) < 0) {
+		goto failrm;
+	}
+
+	return 0;
+
+failclose:
+	(void) closefile(f);
+failrm:
+	remove(tmpname);
+
+	return -1;
+}
+
+int syncwrite(const char *filename,int secret,const u8 *data,size_t datalen)
+{
+	//fprintf(stderr,"filename = %s\n",filename);
+
+	size_t fnlen = strlen(filename);
+
+	VEC_STRUCT(,char) tmpnamebuf;
+	VEC_INIT(tmpnamebuf);
+	VEC_ADDN(tmpnamebuf,fnlen + 4 /* ".tmp" */ + 1 /* "\0" */);
+	memcpy(&VEC_BUF(tmpnamebuf,0),filename,fnlen);
+	strcpy(&VEC_BUF(tmpnamebuf,fnlen),".tmp");
+	const char *tmpname = &VEC_BUF(tmpnamebuf,0);
+
+	//fprintf(stderr,"tmpname = %s\n",tmpname);
+
+	int r = syncwritefile(filename,tmpname,secret,data,datalen);
+
+	VEC_FREE(tmpnamebuf);
+
+	if (r < 0)
+		return r;
+
+	VEC_STRUCT(,char) dirnamebuf;
+	VEC_INIT(dirnamebuf);
+	const char *dirname;
+
+	for (ssize_t x = ((ssize_t)fnlen) - 1;x >= 0;--x) {
+		if (filename[x] == '/') {
+			if (x)
+				--x;
+			++x;
+			VEC_ADDN(dirnamebuf,x + 1);
+			memcpy(&VEC_BUF(dirnamebuf,0),filename,x);
+			VEC_BUF(dirnamebuf,x) = '\0';
+			dirname = &VEC_BUF(dirnamebuf,0);
+			goto foundslash;
+		}
+	}
+	/* not found slash, fall back to "." */
+	dirname = ".";
+
+foundslash:
+	//fprintf(stderr,"dirname = %s\n",dirname);
+	;
+
+	int dirf;
+	do {
+		dirf = open(dirname,O_RDONLY);
+		if (dirf < 0) {
+			if (errno == EINTR)
+				continue;
+
+			// failed for non-eintr reasons
+			goto skipdsync; // don't really care enough
+		}
+	} while (0);
+
+	int sret;
+	do {
+		sret = fsync(dirf);
+		if (sret < 0) {
+			if (errno == EINTR)
+				continue;
+
+			// failed for non-eintr reasons
+			break; // don't care
+		}
+	} while (0);
+
+	(void) closefile(dirf); // don't care
+
+skipdsync:
+	VEC_FREE(dirnamebuf);
+
+	return 0;
+}
+
 #else
 
 int writeall(FH fd,const u8 *data,size_t len)
@@ -99,6 +218,61 @@ int createdir(const char *path,int secret)
 	return CreateDirectoryA(path,0) ? 0 : -1;
 }
 
+static int syncwritefile(const char *filename,const char *tmpname,int secret,const char *data,size_t datalen)
+{
+	FH f = createfile(tmpnamestr,secret)
+	if (f == FH_invalid)
+		return -1;
+
+	if (writeall(f,data,datalen) < 0) {
+		goto failclose;
+	}
+
+	if (FlushFileBuffers(f) == 0) {
+		goto failclose;
+	}
+
+	if (closefile(f) < 0) {
+		goto failrm;
+	}
+
+	if (MoveFileA(tmpnamestr,filename) == 0) {
+		goto failrm;
+	}
+
+	return 0;
+
+failclose:
+	(void) closefile(f);
+failrm:
+	remove(tmpnamestr);
+
+	return -1;
+}
+
+int syncwrite(const char *filename,int secret,const char *data,size_t datalen)
+{
+	size_t fnlen = strlen(filename);
+
+	VEC_STRUCT(,char) tmpnamebuf;
+	VEC_INIT(tmpnamebuf);
+	VEC_ADDN(tmpnamebuf,fnlen + 4 /* ".tmp" */ + 1 /* "\0" */);
+	memcpy(&VEC_BUF(tmpnamebuf,0),filename,fnlen);
+	strcpy(&VEC_BUF(tmpnamebuf,fnlen),".tmp");
+	const char *tmpname = &VEC_BUF(tmpnamebuf,0);
+
+	int r = syncwritefile(filename,tmpname,secret,data,datalen);
+
+	VEC_FREE(tmpnamebuf);
+
+	if (r < 0)
+		return r;
+
+	// can't fsync parent dir on windows so just end here
+
+	return 0;
+}
+
 #endif
 
 int writetofile(const char *path,const u8 *data,size_t len,int secret)

ioutil.h

@@ -18,3 +18,4 @@ int closefile(FH fd);
 int writeall(FH,const u8 *data,size_t len);
 int writetofile(const char *path,const u8 *data,size_t len,int secret);
 int createdir(const char *path,int secret);
+int syncwrite(const char *filename,int secret,const u8 *data,size_t datalen);

main.c

@@ -29,6 +29,8 @@
 
 #include "worker.h"
 
+#include "likely.h"
+
 #ifndef _WIN32
 #define FSZ "%zu"
 #else
@@ -58,6 +60,11 @@ size_t printlen;      // precalculated, related to printstartpos
 pthread_mutex_t fout_mutex;
 FILE *fout;
 
+#ifdef PASSPHRASE
+u8 orig_determseed[SEED_LEN];
+const char *checkpointfile = 0;
+#endif
+
 static void termhandler(int sig)
 {
 	switch (sig) {
@@ -82,6 +89,8 @@ VEC_STRUCT(tstatsvec,struct tstatstruct);
 
 static void printhelp(FILE *out,const char *progname)
 {
+	//   0         1         2         3         4         5         6         7
+	//   01234567890123456789012345678901234567890123456789012345678901234567890123456789
 	fprintf(out,
 		"Usage: %s FILTER [FILTER...] [OPTION]\n"
 		"       %s -f FILTERFILE [OPTION]\n"
@@ -117,6 +126,9 @@ static void printhelp(FILE *out,const char *progname)
 		"  -p PASSPHRASE         use passphrase to initialize the random seed with\n"
 		"  -P                    same as -p, but takes passphrase from PASSPHRASE\n"
 		"                        environment variable\n"
+		"  --checkpoint filename\n"
+		"                        load/save checkpoint of progress to specified file\n"
+		"                        (requires passphrase)\n"
 #endif
 		"      --rawyaml         raw (unprefixed) public/secret keys for -y/-Y\n"
 		"                        (may be useful for tor controller API)\n"
@@ -203,9 +215,58 @@ static void setpassphrase(const char *pass)
 	}
 	fprintf(stderr," done.\n");
 }
+
+static void savecheckpoint(void)
+{
+	u8 checkpoint[SEED_LEN];
+	bool carry = 0;
+	pthread_mutex_lock(&determseed_mutex);
+	for (int i = 0; i < SEED_LEN; i++) {
+		checkpoint[i] = determseed[i] - orig_determseed[i] - carry;
+		carry = checkpoint[i] > determseed[i];
+	}
+	pthread_mutex_unlock(&determseed_mutex);
+
+	if (syncwrite(checkpointfile,1,checkpoint,SEED_LEN) < 0) {
+		pthread_mutex_lock(&fout_mutex);
+		fprintf(stderr,"ERROR: could not save checkpoint\n");
+		pthread_mutex_unlock(&fout_mutex);
+	}
+}
+
+static volatile int checkpointer_endwork = 0;
+
+static void *checkpointworker(void *arg)
+{
+	(void) arg;
+
+	struct timespec ts;
+	memset(&ts,0,sizeof(ts));
+	ts.tv_nsec = 100000000;
+
+	struct timespec nowtime;
+	u64 ilasttime,inowtime;
+	clock_gettime(CLOCK_MONOTONIC,&nowtime);
+	ilasttime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000);
+
+	while (!unlikely(checkpointer_endwork)) {
+
+		clock_gettime(CLOCK_MONOTONIC,&nowtime);
+		inowtime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000);
+
+		if ((i64)(inowtime - ilasttime) >= 300 * 1000000 /* 5 minutes */) {
+			savecheckpoint();
+			ilasttime = inowtime;
+		}
+	}
+
+	savecheckpoint();
+
+	return 0;
+}
 #endif
 
-VEC_STRUCT(threadvec, pthread_t);
+VEC_STRUCT(threadvec,pthread_t);
 
 #include "filters_inc.inc.h"
 #include "filters_main.inc.h"
@@ -276,12 +337,20 @@ int main(int argc,char **argv)
 					printhelp(stdout,progname);
 					exit(0);
 				}
-				else if (!strcmp(arg,"rawyaml"))
-					yamlraw = 1;
 				else if (!strcmp(arg,"version")) {
 					printversion();
 					exit(0);
 				}
+				else if (!strcmp(arg,"rawyaml"))
+					yamlraw = 1;
+#ifdef PASSPHRASE
+				else if (!strcmp(arg,"checkpoint")) {
+					if (argc--)
+						checkpointfile = *argv++;
+					else
+						e_additional();
+				}
+#endif // PASSPHRASE
 				else {
 					fprintf(stderr,"unrecognised argument: --%s\n",arg);
 					exit(1);
@@ -453,6 +522,11 @@ int main(int argc,char **argv)
 		exit(1);
 	}
 
+	if (checkpointfile && !deterministic) {
+		fprintf(stderr,"--checkpoint requires passphrase\n");
+		exit(1);
+	}
+
 	if (outfile) {
 		fout = fopen(outfile,!outfileoverwrite ? "a" : "w");
 		if (!fout) {
@@ -538,8 +612,27 @@ int main(int argc,char **argv)
 			numthreads,numthreads == 1 ? "thread" : "threads");
 
 #ifdef PASSPHRASE
-	if (!quietflag && deterministic && numneedgenerate != 1)
-		fprintf(stderr,"CAUTION: avoid using keys generated with same password for unrelated services, as single leaked key may help attacker to regenerate related keys.\n");
+	if (deterministic) {
+		if (!quietflag && numneedgenerate != 1)
+			fprintf(stderr,"CAUTION: avoid using keys generated with same password for unrelated services, as single leaked key may help attacker to regenerate related keys.\n");
+		if (checkpointfile) {
+			memcpy(orig_determseed,determseed,sizeof(determseed));
+			// Read current checkpoint position if file exists
+			FILE *checkout = fopen(checkpointfile,"r");
+			if (checkout) {
+				u8 checkpoint[SEED_LEN];
+				if(fread(checkpoint,1,SEED_LEN,checkout) != SEED_LEN) {
+					fprintf(stderr,"failed to read checkpoint file\n");
+					exit(1);
+				}
+				fclose(checkout);
+
+				// Apply checkpoint to determseed
+				for (int i = 0; i < SEED_LEN; i++)
+					determseed[i] += checkpoint[i];
+			}
+		}
+	}
 #endif
 
 	signal(SIGTERM,termhandler);
@@ -610,6 +703,18 @@ int main(int argc,char **argv)
 			perror("pthread_attr_destroy");
 	}
 
+#if PASSPHRASE
+	pthread_t checkpoint_thread;
+
+	if (checkpointfile) {
+		tret = pthread_create(&checkpoint_thread,NULL,checkpointworker,NULL);
+		if (tret) {
+			fprintf(stderr,"error while making checkpoint thread: %s\n",strerror(tret));
+			exit(1);
+		}
+	}
+#endif
+
 #ifdef STATISTICS
 	struct timespec nowtime;
 	u64 istarttime,inowtime,ireporttime = 0,elapsedoffset = 0;
@@ -619,6 +724,7 @@ int main(int argc,char **argv)
 	}
 	istarttime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000);
 #endif
+
 	struct timespec ts;
 	memset(&ts,0,sizeof(ts));
 	ts.tv_nsec = 100000000;
@@ -654,7 +760,7 @@ int main(int argc,char **argv)
 			VEC_BUF(tstats,i).numrestart += (u64)tdiff;
 			sumrestart += VEC_BUF(tstats,i).numrestart;
 		}
-		if (reportdelay && (!ireporttime || inowtime - ireporttime >= reportdelay)) {
+		if (reportdelay && (!ireporttime || (i64)(inowtime - ireporttime) >= (i64)reportdelay)) {
 			if (ireporttime)
 				ireporttime += reportdelay;
 			else
@@ -694,8 +800,16 @@ int main(int argc,char **argv)
 
 	if (!quietflag)
 		fprintf(stderr,"waiting for threads to finish...");
+
 	for (size_t i = 0;i < VEC_LENGTH(threads);++i)
 		pthread_join(VEC_BUF(threads,i),0);
+#ifdef PASSPHRASE
+	if (checkpointfile) {
+		checkpointer_endwork = 1;
+		pthread_join(checkpoint_thread,0);
+	}
+#endif
+
 	if (!quietflag)
 		fprintf(stderr," done.\n");
 

worker.c

@@ -98,9 +98,9 @@ static void onionready(char *sname,const u8 *secret,const u8 *pubonion)
 	// Sanity check that the public key matches the private one.
 	ge_p3 ALIGN(16) point;
 	u8 testpk[PUBLIC_LEN];
-	ge_scalarmult_base(&point, secret);
-	ge_p3_tobytes(testpk, &point);
-	if (!memcmp(testpk, pubonion, PUBLIC_LEN))
+	ge_scalarmult_base(&point,&secret[SKPREFIX_SIZE]);
+	ge_p3_tobytes(testpk,&point);
+	if (memcmp(testpk,&pubonion[PKPREFIX_SIZE],PUBLIC_LEN) != 0)
 		abort();
 #endif
 

worker_batch.inc.h

@@ -21,6 +21,8 @@ void *worker_batch(void *task)
 
 #ifdef STATISTICS
 	struct statstruct *st = (struct statstruct *)task;
+#else
+	(void) task;
 #endif
 
 	PREFILTER
@@ -36,6 +38,7 @@ void *worker_batch(void *task)
 	sname = makesname();
 
 initseed:
+
 #ifdef STATISTICS
 	++st->numrestart.v;
 #endif
@@ -112,8 +115,11 @@ initseed:
 
 end:
 	free(sname);
+
 	POSTFILTER
+
 	sodium_memzero(secret,sizeof(secret));
 	sodium_memzero(seed,sizeof(seed));
+
 	return 0;
 }

worker_batch_pass.inc.h

@@ -22,6 +22,8 @@ void *worker_batch_pass(void *task)
 
 #ifdef STATISTICS
 	struct statstruct *st = (struct statstruct *)task;
+#else
+	(void) task;
 #endif
 
 	PREFILTER
@@ -37,6 +39,7 @@ void *worker_batch_pass(void *task)
 	sname = makesname();
 
 initseed:
+
 #ifdef STATISTICS
 	++st->numrestart.v;
 #endif
@@ -133,7 +136,7 @@ initseed:
 			ge_p1p1_to_p3(&ge_public,&sum);
 		}
 		// NOTE: leaves unfinished one bit at the very end
-		ge_p3_batchtobytes_destructive_1(pk_batch,ge_batch,batchgez,tmp_batch,remaining);
+		ge_p3_batchtobytes_destructive_1(pk_batch,ge_batch,tmp_batch,remaining);
 
 #ifdef STATISTICS
 		st->numcalc.v += remaining;
@@ -189,9 +192,12 @@ initseed:
 
 end:
 	free(sname);
+
 	POSTFILTER
+
 	sodium_memzero(secret,sizeof(secret));
 	sodium_memzero(seed,sizeof(seed));
+
 	return 0;
 }
 #endif // PASSPHRASE

worker_fast.inc.h

@@ -33,6 +33,7 @@ void *worker_fast(void *task)
 	sname = makesname();
 
 initseed:
+
 #ifdef STATISTICS
 	++st->numrestart.v;
 #endif
@@ -96,8 +97,11 @@ initseed:
 
 end:
 	free(sname);
+
 	POSTFILTER
+
 	sodium_memzero(secret,sizeof(secret));
 	sodium_memzero(seed,sizeof(seed));
+
 	return 0;
 }

worker_fast_pass.inc.h

@@ -34,6 +34,7 @@ void *worker_fast_pass(void *task)
 	sname = makesname();
 
 initseed:
+
 #ifdef STATISTICS
 	++st->numrestart.v;
 #endif
@@ -104,9 +105,12 @@ initseed:
 
 end:
 	free(sname);
+
 	POSTFILTER
+
 	sodium_memzero(secret,sizeof(secret));
 	sodium_memzero(seed,sizeof(seed));
+
 	return 0;
 }
 #endif // PASSPHRASE

worker_slow.inc.h

@@ -17,6 +17,7 @@ void *worker_slow(void *task)
 #else
 	(void) task;
 #endif
+
 	PREFILTER
 
 	memcpy(secret,skprefix,SKPREFIX_SIZE);
@@ -32,6 +33,7 @@ void *worker_slow(void *task)
 initseed:
 	randombytes(seed,sizeof(seed));
 	ed25519_seckey_expand(sk,seed);
+
 #ifdef STATISTICS
 	++st->numrestart.v;
 #endif
@@ -82,8 +84,11 @@ next:
 
 end:
 	free(sname);
+
 	POSTFILTER
+
 	sodium_memzero(secret,sizeof(secret));
 	sodium_memzero(seed,sizeof(seed));
+
 	return 0;
 }