diff --git a/.gitignore b/.gitignore index 76500c6..4d8142e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ # output files mkp224o +calcest +test_* *.o # generated onions @@ -7,6 +9,7 @@ mkp224o # garbage configure +configure~ GNUmakefile GNUmakefile.in.bak config.status diff --git a/GNUmakefile.in b/GNUmakefile.in index 71beef6..d566837 100644 --- a/GNUmakefile.in +++ b/GNUmakefile.in @@ -57,6 +57,7 @@ TEST_ED25519_OBJ= \ ALL_O= $(sort \ $(MAIN_OBJ) \ + $(UTIL_CALCEST_OBJ) \ $(TEST_BASE64_OBJ) \ $(TEST_BASE32_OBJ) \ $(TEST_BASE16_OBJ) \ diff --git a/calcest.c b/calcest.c index 28d9c98..0315a3a 100644 --- a/calcest.c +++ b/calcest.c @@ -15,19 +15,20 @@ */ const double probs[] = { 0.5, 0.8, 0.9, 0.95, 0.99 }; -const int charcounts[] = { 2, 3, 4, 5, 6, 7 }; +const int charcounts[] = { 2, 3, 4, 5, 6, 7, 8, 9, 10 }; -int main(void) +int main(int argc,char **argv) { + printf(" |"); for (size_t i = 0; i < sizeof(probs)/sizeof(probs[0]); ++i) { - printf(" %11d%% |",(int)((probs[i]*100)+0.5)); + printf(" %15d%% |",(int)((probs[i]*100)+0.5)); } printf("\n"); printf("---+"); for (size_t i = 0; i < sizeof(probs)/sizeof(probs[0]); ++i) { - printf("--------------+"); + printf("------------------+"); } printf("\n"); @@ -35,7 +36,7 @@ int main(void) printf("%2d |",charcounts[i]); for (size_t j = 0; j < sizeof(probs)/sizeof(probs[0]); ++j) { double t = log2(1 - probs[j]) / log2(1 - (1 / pow(32,charcounts[i]))); - printf(" %12.0f |",t); + printf(" %16.0f |",t); } printf("\n"); } diff --git a/configure.ac b/configure.ac index aad12af..35a488e 100644 --- a/configure.ac +++ b/configure.ac @@ -112,7 +112,7 @@ AC_ARG_ENABLE([donna-sse2], ) # default -AS_IF([test "x$ed25519impl" == "x"],[ed25519impl="donna"]) +AS_IF([test "x$ed25519impl" = "x"],[ed25519impl="donna"]) if test "$ed25519impl" = "donna-sse2" then diff --git a/ioutil.c b/ioutil.c index 4e6aa55..cc430df 100644 --- a/ioutil.c +++ b/ioutil.c @@ -1,7 +1,10 @@ #include #include +#include #include "types.h" #include "ioutil.h" +#include "vec.h" +#include #ifndef _WIN32 @@ -31,7 +34,7 @@ FH createfile(const char *path,int secret) int fd; do { fd = open(path,O_WRONLY | O_CREAT | O_TRUNC,secret ? 0600 : 0666); - if (fd == -1) { + if (fd < 0) { if (errno == EINTR) continue; return -1; @@ -45,7 +48,7 @@ int closefile(FH fd) int cret; do { cret = close(fd); - if (cret == -1) { + if (cret < 0) { if (errno == EINTR) continue; return -1; @@ -59,6 +62,122 @@ int createdir(const char *path,int secret) return mkdir(path,secret ? 0700 : 0777); } +static int syncwritefile(const char *filename,const char *tmpname,int secret,const u8 *data,size_t datalen) +{ + FH f = createfile(tmpname,secret); + if (f == FH_invalid) + return -1; + + if (writeall(f,data,datalen) < 0) { + goto failclose; + } + + int sret; + do { + sret = fsync(f); + if (sret < 0) { + if (errno == EINTR) + continue; + + goto failclose; + } + } while (0); + + if (closefile(f) < 0) { + goto failrm; + } + + if (rename(tmpname,filename) < 0) { + goto failrm; + } + + return 0; + +failclose: + (void) closefile(f); +failrm: + remove(tmpname); + + return -1; +} + +int syncwrite(const char *filename,int secret,const u8 *data,size_t datalen) +{ + //fprintf(stderr,"filename = %s\n",filename); + + size_t fnlen = strlen(filename); + + VEC_STRUCT(,char) tmpnamebuf; + VEC_INIT(tmpnamebuf); + VEC_ADDN(tmpnamebuf,fnlen + 4 /* ".tmp" */ + 1 /* "\0" */); + memcpy(&VEC_BUF(tmpnamebuf,0),filename,fnlen); + strcpy(&VEC_BUF(tmpnamebuf,fnlen),".tmp"); + const char *tmpname = &VEC_BUF(tmpnamebuf,0); + + //fprintf(stderr,"tmpname = %s\n",tmpname); + + int r = syncwritefile(filename,tmpname,secret,data,datalen); + + VEC_FREE(tmpnamebuf); + + if (r < 0) + return r; + + VEC_STRUCT(,char) dirnamebuf; + VEC_INIT(dirnamebuf); + const char *dirname; + + for (ssize_t x = ((ssize_t)fnlen) - 1;x >= 0;--x) { + if (filename[x] == '/') { + if (x) + --x; + ++x; + VEC_ADDN(dirnamebuf,x + 1); + memcpy(&VEC_BUF(dirnamebuf,0),filename,x); + VEC_BUF(dirnamebuf,x) = '\0'; + dirname = &VEC_BUF(dirnamebuf,0); + goto foundslash; + } + } + /* not found slash, fall back to "." */ + dirname = "."; + +foundslash: + //fprintf(stderr,"dirname = %s\n",dirname); + ; + + int dirf; + do { + dirf = open(dirname,O_RDONLY); + if (dirf < 0) { + if (errno == EINTR) + continue; + + // failed for non-eintr reasons + goto skipdsync; // don't really care enough + } + } while (0); + + int sret; + do { + sret = fsync(dirf); + if (sret < 0) { + if (errno == EINTR) + continue; + + // failed for non-eintr reasons + break; // don't care + } + } while (0); + + (void) closefile(dirf); // don't care + +skipdsync: + VEC_FREE(dirnamebuf); + + return 0; +} + #else int writeall(FH fd,const u8 *data,size_t len) @@ -99,6 +218,61 @@ int createdir(const char *path,int secret) return CreateDirectoryA(path,0) ? 0 : -1; } +static int syncwritefile(const char *filename,const char *tmpname,int secret,const char *data,size_t datalen) +{ + FH f = createfile(tmpnamestr,secret) + if (f == FH_invalid) + return -1; + + if (writeall(f,data,datalen) < 0) { + goto failclose; + } + + if (FlushFileBuffers(f) == 0) { + goto failclose; + } + + if (closefile(f) < 0) { + goto failrm; + } + + if (MoveFileA(tmpnamestr,filename) == 0) { + goto failrm; + } + + return 0; + +failclose: + (void) closefile(f); +failrm: + remove(tmpnamestr); + + return -1; +} + +int syncwrite(const char *filename,int secret,const char *data,size_t datalen) +{ + size_t fnlen = strlen(filename); + + VEC_STRUCT(,char) tmpnamebuf; + VEC_INIT(tmpnamebuf); + VEC_ADDN(tmpnamebuf,fnlen + 4 /* ".tmp" */ + 1 /* "\0" */); + memcpy(&VEC_BUF(tmpnamebuf,0),filename,fnlen); + strcpy(&VEC_BUF(tmpnamebuf,fnlen),".tmp"); + const char *tmpname = &VEC_BUF(tmpnamebuf,0); + + int r = syncwritefile(filename,tmpname,secret,data,datalen); + + VEC_FREE(tmpnamebuf); + + if (r < 0) + return r; + + // can't fsync parent dir on windows so just end here + + return 0; +} + #endif int writetofile(const char *path,const u8 *data,size_t len,int secret) diff --git a/ioutil.h b/ioutil.h index c7a1dab..5244508 100644 --- a/ioutil.h +++ b/ioutil.h @@ -18,3 +18,4 @@ int closefile(FH fd); int writeall(FH,const u8 *data,size_t len); int writetofile(const char *path,const u8 *data,size_t len,int secret); int createdir(const char *path,int secret); +int syncwrite(const char *filename,int secret,const u8 *data,size_t datalen); diff --git a/main.c b/main.c index e334e61..486a433 100644 --- a/main.c +++ b/main.c @@ -29,6 +29,8 @@ #include "worker.h" +#include "likely.h" + #ifndef _WIN32 #define FSZ "%zu" #else @@ -58,6 +60,11 @@ size_t printlen; // precalculated, related to printstartpos pthread_mutex_t fout_mutex; FILE *fout; +#ifdef PASSPHRASE +u8 orig_determseed[SEED_LEN]; +const char *checkpointfile = 0; +#endif + static void termhandler(int sig) { switch (sig) { @@ -82,6 +89,8 @@ VEC_STRUCT(tstatsvec,struct tstatstruct); static void printhelp(FILE *out,const char *progname) { + // 0 1 2 3 4 5 6 7 + // 01234567890123456789012345678901234567890123456789012345678901234567890123456789 fprintf(out, "Usage: %s FILTER [FILTER...] [OPTION]\n" " %s -f FILTERFILE [OPTION]\n" @@ -117,6 +126,9 @@ static void printhelp(FILE *out,const char *progname) " -p PASSPHRASE use passphrase to initialize the random seed with\n" " -P same as -p, but takes passphrase from PASSPHRASE\n" " environment variable\n" + " --checkpoint filename\n" + " load/save checkpoint of progress to specified file\n" + " (requires passphrase)\n" #endif " --rawyaml raw (unprefixed) public/secret keys for -y/-Y\n" " (may be useful for tor controller API)\n" @@ -203,9 +215,58 @@ static void setpassphrase(const char *pass) } fprintf(stderr," done.\n"); } + +static void savecheckpoint(void) +{ + u8 checkpoint[SEED_LEN]; + bool carry = 0; + pthread_mutex_lock(&determseed_mutex); + for (int i = 0; i < SEED_LEN; i++) { + checkpoint[i] = determseed[i] - orig_determseed[i] - carry; + carry = checkpoint[i] > determseed[i]; + } + pthread_mutex_unlock(&determseed_mutex); + + if (syncwrite(checkpointfile,1,checkpoint,SEED_LEN) < 0) { + pthread_mutex_lock(&fout_mutex); + fprintf(stderr,"ERROR: could not save checkpoint\n"); + pthread_mutex_unlock(&fout_mutex); + } +} + +static volatile int checkpointer_endwork = 0; + +static void *checkpointworker(void *arg) +{ + (void) arg; + + struct timespec ts; + memset(&ts,0,sizeof(ts)); + ts.tv_nsec = 100000000; + + struct timespec nowtime; + u64 ilasttime,inowtime; + clock_gettime(CLOCK_MONOTONIC,&nowtime); + ilasttime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000); + + while (!unlikely(checkpointer_endwork)) { + + clock_gettime(CLOCK_MONOTONIC,&nowtime); + inowtime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000); + + if ((i64)(inowtime - ilasttime) >= 300 * 1000000 /* 5 minutes */) { + savecheckpoint(); + ilasttime = inowtime; + } + } + + savecheckpoint(); + + return 0; +} #endif -VEC_STRUCT(threadvec, pthread_t); +VEC_STRUCT(threadvec,pthread_t); #include "filters_inc.inc.h" #include "filters_main.inc.h" @@ -276,12 +337,20 @@ int main(int argc,char **argv) printhelp(stdout,progname); exit(0); } - else if (!strcmp(arg,"rawyaml")) - yamlraw = 1; else if (!strcmp(arg,"version")) { printversion(); exit(0); } + else if (!strcmp(arg,"rawyaml")) + yamlraw = 1; +#ifdef PASSPHRASE + else if (!strcmp(arg,"checkpoint")) { + if (argc--) + checkpointfile = *argv++; + else + e_additional(); + } +#endif // PASSPHRASE else { fprintf(stderr,"unrecognised argument: --%s\n",arg); exit(1); @@ -453,6 +522,11 @@ int main(int argc,char **argv) exit(1); } + if (checkpointfile && !deterministic) { + fprintf(stderr,"--checkpoint requires passphrase\n"); + exit(1); + } + if (outfile) { fout = fopen(outfile,!outfileoverwrite ? "a" : "w"); if (!fout) { @@ -538,8 +612,27 @@ int main(int argc,char **argv) numthreads,numthreads == 1 ? "thread" : "threads"); #ifdef PASSPHRASE - if (!quietflag && deterministic && numneedgenerate != 1) - fprintf(stderr,"CAUTION: avoid using keys generated with same password for unrelated services, as single leaked key may help attacker to regenerate related keys.\n"); + if (deterministic) { + if (!quietflag && numneedgenerate != 1) + fprintf(stderr,"CAUTION: avoid using keys generated with same password for unrelated services, as single leaked key may help attacker to regenerate related keys.\n"); + if (checkpointfile) { + memcpy(orig_determseed,determseed,sizeof(determseed)); + // Read current checkpoint position if file exists + FILE *checkout = fopen(checkpointfile,"r"); + if (checkout) { + u8 checkpoint[SEED_LEN]; + if(fread(checkpoint,1,SEED_LEN,checkout) != SEED_LEN) { + fprintf(stderr,"failed to read checkpoint file\n"); + exit(1); + } + fclose(checkout); + + // Apply checkpoint to determseed + for (int i = 0; i < SEED_LEN; i++) + determseed[i] += checkpoint[i]; + } + } + } #endif signal(SIGTERM,termhandler); @@ -610,6 +703,18 @@ int main(int argc,char **argv) perror("pthread_attr_destroy"); } +#if PASSPHRASE + pthread_t checkpoint_thread; + + if (checkpointfile) { + tret = pthread_create(&checkpoint_thread,NULL,checkpointworker,NULL); + if (tret) { + fprintf(stderr,"error while making checkpoint thread: %s\n",strerror(tret)); + exit(1); + } + } +#endif + #ifdef STATISTICS struct timespec nowtime; u64 istarttime,inowtime,ireporttime = 0,elapsedoffset = 0; @@ -619,6 +724,7 @@ int main(int argc,char **argv) } istarttime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000); #endif + struct timespec ts; memset(&ts,0,sizeof(ts)); ts.tv_nsec = 100000000; @@ -654,7 +760,7 @@ int main(int argc,char **argv) VEC_BUF(tstats,i).numrestart += (u64)tdiff; sumrestart += VEC_BUF(tstats,i).numrestart; } - if (reportdelay && (!ireporttime || inowtime - ireporttime >= reportdelay)) { + if (reportdelay && (!ireporttime || (i64)(inowtime - ireporttime) >= (i64)reportdelay)) { if (ireporttime) ireporttime += reportdelay; else @@ -694,8 +800,16 @@ int main(int argc,char **argv) if (!quietflag) fprintf(stderr,"waiting for threads to finish..."); + for (size_t i = 0;i < VEC_LENGTH(threads);++i) pthread_join(VEC_BUF(threads,i),0); +#ifdef PASSPHRASE + if (checkpointfile) { + checkpointer_endwork = 1; + pthread_join(checkpoint_thread,0); + } +#endif + if (!quietflag) fprintf(stderr," done.\n"); diff --git a/worker.c b/worker.c index f0aaf62..7f8cb03 100644 --- a/worker.c +++ b/worker.c @@ -98,9 +98,9 @@ static void onionready(char *sname,const u8 *secret,const u8 *pubonion) // Sanity check that the public key matches the private one. ge_p3 ALIGN(16) point; u8 testpk[PUBLIC_LEN]; - ge_scalarmult_base(&point, secret); - ge_p3_tobytes(testpk, &point); - if (!memcmp(testpk, pubonion, PUBLIC_LEN)) + ge_scalarmult_base(&point,&secret[SKPREFIX_SIZE]); + ge_p3_tobytes(testpk,&point); + if (memcmp(testpk,&pubonion[PKPREFIX_SIZE],PUBLIC_LEN) != 0) abort(); #endif diff --git a/worker_batch.inc.h b/worker_batch.inc.h index 8de5330..2e0788f 100644 --- a/worker_batch.inc.h +++ b/worker_batch.inc.h @@ -21,6 +21,8 @@ void *worker_batch(void *task) #ifdef STATISTICS struct statstruct *st = (struct statstruct *)task; +#else + (void) task; #endif PREFILTER @@ -36,6 +38,7 @@ void *worker_batch(void *task) sname = makesname(); initseed: + #ifdef STATISTICS ++st->numrestart.v; #endif @@ -112,8 +115,11 @@ initseed: end: free(sname); + POSTFILTER + sodium_memzero(secret,sizeof(secret)); sodium_memzero(seed,sizeof(seed)); + return 0; } diff --git a/worker_batch_pass.inc.h b/worker_batch_pass.inc.h index 3676eec..44006ff 100644 --- a/worker_batch_pass.inc.h +++ b/worker_batch_pass.inc.h @@ -22,6 +22,8 @@ void *worker_batch_pass(void *task) #ifdef STATISTICS struct statstruct *st = (struct statstruct *)task; +#else + (void) task; #endif PREFILTER @@ -37,6 +39,7 @@ void *worker_batch_pass(void *task) sname = makesname(); initseed: + #ifdef STATISTICS ++st->numrestart.v; #endif @@ -133,7 +136,7 @@ initseed: ge_p1p1_to_p3(&ge_public,&sum); } // NOTE: leaves unfinished one bit at the very end - ge_p3_batchtobytes_destructive_1(pk_batch,ge_batch,batchgez,tmp_batch,remaining); + ge_p3_batchtobytes_destructive_1(pk_batch,ge_batch,tmp_batch,remaining); #ifdef STATISTICS st->numcalc.v += remaining; @@ -189,9 +192,12 @@ initseed: end: free(sname); + POSTFILTER + sodium_memzero(secret,sizeof(secret)); sodium_memzero(seed,sizeof(seed)); + return 0; } #endif // PASSPHRASE diff --git a/worker_fast.inc.h b/worker_fast.inc.h index 141a34b..1d7443f 100644 --- a/worker_fast.inc.h +++ b/worker_fast.inc.h @@ -33,6 +33,7 @@ void *worker_fast(void *task) sname = makesname(); initseed: + #ifdef STATISTICS ++st->numrestart.v; #endif @@ -96,8 +97,11 @@ initseed: end: free(sname); + POSTFILTER + sodium_memzero(secret,sizeof(secret)); sodium_memzero(seed,sizeof(seed)); + return 0; } diff --git a/worker_fast_pass.inc.h b/worker_fast_pass.inc.h index 2d482b3..132aed6 100644 --- a/worker_fast_pass.inc.h +++ b/worker_fast_pass.inc.h @@ -34,6 +34,7 @@ void *worker_fast_pass(void *task) sname = makesname(); initseed: + #ifdef STATISTICS ++st->numrestart.v; #endif @@ -104,9 +105,12 @@ initseed: end: free(sname); + POSTFILTER + sodium_memzero(secret,sizeof(secret)); sodium_memzero(seed,sizeof(seed)); + return 0; } #endif // PASSPHRASE diff --git a/worker_slow.inc.h b/worker_slow.inc.h index 799e4aa..278b58b 100644 --- a/worker_slow.inc.h +++ b/worker_slow.inc.h @@ -17,6 +17,7 @@ void *worker_slow(void *task) #else (void) task; #endif + PREFILTER memcpy(secret,skprefix,SKPREFIX_SIZE); @@ -32,6 +33,7 @@ void *worker_slow(void *task) initseed: randombytes(seed,sizeof(seed)); ed25519_seckey_expand(sk,seed); + #ifdef STATISTICS ++st->numrestart.v; #endif @@ -82,8 +84,11 @@ next: end: free(sname); + POSTFILTER + sodium_memzero(secret,sizeof(secret)); sodium_memzero(seed,sizeof(seed)); + return 0; }