.

aes.h

@@ -68,7 +68,10 @@ static inline void ZTLF_AES256CFB_crypt(ZTLF_AES256CFB *c,void *out,const void *
 	}
 }
 
-static inline void ZTLF_AES256CFB_destroy(ZTLF_AES256CFB *c) { CCCryptorRelease(*c); }
+static inline void ZTLF_AES256CFB_destroy(ZTLF_AES256CFB *c)
+{
+	CCCryptorRelease(*c);
+}
 
 #endif /* __APPLE__ */
 
@@ -79,6 +82,47 @@ static inline void ZTLF_AES256CFB_destroy(ZTLF_AES256CFB *c) { CCCryptorRelease(
 /* If we don't have Apple or Windows, use OpenSSL/LibreSSL libcrypto */
 #ifndef ZTLF_HAVE_AES_IMPL
 
+#include <openssl/aes.h>
+#include <openssl/evp.h>
+
+typedef EVP_CIPHER_CTX ZTLF_AES256CFB;
+
+static inline void ZTLF_AES256ECB_encrypt(const void *key,void *out,const void *in)
+{
+	AES_KEY c;
+	AES_set_encrypt_key((const unsigned char *)key,256,&c);
+	AES_encrypt((const unsigned char *)in,(unsigned char *)out,&c);
+}
+
+static inline void ZTLF_AES256CFB_init(ZTLF_AES256CFB *c,const void *key,const void *iv,bool encrypt)
+{
+	EVP_CIPHER_CTX_init(c);
+	if (encrypt) {
+		EVP_EncryptInit_ex(c,EVP_aes_256_cfb128(),NULL,key,iv);
+	} else {
+		EVP_DecryptInit_ex(c,EVP_aes_256_cfb128(),NULL,key,iv);
+	}
+}
+
+static inline void ZTLF_AES256CFB_crypt(ZTLF_AES256CFB *c,void *out,const void *in,const unsigned long len)
+{
+	if (len) {
+		int outl = (int)len;
+		if (c->encrypt) {
+			EVP_EncryptUpdate(c,out,&outl,in,(int)len);
+		} else {
+			EVP_DecryptUpdate(c,out,&outl,in,(int)len);
+		}
+		if (outl != (int)len)
+			abort();
+	}
+}
+
+static inline void ZTLF_AES256CFB_destroy(ZTLF_AES256CFB *c)
+{
+	EVP_CIPHER_CTX_cleanup(c);
+}
+
 #endif
 
 #endif

common.h

@@ -152,10 +152,21 @@ static inline uint64_t ZTLF_htonll(uint64_t n)
 #else
 #define ZTLF_htonll(n) ((uint64_t)(n))
 #endif
-
 #define ZTLF_ntohll(n) ZTLF_htonll((n))
 
-#define ZTLF_MALLOC_CHECK(m) if (unlikely(!((m)))) { fprintf(stderr,"FATAL: malloc() failed!\n"); abort(); }
+#define ZTLF_ERR_NONE                         0
+#define ZTLF_ERR_OUT_OF_MEMORY                1
+#define ZTLF_ERR_ABORTED                      2
+#define ZTLF_ERR_OBJECT_TOO_LARGE             3
+
+#define ZTLF_MALLOC_CHECK(m) if (unlikely(!((m)))) { ZTLF_L_fatal("malloc() failed!"); abort(); }
+
+/* Macro to safely assign identical primitive types to unaligned variables */
+#if defined(_M_AMD64) || defined(__amd64__) || defined(__x86_64__) || defined(__amd64) || defined(__x86_64)
+#define ZTLF_UNALIGNED_ASSIGN(d,s) (d) = (s)
+#else
+#define ZTLF_UNALIGNED_ASSIGN(d,s) for(int _X=0;_X<sizeof(s);++_X) ((uint8_t *)&(d))[_X] = ((const uint8_t *)&(s))[_X]
+#endif
 
 #define ZTLF_NEG(e) (((e) <= 0) ? (e) : -(e))
 #define ZTLF_POS(e) (((e) >= 0) ? (e) : -(e))

db.c

@@ -34,7 +34,7 @@
 ZTLF_PACKED_STRUCT(struct ZTLF_DB_GraphNode
 {
 	double totalWeight;
-	uint16_t linkCount;
+	uint8_t linkCount;
 	int64_t linkedRecordGoff[];
 });
 
@@ -330,7 +330,9 @@ long ZTLF_DB_getRecord(struct ZTLF_DB *const db,struct ZTLF_Record *r,double *ag
 				o->eof = true;
 			} else {
 				/* Total weight of this owner's set of IDs is the sum of the total weights of each update in the set. */
-				o->aggregatedTotalWeight += ((struct ZTLF_DB_GraphNode *)(db->gfm + (uintptr_t)sqlite3_column_int64(db->sGetRecordHistoryById,2)))->totalWeight;
+				double tw;
+				ZTLF_UNALIGNED_ASSIGN(tw,((struct ZTLF_DB_GraphNode *)(db->gfm + (uintptr_t)sqlite3_column_int64(db->sGetRecordHistoryById,2)))->totalWeight);
+				o->aggregatedTotalWeight += tw;
 			}
 
 			/* "Next" timestamp is previous row since we iterate backwards through timestamp history. */
@@ -455,14 +457,15 @@ int ZTLF_DB_putRecord(struct ZTLF_DB *db,struct ZTLF_ExpandedRecord *const er)
 	}
 
 	/* Set links from this record in graph node or create dangling link entries. */
-	graphNode->linkCount = er->linkCount;
+	graphNode->linkCount = (uint8_t)er->linkCount;
+	const int64_t neg1 = -1;
 	for(unsigned int i=0,j=er->linkCount;i<j;++i) {
 		const uint8_t *l = er->links + (i * 32);
 		sqlite3_reset(db->sGetRecordInfoByHash);
 		sqlite3_bind_blob(db->sGetRecordInfoByHash,1,l,32,SQLITE_STATIC);
 		if (sqlite3_step(db->sGetRecordInfoByHash) == SQLITE_ROW) {
 			const int64_t linkedGoff = sqlite3_column_int64(db->sGetRecordInfoByHash,1);
-			graphNode->linkedRecordGoff[i] = linkedGoff;
+			ZTLF_UNALIGNED_ASSIGN(graphNode->linkedRecordGoff[i],linkedGoff);
 			ZTLF_Vector_i64_append(&graphTraversalQueue,linkedGoff);
 		} else {
 			sqlite3_reset(db->sAddDanglingLink);
@@ -470,7 +473,7 @@ int ZTLF_DB_putRecord(struct ZTLF_DB *db,struct ZTLF_ExpandedRecord *const er)
 			sqlite3_bind_int64(db->sAddDanglingLink,2,doff);
 			if ((e = sqlite3_step(db->sAddDanglingLink)) != SQLITE_DONE)
 				fprintf(stderr,"WARNING: database error adding dangling link: %d\n",e);
-			graphNode->linkedRecordGoff[i] = -1;
+			ZTLF_UNALIGNED_ASSIGN(graphNode->linkedRecordGoff[i],neg1);
 		}
 	}
 
@@ -482,10 +485,14 @@ int ZTLF_DB_putRecord(struct ZTLF_DB *db,struct ZTLF_ExpandedRecord *const er)
 	sqlite3_bind_blob(db->sGetDanglingLinks,1,er->hash,32,SQLITE_STATIC);
 	while (sqlite3_step(db->sGetDanglingLinks) == SQLITE_ROW) {
 		volatile struct ZTLF_DB_GraphNode *const linkingRecordGraphNode = (volatile struct ZTLF_DB_GraphNode *)(db->gfm + (uintptr_t)sqlite3_column_int64(db->sGetDanglingLinks,0));
-		totalWeight += linkingRecordGraphNode->totalWeight;
+		double tw;
+		ZTLF_UNALIGNED_ASSIGN(tw,linkingRecordGraphNode->totalWeight);
+		totalWeight += tw;
 		for(unsigned int j=0,k=linkingRecordGraphNode->linkCount;j<k;++j) {
-			if (linkingRecordGraphNode->linkedRecordGoff[j] < 0) {
-				linkingRecordGraphNode->linkedRecordGoff[j] = goff;
+			int64_t lrgoff;
+			ZTLF_UNALIGNED_ASSIGN(lrgoff,linkingRecordGraphNode->linkedRecordGoff[j]);
+			if (lrgoff < 0) {
+				ZTLF_UNALIGNED_ASSIGN(linkingRecordGraphNode->linkedRecordGoff[j],goff);
 				break;
 			}
 		}
@@ -498,7 +505,7 @@ int ZTLF_DB_putRecord(struct ZTLF_DB *db,struct ZTLF_ExpandedRecord *const er)
 		fprintf(stderr,"WARNING: database error deleting dangling links for received record: %d\n",e);
 
 	/* Set this record's initial total weight in its graph node. */
-	graphNode->totalWeight = totalWeight;
+	ZTLF_UNALIGNED_ASSIGN(graphNode->totalWeight,totalWeight);
 
 	/* SQLite database work is now done. */
 	pthread_mutex_unlock(&db->dbcLock);
@@ -512,9 +519,13 @@ int ZTLF_DB_putRecord(struct ZTLF_DB *db,struct ZTLF_ExpandedRecord *const er)
 		const int64_t goff = graphTraversalQueue.v[i++];
 		if (ZTLF_ISet_put(visited,goff)) {
 			volatile struct ZTLF_DB_GraphNode *const gn = (volatile struct ZTLF_DB_GraphNode *)(db->gfm + (uintptr_t)goff);
-			gn->totalWeight += wtmp;
+			double tw;
+			ZTLF_UNALIGNED_ASSIGN(tw,gn->totalWeight);
+			tw += wtmp;
+			ZTLF_UNALIGNED_ASSIGN(gn->totalWeight,tw);
 			for(unsigned int j=0,k=gn->linkCount;j<k;++j) {
-				const int64_t tmp = gn->linkedRecordGoff[j];
+				int64_t tmp;
+				ZTLF_UNALIGNED_ASSIGN(tmp,gn->linkedRecordGoff[j]);
 				if (tmp >= 0) {
 					ZTLF_Vector_i64_append(&graphTraversalQueue,tmp);
 				}

record.c

@@ -34,27 +34,34 @@ void ZTLF_Record_keyToId(uint64_t id[4],const void *k,const unsigned long klen)
 	ZTLF_Ed25519CreateKeypair((unsigned char *)id,priv,seed); /* only the first 32 bytes of the hash are used here */
 }
 
-bool ZTLF_Record_createInit(
+int ZTLF_Record_create(
 	struct ZTLF_RecordBuffer *rb,
 	const void *plainTextKey,
-	const unsigned long plainTextKeyLength,
+	unsigned int plainTextKeyLength,
 	const void *value,
-	const unsigned long valueLength,
+	unsigned int valueLength,
 	const void *ownerPublicKey,
-	const uint64_t timestamp,
-	const uint64_t ttl,
-	const bool encryptValue,
-	uint8_t *linkHashPrefix,
-	unsigned int linkHashPrefixLength)
+	const void *ownerPrivateKey,
+	const void *links,
+	unsigned int linkCount,
+	uint64_t timestamp,
+	uint64_t ttl,
+	bool encryptValue,
+	bool (*statusCallback)(uint32_t,uint32_t))
 {
 	if (valueLength > ZTLF_RECORD_MAX_VALUE_SIZE)
-		return false;
+		return ZTLF_ERR_OBJECT_TOO_LARGE;
+	if (linkCount > 0x1f)
+		linkCount = 0x1f;
 
-	uint8_t seed[64]; /* Ed25519 seed (first 32 bytes), secret key for value masking (second 32 bytes) */
+	/* Create ID and ID claim signing secret by using a hash of the plain text key to generate an ed25519 key pair. */
+	uint8_t seed[64]; /* first 32 bytes are used as ed25519 seed to generate pair, second 32 bytes are used for optional value masking encryption */
 	ZTLF_SHA512(seed,plainTextKey,plainTextKeyLength);
 	ZTLF_Ed25519CreateKeypair((unsigned char *)rb->data.r.id,(unsigned char *)rb->idClaimPrivateKey,(const unsigned char *)seed);
 
+	/* Owner is owner public key (no hashing is needed for ed25519 since key size == owner field size) */
 	memcpy(rb->data.r.owner,ownerPublicKey,ZTLF_ED25519_PUBLIC_KEY_SIZE);
+
 	rb->data.r.timestamp[0] = (uint8_t)((timestamp >> 32) & 0xff);
 	rb->data.r.timestamp[1] = (uint8_t)((timestamp >> 24) & 0xff);
 	rb->data.r.timestamp[2] = (uint8_t)((timestamp >> 16) & 0xff);
@@ -73,47 +80,68 @@ bool ZTLF_Record_createInit(
 		(ZTLF_RECORD_ALG_SIG_ED25519 << 2) |
 		(ZTLF_RECORD_ALG_SIG_ED25519)
 	);
-	rb->data.r.valueSize[0] = (uint8_t)(((valueLength) >> 8) & 0xff);
-	rb->data.r.valueSize[1] = (uint8_t)(valueLength & 0xff);
+	const unsigned long vlSize = ((linkCount & 0x1f) << 11) | (valueLength & 0x7ff);
+	rb->data.r.vlSize[0] = (uint8_t)(((vlSize) >> 8) & 0xff);
+	rb->data.r.vlSize[1] = (uint8_t)(vlSize & 0xff);
 
 	unsigned int s = sizeof(struct ZTLF_Record);
-	for(unsigned long i=0;i<valueLength;++i)
-		rb->data.b[s++] = ((const uint8_t *)value)[i];
+	if (encryptValue) {
+		uint8_t ivHash[48];
+		ZTLF_SHA384(ivHash,rb->data.b,sizeof(struct ZTLF_Record));
+		ZTLF_AES256CFB c;
+		ZTLF_AES256CFB_init(&c,seed + 32,ivHash,true);
+		ZTLF_AES256CFB_crypt(&c,rb->data.b + s,value,valueLength);
+		ZTLF_AES256CFB_destroy(&c);
+		s += valueLength;
+	} else {
+		for(unsigned int i=0;i<valueLength;++i)
+			rb->data.b[s++] = ((const uint8_t *)value)[i];
+	}
+	for(unsigned int i=0,j=(linkCount*32);i<j;++i)
+		rb->data.b[s++] = ((const uint8_t *)links)[i];
 
-	uint8_t workHash[48],work[ZTLF_WHARRGARBL_POW_BYTES],bestWork[ZTLF_WHARRGARBL_POW_BYTES];
-	uint8_t scoringHash[48];
-	ZTLF_SHA384(workHash,rb->data.b,s);
-	void *workMemory = malloc(ZTLF_RECORD_WHARRGARBL_POW_ITERATION_MEMORY);
+	const unsigned int neededBytes = s + ZTLF_WHARRGARBL_POW_BYTES + ZTLF_ED25519_SIGNATURE_SIZE + ZTLF_ED25519_SIGNATURE_SIZE;
+	if (neededBytes > ZTLF_RECORD_MAX_SIZE) /* sanity check, should be impossible */
+		return ZTLF_ERR_OBJECT_TOO_LARGE;
+	uint64_t neededScore64 = (uint64_t)neededBytes * (uint64_t)ZTLF_RECORD_WORK_COST_DIVISOR;
+	const uint32_t neededScore = (neededScore64 > 0xffffffffULL) ? (uint32_t)0xffffffff : (uint32_t)neededScore64;
+
+	uint8_t workHash[64],scoringHash[48];
+	ZTLF_SHA512(workHash,rb->data.b,s);
+	void *const workMemory = malloc(ZTLF_RECORD_WHARRGARBL_POW_ITERATION_MEMORY);
 	if (!workMemory)
-		return false;
-	ZTLF_SHA384_CTX sh;
-	for(uint32_t bestScore=0;;) {
-		ZTLF_wharrgarbl(work,workHash,sizeof(workHash),ZTLF_RECORD_WHARRGARBL_POW_ITERATION_DIFFICULTY,workMemory,ZTLF_RECORD_WHARRGARBL_POW_ITERATION_MEMORY,0);
-
-		ZTLF_SHA384_init(&sh);
-		ZTLF_SHA384_update(&sh,rb->data.b,s);
-		ZTLF_SHA384_update(&sh,work,ZTLF_WHARRGARBL_POW_BYTES);
-		ZTLF_SHA384_final(&sh,scoringHash);
+		return ZTLF_ERR_OUT_OF_MEMORY;
+	uint32_t bestScoreSoFar = 0;
+	for(;;) {
+		ZTLF_wharrgarbl(rb->data.b + s,workHash,sizeof(workHash),ZTLF_RECORD_WHARRGARBL_POW_ITERATION_DIFFICULTY,workMemory,ZTLF_RECORD_WHARRGARBL_POW_ITERATION_MEMORY,0);
+		ZTLF_SHA384(scoringHash,rb->data.b,s + ZTLF_WHARRGARBL_POW_BYTES);
 		const uint32_t score = ZTLF_score(scoringHash);
-
-		if (score >= bestScore) {
-			bestScore = score;
-			memcpy(bestWork,work,ZTLF_WHARRGARBL_POW_BYTES);
+		if (score >= neededScore)
+			break;
+		if (score > bestScoreSoFar)
+			bestScoreSoFar = score;
+		if (statusCallback) {
+			if (!statusCallback(bestScoreSoFar,neededScore)) {
+				free(workMemory);
+				return ZTLF_ERR_ABORTED;
+			}
 		}
 	}
-	memcpy(rb->data.b + s,bestWork,ZTLF_WHARRGARBL_POW_BYTES);
+	free(workMemory);
 	s += ZTLF_WHARRGARBL_POW_BYTES;
 
-	rb->size = s;
-	return true;
-}
+	uint8_t signHash[64];
+	ZTLF_SHA512(signHash,rb->data.b,s);
+	ZTLF_Ed25519Sign((unsigned char *)(rb->data.b + s),signHash,sizeof(signHash),(const unsigned char *)rb->data.r.id,(const unsigned char *)rb->idClaimPrivateKey);
+	s += ZTLF_ED25519_SIGNATURE_SIZE;
 
-bool ZTLF_Record_createFinal(
-	struct ZTLF_RecordBuffer *rb,
-	const void *links,
-	const unsigned int linkCount,
-	const void *ownerPrivateKey)
-{
+	ZTLF_SHA512(signHash,rb->data.b,s);
+	ZTLF_Ed25519Sign((unsigned char *)(rb->data.b + s),signHash,sizeof(signHash),(const unsigned char *)ownerPublicKey,(const unsigned char *)ownerPrivateKey);
+	s += ZTLF_ED25519_SIGNATURE_SIZE;
+
+	rb->size = s;
+
+	return ZTLF_ERR_NONE;
 }
 
 bool ZTLF_Record_expand(struct ZTLF_ExpandedRecord *const er,const struct ZTLF_Record *const r,const unsigned int rsize)
@@ -128,8 +156,10 @@ bool ZTLF_Record_expand(struct ZTLF_ExpandedRecord *const er,const struct ZTLF_R
 	er->ttl = ZTLF_Record_ttl(r);
 	er->size = rsize;
 
-	er->valueSize = ((unsigned int)r->valueSize[0]) << 8;
-	er->valueSize |= (unsigned int)r->valueSize[1];
+	const unsigned int vlSize = (((unsigned int)r->vlSize[0]) << 8) | (unsigned int)r->vlSize[1];
+
+	er->valueSize = vlSize & 0x7ff;
+	er->linkCount = vlSize >> 11;
 	if (er->valueSize > ZTLF_RECORD_MAX_VALUE_SIZE)
 		return false;
 	switch (er->workAlgorithm) {
@@ -159,19 +189,24 @@ bool ZTLF_Record_expand(struct ZTLF_ExpandedRecord *const er,const struct ZTLF_R
 
 	er->r = r;
 
-	er->value = r->data;
-	er->work = ((const uint8_t *)er->value) + er->valueSize;
-	if ((((const uint8_t *)er->value) + er->valueSize) >= (((const uint8_t *)r) + rsize)) return false;
-	er->linkCount = *(((const uint8_t *)er->work) + er->workSize);
-	er->links = ((const uint8_t *)er->work) + er->workSize + 1;
-	er->idClaimSignature = ((const uint8_t *)er->links) + (32 * er->linkCount);
-	er->ownerSignature = ((const uint8_t *)er->idClaimSignature) + er->idClaimSignatureSize;
-	if ((((const uint8_t *)er->ownerSignature) + er->ownerSignatureSize) > (((const uint8_t *)r) + rsize)) return false;
+	const uint8_t *dp = r->data;
+	er->value = dp;
+	dp += er->valueSize;
+	er->links = dp;
+	dp += (32 * er->linkCount);
+	er->work = dp;
+	dp += er->workSize;
+	er->idClaimSignature = dp;
+	dp += er->idClaimSignatureSize;
+	er->ownerSignature = dp;
+	dp += er->ownerSignatureSize;
+	if (dp > (((const uint8_t *)r) + rsize))
+		return false;
 
 	if (er->workSize > 0) {
-		uint8_t scoringHash[48];
-		ZTLF_SHA384(scoringHash,er->value,er->valueSize + er->workSize);
-		er->weight = 1.0 + (((double)ZTLF_score(scoringHash)) / 4294967295.0);
+		ZTLF_SHA384(er->scoringHash,r,sizeof(struct ZTLF_Record) + er->valueSize + (32 * er->linkCount) + er->workSize);
+		er->workScore = ZTLF_score(er->scoringHash);
+		er->weight = 1.0 + (((double)er->workScore) / 4294967295.0);
 	} else {
 		er->weight = 1.0;
 	}

record.h

@@ -42,45 +42,55 @@
 /**
  * Minimum size of a record (simply size of header)
  */
-#define ZTLF_RECORD_MIN_SIZE                         (sizeof(struct ZTLF_Record))
+#define ZTLF_RECORD_MIN_SIZE sizeof(struct ZTLF_Record)
 
 /**
  * Overall maximum allowed record size (sanity limit, cannot be changed)
  */
-#define ZTLF_RECORD_MAX_SIZE                         4096
+#define ZTLF_RECORD_MAX_SIZE 4096
 
 /**
  * Maximum record value size (cannot be changed without network-wide upgrade)
  */
-#define ZTLF_RECORD_MAX_VALUE_SIZE                   1024
+#define ZTLF_RECORD_MAX_VALUE_SIZE 1024
+
+/**
+ * Minimum number of links for a non-genesis record
+ */
+#define ZTLF_RECORD_MIN_LINKS 3
 
 /**
  * Unit for TTL in seconds (cannot be changed)
  */
-#define ZTLF_RECORD_TTL_INCREMENT_SEC                123671
+#define ZTLF_RECORD_TTL_INCREMENT_SEC 123671
 
 /**
- * Wharrgarbl difficulty per iteration for record PoW
+ * Wharrgarbl difficulty per iteration for record PoW (takes ~1-3 sec on a quad-core Core i7 in 2018)
  */
-#define ZTLF_RECORD_WHARRGARBL_POW_ITERATION_DIFFICULTY 0x1000
+#define ZTLF_RECORD_WHARRGARBL_POW_ITERATION_DIFFICULTY 0x10000
 
 /**
  * Wharrgarbl memory per iteration for record PoW
  */
-#define ZTLF_RECORD_WHARRGARBL_POW_ITERATION_MEMORY 67108864
+#define ZTLF_RECORD_WHARRGARBL_POW_ITERATION_MEMORY 268435456
+
+/**
+ * Number by which 32-bit scoring hash score (see score.h) is divided to get the maximum number of bytes "paid for" by a record's work
+ */
+#define ZTLF_RECORD_WORK_COST_DIVISOR 32768
 
 /**
  * Packed record as it appears on the wire and in the database
  */
 ZTLF_PACKED_STRUCT(struct ZTLF_Record
 {
-	uint64_t id[4];                            /* public key (or hash thereof) derived from record key */
-	uint64_t owner[4];                         /* public key (or hash thereof) of owner */
+	uint8_t id[32];                            /* public key (or hash thereof) derived from record key */
+	uint8_t owner[32];                         /* public key (or hash thereof) of owner */
 	uint8_t timestamp[5];                      /* 40-bit (big-endian) timestamp in seconds since epoch */
 	uint8_t ttl;                               /* TTL in 123671 second (~34 hour) increments or 0 to relinquish ID ownership now */
 	uint8_t algorithms;                        /* VVWWIIOO: VV=value cipher,WW=work,II=id claim signature,OO=owner signature */
-	uint8_t valueSize[2];                      /* size of value in bytes (16-bit) */
-	uint8_t data[];                            /* value, work, links, id claim signature, owner signature */
+	uint8_t vlSize[2];                         /* number of links (most significant 5 bits) and size of value (least significant 11 bits) */
+	uint8_t data[];                            /* value, links, work, id claim signature, owner signature */
 });
 
 /**
@@ -104,8 +114,8 @@ struct ZTLF_ExpandedRecord
 	const struct ZTLF_Record *r;
 
 	const void *value;
-	const void *work;
 	const void *links; /* size in bytes is 32*r->linkCount */
+	const void *work;
 	const void *idClaimSignature;
 	const void *ownerSignature;
 
@@ -114,13 +124,17 @@ struct ZTLF_ExpandedRecord
 	uint64_t timestamp;
 	uint64_t ttl;
 	double weight;
+
 	unsigned int size;
 
 	unsigned int valueSize;
+	unsigned int linkCount;
 	unsigned int workSize;
 	unsigned int idClaimSignatureSize;
 	unsigned int ownerSignatureSize;
-	unsigned int linkCount;
+
+	uint8_t scoringHash[48]; /* only first 32 bytes are used to compute score */
+	uint32_t workScore;
 
 	uint8_t valueCipher;
 	uint8_t workAlgorithm;
@@ -138,48 +152,43 @@ struct ZTLF_ExpandedRecord
 void ZTLF_Record_keyToId(uint64_t id[4],const void *k,const unsigned long klen);
 
 /**
- * Create record, phase one -- set value, compute work
+ * Create a record
  * 
- * This can be a very time consuming operation due to work.
+ * This can be a very time consuming operation due to proof of work. The status callback
+ * can be used to provide some user feedback and cancel long running jobs. Its parameters
+ * are the best PoW score so far and the target PoW score, which is based on the size
+ * of this record in bytes (including overhead). If the status callback returns false
+ * the search is aborted and this function returns false.
  * 
  * @param rb Record buffer (existing contents will be lost)
  * @param plainTextKey Plain text key
  * @param plainTextKeyLength Plain text key length
  * @param value Plain text value
  * @param valueLength Length of value in bytes
+ * @param ownerPublicKey Public key of owner (currently must be 32-byte ed25519 public key)
+ * @param ownerPrivateKey Private key of owner (currently must be 32-byte ed25519 private key)
+ * @param links Links (must be 32*linkCount bytes in size)
+ * @param linkCount Number of links (theoretical max 255, links beyond this are ignored)
  * @param timestamp Timestamp in seconds since epoch
  * @param ttl TTL in seconds (will be quantized to ZTLF_RECORD_TTL_INCREMENT_SEC)
  * @param encryptValue If true, value will be hidden from anyone who doesn't know the plain text key (default behavior)
- * @param linkHashPrefix Buffer to fill with bytes to select prefixed for link hashes
- * @param linkHashPrefixLength Length of linkHashPrefix in bytes (up to 32)
- * @return True if parameters were valid
+ * @param statusCallback If non-NULL call this periodically and if it returns false terminate work and return false from create
+ * @return 0 on success or error code
  */
-bool ZTLF_Record_createInit(
+int ZTLF_Record_create(
 	struct ZTLF_RecordBuffer *rb,
 	const void *plainTextKey,
-	const unsigned long plainTextKeyLength,
+	unsigned int plainTextKeyLength,
 	const void *value,
-	const unsigned long valueLength,
+	unsigned int valueLength,
 	const void *ownerPublicKey,
-	const uint64_t timestamp,
-	const uint64_t ttl,
-	const bool encryptValue,
-	uint8_t *linkHashPrefix,
-	unsigned int linkHashPrefixLength);
-
-/**
- * Create record, phase two -- add links, sign with ID claim key and owner key
- * 
- * @param rb Record buffer containing the results of createInit()
- * @param links Links (size must be 32*linkCount)
- * @param linkCount Number of links
- * @param ownerPrivateKey Private key to sign this record
- */
-bool ZTLF_Record_createFinal(
-	struct ZTLF_RecordBuffer *rb,
+	const void *ownerPrivateKey,
 	const void *links,
-	const unsigned int linkCount,
-	const void *ownerPrivateKey);
+	unsigned int linkCount,
+	uint64_t timestamp,
+	uint64_t ttl,
+	bool encryptValue,
+	bool (*statusCallback)(uint32_t,uint32_t));
 
 /**
  * Extract record timestamp from record

selftest.c

@@ -54,27 +54,24 @@ bool ZTLF_selftest_wharrgarbl(FILE *o)
 		tmp[i] = (uint8_t)i;
 	uint8_t pow[ZTLF_WHARRGARBL_POW_BYTES];
 
-	/*
 	fprintf(o,"Testing and benchmarking wharrgarbl proof of work..." ZTLF_EOL);
 	fprintf(o,"-----------------------------------------------------" ZTLF_EOL);
 	fprintf(o,"Difficulty (hex)  Threads    Iterations  Avg Time (s)" ZTLF_EOL);
 	fprintf(o,"-----------------------------------------------------" ZTLF_EOL);
 	const unsigned int thr = ZTLF_ncpus();
-	for(int i=10;i<14;++i) {
-		uint32_t diff32 = 1 << i;
+	for(int i=0;i<4;++i) {
 		uint64_t iter = 0;
 		uint64_t start = ZTLF_timeMs();
-		for(int k=0;k<10;++k)
-			iter += ZTLF_wharrgarbl(pow,tmp,sizeof(tmp),diff32,foo,mem,0);
+		for(int k=0;k<5;++k)
+			iter += ZTLF_wharrgarbl(pow,tmp,sizeof(tmp),ZTLF_RECORD_WHARRGARBL_POW_ITERATION_DIFFICULTY,foo,mem,0);
 		uint64_t end = ZTLF_timeMs();
 		if (!ZTLF_wharrgarblVerify(pow,tmp,sizeof(tmp))) {
 			fprintf(o,"FAILED! (verify)" ZTLF_EOL);
 			free(foo);
 			return false;
 		}
-		fprintf(o,"   %12x  %8u  %12llu   %.8f" ZTLF_EOL,diff32,thr,iter / 10,((double)(end - start)) / 10.0 / 1000.0);
+		fprintf(o,"   %12x  %8u  %12llu   %.8f" ZTLF_EOL,ZTLF_RECORD_WHARRGARBL_POW_ITERATION_DIFFICULTY,thr,iter / 5,((double)(end - start)) / 5.0 / 1000.0);
 	}
-	*/
 
 	int icols = -1;
 	char *cols = getenv("COLUMNS");
@@ -83,7 +80,7 @@ bool ZTLF_selftest_wharrgarbl(FILE *o)
 		if (icols <= 0)
 			icols = -1;
 	}
-	icols /= 14;
+	icols /= 15;
 	fprintf(o,ZTLF_EOL "Determining time for record work by record length..." ZTLF_EOL);
 	uint8_t scoringHash[48];
 	double avgTime[ZTLF_RECORD_MAX_SIZE][2];
@@ -97,7 +94,7 @@ bool ZTLF_selftest_wharrgarbl(FILE *o)
 		ZTLF_SHA384(scoringHash,pow,sizeof(pow));
 		const uint32_t score = ZTLF_score(scoringHash);
 
-		uint32_t recordLengthAchieved = score / 61440;
+		uint32_t recordLengthAchieved = score / ZTLF_RECORD_WORK_COST_DIVISOR;
 		if (recordLengthAchieved > ZTLF_RECORD_MAX_SIZE)
 			recordLengthAchieved = ZTLF_RECORD_MAX_SIZE;
 		if (recordLengthAchieved == 0)
@@ -111,14 +108,16 @@ bool ZTLF_selftest_wharrgarbl(FILE *o)
 			avgTime[i][0] += ((double)elapsed) / 1000.0;
 			avgTime[i][1] += 1.0;
 		}
+		for(int i=0;i<16;++i)
+			fprintf(o,ZTLF_EOL);
 		for(unsigned int i=0;i<ZTLF_RECORD_MAX_SIZE;++i) {
-			fprintf(o,"%5u %8.3f",i,(avgTime[i][1] > 0.0) ? (avgTime[i][0] / avgTime[i][1]) : 0.0);
+			fprintf(o,"|%5u %7.2f ",i+1,(avgTime[i][1] > 0.0) ? (avgTime[i][0] / avgTime[i][1]) : 0.0);
 			if (icols > 0) {
 				if ((i % icols) == (icols-1))
 					fprintf(o,ZTLF_EOL);
 			}
 		}
-		fprintf(o,ZTLF_EOL ZTLF_EOL);
+		fprintf(o,ZTLF_EOL);
 	}
 
 	free(foo);

sha.h

@@ -34,6 +34,8 @@
 
 #include <CommonCrypto/CommonDigest.h>
 
+#define ZTLF_HAVE_SHA_IMPL 1
+
 static inline void ZTLF_SHA384(void *d,const void *b,const unsigned long s)
 {
 	CC_SHA512_CTX s384;
@@ -62,34 +64,45 @@ static inline void ZTLF_SHA512(void *d,const void *b,const unsigned long s)
 
 #endif /* __APPLE__ */
 
+#endif
+
+#ifndef ZTLF_HAVE_SHA_IMPL
+
+#include <openssl/sha.h>
+
+static inline void ZTLF_SHA384(void *d,const void *b,const unsigned long s)
+{
+	SHA512_CTX h;
+	SHA384_Init(&h);
+	SHA384_Update(&h,b,(size_t)s);
+	SHA384_Final((unsigned char *)d,&h);
+}
+
+static inline void ZTLF_SHA512(void *d,const void *b,const unsigned long s)
+{
+	SHA512_CTX h;
+	SHA512_Init(&h);
+	SHA512_Update(&h,b,(size_t)s);
+	SHA512_Final((unsigned char *)d,&h);
+}
+
+#define ZTLF_SHA384_CTX SHA512_CTX
+#define ZTLF_SHA384_init(ctx) SHA384_Init(ctx)
+#define ZTLF_SHA384_update(ctx,b,l) SHA384_Update(ctx,(const void *)(b),(size_t)(l))
+#define ZTLF_SHA384_final(ctx,d) SHA384_Final((unsigned char *)(d),ctx)
+
+#define ZTLF_SHA512_CTX SHA512_CTX
+#define ZTLF_SHA512_init(ctx) SHA512_Init(ctx)
+#define ZTLF_SHA512_update(ctx,b,l) SHA512_Update(ctx,(const void *)(b),(size_t)(l))
+#define ZTLF_SHA512_final(ctx,d) SHA512_Final((unsigned char *)(d),ctx)
+
+#endif
+
 static inline void ZTLF_Shandwich256(void *d,const void *b,const unsigned long s)
 {
 	uint64_t s512a[8],s512b[8];
 	ZTLF_SHA512(s512a,b,s);
 	ZTLF_SHA512(s512b,s512a,64);
 	ZTLF_AES256ECB_encrypt(s512a,d,s512b);
-	ZTLF_AES256ECB_encrypt(s512a + 2,((uint8_t *)d) + 16,s512b + 2);
+	ZTLF_AES256ECB_encrypt(s512a + 4,((uint8_t *)d) + 16,s512b + 2);
 }
-
-static inline void ZTLF_Shandwich384(void *d,const void *b,const unsigned long s)
-{
-	uint64_t s512a[8],s512b[8];
-	ZTLF_SHA512(s512a,b,s);
-	ZTLF_SHA512(s512b,s512a,64);
-	ZTLF_AES256ECB_encrypt(s512a,d,s512b);
-	ZTLF_AES256ECB_encrypt(s512a + 2,((uint8_t *)d) + 16,s512b + 2);
-	ZTLF_AES256ECB_encrypt(s512a + 4,((uint8_t *)d) + 32,s512b + 4);
-}
-
-static inline void ZTLF_Shandwich512(void *d,const void *b,const unsigned long s)
-{
-	uint64_t s512a[8],s512b[8];
-	ZTLF_SHA512(s512a,b,s);
-	ZTLF_SHA512(s512b,s512a,64);
-	ZTLF_AES256ECB_encrypt(s512a,d,s512b);
-	ZTLF_AES256ECB_encrypt(s512a + 2,((uint8_t *)d) + 16,s512b + 2);
-	ZTLF_AES256ECB_encrypt(s512a + 4,((uint8_t *)d) + 32,s512b + 4);
-	ZTLF_AES256ECB_encrypt(s512a + 6,((uint8_t *)d) + 48,s512b + 6);
-}
-
-#endif