void/ZeroTierOne
2
0
Fork 0
mirror of https://github.com/zerotier/ZeroTierOne.git synced 2025-06-07 13:03:45 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Dump huffman, doesnt add much and complicates porting to other languages. Also fix compile error in idtool.

Browse source
This commit is contained in:
Adam Ierymenko 2013-07-27 14:01:19 -04:00
parent e6e825da70
commit a816f56426
6 changed files with 8 additions and 364 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

252
ext/huffandpuff/huffman.c
View file

@ -1,252 +0,0 @@
/*
* Huffandpuff minimal Huffman coder
*
* (c)2013 Adam Ierymenko <adam.ierymenko@zerotier.com>
* This code is in the public domain and is distributed with NO WARRANTY.
*/
#include "huffman.h"
struct _huffman_node
{
struct _huffman_node *lr[2];
struct _huffman_node *qprev,*qnext;
double prob;
unsigned long c;
};
struct _huffman_encode_table
{
unsigned long code;
unsigned long bits;
};
static void _huffman_write_tree_and_make_encode_table(unsigned char *out,unsigned long *outbitctr,unsigned long outlen,struct _huffman_encode_table *et,unsigned long code,unsigned int bits,struct _huffman_node *t)
{
struct _huffman_encode_table *eti;
unsigned int i;
unsigned long byte_index;
byte_index = (*outbitctr)++ >> 3;
byte_index *= (byte_index < outlen);
if (t->lr[0]) {
out[byte_index] <<= 1;
_huffman_write_tree_and_make_encode_table(out,outbitctr,outlen,et,code,bits + 1,t->lr[0]);
_huffman_write_tree_and_make_encode_table(out,outbitctr,outlen,et,code | (1 << bits),bits + 1,t->lr[1]);
} else {
out[byte_index] = (out[byte_index] << 1) | 1;
for(i=0;i<9;++i) {
byte_index = (*outbitctr)++ >> 3;
if (byte_index >= outlen) return;
out[byte_index] = (out[byte_index] << 1) | ((unsigned char)((t->c >> i) & 1));
}
eti = &(et[t->c]);
eti->code = code;
eti->bits = bits;
}
}
static struct _huffman_node *_huffman_read_tree(const unsigned char *in,unsigned long *inbitctr,unsigned long inlen,unsigned char **heapptr,unsigned char *heapend)
{
struct _huffman_node *n;
unsigned int i;
unsigned long byte_index;
n = (struct _huffman_node *)(*heapptr);
*heapptr += sizeof(struct _huffman_node);
if (*heapptr > heapend) return (struct _huffman_node *)0;
byte_index = *inbitctr >> 3;
byte_index *= (byte_index < inlen);
if (((in[byte_index] >> (~((*inbitctr)++) & 7)) & 1)) {
n->lr[0] = (struct _huffman_node *)0;
n->lr[1] = (struct _huffman_node *)0;
n->c = 0;
for(i=0;i<9;++i) {
byte_index = *inbitctr >> 3;
if (byte_index >= inlen) return (struct _huffman_node *)0;
n->c |= (((unsigned int)(in[byte_index] >> (~((*inbitctr)++) & 7))) & 1) << i;
}
} else {
n->lr[0] = _huffman_read_tree(in,inbitctr,inlen,heapptr,heapend);
n->lr[1] = _huffman_read_tree(in,inbitctr,inlen,heapptr,heapend);
if (!((n->lr[0])&&(n->lr[1])))
return (struct _huffman_node *)0;
}
return n;
}
unsigned long huffman_compress(const unsigned char *in,unsigned long inlen,unsigned char *out,unsigned long outlen,void *huffheap)
{
struct _huffman_encode_table *et,*eti;
struct _huffman_node *t,*n;
struct _huffman_node *pair[2];
unsigned char *heapptr = (unsigned char *)huffheap;
unsigned long i,code,byte_index,outbitctr;
unsigned int bits,b;
double *counts,lowest_prob,total_symbols;
counts = (double *)heapptr;
heapptr += (sizeof(double) * 257);
for(i=0;i<256;++i)
counts[i] = 0.0;
counts[256] = 1.0; /* one stop code at end */
for(i=0;i<inlen;++i)
counts[(unsigned long)in[i]] += 1.0;
t = (struct _huffman_node *)0;
total_symbols = (double)(inlen + 1);
for(i=0;i<=256;++i) {
if (counts[i] > 0.0) {
n = (struct _huffman_node *)heapptr;
heapptr += sizeof(struct _huffman_node);
if (t)
t->qprev = n;
n->qprev = (struct _huffman_node *)0;
n->qnext = t;
n->lr[0] = (struct _huffman_node *)0;
n->lr[1] = (struct _huffman_node *)0;
n->prob = counts[i] / total_symbols;
n->c = (unsigned int)i;
t = n;
}
}
while (t->qnext) {
for(i=0;i<2;++i) {
lowest_prob = 1.0;
pair[i] = (struct _huffman_node *)0;
n = t;
while (n) {
if (n->prob <= lowest_prob) {
lowest_prob = n->prob;
pair[i] = n;
}
n = n->qnext;
}
if (pair[i]->qprev)
pair[i]->qprev->qnext = pair[i]->qnext;
else t = pair[i]->qnext;
if (pair[i]->qnext)
pair[i]->qnext->qprev = pair[i]->qprev;
}
n = (struct _huffman_node *)heapptr;
heapptr += sizeof(struct _huffman_node);
n->lr[0] = pair[0];
n->lr[1] = pair[1];
n->prob = pair[0]->prob + pair[1]->prob;
if (t)
t->qprev = n;
n->qprev = (struct _huffman_node *)0;
n->qnext = t;
t = n;
}
et = (struct _huffman_encode_table *)heapptr;
heapptr += (sizeof(struct _huffman_encode_table) * 257);
outbitctr = 0;
_huffman_write_tree_and_make_encode_table(out,&outbitctr,outlen,et,0,0,t);
for(i=0;i<inlen;++i) {
eti = &(et[(unsigned long)in[i]]);
code = eti->code;
bits = eti->bits;
for(b=0;b<bits;++b) {
byte_index = outbitctr++ >> 3;
if (byte_index >= outlen) return 0;
out[byte_index] = (out[byte_index] << 1) | (unsigned char)(code & 1);
code >>= 1;
}
}
code = et[256].code;
bits = et[256].bits;
for(b=0;b<bits;++b) {
byte_index = outbitctr++ >> 3;
if (byte_index >= outlen) return 0;
out[byte_index] = (out[byte_index] << 1) | (unsigned char)(code & 1);
code >>= 1;
}
if (outbitctr > (outlen << 3))
return 0;
else if ((outbitctr & 7)) {
out[i = (outbitctr >> 3)] <<= 8 - (outbitctr & 7);
return (i + 1);
} else return (outbitctr >> 3);
}
unsigned long huffman_decompress(const unsigned char *in,unsigned long inlen,unsigned char *out,unsigned long outlen,void *huffheap)
{
struct _huffman_node *t,*n;
unsigned char *heapptr = (unsigned char *)huffheap;
unsigned long inbitctr,outptr,byte_index = 0;
inbitctr = 0;
t = _huffman_read_tree(in,&inbitctr,inlen,&heapptr,heapptr + HUFFHEAP_SIZE);
if (!t) return 0;
outptr = 0;
for(;;) {
n = t;
while (n->lr[0]) {
byte_index = inbitctr >> 3;
if (byte_index >= inlen) return 0;
n = n->lr[((unsigned long)(in[byte_index] >> (~(inbitctr++) & 7))) & 1];
}
if (n->c == 256) return outptr;
if (outptr == outlen) return 0;
out[outptr++] = (unsigned char)n->c;
}
}
#ifdef HUFFANDPUFF_TEST
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#define HUFFANDPUFF_TEST_MAXLEN 1048576
#define HUFFANDPUFF_TEST_ITER 1024
static unsigned char testin[HUFFANDPUFF_TEST_MAXLEN];
static unsigned char testout[HUFFANDPUFF_TEST_MAXLEN * 2];
static unsigned char testver[HUFFANDPUFF_TEST_MAXLEN];
static unsigned char huffbuf[HUFFHEAP_SIZE];
int main(int argc,char **argv)
{
unsigned long i,k,l,cl,dcl;
int v;
unsigned char mask;
srand(time(0));
for(k=0;k<HUFFANDPUFF_TEST_ITER;++k) {
l = (rand() % HUFFANDPUFF_TEST_MAXLEN) + 1;
mask = (rand() & 0xff);
for(i=0;i<l;++i)
testin[i] = (unsigned char)(rand() & 0xff) & mask;
cl = huffman_compress(testin,l,testout,sizeof(testout),huffbuf);
if (cl) {
memset(testver,0,sizeof(testver));
dcl = huffman_decompress(testout,cl,testver,sizeof(testver),huffbuf);
v = ((dcl)&&(!memcmp(testver,testin,l)));
printf("[%d] in: %d, out: %d, verified: %s\n",(int)k,(int)l,(int)cl,(v) ? "OK" : "FAIL");
} else printf("[%d] in: %d, out: FAIL\n",(int)k,(int)l);
}
printf("\nFuzzing decompress function...\n");
for(;;) {
l = (rand() % HUFFANDPUFF_TEST_MAXLEN) + 1;
mask = (rand() & 0xff);
for(i=0;i<l;++i)
testin[i] = (unsigned char)(rand() & 0xff) & mask;
huffman_decompress(testin,l,testver,sizeof(testver),huffbuf);
printf("."); fflush(stdout);
}
return 0;
}
#endif

52
ext/huffandpuff/huffman.h
View file

@ -1,52 +0,0 @@
/*
* Huffandpuff minimal Huffman coder
*
* (c)2013 Adam Ierymenko <adam.ierymenko@zerotier.com>
* This code is in the public domain and is distributed with NO WARRANTY.
*/
#ifndef ____HUFFMAN_H
#define ____HUFFMAN_H
#ifdef __cplusplus
extern "C" {
#endif
/**
* Required size of huffheap parameter to compress and decompress
*
* Note: if you change any of the data types in the _huffman_node
* or _huffman_encode_table structs in huffman.c, this also must be
* changed.
*/
#define HUFFHEAP_SIZE ((sizeof(double) * 257) + (((sizeof(void *) * 4) + sizeof(double) + sizeof(unsigned long)) * (257 * 3)) + ((sizeof(unsigned long) + sizeof(unsigned long)) * 257))
/**
* Huffman encode a block of data
*
* @param in Input data
* @param inlen Input data length
* @param out Output buffer
* @param outlen Output buffer length
* @param huffheap Heap memory to use for compression (must be HUFFHEAP_SIZE in size)
* @return Size of encoded result or 0 on out buffer overrun
*/
extern unsigned long huffman_compress(const unsigned char *in,unsigned long inlen,unsigned char *out,unsigned long outlen,void *huffheap);
/**
* Huffman decode a block of data
*
* @param in Input data
* @param inlen Length of input data
* @param out Output buffer
* @param outlen Length of output buffer
* @param huffheap Heap memory to use for decompression (must be HUFFHEAP_SIZE in size)
* @return Size of decoded result or 0 on out buffer overrun or corrupt input data
*/
extern unsigned long huffman_decompress(const unsigned char *in,unsigned long inlen,unsigned char *out,unsigned long outlen,void *huffheap);
#ifdef __cplusplus
}
#endif
#endif

2
idtool.cpp
View file

@ -159,7 +159,7 @@ int main(int argc,char **argv)
} }
std::string signature(Utils::base64Decode(argv[4],strlen(argv[4]))); std::string signature(Utils::base64Decode(argv[4],strlen(argv[4])));
if ((signature.length() > ZEROTIER_ADDRESS_LENGTH)&&(id.verifySignature(inf.data(),inf.length(),signature))) { if ((signature.length() > ZT_ADDRESS_LENGTH)&&(id.verifySignature(inf.data(),inf.length(),signature.data(),signature.length()))) {
std::cout << argv[3] << " signature valid" << std::endl; std::cout << argv[3] << " signature valid" << std::endl;
} else { } else {
std::cerr << argv[3] << " signature check FAILED" << std::endl; std::cerr << argv[3] << " signature check FAILED" << std::endl;

23
node/Utils.hpp
View file

@ -42,7 +42,6 @@
#include "../ext/lz4/lz4.h" #include "../ext/lz4/lz4.h"
#include "../ext/lz4/lz4hc.h" #include "../ext/lz4/lz4hc.h"
#include "../ext/huffandpuff/huffman.h"
#include "Constants.hpp" #include "Constants.hpp"
@ -176,7 +175,6 @@ public:
template<typename I,typename O> template<typename I,typename O>
static inline void compress(I begin,I end,O out) static inline void compress(I begin,I end,O out)
{ {
char huffheap[HUFFHEAP_SIZE];
unsigned int bufLen = LZ4_compressBound(ZT_COMPRESSION_BLOCK_SIZE); unsigned int bufLen = LZ4_compressBound(ZT_COMPRESSION_BLOCK_SIZE);
char *buf = new char[bufLen * 2]; char *buf = new char[bufLen * 2];
char *buf2 = buf + bufLen; char *buf2 = buf + bufLen;
@ -210,16 +208,9 @@ public:
continue; continue;
} }
unsigned long huffCompressedLen = huffman_compress((const unsigned char *)buf2,lz4CompressedLen,(unsigned char *)buf,bufLen,huffheap);
if ((!huffCompressedLen)||((int)huffCompressedLen >= lz4CompressedLen)) {
l = hton((uint32_t)lz4CompressedLen); // lz4 only l = hton((uint32_t)lz4CompressedLen); // lz4 only
out((const void *)&l,4); out((const void *)&l,4);
out((const void *)buf2,(unsigned int)lz4CompressedLen); out((const void *)buf2,(unsigned int)lz4CompressedLen);
} else {
l = hton((uint32_t)0x80000000 | (uint32_t)huffCompressedLen); // lz4 with huffman
out((const void *)&l,4);
out((const void *)buf,(unsigned int)huffCompressedLen);
}
} }
delete [] buf; delete [] buf;
@ -242,7 +233,6 @@ public:
template<typename I,typename O> template<typename I,typename O>
static inline bool decompress(I begin,I end,O out) static inline bool decompress(I begin,I end,O out)
{ {
char huffheap[HUFFHEAP_SIZE];
volatile char i32c[4]; volatile char i32c[4];
void *const i32cp = (void *)i32c; void *const i32cp = (void *)i32c;
unsigned int bufLen = LZ4_compressBound(ZT_COMPRESSION_BLOCK_SIZE); unsigned int bufLen = LZ4_compressBound(ZT_COMPRESSION_BLOCK_SIZE);
@ -279,23 +269,10 @@ public:
return false; return false;
} }
if ((_compressedSize & 0x80000000)) { // lz4 and huffman
unsigned long lz4CompressedSize = huffman_decompress((const unsigned char *)buf,compressedSize,(unsigned char *)buf2,bufLen,huffheap);
if (lz4CompressedSize) {
if (LZ4_uncompress_unknownOutputSize(buf2,buf,lz4CompressedSize,bufLen) != (int)originalSize) {
delete [] buf;
return false;
} else out((const void *)buf,(unsigned int)originalSize);
} else {
delete [] buf;
return false;
}
} else { // lz4 only
if (LZ4_uncompress_unknownOutputSize(buf,buf2,compressedSize,bufLen) != (int)originalSize) { if (LZ4_uncompress_unknownOutputSize(buf,buf2,compressedSize,bufLen) != (int)originalSize) {
delete [] buf; delete [] buf;
return false; return false;
} else out((const void *)buf2,(unsigned int)originalSize); } else out((const void *)buf2,(unsigned int)originalSize);
}
} else { // stored } else { // stored
if (originalSize > bufLen) { if (originalSize > bufLen) {
delete [] buf; delete [] buf;

1
objects.mk
View file

@ -1,5 +1,4 @@
OBJS=\ OBJS=\
ext/huffandpuff/huffman.o \
ext/kissdb/kissdb.o \ ext/kissdb/kissdb.o \
ext/lz4/lz4hc.o \ ext/lz4/lz4hc.o \
ext/lz4/lz4.o \ ext/lz4/lz4.o \

28
selftest.cpp
View file

@ -43,7 +43,6 @@
#include "node/HMAC.hpp" #include "node/HMAC.hpp"
#include "node/MAC.hpp" #include "node/MAC.hpp"
#include "node/Peer.hpp" #include "node/Peer.hpp"
#include "node/Http.hpp"
#include "node/Condition.hpp" #include "node/Condition.hpp"
#include "node/NodeConfig.hpp" #include "node/NodeConfig.hpp"
@ -302,39 +301,12 @@ static int testOther()
return 0; return 0;
} }
static Condition testHttpDoneCondition;
static bool testHttpHandler(Http::Request *req,void *arg,const std::string &url,int code,const std::map<std::string,std::string> &headers,const std::string &body)
{
if (code)
std::cout << "[net] " << url << " " << code << " bytes: " << body.length() << std::endl;
else std::cout << "[net] " << url << " FAILED: " << body << std::endl;
testHttpDoneCondition.signal();
return false;
}
static int testNet()
{
std::cout << "[net] GET http://www.uc.edu/" << std::endl;
new Http::Request(Http::HTTP_METHOD_GET,"http://www.uc.edu/",Http::EMPTY_HEADERS,std::string(),&testHttpHandler,(void *)0);
testHttpDoneCondition.wait();
std::cout << "[net] GET http://zerotier.com/" << std::endl;
new Http::Request(Http::HTTP_METHOD_GET,"http://zerotier.com/",Http::EMPTY_HEADERS,std::string(),&testHttpHandler,(void *)0);
testHttpDoneCondition.wait();
std::cout << "[net] GET http://www.google.com/" << std::endl;
new Http::Request(Http::HTTP_METHOD_GET,"http://www.google.com/",Http::EMPTY_HEADERS,std::string(),&testHttpHandler,(void *)0);
testHttpDoneCondition.wait();
return 0;
}
int main(int argc,char **argv) int main(int argc,char **argv)
{ {
int r = 0; int r = 0;
srand(time(0)); srand(time(0));
r |= testNet();
r |= testCrypto(); r |= testCrypto();
r |= testPacket(); r |= testPacket();
r |= testOther(); r |= testOther();