Subversion Repositories planix.SVN

#include <u.h>

#include <libc.h>

#include <bio.h>

#include <ctype.h>

#include "code.h"

/* read an annotated spelling list in the form

	word <tab> affixcode [ , affixcode ] ...

   print a reencoded version

	octal <tab> word

 */

typedef	struct	Dict	Dict;

struct	Dict

{

	char*	word;

	int	encode;

};

Dict	words[200000];

char	space[500000];

long	encodes[4094];

long	nspace;

long	nwords;

int	ncodes;

Biobuf	bout;

void	readinput(int f);

long	typecode(char *str);

int	wcmp(void*, void*);

void	pdict(void);

void	sput(int);

void

main(int argc, char *argv[])

{

	int f;

	Binit(&bout, 1, OWRITE);

	nwords = 0;

	nspace = 0;

	ncodes = 0;

	if(argc <= 1)

		readinput(0);

	while(argc > 1) {

		f = open(argv[1], 0);

		if(f < 0) {

			fprint(2, "Cannot open %s\n", argv[1]);

			exits("open");

		}

		readinput(f);

		argc--;

		argv++;

	}

	fprint(2, "words = %ld; space = %ld; codes = %d\n",

		nwords, nspace, ncodes);

	qsort(words, nwords, sizeof(words[0]), wcmp);

	pdict();

	exits(0);

}

wcmp(void *a, void *b)

{

	return strcmp(((Dict*)a)->word, ((Dict*)b)->word);

}

void

readinput(int f)

{

	long i;

	char *code, *line, *bword;

	Biobuf buf;

	long lineno = 0;

	Binit(&buf, f, OREAD);

	while(line = Brdline(&buf, '\n')) {

		line[Blinelen(&buf)-1] = 0;

		lineno++;

		code = line;

		while(isspace(*code))

			code++;

		bword = code;

		while(*code && !isspace(*code))

			code++;

		i = code-bword;

		memmove(space+nspace, bword, i);

		words[nwords].word = space+nspace;

		nspace += i;

		space[nspace] = 0;

		nspace++;

		if(*code) {

			*code++ = 0;

			while(isspace(*code))

				code++;

		}

		words[nwords].encode = typecode(code);

		nwords++;

		if(nwords >= sizeof(words)/sizeof(words[0])) {

			fprint(2, "words array too small\n");

			exits("words");

		}

		if(nspace >= sizeof(space)/sizeof(space[0])) {

			fprint(2, "space array too small\n");

			exits("space");

		}

	}

	Bterm(&buf);

}

typedef	struct	Class	Class;

struct	Class

{

	char*	codename;

	long	bits;

};

Class	codea[]  =

{

	{ "a", ADJ },

	{ "adv", ADV },

};

Class	codec[] =

{

	{ "comp", COMP },

};

Class	coded[] =

{

	{ "d", DONT_TOUCH},

};

Class	codee[] =

{

	{ "ed",	ED },

	{ "er", ACTOR },

};

Class	codei[] =

{

	{ "in", IN },

	{ "ion", ION },

};

Class	codem[] =

{

	{ "man", MAN },

	{ "ms", MONO },

};

Class	coden[] =

{

	{ "n", NOUN },

	{ "na", N_AFFIX },

	{ "nopref", NOPREF },

};

Class	codep[] =

{

	{ "pc", PROP_COLLECT },

};

Class	codes[] =

{

	{ "s", STOP },

};

Class	codev[] =

{

	{ "v", VERB },

	{ "va", V_AFFIX },

	{ "vi", V_IRREG },

};

Class	codey[] =

{

	{ "y", _Y },

};

Class	codez[] =

{

};

Class*	codetab[] =

{

	codea,

	codez,

	codec,

	coded,

	codee,

	codez,

	codez,

	codez,

	codei,

	codez,

	codez,

	codez,

	codem,

	coden,

	codez,

	codep,

	codez,

	codez,

	codes,

	codez,

	codez,

	codev,

	codez,

	codez,

	codey,

	codez,

};

long

typecode(char *str)

{

	Class *p;

	long code;

	int n, i;

	char *s, *sp, *st;

	code = 0;

loop:

	for(s=str; *s != 0 && *s != ','; s++)

		;

	for(p = codetab[*str-'a']; sp = p->codename; p++) {

		st = str;

		for(n=s-str;; st++,sp++) {

			if(*st != *sp)

				goto cont;

			n--;

			if(n == 0)

				break;

		}

		code |= p->bits;

		if(*s == 0)

			goto out;

		str = s+1;

		goto loop;

	cont:;

	}

	fprint(2, "Unknown affix code \"%s\"\n", str);

	return 0;

out:

	for(i=0; i<ncodes; i++)

		if(encodes[i] == code)

			return i;

	encodes[i] = code;

	ncodes++;

	return i;

}

void

sput(int s)

{

	Bputc(&bout, s>>8);

	Bputc(&bout, s);

}

void

lput(long l)

{

	Bputc(&bout, l>>24);

	Bputc(&bout, l>>16);

	Bputc(&bout, l>>8);

	Bputc(&bout, l);

}

/*

 * spit out the encoded dictionary

 * all numbers are encoded big-endian.

 *	struct

 *	{

 *		short	ncodes;

 *		long	encodes[ncodes];

 *		struct

 *		{

 *			short	encode;

 *			char	word[*];

 *		} words[*];

 *	};

 * 0x8000 flag for code word

 * 0x7800 count of number of common bytes with previous word

 * 0x07ff index into codes array for affixes

 */

void

pdict(void)

{

	long i, count;

	int encode, j, c;

	char *lastword, *thisword, *word;

	sput(ncodes);

	for(i=0; i<ncodes; i++)

		lput(encodes[i]);

	count = ncodes*4 + 2;

	lastword = "";

	for(i=0; i<nwords; i++) {

		word = words[i].word;

		thisword = word;

		for(j=0; *thisword == *lastword; j++) {

			if(*thisword == 0) {

				fprint(2, "identical words: %s\n", word);

				break;

			}

			thisword++;

			lastword++;

		}

		if(j > 15)

			j = 15;

		encode = words[i].encode;

		c = (1<<15) | (j<<11) | encode;

		sput(c);

		count += 2;

		for(thisword=word+j; c = *thisword; thisword++) {

			Bputc(&bout, c);

			count++;

		}

		lastword = word;

	}

	fprint(2, "output bytes = %ld\n", count);

}