Subversion Repositories planix.SVN

#include <u.h>

#include <libc.h>

#include <bio.h>

#include <ctype.h>

#include "msgdb.h"

void

usage(void)

{

	fprint(2, "usage: upas/msgclass [-a] [-d name dbfile]... [-l lockfile] [-m mul] [-t thresh] [tokenfile ...]\n");

	exits("usage");

}

enum

{

	MAXBEST = 32,

	MAXLEN = 64,

	MAXTAB = 256,

};

typedef struct Ndb Ndb;

struct Ndb

{

	char *name;

	char *file;

	Msgdb *db;

	double p;

	long nmsg;

};

typedef struct Word Word;

struct Word

{

	char s[MAXLEN];

	int count[MAXTAB];

	double p[MAXTAB];

	double mp;

	int mi; /* w.p[w.mi] = w.mp */

	int nmsg;

};

Ndb db[MAXTAB];

int ndb;

int add;

int mul;

Msgdb *indb;

Word best[MAXBEST];

int mbest = 15;

int nbest;

void process(Biobuf*, char*);

void lockfile(char*);

void

noteword(Word *w, char *s)

{

	int i;

	for(i=nbest-1; i>=0; i--)

		if(w->mp < best[i].mp)

			break;

	i++;

	if(i >= mbest)

		return;

	if(nbest == mbest)

		nbest--;

	if(i < nbest)

		memmove(&best[i+1], &best[i], (nbest-i)*sizeof(best[0]));

	best[i] = *w;

	strecpy(best[i].s, best[i].s+MAXLEN, s);

	nbest++;

}

void

main(int argc, char **argv)

{

	int i, bad, m, tot, nn, j;

	Biobuf bin, *b, bout;

	char *s, *lf;

	double totp, p, thresh;

	long n;

	Word w;

	lf = nil;

	thresh = 0;

	ARGBEGIN{

	case 'a':

		add = 1;

		break;

	case 'd':

		if(ndb >= MAXTAB)

			sysfatal("too many db classes");

		db[ndb].name = EARGF(usage());

		db[ndb].file = EARGF(usage());

		ndb++;

		break;

	case 'l':

		lf = EARGF(usage());

		break;

	case 'm':

		mul = atoi(EARGF(usage()));

		break;

	case 't':

		thresh = atof(EARGF(usage()));

		break;

	default:

		usage();

	}ARGEND

	if(ndb == 0){

		fprint(2, "must have at least one -d option\n");

		usage();

	}

	indb = mdopen(nil, 1);

	if(argc == 0){

		Binit(&bin, 0, OREAD);

		process(&bin, "<stdin>");

		Bterm(&bin);

	}else{

		bad = 0;

		for(i=0; i<argc; i++){

			if((b = Bopen(argv[i], OREAD)) == nil){

				fprint(2, "opening %s: %r\n", argv[i]);

				bad = 1;

				continue;

			}

			process(b, argv[i]);

			Bterm(b);

		}

		if(bad)

			exits("open inputs");

	}

	lockfile(lf);

	bad = 0;

	for(i=0; i<ndb; i++){

		if((db[i].db = mdopen(db[i].file, 0)) == nil){

			fprint(2, "opendb %s: %r\n", db[i].file);

			bad = 1;

		}

		db[i].nmsg = mdget(db[i].db, "*From*");

	}

	if(bad)

		exits("open databases");

	/* run conditional probabilities of input words, getting 15 most specific */

	mdenum(indb);

	nbest = 0;

	while(mdnext(indb, &s, &n) >= 0){

		tot = 0;

		totp = 0.0;

		for(i=0; i<ndb; i++){

			nn = mdget(db[i].db, s)*(i==0 ? 3 : 1);

			tot += nn;

			w.count[i] = nn;

			p = w.count[i]/(double)db[i].nmsg;

			if(p >= 1.0)

				p = 1.0;

			w.p[i] = p;

			totp += p;

		}

//fprint(2, "%s tot %d totp %g\n", s, tot, totp);

		if(tot < 2)

			continue;

		w.mp = 0.0;

		for(i=0; i<ndb; i++){

			p = w.p[i];

			p /= totp;

			if(p < 0.001)

				p = 0.001;

			else if(p > 0.999)

				p = 0.999;

			if(p > w.mp){

				w.mp = p;

				w.mi = i;

			}

			w.p[i] = p;

		}

		noteword(&w, s);

	}

	/* compute conditional probabilities of message classes using 15 most specific */

	totp = 0.0;

	for(i=0; i<ndb; i++){

		p = 1.0;

		for(j=0; j<nbest; j++)

			p *= best[j].p[i];

		db[i].p = p;

		totp += p;

	}

	for(i=0; i<ndb; i++)

		db[i].p /= totp;

	m = 0;

	for(i=1; i<ndb; i++)

		if(db[i].p > db[m].p)

			m = i;

	Binit(&bout, 1, OWRITE);

	if(db[m].p < thresh)

		m = -1;

	if(m >= 0)

		Bprint(&bout, "%s", db[m].name);

	else

		Bprint(&bout, "inconclusive");

	for(j=0; j<ndb; j++)

		Bprint(&bout, " %s=%g", db[j].name, db[j].p);

	Bprint(&bout, "\n");

	for(i=0; i<nbest; i++){

		Bprint(&bout, "%s", best[i].s);

		for(j=0; j<ndb; j++)

			Bprint(&bout, " %s=%g", db[j].name, best[i].p[j]);

		Bprint(&bout, "\n");

	}

		Bprint(&bout, "%s %g\n", best[i].s, best[i].p[m]);

	Bterm(&bout);

	if(m >= 0 && add){

		mdenum(indb);

		while(mdnext(indb, &s, &n) >= 0)

			mdput(db[m].db, s, mdget(db[m].db, s)+n*mul);

		mdclose(db[m].db);

	}

	exits(nil);

}

void

process(Biobuf *b, char*)

{

	char *s;

	char *p;

	long n;

	while((s = Brdline(b, '\n')) != nil){

		s[Blinelen(b)-1] = 0;

		if((p = strrchr(s, ' ')) != nil){

			*p++ = 0;

			n = atoi(p);

		}else

			n = 1;

		mdput(indb, s, mdget(indb, s)+n);

	}

}

int tpid;

void

killtickle(void)

{

	postnote(PNPROC, tpid, "die");

}

void

lockfile(char *s)

{

	int fd, t, w;

	char err[ERRMAX];

	if(s == nil)

		return;

	w = 50;

	t = 0;

	for(;;){

		fd = open(s, OREAD);

		if(fd >= 0)

			break;

		rerrstr(err, sizeof err);

		if(strstr(err, "file is locked")==nil && strstr(err, "exclusive lock")==nil))

			break;

		sleep(w);

		t += w;

		if(w < 1000)

			w = (w*3)/2;

		if(t > 120*1000)

			break;

	}

	if(fd < 0)

		sysfatal("could not lock %s", s);

	switch(tpid = fork()){

	case -1:

		sysfatal("fork: %r");

	case 0:

		for(;;){

			sleep(30*1000);

			free(dirfstat(fd));

		}

		_exits(nil);

	default:

		break;

	}

	close(fd);

	atexit(killtickle);

}