Subversion Repositories planix.SVN

Rev

Details | Last modification | View Log | RSS feed

Rev Author Line No. Line
2 - 1
#include "stdinc.h"
2
#include "dat.h"
3
#include "fns.h"
4
 
5
/*
6
 * disk structure conversion routines
7
 */
8
#define	U8GET(p)	((p)[0])
9
#define	U16GET(p)	(((p)[0]<<8)|(p)[1])
10
#define	U32GET(p)	((u32int)(((p)[0]<<24)|((p)[1]<<16)|((p)[2]<<8)|(p)[3]))
11
#define	U64GET(p)	(((u64int)U32GET(p)<<32)|(u64int)U32GET((p)+4))
12
 
13
#define	U8PUT(p,v)	(p)[0]=(v)&0xFF
14
#define	U16PUT(p,v)	(p)[0]=((v)>>8)&0xFF;(p)[1]=(v)&0xFF
15
#define	U32PUT(p,v)	(p)[0]=((v)>>24)&0xFF;(p)[1]=((v)>>16)&0xFF;(p)[2]=((v)>>8)&0xFF;(p)[3]=(v)&0xFF
16
#define	U64PUT(p,v,t32)	t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32)
17
 
18
int debugarena = -1;		/* hack to improve error reporting */
19
 
20
static struct {
21
	u32int m;
22
	char *s;
23
} magics[] = {
24
	ArenaPartMagic, "ArenaPartMagic",
25
	ArenaHeadMagic, "ArenaHeadMagic",
26
	ArenaMagic, "ArenaMagic",
27
	ISectMagic, "ISectMagic",
28
	BloomMagic, "BloomMagic",
29
};
30
 
31
static char*
32
fmtmagic(char *s, u32int m)
33
{
34
	int i;
35
 
36
	for(i=0; i<nelem(magics); i++)
37
		if(magics[i].m == m)
38
			return magics[i].s;
39
	sprint(s, "%#08ux", m);
40
	return s;
41
}
42
 
43
u32int
44
unpackmagic(u8int *buf)
45
{
46
	return U32GET(buf);
47
}
48
 
49
void
50
packmagic(u32int magic, u8int *buf)
51
{
52
	U32PUT(buf, magic);
53
}
54
 
55
int
56
unpackarenapart(ArenaPart *ap, u8int *buf)
57
{
58
	u8int *p;
59
	u32int m;
60
	char fbuf[20];
61
 
62
	p = buf;
63
 
64
	m = U32GET(p);
65
	if(m != ArenaPartMagic){
66
		seterr(ECorrupt, "arena set has wrong magic number: %s expected ArenaPartMagic (%#lux)", fmtmagic(fbuf, m), ArenaPartMagic);
67
		return -1;
68
	}
69
	p += U32Size;
70
	ap->version = U32GET(p);
71
	p += U32Size;
72
	ap->blocksize = U32GET(p);
73
	p += U32Size;
74
	ap->arenabase = U32GET(p);
75
	p += U32Size;
76
 
77
	if(buf + ArenaPartSize != p)
78
		sysfatal("unpackarenapart unpacked wrong amount");
79
 
80
	return 0;
81
}
82
 
83
int
84
packarenapart(ArenaPart *ap, u8int *buf)
85
{
86
	u8int *p;
87
 
88
	p = buf;
89
 
90
	U32PUT(p, ArenaPartMagic);
91
	p += U32Size;
92
	U32PUT(p, ap->version);
93
	p += U32Size;
94
	U32PUT(p, ap->blocksize);
95
	p += U32Size;
96
	U32PUT(p, ap->arenabase);
97
	p += U32Size;
98
 
99
	if(buf + ArenaPartSize != p)
100
		sysfatal("packarenapart packed wrong amount");
101
 
102
	return 0;
103
}
104
 
105
int
106
unpackarena(Arena *arena, u8int *buf)
107
{
108
	int sz;
109
	u8int *p;
110
	u32int m;
111
	char fbuf[20];
112
 
113
	p = buf;
114
 
115
	m = U32GET(p);
116
	if(m != ArenaMagic){
117
		seterr(ECorrupt, "arena %d has wrong magic number: %s "
118
			"expected ArenaMagic (%#lux)", debugarena,
119
			fmtmagic(fbuf, m), ArenaMagic);
120
		return -1;
121
	}
122
	p += U32Size;
123
	arena->version = U32GET(p);
124
	p += U32Size;
125
	namecp(arena->name, (char*)p);
126
	p += ANameSize;
127
	arena->diskstats.clumps = U32GET(p);
128
	p += U32Size;
129
	arena->diskstats.cclumps = U32GET(p);
130
	p += U32Size;
131
	arena->ctime = U32GET(p);
132
	p += U32Size;
133
	arena->wtime = U32GET(p);
134
	p += U32Size;
135
	if(arena->version == ArenaVersion5){
136
		arena->clumpmagic = U32GET(p);
137
		p += U32Size;
138
	}
139
	arena->diskstats.used = U64GET(p);
140
	p += U64Size;
141
	arena->diskstats.uncsize = U64GET(p);
142
	p += U64Size;
143
	arena->diskstats.sealed = U8GET(p);
144
	p += U8Size;
145
	switch(arena->version){
146
	case ArenaVersion4:
147
		sz = ArenaSize4;
148
		arena->clumpmagic = _ClumpMagic;
149
		break;
150
	case ArenaVersion5:
151
		sz = ArenaSize5;
152
		break;
153
	default:
154
		seterr(ECorrupt, "arena has bad version number %d", arena->version);
155
		return -1;
156
	}
157
	/*
158
	 * Additional fields for the memstats version of the stats.
159
	 * Diskstats reflects what is committed to the index.
160
	 * Memstats reflects what is in the arena.  Originally intended
161
	 * this to be a version 5 extension, but might as well use for
162
	 * all the existing version 4 arenas too.
163
	 *
164
	 * To maintain backwards compatibility with existing venti
165
	 * installations using the older format, we define that if 
166
	 * memstats == diskstats, then the extension fields are not
167
	 * included (see packarena below).  That is, only partially
168
	 * indexed arenas have these fields.  Fully indexed arenas
169
	 * (in particular, sealed arenas) do not.
170
	 */
171
	if(U8GET(p) == 1){
172
		sz += ArenaSize5a-ArenaSize5;
173
		p += U8Size;
174
		arena->memstats.clumps = U32GET(p);
175
		p += U32Size;
176
		arena->memstats.cclumps = U32GET(p);
177
		p += U32Size;
178
		arena->memstats.used = U64GET(p);
179
		p += U64Size;
180
		arena->memstats.uncsize = U64GET(p);
181
		p += U64Size;
182
		arena->memstats.sealed = U8GET(p);
183
		p += U8Size;
184
 
185
		/*
186
		 * 2008/4/2
187
		 * Packarena (below) used to have a bug in which it would
188
		 * not zero out any existing extension fields when writing
189
		 * the arena metadata.  This would manifest itself as arenas
190
		 * with arena->diskstats.sealed == 1 but arena->memstats.sealed == 0
191
		 * after a server restart.  Because arena->memstats.sealed wouldn't
192
		 * be set, the server might try to fit another block into the arena
193
		 * (and succeed), violating the append-only structure of the log
194
		 * and invalidating any already-computed seal on the arena.
195
		 *
196
		 * It might end up that other fields in arena->memstats end up
197
		 * behind arena->diskstats too, but that would be considerably
198
		 * more rare, and the bug is fixed now.  The case we need to
199
		 * handle is just the sealed mismatch.
200
		 *
201
		 * If we encounter such a bogus arena, fix the sealed field.
202
		 */
203
		if(arena->diskstats.sealed)
204
			arena->memstats.sealed = 1;
205
	}else
206
		arena->memstats = arena->diskstats;
207
	if(buf + sz != p)
208
		sysfatal("unpackarena unpacked wrong amount");
209
 
210
	return 0;
211
}
212
 
213
int
214
packarena(Arena *arena, u8int *buf)
215
{
216
	return _packarena(arena, buf, 0);
217
}
218
 
219
int
220
_packarena(Arena *arena, u8int *buf, int forceext)
221
{
222
	int sz;
223
	u8int *p;
224
	u32int t32;
225
 
226
	switch(arena->version){
227
	case ArenaVersion4:
228
		sz = ArenaSize4;
229
		if(arena->clumpmagic != _ClumpMagic)
230
			fprint(2, "warning: writing old arena tail loses clump magic 0x%lux != 0x%lux\n",
231
				(ulong)arena->clumpmagic, (ulong)_ClumpMagic);
232
		break;
233
	case ArenaVersion5:
234
		sz = ArenaSize5;
235
		break;
236
	default:
237
		sysfatal("packarena unknown version %d", arena->version);
238
		return -1;
239
	}
240
 
241
	p = buf;
242
 
243
	U32PUT(p, ArenaMagic);
244
	p += U32Size;
245
	U32PUT(p, arena->version);
246
	p += U32Size;
247
	namecp((char*)p, arena->name);
248
	p += ANameSize;
249
	U32PUT(p, arena->diskstats.clumps);
250
	p += U32Size;
251
	U32PUT(p, arena->diskstats.cclumps);
252
	p += U32Size;
253
	U32PUT(p, arena->ctime);
254
	p += U32Size;
255
	U32PUT(p, arena->wtime);
256
	p += U32Size;
257
	if(arena->version == ArenaVersion5){
258
		U32PUT(p, arena->clumpmagic);
259
		p += U32Size;
260
	}
261
	U64PUT(p, arena->diskstats.used, t32);
262
	p += U64Size;
263
	U64PUT(p, arena->diskstats.uncsize, t32);
264
	p += U64Size;
265
	U8PUT(p, arena->diskstats.sealed);
266
	p += U8Size;
267
 
268
	/*
269
	 * Extension fields; see above.
270
	 */
271
	if(forceext
272
	|| arena->memstats.clumps != arena->diskstats.clumps
273
	|| arena->memstats.cclumps != arena->diskstats.cclumps
274
	|| arena->memstats.used != arena->diskstats.used
275
	|| arena->memstats.uncsize != arena->diskstats.uncsize
276
	|| arena->memstats.sealed != arena->diskstats.sealed){
277
		sz += ArenaSize5a - ArenaSize5;
278
		U8PUT(p, 1);
279
		p += U8Size;
280
		U32PUT(p, arena->memstats.clumps);
281
		p += U32Size;
282
		U32PUT(p, arena->memstats.cclumps);
283
		p += U32Size;
284
		U64PUT(p, arena->memstats.used, t32);	
285
		p += U64Size;
286
		U64PUT(p, arena->memstats.uncsize, t32);
287
		p += U64Size;
288
		U8PUT(p, arena->memstats.sealed);
289
		p += U8Size;
290
	}else{
291
		/* Clear any extension fields already on disk. */
292
		memset(p, 0, ArenaSize5a - ArenaSize5);
293
		p += ArenaSize5a - ArenaSize5;
294
		sz += ArenaSize5a - ArenaSize5;
295
	}
296
 
297
	if(buf + sz != p)
298
		sysfatal("packarena packed wrong amount");
299
 
300
	return 0;
301
}
302
 
303
int
304
unpackarenahead(ArenaHead *head, u8int *buf)
305
{
306
	u8int *p;
307
	u32int m;
308
	int sz;
309
	char fbuf[20];
310
 
311
	p = buf;
312
 
313
	m = U32GET(p);
314
	if(m != ArenaHeadMagic){
315
		seterr(ECorrupt, "arena %d head has wrong magic number: %s "
316
			"expected ArenaHeadMagic (%#lux)", debugarena,
317
			fmtmagic(fbuf, m), ArenaHeadMagic);
318
		return -1;
319
	}
320
 
321
	p += U32Size;
322
	head->version = U32GET(p);
323
	p += U32Size;
324
	namecp(head->name, (char*)p);
325
	p += ANameSize;
326
	head->blocksize = U32GET(p);
327
	p += U32Size;
328
	head->size = U64GET(p);
329
	p += U64Size;
330
	if(head->version == ArenaVersion5){
331
		head->clumpmagic = U32GET(p);
332
		p += U32Size;
333
	}
334
 
335
	switch(head->version){
336
	case ArenaVersion4:
337
		sz = ArenaHeadSize4;
338
		head->clumpmagic = _ClumpMagic;
339
		break;
340
	case ArenaVersion5:
341
		sz = ArenaHeadSize5;
342
		break;
343
	default:
344
		seterr(ECorrupt, "arena head has unexpected version %d", head->version);
345
		return -1;
346
	}
347
 
348
	if(buf + sz != p)
349
		sysfatal("unpackarenahead unpacked wrong amount");
350
 
351
	return 0;
352
}
353
 
354
int
355
packarenahead(ArenaHead *head, u8int *buf)
356
{
357
	u8int *p;
358
	int sz;
359
	u32int t32;
360
 
361
	switch(head->version){
362
	case ArenaVersion4:
363
		sz = ArenaHeadSize4;
364
		if(head->clumpmagic != _ClumpMagic)
365
			fprint(2, "warning: writing old arena header loses clump magic 0x%lux != 0x%lux\n",
366
				(ulong)head->clumpmagic, (ulong)_ClumpMagic);
367
		break;
368
	case ArenaVersion5:
369
		sz = ArenaHeadSize5;
370
		break;
371
	default:
372
		sysfatal("packarenahead unknown version %d", head->version);
373
		return -1;
374
	}
375
 
376
	p = buf;
377
 
378
	U32PUT(p, ArenaHeadMagic);
379
	p += U32Size;
380
	U32PUT(p, head->version);
381
	p += U32Size;
382
	namecp((char*)p, head->name);
383
	p += ANameSize;
384
	U32PUT(p, head->blocksize);
385
	p += U32Size;
386
	U64PUT(p, head->size, t32);
387
	p += U64Size;
388
	if(head->version == ArenaVersion5){
389
		U32PUT(p, head->clumpmagic);
390
		p += U32Size;
391
	}
392
	if(buf + sz != p)
393
		sysfatal("packarenahead packed wrong amount");
394
 
395
	return 0;
396
}
397
 
398
static int
399
checkclump(Clump *w)
400
{
401
	if(w->encoding == ClumpENone){
402
		if(w->info.size != w->info.uncsize){
403
			seterr(ECorrupt, "uncompressed wad size mismatch");
404
			return -1;
405
		}
406
	}else if(w->encoding == ClumpECompress){
407
		if(w->info.size >= w->info.uncsize){
408
			seterr(ECorrupt, "compressed lump has inconsistent block sizes %d %d", w->info.size, w->info.uncsize);
409
			return -1;
410
		}
411
	}else{
412
		seterr(ECorrupt, "clump has illegal encoding");
413
		return -1;
414
	}
415
 
416
	return 0;
417
}
418
 
419
int
420
unpackclump(Clump *c, u8int *buf, u32int cmagic)
421
{
422
	u8int *p;
423
	u32int magic;
424
 
425
	p = buf;
426
	magic = U32GET(p);
427
	if(magic != cmagic){
428
		seterr(ECorrupt, "clump has bad magic number=%#8.8ux != %#8.8ux", magic, cmagic);
429
		return -1;
430
	}
431
	p += U32Size;
432
 
433
	c->info.type = vtfromdisktype(U8GET(p));
434
	p += U8Size;
435
	c->info.size = U16GET(p);
436
	p += U16Size;
437
	c->info.uncsize = U16GET(p);
438
	p += U16Size;
439
	scorecp(c->info.score, p);
440
	p += VtScoreSize;
441
 
442
	c->encoding = U8GET(p);
443
	p += U8Size;
444
	c->creator = U32GET(p);
445
	p += U32Size;
446
	c->time = U32GET(p);
447
	p += U32Size;
448
 
449
	if(buf + ClumpSize != p)
450
		sysfatal("unpackclump unpacked wrong amount");
451
 
452
	return checkclump(c);
453
}
454
 
455
int
456
packclump(Clump *c, u8int *buf, u32int magic)
457
{
458
	u8int *p;
459
 
460
	p = buf;
461
	U32PUT(p, magic);
462
	p += U32Size;
463
 
464
	U8PUT(p, vttodisktype(c->info.type));
465
	p += U8Size;
466
	U16PUT(p, c->info.size);
467
	p += U16Size;
468
	U16PUT(p, c->info.uncsize);
469
	p += U16Size;
470
	scorecp(p, c->info.score);
471
	p += VtScoreSize;
472
 
473
	U8PUT(p, c->encoding);
474
	p += U8Size;
475
	U32PUT(p, c->creator);
476
	p += U32Size;
477
	U32PUT(p, c->time);
478
	p += U32Size;
479
 
480
	if(buf + ClumpSize != p)
481
		sysfatal("packclump packed wrong amount");
482
 
483
	return checkclump(c);
484
}
485
 
486
void
487
unpackclumpinfo(ClumpInfo *ci, u8int *buf)
488
{
489
	u8int *p;
490
 
491
	p = buf;
492
	ci->type = vtfromdisktype(U8GET(p));
493
	p += U8Size;
494
	ci->size = U16GET(p);
495
	p += U16Size;
496
	ci->uncsize = U16GET(p);
497
	p += U16Size;
498
	scorecp(ci->score, p);
499
	p += VtScoreSize;
500
 
501
	if(buf + ClumpInfoSize != p)
502
		sysfatal("unpackclumpinfo unpacked wrong amount");
503
}
504
 
505
void
506
packclumpinfo(ClumpInfo *ci, u8int *buf)
507
{
508
	u8int *p;
509
 
510
	p = buf;
511
	U8PUT(p, vttodisktype(ci->type));
512
	p += U8Size;
513
	U16PUT(p, ci->size);
514
	p += U16Size;
515
	U16PUT(p, ci->uncsize);
516
	p += U16Size;
517
	scorecp(p, ci->score);
518
	p += VtScoreSize;
519
 
520
	if(buf + ClumpInfoSize != p)
521
		sysfatal("packclumpinfo packed wrong amount");
522
}
523
 
524
int
525
unpackisect(ISect *is, u8int *buf)
526
{
527
	u8int *p;
528
	u32int m;
529
	char fbuf[20];
530
 
531
	p = buf;
532
 
533
 
534
	m = U32GET(p);
535
	if(m != ISectMagic){
536
		seterr(ECorrupt, "index section has wrong magic number: %s expected ISectMagic (%#lux)",
537
			fmtmagic(fbuf, m), ISectMagic);
538
		return -1;
539
	}
540
	p += U32Size;
541
	is->version = U32GET(p);
542
	p += U32Size;
543
	namecp(is->name, (char*)p);
544
	p += ANameSize;
545
	namecp(is->index, (char*)p);
546
	p += ANameSize;
547
	is->blocksize = U32GET(p);
548
	p += U32Size;
549
	is->blockbase = U32GET(p);
550
	p += U32Size;
551
	is->blocks = U32GET(p);
552
	p += U32Size;
553
	is->start = U32GET(p);
554
	p += U32Size;
555
	is->stop = U32GET(p);
556
	p += U32Size;
557
	if(buf + ISectSize1 != p)
558
		sysfatal("unpackisect unpacked wrong amount");
559
	is->bucketmagic = 0;
560
	if(is->version == ISectVersion2){
561
		is->bucketmagic = U32GET(p);
562
		p += U32Size;
563
		if(buf + ISectSize2 != p)
564
			sysfatal("unpackisect unpacked wrong amount");
565
	}
566
 
567
	return 0;
568
}
569
 
570
int
571
packisect(ISect *is, u8int *buf)
572
{
573
	u8int *p;
574
 
575
	p = buf;
576
 
577
	U32PUT(p, ISectMagic);
578
	p += U32Size;
579
	U32PUT(p, is->version);
580
	p += U32Size;
581
	namecp((char*)p, is->name);
582
	p += ANameSize;
583
	namecp((char*)p, is->index);
584
	p += ANameSize;
585
	U32PUT(p, is->blocksize);
586
	p += U32Size;
587
	U32PUT(p, is->blockbase);
588
	p += U32Size;
589
	U32PUT(p, is->blocks);
590
	p += U32Size;
591
	U32PUT(p, is->start);
592
	p += U32Size;
593
	U32PUT(p, is->stop);
594
	p += U32Size;
595
	if(buf + ISectSize1 != p)
596
		sysfatal("packisect packed wrong amount");
597
	if(is->version == ISectVersion2){
598
		U32PUT(p, is->bucketmagic);
599
		p += U32Size;
600
		if(buf + ISectSize2 != p)
601
			sysfatal("packisect packed wrong amount");
602
	}
603
 
604
	return 0;
605
}
606
 
607
void
608
unpackientry(IEntry *ie, u8int *buf)
609
{
610
	u8int *p;
611
 
612
	p = buf;
613
 
614
	scorecp(ie->score, p);
615
	p += VtScoreSize;
616
	/* ie->wtime = U32GET(p); */
617
	p += U32Size;
618
	/* ie->train = U16GET(p); */
619
	p += U16Size;
620
	if(p - buf != IEntryAddrOff)
621
		sysfatal("unpackentry bad IEntryAddrOff amount");
622
	ie->ia.addr = U64GET(p);
623
if(ie->ia.addr>>56) print("%.8H => %llux\n", p, ie->ia.addr);
624
	p += U64Size;
625
	ie->ia.size = U16GET(p);
626
	p += U16Size;
627
	if(p - buf != IEntryTypeOff)
628
		sysfatal("unpackientry bad IEntryTypeOff amount");
629
	ie->ia.type = vtfromdisktype(U8GET(p));
630
	p += U8Size;
631
	ie->ia.blocks = U8GET(p);
632
	p += U8Size;
633
 
634
	if(p - buf != IEntrySize)
635
		sysfatal("unpackientry unpacked wrong amount");
636
}
637
 
638
void
639
packientry(IEntry *ie, u8int *buf)
640
{
641
	u32int t32;
642
	u8int *p;
643
 
644
	p = buf;
645
 
646
	scorecp(p, ie->score);
647
	p += VtScoreSize;
648
	U32PUT(p, 0); /* wtime */
649
	p += U32Size;
650
	U16PUT(p, 0); /* train */
651
	p += U16Size;
652
	U64PUT(p, ie->ia.addr, t32);
653
	p += U64Size;
654
	U16PUT(p, ie->ia.size);
655
	p += U16Size;
656
	U8PUT(p, vttodisktype(ie->ia.type));
657
	p += U8Size;
658
	U8PUT(p, ie->ia.blocks);
659
	p += U8Size;
660
 
661
	if(p - buf != IEntrySize)
662
		sysfatal("packientry packed wrong amount");
663
}
664
 
665
void
666
unpackibucket(IBucket *b, u8int *buf, u32int magic)
667
{
668
	b->n = U16GET(buf);
669
	b->data = buf + IBucketSize;
670
	if(magic && magic != U32GET(buf+U16Size))
671
		b->n = 0;
672
}		
673
 
674
void
675
packibucket(IBucket *b, u8int *buf, u32int magic)
676
{
677
	U16PUT(buf, b->n);
678
	U32PUT(buf+U16Size, magic);
679
}
680
 
681
void
682
packbloomhead(Bloom *b, u8int *buf)
683
{
684
	u8int *p;
685
 
686
	p = buf;
687
	U32PUT(p, BloomMagic);
688
	U32PUT(p+4, BloomVersion);
689
	U32PUT(p+8, b->nhash);
690
	U32PUT(p+12, b->size);
691
}
692
 
693
int
694
unpackbloomhead(Bloom *b, u8int *buf)
695
{
696
	u8int *p;
697
	u32int m;
698
	char fbuf[20];
699
 
700
	p = buf;
701
 
702
	m = U32GET(p);
703
	if(m != BloomMagic){
704
		seterr(ECorrupt, "bloom filter has wrong magic number: %s expected BloomMagic (%#lux)", fmtmagic(fbuf, m), (ulong)BloomMagic);
705
		return -1;
706
	}
707
	p += U32Size;
708
 
709
	m = U32GET(p);
710
	if(m != BloomVersion){
711
		seterr(ECorrupt, "bloom filter has wrong version %ud expected %ud", (uint)m, (uint)BloomVersion);
712
		return -1;
713
	}
714
	p += U32Size;
715
 
716
	b->nhash = U32GET(p);
717
	p += U32Size;
718
 
719
	b->size = U32GET(p);
720
	p += U32Size;
721
	if(b->size < BloomHeadSize || b->size > MaxBloomSize || (b->size&(b->size-1))){
722
		seterr(ECorrupt, "bloom filter has invalid size %#lux", b->size);
723
		return -1;
724
	}
725
 
726
	if(buf + BloomHeadSize != p)
727
		sysfatal("unpackarena unpacked wrong amount");
728
 
729
	return 0;
730
}