Commit | Line | Data |
---|---|---|
5ca28f79 L |
1 | /* gzjoin -- command to join gzip files into one gzip file |
2 | ||
da09a436 JB |
3 | Copyright (C) 2004, 2005, 2012 Mark Adler, all rights reserved |
4 | version 1.2, 14 Aug 2012 | |
5ca28f79 L |
5 | |
6 | This software is provided 'as-is', without any express or implied | |
7 | warranty. In no event will the author be held liable for any damages | |
8 | arising from the use of this software. | |
9 | ||
10 | Permission is granted to anyone to use this software for any purpose, | |
11 | including commercial applications, and to alter it and redistribute it | |
12 | freely, subject to the following restrictions: | |
13 | ||
14 | 1. The origin of this software must not be misrepresented; you must not | |
15 | claim that you wrote the original software. If you use this software | |
16 | in a product, an acknowledgment in the product documentation would be | |
17 | appreciated but is not required. | |
18 | 2. Altered source versions must be plainly marked as such, and must not be | |
19 | misrepresented as being the original software. | |
20 | 3. This notice may not be removed or altered from any source distribution. | |
21 | ||
22 | Mark Adler madler@alumni.caltech.edu | |
23 | */ | |
24 | ||
25 | /* | |
26 | * Change history: | |
27 | * | |
28 | * 1.0 11 Dec 2004 - First version | |
29 | * 1.1 12 Jun 2005 - Changed ssize_t to long for portability | |
da09a436 | 30 | * 1.2 14 Aug 2012 - Clean up for z_const usage |
5ca28f79 L |
31 | */ |
32 | ||
33 | /* | |
34 | gzjoin takes one or more gzip files on the command line and writes out a | |
35 | single gzip file that will uncompress to the concatenation of the | |
36 | uncompressed data from the individual gzip files. gzjoin does this without | |
37 | having to recompress any of the data and without having to calculate a new | |
38 | crc32 for the concatenated uncompressed data. gzjoin does however have to | |
39 | decompress all of the input data in order to find the bits in the compressed | |
40 | data that need to be modified to concatenate the streams. | |
41 | ||
42 | gzjoin does not do an integrity check on the input gzip files other than | |
43 | checking the gzip header and decompressing the compressed data. They are | |
44 | otherwise assumed to be complete and correct. | |
45 | ||
46 | Each joint between gzip files removes at least 18 bytes of previous trailer | |
47 | and subsequent header, and inserts an average of about three bytes to the | |
48 | compressed data in order to connect the streams. The output gzip file | |
49 | has a minimal ten-byte gzip header with no file name or modification time. | |
50 | ||
51 | This program was written to illustrate the use of the Z_BLOCK option of | |
52 | inflate() and the crc32_combine() function. gzjoin will not compile with | |
53 | versions of zlib earlier than 1.2.3. | |
54 | */ | |
55 | ||
56 | #include <stdio.h> /* fputs(), fprintf(), fwrite(), putc() */ | |
57 | #include <stdlib.h> /* exit(), malloc(), free() */ | |
58 | #include <fcntl.h> /* open() */ | |
59 | #include <unistd.h> /* close(), read(), lseek() */ | |
60 | #include "zlib.h" | |
61 | /* crc32(), crc32_combine(), inflateInit2(), inflate(), inflateEnd() */ | |
62 | ||
63 | #define local static | |
64 | ||
65 | /* exit with an error (return a value to allow use in an expression) */ | |
66 | local int bail(char *why1, char *why2) | |
67 | { | |
68 | fprintf(stderr, "gzjoin error: %s%s, output incomplete\n", why1, why2); | |
69 | exit(1); | |
70 | return 0; | |
71 | } | |
72 | ||
73 | /* -- simple buffered file input with access to the buffer -- */ | |
74 | ||
75 | #define CHUNK 32768 /* must be a power of two and fit in unsigned */ | |
76 | ||
77 | /* bin buffered input file type */ | |
78 | typedef struct { | |
79 | char *name; /* name of file for error messages */ | |
80 | int fd; /* file descriptor */ | |
81 | unsigned left; /* bytes remaining at next */ | |
82 | unsigned char *next; /* next byte to read */ | |
83 | unsigned char *buf; /* allocated buffer of length CHUNK */ | |
84 | } bin; | |
85 | ||
86 | /* close a buffered file and free allocated memory */ | |
87 | local void bclose(bin *in) | |
88 | { | |
89 | if (in != NULL) { | |
90 | if (in->fd != -1) | |
91 | close(in->fd); | |
92 | if (in->buf != NULL) | |
93 | free(in->buf); | |
94 | free(in); | |
95 | } | |
96 | } | |
97 | ||
98 | /* open a buffered file for input, return a pointer to type bin, or NULL on | |
99 | failure */ | |
100 | local bin *bopen(char *name) | |
101 | { | |
102 | bin *in; | |
103 | ||
104 | in = malloc(sizeof(bin)); | |
105 | if (in == NULL) | |
106 | return NULL; | |
107 | in->buf = malloc(CHUNK); | |
108 | in->fd = open(name, O_RDONLY, 0); | |
109 | if (in->buf == NULL || in->fd == -1) { | |
110 | bclose(in); | |
111 | return NULL; | |
112 | } | |
113 | in->left = 0; | |
114 | in->next = in->buf; | |
115 | in->name = name; | |
116 | return in; | |
117 | } | |
118 | ||
119 | /* load buffer from file, return -1 on read error, 0 or 1 on success, with | |
120 | 1 indicating that end-of-file was reached */ | |
121 | local int bload(bin *in) | |
122 | { | |
123 | long len; | |
124 | ||
125 | if (in == NULL) | |
126 | return -1; | |
127 | if (in->left != 0) | |
128 | return 0; | |
129 | in->next = in->buf; | |
130 | do { | |
131 | len = (long)read(in->fd, in->buf + in->left, CHUNK - in->left); | |
132 | if (len < 0) | |
133 | return -1; | |
134 | in->left += (unsigned)len; | |
135 | } while (len != 0 && in->left < CHUNK); | |
136 | return len == 0 ? 1 : 0; | |
137 | } | |
138 | ||
139 | /* get a byte from the file, bail if end of file */ | |
140 | #define bget(in) (in->left ? 0 : bload(in), \ | |
141 | in->left ? (in->left--, *(in->next)++) : \ | |
142 | bail("unexpected end of file on ", in->name)) | |
143 | ||
144 | /* get a four-byte little-endian unsigned integer from file */ | |
145 | local unsigned long bget4(bin *in) | |
146 | { | |
147 | unsigned long val; | |
148 | ||
149 | val = bget(in); | |
150 | val += (unsigned long)(bget(in)) << 8; | |
151 | val += (unsigned long)(bget(in)) << 16; | |
152 | val += (unsigned long)(bget(in)) << 24; | |
153 | return val; | |
154 | } | |
155 | ||
156 | /* skip bytes in file */ | |
157 | local void bskip(bin *in, unsigned skip) | |
158 | { | |
159 | /* check pointer */ | |
160 | if (in == NULL) | |
161 | return; | |
162 | ||
163 | /* easy case -- skip bytes in buffer */ | |
164 | if (skip <= in->left) { | |
165 | in->left -= skip; | |
166 | in->next += skip; | |
167 | return; | |
168 | } | |
169 | ||
170 | /* skip what's in buffer, discard buffer contents */ | |
171 | skip -= in->left; | |
172 | in->left = 0; | |
173 | ||
174 | /* seek past multiples of CHUNK bytes */ | |
175 | if (skip > CHUNK) { | |
176 | unsigned left; | |
177 | ||
178 | left = skip & (CHUNK - 1); | |
179 | if (left == 0) { | |
180 | /* exact number of chunks: seek all the way minus one byte to check | |
181 | for end-of-file with a read */ | |
182 | lseek(in->fd, skip - 1, SEEK_CUR); | |
183 | if (read(in->fd, in->buf, 1) != 1) | |
184 | bail("unexpected end of file on ", in->name); | |
185 | return; | |
186 | } | |
187 | ||
188 | /* skip the integral chunks, update skip with remainder */ | |
189 | lseek(in->fd, skip - left, SEEK_CUR); | |
190 | skip = left; | |
191 | } | |
192 | ||
193 | /* read more input and skip remainder */ | |
194 | bload(in); | |
195 | if (skip > in->left) | |
196 | bail("unexpected end of file on ", in->name); | |
197 | in->left -= skip; | |
198 | in->next += skip; | |
199 | } | |
200 | ||
201 | /* -- end of buffered input functions -- */ | |
202 | ||
203 | /* skip the gzip header from file in */ | |
204 | local void gzhead(bin *in) | |
205 | { | |
206 | int flags; | |
207 | ||
208 | /* verify gzip magic header and compression method */ | |
209 | if (bget(in) != 0x1f || bget(in) != 0x8b || bget(in) != 8) | |
210 | bail(in->name, " is not a valid gzip file"); | |
211 | ||
212 | /* get and verify flags */ | |
213 | flags = bget(in); | |
214 | if ((flags & 0xe0) != 0) | |
215 | bail("unknown reserved bits set in ", in->name); | |
216 | ||
217 | /* skip modification time, extra flags, and os */ | |
218 | bskip(in, 6); | |
219 | ||
220 | /* skip extra field if present */ | |
221 | if (flags & 4) { | |
222 | unsigned len; | |
223 | ||
224 | len = bget(in); | |
225 | len += (unsigned)(bget(in)) << 8; | |
226 | bskip(in, len); | |
227 | } | |
228 | ||
229 | /* skip file name if present */ | |
230 | if (flags & 8) | |
231 | while (bget(in) != 0) | |
232 | ; | |
233 | ||
234 | /* skip comment if present */ | |
235 | if (flags & 16) | |
236 | while (bget(in) != 0) | |
237 | ; | |
238 | ||
239 | /* skip header crc if present */ | |
240 | if (flags & 2) | |
241 | bskip(in, 2); | |
242 | } | |
243 | ||
244 | /* write a four-byte little-endian unsigned integer to out */ | |
245 | local void put4(unsigned long val, FILE *out) | |
246 | { | |
247 | putc(val & 0xff, out); | |
248 | putc((val >> 8) & 0xff, out); | |
249 | putc((val >> 16) & 0xff, out); | |
250 | putc((val >> 24) & 0xff, out); | |
251 | } | |
252 | ||
253 | /* Load up zlib stream from buffered input, bail if end of file */ | |
254 | local void zpull(z_streamp strm, bin *in) | |
255 | { | |
256 | if (in->left == 0) | |
257 | bload(in); | |
258 | if (in->left == 0) | |
259 | bail("unexpected end of file on ", in->name); | |
260 | strm->avail_in = in->left; | |
261 | strm->next_in = in->next; | |
262 | } | |
263 | ||
264 | /* Write header for gzip file to out and initialize trailer. */ | |
265 | local void gzinit(unsigned long *crc, unsigned long *tot, FILE *out) | |
266 | { | |
267 | fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out); | |
268 | *crc = crc32(0L, Z_NULL, 0); | |
269 | *tot = 0; | |
270 | } | |
271 | ||
272 | /* Copy the compressed data from name, zeroing the last block bit of the last | |
273 | block if clr is true, and adding empty blocks as needed to get to a byte | |
274 | boundary. If clr is false, then the last block becomes the last block of | |
275 | the output, and the gzip trailer is written. crc and tot maintains the | |
276 | crc and length (modulo 2^32) of the output for the trailer. The resulting | |
277 | gzip file is written to out. gzinit() must be called before the first call | |
278 | of gzcopy() to write the gzip header and to initialize crc and tot. */ | |
279 | local void gzcopy(char *name, int clr, unsigned long *crc, unsigned long *tot, | |
280 | FILE *out) | |
281 | { | |
282 | int ret; /* return value from zlib functions */ | |
283 | int pos; /* where the "last block" bit is in byte */ | |
284 | int last; /* true if processing the last block */ | |
285 | bin *in; /* buffered input file */ | |
286 | unsigned char *start; /* start of compressed data in buffer */ | |
287 | unsigned char *junk; /* buffer for uncompressed data -- discarded */ | |
288 | z_off_t len; /* length of uncompressed data (support > 4 GB) */ | |
289 | z_stream strm; /* zlib inflate stream */ | |
290 | ||
291 | /* open gzip file and skip header */ | |
292 | in = bopen(name); | |
293 | if (in == NULL) | |
294 | bail("could not open ", name); | |
295 | gzhead(in); | |
296 | ||
297 | /* allocate buffer for uncompressed data and initialize raw inflate | |
298 | stream */ | |
299 | junk = malloc(CHUNK); | |
300 | strm.zalloc = Z_NULL; | |
301 | strm.zfree = Z_NULL; | |
302 | strm.opaque = Z_NULL; | |
303 | strm.avail_in = 0; | |
304 | strm.next_in = Z_NULL; | |
305 | ret = inflateInit2(&strm, -15); | |
306 | if (junk == NULL || ret != Z_OK) | |
307 | bail("out of memory", ""); | |
308 | ||
309 | /* inflate and copy compressed data, clear last-block bit if requested */ | |
310 | len = 0; | |
311 | zpull(&strm, in); | |
da09a436 | 312 | start = in->next; |
5ca28f79 L |
313 | last = start[0] & 1; |
314 | if (last && clr) | |
315 | start[0] &= ~1; | |
316 | strm.avail_out = 0; | |
317 | for (;;) { | |
318 | /* if input used and output done, write used input and get more */ | |
319 | if (strm.avail_in == 0 && strm.avail_out != 0) { | |
320 | fwrite(start, 1, strm.next_in - start, out); | |
321 | start = in->buf; | |
322 | in->left = 0; | |
323 | zpull(&strm, in); | |
324 | } | |
325 | ||
326 | /* decompress -- return early when end-of-block reached */ | |
327 | strm.avail_out = CHUNK; | |
328 | strm.next_out = junk; | |
329 | ret = inflate(&strm, Z_BLOCK); | |
330 | switch (ret) { | |
331 | case Z_MEM_ERROR: | |
332 | bail("out of memory", ""); | |
333 | case Z_DATA_ERROR: | |
334 | bail("invalid compressed data in ", in->name); | |
335 | } | |
336 | ||
337 | /* update length of uncompressed data */ | |
338 | len += CHUNK - strm.avail_out; | |
339 | ||
340 | /* check for block boundary (only get this when block copied out) */ | |
341 | if (strm.data_type & 128) { | |
342 | /* if that was the last block, then done */ | |
343 | if (last) | |
344 | break; | |
345 | ||
346 | /* number of unused bits in last byte */ | |
347 | pos = strm.data_type & 7; | |
348 | ||
349 | /* find the next last-block bit */ | |
350 | if (pos != 0) { | |
351 | /* next last-block bit is in last used byte */ | |
352 | pos = 0x100 >> pos; | |
353 | last = strm.next_in[-1] & pos; | |
354 | if (last && clr) | |
da09a436 | 355 | in->buf[strm.next_in - in->buf - 1] &= ~pos; |
5ca28f79 L |
356 | } |
357 | else { | |
358 | /* next last-block bit is in next unused byte */ | |
359 | if (strm.avail_in == 0) { | |
360 | /* don't have that byte yet -- get it */ | |
361 | fwrite(start, 1, strm.next_in - start, out); | |
362 | start = in->buf; | |
363 | in->left = 0; | |
364 | zpull(&strm, in); | |
365 | } | |
366 | last = strm.next_in[0] & 1; | |
367 | if (last && clr) | |
da09a436 | 368 | in->buf[strm.next_in - in->buf] &= ~1; |
5ca28f79 L |
369 | } |
370 | } | |
371 | } | |
372 | ||
373 | /* update buffer with unused input */ | |
374 | in->left = strm.avail_in; | |
da09a436 | 375 | in->next = in->buf + (strm.next_in - in->buf); |
5ca28f79 L |
376 | |
377 | /* copy used input, write empty blocks to get to byte boundary */ | |
378 | pos = strm.data_type & 7; | |
379 | fwrite(start, 1, in->next - start - 1, out); | |
380 | last = in->next[-1]; | |
381 | if (pos == 0 || !clr) | |
382 | /* already at byte boundary, or last file: write last byte */ | |
383 | putc(last, out); | |
384 | else { | |
385 | /* append empty blocks to last byte */ | |
386 | last &= ((0x100 >> pos) - 1); /* assure unused bits are zero */ | |
387 | if (pos & 1) { | |
388 | /* odd -- append an empty stored block */ | |
389 | putc(last, out); | |
390 | if (pos == 1) | |
391 | putc(0, out); /* two more bits in block header */ | |
392 | fwrite("\0\0\xff\xff", 1, 4, out); | |
393 | } | |
394 | else { | |
395 | /* even -- append 1, 2, or 3 empty fixed blocks */ | |
396 | switch (pos) { | |
397 | case 6: | |
398 | putc(last | 8, out); | |
399 | last = 0; | |
400 | case 4: | |
401 | putc(last | 0x20, out); | |
402 | last = 0; | |
403 | case 2: | |
404 | putc(last | 0x80, out); | |
405 | putc(0, out); | |
406 | } | |
407 | } | |
408 | } | |
409 | ||
410 | /* update crc and tot */ | |
411 | *crc = crc32_combine(*crc, bget4(in), len); | |
412 | *tot += (unsigned long)len; | |
413 | ||
414 | /* clean up */ | |
415 | inflateEnd(&strm); | |
416 | free(junk); | |
417 | bclose(in); | |
418 | ||
419 | /* write trailer if this is the last gzip file */ | |
420 | if (!clr) { | |
421 | put4(*crc, out); | |
422 | put4(*tot, out); | |
423 | } | |
424 | } | |
425 | ||
426 | /* join the gzip files on the command line, write result to stdout */ | |
427 | int main(int argc, char **argv) | |
428 | { | |
429 | unsigned long crc, tot; /* running crc and total uncompressed length */ | |
430 | ||
431 | /* skip command name */ | |
432 | argc--; | |
433 | argv++; | |
434 | ||
435 | /* show usage if no arguments */ | |
436 | if (argc == 0) { | |
437 | fputs("gzjoin usage: gzjoin f1.gz [f2.gz [f3.gz ...]] > fjoin.gz\n", | |
438 | stderr); | |
439 | return 0; | |
440 | } | |
441 | ||
442 | /* join gzip files on command line and write to stdout */ | |
443 | gzinit(&crc, &tot, stdout); | |
444 | while (argc--) | |
445 | gzcopy(*argv++, argc, &crc, &tot, stdout); | |
446 | ||
447 | /* done */ | |
448 | return 0; | |
449 | } |