Line data Source code
1 : /* LTO IL compression streams.
2 :
3 : Copyright (C) 2009-2026 Free Software Foundation, Inc.
4 : Contributed by Simon Baldwin <simonb@google.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it
9 : under the terms of the GNU General Public License as published by
10 : the Free Software Foundation; either version 3, or (at your option)
11 : any later version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT
14 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15 : or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
16 : License for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #include "system.h"
24 : #include "coretypes.h"
25 : #include "backend.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "cgraph.h"
29 : #include "lto-streamer.h"
30 : /* zlib.h includes other system headers. Those headers may test feature
31 : test macros. config.h may define feature test macros. For this reason,
32 : zlib.h needs to be included after, rather than before, config.h and
33 : system.h. */
34 : #include <zlib.h>
35 : #include "lto-compress.h"
36 : #include "timevar.h"
37 :
38 : #ifdef HAVE_ZSTD_H
39 : #include <zstd.h>
40 : #endif
41 :
42 : /* Compression stream structure, holds the flush callback and opaque token,
43 : the buffered data, and a note of whether compressing or uncompressing. */
44 :
45 : struct lto_compression_stream
46 : {
47 : void (*callback) (const char *, unsigned, void *);
48 : void *opaque;
49 : char *buffer;
50 : size_t bytes;
51 : size_t allocation;
52 : bool is_compression;
53 : };
54 :
55 : /* Overall compression constants for zlib. */
56 :
57 : static const size_t Z_BUFFER_LENGTH = 4096;
58 : static const size_t MIN_STREAM_ALLOCATION = 1024;
59 :
60 : /* For zlib, allocate SIZE count of ITEMS and return the address, OPAQUE
61 : is unused. */
62 :
63 : static void *
64 0 : lto_zalloc (void *opaque, unsigned items, unsigned size)
65 : {
66 0 : gcc_assert (opaque == Z_NULL);
67 0 : return xmalloc (items * size);
68 : }
69 :
70 : /* For zlib, free memory at ADDRESS, OPAQUE is unused. */
71 :
72 : static void
73 0 : lto_zfree (void *opaque, void *address)
74 : {
75 0 : gcc_assert (opaque == Z_NULL);
76 0 : free (address);
77 0 : }
78 :
79 : /* Return a zlib compression level that zlib will not reject. Normalizes
80 : the compression level from the command line flag, clamping non-default
81 : values to the appropriate end of their valid range. */
82 :
83 : static int
84 0 : lto_normalized_zlib_level (void)
85 : {
86 0 : int level = flag_lto_compression_level;
87 :
88 0 : if (level != Z_DEFAULT_COMPRESSION)
89 : {
90 0 : if (level < Z_NO_COMPRESSION)
91 : level = Z_NO_COMPRESSION;
92 0 : else if (level > Z_BEST_COMPRESSION)
93 : level = Z_BEST_COMPRESSION;
94 : }
95 :
96 0 : return level;
97 : }
98 :
99 : /* Free the buffer and memory associated with STREAM. */
100 :
101 : static void
102 539519 : lto_destroy_compression_stream (struct lto_compression_stream *stream)
103 : {
104 539519 : free (stream->buffer);
105 539519 : free (stream);
106 539519 : }
107 :
108 : #ifdef HAVE_ZSTD_H
109 : /* Return a zstd compression level that zstd will not reject. Normalizes
110 : the compression level from the command line flag, clamping non-default
111 : values to the appropriate end of their valid range. */
112 :
113 : static int
114 330801 : lto_normalized_zstd_level (void)
115 : {
116 330801 : int level = flag_lto_compression_level;
117 :
118 330801 : if (level < 0)
119 : level = 0;
120 0 : else if (level > ZSTD_maxCLevel ())
121 0 : level = ZSTD_maxCLevel ();
122 :
123 330801 : return level;
124 : }
125 :
126 : /* Compress STREAM using ZSTD algorithm. */
127 :
128 : static void
129 330801 : lto_compression_zstd (struct lto_compression_stream *stream)
130 : {
131 330801 : unsigned char *cursor = (unsigned char *) stream->buffer;
132 330801 : size_t size = stream->bytes;
133 :
134 330801 : timevar_push (TV_IPA_LTO_COMPRESS);
135 330801 : size_t const outbuf_length = ZSTD_compressBound (size);
136 330801 : char *outbuf = (char *) xmalloc (outbuf_length);
137 :
138 330801 : size_t const csize = ZSTD_compress (outbuf, outbuf_length, cursor, size,
139 : lto_normalized_zstd_level ());
140 :
141 330801 : if (ZSTD_isError (csize))
142 0 : internal_error ("compressed stream: %s", ZSTD_getErrorName (csize));
143 :
144 330801 : lto_stats.num_compressed_il_bytes += csize;
145 330801 : stream->callback (outbuf, csize, NULL);
146 :
147 330801 : lto_destroy_compression_stream (stream);
148 330801 : free (outbuf);
149 330801 : timevar_pop (TV_IPA_LTO_COMPRESS);
150 330801 : }
151 :
152 : /* Uncompress STREAM using ZSTD algorithm. */
153 :
154 : static void
155 208718 : lto_uncompression_zstd (struct lto_compression_stream *stream)
156 : {
157 208718 : unsigned char *cursor = (unsigned char *) stream->buffer;
158 208718 : size_t size = stream->bytes;
159 :
160 208718 : timevar_push (TV_IPA_LTO_DECOMPRESS);
161 208718 : unsigned long long const rsize = ZSTD_getFrameContentSize (cursor, size);
162 208718 : if (rsize == ZSTD_CONTENTSIZE_ERROR)
163 0 : internal_error ("original not compressed with zstd");
164 208718 : else if (rsize == ZSTD_CONTENTSIZE_UNKNOWN)
165 0 : internal_error ("original size unknown");
166 :
167 208718 : char *outbuf = (char *) xmalloc (rsize);
168 208718 : size_t const dsize = ZSTD_decompress (outbuf, rsize, cursor, size);
169 :
170 208718 : if (ZSTD_isError (dsize))
171 0 : internal_error ("decompressed stream: %s", ZSTD_getErrorName (dsize));
172 :
173 208718 : lto_stats.num_uncompressed_il_bytes += dsize;
174 208718 : stream->callback (outbuf, dsize, stream->opaque);
175 :
176 208718 : lto_destroy_compression_stream (stream);
177 208718 : free (outbuf);
178 208718 : timevar_pop (TV_IPA_LTO_DECOMPRESS);
179 208718 : }
180 :
181 : #endif
182 :
183 : /* Create a new compression stream, with CALLBACK flush function passed
184 : OPAQUE token, IS_COMPRESSION indicates if compressing or uncompressing. */
185 :
186 : static struct lto_compression_stream *
187 539519 : lto_new_compression_stream (void (*callback) (const char *, unsigned, void *),
188 : void *opaque, bool is_compression)
189 : {
190 539519 : struct lto_compression_stream *stream
191 539519 : = (struct lto_compression_stream *) xmalloc (sizeof (*stream));
192 :
193 539519 : memset (stream, 0, sizeof (*stream));
194 539519 : stream->callback = callback;
195 539519 : stream->opaque = opaque;
196 539519 : stream->is_compression = is_compression;
197 :
198 539519 : return stream;
199 : }
200 :
201 : /* Append NUM_CHARS from address BASE to STREAM. */
202 :
203 : static void
204 1446117 : lto_append_to_compression_stream (struct lto_compression_stream *stream,
205 : const char *base, size_t num_chars)
206 : {
207 1446117 : size_t required = stream->bytes + num_chars;
208 :
209 1446117 : if (stream->allocation < required)
210 : {
211 594360 : if (stream->allocation == 0)
212 539519 : stream->allocation = MIN_STREAM_ALLOCATION;
213 670108 : while (stream->allocation < required)
214 75748 : stream->allocation *= 2;
215 :
216 594360 : stream->buffer = (char *) xrealloc (stream->buffer, stream->allocation);
217 : }
218 :
219 1446117 : memcpy (stream->buffer + stream->bytes, base, num_chars);
220 1446117 : stream->bytes += num_chars;
221 1446117 : }
222 :
223 : /* Return a new compression stream, with CALLBACK flush function passed
224 : OPAQUE token. */
225 :
226 : struct lto_compression_stream *
227 330801 : lto_start_compression (void (*callback) (const char *, unsigned, void *),
228 : void *opaque)
229 : {
230 330801 : return lto_new_compression_stream (callback, opaque, true);
231 : }
232 :
233 : /* Append NUM_CHARS from address BASE to STREAM. */
234 :
235 : void
236 1237399 : lto_compress_block (struct lto_compression_stream *stream,
237 : const char *base, size_t num_chars)
238 : {
239 1237399 : gcc_assert (stream->is_compression);
240 :
241 1237399 : lto_append_to_compression_stream (stream, base, num_chars);
242 1237399 : lto_stats.num_output_il_bytes += num_chars;
243 1237399 : }
244 :
245 : static void ATTRIBUTE_UNUSED
246 0 : lto_compression_zlib (struct lto_compression_stream *stream)
247 : {
248 0 : unsigned char *cursor = (unsigned char *) stream->buffer;
249 0 : size_t remaining = stream->bytes;
250 0 : const size_t outbuf_length = Z_BUFFER_LENGTH;
251 0 : unsigned char *outbuf = (unsigned char *) xmalloc (outbuf_length);
252 0 : z_stream out_stream;
253 0 : int status;
254 :
255 0 : gcc_assert (stream->is_compression);
256 :
257 0 : timevar_push (TV_IPA_LTO_COMPRESS);
258 :
259 0 : out_stream.next_out = outbuf;
260 0 : out_stream.avail_out = outbuf_length;
261 0 : out_stream.next_in = cursor;
262 0 : out_stream.avail_in = remaining;
263 0 : out_stream.zalloc = lto_zalloc;
264 0 : out_stream.zfree = lto_zfree;
265 0 : out_stream.opaque = Z_NULL;
266 :
267 0 : status = deflateInit (&out_stream, lto_normalized_zlib_level ());
268 0 : if (status != Z_OK)
269 0 : internal_error ("compressed stream: %s", zError (status));
270 :
271 0 : do
272 : {
273 0 : size_t in_bytes, out_bytes;
274 :
275 0 : status = deflate (&out_stream, Z_FINISH);
276 0 : if (status != Z_OK && status != Z_STREAM_END)
277 0 : internal_error ("compressed stream: %s", zError (status));
278 :
279 0 : in_bytes = remaining - out_stream.avail_in;
280 0 : out_bytes = outbuf_length - out_stream.avail_out;
281 :
282 0 : stream->callback ((const char *) outbuf, out_bytes, stream->opaque);
283 0 : lto_stats.num_compressed_il_bytes += out_bytes;
284 :
285 0 : cursor += in_bytes;
286 0 : remaining -= in_bytes;
287 :
288 0 : out_stream.next_out = outbuf;
289 0 : out_stream.avail_out = outbuf_length;
290 0 : out_stream.next_in = cursor;
291 0 : out_stream.avail_in = remaining;
292 : }
293 0 : while (status != Z_STREAM_END);
294 :
295 0 : status = deflateEnd (&out_stream);
296 0 : if (status != Z_OK)
297 0 : internal_error ("compressed stream: %s", zError (status));
298 :
299 0 : lto_destroy_compression_stream (stream);
300 0 : free (outbuf);
301 0 : timevar_pop (TV_IPA_LTO_COMPRESS);
302 0 : }
303 :
304 : void
305 330801 : lto_end_compression (struct lto_compression_stream *stream)
306 : {
307 : #ifdef HAVE_ZSTD_H
308 330801 : lto_compression_zstd (stream);
309 : #else
310 : lto_compression_zlib (stream);
311 : #endif
312 330801 : }
313 :
314 : /* Return a new uncompression stream, with CALLBACK flush function passed
315 : OPAQUE token. */
316 :
317 : struct lto_compression_stream *
318 208718 : lto_start_uncompression (void (*callback) (const char *, unsigned, void *),
319 : void *opaque)
320 : {
321 208718 : return lto_new_compression_stream (callback, opaque, false);
322 : }
323 :
324 : /* Append NUM_CHARS from address BASE to STREAM. */
325 :
326 : void
327 208718 : lto_uncompress_block (struct lto_compression_stream *stream,
328 : const char *base, size_t num_chars)
329 : {
330 208718 : gcc_assert (!stream->is_compression);
331 :
332 208718 : lto_append_to_compression_stream (stream, base, num_chars);
333 208718 : lto_stats.num_input_il_bytes += num_chars;
334 208718 : }
335 :
336 : static void
337 0 : lto_uncompression_zlib (struct lto_compression_stream *stream)
338 : {
339 0 : unsigned char *cursor = (unsigned char *) stream->buffer;
340 0 : size_t remaining = stream->bytes;
341 0 : const size_t outbuf_length = Z_BUFFER_LENGTH;
342 0 : unsigned char *outbuf = (unsigned char *) xmalloc (outbuf_length);
343 :
344 0 : gcc_assert (!stream->is_compression);
345 0 : timevar_push (TV_IPA_LTO_DECOMPRESS);
346 :
347 0 : while (remaining > 0)
348 : {
349 0 : z_stream in_stream;
350 0 : size_t out_bytes;
351 0 : int status;
352 :
353 0 : in_stream.next_out = outbuf;
354 0 : in_stream.avail_out = outbuf_length;
355 0 : in_stream.next_in = cursor;
356 0 : in_stream.avail_in = remaining;
357 0 : in_stream.zalloc = lto_zalloc;
358 0 : in_stream.zfree = lto_zfree;
359 0 : in_stream.opaque = Z_NULL;
360 :
361 0 : status = inflateInit (&in_stream);
362 0 : if (status != Z_OK)
363 0 : internal_error ("compressed stream: %s", zError (status));
364 :
365 0 : do
366 : {
367 0 : size_t in_bytes;
368 :
369 0 : status = inflate (&in_stream, Z_SYNC_FLUSH);
370 0 : if (status != Z_OK && status != Z_STREAM_END)
371 0 : internal_error ("compressed stream: %s", zError (status));
372 :
373 0 : in_bytes = remaining - in_stream.avail_in;
374 0 : out_bytes = outbuf_length - in_stream.avail_out;
375 :
376 0 : stream->callback ((const char *) outbuf, out_bytes, stream->opaque);
377 0 : lto_stats.num_uncompressed_il_bytes += out_bytes;
378 :
379 0 : cursor += in_bytes;
380 0 : remaining -= in_bytes;
381 :
382 0 : in_stream.next_out = outbuf;
383 0 : in_stream.avail_out = outbuf_length;
384 0 : in_stream.next_in = cursor;
385 0 : in_stream.avail_in = remaining;
386 : }
387 0 : while (!(status == Z_STREAM_END && out_bytes == 0));
388 :
389 0 : status = inflateEnd (&in_stream);
390 0 : if (status != Z_OK)
391 0 : internal_error ("compressed stream: %s", zError (status));
392 : }
393 :
394 0 : lto_destroy_compression_stream (stream);
395 0 : free (outbuf);
396 0 : timevar_pop (TV_IPA_LTO_DECOMPRESS);
397 0 : }
398 :
399 : void
400 208718 : lto_end_uncompression (struct lto_compression_stream *stream,
401 : lto_compression compression)
402 : {
403 : #ifdef HAVE_ZSTD_H
404 208718 : if (compression == ZSTD)
405 : {
406 208718 : lto_uncompression_zstd (stream);
407 208718 : return;
408 : }
409 : #endif
410 0 : if (compression == ZSTD)
411 : fatal_error (UNKNOWN_LOCATION, "compiler does not support ZSTD LTO compression");
412 :
413 0 : lto_uncompression_zlib (stream);
414 : }
|
