Line data Source code
1 : /* File caching.
2 : Copyright (C) 2023-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #define INCLUDE_ALGORITHM
21 : #define INCLUDE_STRING
22 : #define INCLUDE_ARRAY
23 : #define INCLUDE_MAP
24 : #define INCLUDE_VECTOR
25 : #include "config.h"
26 : #include "system.h"
27 : #include "sha1.h"
28 : #include "lto-ltrans-cache.h"
29 :
30 : static const checksum_t NULL_CHECKSUM = {
31 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
32 : 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
33 : };
34 :
35 : /* Computes checksum for given file, returns NULL_CHECKSUM if not
36 : possible. */
37 : static checksum_t
38 0 : file_checksum (char const *filename)
39 : {
40 0 : FILE *file = fopen (filename, "rb");
41 :
42 0 : if (!file)
43 0 : return NULL_CHECKSUM;
44 :
45 0 : checksum_t result = NULL_CHECKSUM;
46 :
47 0 : int ret = sha1_stream (file, &result);
48 :
49 0 : if (ret)
50 0 : result = NULL_CHECKSUM;
51 :
52 0 : fclose (file);
53 :
54 0 : return result;
55 : }
56 :
57 : /* Checks identity of two files. */
58 : static bool
59 0 : files_identical (char const *first_filename, char const *second_filename)
60 : {
61 0 : bool ret = true;
62 :
63 : #if HAVE_MMAP_FILE
64 0 : struct stat st;
65 0 : if (stat (first_filename, &st) < 0 || !S_ISREG (st.st_mode))
66 : return false;
67 0 : size_t len = st.st_size;
68 0 : if (stat (second_filename, &st) < 0 || !S_ISREG (st.st_mode))
69 : return false;
70 0 : if (len != (size_t) st.st_size)
71 : return false;
72 :
73 0 : int fd;
74 0 : void *map[2] = { NULL, NULL };
75 :
76 0 : fd = open (first_filename, O_RDONLY);
77 0 : if (fd < 0)
78 0 : goto fail_mmap;
79 0 : map[0] = mmap (NULL, len, PROT_READ, MAP_PRIVATE, fd, 0);
80 0 : close (fd);
81 0 : if (map[0] == MAP_FAILED)
82 0 : goto fail_mmap;
83 :
84 0 : fd = open (second_filename, O_RDONLY);
85 0 : if (fd < 0)
86 0 : goto fail_mmap2;
87 0 : map[1] = mmap (NULL, len, PROT_READ, MAP_PRIVATE, fd, 0);
88 0 : close (fd);
89 0 : if (map[1] == MAP_FAILED)
90 0 : goto fail_mmap2;
91 :
92 0 : ret = memcmp (map[0], map[1], len) == 0;
93 0 : munmap (map[1], len);
94 0 : munmap (map[0], len);
95 0 : return ret;
96 :
97 0 : fail_mmap2:
98 0 : munmap (map[0], len);
99 0 : fail_mmap:
100 : #endif
101 :
102 : /* Fallback. */
103 :
104 0 : FILE *f_first = fopen (first_filename, "rb");
105 0 : if (!f_first)
106 : return false;
107 :
108 0 : FILE *f_second = fopen (second_filename, "rb");
109 0 : if (!f_second)
110 : {
111 0 : fclose (f_first);
112 0 : return false;
113 : }
114 :
115 0 : const size_t buffer_len = 64 * 1024;
116 0 : uint8_t *buffer1 = (uint8_t*)xmalloc (buffer_len);
117 0 : uint8_t *buffer2 = (uint8_t*)xmalloc (buffer_len);
118 :
119 0 : size_t len1, len2;
120 :
121 0 : do
122 : {
123 0 : len1 = fread (buffer1, 1, buffer_len, f_first);
124 0 : len2 = fread (buffer2, 1, buffer_len, f_second);
125 :
126 0 : if (len1 != len2 || memcmp (buffer1, buffer2, len1) != 0)
127 : {
128 : ret = false;
129 : break;
130 : }
131 : }
132 0 : while (len1 != 0);
133 :
134 0 : free (buffer2);
135 0 : free (buffer1);
136 0 : fclose (f_first);
137 0 : fclose (f_second);
138 0 : return ret;
139 : }
140 :
141 : /* Contructor of cache item. */
142 0 : ltrans_file_cache::item::item (std::string input, std::string output,
143 : checksum_t input_checksum,
144 0 : uint32_t last_used):
145 0 : input (std::move (input)), output (std::move (output)),
146 0 : input_checksum (input_checksum), last_used (last_used)
147 : {
148 0 : lock = lockfile (this->input + ".lock");
149 0 : }
150 : /* Destructor of cache item. */
151 0 : ltrans_file_cache::item::~item ()
152 : {
153 0 : lock.unlock ();
154 0 : }
155 :
156 : /* Reads next cache item from cachedata file.
157 : Adds `dir/` prefix to filenames. */
158 : static ltrans_file_cache::item*
159 0 : read_cache_item (FILE* f, const char* dir)
160 : {
161 0 : checksum_t checksum;
162 0 : uint32_t last_used;
163 :
164 0 : if (fread (&checksum, 1, checksum.size (), f) != checksum.size ())
165 : return NULL;
166 0 : if (fread (&last_used, sizeof (last_used), 1, f) != 1)
167 : return NULL;
168 :
169 0 : std::string input (dir);
170 0 : input.push_back ('/');
171 0 : std::string output = input; /* Copy. */
172 :
173 0 : int c;
174 0 : while ((c = getc (f)))
175 : {
176 0 : if (c == EOF)
177 : return NULL;
178 0 : input.push_back (c);
179 : }
180 0 : while ((c = getc (f)))
181 : {
182 0 : if (c == EOF)
183 : return NULL;
184 0 : output.push_back (c);
185 : }
186 :
187 0 : return new ltrans_file_cache::item (&input[0], &output[0], checksum,
188 0 : last_used);
189 0 : }
190 :
191 : /* Writes cache item to cachedata file.
192 : Removes `dir/` prefix from filenames. */
193 : static void
194 0 : write_cache_item (FILE* f, ltrans_file_cache::item *item, const char* dir)
195 : {
196 0 : fwrite (&item->input_checksum, 1, item->input_checksum.size (), f);
197 0 : fwrite (&item->last_used, sizeof (item->last_used), 1, f);
198 :
199 0 : gcc_assert (item->input.size () > strlen (dir));
200 0 : fputs (item->input.c_str () + strlen (dir) + 1, f);
201 0 : fputc (0, f);
202 :
203 0 : gcc_assert (item->output.size () > strlen (dir));
204 0 : fputs (item->output.c_str () + strlen (dir) + 1, f);
205 0 : fputc (0, f);
206 0 : }
207 :
208 : /* Constructor. Resulting cache item filenames will be
209 : in format `prefix%d[.ltrans]suffix`. */
210 7817 : ltrans_file_cache::ltrans_file_cache (const char* dir, const char* prefix,
211 : const char* suffix,
212 7817 : size_t soft_cache_size):
213 7817 : dir (dir), suffix (suffix), soft_cache_size (soft_cache_size)
214 : {
215 7817 : if (!dir) return;
216 :
217 0 : creation_lock = lockfile (std::string (dir) + "/lockfile_creation");
218 0 : deletion_lock = lockfile (std::string (dir) + "/lockfile_deletion");
219 :
220 0 : cache_prefix = std::string (dir) + "/" + prefix;
221 0 : cache_free_idx = 0;
222 :
223 0 : str_buffer = (char *)xmalloc (cache_prefix.size ()
224 : + sizeof ("4294967295.ltrans")
225 0 : + strlen (suffix));
226 : }
227 :
228 : /* Destructor. */
229 7817 : ltrans_file_cache::~ltrans_file_cache ()
230 : {
231 7817 : if (!*this)
232 7817 : return;
233 :
234 0 : cleanup ();
235 0 : free (str_buffer);
236 7817 : }
237 :
238 : /* Adds given cache item to all relevant datastructures. */
239 : void
240 0 : ltrans_file_cache::add_item (ltrans_file_cache::item* item)
241 : {
242 0 : items.push_back (item);
243 0 : map_checksum[item->input_checksum] = item;
244 0 : map_input[item->input] = item;
245 :
246 0 : usage_counter = std::max (usage_counter, item->last_used);
247 0 : }
248 :
249 : /* Creates cachedata filename for save/load. */
250 : std::string
251 0 : ltrans_file_cache::filename_cachedata ()
252 : {
253 0 : return std::string (dir) + "/cachedata";
254 : }
255 :
256 : /* Loads data about previously cached items from cachedata file.
257 : Sorts items by last_used and remaps last_used to small integers.
258 :
259 : Must be called with creation_lock or deletion_lock held to
260 : prevent data race. */
261 : void
262 0 : ltrans_file_cache::load_cache ()
263 : {
264 0 : cleanup ();
265 :
266 0 : std::string filename = filename_cachedata ();
267 0 : FILE *file = fopen (filename.c_str (), "rb");
268 :
269 0 : if (!file)
270 0 : return;
271 :
272 : ltrans_file_cache::item* _item;
273 0 : while ((_item = read_cache_item (file, dir)))
274 0 : add_item (_item);
275 :
276 0 : fclose (file);
277 :
278 0 : std::sort (items.begin (), items.end (),
279 0 : [](item* a, item* b)
280 0 : {return a->last_used < b->last_used;});
281 :
282 0 : for (size_t i = 0; i < items.size (); ++i)
283 0 : items[i]->last_used = (uint32_t) i;
284 0 : usage_counter = (uint32_t) items.size ();
285 0 : }
286 :
287 : /* Rewrites data about cache items into cachedata file.
288 :
289 : Must be only called when creation_lock or deletion_lock was held since last
290 : call to load_cache. */
291 : void
292 0 : ltrans_file_cache::save_cache ()
293 : {
294 0 : std::string filename = filename_cachedata ();
295 0 : FILE *file = fopen (filename.c_str (), "wb");
296 :
297 0 : if (!file)
298 0 : return;
299 :
300 0 : for (item* _item: items)
301 0 : write_cache_item (file, _item, dir);
302 :
303 0 : fclose (file);
304 0 : }
305 :
306 : /* Creates new cache item with given checksum.
307 : New input/output files are chosen to not collide with other items.
308 :
309 : Must be called with creation_lock held to prevent data race. */
310 : ltrans_file_cache::item*
311 0 : ltrans_file_cache::create_item (const checksum_t& checksum)
312 : {
313 0 : size_t prefix_len = cache_prefix.size ();
314 :
315 0 : strcpy (str_buffer, cache_prefix.c_str ());
316 :
317 0 : for (;;)
318 : {
319 0 : sprintf (str_buffer + prefix_len, "%04u%s", cache_free_idx, suffix);
320 :
321 0 : if (map_input.find (str_buffer) == map_input.end ())
322 : break;
323 0 : cache_free_idx++;
324 : }
325 :
326 0 : std::string input = str_buffer;
327 :
328 0 : sprintf (str_buffer + prefix_len, "%04u.ltrans%s", cache_free_idx, suffix);
329 :
330 0 : return new item (std::move (input), str_buffer, checksum, usage_counter++);
331 0 : }
332 :
333 : /* Adds input file into cache. Cache item with input file identical to
334 : added input file will be returned as _item.
335 : If the file was already cached, `true` is returned, `false` otherwise.
336 : The added input file is deleted (or moved).
337 :
338 : Must be called with creation_lock held to prevent data race. */
339 : bool
340 0 : ltrans_file_cache::add_to_cache (const char* filename, item*& _item)
341 : {
342 0 : checksum_t checksum = file_checksum (filename);
343 :
344 0 : auto it = map_checksum.find (checksum);
345 :
346 0 : if (it != map_checksum.end ()
347 0 : && files_identical (filename, it->second->input.c_str ()))
348 : {
349 0 : unlink (filename);
350 0 : _item = it->second;
351 0 : _item->last_used = usage_counter++;
352 0 : return true;
353 : }
354 : else
355 : {
356 : /* Cache miss. Move into cache dir. */
357 0 : _item = create_item (checksum);
358 0 : add_item (_item);
359 :
360 0 : rename (filename, _item->input.c_str ());
361 0 : return false;
362 : }
363 : }
364 :
365 : /* If exists, returns cache item corresponding to cached input file. */
366 : ltrans_file_cache::item*
367 0 : ltrans_file_cache::get_item (const char* input)
368 : {
369 0 : auto it = map_input.find (input);
370 0 : if (it == map_input.end ())
371 : return NULL;
372 0 : return it->second;
373 : }
374 :
375 : /* If no other process holds the deletion_lock, prunes oldest unused cache
376 : items over limit. */
377 : void
378 0 : ltrans_file_cache::try_prune ()
379 : {
380 0 : if (deletion_lock.try_lock_write () == 0)
381 : {
382 0 : prune ();
383 0 : deletion_lock.unlock ();
384 : }
385 0 : }
386 :
387 : /* Returns true if file exists. */
388 : static int
389 0 : file_exists (char const *name)
390 : {
391 0 : return access (name, R_OK) == 0;
392 : }
393 :
394 : /* Prunes oldest unused cache items over limit.
395 : Must be called with deletion_lock held to prevent data race. */
396 : void
397 0 : ltrans_file_cache::prune ()
398 : {
399 0 : load_cache ();
400 0 : if (items.size () > soft_cache_size)
401 : {
402 0 : std::vector<item*> sorted_items = std::move (items);
403 :
404 0 : cleanup ();
405 :
406 0 : for (size_t i = 0; i < sorted_items.size (); ++i)
407 : {
408 0 : ltrans_file_cache::item* item = sorted_items[i];
409 0 : if ((i < sorted_items.size () - soft_cache_size)
410 0 : || !file_exists (item->input.c_str ())
411 0 : || !file_exists (item->output.c_str ()))
412 : {
413 0 : unlink (item->input.c_str ());
414 0 : unlink (item->output.c_str ());
415 0 : delete item;
416 : }
417 : else
418 0 : add_item (item);
419 : }
420 0 : }
421 0 : save_cache ();
422 0 : }
423 :
424 : /* Clears cache class, as if only constructor was called. */
425 : void
426 0 : ltrans_file_cache::cleanup ()
427 : {
428 0 : map_checksum.clear ();
429 0 : map_input.clear ();
430 :
431 0 : for (ltrans_file_cache::item* item: items)
432 0 : delete item;
433 0 : items.clear ();
434 :
435 0 : usage_counter = 0;
436 0 : }
|
