Line data Source code
1 : /* Data and functions related to line maps and input files.
2 : Copyright (C) 2004-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "intl.h"
24 : #include "diagnostic.h"
25 : #include "diagnostics/file-cache.h"
26 : #include "selftest.h"
27 : #include "cpplib.h"
28 :
29 : #ifndef HAVE_ICONV
30 : #define HAVE_ICONV 0
31 : #endif
32 :
33 : const char *
34 7047978 : special_fname_builtin ()
35 : {
36 7047978 : return _("<built-in>");
37 : }
38 :
39 : /* Current position in real source file. */
40 :
41 : location_t input_location = UNKNOWN_LOCATION;
42 :
43 : class line_maps *line_table;
44 :
45 : /* A stashed copy of "line_table" for use by selftest::line_table_test.
46 : This needs to be a global so that it can be a GC root, and thus
47 : prevent the stashed copy from being garbage-collected if the GC runs
48 : during a line_table_test. */
49 :
50 : class line_maps *saved_line_table;
51 :
52 : /* Expand the source location LOC into a human readable location. If
53 : LOC resolves to a builtin location, the file name of the readable
54 : location is set to the string "<built-in>". If EXPANSION_POINT_P is
55 : TRUE and LOC is virtual, then it is resolved to the expansion
56 : point of the involved macro. Otherwise, it is resolved to the
57 : spelling location of the token.
58 :
59 : When resolving to the spelling location of the token, if the
60 : resulting location is for a built-in location (that is, it has no
61 : associated line/column) in the context of a macro expansion, the
62 : returned location is the first one (while unwinding the macro
63 : location towards its expansion point) that is in real source
64 : code.
65 :
66 : ASPECT controls which part of the location to use. */
67 :
68 : static expanded_location
69 975181829 : expand_location_1 (const line_maps *set,
70 : location_t loc,
71 : bool expansion_point_p,
72 : enum location_aspect aspect)
73 : {
74 975181829 : expanded_location xloc;
75 975181829 : const line_map_ordinary *map;
76 975181829 : enum location_resolution_kind lrk = LRK_MACRO_EXPANSION_POINT;
77 975181829 : tree block = NULL;
78 :
79 975181829 : if (IS_ADHOC_LOC (loc))
80 : {
81 271601607 : block = LOCATION_BLOCK (loc);
82 271601607 : loc = LOCATION_LOCUS (loc);
83 : }
84 :
85 975181829 : memset (&xloc, 0, sizeof (xloc));
86 :
87 975181829 : if (loc >= RESERVED_LOCATION_COUNT)
88 : {
89 917754809 : if (!expansion_point_p)
90 : {
91 : /* We want to resolve LOC to its spelling location.
92 :
93 : But if that spelling location is a reserved location that
94 : appears in the context of a macro expansion (like for a
95 : location for a built-in token), let's consider the first
96 : location (toward the expansion point) that is not reserved;
97 : that is, the first location that is in real source code. */
98 2286788 : loc = linemap_unwind_to_first_non_reserved_loc (set,
99 : loc, NULL);
100 2286788 : lrk = LRK_SPELLING_LOCATION;
101 : }
102 917754809 : loc = linemap_resolve_location (set, loc, lrk, &map);
103 :
104 : /* loc is now either in an ordinary map, or is a reserved location.
105 : If it is a compound location, the caret is in a spelling location,
106 : but the start/finish might still be a virtual location.
107 : Depending of what the caller asked for, we may need to recurse
108 : one level in order to resolve any virtual locations in the
109 : end-points. */
110 917754809 : switch (aspect)
111 : {
112 0 : default:
113 0 : gcc_unreachable ();
114 : /* Fall through. */
115 : case location_aspect::caret:
116 : break;
117 449995 : case location_aspect::start:
118 449995 : {
119 449995 : location_t start = get_start (loc);
120 449995 : if (start != loc)
121 1183 : return expand_location_1 (set, start, expansion_point_p, aspect);
122 : }
123 : break;
124 102042 : case location_aspect::finish:
125 102042 : {
126 102042 : location_t finish = get_finish (loc);
127 102042 : if (finish != loc)
128 1141 : return expand_location_1 (set, finish, expansion_point_p, aspect);
129 : }
130 : break;
131 : }
132 917752485 : xloc = linemap_expand_location (set, map, loc);
133 : }
134 :
135 975179505 : xloc.data = block;
136 975179505 : if (loc <= BUILTINS_LOCATION)
137 57427020 : xloc.file = loc == UNKNOWN_LOCATION ? NULL : special_fname_builtin ();
138 :
139 975179505 : return xloc;
140 : }
141 :
142 : /* Return a NUL-terminated copy of the source text between two locations, or
143 : NULL if the arguments are invalid. The caller is responsible for freeing
144 : the return value. */
145 :
146 : char *
147 1584 : get_source_text_between (diagnostics::file_cache &fc,
148 : location_t start, location_t end)
149 : {
150 1584 : expanded_location expstart
151 1584 : = expand_location_to_spelling_point (start, location_aspect::start);
152 1584 : expanded_location expend
153 1584 : = expand_location_to_spelling_point (end, location_aspect::finish);
154 :
155 : /* If the locations are in different files or the end comes before the
156 : start, give up and return nothing. */
157 1584 : if (!expstart.file || !expend.file)
158 : return NULL;
159 1582 : if (strcmp (expstart.file, expend.file) != 0)
160 : return NULL;
161 1582 : if (expstart.line > expend.line)
162 : return NULL;
163 1582 : if (expstart.line == expend.line
164 1577 : && expstart.column > expend.column)
165 : return NULL;
166 : /* These aren't real column numbers, give up. */
167 1582 : if (expstart.column == 0 || expend.column == 0)
168 : return NULL;
169 :
170 : /* For a single line we need to trim both edges. */
171 1582 : if (expstart.line == expend.line)
172 : {
173 1577 : diagnostics::char_span line
174 1577 : = fc.get_source_line (expstart.file, expstart.line);
175 1577 : if (line.length () < 1)
176 : return NULL;
177 1577 : int s = expstart.column - 1;
178 1577 : int len = expend.column - s;
179 1577 : if (line.length () < (size_t)expend.column)
180 : return NULL;
181 1577 : return line.subspan (s, len).xstrdup ();
182 : }
183 :
184 5 : struct obstack buf_obstack;
185 5 : obstack_init (&buf_obstack);
186 :
187 : /* Loop through all lines in the range and append each to buf; may trim
188 : parts of the start and end lines off depending on column values. */
189 48 : for (int lnum = expstart.line; lnum <= expend.line; ++lnum)
190 : {
191 38 : diagnostics::char_span line = fc.get_source_line (expstart.file, lnum);
192 38 : if (line.length () < 1 && (lnum != expstart.line && lnum != expend.line))
193 0 : continue;
194 :
195 : /* For the first line in the range, only start at expstart.column */
196 38 : if (lnum == expstart.line)
197 : {
198 5 : unsigned off = expstart.column - 1;
199 5 : if (line.length () < off)
200 0 : return NULL;
201 5 : line = line.subspan (off, line.length() - off);
202 : }
203 : /* For the last line, don't go past expend.column */
204 33 : else if (lnum == expend.line)
205 : {
206 5 : if (line.length () < (size_t)expend.column)
207 : return NULL;
208 5 : line = line.subspan (0, expend.column);
209 : }
210 :
211 : /* Combine spaces at the beginning of later lines. */
212 38 : if (lnum > expstart.line)
213 : {
214 : unsigned off;
215 383 : for (off = 0; off < line.length(); ++off)
216 383 : if (line[off] != ' ' && line[off] != '\t')
217 : break;
218 33 : if (off > 0)
219 : {
220 33 : obstack_1grow (&buf_obstack, ' ');
221 33 : line = line.subspan (off, line.length() - off);
222 : }
223 : }
224 :
225 : /* This does not include any trailing newlines. */
226 38 : obstack_grow (&buf_obstack, line.get_buffer (), line.length ());
227 : }
228 :
229 : /* NUL-terminate and finish the buf obstack. */
230 5 : obstack_1grow (&buf_obstack, 0);
231 5 : const char *buf = (const char *) obstack_finish (&buf_obstack);
232 :
233 5 : return xstrdup (buf);
234 : }
235 :
236 : /* Test if the location originates from the spelling location of a
237 : builtin-tokens. That is, return TRUE if LOC is a (possibly
238 : virtual) location of a built-in token that appears in the expansion
239 : list of a macro. Please note that this function also works on
240 : tokens that result from built-in tokens. For instance, the
241 : function would return true if passed a token "4" that is the result
242 : of the expansion of the built-in __LINE__ macro. */
243 : bool
244 16612 : is_location_from_builtin_token (location_t loc)
245 : {
246 16612 : const line_map_ordinary *map = NULL;
247 16612 : loc = linemap_resolve_location (line_table, loc,
248 : LRK_SPELLING_LOCATION, &map);
249 16612 : return loc == BUILTINS_LOCATION;
250 : }
251 :
252 : /* Expand the source location LOC into a human readable location. If
253 : LOC is virtual, it resolves to the expansion point of the involved
254 : macro. If LOC resolves to a builtin location, the file name of the
255 : readable location is set to the string "<built-in>". */
256 :
257 : expanded_location
258 972890526 : expand_location (location_t loc)
259 : {
260 972890526 : return expand_location_1 (line_table, loc, /*expansion_point_p=*/true,
261 972890526 : location_aspect::caret);
262 : }
263 :
264 : /* Expand the source location LOC into a human readable location. If
265 : LOC is virtual, it resolves to the expansion location of the
266 : relevant macro. If LOC resolves to a builtin location, the file
267 : name of the readable location is set to the string
268 : "<built-in>". */
269 :
270 : expanded_location
271 85926 : expand_location_to_spelling_point (location_t loc,
272 : enum location_aspect aspect)
273 : {
274 85926 : return expand_location_1 (line_table, loc, /*expansion_point_p=*/false,
275 85926 : aspect);
276 : }
277 :
278 : /* The rich_location class within libcpp requires a way to expand
279 : location_t instances, and relies on the client code
280 : providing a symbol named
281 : linemap_client_expand_location_to_spelling_point
282 : to do this.
283 :
284 : This is the implementation for libcommon.a (all host binaries),
285 : which simply calls into expand_location_1. */
286 :
287 : expanded_location
288 2203053 : linemap_client_expand_location_to_spelling_point (const line_maps *set,
289 : location_t loc,
290 : enum location_aspect aspect)
291 : {
292 2203053 : return expand_location_1 (set, loc, /*expansion_point_p=*/false, aspect);
293 : }
294 :
295 :
296 : /* If LOCATION is in a system header and if it is a virtual location
297 : for a token coming from the expansion of a macro, unwind it to
298 : the location of the expansion point of the macro. If the expansion
299 : point is also in a system header return the original LOCATION.
300 : Otherwise, return the location of the expansion point.
301 :
302 : This is used for instance when we want to emit diagnostics about a
303 : token that may be located in a macro that is itself defined in a
304 : system header, for example, for the NULL macro. In such a case, if
305 : LOCATION were passed directly to diagnostic functions such as
306 : warning_at, the diagnostic would be suppressed (unless
307 : -Wsystem-headers). */
308 :
309 : location_t
310 503979564 : expansion_point_location_if_in_system_header (location_t location)
311 : {
312 503979564 : if (!in_system_header_at (location))
313 : return location;
314 :
315 382448292 : location_t xloc = linemap_resolve_location (line_table, location,
316 : LRK_MACRO_EXPANSION_POINT,
317 : NULL);
318 382448292 : return in_system_header_at (xloc) ? location : xloc;
319 : }
320 :
321 : /* If LOCATION is a virtual location for a token coming from the expansion
322 : of a macro, unwind to the location of the expansion point of the macro. */
323 :
324 : location_t
325 3120727 : expansion_point_location (location_t location)
326 : {
327 3120727 : return linemap_resolve_location (line_table, location,
328 3120727 : LRK_MACRO_EXPANSION_POINT, NULL);
329 : }
330 :
331 : /* Construct a location with caret at CARET, ranging from START to
332 : FINISH.
333 :
334 : For example, consider:
335 :
336 : 11111111112
337 : 12345678901234567890
338 : 522
339 : 523 return foo + bar;
340 : ~~~~^~~~~
341 : 524
342 :
343 : The location's caret is at the "+", line 523 column 15, but starts
344 : earlier, at the "f" of "foo" at column 11. The finish is at the "r"
345 : of "bar" at column 19. */
346 :
347 : location_t
348 2853449549 : make_location (location_t caret, location_t start, location_t finish)
349 : {
350 2853449549 : return line_table->make_location (caret, start, finish);
351 : }
352 :
353 : /* Same as above, but taking a source range rather than two locations. */
354 :
355 : location_t
356 1986418584 : make_location (location_t caret, source_range src_range)
357 : {
358 1986418584 : location_t pure_loc = get_pure_location (caret);
359 1986418584 : return line_table->get_or_create_combined_loc (pure_loc, src_range,
360 1986418584 : nullptr, 0);
361 : }
362 :
363 : /* An expanded_location stores the column in byte units. This function
364 : converts that column to display units. That requires reading the associated
365 : source line in order to calculate the display width. If that cannot be done
366 : for any reason, then returns the byte column as a fallback. */
367 : int
368 788240 : location_compute_display_column (diagnostics::file_cache &fc,
369 : expanded_location exploc,
370 : const cpp_char_column_policy &policy)
371 : {
372 788240 : if (!(exploc.file && *exploc.file && exploc.line && exploc.column))
373 : return exploc.column;
374 752730 : diagnostics::char_span line = fc.get_source_line (exploc.file, exploc.line);
375 : /* If line is NULL, this function returns exploc.column which is the
376 : desired fallback. */
377 752730 : return cpp_byte_column_to_display_column (line.get_buffer (), line.length (),
378 752730 : exploc.column, policy);
379 : }
380 :
381 : /* Dump statistics to stderr about the memory usage of the line_table
382 : set of line maps. This also displays some statistics about macro
383 : expansion. */
384 :
385 : void
386 0 : dump_line_table_statistics (void)
387 : {
388 0 : struct linemap_stats s;
389 0 : long total_used_map_size,
390 : macro_maps_size,
391 : total_allocated_map_size;
392 :
393 0 : memset (&s, 0, sizeof (s));
394 :
395 0 : linemap_get_statistics (line_table, &s);
396 :
397 0 : macro_maps_size = s.macro_maps_used_size
398 0 : + s.macro_maps_locations_size;
399 :
400 0 : total_allocated_map_size = s.ordinary_maps_allocated_size
401 0 : + s.macro_maps_allocated_size
402 : + s.macro_maps_locations_size;
403 :
404 0 : total_used_map_size = s.ordinary_maps_used_size
405 0 : + s.macro_maps_used_size
406 : + s.macro_maps_locations_size;
407 :
408 0 : fprintf (stderr, "Number of expanded macros: %5ld\n",
409 : s.num_expanded_macros);
410 0 : if (s.num_expanded_macros != 0)
411 0 : fprintf (stderr, "Average number of tokens per macro expansion: %5ld\n",
412 0 : s.num_macro_tokens / s.num_expanded_macros);
413 0 : fprintf (stderr,
414 : "\nLine Table allocations during the "
415 : "compilation process\n");
416 0 : fprintf (stderr, "Number of ordinary maps used: " PRsa (5) "\n",
417 0 : SIZE_AMOUNT (s.num_ordinary_maps_used));
418 0 : fprintf (stderr, "Ordinary map used size: " PRsa (5) "\n",
419 0 : SIZE_AMOUNT (s.ordinary_maps_used_size));
420 0 : fprintf (stderr, "Number of ordinary maps allocated: " PRsa (5) "\n",
421 0 : SIZE_AMOUNT (s.num_ordinary_maps_allocated));
422 0 : fprintf (stderr, "Ordinary maps allocated size: " PRsa (5) "\n",
423 0 : SIZE_AMOUNT (s.ordinary_maps_allocated_size));
424 0 : fprintf (stderr, "Number of macro maps used: " PRsa (5) "\n",
425 0 : SIZE_AMOUNT (s.num_macro_maps_used));
426 0 : fprintf (stderr, "Macro maps used size: " PRsa (5) "\n",
427 0 : SIZE_AMOUNT (s.macro_maps_used_size));
428 0 : fprintf (stderr, "Macro maps locations size: " PRsa (5) "\n",
429 0 : SIZE_AMOUNT (s.macro_maps_locations_size));
430 0 : fprintf (stderr, "Macro maps size: " PRsa (5) "\n",
431 0 : SIZE_AMOUNT (macro_maps_size));
432 0 : fprintf (stderr, "Duplicated maps locations size: " PRsa (5) "\n",
433 0 : SIZE_AMOUNT (s.duplicated_macro_maps_locations_size));
434 0 : fprintf (stderr, "Total allocated maps size: " PRsa (5) "\n",
435 0 : SIZE_AMOUNT (total_allocated_map_size));
436 0 : fprintf (stderr, "Total used maps size: " PRsa (5) "\n",
437 0 : SIZE_AMOUNT (total_used_map_size));
438 0 : fprintf (stderr, "Ad-hoc table size: " PRsa (5) "\n",
439 0 : SIZE_AMOUNT (s.adhoc_table_size));
440 0 : fprintf (stderr, "Ad-hoc table entries used: " PRsa (5) "\n",
441 0 : SIZE_AMOUNT (s.adhoc_table_entries_used));
442 0 : fprintf (stderr, "optimized_ranges: " PRsa (5) "\n",
443 0 : SIZE_AMOUNT (line_table->m_num_optimized_ranges));
444 0 : fprintf (stderr, "unoptimized_ranges: " PRsa (5) "\n",
445 0 : SIZE_AMOUNT (line_table->m_num_unoptimized_ranges));
446 :
447 0 : fprintf (stderr, "\n");
448 0 : }
449 :
450 : /* Get location one beyond the final location in ordinary map IDX. */
451 :
452 : static location_t
453 6 : get_end_location (class line_maps *set, line_map_uint_t idx)
454 : {
455 6 : if (idx == LINEMAPS_ORDINARY_USED (set) - 1)
456 1 : return set->highest_location;
457 :
458 5 : struct line_map *next_map = LINEMAPS_ORDINARY_MAP_AT (set, idx + 1);
459 5 : return MAP_START_LOCATION (next_map);
460 : }
461 :
462 : /* Helper function for write_digit_row. */
463 :
464 : static void
465 11500 : write_digit (FILE *stream, int digit)
466 : {
467 0 : fputc ('0' + digit, stream);
468 0 : }
469 :
470 : /* Helper function for dump_location_info.
471 : Write a row of numbers to STREAM, numbering a source line,
472 : giving the units, tens, hundreds etc of the column number. */
473 :
474 : static void
475 296 : write_digit_row (FILE *stream, int indent,
476 : const line_map_ordinary *map,
477 : location_t loc, int max_col, int divisor)
478 : {
479 296 : fprintf (stream, "%*c", indent, ' ');
480 296 : fprintf (stream, "|");
481 12092 : for (int column = 1; column < max_col; column++)
482 : {
483 11500 : location_t column_loc = loc + (location_t (column) << map->m_range_bits);
484 11500 : write_digit (stream, (column_loc / divisor) % 10);
485 : }
486 296 : fprintf (stream, "\n");
487 296 : }
488 :
489 : /* Write a half-closed (START) / half-open (END) interval of
490 : location_t to STREAM. */
491 :
492 : static void
493 12 : dump_location_range (FILE *stream,
494 : location_t start, location_t end)
495 : {
496 6 : fprintf (stream,
497 : " location_t interval: %llu <= loc < %llu\n",
498 : (unsigned long long) start, (unsigned long long) end);
499 0 : }
500 :
501 : /* Write a labelled description of a half-closed (START) / half-open (END)
502 : interval of location_t to STREAM. */
503 :
504 : static void
505 4 : dump_labelled_location_range (FILE *stream,
506 : const char *name,
507 : location_t start, location_t end)
508 : {
509 4 : fprintf (stream, "%s\n", name);
510 4 : dump_location_range (stream, start, end);
511 4 : fprintf (stream, "\n");
512 4 : }
513 :
514 : /* Write a visualization of the locations in the line_table to STREAM. */
515 :
516 : void
517 1 : dump_location_info (FILE *stream)
518 : {
519 1 : diagnostics::file_cache fc;
520 :
521 : /* Visualize the reserved locations. */
522 1 : dump_labelled_location_range (stream, "RESERVED LOCATIONS",
523 : 0, RESERVED_LOCATION_COUNT);
524 :
525 1 : using ULL = unsigned long long;
526 :
527 : /* Visualize the ordinary line_map instances, rendering the sources. */
528 8 : for (line_map_uint_t idx = 0; idx < LINEMAPS_ORDINARY_USED (line_table);
529 : idx++)
530 : {
531 6 : location_t end_location = get_end_location (line_table, idx);
532 : /* half-closed: doesn't include this one. */
533 :
534 6 : const line_map_ordinary *map
535 6 : = LINEMAPS_ORDINARY_MAP_AT (line_table, idx);
536 6 : fprintf (stream, "ORDINARY MAP: %llu\n", (ULL) idx);
537 6 : dump_location_range (stream,
538 : MAP_START_LOCATION (map), end_location);
539 6 : fprintf (stream, " file: %s\n", ORDINARY_MAP_FILE_NAME (map));
540 6 : fprintf (stream, " starting at line: %i\n",
541 : ORDINARY_MAP_STARTING_LINE_NUMBER (map));
542 6 : fprintf (stream, " column and range bits: %i\n",
543 6 : map->m_column_and_range_bits);
544 6 : fprintf (stream, " column bits: %i\n",
545 6 : map->m_column_and_range_bits - map->m_range_bits);
546 6 : fprintf (stream, " range bits: %i\n",
547 6 : map->m_range_bits);
548 6 : const char * reason;
549 6 : switch (map->reason) {
550 : case LC_ENTER:
551 : reason = "LC_ENTER";
552 : break;
553 1 : case LC_LEAVE:
554 1 : reason = "LC_LEAVE";
555 1 : break;
556 3 : case LC_RENAME:
557 3 : reason = "LC_RENAME";
558 3 : break;
559 0 : case LC_RENAME_VERBATIM:
560 0 : reason = "LC_RENAME_VERBATIM";
561 0 : break;
562 0 : case LC_ENTER_MACRO:
563 0 : reason = "LC_RENAME_MACRO";
564 0 : break;
565 0 : default:
566 0 : reason = "Unknown";
567 : }
568 6 : fprintf (stream, " reason: %d (%s)\n", map->reason, reason);
569 :
570 6 : const line_map_ordinary *includer_map
571 6 : = linemap_included_from_linemap (line_table, map);
572 6 : fprintf (stream, " included from location: %llu",
573 6 : (ULL) linemap_included_from (map));
574 6 : if (includer_map) {
575 1 : fprintf (stream, " (in ordinary map %llu)",
576 1 : ULL (includer_map - line_table->info_ordinary.maps));
577 : }
578 6 : fprintf (stream, "\n");
579 :
580 : /* Render the span of source lines that this "map" covers. */
581 6 : for (location_t loc = MAP_START_LOCATION (map);
582 9358 : loc < end_location;
583 9352 : loc += (location_t (1) << map->m_range_bits))
584 : {
585 9356 : gcc_assert (pure_location_p (line_table, loc) );
586 :
587 9356 : expanded_location exploc
588 9356 : = linemap_expand_location (line_table, map, loc);
589 :
590 9356 : if (exploc.column == 0)
591 : {
592 : /* Beginning of a new source line: draw the line. */
593 :
594 78 : diagnostics::char_span line_text
595 78 : = fc.get_source_line (exploc.file, exploc.line);
596 78 : if (!line_text)
597 : break;
598 74 : fprintf (stream,
599 : "%s:%3i|loc:%5llu|%.*s\n",
600 : exploc.file, exploc.line,
601 : (ULL) loc,
602 74 : (int)line_text.length (), line_text.get_buffer ());
603 :
604 : /* "loc" is at column 0, which means "the whole line".
605 : Render the locations *within* the line, by underlining
606 : it, showing the location_t numeric values
607 : at each column. */
608 74 : auto max_col = (ULL (1) << map->m_column_and_range_bits) - 1;
609 74 : if (max_col > line_text.length ())
610 74 : max_col = line_text.length () + 1;
611 :
612 74 : int len_lnum = diagnostics::num_digits (exploc.line);
613 74 : if (len_lnum < 3)
614 : len_lnum = 3;
615 74 : int len_loc = diagnostics::num_digits (loc);
616 74 : if (len_loc < 5)
617 : len_loc = 5;
618 :
619 74 : int indent = 6 + strlen (exploc.file) + len_lnum + len_loc;
620 :
621 : /* Thousands. */
622 74 : if (end_location > 999)
623 74 : write_digit_row (stream, indent, map, loc, max_col, 1000);
624 :
625 : /* Hundreds. */
626 74 : if (end_location > 99)
627 74 : write_digit_row (stream, indent, map, loc, max_col, 100);
628 :
629 : /* Tens. */
630 74 : write_digit_row (stream, indent, map, loc, max_col, 10);
631 :
632 : /* Units. */
633 74 : write_digit_row (stream, indent, map, loc, max_col, 1);
634 : }
635 : }
636 6 : fprintf (stream, "\n");
637 : }
638 :
639 : /* Visualize unallocated values. */
640 1 : dump_labelled_location_range (stream, "UNALLOCATED LOCATIONS",
641 : line_table->highest_location,
642 : LINEMAPS_MACRO_LOWEST_LOCATION (line_table));
643 :
644 : /* Visualize the macro line_map instances, rendering the sources. */
645 4 : for (line_map_uint_t i = 0; i < LINEMAPS_MACRO_USED (line_table); i++)
646 : {
647 : /* Each macro map that is allocated owns location_t values
648 : that are *lower* that the one before them.
649 : Hence it's meaningful to view them either in order of ascending
650 : source locations, or in order of ascending macro map index. */
651 2 : const bool ascending_location_ts = true;
652 2 : auto idx = (ascending_location_ts
653 2 : ? (LINEMAPS_MACRO_USED (line_table) - (i + 1))
654 2 : : i);
655 2 : const line_map_macro *map = LINEMAPS_MACRO_MAP_AT (line_table, idx);
656 2 : fprintf (stream, "MACRO %llu: %s (%u tokens)\n",
657 : (ULL) idx,
658 : linemap_map_get_macro_name (map),
659 : MACRO_MAP_NUM_MACRO_TOKENS (map));
660 4 : dump_location_range (stream,
661 2 : map->start_location,
662 2 : (map->start_location
663 2 : + MACRO_MAP_NUM_MACRO_TOKENS (map)));
664 2 : inform (map->get_expansion_point_location (),
665 : "expansion point is location %llu",
666 2 : (ULL) map->get_expansion_point_location ());
667 2 : fprintf (stream, " map->start_location: %llu\n",
668 2 : (ULL) map->start_location);
669 :
670 2 : fprintf (stream, " macro_locations:\n");
671 6 : for (unsigned int i = 0; i < MACRO_MAP_NUM_MACRO_TOKENS (map); i++)
672 : {
673 2 : location_t x = MACRO_MAP_LOCATIONS (map)[2 * i];
674 2 : location_t y = MACRO_MAP_LOCATIONS (map)[(2 * i) + 1];
675 :
676 : /* linemap_add_macro_token encodes token numbers in an expansion
677 : by putting them after MAP_START_LOCATION. */
678 :
679 : /* I'm typically seeing 4 uninitialized entries at the end of
680 : 0xafafafaf.
681 : This appears to be due to macro.cc:replace_args
682 : adding 2 extra args for padding tokens; presumably there may
683 : be a leading and/or trailing padding token injected,
684 : each for 2 more location slots.
685 : This would explain there being up to 4 location_ts slots
686 : that may be uninitialized. */
687 :
688 2 : fprintf (stream, " %u: %llu, %llu\n",
689 : i,
690 : (ULL) x,
691 : (ULL) y);
692 2 : if (x == y)
693 : {
694 2 : if (x < MAP_START_LOCATION (map))
695 2 : inform (x, "token %u has %<x-location == y-location == %llu%>",
696 : i, (ULL) x);
697 : else
698 0 : fprintf (stream,
699 : "x-location == y-location == %llu"
700 : " encodes token # %u\n",
701 : (ULL) x,
702 0 : (unsigned int)(x - MAP_START_LOCATION (map)));
703 : }
704 : else
705 : {
706 0 : inform (x, "token %u has %<x-location == %llu%>", i, (ULL) x);
707 0 : inform (x, "token %u has %<y-location == %llu%>", i, (ULL) y);
708 : }
709 : }
710 2 : fprintf (stream, "\n");
711 : }
712 :
713 : /* It appears that MAX_LOCATION_T itself is never assigned to a
714 : macro map, presumably due to an off-by-one error somewhere
715 : between the logic in linemap_enter_macro and
716 : LINEMAPS_MACRO_LOWEST_LOCATION. */
717 1 : dump_labelled_location_range (stream, "MAX_LOCATION_T",
718 : MAX_LOCATION_T,
719 : MAX_LOCATION_T + 1);
720 :
721 : /* Visualize ad-hoc values. */
722 1 : dump_labelled_location_range (stream, "AD-HOC LOCATIONS",
723 : MAX_LOCATION_T + 1, location_t (-1));
724 1 : }
725 :
726 : /* string_concat's constructor. */
727 :
728 3373795 : string_concat::string_concat (int num, location_t *locs)
729 3373795 : : m_num (num)
730 : {
731 3373795 : m_locs = ggc_vec_alloc <location_t> (num);
732 37926578 : for (int i = 0; i < num; i++)
733 34552783 : m_locs[i] = locs[i];
734 3373795 : }
735 :
736 : /* string_concat_db's constructor. */
737 :
738 215856 : string_concat_db::string_concat_db ()
739 : {
740 215856 : m_table = hash_map <location_hash, string_concat *>::create_ggc (64);
741 215856 : }
742 :
743 : /* Record that a string concatenation occurred, covering NUM
744 : string literal tokens. LOCS is an array of size NUM, containing the
745 : locations of the tokens. A copy of LOCS is taken. */
746 :
747 : void
748 3373801 : string_concat_db::record_string_concatenation (int num, location_t *locs)
749 : {
750 3373801 : gcc_assert (num > 1);
751 3373801 : gcc_assert (locs);
752 :
753 3373801 : location_t key_loc = get_key_loc (locs[0]);
754 : /* We don't record data for 'RESERVED_LOCATION_P (key_loc)' key values:
755 : any data now recorded under key 'key_loc' would be overwritten by a
756 : subsequent call with the same key 'key_loc'. */
757 3373801 : if (RESERVED_LOCATION_P (key_loc))
758 : return;
759 :
760 3373795 : string_concat *concat
761 3373795 : = new (ggc_alloc <string_concat> ()) string_concat (num, locs);
762 3373795 : m_table->put (key_loc, concat);
763 : }
764 :
765 : /* Determine if LOC was the location of the initial token of a
766 : concatenation of string literal tokens.
767 : If so, *OUT_NUM is written to with the number of tokens, and
768 : *OUT_LOCS with the location of an array of locations of the
769 : tokens, and return true. *OUT_LOCS is a borrowed pointer to
770 : storage owned by the string_concat_db.
771 : Otherwise, return false. */
772 :
773 : bool
774 34605 : string_concat_db::get_string_concatenation (location_t loc,
775 : int *out_num,
776 : location_t **out_locs)
777 : {
778 34605 : gcc_assert (out_num);
779 34605 : gcc_assert (out_locs);
780 :
781 34605 : location_t key_loc = get_key_loc (loc);
782 : /* We don't record data for 'RESERVED_LOCATION_P (key_loc)' key values; see
783 : discussion in 'string_concat_db::record_string_concatenation'. */
784 34605 : if (RESERVED_LOCATION_P (key_loc))
785 : return false;
786 :
787 34603 : string_concat **concat = m_table->get (key_loc);
788 34603 : if (!concat)
789 : return false;
790 :
791 4352 : *out_num = (*concat)->m_num;
792 4352 : *out_locs =(*concat)->m_locs;
793 4352 : return true;
794 : }
795 :
796 : /* Internal function. Canonicalize LOC into a form suitable for
797 : use as a key within the database, stripping away macro expansion,
798 : ad-hoc information, and range information, using the location of
799 : the start of LOC within an ordinary linemap. */
800 :
801 : location_t
802 3408406 : string_concat_db::get_key_loc (location_t loc)
803 : {
804 3408406 : loc = linemap_resolve_location (line_table, loc, LRK_SPELLING_LOCATION,
805 : NULL);
806 :
807 3408406 : loc = get_range_from_loc (line_table, loc).m_start;
808 :
809 3408406 : return loc;
810 : }
811 :
812 : /* Helper class for use within get_substring_ranges_for_loc.
813 : An vec of cpp_string with responsibility for releasing all of the
814 : str->text for each str in the vector. */
815 :
816 : class auto_cpp_string_vec : public auto_vec <cpp_string>
817 : {
818 : public:
819 34605 : auto_cpp_string_vec (int alloc)
820 69210 : : auto_vec <cpp_string> (alloc) {}
821 :
822 34605 : ~auto_cpp_string_vec ()
823 : {
824 : /* Clean up the copies within this vec. */
825 34605 : int i;
826 34605 : cpp_string *str;
827 70168 : FOR_EACH_VEC_ELT (*this, i, str)
828 35563 : free (const_cast <unsigned char *> (str->text));
829 34605 : }
830 : };
831 :
832 : /* Attempt to populate RANGES with source location information on the
833 : individual characters within the string literal found at STRLOC.
834 : If CONCATS is non-NULL, then any string literals that the token at
835 : STRLOC was concatenated with are also added to RANGES.
836 :
837 : Return NULL if successful, or an error message if any errors occurred (in
838 : which case RANGES may be only partially populated and should not
839 : be used).
840 :
841 : This is implemented by re-parsing the relevant source line(s). */
842 :
843 : static const char *
844 36851 : get_substring_ranges_for_loc (cpp_reader *pfile,
845 : diagnostics::file_cache &fc,
846 : string_concat_db *concats,
847 : location_t strloc,
848 : enum cpp_ttype type,
849 : cpp_substring_ranges &ranges)
850 : {
851 36851 : gcc_assert (pfile);
852 :
853 36851 : if (strloc == UNKNOWN_LOCATION)
854 : return "unknown location";
855 :
856 : /* Reparsing the strings requires accurate location information.
857 : If -ftrack-macro-expansion has been overridden from its default
858 : of 2, then we might have a location of a macro expansion point,
859 : rather than the location of the literal itself.
860 : Avoid this by requiring that we have full macro expansion tracking
861 : for substring locations to be available. */
862 36851 : if (cpp_get_options (pfile)->track_macro_expansion != 2)
863 : return "track_macro_expansion != 2";
864 :
865 : /* If #line or # 44 "file"-style directives are present, then there's
866 : no guarantee that the line numbers we have can be used to locate
867 : the strings. For example, we might have a .i file with # directives
868 : pointing back to lines within a .c file, but the .c file might
869 : have been edited since the .i file was created.
870 : In such a case, the safest course is to disable on-demand substring
871 : locations. */
872 34608 : if (line_table->seen_line_directive)
873 : return "seen line directive";
874 :
875 : /* If string concatenation has occurred at STRLOC, get the locations
876 : of all of the literal tokens making up the compound string.
877 : Otherwise, just use STRLOC. */
878 34605 : int num_locs = 1;
879 34605 : location_t *strlocs = &strloc;
880 34605 : if (concats)
881 34605 : concats->get_string_concatenation (strloc, &num_locs, &strlocs);
882 :
883 34605 : auto_cpp_string_vec strs (num_locs);
884 34605 : auto_vec <cpp_string_location_reader> loc_readers (num_locs);
885 104766 : for (int i = 0; i < num_locs; i++)
886 : {
887 : /* Get range of strloc. We will use it to locate the start and finish
888 : of the literal token within the line. */
889 41650 : source_range src_range = get_range_from_loc (line_table, strlocs[i]);
890 :
891 41650 : if (src_range.m_start >= LINEMAPS_MACRO_LOWEST_LOCATION (line_table))
892 : {
893 : /* If the string token was within a macro expansion, then we can
894 : cope with it for the simple case where we have a single token.
895 : Otherwise, bail out. */
896 1173 : if (src_range.m_start != src_range.m_finish)
897 6094 : return "macro expansion";
898 : }
899 : else
900 : {
901 40477 : if (src_range.m_start >= LINE_MAP_MAX_LOCATION_WITH_COLS)
902 : /* If so, we can't reliably determine where the token started within
903 : its line. */
904 : return "range starts after LINE_MAP_MAX_LOCATION_WITH_COLS";
905 :
906 34805 : if (src_range.m_finish >= LINE_MAP_MAX_LOCATION_WITH_COLS)
907 : /* If so, we can't reliably determine where the token finished
908 : within its line. */
909 : return "range ends after LINE_MAP_MAX_LOCATION_WITH_COLS";
910 : }
911 :
912 35807 : expanded_location start
913 35807 : = expand_location_to_spelling_point (src_range.m_start,
914 : location_aspect::start);
915 35807 : expanded_location finish
916 35807 : = expand_location_to_spelling_point (src_range.m_finish,
917 : location_aspect::finish);
918 35807 : if (start.file != finish.file)
919 : return "range endpoints are in different files";
920 35807 : if (start.line != finish.line)
921 : return "range endpoints are on different lines";
922 35564 : if (start.column > finish.column)
923 : return "range endpoints are reversed";
924 :
925 35564 : diagnostics::char_span line = fc.get_source_line (start.file, start.line);
926 35564 : if (!line)
927 : return "unable to read source line";
928 :
929 : /* Determine the location of the literal (including quotes
930 : and leading prefix chars, such as the 'u' in a u""
931 : token). */
932 35564 : size_t literal_length = finish.column - start.column + 1;
933 :
934 : /* Ensure that we don't crash if we got the wrong location. */
935 35564 : if (start.column < 1)
936 : return "zero start column";
937 35564 : if (line.length () < (start.column - 1 + literal_length))
938 : return "line is not wide enough";
939 :
940 35563 : diagnostics::char_span literal
941 35563 : = line.subspan (start.column - 1, literal_length);
942 :
943 35563 : cpp_string from;
944 35563 : from.len = literal_length;
945 : /* Make a copy of the literal, to avoid having to rely on
946 : the lifetime of the copy of the line within the cache.
947 : This will be released by the auto_cpp_string_vec dtor. */
948 35563 : from.text = (unsigned char *)literal.xstrdup ();
949 35563 : strs.safe_push (from);
950 :
951 : /* For very long lines, a new linemap could have started
952 : halfway through the token.
953 : Ensure that the loc_reader uses the linemap of the
954 : *end* of the token for its start location. */
955 35563 : const line_map_ordinary *start_ord_map;
956 35563 : linemap_resolve_location (line_table, src_range.m_start,
957 : LRK_SPELLING_LOCATION, &start_ord_map);
958 35563 : const line_map_ordinary *final_ord_map;
959 35563 : linemap_resolve_location (line_table, src_range.m_finish,
960 : LRK_SPELLING_LOCATION, &final_ord_map);
961 35563 : if (start_ord_map == NULL || final_ord_map == NULL)
962 : return "failed to get ordinary maps";
963 : /* Bulletproofing. We ought to only have different ordinary maps
964 : for start vs finish due to line-length jumps. */
965 35562 : if (start_ord_map != final_ord_map
966 6865 : && start_ord_map->to_file != final_ord_map->to_file)
967 : return "start and finish are spelled in different ordinary maps";
968 : /* The file from linemap_resolve_location ought to match that from
969 : expand_location_to_spelling_point. */
970 35562 : if (start_ord_map->to_file != start.file)
971 : return "mismatching file after resolving linemap";
972 :
973 35556 : location_t start_loc
974 35556 : = linemap_position_for_line_and_column (line_table, final_ord_map,
975 : start.line, start.column);
976 :
977 35556 : cpp_string_location_reader loc_reader (start_loc, line_table);
978 35556 : loc_readers.safe_push (loc_reader);
979 : }
980 :
981 : /* Rerun cpp_interpret_string, or rather, a modified version of it. */
982 57022 : const char *err = cpp_interpret_string_ranges (pfile, strs.address (),
983 : loc_readers.address (),
984 : num_locs, &ranges, type);
985 28511 : if (err)
986 : return err;
987 :
988 : /* Success: "ranges" should now contain information on the string. */
989 : return NULL;
990 34605 : }
991 :
992 : /* Attempt to populate *OUT_LOC with source location information on the
993 : given characters within the string literal found at STRLOC.
994 : CARET_IDX, START_IDX, and END_IDX refer to offsets within the execution
995 : character set.
996 :
997 : For example, given CARET_IDX = 4, START_IDX = 3, END_IDX = 7
998 : and string literal "012345\n789"
999 : *OUT_LOC is written to with:
1000 : "012345\n789"
1001 : ~^~~~~
1002 :
1003 : If CONCATS is non-NULL, then any string literals that the token at
1004 : STRLOC was concatenated with are also considered.
1005 :
1006 : This is implemented by re-parsing the relevant source line(s).
1007 :
1008 : Return NULL if successful, or an error message if any errors occurred.
1009 : Error messages are intended for GCC developers (to help debugging) rather
1010 : than for end-users. */
1011 :
1012 : const char *
1013 11199 : get_location_within_string (cpp_reader *pfile,
1014 : diagnostics::file_cache &fc,
1015 : string_concat_db *concats,
1016 : location_t strloc,
1017 : enum cpp_ttype type,
1018 : int caret_idx, int start_idx, int end_idx,
1019 : location_t *out_loc)
1020 : {
1021 11199 : gcc_checking_assert (caret_idx >= 0);
1022 11199 : gcc_checking_assert (start_idx >= 0);
1023 11199 : gcc_checking_assert (end_idx >= 0);
1024 11199 : gcc_assert (out_loc);
1025 :
1026 11199 : cpp_substring_ranges ranges;
1027 11199 : const char *err
1028 11199 : = get_substring_ranges_for_loc (pfile, fc, concats, strloc, type, ranges);
1029 11199 : if (err)
1030 : return err;
1031 :
1032 8491 : if (caret_idx >= ranges.get_num_ranges ())
1033 : return "caret_idx out of range";
1034 8491 : if (start_idx >= ranges.get_num_ranges ())
1035 : return "start_idx out of range";
1036 8491 : if (end_idx >= ranges.get_num_ranges ())
1037 : return "end_idx out of range";
1038 :
1039 8491 : *out_loc = make_location (ranges.get_range (caret_idx).m_start,
1040 8491 : ranges.get_range (start_idx).m_start,
1041 8491 : ranges.get_range (end_idx).m_finish);
1042 8491 : return NULL;
1043 11199 : }
1044 :
1045 : /* Associate the DISCRIMINATOR with LOCUS, and return a new locus. */
1046 :
1047 : location_t
1048 57291814 : location_with_discriminator (location_t locus, int discriminator)
1049 : {
1050 57291814 : tree block = LOCATION_BLOCK (locus);
1051 57291814 : source_range src_range = get_range_from_loc (line_table, locus);
1052 57291814 : locus = get_pure_location (locus);
1053 :
1054 57291814 : if (locus == UNKNOWN_LOCATION)
1055 : return locus;
1056 :
1057 32657019 : return line_table->get_or_create_combined_loc (locus, src_range, block,
1058 32657019 : discriminator);
1059 : }
1060 :
1061 : /* Return TRUE if LOCUS represents a location with a discriminator. */
1062 :
1063 : bool
1064 80165647 : has_discriminator (location_t locus)
1065 : {
1066 80165647 : return get_discriminator_from_loc (locus) != 0;
1067 : }
1068 :
1069 : /* Return the discriminator for LOCUS. */
1070 :
1071 : int
1072 420222350 : get_discriminator_from_loc (location_t locus)
1073 : {
1074 420222350 : return get_discriminator_from_loc (line_table, locus);
1075 : }
1076 :
1077 : /* Create a location with hierarchical discriminator components. */
1078 :
1079 : location_t
1080 3772409 : location_with_discriminator_components (location_t locus,
1081 : const discriminator_components &comp)
1082 : {
1083 3772409 : gcc_assert (comp.base <= DISCR_BASE_MAX);
1084 3772409 : gcc_assert (comp.multiplicity <= DISCR_MULTIPLICITY_MAX);
1085 3772409 : gcc_assert (comp.copyid <= DISCR_COPYID_MAX);
1086 3772409 : unsigned int discriminator = (comp.base << DISCR_BASE_SHIFT)
1087 3772409 : | (comp.multiplicity << DISCR_MULTIPLICITY_SHIFT)
1088 3772409 : | (comp.copyid << DISCR_COPYID_SHIFT);
1089 3772409 : return location_with_discriminator (locus, discriminator);
1090 : }
1091 :
1092 : /* Get hierarchical discriminator components from a location. */
1093 :
1094 : discriminator_components
1095 3772409 : get_discriminator_components_from_loc (location_t locus)
1096 : {
1097 3772409 : unsigned int discriminator = get_discriminator_from_loc (locus);
1098 3772409 : discriminator_components comp;
1099 3772409 : comp.base = discriminator & DISCR_BASE_MASK;
1100 3772409 : comp.multiplicity = (discriminator >> DISCR_MULTIPLICITY_SHIFT)
1101 3772409 : & DISCR_MULTIPLICITY_MASK;
1102 3772409 : comp.copyid = (discriminator >> DISCR_COPYID_SHIFT) & DISCR_COPYID_MASK;
1103 3772409 : return comp;
1104 : }
1105 :
1106 : #if CHECKING_P
1107 :
1108 : namespace selftest {
1109 :
1110 : /* Selftests of location handling. */
1111 :
1112 : /* Attempt to populate *OUT_RANGE with source location information on the
1113 : given character within the string literal found at STRLOC.
1114 : CHAR_IDX refers to an offset within the execution character set.
1115 : If CONCATS is non-NULL, then any string literals that the token at
1116 : STRLOC was concatenated with are also considered.
1117 :
1118 : This is implemented by re-parsing the relevant source line(s).
1119 :
1120 : Return NULL if successful, or an error message if any errors occurred.
1121 : Error messages are intended for GCC developers (to help debugging) rather
1122 : than for end-users. */
1123 :
1124 : static const char *
1125 23748 : get_source_range_for_char (cpp_reader *pfile,
1126 : diagnostics::file_cache &fc,
1127 : string_concat_db *concats,
1128 : location_t strloc,
1129 : enum cpp_ttype type,
1130 : int char_idx,
1131 : source_range *out_range)
1132 : {
1133 23748 : gcc_checking_assert (char_idx >= 0);
1134 23748 : gcc_assert (out_range);
1135 :
1136 23748 : cpp_substring_ranges ranges;
1137 23748 : const char *err
1138 23748 : = get_substring_ranges_for_loc (pfile, fc, concats, strloc, type, ranges);
1139 23748 : if (err)
1140 : return err;
1141 :
1142 18652 : if (char_idx >= ranges.get_num_ranges ())
1143 : return "char_idx out of range";
1144 :
1145 18652 : *out_range = ranges.get_range (char_idx);
1146 18652 : return NULL;
1147 23748 : }
1148 :
1149 : /* As get_source_range_for_char, but write to *OUT the number
1150 : of ranges that are available. */
1151 :
1152 : static const char *
1153 1268 : get_num_source_ranges_for_substring (cpp_reader *pfile,
1154 : diagnostics::file_cache &fc,
1155 : string_concat_db *concats,
1156 : location_t strloc,
1157 : enum cpp_ttype type,
1158 : int *out)
1159 : {
1160 1268 : gcc_assert (out);
1161 :
1162 1268 : cpp_substring_ranges ranges;
1163 1268 : const char *err
1164 1268 : = get_substring_ranges_for_loc (pfile, fc, concats, strloc, type, ranges);
1165 :
1166 1268 : if (err)
1167 : return err;
1168 :
1169 884 : *out = ranges.get_num_ranges ();
1170 884 : return NULL;
1171 1268 : }
1172 :
1173 : /* Selftests of location handling. */
1174 :
1175 : /* Verify that compare() on linenum_type handles comparisons over the full
1176 : range of the type. */
1177 :
1178 : static void
1179 4 : test_linenum_comparisons ()
1180 : {
1181 4 : linenum_type min_line (0);
1182 4 : linenum_type max_line (0xffffffff);
1183 4 : ASSERT_EQ (0, compare (min_line, min_line));
1184 4 : ASSERT_EQ (0, compare (max_line, max_line));
1185 :
1186 4 : ASSERT_GT (compare (max_line, min_line), 0);
1187 4 : ASSERT_LT (compare (min_line, max_line), 0);
1188 4 : }
1189 :
1190 : /* Helper function for verifying location data: when location_t
1191 : values are > LINE_MAP_MAX_LOCATION_WITH_COLS, they are treated
1192 : as having column 0. */
1193 :
1194 : static bool
1195 65424 : should_have_column_data_p (location_t loc)
1196 : {
1197 65424 : if (IS_ADHOC_LOC (loc))
1198 20240 : loc = get_location_from_adhoc_loc (line_table, loc);
1199 65424 : if (loc > LINE_MAP_MAX_LOCATION_WITH_COLS)
1200 6572 : return false;
1201 : return true;
1202 : }
1203 :
1204 : /* Selftest for should_have_column_data_p. */
1205 :
1206 : static void
1207 4 : test_should_have_column_data_p ()
1208 : {
1209 4 : ASSERT_TRUE (should_have_column_data_p (RESERVED_LOCATION_COUNT));
1210 4 : ASSERT_TRUE
1211 : (should_have_column_data_p (LINE_MAP_MAX_LOCATION_WITH_COLS));
1212 4 : ASSERT_FALSE
1213 : (should_have_column_data_p (LINE_MAP_MAX_LOCATION_WITH_COLS + 1));
1214 4 : }
1215 :
1216 : /* Verify the result of LOCATION_FILE/LOCATION_LINE/LOCATION_COLUMN
1217 : on LOC. */
1218 :
1219 : static void
1220 1356 : assert_loceq (const char *exp_filename, int exp_linenum, int exp_colnum,
1221 : location_t loc)
1222 : {
1223 1356 : ASSERT_STREQ (exp_filename, LOCATION_FILE (loc));
1224 1356 : ASSERT_EQ (exp_linenum, LOCATION_LINE (loc));
1225 : /* If location_t values are sufficiently high, then column numbers
1226 : will be unavailable and LOCATION_COLUMN (loc) will be 0.
1227 : When close to the threshold, column numbers *may* be present: if
1228 : the final linemap before the threshold contains a line that straddles
1229 : the threshold, locations in that line have column information. */
1230 1356 : if (should_have_column_data_p (loc))
1231 828 : ASSERT_EQ (exp_colnum, LOCATION_COLUMN (loc));
1232 1356 : }
1233 :
1234 : /* Various selftests involve constructing a line table and one or more
1235 : line maps within it.
1236 :
1237 : For maximum test coverage we want to run these tests with a variety
1238 : of situations:
1239 : - line_table->default_range_bits: some frontends use a non-zero value
1240 : and others use zero
1241 : - the fallback modes within line-map.cc: there are various threshold
1242 : values for location_t beyond line-map.cc changes
1243 : behavior (disabling of the range-packing optimization, disabling
1244 : of column-tracking). We can exercise these by starting the line_table
1245 : at interesting values at or near these thresholds.
1246 :
1247 : The following struct describes a particular case within our test
1248 : matrix. */
1249 :
1250 : class line_table_case
1251 : {
1252 : public:
1253 5860 : line_table_case (int default_range_bits, location_t base_location)
1254 5860 : : m_default_range_bits (default_range_bits),
1255 5860 : m_base_location (base_location)
1256 : {}
1257 :
1258 : int m_default_range_bits;
1259 : location_t m_base_location;
1260 : };
1261 :
1262 : /* Constructor. Store the old value of line_table, and create a new
1263 : one, using sane defaults. */
1264 :
1265 21 : line_table_test::line_table_test ()
1266 : {
1267 21 : gcc_assert (saved_line_table == NULL);
1268 21 : saved_line_table = line_table;
1269 21 : line_table = ggc_alloc<line_maps> ();
1270 21 : linemap_init (line_table, BUILTINS_LOCATION);
1271 21 : gcc_assert (saved_line_table->m_reallocator);
1272 21 : line_table->m_reallocator = saved_line_table->m_reallocator;
1273 21 : gcc_assert (saved_line_table->m_round_alloc_size);
1274 21 : line_table->m_round_alloc_size = saved_line_table->m_round_alloc_size;
1275 21 : line_table->default_range_bits = 0;
1276 21 : }
1277 :
1278 : /* Constructor. Store the old value of line_table, and create a new
1279 : one, using the situation described in CASE_. */
1280 :
1281 6724 : line_table_test::line_table_test (const line_table_case &case_)
1282 : {
1283 6724 : gcc_assert (saved_line_table == NULL);
1284 6724 : saved_line_table = line_table;
1285 6724 : line_table = ggc_alloc<line_maps> ();
1286 6724 : linemap_init (line_table, BUILTINS_LOCATION);
1287 6724 : gcc_assert (saved_line_table->m_reallocator);
1288 6724 : line_table->m_reallocator = saved_line_table->m_reallocator;
1289 6724 : gcc_assert (saved_line_table->m_round_alloc_size);
1290 6724 : line_table->m_round_alloc_size = saved_line_table->m_round_alloc_size;
1291 6724 : line_table->default_range_bits = case_.m_default_range_bits;
1292 6724 : if (case_.m_base_location)
1293 : {
1294 6160 : line_table->highest_location = case_.m_base_location;
1295 6160 : line_table->highest_line = case_.m_base_location;
1296 : }
1297 6724 : }
1298 :
1299 : /* Destructor. Restore the old value of line_table. */
1300 :
1301 6745 : line_table_test::~line_table_test ()
1302 : {
1303 6745 : gcc_assert (saved_line_table != NULL);
1304 6745 : line_table = saved_line_table;
1305 6745 : saved_line_table = NULL;
1306 6745 : }
1307 :
1308 : /* Verify basic operation of ordinary linemaps. */
1309 :
1310 : static void
1311 96 : test_accessing_ordinary_linemaps (const line_table_case &case_)
1312 : {
1313 96 : line_table_test ltt (case_);
1314 :
1315 : /* Build a simple linemap describing some locations. */
1316 96 : linemap_add (line_table, LC_ENTER, false, "foo.c", 0);
1317 :
1318 96 : linemap_line_start (line_table, 1, 100);
1319 96 : location_t loc_a = linemap_position_for_column (line_table, 1);
1320 96 : location_t loc_b = linemap_position_for_column (line_table, 23);
1321 :
1322 96 : linemap_line_start (line_table, 2, 100);
1323 96 : location_t loc_c = linemap_position_for_column (line_table, 1);
1324 96 : location_t loc_d = linemap_position_for_column (line_table, 17);
1325 :
1326 : /* Example of a very long line. */
1327 96 : linemap_line_start (line_table, 3, 2000);
1328 96 : location_t loc_e = linemap_position_for_column (line_table, 700);
1329 :
1330 : /* Transitioning back to a short line. */
1331 96 : linemap_line_start (line_table, 4, 0);
1332 96 : location_t loc_back_to_short = linemap_position_for_column (line_table, 100);
1333 :
1334 96 : if (should_have_column_data_p (loc_back_to_short))
1335 : {
1336 : /* Verify that we switched to short lines in the linemap. */
1337 56 : line_map_ordinary *map = LINEMAPS_LAST_ORDINARY_MAP (line_table);
1338 56 : ASSERT_EQ (7, map->m_column_and_range_bits - map->m_range_bits);
1339 : }
1340 :
1341 : /* Example of a line that will eventually be seen to be longer
1342 : than LINE_MAP_MAX_COLUMN_NUMBER; the initially seen width is
1343 : below that. */
1344 96 : linemap_line_start (line_table, 5, 2000);
1345 :
1346 96 : location_t loc_start_of_very_long_line
1347 96 : = linemap_position_for_column (line_table, 2000);
1348 96 : location_t loc_too_wide
1349 96 : = linemap_position_for_column (line_table, LINE_MAP_MAX_COLUMN_NUMBER + 1);
1350 96 : location_t loc_too_wide_2
1351 96 : = linemap_position_for_column (line_table, LINE_MAP_MAX_COLUMN_NUMBER + 2);
1352 :
1353 : /* ...and back to a sane line length. */
1354 96 : linemap_line_start (line_table, 6, 100);
1355 96 : location_t loc_sane_again = linemap_position_for_column (line_table, 10);
1356 :
1357 96 : linemap_add (line_table, LC_LEAVE, false, NULL, 0);
1358 :
1359 : /* Multiple files. */
1360 96 : linemap_add (line_table, LC_ENTER, false, "bar.c", 0);
1361 96 : linemap_line_start (line_table, 1, 200);
1362 96 : location_t loc_f = linemap_position_for_column (line_table, 150);
1363 96 : linemap_add (line_table, LC_LEAVE, false, NULL, 0);
1364 :
1365 : /* Verify that we can recover the location info. */
1366 96 : assert_loceq ("foo.c", 1, 1, loc_a);
1367 96 : assert_loceq ("foo.c", 1, 23, loc_b);
1368 96 : assert_loceq ("foo.c", 2, 1, loc_c);
1369 96 : assert_loceq ("foo.c", 2, 17, loc_d);
1370 96 : assert_loceq ("foo.c", 3, 700, loc_e);
1371 96 : assert_loceq ("foo.c", 4, 100, loc_back_to_short);
1372 :
1373 : /* In the very wide line, the initial location should be fully tracked. */
1374 96 : assert_loceq ("foo.c", 5, 2000, loc_start_of_very_long_line);
1375 : /* ...but once we exceed LINE_MAP_MAX_COLUMN_NUMBER column-tracking should
1376 : be disabled. */
1377 96 : assert_loceq ("foo.c", 5, 0, loc_too_wide);
1378 96 : assert_loceq ("foo.c", 5, 0, loc_too_wide_2);
1379 : /*...and column-tracking should be re-enabled for subsequent lines. */
1380 96 : assert_loceq ("foo.c", 6, 10, loc_sane_again);
1381 :
1382 96 : assert_loceq ("bar.c", 1, 150, loc_f);
1383 :
1384 96 : ASSERT_FALSE (is_location_from_builtin_token (loc_a));
1385 96 : ASSERT_TRUE (pure_location_p (line_table, loc_a));
1386 :
1387 : /* Verify using make_location to build a range, and extracting data
1388 : back from it. */
1389 96 : location_t range_c_b_d = make_location (loc_c, loc_b, loc_d);
1390 96 : ASSERT_FALSE (pure_location_p (line_table, range_c_b_d));
1391 96 : ASSERT_EQ (loc_c, get_location_from_adhoc_loc (line_table, range_c_b_d));
1392 96 : source_range src_range = get_range_from_loc (line_table, range_c_b_d);
1393 96 : ASSERT_EQ (loc_b, src_range.m_start);
1394 96 : ASSERT_EQ (loc_d, src_range.m_finish);
1395 :
1396 : /* Verify raw line map usage. The map should have the requested properties,
1397 : irrespective of limits like LINE_MAP_MAX_LOCATION_WITH_COLS. */
1398 384 : for (int sysp = 0; sysp != 3; ++sysp)
1399 : {
1400 288 : constexpr int column_bits = 8, range_bits = 7;
1401 288 : const int line = 1 + sysp;
1402 576 : const auto map = linemap_add_raw_map (line_table, LC_RENAME,
1403 288 : line_table->highest_location + 1,
1404 : sysp, column_bits + range_bits,
1405 : range_bits, "foo2.c", line, 1);
1406 288 : ASSERT_NE (map, nullptr);
1407 288 : ASSERT_EQ (map->reason, LC_RENAME);
1408 288 : ASSERT_EQ (map->m_column_and_range_bits, column_bits + range_bits);
1409 288 : ASSERT_EQ (map->m_range_bits, range_bits);
1410 288 : const line_map_uint_t col = 200 + sysp;
1411 288 : const location_t loc = MAP_START_LOCATION (map) + (col << range_bits);
1412 288 : ASSERT_GT (loc, line_table->highest_line);
1413 288 : ASSERT_LT (loc, line_table->highest_location);
1414 288 : ASSERT_EQ (in_system_header_at (loc), sysp);
1415 288 : assert_loceq ("foo2.c", line, col, loc);
1416 : }
1417 96 : }
1418 :
1419 : /* Verify various properties of UNKNOWN_LOCATION. */
1420 :
1421 : static void
1422 4 : test_unknown_location ()
1423 : {
1424 4 : ASSERT_EQ (NULL, LOCATION_FILE (UNKNOWN_LOCATION));
1425 4 : ASSERT_EQ (0, LOCATION_LINE (UNKNOWN_LOCATION));
1426 4 : ASSERT_EQ (0, LOCATION_COLUMN (UNKNOWN_LOCATION));
1427 4 : }
1428 :
1429 : /* Verify various properties of BUILTINS_LOCATION. */
1430 :
1431 : static void
1432 4 : test_builtins ()
1433 : {
1434 4 : assert_loceq (special_fname_builtin (), 0, 0, BUILTINS_LOCATION);
1435 4 : ASSERT_PRED1 (is_location_from_builtin_token, BUILTINS_LOCATION);
1436 4 : }
1437 :
1438 : /* Regression test for make_location.
1439 : Ensure that we use pure locations for the start/finish of the range,
1440 : rather than storing a packed or ad-hoc range as the start/finish. */
1441 :
1442 : static void
1443 96 : test_make_location_nonpure_range_endpoints (const line_table_case &case_)
1444 : {
1445 : /* Issue seen with testsuite/c-c++-common/Wlogical-not-parentheses-2.c
1446 : with C++ frontend.
1447 : ....................0000000001111111111222.
1448 : ....................1234567890123456789012. */
1449 96 : const char *content = " r += !aaa == bbb;\n";
1450 96 : temp_source_file tmp (SELFTEST_LOCATION, ".C", content);
1451 96 : line_table_test ltt (case_);
1452 96 : linemap_add (line_table, LC_ENTER, false, tmp.get_filename (), 1);
1453 :
1454 96 : const location_t c11 = linemap_position_for_column (line_table, 11);
1455 96 : const location_t c12 = linemap_position_for_column (line_table, 12);
1456 96 : const location_t c13 = linemap_position_for_column (line_table, 13);
1457 96 : const location_t c14 = linemap_position_for_column (line_table, 14);
1458 96 : const location_t c21 = linemap_position_for_column (line_table, 21);
1459 :
1460 96 : if (c21 > LINE_MAP_MAX_LOCATION_WITH_COLS)
1461 32 : return;
1462 :
1463 : /* Use column 13 for the caret location, arbitrarily, to verify that we
1464 : handle start != caret. */
1465 64 : const location_t aaa = make_location (c13, c12, c14);
1466 64 : ASSERT_EQ (c13, get_pure_location (aaa));
1467 64 : ASSERT_EQ (c12, get_start (aaa));
1468 64 : ASSERT_FALSE (IS_ADHOC_LOC (get_start (aaa)));
1469 64 : ASSERT_EQ (c14, get_finish (aaa));
1470 64 : ASSERT_FALSE (IS_ADHOC_LOC (get_finish (aaa)));
1471 :
1472 : /* Make a location using a location with a range as the start-point. */
1473 64 : const location_t not_aaa = make_location (c11, aaa, c14);
1474 64 : ASSERT_EQ (c11, get_pure_location (not_aaa));
1475 : /* It should use the start location of the range, not store the range
1476 : itself. */
1477 64 : ASSERT_EQ (c12, get_start (not_aaa));
1478 64 : ASSERT_FALSE (IS_ADHOC_LOC (get_start (not_aaa)));
1479 64 : ASSERT_EQ (c14, get_finish (not_aaa));
1480 64 : ASSERT_FALSE (IS_ADHOC_LOC (get_finish (not_aaa)));
1481 :
1482 : /* Similarly, make a location with a range as the end-point. */
1483 64 : const location_t aaa_eq_bbb = make_location (c12, c12, c21);
1484 64 : ASSERT_EQ (c12, get_pure_location (aaa_eq_bbb));
1485 64 : ASSERT_EQ (c12, get_start (aaa_eq_bbb));
1486 64 : ASSERT_FALSE (IS_ADHOC_LOC (get_start (aaa_eq_bbb)));
1487 64 : ASSERT_EQ (c21, get_finish (aaa_eq_bbb));
1488 64 : ASSERT_FALSE (IS_ADHOC_LOC (get_finish (aaa_eq_bbb)));
1489 64 : const location_t not_aaa_eq_bbb = make_location (c11, c12, aaa_eq_bbb);
1490 : /* It should use the finish location of the range, not store the range
1491 : itself. */
1492 64 : ASSERT_EQ (c11, get_pure_location (not_aaa_eq_bbb));
1493 64 : ASSERT_EQ (c12, get_start (not_aaa_eq_bbb));
1494 64 : ASSERT_FALSE (IS_ADHOC_LOC (get_start (not_aaa_eq_bbb)));
1495 64 : ASSERT_EQ (c21, get_finish (not_aaa_eq_bbb));
1496 64 : ASSERT_FALSE (IS_ADHOC_LOC (get_finish (not_aaa_eq_bbb)));
1497 96 : }
1498 :
1499 : /* Tests of lexing. */
1500 :
1501 : /* Verify that token TOK from PARSER has cpp_token_as_text
1502 : equal to EXPECTED_TEXT. */
1503 :
1504 : #define ASSERT_TOKEN_AS_TEXT_EQ(PARSER, TOK, EXPECTED_TEXT) \
1505 : SELFTEST_BEGIN_STMT \
1506 : unsigned char *actual_txt = cpp_token_as_text ((PARSER), (TOK)); \
1507 : ASSERT_STREQ ((EXPECTED_TEXT), (const char *)actual_txt); \
1508 : SELFTEST_END_STMT
1509 :
1510 : /* Verify that TOK's src_loc is within EXP_FILENAME at EXP_LINENUM,
1511 : and ranges from EXP_START_COL to EXP_FINISH_COL.
1512 : Use LOC as the effective location of the selftest. */
1513 :
1514 : static void
1515 576 : assert_token_loc_eq (const location &loc,
1516 : const cpp_token *tok,
1517 : const char *exp_filename, int exp_linenum,
1518 : int exp_start_col, int exp_finish_col)
1519 : {
1520 576 : location_t tok_loc = tok->src_loc;
1521 576 : ASSERT_STREQ_AT (loc, exp_filename, LOCATION_FILE (tok_loc));
1522 576 : ASSERT_EQ_AT (loc, exp_linenum, LOCATION_LINE (tok_loc));
1523 :
1524 : /* If location_t values are sufficiently high, then column numbers
1525 : will be unavailable. */
1526 576 : if (!should_have_column_data_p (tok_loc))
1527 196 : return;
1528 :
1529 380 : ASSERT_EQ_AT (loc, exp_start_col, LOCATION_COLUMN (tok_loc));
1530 380 : source_range tok_range = get_range_from_loc (line_table, tok_loc);
1531 380 : ASSERT_EQ_AT (loc, exp_start_col, LOCATION_COLUMN (tok_range.m_start));
1532 380 : ASSERT_EQ_AT (loc, exp_finish_col, LOCATION_COLUMN (tok_range.m_finish));
1533 : }
1534 :
1535 : /* Use assert_token_loc_eq to verify the TOK->src_loc, using
1536 : SELFTEST_LOCATION as the effective location of the selftest. */
1537 :
1538 : #define ASSERT_TOKEN_LOC_EQ(TOK, EXP_FILENAME, EXP_LINENUM, \
1539 : EXP_START_COL, EXP_FINISH_COL) \
1540 : assert_token_loc_eq (SELFTEST_LOCATION, (TOK), (EXP_FILENAME), \
1541 : (EXP_LINENUM), (EXP_START_COL), (EXP_FINISH_COL))
1542 :
1543 : /* Test of lexing a file using libcpp, verifying tokens and their
1544 : location information. */
1545 :
1546 : static void
1547 96 : test_lexer (const line_table_case &case_)
1548 : {
1549 : /* Create a tempfile and write some text to it. */
1550 96 : const char *content =
1551 : /*00000000011111111112222222222333333.3333444444444.455555555556
1552 : 12345678901234567890123456789012345.6789012345678.901234567890. */
1553 : ("test_name /* c-style comment */\n"
1554 : " \"test literal\"\n"
1555 : " // test c++-style comment\n"
1556 : " 42\n");
1557 96 : temp_source_file tmp (SELFTEST_LOCATION, ".txt", content);
1558 :
1559 96 : line_table_test ltt (case_);
1560 :
1561 96 : cpp_reader *parser = cpp_create_reader (CLK_GNUC89, NULL, line_table);
1562 :
1563 96 : const char *fname = cpp_read_main_file (parser, tmp.get_filename ());
1564 96 : ASSERT_NE (fname, NULL);
1565 :
1566 : /* Verify that we get the expected tokens back, with the correct
1567 : location information. */
1568 :
1569 96 : location_t loc;
1570 96 : const cpp_token *tok;
1571 96 : tok = cpp_get_token_with_location (parser, &loc);
1572 96 : ASSERT_NE (tok, NULL);
1573 96 : ASSERT_EQ (tok->type, CPP_NAME);
1574 96 : ASSERT_TOKEN_AS_TEXT_EQ (parser, tok, "test_name");
1575 96 : ASSERT_TOKEN_LOC_EQ (tok, tmp.get_filename (), 1, 1, 9);
1576 :
1577 96 : tok = cpp_get_token_with_location (parser, &loc);
1578 96 : ASSERT_NE (tok, NULL);
1579 96 : ASSERT_EQ (tok->type, CPP_STRING);
1580 96 : ASSERT_TOKEN_AS_TEXT_EQ (parser, tok, "\"test literal\"");
1581 96 : ASSERT_TOKEN_LOC_EQ (tok, tmp.get_filename (), 2, 35, 48);
1582 :
1583 96 : tok = cpp_get_token_with_location (parser, &loc);
1584 96 : ASSERT_NE (tok, NULL);
1585 96 : ASSERT_EQ (tok->type, CPP_NUMBER);
1586 96 : ASSERT_TOKEN_AS_TEXT_EQ (parser, tok, "42");
1587 96 : ASSERT_TOKEN_LOC_EQ (tok, tmp.get_filename (), 4, 4, 5);
1588 :
1589 96 : tok = cpp_get_token_with_location (parser, &loc);
1590 96 : ASSERT_NE (tok, NULL);
1591 96 : ASSERT_EQ (tok->type, CPP_EOF);
1592 :
1593 96 : cpp_finish (parser, NULL);
1594 96 : cpp_destroy (parser);
1595 96 : }
1596 :
1597 : /* Forward decls. */
1598 :
1599 : class lexer_test;
1600 : class lexer_test_options;
1601 :
1602 : /* A class for specifying options of a lexer_test.
1603 : The "apply" vfunc is called during the lexer_test constructor. */
1604 :
1605 192 : class lexer_test_options
1606 : {
1607 : public:
1608 : virtual void apply (lexer_test &) = 0;
1609 : };
1610 :
1611 : /* Wrapper around an cpp_reader *, which calls cpp_finish and cpp_destroy
1612 : in its dtor.
1613 :
1614 : This is needed by struct lexer_test to ensure that the cleanup of the
1615 : cpp_reader happens *after* the cleanup of the temp_source_file. */
1616 :
1617 : class cpp_reader_ptr
1618 : {
1619 : public:
1620 2304 : cpp_reader_ptr (cpp_reader *ptr) : m_ptr (ptr) {}
1621 :
1622 2304 : ~cpp_reader_ptr ()
1623 : {
1624 2304 : cpp_finish (m_ptr, NULL);
1625 2304 : cpp_destroy (m_ptr);
1626 2304 : }
1627 :
1628 2304 : operator cpp_reader * () const { return m_ptr; }
1629 :
1630 : private:
1631 : cpp_reader *m_ptr;
1632 : };
1633 :
1634 : /* A struct for writing lexer tests. */
1635 :
1636 : class lexer_test
1637 : {
1638 : public:
1639 : lexer_test (const line_table_case &case_, const char *content,
1640 : lexer_test_options *options);
1641 : ~lexer_test ();
1642 :
1643 : const cpp_token *get_token ();
1644 :
1645 : /* The ordering of these fields matters.
1646 : The line_table_test must be first, since the cpp_reader_ptr
1647 : uses it.
1648 : The cpp_reader must be cleaned up *after* the temp_source_file
1649 : since the filenames in input.cc's input cache are owned by the
1650 : cpp_reader; in particular, when ~temp_source_file evicts the
1651 : filename the filenames must still be alive. */
1652 : line_table_test m_ltt;
1653 : cpp_reader_ptr m_parser;
1654 : temp_source_file m_tempfile;
1655 : diagnostics::file_cache m_file_cache;
1656 : string_concat_db m_concats;
1657 : bool m_implicitly_expect_EOF;
1658 : };
1659 :
1660 : /* Use an EBCDIC encoding for the execution charset, specifically
1661 : IBM1047-encoded (aka "EBCDIC 1047", or "Code page 1047").
1662 :
1663 : This exercises iconv integration within libcpp.
1664 : Not every build of iconv supports the given charset,
1665 : so we need to flag this error and handle it gracefully. */
1666 :
1667 : class ebcdic_execution_charset : public lexer_test_options
1668 : {
1669 : public:
1670 96 : ebcdic_execution_charset () : m_num_iconv_errors (0)
1671 : {
1672 96 : gcc_assert (s_singleton == NULL);
1673 96 : s_singleton = this;
1674 96 : }
1675 96 : ~ebcdic_execution_charset ()
1676 96 : {
1677 96 : gcc_assert (s_singleton == this);
1678 96 : s_singleton = NULL;
1679 96 : }
1680 :
1681 96 : void apply (lexer_test &test) final override
1682 : {
1683 96 : cpp_options *cpp_opts = cpp_get_options (test.m_parser);
1684 96 : cpp_opts->narrow_charset = "IBM1047";
1685 :
1686 96 : cpp_callbacks *callbacks = cpp_get_callbacks (test.m_parser);
1687 96 : callbacks->diagnostic = on_diagnostic;
1688 96 : }
1689 :
1690 0 : static bool on_diagnostic (cpp_reader *pfile ATTRIBUTE_UNUSED,
1691 : enum cpp_diagnostic_level level ATTRIBUTE_UNUSED,
1692 : enum cpp_warning_reason reason ATTRIBUTE_UNUSED,
1693 : rich_location *richloc ATTRIBUTE_UNUSED,
1694 : const char *msgid, va_list *ap ATTRIBUTE_UNUSED)
1695 : ATTRIBUTE_FPTR_PRINTF(5,0)
1696 : {
1697 0 : gcc_assert (s_singleton);
1698 : /* Avoid exgettext from picking this up, it is translated in libcpp. */
1699 0 : const char *msg = "conversion from %s to %s not supported by iconv";
1700 : #ifdef ENABLE_NLS
1701 0 : msg = dgettext ("cpplib", msg);
1702 : #endif
1703 : /* Detect and record errors emitted by libcpp/charset.cc:init_iconv_desc
1704 : when the local iconv build doesn't support the conversion. */
1705 0 : if (strcmp (msgid, msg) == 0)
1706 : {
1707 0 : s_singleton->m_num_iconv_errors++;
1708 0 : return true;
1709 : }
1710 :
1711 : /* Otherwise, we have an unexpected error. */
1712 0 : abort ();
1713 : }
1714 :
1715 96 : bool iconv_errors_occurred_p () const { return m_num_iconv_errors > 0; }
1716 :
1717 : private:
1718 : static ebcdic_execution_charset *s_singleton;
1719 : int m_num_iconv_errors;
1720 : };
1721 :
1722 : ebcdic_execution_charset *ebcdic_execution_charset::s_singleton;
1723 :
1724 : /* A lexer_test_options subclass that records a list of diagnostic
1725 : messages emitted by the lexer. */
1726 :
1727 : class lexer_diagnostic_sink : public lexer_test_options
1728 : {
1729 : public:
1730 96 : lexer_diagnostic_sink ()
1731 96 : {
1732 96 : gcc_assert (s_singleton == NULL);
1733 96 : s_singleton = this;
1734 96 : }
1735 96 : ~lexer_diagnostic_sink ()
1736 96 : {
1737 96 : gcc_assert (s_singleton == this);
1738 96 : s_singleton = NULL;
1739 :
1740 96 : int i;
1741 96 : char *str;
1742 192 : FOR_EACH_VEC_ELT (m_diagnostics, i, str)
1743 96 : free (str);
1744 96 : }
1745 :
1746 96 : void apply (lexer_test &test) final override
1747 : {
1748 96 : cpp_callbacks *callbacks = cpp_get_callbacks (test.m_parser);
1749 96 : callbacks->diagnostic = on_diagnostic;
1750 96 : }
1751 :
1752 96 : static bool on_diagnostic (cpp_reader *pfile ATTRIBUTE_UNUSED,
1753 : enum cpp_diagnostic_level level ATTRIBUTE_UNUSED,
1754 : enum cpp_warning_reason reason ATTRIBUTE_UNUSED,
1755 : rich_location *richloc ATTRIBUTE_UNUSED,
1756 : const char *msgid, va_list *ap)
1757 : ATTRIBUTE_FPTR_PRINTF(5,0)
1758 : {
1759 96 : char *msg = xvasprintf (msgid, *ap);
1760 96 : s_singleton->m_diagnostics.safe_push (msg);
1761 96 : return true;
1762 : }
1763 :
1764 : auto_vec<char *> m_diagnostics;
1765 :
1766 : private:
1767 : static lexer_diagnostic_sink *s_singleton;
1768 : };
1769 :
1770 : lexer_diagnostic_sink *lexer_diagnostic_sink::s_singleton;
1771 :
1772 : /* Constructor. Override line_table with a new instance based on CASE_,
1773 : and write CONTENT to a tempfile. Create a cpp_reader, and use it to
1774 : start parsing the tempfile. */
1775 :
1776 2304 : lexer_test::lexer_test (const line_table_case &case_, const char *content,
1777 : lexer_test_options *options)
1778 2304 : : m_ltt (case_),
1779 2304 : m_parser (cpp_create_reader (CLK_GNUC99, NULL, line_table)),
1780 : /* Create a tempfile and write the text to it. */
1781 2304 : m_tempfile (SELFTEST_LOCATION, ".c", content),
1782 2304 : m_concats (),
1783 2304 : m_implicitly_expect_EOF (true)
1784 : {
1785 2304 : if (options)
1786 192 : options->apply (*this);
1787 :
1788 2304 : cpp_init_iconv (m_parser);
1789 :
1790 : /* Parse the file. */
1791 2304 : const char *fname = cpp_read_main_file (m_parser,
1792 : m_tempfile.get_filename ());
1793 2304 : ASSERT_NE (fname, NULL);
1794 2304 : }
1795 :
1796 : /* Destructor. By default, verify that the next token in m_parser is EOF. */
1797 :
1798 2304 : lexer_test::~lexer_test ()
1799 : {
1800 2304 : location_t loc;
1801 2304 : const cpp_token *tok;
1802 :
1803 2304 : if (m_implicitly_expect_EOF)
1804 : {
1805 2208 : tok = cpp_get_token_with_location (m_parser, &loc);
1806 2208 : ASSERT_NE (tok, NULL);
1807 2208 : ASSERT_EQ (tok->type, CPP_EOF);
1808 : }
1809 2304 : }
1810 :
1811 : /* Get the next token from m_parser. */
1812 :
1813 : const cpp_token *
1814 3936 : lexer_test::get_token ()
1815 : {
1816 3936 : location_t loc;
1817 3936 : const cpp_token *tok;
1818 :
1819 3936 : tok = cpp_get_token_with_location (m_parser, &loc);
1820 3936 : ASSERT_NE (tok, NULL);
1821 3936 : return tok;
1822 : }
1823 :
1824 : /* Verify that locations within string literals are correctly handled. */
1825 :
1826 : /* Verify get_source_range_for_substring for token(s) at STRLOC,
1827 : using the string concatenation database for TEST.
1828 :
1829 : Assert that the character at index IDX is on EXPECTED_LINE,
1830 : and that it begins at column EXPECTED_START_COL and ends at
1831 : EXPECTED_FINISH_COL (unless the locations are beyond
1832 : LINE_MAP_MAX_LOCATION_WITH_COLS, in which case don't check their
1833 : columns). */
1834 :
1835 : static void
1836 23740 : assert_char_at_range (const location &loc,
1837 : lexer_test& test,
1838 : location_t strloc, enum cpp_ttype type, int idx,
1839 : int expected_line, int expected_start_col,
1840 : int expected_finish_col)
1841 : {
1842 23740 : cpp_reader *pfile = test.m_parser;
1843 23740 : string_concat_db *concats = &test.m_concats;
1844 :
1845 23740 : source_range actual_range = source_range();
1846 23740 : const char *err
1847 23740 : = get_source_range_for_char (pfile, test.m_file_cache,
1848 : concats, strloc, type, idx,
1849 : &actual_range);
1850 23740 : if (should_have_column_data_p (strloc))
1851 18652 : ASSERT_EQ_AT (loc, NULL, err);
1852 : else
1853 : {
1854 5088 : ASSERT_STREQ_AT (loc,
1855 : "range starts after LINE_MAP_MAX_LOCATION_WITH_COLS",
1856 : err);
1857 5088 : return;
1858 : }
1859 :
1860 18652 : int actual_start_line = LOCATION_LINE (actual_range.m_start);
1861 18652 : ASSERT_EQ_AT (loc, expected_line, actual_start_line);
1862 18652 : int actual_finish_line = LOCATION_LINE (actual_range.m_finish);
1863 18652 : ASSERT_EQ_AT (loc, expected_line, actual_finish_line);
1864 :
1865 18652 : if (should_have_column_data_p (actual_range.m_start))
1866 : {
1867 18652 : int actual_start_col = LOCATION_COLUMN (actual_range.m_start);
1868 18652 : ASSERT_EQ_AT (loc, expected_start_col, actual_start_col);
1869 : }
1870 18652 : if (should_have_column_data_p (actual_range.m_finish))
1871 : {
1872 18652 : int actual_finish_col = LOCATION_COLUMN (actual_range.m_finish);
1873 18652 : ASSERT_EQ_AT (loc, expected_finish_col, actual_finish_col);
1874 : }
1875 : }
1876 :
1877 : /* Macro for calling assert_char_at_range, supplying SELFTEST_LOCATION for
1878 : the effective location of any errors. */
1879 :
1880 : #define ASSERT_CHAR_AT_RANGE(LEXER_TEST, STRLOC, TYPE, IDX, EXPECTED_LINE, \
1881 : EXPECTED_START_COL, EXPECTED_FINISH_COL) \
1882 : assert_char_at_range (SELFTEST_LOCATION, (LEXER_TEST), (STRLOC), (TYPE), \
1883 : (IDX), (EXPECTED_LINE), (EXPECTED_START_COL), \
1884 : (EXPECTED_FINISH_COL))
1885 :
1886 : /* Verify get_num_source_ranges_for_substring for token(s) at STRLOC,
1887 : using the string concatenation database for TEST.
1888 :
1889 : Assert that the token(s) at STRLOC contain EXPECTED_NUM_RANGES. */
1890 :
1891 : static void
1892 1268 : assert_num_substring_ranges (const location &loc,
1893 : lexer_test& test,
1894 : location_t strloc,
1895 : enum cpp_ttype type,
1896 : int expected_num_ranges)
1897 : {
1898 1268 : cpp_reader *pfile = test.m_parser;
1899 1268 : string_concat_db *concats = &test.m_concats;
1900 :
1901 1268 : int actual_num_ranges = -1;
1902 1268 : const char *err
1903 1268 : = get_num_source_ranges_for_substring (pfile, test.m_file_cache,
1904 : concats, strloc, type,
1905 : &actual_num_ranges);
1906 1268 : if (should_have_column_data_p (strloc))
1907 884 : ASSERT_EQ_AT (loc, NULL, err);
1908 : else
1909 : {
1910 384 : ASSERT_STREQ_AT (loc,
1911 : "range starts after LINE_MAP_MAX_LOCATION_WITH_COLS",
1912 : err);
1913 384 : return;
1914 : }
1915 884 : ASSERT_EQ_AT (loc, expected_num_ranges, actual_num_ranges);
1916 : }
1917 :
1918 : /* Macro for calling assert_num_substring_ranges, supplying
1919 : SELFTEST_LOCATION for the effective location of any errors. */
1920 :
1921 : #define ASSERT_NUM_SUBSTRING_RANGES(LEXER_TEST, STRLOC, TYPE, \
1922 : EXPECTED_NUM_RANGES) \
1923 : assert_num_substring_ranges (SELFTEST_LOCATION, (LEXER_TEST), (STRLOC), \
1924 : (TYPE), (EXPECTED_NUM_RANGES))
1925 :
1926 :
1927 : /* Verify that get_num_source_ranges_for_substring for token(s) at STRLOC
1928 : returns an error (using the string concatenation database for TEST). */
1929 :
1930 : static void
1931 636 : assert_has_no_substring_ranges (const location &loc,
1932 : lexer_test& test,
1933 : location_t strloc,
1934 : enum cpp_ttype type,
1935 : const char *expected_err)
1936 : {
1937 636 : cpp_reader *pfile = test.m_parser;
1938 636 : string_concat_db *concats = &test.m_concats;
1939 636 : cpp_substring_ranges ranges;
1940 636 : const char *actual_err
1941 636 : = get_substring_ranges_for_loc (pfile, test.m_file_cache, concats, strloc,
1942 : type, ranges);
1943 636 : if (should_have_column_data_p (strloc))
1944 444 : ASSERT_STREQ_AT (loc, expected_err, actual_err);
1945 : else
1946 192 : ASSERT_STREQ_AT (loc,
1947 : "range starts after LINE_MAP_MAX_LOCATION_WITH_COLS",
1948 : actual_err);
1949 636 : }
1950 :
1951 : #define ASSERT_HAS_NO_SUBSTRING_RANGES(LEXER_TEST, STRLOC, TYPE, ERR) \
1952 : assert_has_no_substring_ranges (SELFTEST_LOCATION, (LEXER_TEST), \
1953 : (STRLOC), (TYPE), (ERR))
1954 :
1955 : /* Lex a simple string literal. Verify the substring location data, before
1956 : and after running cpp_interpret_string on it. */
1957 :
1958 : static void
1959 96 : test_lexer_string_locations_simple (const line_table_case &case_)
1960 : {
1961 : /* Digits 0-9 (with 0 at column 10), the simple way.
1962 : ....................000000000.11111111112.2222222223333333333
1963 : ....................123456789.01234567890.1234567890123456789
1964 : We add a trailing comment to ensure that we correctly locate
1965 : the end of the string literal token. */
1966 96 : const char *content = " \"0123456789\" /* not a string */\n";
1967 96 : lexer_test test (case_, content, NULL);
1968 :
1969 : /* Verify that we get the expected token back, with the correct
1970 : location information. */
1971 96 : const cpp_token *tok = test.get_token ();
1972 96 : ASSERT_EQ (tok->type, CPP_STRING);
1973 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"0123456789\"");
1974 96 : ASSERT_TOKEN_LOC_EQ (tok, test.m_tempfile.get_filename (), 1, 9, 20);
1975 :
1976 : /* At this point in lexing, the quote characters are treated as part of
1977 : the string (they are stripped off by cpp_interpret_string). */
1978 :
1979 96 : ASSERT_EQ (tok->val.str.len, 12);
1980 :
1981 : /* Verify that cpp_interpret_string works. */
1982 96 : cpp_string dst_string;
1983 96 : const enum cpp_ttype type = CPP_STRING;
1984 96 : bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
1985 : &dst_string, type);
1986 96 : ASSERT_TRUE (result);
1987 96 : ASSERT_STREQ ("0123456789", (const char *)dst_string.text);
1988 96 : free (const_cast <unsigned char *> (dst_string.text));
1989 :
1990 : /* Verify ranges of individual characters. This no longer includes the
1991 : opening quote, but does include the closing quote. */
1992 1152 : for (int i = 0; i <= 10; i++)
1993 1056 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1,
1994 : 10 + i, 10 + i);
1995 :
1996 96 : ASSERT_NUM_SUBSTRING_RANGES (test, tok->src_loc, type, 11);
1997 96 : }
1998 :
1999 : /* As test_lexer_string_locations_simple, but use an EBCDIC execution
2000 : encoding. */
2001 :
2002 : static void
2003 96 : test_lexer_string_locations_ebcdic (const line_table_case &case_)
2004 : {
2005 : /* EBCDIC support requires iconv. */
2006 96 : if (!HAVE_ICONV)
2007 0 : return;
2008 :
2009 : /* Digits 0-9 (with 0 at column 10), the simple way.
2010 : ....................000000000.11111111112.2222222223333333333
2011 : ....................123456789.01234567890.1234567890123456789
2012 : We add a trailing comment to ensure that we correctly locate
2013 : the end of the string literal token. */
2014 96 : const char *content = " \"0123456789\" /* not a string */\n";
2015 96 : ebcdic_execution_charset use_ebcdic;
2016 96 : lexer_test test (case_, content, &use_ebcdic);
2017 :
2018 : /* Verify that we get the expected token back, with the correct
2019 : location information. */
2020 96 : const cpp_token *tok = test.get_token ();
2021 96 : ASSERT_EQ (tok->type, CPP_STRING);
2022 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"0123456789\"");
2023 96 : ASSERT_TOKEN_LOC_EQ (tok, test.m_tempfile.get_filename (), 1, 9, 20);
2024 :
2025 : /* At this point in lexing, the quote characters are treated as part of
2026 : the string (they are stripped off by cpp_interpret_string). */
2027 :
2028 96 : ASSERT_EQ (tok->val.str.len, 12);
2029 :
2030 : /* The remainder of the test requires an iconv implementation that
2031 : can convert from UTF-8 to the EBCDIC encoding requested above. */
2032 96 : if (use_ebcdic.iconv_errors_occurred_p ())
2033 0 : return;
2034 :
2035 : /* Verify that cpp_interpret_string works. */
2036 96 : cpp_string dst_string;
2037 96 : const enum cpp_ttype type = CPP_STRING;
2038 96 : bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
2039 : &dst_string, type);
2040 96 : ASSERT_TRUE (result);
2041 : /* We should now have EBCDIC-encoded text, specifically
2042 : IBM1047-encoded (aka "EBCDIC 1047", or "Code page 1047").
2043 : The digits 0-9 are encoded as 240-249 i.e. 0xf0-0xf9. */
2044 96 : ASSERT_STREQ ("\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9",
2045 : (const char *)dst_string.text);
2046 96 : free (const_cast <unsigned char *> (dst_string.text));
2047 :
2048 : /* Verify that we don't attempt to record substring location information
2049 : for such cases. */
2050 96 : ASSERT_HAS_NO_SUBSTRING_RANGES
2051 : (test, tok->src_loc, type,
2052 : "execution character set != source character set");
2053 96 : }
2054 :
2055 : /* Lex a string literal containing a hex-escaped character.
2056 : Verify the substring location data, before and after running
2057 : cpp_interpret_string on it. */
2058 :
2059 : static void
2060 96 : test_lexer_string_locations_hex (const line_table_case &case_)
2061 : {
2062 : /* Digits 0-9, expressing digit 5 in ASCII as "\x35"
2063 : and with a space in place of digit 6, to terminate the escaped
2064 : hex code.
2065 : ....................000000000.111111.11112222.
2066 : ....................123456789.012345.67890123. */
2067 96 : const char *content = " \"01234\\x35 789\"\n";
2068 96 : lexer_test test (case_, content, NULL);
2069 :
2070 : /* Verify that we get the expected token back, with the correct
2071 : location information. */
2072 96 : const cpp_token *tok = test.get_token ();
2073 96 : ASSERT_EQ (tok->type, CPP_STRING);
2074 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"01234\\x35 789\"");
2075 96 : ASSERT_TOKEN_LOC_EQ (tok, test.m_tempfile.get_filename (), 1, 9, 23);
2076 :
2077 : /* At this point in lexing, the quote characters are treated as part of
2078 : the string (they are stripped off by cpp_interpret_string). */
2079 96 : ASSERT_EQ (tok->val.str.len, 15);
2080 :
2081 : /* Verify that cpp_interpret_string works. */
2082 96 : cpp_string dst_string;
2083 96 : const enum cpp_ttype type = CPP_STRING;
2084 96 : bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
2085 : &dst_string, type);
2086 96 : ASSERT_TRUE (result);
2087 96 : ASSERT_STREQ ("012345 789", (const char *)dst_string.text);
2088 96 : free (const_cast <unsigned char *> (dst_string.text));
2089 :
2090 : /* Verify ranges of individual characters. This no longer includes the
2091 : opening quote, but does include the closing quote. */
2092 576 : for (int i = 0; i <= 4; i++)
2093 480 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 10 + i, 10 + i);
2094 96 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, 5, 1, 15, 18);
2095 576 : for (int i = 6; i <= 10; i++)
2096 480 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 13 + i, 13 + i);
2097 :
2098 96 : ASSERT_NUM_SUBSTRING_RANGES (test, tok->src_loc, type, 11);
2099 96 : }
2100 :
2101 : /* Lex a string literal containing an octal-escaped character.
2102 : Verify the substring location data after running cpp_interpret_string
2103 : on it. */
2104 :
2105 : static void
2106 96 : test_lexer_string_locations_oct (const line_table_case &case_)
2107 : {
2108 : /* Digits 0-9, expressing digit 5 in ASCII as "\065"
2109 : and with a space in place of digit 6, to terminate the escaped
2110 : octal code.
2111 : ....................000000000.111111.11112222.2222223333333333444
2112 : ....................123456789.012345.67890123.4567890123456789012 */
2113 96 : const char *content = " \"01234\\065 789\" /* not a string */\n";
2114 96 : lexer_test test (case_, content, NULL);
2115 :
2116 : /* Verify that we get the expected token back, with the correct
2117 : location information. */
2118 96 : const cpp_token *tok = test.get_token ();
2119 96 : ASSERT_EQ (tok->type, CPP_STRING);
2120 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"01234\\065 789\"");
2121 :
2122 : /* Verify that cpp_interpret_string works. */
2123 96 : cpp_string dst_string;
2124 96 : const enum cpp_ttype type = CPP_STRING;
2125 96 : bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
2126 : &dst_string, type);
2127 96 : ASSERT_TRUE (result);
2128 96 : ASSERT_STREQ ("012345 789", (const char *)dst_string.text);
2129 96 : free (const_cast <unsigned char *> (dst_string.text));
2130 :
2131 : /* Verify ranges of individual characters. This no longer includes the
2132 : opening quote, but does include the closing quote. */
2133 576 : for (int i = 0; i < 5; i++)
2134 480 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 10 + i, 10 + i);
2135 96 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, 5, 1, 15, 18);
2136 576 : for (int i = 6; i <= 10; i++)
2137 480 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 13 + i, 13 + i);
2138 :
2139 96 : ASSERT_NUM_SUBSTRING_RANGES (test, tok->src_loc, type, 11);
2140 96 : }
2141 :
2142 : /* Test of string literal containing letter escapes. */
2143 :
2144 : static void
2145 96 : test_lexer_string_locations_letter_escape_1 (const line_table_case &case_)
2146 : {
2147 : /* The string "\tfoo\\\nbar" i.e. tab, "foo", backslash, newline, bar.
2148 : .....................000000000.1.11111.1.1.11222.22222223333333
2149 : .....................123456789.0.12345.6.7.89012.34567890123456. */
2150 96 : const char *content = (" \"\\tfoo\\\\\\nbar\" /* non-str */\n");
2151 96 : lexer_test test (case_, content, NULL);
2152 :
2153 : /* Verify that we get the expected tokens back. */
2154 96 : const cpp_token *tok = test.get_token ();
2155 96 : ASSERT_EQ (tok->type, CPP_STRING);
2156 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"\\tfoo\\\\\\nbar\"");
2157 :
2158 : /* Verify ranges of individual characters. */
2159 : /* "\t". */
2160 96 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, CPP_STRING,
2161 : 0, 1, 10, 11);
2162 : /* "foo". */
2163 384 : for (int i = 1; i <= 3; i++)
2164 288 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, CPP_STRING,
2165 : i, 1, 11 + i, 11 + i);
2166 : /* "\\" and "\n". */
2167 96 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, CPP_STRING,
2168 : 4, 1, 15, 16);
2169 96 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, CPP_STRING,
2170 : 5, 1, 17, 18);
2171 :
2172 : /* "bar" and closing quote for nul-terminator. */
2173 480 : for (int i = 6; i <= 9; i++)
2174 384 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, CPP_STRING,
2175 : i, 1, 13 + i, 13 + i);
2176 :
2177 96 : ASSERT_NUM_SUBSTRING_RANGES (test, tok->src_loc, CPP_STRING, 10);
2178 96 : }
2179 :
2180 : /* Another test of a string literal containing a letter escape.
2181 : Based on string seen in
2182 : printf ("%-%\n");
2183 : in gcc.dg/format/c90-printf-1.c. */
2184 :
2185 : static void
2186 96 : test_lexer_string_locations_letter_escape_2 (const line_table_case &case_)
2187 : {
2188 : /* .....................000000000.1111.11.1111.22222222223.
2189 : .....................123456789.0123.45.6789.01234567890. */
2190 96 : const char *content = (" \"%-%\\n\" /* non-str */\n");
2191 96 : lexer_test test (case_, content, NULL);
2192 :
2193 : /* Verify that we get the expected tokens back. */
2194 96 : const cpp_token *tok = test.get_token ();
2195 96 : ASSERT_EQ (tok->type, CPP_STRING);
2196 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"%-%\\n\"");
2197 :
2198 : /* Verify ranges of individual characters. */
2199 : /* "%-%". */
2200 384 : for (int i = 0; i < 3; i++)
2201 288 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, CPP_STRING,
2202 : i, 1, 10 + i, 10 + i);
2203 : /* "\n". */
2204 96 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, CPP_STRING,
2205 : 3, 1, 13, 14);
2206 :
2207 : /* Closing quote for nul-terminator. */
2208 96 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, CPP_STRING,
2209 : 4, 1, 15, 15);
2210 :
2211 96 : ASSERT_NUM_SUBSTRING_RANGES (test, tok->src_loc, CPP_STRING, 5);
2212 96 : }
2213 :
2214 : /* Lex a string literal containing UCN 4 characters.
2215 : Verify the substring location data after running cpp_interpret_string
2216 : on it. */
2217 :
2218 : static void
2219 96 : test_lexer_string_locations_ucn4 (const line_table_case &case_)
2220 : {
2221 : /* Digits 0-9, expressing digits 5 and 6 as Roman numerals expressed
2222 : as UCN 4.
2223 : ....................000000000.111111.111122.222222223.33333333344444
2224 : ....................123456789.012345.678901.234567890.12345678901234 */
2225 96 : const char *content = " \"01234\\u2174\\u2175789\" /* non-str */\n";
2226 96 : lexer_test test (case_, content, NULL);
2227 :
2228 : /* Verify that we get the expected token back, with the correct
2229 : location information. */
2230 96 : const cpp_token *tok = test.get_token ();
2231 96 : ASSERT_EQ (tok->type, CPP_STRING);
2232 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"01234\\u2174\\u2175789\"");
2233 :
2234 : /* Verify that cpp_interpret_string works.
2235 : The string should be encoded in the execution character
2236 : set. Assuming that is UTF-8, we should have the following:
2237 : ----------- ---- ----- ------- ----------------
2238 : Byte offset Byte Octal Unicode Source Column(s)
2239 : ----------- ---- ----- ------- ----------------
2240 : 0 0x30 '0' 10
2241 : 1 0x31 '1' 11
2242 : 2 0x32 '2' 12
2243 : 3 0x33 '3' 13
2244 : 4 0x34 '4' 14
2245 : 5 0xE2 \342 U+2174 15-20
2246 : 6 0x85 \205 (cont) 15-20
2247 : 7 0xB4 \264 (cont) 15-20
2248 : 8 0xE2 \342 U+2175 21-26
2249 : 9 0x85 \205 (cont) 21-26
2250 : 10 0xB5 \265 (cont) 21-26
2251 : 11 0x37 '7' 27
2252 : 12 0x38 '8' 28
2253 : 13 0x39 '9' 29
2254 : 14 0x00 30 (closing quote)
2255 : ----------- ---- ----- ------- ---------------. */
2256 :
2257 96 : cpp_string dst_string;
2258 96 : const enum cpp_ttype type = CPP_STRING;
2259 96 : bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
2260 : &dst_string, type);
2261 96 : ASSERT_TRUE (result);
2262 96 : ASSERT_STREQ ("01234\342\205\264\342\205\265789",
2263 : (const char *)dst_string.text);
2264 96 : free (const_cast <unsigned char *> (dst_string.text));
2265 :
2266 : /* Verify ranges of individual characters. This no longer includes the
2267 : opening quote, but does include the closing quote.
2268 : '01234'. */
2269 576 : for (int i = 0; i <= 4; i++)
2270 480 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 10 + i, 10 + i);
2271 : /* U+2174. */
2272 384 : for (int i = 5; i <= 7; i++)
2273 288 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 15, 20);
2274 : /* U+2175. */
2275 384 : for (int i = 8; i <= 10; i++)
2276 288 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 21, 26);
2277 : /* '789' and nul terminator */
2278 480 : for (int i = 11; i <= 14; i++)
2279 384 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 16 + i, 16 + i);
2280 :
2281 96 : ASSERT_NUM_SUBSTRING_RANGES (test, tok->src_loc, type, 15);
2282 96 : }
2283 :
2284 : /* Lex a string literal containing UCN 8 characters.
2285 : Verify the substring location data after running cpp_interpret_string
2286 : on it. */
2287 :
2288 : static void
2289 96 : test_lexer_string_locations_ucn8 (const line_table_case &case_)
2290 : {
2291 : /* Digits 0-9, expressing digits 5 and 6 as Roman numerals as UCN 8.
2292 : ....................000000000.111111.1111222222.2222333333333.344444
2293 : ....................123456789.012345.6789012345.6789012345678.901234 */
2294 96 : const char *content = " \"01234\\U00002174\\U00002175789\" /* */\n";
2295 96 : lexer_test test (case_, content, NULL);
2296 :
2297 : /* Verify that we get the expected token back, with the correct
2298 : location information. */
2299 96 : const cpp_token *tok = test.get_token ();
2300 96 : ASSERT_EQ (tok->type, CPP_STRING);
2301 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok,
2302 : "\"01234\\U00002174\\U00002175789\"");
2303 :
2304 : /* Verify that cpp_interpret_string works.
2305 : The UTF-8 encoding of the string is identical to that from
2306 : the ucn4 testcase above; the only difference is the column
2307 : locations. */
2308 96 : cpp_string dst_string;
2309 96 : const enum cpp_ttype type = CPP_STRING;
2310 96 : bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
2311 : &dst_string, type);
2312 96 : ASSERT_TRUE (result);
2313 96 : ASSERT_STREQ ("01234\342\205\264\342\205\265789",
2314 : (const char *)dst_string.text);
2315 96 : free (const_cast <unsigned char *> (dst_string.text));
2316 :
2317 : /* Verify ranges of individual characters. This no longer includes the
2318 : opening quote, but does include the closing quote.
2319 : '01234'. */
2320 576 : for (int i = 0; i <= 4; i++)
2321 480 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 10 + i, 10 + i);
2322 : /* U+2174. */
2323 384 : for (int i = 5; i <= 7; i++)
2324 288 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 15, 24);
2325 : /* U+2175. */
2326 384 : for (int i = 8; i <= 10; i++)
2327 288 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 25, 34);
2328 : /* '789' at columns 35-37 */
2329 384 : for (int i = 11; i <= 13; i++)
2330 288 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 24 + i, 24 + i);
2331 : /* Closing quote/nul-terminator at column 38. */
2332 96 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, 14, 1, 38, 38);
2333 :
2334 96 : ASSERT_NUM_SUBSTRING_RANGES (test, tok->src_loc, type, 15);
2335 96 : }
2336 :
2337 : /* Fetch a big-endian 32-bit value and convert to host endianness. */
2338 :
2339 : static uint32_t
2340 768 : uint32_from_big_endian (const uint32_t *ptr_be_value)
2341 : {
2342 768 : const unsigned char *buf = (const unsigned char *)ptr_be_value;
2343 768 : return (((uint32_t) buf[0] << 24)
2344 768 : | ((uint32_t) buf[1] << 16)
2345 768 : | ((uint32_t) buf[2] << 8)
2346 768 : | (uint32_t) buf[3]);
2347 : }
2348 :
2349 : /* Lex a wide string literal and verify that attempts to read substring
2350 : location data from it fail gracefully. */
2351 :
2352 : static void
2353 96 : test_lexer_string_locations_wide_string (const line_table_case &case_)
2354 : {
2355 : /* Digits 0-9.
2356 : ....................000000000.11111111112.22222222233333
2357 : ....................123456789.01234567890.12345678901234 */
2358 96 : const char *content = " L\"0123456789\" /* non-str */\n";
2359 96 : lexer_test test (case_, content, NULL);
2360 :
2361 : /* Verify that we get the expected token back, with the correct
2362 : location information. */
2363 96 : const cpp_token *tok = test.get_token ();
2364 96 : ASSERT_EQ (tok->type, CPP_WSTRING);
2365 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "L\"0123456789\"");
2366 :
2367 : /* Verify that cpp_interpret_string works, using CPP_WSTRING. */
2368 96 : cpp_string dst_string;
2369 96 : const enum cpp_ttype type = CPP_WSTRING;
2370 96 : bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
2371 : &dst_string, type);
2372 96 : ASSERT_TRUE (result);
2373 : /* The cpp_reader defaults to big-endian with
2374 : CHAR_BIT * sizeof (int) for the wchar_precision, so dst_string should
2375 : now be encoded as UTF-32BE. */
2376 96 : const uint32_t *be32_chars = (const uint32_t *)dst_string.text;
2377 96 : ASSERT_EQ ('0', uint32_from_big_endian (&be32_chars[0]));
2378 96 : ASSERT_EQ ('5', uint32_from_big_endian (&be32_chars[5]));
2379 96 : ASSERT_EQ ('9', uint32_from_big_endian (&be32_chars[9]));
2380 96 : ASSERT_EQ (0, uint32_from_big_endian (&be32_chars[10]));
2381 96 : free (const_cast <unsigned char *> (dst_string.text));
2382 :
2383 : /* We don't yet support generating substring location information
2384 : for L"" strings. */
2385 96 : ASSERT_HAS_NO_SUBSTRING_RANGES
2386 : (test, tok->src_loc, type,
2387 : "execution character set != source character set");
2388 96 : }
2389 :
2390 : /* Fetch a big-endian 16-bit value and convert to host endianness. */
2391 :
2392 : static uint16_t
2393 384 : uint16_from_big_endian (const uint16_t *ptr_be_value)
2394 : {
2395 384 : const unsigned char *buf = (const unsigned char *)ptr_be_value;
2396 384 : return ((uint16_t) buf[0] << 8) | (uint16_t) buf[1];
2397 : }
2398 :
2399 : /* Lex a u"" string literal and verify that attempts to read substring
2400 : location data from it fail gracefully. */
2401 :
2402 : static void
2403 96 : test_lexer_string_locations_string16 (const line_table_case &case_)
2404 : {
2405 : /* Digits 0-9.
2406 : ....................000000000.11111111112.22222222233333
2407 : ....................123456789.01234567890.12345678901234 */
2408 96 : const char *content = " u\"0123456789\" /* non-str */\n";
2409 96 : lexer_test test (case_, content, NULL);
2410 :
2411 : /* Verify that we get the expected token back, with the correct
2412 : location information. */
2413 96 : const cpp_token *tok = test.get_token ();
2414 96 : ASSERT_EQ (tok->type, CPP_STRING16);
2415 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "u\"0123456789\"");
2416 :
2417 : /* Verify that cpp_interpret_string works, using CPP_STRING16. */
2418 96 : cpp_string dst_string;
2419 96 : const enum cpp_ttype type = CPP_STRING16;
2420 96 : bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
2421 : &dst_string, type);
2422 96 : ASSERT_TRUE (result);
2423 :
2424 : /* The cpp_reader defaults to big-endian, so dst_string should
2425 : now be encoded as UTF-16BE. */
2426 96 : const uint16_t *be16_chars = (const uint16_t *)dst_string.text;
2427 96 : ASSERT_EQ ('0', uint16_from_big_endian (&be16_chars[0]));
2428 96 : ASSERT_EQ ('5', uint16_from_big_endian (&be16_chars[5]));
2429 96 : ASSERT_EQ ('9', uint16_from_big_endian (&be16_chars[9]));
2430 96 : ASSERT_EQ (0, uint16_from_big_endian (&be16_chars[10]));
2431 96 : free (const_cast <unsigned char *> (dst_string.text));
2432 :
2433 : /* We don't yet support generating substring location information
2434 : for L"" strings. */
2435 96 : ASSERT_HAS_NO_SUBSTRING_RANGES
2436 : (test, tok->src_loc, type,
2437 : "execution character set != source character set");
2438 96 : }
2439 :
2440 : /* Lex a U"" string literal and verify that attempts to read substring
2441 : location data from it fail gracefully. */
2442 :
2443 : static void
2444 96 : test_lexer_string_locations_string32 (const line_table_case &case_)
2445 : {
2446 : /* Digits 0-9.
2447 : ....................000000000.11111111112.22222222233333
2448 : ....................123456789.01234567890.12345678901234 */
2449 96 : const char *content = " U\"0123456789\" /* non-str */\n";
2450 96 : lexer_test test (case_, content, NULL);
2451 :
2452 : /* Verify that we get the expected token back, with the correct
2453 : location information. */
2454 96 : const cpp_token *tok = test.get_token ();
2455 96 : ASSERT_EQ (tok->type, CPP_STRING32);
2456 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "U\"0123456789\"");
2457 :
2458 : /* Verify that cpp_interpret_string works, using CPP_STRING32. */
2459 96 : cpp_string dst_string;
2460 96 : const enum cpp_ttype type = CPP_STRING32;
2461 96 : bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
2462 : &dst_string, type);
2463 96 : ASSERT_TRUE (result);
2464 :
2465 : /* The cpp_reader defaults to big-endian, so dst_string should
2466 : now be encoded as UTF-32BE. */
2467 96 : const uint32_t *be32_chars = (const uint32_t *)dst_string.text;
2468 96 : ASSERT_EQ ('0', uint32_from_big_endian (&be32_chars[0]));
2469 96 : ASSERT_EQ ('5', uint32_from_big_endian (&be32_chars[5]));
2470 96 : ASSERT_EQ ('9', uint32_from_big_endian (&be32_chars[9]));
2471 96 : ASSERT_EQ (0, uint32_from_big_endian (&be32_chars[10]));
2472 96 : free (const_cast <unsigned char *> (dst_string.text));
2473 :
2474 : /* We don't yet support generating substring location information
2475 : for L"" strings. */
2476 96 : ASSERT_HAS_NO_SUBSTRING_RANGES
2477 : (test, tok->src_loc, type,
2478 : "execution character set != source character set");
2479 96 : }
2480 :
2481 : /* Lex a u8-string literal.
2482 : Verify the substring location data after running cpp_interpret_string
2483 : on it. */
2484 :
2485 : static void
2486 96 : test_lexer_string_locations_u8 (const line_table_case &case_)
2487 : {
2488 : /* Digits 0-9.
2489 : ....................000000000.11111111112.22222222233333
2490 : ....................123456789.01234567890.12345678901234 */
2491 96 : const char *content = " u8\"0123456789\" /* non-str */\n";
2492 96 : lexer_test test (case_, content, NULL);
2493 :
2494 : /* Verify that we get the expected token back, with the correct
2495 : location information. */
2496 96 : const cpp_token *tok = test.get_token ();
2497 96 : ASSERT_EQ (tok->type, CPP_UTF8STRING);
2498 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "u8\"0123456789\"");
2499 :
2500 : /* Verify that cpp_interpret_string works. */
2501 96 : cpp_string dst_string;
2502 96 : const enum cpp_ttype type = CPP_STRING;
2503 96 : bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
2504 : &dst_string, type);
2505 96 : ASSERT_TRUE (result);
2506 96 : ASSERT_STREQ ("0123456789", (const char *)dst_string.text);
2507 96 : free (const_cast <unsigned char *> (dst_string.text));
2508 :
2509 : /* Verify ranges of individual characters. This no longer includes the
2510 : opening quote, but does include the closing quote. */
2511 1152 : for (int i = 0; i <= 10; i++)
2512 1056 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 10 + i, 10 + i);
2513 96 : }
2514 :
2515 : /* Lex a string literal containing UTF-8 source characters.
2516 : Verify the substring location data after running cpp_interpret_string
2517 : on it. */
2518 :
2519 : static void
2520 96 : test_lexer_string_locations_utf8_source (const line_table_case &case_)
2521 : {
2522 : /* This string literal is written out to the source file as UTF-8,
2523 : and is of the form "before mojibake after", where "mojibake"
2524 : is written as the following four unicode code points:
2525 : U+6587 CJK UNIFIED IDEOGRAPH-6587
2526 : U+5B57 CJK UNIFIED IDEOGRAPH-5B57
2527 : U+5316 CJK UNIFIED IDEOGRAPH-5316
2528 : U+3051 HIRAGANA LETTER KE.
2529 : Each of these is 3 bytes wide when encoded in UTF-8, whereas the
2530 : "before" and "after" are 1 byte per unicode character.
2531 :
2532 : The numbering shown are "columns", which are *byte* numbers within
2533 : the line, rather than unicode character numbers.
2534 :
2535 : .................... 000000000.1111111.
2536 : .................... 123456789.0123456. */
2537 96 : const char *content = (" \"before "
2538 : /* U+6587 CJK UNIFIED IDEOGRAPH-6587
2539 : UTF-8: 0xE6 0x96 0x87
2540 : C octal escaped UTF-8: \346\226\207
2541 : "column" numbers: 17-19. */
2542 : "\346\226\207"
2543 :
2544 : /* U+5B57 CJK UNIFIED IDEOGRAPH-5B57
2545 : UTF-8: 0xE5 0xAD 0x97
2546 : C octal escaped UTF-8: \345\255\227
2547 : "column" numbers: 20-22. */
2548 : "\345\255\227"
2549 :
2550 : /* U+5316 CJK UNIFIED IDEOGRAPH-5316
2551 : UTF-8: 0xE5 0x8C 0x96
2552 : C octal escaped UTF-8: \345\214\226
2553 : "column" numbers: 23-25. */
2554 : "\345\214\226"
2555 :
2556 : /* U+3051 HIRAGANA LETTER KE
2557 : UTF-8: 0xE3 0x81 0x91
2558 : C octal escaped UTF-8: \343\201\221
2559 : "column" numbers: 26-28. */
2560 : "\343\201\221"
2561 :
2562 : /* column numbers 29 onwards
2563 : 2333333.33334444444444
2564 : 9012345.67890123456789. */
2565 : " after\" /* non-str */\n");
2566 96 : lexer_test test (case_, content, NULL);
2567 :
2568 : /* Verify that we get the expected token back, with the correct
2569 : location information. */
2570 96 : const cpp_token *tok = test.get_token ();
2571 96 : ASSERT_EQ (tok->type, CPP_STRING);
2572 96 : ASSERT_TOKEN_AS_TEXT_EQ
2573 : (test.m_parser, tok,
2574 : "\"before \346\226\207\345\255\227\345\214\226\343\201\221 after\"");
2575 :
2576 : /* Verify that cpp_interpret_string works. */
2577 96 : cpp_string dst_string;
2578 96 : const enum cpp_ttype type = CPP_STRING;
2579 96 : bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
2580 : &dst_string, type);
2581 96 : ASSERT_TRUE (result);
2582 96 : ASSERT_STREQ
2583 : ("before \346\226\207\345\255\227\345\214\226\343\201\221 after",
2584 : (const char *)dst_string.text);
2585 96 : free (const_cast <unsigned char *> (dst_string.text));
2586 :
2587 : /* Verify ranges of individual characters. This no longer includes the
2588 : opening quote, but does include the closing quote.
2589 : Assuming that both source and execution encodings are UTF-8, we have
2590 : a run of 25 octets in each, plus the NUL terminator. */
2591 2496 : for (int i = 0; i < 25; i++)
2592 2400 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, i, 1, 10 + i, 10 + i);
2593 : /* NUL-terminator should use the closing quote at column 35. */
2594 96 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, type, 25, 1, 35, 35);
2595 :
2596 96 : ASSERT_NUM_SUBSTRING_RANGES (test, tok->src_loc, type, 26);
2597 96 : }
2598 :
2599 : /* Test of string literal concatenation. */
2600 :
2601 : static void
2602 96 : test_lexer_string_locations_concatenation_1 (const line_table_case &case_)
2603 : {
2604 : /* Digits 0-9.
2605 : .....................000000000.111111.11112222222222
2606 : .....................123456789.012345.67890123456789. */
2607 96 : const char *content = (" \"01234\" /* non-str */\n"
2608 : " \"56789\" /* non-str */\n");
2609 96 : lexer_test test (case_, content, NULL);
2610 :
2611 96 : location_t input_locs[2];
2612 :
2613 : /* Verify that we get the expected tokens back. */
2614 96 : auto_vec <cpp_string> input_strings;
2615 96 : const cpp_token *tok_a = test.get_token ();
2616 96 : ASSERT_EQ (tok_a->type, CPP_STRING);
2617 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok_a, "\"01234\"");
2618 96 : input_strings.safe_push (tok_a->val.str);
2619 96 : input_locs[0] = tok_a->src_loc;
2620 :
2621 96 : const cpp_token *tok_b = test.get_token ();
2622 96 : ASSERT_EQ (tok_b->type, CPP_STRING);
2623 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok_b, "\"56789\"");
2624 96 : input_strings.safe_push (tok_b->val.str);
2625 96 : input_locs[1] = tok_b->src_loc;
2626 :
2627 : /* Verify that cpp_interpret_string works. */
2628 96 : cpp_string dst_string;
2629 96 : const enum cpp_ttype type = CPP_STRING;
2630 96 : bool result = cpp_interpret_string (test.m_parser,
2631 96 : input_strings.address (), 2,
2632 : &dst_string, type);
2633 96 : ASSERT_TRUE (result);
2634 96 : ASSERT_STREQ ("0123456789", (const char *)dst_string.text);
2635 96 : free (const_cast <unsigned char *> (dst_string.text));
2636 :
2637 : /* Simulate c-lex.cc's lex_string in order to record concatenation. */
2638 96 : test.m_concats.record_string_concatenation (2, input_locs);
2639 :
2640 96 : location_t initial_loc = input_locs[0];
2641 :
2642 : /* "01234" on line 1. */
2643 576 : for (int i = 0; i <= 4; i++)
2644 480 : ASSERT_CHAR_AT_RANGE (test, initial_loc, type, i, 1, 10 + i, 10 + i);
2645 : /* "56789" in line 2, plus its closing quote for the nul terminator. */
2646 672 : for (int i = 5; i <= 10; i++)
2647 576 : ASSERT_CHAR_AT_RANGE (test, initial_loc, type, i, 2, 5 + i, 5 + i);
2648 :
2649 96 : ASSERT_NUM_SUBSTRING_RANGES (test, initial_loc, type, 11);
2650 96 : }
2651 :
2652 : /* Another test of string literal concatenation. */
2653 :
2654 : static void
2655 96 : test_lexer_string_locations_concatenation_2 (const line_table_case &case_)
2656 : {
2657 : /* Digits 0-9.
2658 : .....................000000000.111.11111112222222
2659 : .....................123456789.012.34567890123456. */
2660 96 : const char *content = (" \"01\" /* non-str */\n"
2661 : " \"23\" /* non-str */\n"
2662 : " \"45\" /* non-str */\n"
2663 : " \"67\" /* non-str */\n"
2664 : " \"89\" /* non-str */\n");
2665 96 : lexer_test test (case_, content, NULL);
2666 :
2667 96 : auto_vec <cpp_string> input_strings;
2668 96 : location_t input_locs[5];
2669 :
2670 : /* Verify that we get the expected tokens back. */
2671 576 : for (int i = 0; i < 5; i++)
2672 : {
2673 480 : const cpp_token *tok = test.get_token ();
2674 480 : ASSERT_EQ (tok->type, CPP_STRING);
2675 480 : input_strings.safe_push (tok->val.str);
2676 480 : input_locs[i] = tok->src_loc;
2677 : }
2678 :
2679 : /* Verify that cpp_interpret_string works. */
2680 96 : cpp_string dst_string;
2681 96 : const enum cpp_ttype type = CPP_STRING;
2682 96 : bool result = cpp_interpret_string (test.m_parser,
2683 96 : input_strings.address (), 5,
2684 : &dst_string, type);
2685 96 : ASSERT_TRUE (result);
2686 96 : ASSERT_STREQ ("0123456789", (const char *)dst_string.text);
2687 96 : free (const_cast <unsigned char *> (dst_string.text));
2688 :
2689 : /* Simulate c-lex.cc's lex_string in order to record concatenation. */
2690 96 : test.m_concats.record_string_concatenation (5, input_locs);
2691 :
2692 96 : location_t initial_loc = input_locs[0];
2693 :
2694 : /* Within ASSERT_CHAR_AT_RANGE (actually assert_char_at_range), we can
2695 : detect if the initial loc is after LINE_MAP_MAX_LOCATION_WITH_COLS
2696 : and expect get_source_range_for_substring to fail.
2697 : However, for a string concatenation test, we can have a case
2698 : where the initial string is fully before LINE_MAP_MAX_LOCATION_WITH_COLS,
2699 : but subsequent strings can be after it.
2700 : Attempting to detect this within assert_char_at_range
2701 : would overcomplicate the logic for the common test cases, so
2702 : we detect it here. */
2703 96 : if (should_have_column_data_p (input_locs[0])
2704 96 : && !should_have_column_data_p (input_locs[4]))
2705 : {
2706 : /* Verify that get_source_range_for_substring gracefully rejects
2707 : this case. */
2708 8 : source_range actual_range;
2709 8 : const char *err
2710 8 : = get_source_range_for_char (test.m_parser, test.m_file_cache,
2711 : &test.m_concats,
2712 : initial_loc, type, 0, &actual_range);
2713 8 : ASSERT_STREQ ("range starts after LINE_MAP_MAX_LOCATION_WITH_COLS", err);
2714 8 : return;
2715 : }
2716 :
2717 528 : for (int i = 0; i < 5; i++)
2718 1320 : for (int j = 0; j < 2; j++)
2719 880 : ASSERT_CHAR_AT_RANGE (test, initial_loc, type, (i * 2) + j,
2720 : i + 1, 10 + j, 10 + j);
2721 :
2722 : /* NUL-terminator should use the final closing quote at line 5 column 12. */
2723 88 : ASSERT_CHAR_AT_RANGE (test, initial_loc, type, 10, 5, 12, 12);
2724 :
2725 88 : ASSERT_NUM_SUBSTRING_RANGES (test, initial_loc, type, 11);
2726 96 : }
2727 :
2728 : /* Another test of string literal concatenation, this time combined with
2729 : various kinds of escaped characters. */
2730 :
2731 : static void
2732 96 : test_lexer_string_locations_concatenation_3 (const line_table_case &case_)
2733 : {
2734 : /* Digits 0-9, expressing digit 5 in ASCII as hex "\x35"
2735 : digit 6 in ASCII as octal "\066", concatenating multiple strings. */
2736 96 : const char *content
2737 : /* .000000000.111111.111.1.2222.222.2.2233.333.3333.34444444444555
2738 : .123456789.012345.678.9.0123.456.7.8901.234.5678.90123456789012. */
2739 : = (" \"01234\" \"\\x35\" \"\\066\" \"789\" /* non-str */\n");
2740 96 : lexer_test test (case_, content, NULL);
2741 :
2742 96 : auto_vec <cpp_string> input_strings;
2743 96 : location_t input_locs[4];
2744 :
2745 : /* Verify that we get the expected tokens back. */
2746 480 : for (int i = 0; i < 4; i++)
2747 : {
2748 384 : const cpp_token *tok = test.get_token ();
2749 384 : ASSERT_EQ (tok->type, CPP_STRING);
2750 384 : input_strings.safe_push (tok->val.str);
2751 384 : input_locs[i] = tok->src_loc;
2752 : }
2753 :
2754 : /* Verify that cpp_interpret_string works. */
2755 96 : cpp_string dst_string;
2756 96 : const enum cpp_ttype type = CPP_STRING;
2757 96 : bool result = cpp_interpret_string (test.m_parser,
2758 96 : input_strings.address (), 4,
2759 : &dst_string, type);
2760 96 : ASSERT_TRUE (result);
2761 96 : ASSERT_STREQ ("0123456789", (const char *)dst_string.text);
2762 96 : free (const_cast <unsigned char *> (dst_string.text));
2763 :
2764 : /* Simulate c-lex.cc's lex_string in order to record concatenation. */
2765 96 : test.m_concats.record_string_concatenation (4, input_locs);
2766 :
2767 96 : location_t initial_loc = input_locs[0];
2768 :
2769 576 : for (int i = 0; i <= 4; i++)
2770 480 : ASSERT_CHAR_AT_RANGE (test, initial_loc, type, i, 1, 10 + i, 10 + i);
2771 96 : ASSERT_CHAR_AT_RANGE (test, initial_loc, type, 5, 1, 19, 22);
2772 96 : ASSERT_CHAR_AT_RANGE (test, initial_loc, type, 6, 1, 27, 30);
2773 384 : for (int i = 7; i <= 9; i++)
2774 288 : ASSERT_CHAR_AT_RANGE (test, initial_loc, type, i, 1, 28 + i, 28 + i);
2775 :
2776 : /* NUL-terminator should use the location of the final closing quote. */
2777 96 : ASSERT_CHAR_AT_RANGE (test, initial_loc, type, 10, 1, 38, 38);
2778 :
2779 96 : ASSERT_NUM_SUBSTRING_RANGES (test, initial_loc, type, 11);
2780 96 : }
2781 :
2782 : /* Test of string literal in a macro. */
2783 :
2784 : static void
2785 96 : test_lexer_string_locations_macro (const line_table_case &case_)
2786 : {
2787 : /* Digits 0-9.
2788 : .....................0000000001111111111.22222222223.
2789 : .....................1234567890123456789.01234567890. */
2790 96 : const char *content = ("#define MACRO \"0123456789\" /* non-str */\n"
2791 : " MACRO");
2792 96 : lexer_test test (case_, content, NULL);
2793 :
2794 : /* Verify that we get the expected tokens back. */
2795 96 : const cpp_token *tok = test.get_token ();
2796 96 : ASSERT_EQ (tok->type, CPP_PADDING);
2797 :
2798 96 : tok = test.get_token ();
2799 96 : ASSERT_EQ (tok->type, CPP_STRING);
2800 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"0123456789\"");
2801 :
2802 : /* Verify ranges of individual characters. We ought to
2803 : see columns within the macro definition. */
2804 1152 : for (int i = 0; i <= 10; i++)
2805 1056 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, CPP_STRING,
2806 : i, 1, 20 + i, 20 + i);
2807 :
2808 96 : ASSERT_NUM_SUBSTRING_RANGES (test, tok->src_loc, CPP_STRING, 11);
2809 :
2810 96 : tok = test.get_token ();
2811 96 : ASSERT_EQ (tok->type, CPP_PADDING);
2812 96 : }
2813 :
2814 : /* Test of stringification of a macro argument. */
2815 :
2816 : static void
2817 96 : test_lexer_string_locations_stringified_macro_argument
2818 : (const line_table_case &case_)
2819 : {
2820 : /* .....................000000000111111111122222222223.
2821 : .....................123456789012345678901234567890. */
2822 96 : const char *content = ("#define MACRO(X) #X /* non-str */\n"
2823 : "MACRO(foo)\n");
2824 96 : lexer_test test (case_, content, NULL);
2825 :
2826 : /* Verify that we get the expected token back. */
2827 96 : const cpp_token *tok = test.get_token ();
2828 96 : ASSERT_EQ (tok->type, CPP_PADDING);
2829 :
2830 96 : tok = test.get_token ();
2831 96 : ASSERT_EQ (tok->type, CPP_STRING);
2832 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "\"foo\"");
2833 :
2834 : /* We don't support getting the location of a stringified macro
2835 : argument. Verify that it fails gracefully. */
2836 96 : ASSERT_HAS_NO_SUBSTRING_RANGES (test, tok->src_loc, CPP_STRING,
2837 : "cpp_interpret_string_1 failed");
2838 :
2839 96 : tok = test.get_token ();
2840 96 : ASSERT_EQ (tok->type, CPP_PADDING);
2841 :
2842 96 : tok = test.get_token ();
2843 96 : ASSERT_EQ (tok->type, CPP_PADDING);
2844 96 : }
2845 :
2846 : /* Ensure that we are fail gracefully if something attempts to pass
2847 : in a location that isn't a string literal token. Seen on this code:
2848 :
2849 : const char a[] = " %d ";
2850 : __builtin_printf (a, 0.5);
2851 : ^
2852 :
2853 : when c-format.cc erroneously used the indicated one-character
2854 : location as the format string location, leading to a read past the
2855 : end of a string buffer in cpp_interpret_string_1. */
2856 :
2857 : static void
2858 96 : test_lexer_string_locations_non_string (const line_table_case &case_)
2859 : {
2860 : /* .....................000000000111111111122222222223.
2861 : .....................123456789012345678901234567890. */
2862 96 : const char *content = (" a\n");
2863 96 : lexer_test test (case_, content, NULL);
2864 :
2865 : /* Verify that we get the expected token back. */
2866 96 : const cpp_token *tok = test.get_token ();
2867 96 : ASSERT_EQ (tok->type, CPP_NAME);
2868 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "a");
2869 :
2870 : /* At this point, libcpp is attempting to interpret the name as a
2871 : string literal, despite it not starting with a quote. We don't detect
2872 : that, but we should at least fail gracefully. */
2873 96 : ASSERT_HAS_NO_SUBSTRING_RANGES (test, tok->src_loc, CPP_STRING,
2874 : "cpp_interpret_string_1 failed");
2875 96 : }
2876 :
2877 : /* Ensure that we can read substring information for a token which
2878 : starts in one linemap and ends in another . Adapted from
2879 : gcc.dg/cpp/pr69985.c. */
2880 :
2881 : static void
2882 96 : test_lexer_string_locations_long_line (const line_table_case &case_)
2883 : {
2884 : /* .....................000000.000111111111
2885 : .....................123456.789012346789. */
2886 96 : const char *content = ("/* A very long line, so that we start a new line map. */\n"
2887 : " \"0123456789012345678901234567890123456789"
2888 : "0123456789012345678901234567890123456789"
2889 : "0123456789012345678901234567890123456789"
2890 : "0123456789\"\n");
2891 :
2892 96 : lexer_test test (case_, content, NULL);
2893 :
2894 : /* Verify that we get the expected token back. */
2895 96 : const cpp_token *tok = test.get_token ();
2896 96 : ASSERT_EQ (tok->type, CPP_STRING);
2897 :
2898 96 : if (!should_have_column_data_p (line_table->highest_location))
2899 36 : return;
2900 :
2901 : /* Verify ranges of individual characters. */
2902 60 : ASSERT_NUM_SUBSTRING_RANGES (test, tok->src_loc, CPP_STRING, 131);
2903 7920 : for (int i = 0; i < 131; i++)
2904 7860 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, CPP_STRING,
2905 : i, 2, 7 + i, 7 + i);
2906 96 : }
2907 :
2908 : /* Test of locations within a raw string that doesn't contain a newline. */
2909 :
2910 : static void
2911 96 : test_lexer_string_locations_raw_string_one_line (const line_table_case &case_)
2912 : {
2913 : /* .....................00.0000000111111111122.
2914 : .....................12.3456789012345678901. */
2915 96 : const char *content = ("R\"foo(0123456789)foo\"\n");
2916 96 : lexer_test test (case_, content, NULL);
2917 :
2918 : /* Verify that we get the expected token back. */
2919 96 : const cpp_token *tok = test.get_token ();
2920 96 : ASSERT_EQ (tok->type, CPP_STRING);
2921 :
2922 : /* Verify that cpp_interpret_string works. */
2923 96 : cpp_string dst_string;
2924 96 : const enum cpp_ttype type = CPP_STRING;
2925 96 : bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
2926 : &dst_string, type);
2927 96 : ASSERT_TRUE (result);
2928 96 : ASSERT_STREQ ("0123456789", (const char *)dst_string.text);
2929 96 : free (const_cast <unsigned char *> (dst_string.text));
2930 :
2931 96 : if (!should_have_column_data_p (line_table->highest_location))
2932 32 : return;
2933 :
2934 : /* 0-9, plus the nil terminator. */
2935 64 : ASSERT_NUM_SUBSTRING_RANGES (test, tok->src_loc, CPP_STRING, 11);
2936 768 : for (int i = 0; i < 11; i++)
2937 704 : ASSERT_CHAR_AT_RANGE (test, tok->src_loc, CPP_STRING,
2938 : i, 1, 7 + i, 7 + i);
2939 96 : }
2940 :
2941 : /* Test of locations within a raw string that contains a newline. */
2942 :
2943 : static void
2944 96 : test_lexer_string_locations_raw_string_multiline (const line_table_case &case_)
2945 : {
2946 : /* .....................00.0000.
2947 : .....................12.3456. */
2948 96 : const char *content = ("R\"foo(\n"
2949 : /* .....................00000.
2950 : .....................12345. */
2951 : "hello\n"
2952 : "world\n"
2953 : /* .....................00000.
2954 : .....................12345. */
2955 : ")foo\"\n");
2956 96 : lexer_test test (case_, content, NULL);
2957 :
2958 : /* Verify that we get the expected token back. */
2959 96 : const cpp_token *tok = test.get_token ();
2960 96 : ASSERT_EQ (tok->type, CPP_STRING);
2961 :
2962 : /* Verify that cpp_interpret_string works. */
2963 96 : cpp_string dst_string;
2964 96 : const enum cpp_ttype type = CPP_STRING;
2965 96 : bool result = cpp_interpret_string (test.m_parser, &tok->val.str, 1,
2966 : &dst_string, type);
2967 96 : ASSERT_TRUE (result);
2968 96 : ASSERT_STREQ ("\nhello\nworld\n", (const char *)dst_string.text);
2969 96 : free (const_cast <unsigned char *> (dst_string.text));
2970 :
2971 96 : if (!should_have_column_data_p (line_table->highest_location))
2972 36 : return;
2973 :
2974 : /* Currently we don't support locations within raw strings that
2975 : contain newlines. */
2976 60 : ASSERT_HAS_NO_SUBSTRING_RANGES (test, tok->src_loc, tok->type,
2977 : "range endpoints are on different lines");
2978 96 : }
2979 :
2980 : /* Test of parsing an unterminated raw string. */
2981 :
2982 : static void
2983 96 : test_lexer_string_locations_raw_string_unterminated (const line_table_case &case_)
2984 : {
2985 96 : const char *content = "R\"ouch()ouCh\" /* etc */";
2986 :
2987 96 : lexer_diagnostic_sink diagnostics;
2988 96 : lexer_test test (case_, content, &diagnostics);
2989 96 : test.m_implicitly_expect_EOF = false;
2990 :
2991 : /* Attempt to parse the raw string. */
2992 96 : const cpp_token *tok = test.get_token ();
2993 96 : ASSERT_EQ (tok->type, CPP_EOF);
2994 :
2995 96 : ASSERT_EQ (1, diagnostics.m_diagnostics.length ());
2996 : /* We expect the message "unterminated raw string"
2997 : in the "cpplib" translation domain.
2998 : It's not clear that dgettext is available on all supported hosts,
2999 : so this assertion is commented-out for now.
3000 : ASSERT_STREQ (dgettext ("cpplib", "unterminated raw string"),
3001 : diagnostics.m_diagnostics[0]);
3002 : */
3003 96 : }
3004 :
3005 : /* Test of lexing char constants. */
3006 :
3007 : static void
3008 96 : test_lexer_char_constants (const line_table_case &case_)
3009 : {
3010 : /* Various char constants.
3011 : .....................0000000001111111111.22222222223.
3012 : .....................1234567890123456789.01234567890. */
3013 96 : const char *content = (" 'a'\n"
3014 : " u'a'\n"
3015 : " U'a'\n"
3016 : " L'a'\n"
3017 : " 'abc'\n");
3018 96 : lexer_test test (case_, content, NULL);
3019 :
3020 : /* Verify that we get the expected tokens back. */
3021 : /* 'a'. */
3022 96 : const cpp_token *tok = test.get_token ();
3023 96 : ASSERT_EQ (tok->type, CPP_CHAR);
3024 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "'a'");
3025 :
3026 96 : unsigned int chars_seen;
3027 96 : int unsignedp;
3028 96 : cppchar_t cc = cpp_interpret_charconst (test.m_parser, tok,
3029 : &chars_seen, &unsignedp);
3030 96 : ASSERT_EQ (cc, 'a');
3031 96 : ASSERT_EQ (chars_seen, 1);
3032 :
3033 : /* u'a'. */
3034 96 : tok = test.get_token ();
3035 96 : ASSERT_EQ (tok->type, CPP_CHAR16);
3036 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "u'a'");
3037 :
3038 : /* U'a'. */
3039 96 : tok = test.get_token ();
3040 96 : ASSERT_EQ (tok->type, CPP_CHAR32);
3041 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "U'a'");
3042 :
3043 : /* L'a'. */
3044 96 : tok = test.get_token ();
3045 96 : ASSERT_EQ (tok->type, CPP_WCHAR);
3046 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "L'a'");
3047 :
3048 : /* 'abc' (c-char-sequence). */
3049 96 : tok = test.get_token ();
3050 96 : ASSERT_EQ (tok->type, CPP_CHAR);
3051 96 : ASSERT_TOKEN_AS_TEXT_EQ (test.m_parser, tok, "'abc'");
3052 96 : }
3053 : /* A table of interesting location_t values, giving one axis of our test
3054 : matrix. */
3055 :
3056 : static const location_t boundary_locations[] = {
3057 : /* Zero means "don't override the default values for a new line_table". */
3058 : 0,
3059 :
3060 : /* An arbitrary non-zero value that isn't close to one of
3061 : the boundary values below. */
3062 : 0x10000,
3063 :
3064 : /* Values near LINE_MAP_MAX_LOCATION_WITH_PACKED_RANGES. */
3065 : LINE_MAP_MAX_LOCATION_WITH_PACKED_RANGES - 0x100,
3066 : LINE_MAP_MAX_LOCATION_WITH_PACKED_RANGES - 1,
3067 : LINE_MAP_MAX_LOCATION_WITH_PACKED_RANGES,
3068 : LINE_MAP_MAX_LOCATION_WITH_PACKED_RANGES + 1,
3069 : LINE_MAP_MAX_LOCATION_WITH_PACKED_RANGES + 0x100,
3070 :
3071 : /* Values near LINE_MAP_MAX_LOCATION_WITH_COLS. */
3072 : LINE_MAP_MAX_LOCATION_WITH_COLS - 0x200,
3073 : LINE_MAP_MAX_LOCATION_WITH_COLS - 1,
3074 : LINE_MAP_MAX_LOCATION_WITH_COLS,
3075 : LINE_MAP_MAX_LOCATION_WITH_COLS + 1,
3076 : LINE_MAP_MAX_LOCATION_WITH_COLS + 0x200,
3077 : };
3078 :
3079 : /* Run TESTCASE multiple times, once for each case in our test matrix. */
3080 :
3081 : void
3082 244 : for_each_line_table_case (void (*testcase) (const line_table_case &))
3083 : {
3084 : /* As noted above in the description of struct line_table_case,
3085 : we want to explore a test matrix of interesting line_table
3086 : situations, running various selftests for each case within the
3087 : matrix. */
3088 :
3089 : /* Run all tests with:
3090 : (a) line_table->default_range_bits == 0, and
3091 : (b) line_table->default_range_bits == line_map_suggested_range_bits. */
3092 :
3093 732 : for (int default_range_bits: {0, line_map_suggested_range_bits})
3094 : {
3095 : /* ...and use each of the "interesting" location values as
3096 : the starting location within line_table. */
3097 488 : const int num_boundary_locations = ARRAY_SIZE (boundary_locations);
3098 6344 : for (int loc_idx = 0; loc_idx < num_boundary_locations; loc_idx++)
3099 : {
3100 5856 : line_table_case c (default_range_bits, boundary_locations[loc_idx]);
3101 5856 : testcase (c);
3102 : }
3103 : }
3104 244 : }
3105 :
3106 : /* Verify that when presented with a consecutive pair of locations with
3107 : a very large line offset, we don't attempt to consolidate them into
3108 : a single ordinary linemap where the line offsets within the line map
3109 : would lead to overflow (PR lto/88147). */
3110 :
3111 : static void
3112 4 : test_line_offset_overflow ()
3113 : {
3114 4 : line_table_test ltt (line_table_case (5, 0));
3115 :
3116 4 : linemap_add (line_table, LC_ENTER, false, "foo.c", 0);
3117 4 : linemap_line_start (line_table, 1, 100);
3118 4 : location_t loc_a = linemap_line_start (line_table, 2578, 255);
3119 4 : assert_loceq ("foo.c", 2578, 0, loc_a);
3120 :
3121 4 : const line_map_ordinary *ordmap_a = LINEMAPS_LAST_ORDINARY_MAP (line_table);
3122 4 : ASSERT_EQ (ordmap_a->m_column_and_range_bits, 13);
3123 4 : ASSERT_EQ (ordmap_a->m_range_bits, 5);
3124 :
3125 4 : location_t loc_b = linemap_line_start (line_table, 404198, 512);
3126 4 : assert_loceq ("foo.c", 404198, 0, loc_b);
3127 :
3128 : /* We should have started a new linemap, rather than attempting to store
3129 : a very large line offset. */
3130 4 : const line_map_ordinary *ordmap_b = LINEMAPS_LAST_ORDINARY_MAP (line_table);
3131 4 : ASSERT_NE (ordmap_a, ordmap_b);
3132 4 : }
3133 :
3134 4 : void test_cpp_utf8 ()
3135 : {
3136 4 : const int def_tabstop = 8;
3137 4 : cpp_char_column_policy policy (def_tabstop, cpp_wcwidth);
3138 :
3139 : /* Verify that wcwidth of invalid UTF-8 or control bytes is 1. */
3140 4 : {
3141 4 : int w_bad = cpp_display_width ("\xf0!\x9f!\x98!\x82!", 8, policy);
3142 4 : ASSERT_EQ (8, w_bad);
3143 4 : int w_ctrl = cpp_display_width ("\r\n\v\0\1", 5, policy);
3144 4 : ASSERT_EQ (5, w_ctrl);
3145 : }
3146 :
3147 : /* Verify that wcwidth of valid UTF-8 is as expected. */
3148 4 : {
3149 4 : const int w_pi = cpp_display_width ("\xcf\x80", 2, policy);
3150 4 : ASSERT_EQ (1, w_pi);
3151 4 : const int w_emoji = cpp_display_width ("\xf0\x9f\x98\x82", 4, policy);
3152 4 : ASSERT_EQ (2, w_emoji);
3153 4 : const int w_umlaut_precomposed = cpp_display_width ("\xc3\xbf", 2,
3154 : policy);
3155 4 : ASSERT_EQ (1, w_umlaut_precomposed);
3156 4 : const int w_umlaut_combining = cpp_display_width ("y\xcc\x88", 3,
3157 : policy);
3158 4 : ASSERT_EQ (1, w_umlaut_combining);
3159 4 : const int w_han = cpp_display_width ("\xe4\xb8\xba", 3, policy);
3160 4 : ASSERT_EQ (2, w_han);
3161 4 : const int w_ascii = cpp_display_width ("GCC", 3, policy);
3162 4 : ASSERT_EQ (3, w_ascii);
3163 4 : const int w_mixed = cpp_display_width ("\xcf\x80 = 3.14 \xf0\x9f\x98\x82"
3164 : "\x9f! \xe4\xb8\xba y\xcc\x88",
3165 : 24, policy);
3166 4 : ASSERT_EQ (18, w_mixed);
3167 : }
3168 :
3169 : /* Verify that display width properly expands tabs. */
3170 4 : {
3171 4 : const char *tstr = "\tabc\td";
3172 4 : ASSERT_EQ (6, cpp_display_width (tstr, 6,
3173 : cpp_char_column_policy (1, cpp_wcwidth)));
3174 4 : ASSERT_EQ (10, cpp_display_width (tstr, 6,
3175 : cpp_char_column_policy (3, cpp_wcwidth)));
3176 4 : ASSERT_EQ (17, cpp_display_width (tstr, 6,
3177 : cpp_char_column_policy (8, cpp_wcwidth)));
3178 4 : ASSERT_EQ (1,
3179 : cpp_display_column_to_byte_column
3180 : (tstr, 6, 7, cpp_char_column_policy (8, cpp_wcwidth)));
3181 : }
3182 :
3183 : /* Verify that cpp_byte_column_to_display_column can go past the end,
3184 : and similar edge cases. */
3185 4 : {
3186 4 : const char *str
3187 : /* Display columns.
3188 : 111111112345 */
3189 : = "\xcf\x80 abc";
3190 : /* 111122223456
3191 : Byte columns. */
3192 :
3193 4 : ASSERT_EQ (5, cpp_display_width (str, 6, policy));
3194 4 : ASSERT_EQ (105,
3195 : cpp_byte_column_to_display_column (str, 6, 106, policy));
3196 4 : ASSERT_EQ (10000,
3197 : cpp_byte_column_to_display_column (NULL, 0, 10000, policy));
3198 4 : ASSERT_EQ (0,
3199 : cpp_byte_column_to_display_column (NULL, 10000, 0, policy));
3200 : }
3201 :
3202 : /* Verify that cpp_display_column_to_byte_column can go past the end,
3203 : and similar edge cases, and check invertibility. */
3204 4 : {
3205 4 : const char *str
3206 : /* Display columns.
3207 : 000000000000000000000000000000000000011
3208 : 111111112222222234444444455555555678901 */
3209 : = "\xf0\x9f\x98\x82 \xf0\x9f\x98\x82 hello";
3210 : /* 000000000000000000000000000000000111111
3211 : 111122223333444456666777788889999012345
3212 : Byte columns. */
3213 4 : ASSERT_EQ (4, cpp_display_column_to_byte_column (str, 15, 2, policy));
3214 4 : ASSERT_EQ (15,
3215 : cpp_display_column_to_byte_column (str, 15, 11, policy));
3216 4 : ASSERT_EQ (115,
3217 : cpp_display_column_to_byte_column (str, 15, 111, policy));
3218 4 : ASSERT_EQ (10000,
3219 : cpp_display_column_to_byte_column (NULL, 0, 10000, policy));
3220 4 : ASSERT_EQ (0,
3221 : cpp_display_column_to_byte_column (NULL, 10000, 0, policy));
3222 :
3223 : /* Verify that we do not interrupt a UTF-8 sequence. */
3224 4 : ASSERT_EQ (4, cpp_display_column_to_byte_column (str, 15, 1, policy));
3225 :
3226 64 : for (int byte_col = 1; byte_col <= 15; ++byte_col)
3227 : {
3228 60 : const int disp_col
3229 60 : = cpp_byte_column_to_display_column (str, 15, byte_col, policy);
3230 60 : const int byte_col2
3231 60 : = cpp_display_column_to_byte_column (str, 15, disp_col, policy);
3232 :
3233 : /* If we ask for the display column in the middle of a UTF-8
3234 : sequence, it will return the length of the partial sequence,
3235 : matching the behavior of GCC before display column support.
3236 : Otherwise check the round trip was successful. */
3237 60 : if (byte_col < 4)
3238 12 : ASSERT_EQ (byte_col, disp_col);
3239 48 : else if (byte_col >= 6 && byte_col < 9)
3240 12 : ASSERT_EQ (3 + (byte_col - 5), disp_col);
3241 : else
3242 60 : ASSERT_EQ (byte_col2, byte_col);
3243 : }
3244 : }
3245 4 : }
3246 :
3247 : static bool
3248 36 : check_cpp_valid_utf8_p (const char *str)
3249 : {
3250 36 : return cpp_valid_utf8_p (str, strlen (str));
3251 : }
3252 :
3253 : /* Check that cpp_valid_utf8_p works as expected. */
3254 :
3255 : static void
3256 4 : test_cpp_valid_utf8_p ()
3257 : {
3258 4 : ASSERT_TRUE (check_cpp_valid_utf8_p ("hello world"));
3259 :
3260 : /* 2-byte char (pi). */
3261 4 : ASSERT_TRUE (check_cpp_valid_utf8_p("\xcf\x80"));
3262 :
3263 : /* 3-byte chars (the Japanese word "mojibake"). */
3264 4 : ASSERT_TRUE (check_cpp_valid_utf8_p
3265 : (
3266 : /* U+6587 CJK UNIFIED IDEOGRAPH-6587
3267 : UTF-8: 0xE6 0x96 0x87
3268 : C octal escaped UTF-8: \346\226\207. */
3269 : "\346\226\207"
3270 : /* U+5B57 CJK UNIFIED IDEOGRAPH-5B57
3271 : UTF-8: 0xE5 0xAD 0x97
3272 : C octal escaped UTF-8: \345\255\227. */
3273 : "\345\255\227"
3274 : /* U+5316 CJK UNIFIED IDEOGRAPH-5316
3275 : UTF-8: 0xE5 0x8C 0x96
3276 : C octal escaped UTF-8: \345\214\226. */
3277 : "\345\214\226"
3278 : /* U+3051 HIRAGANA LETTER KE
3279 : UTF-8: 0xE3 0x81 0x91
3280 : C octal escaped UTF-8: \343\201\221. */
3281 : "\343\201\221"));
3282 :
3283 : /* 4-byte char: an emoji. */
3284 4 : ASSERT_TRUE (check_cpp_valid_utf8_p ("\xf0\x9f\x98\x82"));
3285 :
3286 : /* Control codes, including the NUL byte. */
3287 4 : ASSERT_TRUE (cpp_valid_utf8_p ("\r\n\v\0\1", 5));
3288 :
3289 4 : ASSERT_FALSE (check_cpp_valid_utf8_p ("\xf0!\x9f!\x98!\x82!"));
3290 :
3291 : /* Unexpected continuation bytes. */
3292 4 : for (unsigned char continuation_byte = 0x80;
3293 260 : continuation_byte <= 0xbf;
3294 : continuation_byte++)
3295 256 : ASSERT_FALSE (cpp_valid_utf8_p ((const char *)&continuation_byte, 1));
3296 :
3297 : /* "Lonely start characters" for 2-byte sequences. */
3298 4 : {
3299 4 : unsigned char buf[2];
3300 4 : buf[1] = ' ';
3301 4 : for (buf[0] = 0xc0;
3302 132 : buf[0] <= 0xdf;
3303 128 : buf[0]++)
3304 128 : ASSERT_FALSE (cpp_valid_utf8_p ((const char *)buf, 2));
3305 : }
3306 :
3307 : /* "Lonely start characters" for 3-byte sequences. */
3308 4 : {
3309 4 : unsigned char buf[2];
3310 4 : buf[1] = ' ';
3311 4 : for (buf[0] = 0xe0;
3312 68 : buf[0] <= 0xef;
3313 64 : buf[0]++)
3314 64 : ASSERT_FALSE (cpp_valid_utf8_p ((const char *)buf, 2));
3315 : }
3316 :
3317 : /* "Lonely start characters" for 4-byte sequences. */
3318 4 : {
3319 4 : unsigned char buf[2];
3320 4 : buf[1] = ' ';
3321 4 : for (buf[0] = 0xf0;
3322 24 : buf[0] <= 0xf4;
3323 20 : buf[0]++)
3324 20 : ASSERT_FALSE (cpp_valid_utf8_p ((const char *)buf, 2));
3325 : }
3326 :
3327 : /* Invalid start characters (formerly valid for 5-byte and 6-byte
3328 : sequences). */
3329 4 : {
3330 4 : unsigned char buf[2];
3331 4 : buf[1] = ' ';
3332 4 : for (buf[0] = 0xf5;
3333 40 : buf[0] <= 0xfd;
3334 36 : buf[0]++)
3335 36 : ASSERT_FALSE (cpp_valid_utf8_p ((const char *)buf, 2));
3336 : }
3337 :
3338 : /* Impossible bytes. */
3339 4 : ASSERT_FALSE (check_cpp_valid_utf8_p ("\xc0"));
3340 4 : ASSERT_FALSE (check_cpp_valid_utf8_p ("\xc1"));
3341 4 : ASSERT_FALSE (check_cpp_valid_utf8_p ("\xfe"));
3342 4 : ASSERT_FALSE (check_cpp_valid_utf8_p ("\xff"));
3343 4 : }
3344 :
3345 : /* Run all of the selftests within this file. */
3346 :
3347 : void
3348 4 : input_cc_tests ()
3349 : {
3350 4 : test_linenum_comparisons ();
3351 4 : test_should_have_column_data_p ();
3352 4 : test_unknown_location ();
3353 4 : test_builtins ();
3354 4 : for_each_line_table_case (test_make_location_nonpure_range_endpoints);
3355 :
3356 4 : for_each_line_table_case (test_accessing_ordinary_linemaps);
3357 4 : for_each_line_table_case (test_lexer);
3358 4 : for_each_line_table_case (test_lexer_string_locations_simple);
3359 4 : for_each_line_table_case (test_lexer_string_locations_ebcdic);
3360 4 : for_each_line_table_case (test_lexer_string_locations_hex);
3361 4 : for_each_line_table_case (test_lexer_string_locations_oct);
3362 4 : for_each_line_table_case (test_lexer_string_locations_letter_escape_1);
3363 4 : for_each_line_table_case (test_lexer_string_locations_letter_escape_2);
3364 4 : for_each_line_table_case (test_lexer_string_locations_ucn4);
3365 4 : for_each_line_table_case (test_lexer_string_locations_ucn8);
3366 4 : for_each_line_table_case (test_lexer_string_locations_wide_string);
3367 4 : for_each_line_table_case (test_lexer_string_locations_string16);
3368 4 : for_each_line_table_case (test_lexer_string_locations_string32);
3369 4 : for_each_line_table_case (test_lexer_string_locations_u8);
3370 4 : for_each_line_table_case (test_lexer_string_locations_utf8_source);
3371 4 : for_each_line_table_case (test_lexer_string_locations_concatenation_1);
3372 4 : for_each_line_table_case (test_lexer_string_locations_concatenation_2);
3373 4 : for_each_line_table_case (test_lexer_string_locations_concatenation_3);
3374 4 : for_each_line_table_case (test_lexer_string_locations_macro);
3375 4 : for_each_line_table_case (test_lexer_string_locations_stringified_macro_argument);
3376 4 : for_each_line_table_case (test_lexer_string_locations_non_string);
3377 4 : for_each_line_table_case (test_lexer_string_locations_long_line);
3378 4 : for_each_line_table_case (test_lexer_string_locations_raw_string_one_line);
3379 4 : for_each_line_table_case (test_lexer_string_locations_raw_string_multiline);
3380 4 : for_each_line_table_case (test_lexer_string_locations_raw_string_unterminated);
3381 4 : for_each_line_table_case (test_lexer_char_constants);
3382 :
3383 4 : test_line_offset_overflow ();
3384 :
3385 4 : test_cpp_utf8 ();
3386 4 : test_cpp_valid_utf8_p ();
3387 4 : }
3388 :
3389 : } // namespace selftest
3390 :
3391 : #endif /* CHECKING_P */
|
