Line data Source code
1 : /* Preamble and helpers for the autogenerated gimple-match.cc file.
2 : Copyright (C) 2014-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "ssa.h"
29 : #include "cgraph.h"
30 : #include "vec-perm-indices.h"
31 : #include "fold-const.h"
32 : #include "fold-const-call.h"
33 : #include "stor-layout.h"
34 : #include "gimple-iterator.h"
35 : #include "gimple-fold.h"
36 : #include "calls.h"
37 : #include "tree-dfa.h"
38 : #include "builtins.h"
39 : #include "gimple-match.h"
40 : #include "tree-pass.h"
41 : #include "internal-fn.h"
42 : #include "case-cfn-macros.h"
43 : #include "gimplify.h"
44 : #include "memmodel.h"
45 : #include "optabs.h"
46 : #include "optabs-tree.h"
47 : #include "tree-eh.h"
48 : #include "dbgcnt.h"
49 : #include "tm.h"
50 : #include "gimple-range.h"
51 : #include "langhooks.h"
52 : #include "attribs.h"
53 : #include "asan.h"
54 :
55 : tree do_valueize (tree, tree (*)(tree), bool &);
56 : tree do_valueize (tree (*)(tree), tree);
57 :
58 : /* Helper for the autogenerated code, get at the definition of NAME when
59 : VALUEIZE allows that. */
60 :
61 : inline gimple *
62 12784654279 : get_def (tree (*valueize)(tree), tree name)
63 : {
64 12784654279 : if (valueize && ! valueize (name))
65 : return NULL;
66 5259919864 : return SSA_NAME_DEF_STMT (name);
67 : }
68 :
69 : /* Routine to determine if the types T1 and T2 are effectively
70 : the same for GIMPLE. If T1 or T2 is not a type, the test
71 : applies to their TREE_TYPE. */
72 :
73 : static inline bool
74 204892053 : types_match (tree t1, tree t2)
75 : {
76 204892053 : if (!TYPE_P (t1))
77 1335600 : t1 = TREE_TYPE (t1);
78 204892053 : if (!TYPE_P (t2))
79 1979058 : t2 = TREE_TYPE (t2);
80 :
81 204892053 : return types_compatible_p (t1, t2);
82 : }
83 :
84 : /* Routine to determine if the types T1, T2 and T3 are effectively
85 : the same for GIMPLE. If T1, T2 or T2 is not a type, the test
86 : applies to their TREE_TYPE. */
87 :
88 : static inline bool
89 1654 : types_match (tree t1, tree t2, tree t3)
90 : {
91 1654 : return types_match (t1, t2) && types_match (t2, t3);
92 : }
93 :
94 : /* Return if T has a single use. For GIMPLE, we also allow any
95 : non-SSA_NAME (ie constants) and zero uses to cope with uses
96 : that aren't linked up yet. */
97 :
98 : static bool
99 : single_use (const_tree) ATTRIBUTE_PURE;
100 :
101 : static bool
102 11213101 : single_use (const_tree t)
103 : {
104 11213101 : if (TREE_CODE (t) != SSA_NAME)
105 : return true;
106 :
107 : /* Inline return has_zero_uses (t) || has_single_use (t); */
108 11213101 : const ssa_use_operand_t *const head = &(SSA_NAME_IMM_USE_NODE (t));
109 11213101 : const ssa_use_operand_t *ptr;
110 11213101 : bool single = false;
111 :
112 24621019 : for (ptr = head->next; ptr != head; ptr = ptr->next)
113 19520132 : if (USE_STMT(ptr) && !is_gimple_debug (USE_STMT (ptr)))
114 : {
115 17308837 : if (single)
116 : return false;
117 : single = true;
118 : }
119 : return true;
120 : }
121 :
122 : /* Return true if math operations should be canonicalized,
123 : e.g. sqrt(sqrt(x)) -> pow(x, 0.25). */
124 :
125 : static inline bool
126 16474 : canonicalize_math_p ()
127 : {
128 16474 : return !cfun || (cfun->curr_properties & PROP_gimple_opt_math) == 0;
129 : }
130 :
131 : /* Return true if math operations that are beneficial only after
132 : vectorization should be canonicalized. */
133 :
134 : static inline bool
135 46310 : canonicalize_math_after_vectorization_p ()
136 : {
137 46310 : return !cfun || (cfun->curr_properties & PROP_gimple_lvec) != 0;
138 : }
139 :
140 : /* Return true if we can still perform transformations that may introduce
141 : vector operations that are not supported by the target. Vector lowering
142 : normally handles those, but after that pass, it becomes unsafe. */
143 :
144 : static inline bool
145 4130 : optimize_vectors_before_lowering_p ()
146 : {
147 4130 : return !cfun || (cfun->curr_properties & PROP_gimple_lvec) == 0;
148 : }
149 :
150 : /* Returns true if the expression T has no side effects
151 : including not trapping. */
152 : static inline bool
153 5811 : expr_no_side_effects_p (tree t)
154 : {
155 : /* For gimple, there should only be gimple val's here. */
156 5811 : gcc_assert (is_gimple_val (t));
157 5811 : return true;
158 : }
159 :
160 : /* Return true if pow(cst, x) should be optimized into exp(log(cst) * x).
161 : As a workaround for SPEC CPU2017 628.pop2_s, don't do it if arg0
162 : is an exact integer, arg1 = phi_res +/- cst1 and phi_res = PHI <cst2, ...>
163 : where cst2 +/- cst1 is an exact integer, because then pow (arg0, arg1)
164 : will likely be exact, while exp (log (arg0) * arg1) might be not.
165 : Also don't do it if arg1 is phi_res above and cst2 is an exact integer. */
166 :
167 : static bool
168 4 : optimize_pow_to_exp (tree arg0, tree arg1)
169 : {
170 4 : gcc_assert (TREE_CODE (arg0) == REAL_CST);
171 4 : if (!real_isinteger (TREE_REAL_CST_PTR (arg0), TYPE_MODE (TREE_TYPE (arg0))))
172 : return true;
173 :
174 4 : if (TREE_CODE (arg1) != SSA_NAME)
175 : return true;
176 :
177 4 : gimple *def = SSA_NAME_DEF_STMT (arg1);
178 4 : gphi *phi = dyn_cast <gphi *> (def);
179 4 : tree cst1 = NULL_TREE;
180 4 : enum tree_code code = ERROR_MARK;
181 4 : if (!phi)
182 : {
183 4 : if (!is_gimple_assign (def))
184 : return true;
185 4 : code = gimple_assign_rhs_code (def);
186 4 : switch (code)
187 : {
188 4 : case PLUS_EXPR:
189 4 : case MINUS_EXPR:
190 4 : break;
191 : default:
192 : return true;
193 : }
194 4 : if (TREE_CODE (gimple_assign_rhs1 (def)) != SSA_NAME
195 4 : || TREE_CODE (gimple_assign_rhs2 (def)) != REAL_CST)
196 : return true;
197 :
198 4 : cst1 = gimple_assign_rhs2 (def);
199 :
200 4 : phi = dyn_cast <gphi *> (SSA_NAME_DEF_STMT (gimple_assign_rhs1 (def)));
201 : if (!phi)
202 : return true;
203 : }
204 :
205 4 : tree cst2 = NULL_TREE;
206 4 : int n = gimple_phi_num_args (phi);
207 12 : for (int i = 0; i < n; i++)
208 : {
209 8 : tree arg = PHI_ARG_DEF (phi, i);
210 8 : if (TREE_CODE (arg) != REAL_CST)
211 4 : continue;
212 4 : else if (cst2 == NULL_TREE)
213 : cst2 = arg;
214 0 : else if (!operand_equal_p (cst2, arg, 0))
215 : return true;
216 : }
217 :
218 4 : if (cst1 && cst2)
219 4 : cst2 = const_binop (code, TREE_TYPE (cst2), cst2, cst1);
220 4 : if (cst2
221 4 : && TREE_CODE (cst2) == REAL_CST
222 8 : && real_isinteger (TREE_REAL_CST_PTR (cst2),
223 4 : TYPE_MODE (TREE_TYPE (cst2))))
224 4 : return false;
225 : return true;
226 : }
227 :
228 : /* Return true if a division INNER_DIV / DIVISOR where INNER_DIV
229 : is another division can be optimized. Don't optimize if INNER_DIV
230 : is used in a TRUNC_MOD_EXPR with DIVISOR as second operand. */
231 :
232 : static bool
233 15506 : optimize_successive_divisions_p (tree divisor, tree inner_div)
234 : {
235 28468 : if (!gimple_in_ssa_p (cfun))
236 : return false;
237 :
238 15506 : imm_use_iterator imm_iter;
239 15506 : use_operand_p use_p;
240 49169 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, inner_div)
241 : {
242 31119 : gimple *use_stmt = USE_STMT (use_p);
243 31119 : if (!is_gimple_assign (use_stmt)
244 25251 : || gimple_assign_rhs_code (use_stmt) != TRUNC_MOD_EXPR
245 44491 : || !operand_equal_p (gimple_assign_rhs2 (use_stmt), divisor, 0))
246 18157 : continue;
247 12962 : return false;
248 12962 : }
249 2544 : return true;
250 : }
251 :
252 : /* Return true if EXPR1 and EXPR2 have the same value, but not necessarily
253 : same type. The types can differ through nop conversions. */
254 : #define bitwise_equal_p(expr1, expr2) \
255 : gimple_bitwise_equal_p (expr1, expr2, valueize)
256 :
257 : bool gimple_nop_convert (tree, tree *, tree (*) (tree));
258 : bool gimple_maybe_truncate (tree, tree *, tree (*) (tree));
259 :
260 : /* Helper function for bitwise_equal_p macro. */
261 :
262 : static inline bool
263 2111807 : gimple_bitwise_equal_p (tree expr1, tree expr2, tree (*valueize) (tree))
264 : {
265 2111807 : if (expr1 == expr2)
266 : return true;
267 2108603 : if (!tree_nop_conversion_p (TREE_TYPE (expr1), TREE_TYPE (expr2)))
268 : return false;
269 2091285 : if (TREE_CODE (expr1) == INTEGER_CST && TREE_CODE (expr2) == INTEGER_CST)
270 75 : return wi::to_wide (expr1) == wi::to_wide (expr2);
271 2091210 : if (operand_equal_p (expr1, expr2, 0))
272 : return true;
273 2091210 : tree expr3, expr4;
274 2091210 : if (!gimple_nop_convert (expr1, &expr3, valueize))
275 2079730 : expr3 = expr1;
276 2091210 : if (!gimple_nop_convert (expr2, &expr4, valueize))
277 2080119 : expr4 = expr2;
278 2091210 : if (expr1 != expr3)
279 : {
280 11480 : if (operand_equal_p (expr3, expr2, 0))
281 : return true;
282 11240 : if (expr2 != expr4 && operand_equal_p (expr3, expr4, 0))
283 : return true;
284 : }
285 2090956 : if (expr2 != expr4 && operand_equal_p (expr1, expr4, 0))
286 : return true;
287 2090430 : if (gimple_maybe_truncate (expr3, &expr3, valueize)
288 25692 : && gimple_maybe_truncate (expr4, &expr4, valueize)
289 2092242 : && operand_equal_p (expr3, expr4, 0))
290 : return true;
291 : return false;
292 : }
293 :
294 : /* Return true if EXPR1 and EXPR2 have the bitwise opposite value,
295 : but not necessarily same type.
296 : The types can differ through nop conversions. */
297 : #define bitwise_inverted_equal_p(expr1, expr2, wascmp) \
298 : gimple_bitwise_inverted_equal_p (expr1, expr2, wascmp, valueize)
299 :
300 :
301 : bool gimple_bit_not_with_nop (tree, tree *, tree (*) (tree));
302 : bool gimple_maybe_cmp (tree, tree *, tree (*) (tree));
303 : bool gimple_bit_xor_cst (tree, tree *, tree (*) (tree));
304 :
305 : /* Helper function for bitwise_inverted_equal_p macro. */
306 :
307 : static inline bool
308 173054579 : gimple_bitwise_inverted_equal_p (tree expr1, tree expr2, bool &wascmp, tree (*valueize) (tree))
309 : {
310 173054579 : wascmp = false;
311 173054579 : if (expr1 == expr2)
312 : return false;
313 173013584 : if (!tree_nop_conversion_p (TREE_TYPE (expr1), TREE_TYPE (expr2)))
314 : return false;
315 173013584 : tree cst1 = uniform_integer_cst_p (expr1);
316 173013584 : tree cst2 = uniform_integer_cst_p (expr2);
317 173013584 : if (cst1 && cst2)
318 302408 : return wi::to_wide (cst1) == ~wi::to_wide (cst2);
319 172711180 : if (operand_equal_p (expr1, expr2, 0))
320 : return false;
321 :
322 172711180 : tree xor1[2];
323 172711180 : tree xor2[2];
324 : /* `X ^ CST` and `X ^ ~CST` match for ~. */
325 172711180 : if (gimple_bit_xor_cst (expr1, xor1, valueize)
326 172711180 : && gimple_bit_xor_cst (expr2, xor2, valueize))
327 : {
328 363 : if (operand_equal_p (xor1[0], xor2[0], 0)
329 433 : && (wi::to_wide (uniform_integer_cst_p (xor1[1]))
330 433 : == ~wi::to_wide (uniform_integer_cst_p (xor2[1]))))
331 : return true;
332 : }
333 :
334 172711179 : tree other;
335 : /* Try if EXPR1 was defined as ~EXPR2. */
336 172711179 : if (gimple_bit_not_with_nop (expr1, &other, valueize))
337 : {
338 731448 : if (gimple_bitwise_equal_p (other, expr2, valueize))
339 : return true;
340 : }
341 : /* Try if EXPR2 was defined as ~EXPR1. */
342 172710247 : if (gimple_bit_not_with_nop (expr2, &other, valueize))
343 : {
344 615605 : if (gimple_bitwise_equal_p (other, expr1, valueize))
345 : return true;
346 : }
347 :
348 : /* If neither are defined by BIT_NOT, try to see if
349 : both are defined by comparisons and see if they are
350 : complementary (inversion) of each other. */
351 172708865 : tree newexpr1, newexpr2;
352 172708865 : if (!gimple_maybe_cmp (expr1, &newexpr1, valueize))
353 : return false;
354 6207786 : if (!gimple_maybe_cmp (expr2, &newexpr2, valueize))
355 : return false;
356 :
357 5560821 : gimple *d1 = get_def (valueize, newexpr1);
358 5560821 : gassign *a1 = dyn_cast <gassign *> (d1);
359 5560821 : gimple *d2 = get_def (valueize, newexpr2);
360 5560821 : gassign *a2 = dyn_cast <gassign *> (d2);
361 5560821 : tree op10 = do_valueize (valueize, gimple_assign_rhs1 (a1));
362 5560821 : tree op20 = do_valueize (valueize, gimple_assign_rhs1 (a2));
363 5560821 : if (!operand_equal_p (op10, op20))
364 : return false;
365 3217050 : tree op11 = do_valueize (valueize, gimple_assign_rhs2 (a1));
366 3217050 : tree op21 = do_valueize (valueize, gimple_assign_rhs2 (a2));
367 1608525 : if (!operand_equal_p (op11, op21))
368 : return false;
369 41421 : wascmp = true;
370 41421 : tree_code ac1 = gimple_assign_rhs_code (a1);
371 41421 : tree_code ac2 = gimple_assign_rhs_code (a2);
372 : /* Match `^` against `==` but this should only
373 : happen when the type is a 1bit precision integer. */
374 41421 : if (ac1 == BIT_XOR_EXPR)
375 : {
376 41 : tree type = TREE_TYPE (newexpr1);
377 41 : gcc_assert (INTEGRAL_TYPE_P (type) && TYPE_PRECISION (type) == 1);
378 41 : return ac2 == EQ_EXPR;
379 : }
380 41380 : if (ac2 == BIT_XOR_EXPR)
381 : {
382 16 : tree type = TREE_TYPE (newexpr1);
383 16 : gcc_assert (INTEGRAL_TYPE_P (type) && TYPE_PRECISION (type) == 1);
384 16 : return ac1 == EQ_EXPR;
385 : }
386 41364 : if (invert_tree_comparison (ac1, HONOR_NANS (op10)) == ac2)
387 : return true;
388 : return false;
389 : }
390 :
391 : /*
392 : * Return the relevant gcond * of the given phi, as well as the true
393 : * and false TREE args of the phi. Or return nullptr.
394 : *
395 : * If matched the gcond *, the output argument TREE true_arg and false_arg
396 : * will be updated to the relevant args of phi.
397 : *
398 : * If failed to match, nullptr gcond * will be returned, as well as the output
399 : * arguments will be set to NULL_TREE.
400 : */
401 :
402 : static inline gcond *
403 22001743 : match_cond_with_binary_phi (gphi *phi, tree *true_arg, tree *false_arg)
404 : {
405 22001743 : *true_arg = *false_arg = NULL_TREE;
406 :
407 22001743 : if (gimple_phi_num_args (phi) != 2)
408 : return nullptr;
409 :
410 21816463 : basic_block pred_b0 = EDGE_PRED (gimple_bb (phi), 0)->src;
411 21816463 : basic_block pred_b1 = EDGE_PRED (gimple_bb (phi), 1)->src;
412 21816463 : edge edge_for_pred_0 = nullptr;
413 :
414 21816463 : if (EDGE_COUNT (pred_b0->succs) == 2
415 13507507 : && EDGE_COUNT (pred_b1->succs) == 1
416 8450019 : && EDGE_COUNT (pred_b1->preds) == 1
417 28588980 : && pred_b0 == EDGE_PRED (pred_b1, 0)->src)
418 : /*
419 : * +------+
420 : * | b0: |
421 : * | def | +-----+
422 : * | ... | | b1: |
423 : * | cond |------>| def |
424 : * +------+ | ... |
425 : * | +-----+
426 : * # |
427 : * | |
428 : * v |
429 : * +-----+ |
430 : * | b2: | |
431 : * | def |<----------+
432 : * +-----+
433 : * #: edge_for_pred_0.
434 : */
435 : edge_for_pred_0 = EDGE_PRED (gimple_bb (phi), 0);
436 19504552 : else if (EDGE_COUNT (pred_b1->succs) == 2
437 8049324 : && EDGE_COUNT (pred_b0->succs) == 1
438 2999090 : && EDGE_COUNT (pred_b0->preds) == 1
439 21860404 : && pred_b1 == EDGE_PRED (pred_b0, 0)->src)
440 : /*
441 : * +------+
442 : * | b1: |
443 : * +-----+ | def |
444 : * | b0: | | ... |
445 : * | def |<---#---| cond |
446 : * | ... | +------+
447 : * +-----+ |
448 : * | |
449 : * | |
450 : * | |
451 : * v |
452 : * +-----+ |
453 : * | b2: | |
454 : * | def |<----------+
455 : * +-----+
456 : * #: edge_for_pred_0.
457 : */
458 : edge_for_pred_0 = EDGE_PRED (pred_b0, 0);
459 18797508 : else if (EDGE_COUNT (pred_b0->succs) == 1
460 7548592 : && EDGE_COUNT (pred_b1->succs) == 1
461 5253656 : && EDGE_COUNT (pred_b0->preds) == 1
462 4266730 : && EDGE_COUNT (pred_b1->preds) == 1
463 3167024 : && EDGE_COUNT (EDGE_PRED (pred_b0, 0)->src->succs) == 2
464 21823144 : && EDGE_PRED (pred_b0, 0)->src == EDGE_PRED (pred_b1, 0)->src)
465 : /* +------+
466 : * | b0: |
467 : * | ... | +-----+
468 : * | cond |------>| b2: |
469 : * +------+ | ... |
470 : * | +-----+
471 : * # |
472 : * | |
473 : * v |
474 : * +-----+ |
475 : * | b1: | |
476 : * | ... | |
477 : * +-----+ |
478 : * | |
479 : * | |
480 : * v |
481 : * +-----+ |
482 : * | b3: |<----------+
483 : * | ... |
484 : * +-----+
485 : * #: edge_for_pred_0.
486 : */
487 : edge_for_pred_0 = EDGE_PRED (pred_b0, 0);
488 :
489 4502113 : if (!edge_for_pred_0)
490 : return nullptr;
491 :
492 9004226 : gcond *cond = safe_dyn_cast <gcond *> (*gsi_last_bb (edge_for_pred_0->src));
493 :
494 4493503 : if (!cond)
495 : return nullptr;
496 :
497 4493503 : if (edge_for_pred_0->flags & EDGE_TRUE_VALUE)
498 : {
499 2556718 : *true_arg = gimple_phi_arg_def (phi, 0);
500 2556718 : *false_arg = gimple_phi_arg_def (phi, 1);
501 : }
502 : else /* Aka edge_for_pred_0->flags & EDGE_FALSE_VALUE */
503 : {
504 1936785 : *false_arg = gimple_phi_arg_def (phi, 0);
505 1936785 : *true_arg = gimple_phi_arg_def (phi, 1);
506 : }
507 :
508 : return cond;
509 : }
510 :
511 : /* If OP is a SSA_NAME with SSA_NAME_DEF_STMT in the IL, return that
512 : stmt, otherwise NULL. For use in range_of_expr calls. */
513 :
514 : static inline gimple *
515 1042051 : gimple_match_ctx (tree op)
516 : {
517 1042051 : if (TREE_CODE (op) == SSA_NAME
518 1030881 : && SSA_NAME_DEF_STMT (op)
519 2072932 : && gimple_bb (SSA_NAME_DEF_STMT (op)))
520 : return SSA_NAME_DEF_STMT (op);
521 : return NULL;
522 : }
523 :
524 : /* Helper to shorten range queries in match.pd. R is the range to
525 : be queried, OP tree on which it should be queried and CTX is some
526 : capture on which gimple_match_ctx should be called, or NULL for
527 : global range. */
528 :
529 : static inline bool
530 996787 : gimple_match_range_of_expr (vrange &r, tree op, tree ctx = NULL_TREE)
531 : {
532 2990361 : if (!get_range_query (cfun)->range_of_expr (r, op,
533 996787 : ctx ? gimple_match_ctx (ctx)
534 : : NULL))
535 : return false;
536 996787 : return !r.undefined_p ();
537 : }
|
