Line data Source code
1 : /* Schedule GIMPLE vector statements.
2 : Copyright (C) 2020-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by the
8 : Free Software Foundation; either version 3, or (at your option) any
9 : later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "rtl.h"
25 : #include "tree.h"
26 : #include "gimple.h"
27 : #include "tree-pass.h"
28 : #include "ssa.h"
29 : #include "expmed.h"
30 : #include "optabs-tree.h"
31 : #include "tree-eh.h"
32 : #include "gimple-iterator.h"
33 : #include "gimplify-me.h"
34 : #include "gimplify.h"
35 : #include "tree-cfg.h"
36 : #include "bitmap.h"
37 : #include "tree-ssa-dce.h"
38 : #include "memmodel.h"
39 : #include "optabs.h"
40 : #include "gimple-fold.h"
41 : #include "internal-fn.h"
42 : #include "fold-const.h"
43 : #include "gimple-range.h"
44 :
45 :
46 : /* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to
47 : internal function based on vector type of selected expansion.
48 :
49 : For vec_set:
50 :
51 : VIEW_CONVERT_EXPR<int[4]>(u)[_1] = i_4(D);
52 : =>
53 : _7 = u;
54 : _8 = .VEC_SET (_7, i_4(D), _1);
55 : u = _8;
56 :
57 : For vec_extract:
58 :
59 : _3 = VIEW_CONVERT_EXPR<intD.1[4]>(vD.2208)[idx_2(D)];
60 : =>
61 : _4 = vD.2208;
62 : _3 = .VEC_EXTRACT (_4, idx_2(D)); */
63 :
64 : static bool
65 102495153 : gimple_expand_vec_set_extract_expr (struct function *fun,
66 : gimple_stmt_iterator *gsi)
67 : {
68 102495153 : gcall *new_stmt = NULL;
69 102495153 : gassign *ass_stmt = NULL;
70 102495153 : bool cfg_changed = false;
71 :
72 : /* Only consider code == GIMPLE_ASSIGN. */
73 102495153 : gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi));
74 32757988 : if (!stmt)
75 : return false;
76 :
77 32757988 : bool is_extract = false;
78 :
79 32757988 : tree lhs = gimple_assign_lhs (stmt);
80 32757988 : tree rhs = gimple_assign_rhs1 (stmt);
81 32757988 : tree val, ref;
82 32757988 : if (TREE_CODE (lhs) == ARRAY_REF)
83 : {
84 : /* Assume it is a vec_set. */
85 : val = rhs;
86 : ref = lhs;
87 : }
88 32124818 : else if (TREE_CODE (rhs) == ARRAY_REF)
89 : {
90 : /* vec_extract. */
91 : is_extract = true;
92 : val = lhs;
93 : ref = rhs;
94 : }
95 : else
96 : return false;
97 :
98 1212472 : tree op0 = TREE_OPERAND (ref, 0);
99 1212472 : if (TREE_CODE (op0) == VIEW_CONVERT_EXPR
100 33069 : && DECL_P (TREE_OPERAND (op0, 0))
101 27731 : && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0)))
102 1232600 : && (TYPE_MODE (TREE_TYPE (ref))
103 10064 : == TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0))))))
104 : {
105 10060 : tree pos = TREE_OPERAND (ref, 1);
106 :
107 10060 : tree view_op0 = TREE_OPERAND (op0, 0);
108 :
109 10060 : tree idx = TREE_OPERAND (ref, 1);
110 : // if index is a constant, then check the bounds
111 10060 : poly_uint64 idx_poly;
112 10060 : if (poly_int_tree_p (idx, &idx_poly))
113 : {
114 33 : poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (view_op0));
115 33 : if (known_ge (idx_poly, nelts))
116 33 : return false;
117 : }
118 10027 : else if (poly_int_tree_p (idx))
119 : // if idx doesn't fit into poly_uint64, but is constant, it
120 : // must be out of bounds
121 : return false;
122 10020 : machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0));
123 10020 : machine_mode extract_mode = TYPE_MODE (TREE_TYPE (ref));
124 :
125 10020 : if ((auto_var_in_fn_p (view_op0, fun->decl)
126 1311 : || (VAR_P (view_op0) && DECL_HARD_REGISTER (view_op0)))
127 8719 : && !TREE_ADDRESSABLE (view_op0)
128 17010 : && ((!is_extract && can_vec_set_var_idx_p (outermode))
129 : || (is_extract
130 6783 : && can_vec_extract_var_idx_p (outermode, extract_mode))))
131 : {
132 123 : location_t loc = gimple_location (stmt);
133 123 : tree var_src = make_ssa_name (TREE_TYPE (view_op0));
134 :
135 123 : ass_stmt = gimple_build_assign (var_src, view_op0);
136 246 : gimple_set_vuse (ass_stmt, gimple_vuse (stmt));
137 123 : gimple_set_location (ass_stmt, loc);
138 123 : gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
139 :
140 123 : if (!is_extract)
141 : {
142 123 : tree var_dst = make_ssa_name (TREE_TYPE (view_op0));
143 :
144 123 : new_stmt = gimple_build_call_internal (IFN_VEC_SET, 3, var_src,
145 : val, pos);
146 :
147 123 : gimple_call_set_lhs (new_stmt, var_dst);
148 123 : gimple_set_location (new_stmt, loc);
149 123 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
150 :
151 123 : ass_stmt = gimple_build_assign (view_op0, var_dst);
152 123 : gimple_set_location (ass_stmt, loc);
153 123 : gimple_move_vops (ass_stmt, stmt);
154 123 : gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
155 :
156 123 : basic_block bb = gimple_bb (stmt);
157 123 : if (gsi_remove (gsi, true)
158 123 : && gimple_purge_dead_eh_edges (bb))
159 : cfg_changed = true;
160 123 : *gsi = gsi_for_stmt (ass_stmt);
161 : }
162 : else
163 : {
164 0 : new_stmt
165 0 : = gimple_build_call_internal (IFN_VEC_EXTRACT, 2, var_src, pos);
166 0 : gimple_call_set_lhs (new_stmt, lhs);
167 :
168 0 : gsi_replace (gsi, new_stmt, true);
169 0 : cfg_changed = true;
170 : }
171 : }
172 : }
173 :
174 : return cfg_changed;
175 : }
176 :
177 : /* Expand all VEC_COND_EXPR gimple assignments into calls to internal
178 : function based on type of selected expansion. */
179 :
180 : static gimple *
181 102495153 : gimple_expand_vec_cond_expr (gimple_stmt_iterator *gsi)
182 : {
183 102495153 : tree lhs, op0a = NULL_TREE;
184 102495153 : enum tree_code code;
185 102495153 : enum tree_code tcode;
186 :
187 : /* Only consider code == GIMPLE_ASSIGN. */
188 102495153 : gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi));
189 32767649 : if (!stmt)
190 : return NULL;
191 :
192 32767649 : code = gimple_assign_rhs_code (stmt);
193 32767649 : if (code != VEC_COND_EXPR)
194 : return NULL;
195 :
196 22602 : tree op0 = gimple_assign_rhs1 (stmt);
197 22602 : tree op1 = gimple_assign_rhs2 (stmt);
198 22602 : tree op2 = gimple_assign_rhs3 (stmt);
199 22602 : lhs = gimple_assign_lhs (stmt);
200 22602 : machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
201 :
202 : /* Lower mask typed, non-vector mode VEC_COND_EXPRs to bitwise operations.
203 : Those can end up generated by folding and at least for integer mode masks
204 : we cannot expect vcond expanders to exist. We lower a ? b : c
205 : to (b & a) | (c & ~a). */
206 45204 : if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (lhs))
207 22604 : && !VECTOR_MODE_P (mode))
208 : {
209 0 : gcc_assert (types_compatible_p (TREE_TYPE (op0), TREE_TYPE (op1)));
210 0 : gimple_seq stmts = NULL;
211 0 : tree type = TREE_TYPE (lhs);
212 0 : location_t loc = gimple_location (stmt);
213 0 : tree tem0 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op1, op0);
214 0 : tree tem1 = gimple_build (&stmts, loc, BIT_NOT_EXPR, type, op0);
215 0 : tree tem2 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op2, tem1);
216 0 : tree tem3 = gimple_build (&stmts, loc, BIT_IOR_EXPR, type, tem0, tem2);
217 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
218 0 : return gimple_build_assign (lhs, tem3);
219 : }
220 :
221 22602 : bool can_compute_op0 = true;
222 22602 : gcc_assert (!COMPARISON_CLASS_P (op0));
223 22602 : if (TREE_CODE (op0) == SSA_NAME)
224 : {
225 21266 : gassign *def_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op0));
226 21255 : if (def_stmt)
227 : {
228 21255 : tcode = gimple_assign_rhs_code (def_stmt);
229 21255 : op0a = gimple_assign_rhs1 (def_stmt);
230 :
231 21255 : tree op0_type = TREE_TYPE (op0);
232 21255 : tree op0a_type = TREE_TYPE (op0a);
233 21255 : if (TREE_CODE_CLASS (tcode) == tcc_comparison)
234 16417 : can_compute_op0 = expand_vec_cmp_expr_p (op0a_type, op0_type,
235 : tcode);
236 16417 : gcc_assert (can_compute_op0);
237 :
238 21255 : if (can_compute_op0
239 21255 : && TYPE_MODE (TREE_TYPE (lhs)) == TYPE_MODE (TREE_TYPE (op0)))
240 : {
241 : /* Assuming c = x CMP y. */
242 16264 : bool op1_minus_onep = integer_minus_onep (op1);
243 16264 : bool op2_zerop = integer_zerop (op2);
244 16264 : tree vtype = TREE_TYPE (lhs);
245 16264 : machine_mode vmode = TYPE_MODE (vtype);
246 : /* Try to fold r = c ? -1 : 0 to r = c. */
247 16264 : if (op1_minus_onep && op2_zerop)
248 : {
249 5980 : tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
250 5980 : return gimple_build_assign (lhs, conv_op);
251 : }
252 : /* Try to fold r = c ? -1 : z to r = c | z, or
253 : r = c ? c : z. */
254 10284 : if (op1_minus_onep)
255 : {
256 30 : tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
257 30 : tree new_op1 = make_ssa_name (vtype);
258 30 : gassign *new_stmt = gimple_build_assign (new_op1, conv_op);
259 30 : gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT);
260 30 : if (optab_handler (ior_optab, vmode) != CODE_FOR_nothing)
261 : /* r = c | z */
262 30 : return gimple_build_assign (lhs, BIT_IOR_EXPR, new_op1,
263 30 : op2);
264 : /* r = c ? c : z */
265 : op1 = new_op1;
266 : }
267 : /* Try to fold r = c ? z : 0 to r = c & z, or
268 : r = c ? z : c. */
269 10254 : else if (op2_zerop)
270 : {
271 6931 : tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
272 6931 : tree new_op2 = make_ssa_name (vtype);
273 6931 : gassign *new_stmt = gimple_build_assign (new_op2, conv_op);
274 6931 : gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT);
275 6931 : if (optab_handler (and_optab, vmode) != CODE_FOR_nothing)
276 : /* r = c | z */
277 6931 : return gimple_build_assign (lhs, BIT_AND_EXPR, new_op2,
278 6931 : op1);
279 : /* r = c ? z : c */
280 : op2 = new_op2;
281 : }
282 3323 : bool op1_zerop = integer_zerop (op1);
283 3323 : bool op2_minus_onep = integer_minus_onep (op2);
284 : /* Try to fold r = c ? 0 : z to r = .BIT_ANDN (z, c). */
285 3323 : if (op1_zerop
286 3323 : && (direct_internal_fn_supported_p (IFN_BIT_ANDN, vtype,
287 : OPTIMIZE_FOR_BOTH)))
288 : {
289 559 : tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
290 559 : tree new_op = make_ssa_name (vtype);
291 559 : gassign *new_stmt = gimple_build_assign (new_op, conv_op);
292 559 : gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT);
293 559 : return gimple_build_call_internal (IFN_BIT_ANDN, 2, op2,
294 559 : new_op);
295 : }
296 : /* Try to fold r = c ? z : -1 to r = .BIT_IORN (z, c). */
297 2764 : else if (op2_minus_onep
298 2764 : && (direct_internal_fn_supported_p (IFN_BIT_IORN, vtype,
299 : OPTIMIZE_FOR_BOTH)))
300 : {
301 0 : tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
302 0 : tree new_op = make_ssa_name (vtype);
303 0 : gassign *new_stmt = gimple_build_assign (new_op, conv_op);
304 0 : gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT);
305 0 : return gimple_build_call_internal (IFN_BIT_IORN, 2, op1,
306 0 : new_op);
307 : }
308 : }
309 : }
310 : }
311 :
312 9102 : gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
313 18204 : gcc_assert (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
314 : != CODE_FOR_nothing);
315 9102 : return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
316 : }
317 :
318 : /* Duplicate COND_EXPR condition defs of STMT located in BB when they are
319 : comparisons so RTL expansion with the help of TER
320 : can perform better if conversion. */
321 : static void
322 589297 : maybe_duplicate_comparison (gassign *stmt, basic_block bb)
323 : {
324 589297 : imm_use_iterator imm_iter;
325 589297 : use_operand_p use_p;
326 589297 : auto_vec<gassign *, 4> cond_exprs;
327 589297 : tree lhs = gimple_assign_lhs (stmt);
328 589297 : unsigned cnt = 0;
329 :
330 : /* This is should not be used for -O0 nor it is not useful
331 : when ter is turned off. */
332 589297 : if (!optimize || !flag_tree_ter)
333 : return;
334 :
335 955815 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
336 : {
337 493536 : if (is_gimple_debug (USE_STMT (use_p)))
338 13896 : continue;
339 479640 : cnt++;
340 : /* Add the use statement if it was a cond_expr. */
341 479640 : if (gimple_bb (USE_STMT (use_p)) == bb
342 432068 : && is_gimple_assign (USE_STMT (use_p))
343 411553 : && gimple_assign_rhs_code (USE_STMT (use_p)) == COND_EXPR
344 495491 : && gimple_assign_rhs1_ptr (USE_STMT (use_p)) == use_p->use)
345 15441 : cond_exprs.safe_push (as_a <gassign *> (USE_STMT (use_p)));
346 462279 : }
347 :
348 : /* If the comparison has 0 or 1 uses, no reason to do anything. */
349 462279 : if (cnt <= 1)
350 : return;
351 :
352 : /* If we only use the expression inside cond_exprs in that BB, we don't
353 : need to duplicate for one of them so pop the top. */
354 23120 : if (cond_exprs.length () == cnt)
355 203 : cond_exprs.pop();
356 :
357 23604 : while (!cond_exprs.is_empty())
358 : {
359 484 : auto old_top = cond_exprs.pop();
360 484 : gassign *copy = as_a <gassign *> (gimple_copy (stmt));
361 484 : tree new_def = duplicate_ssa_name (lhs, copy);
362 484 : gimple_assign_set_lhs (copy, new_def);
363 484 : auto gsi2 = gsi_for_stmt (old_top);
364 484 : gsi_insert_before (&gsi2, copy, GSI_SAME_STMT);
365 484 : gimple_assign_set_rhs1 (old_top, new_def);
366 484 : update_stmt (old_top);
367 : }
368 589297 : }
369 :
370 : /* match.pd function to match atomic_bit_test_and pattern which
371 : has nop_convert:
372 : _1 = __atomic_fetch_or_4 (&v, 1, 0);
373 : _2 = (int) _1;
374 : _5 = _2 & 1;
375 : */
376 : extern bool gimple_nop_atomic_bit_test_and_p (tree, tree *,
377 : tree (*) (tree));
378 : extern bool gimple_nop_convert (tree, tree*, tree (*) (tree));
379 :
380 : namespace {
381 :
382 : const pass_data pass_data_gimple_isel =
383 : {
384 : GIMPLE_PASS, /* type */
385 : "isel", /* name */
386 : OPTGROUP_VEC, /* optinfo_flags */
387 : TV_NONE, /* tv_id */
388 : PROP_cfg, /* properties_required */
389 : 0, /* properties_provided */
390 : 0, /* properties_destroyed */
391 : 0, /* todo_flags_start */
392 : TODO_update_ssa, /* todo_flags_finish */
393 : };
394 :
395 : class pass_gimple_isel : public gimple_opt_pass
396 : {
397 : public:
398 294196 : pass_gimple_isel (gcc::context *ctxt)
399 588392 : : gimple_opt_pass (pass_data_gimple_isel, ctxt)
400 : {}
401 :
402 : /* opt_pass methods: */
403 1515908 : bool gate (function *) final override
404 : {
405 1515908 : return true;
406 : }
407 :
408 : unsigned int execute (function *fun) final override;
409 : }; // class pass_gimple_isel
410 :
411 :
412 :
413 : /* Convert
414 : _1 = __atomic_fetch_or_* (ptr_6, 1, _3);
415 : _7 = ~_1;
416 : _5 = (_Bool) _7;
417 : to
418 : _1 = __atomic_fetch_or_* (ptr_6, 1, _3);
419 : _8 = _1 & 1;
420 : _5 = _8 == 0;
421 : and convert
422 : _1 = __atomic_fetch_and_* (ptr_6, ~1, _3);
423 : _7 = ~_1;
424 : _4 = (_Bool) _7;
425 : to
426 : _1 = __atomic_fetch_and_* (ptr_6, ~1, _3);
427 : _8 = _1 & 1;
428 : _4 = (_Bool) _8;
429 :
430 : USE_STMT is the gimplt statement which uses the return value of
431 : __atomic_fetch_or_*. LHS is the return value of __atomic_fetch_or_*.
432 : MASK is the mask passed to __atomic_fetch_or_*.
433 : */
434 :
435 : static gimple *
436 14 : convert_atomic_bit_not (enum internal_fn fn, gimple *use_stmt,
437 : tree lhs, tree mask)
438 : {
439 14 : tree and_mask;
440 14 : if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
441 : {
442 : /* MASK must be ~1. */
443 8 : if (!operand_equal_p (build_int_cst (TREE_TYPE (lhs),
444 : ~HOST_WIDE_INT_1), mask, 0))
445 : return nullptr;
446 8 : and_mask = build_int_cst (TREE_TYPE (lhs), 1);
447 : }
448 : else
449 : {
450 : /* MASK must be 1. */
451 6 : if (!operand_equal_p (build_int_cst (TREE_TYPE (lhs), 1), mask, 0))
452 : return nullptr;
453 : and_mask = mask;
454 : }
455 :
456 14 : tree use_lhs = gimple_assign_lhs (use_stmt);
457 :
458 14 : use_operand_p use_p;
459 14 : gimple *use_not_stmt;
460 :
461 14 : if (!single_imm_use (use_lhs, &use_p, &use_not_stmt)
462 14 : || !is_gimple_assign (use_not_stmt))
463 : return nullptr;
464 :
465 14 : if (!CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (use_not_stmt)))
466 : return nullptr;
467 :
468 14 : tree use_not_lhs = gimple_assign_lhs (use_not_stmt);
469 14 : if (TREE_CODE (TREE_TYPE (use_not_lhs)) != BOOLEAN_TYPE)
470 : return nullptr;
471 :
472 14 : gimple_stmt_iterator gsi;
473 14 : tree var = make_ssa_name (TREE_TYPE (lhs));
474 : /* use_stmt need to be removed after use_nop_stmt,
475 : so use_lhs can be released. */
476 14 : gimple *use_stmt_removal = use_stmt;
477 14 : use_stmt = gimple_build_assign (var, BIT_AND_EXPR, lhs, and_mask);
478 14 : gsi = gsi_for_stmt (use_not_stmt);
479 14 : gsi_insert_before (&gsi, use_stmt, GSI_NEW_STMT);
480 14 : lhs = gimple_assign_lhs (use_not_stmt);
481 14 : gimple *g = gimple_build_assign (lhs, EQ_EXPR, var,
482 14 : build_zero_cst (TREE_TYPE (mask)));
483 14 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
484 14 : gsi = gsi_for_stmt (use_not_stmt);
485 14 : gsi_remove (&gsi, true);
486 14 : gsi = gsi_for_stmt (use_stmt_removal);
487 14 : gsi_remove (&gsi, true);
488 14 : return use_stmt;
489 : }
490 :
491 : /* Optimize
492 : mask_2 = 1 << cnt_1;
493 : _4 = __atomic_fetch_or_* (ptr_6, mask_2, _3);
494 : _5 = _4 & mask_2;
495 : to
496 : _4 = .ATOMIC_BIT_TEST_AND_SET (ptr_6, cnt_1, 0, _3);
497 : _5 = _4;
498 : If _5 is only used in _5 != 0 or _5 == 0 comparisons, 1
499 : is passed instead of 0, and the builtin just returns a zero
500 : or 1 value instead of the actual bit.
501 : Similarly for __sync_fetch_and_or_* (without the ", _3" part
502 : in there), and/or if mask_2 is a power of 2 constant.
503 : Similarly for xor instead of or, use ATOMIC_BIT_TEST_AND_COMPLEMENT
504 : in that case. And similarly for and instead of or, except that
505 : the second argument to the builtin needs to be one's complement
506 : of the mask instead of mask. */
507 :
508 : static bool
509 4651 : optimize_atomic_bit_test_and (gimple_stmt_iterator *gsip,
510 : enum internal_fn fn, bool has_model_arg,
511 : bool after)
512 : {
513 4651 : gimple *call = gsi_stmt (*gsip);
514 4651 : tree lhs = gimple_call_lhs (call);
515 4651 : use_operand_p use_p;
516 4651 : gimple *use_stmt;
517 4651 : tree mask;
518 4651 : optab optab;
519 :
520 4651 : if (!flag_inline_atomics
521 4651 : || optimize_debug
522 4651 : || !gimple_call_builtin_p (call, BUILT_IN_NORMAL)
523 4651 : || !lhs
524 3012 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs)
525 3012 : || !single_imm_use (lhs, &use_p, &use_stmt)
526 2982 : || !is_gimple_assign (use_stmt)
527 6411 : || !gimple_vdef (call))
528 : return false;
529 :
530 1760 : switch (fn)
531 : {
532 : case IFN_ATOMIC_BIT_TEST_AND_SET:
533 : optab = atomic_bit_test_and_set_optab;
534 : break;
535 : case IFN_ATOMIC_BIT_TEST_AND_COMPLEMENT:
536 : optab = atomic_bit_test_and_complement_optab;
537 : break;
538 : case IFN_ATOMIC_BIT_TEST_AND_RESET:
539 : optab = atomic_bit_test_and_reset_optab;
540 : break;
541 : default:
542 : return false;
543 : }
544 :
545 1760 : tree bit = nullptr;
546 :
547 1760 : mask = gimple_call_arg (call, 1);
548 1760 : tree_code rhs_code = gimple_assign_rhs_code (use_stmt);
549 1760 : if (rhs_code != BIT_AND_EXPR)
550 : {
551 1468 : if (rhs_code != NOP_EXPR && rhs_code != BIT_NOT_EXPR)
552 1311 : return false;
553 :
554 897 : tree use_lhs = gimple_assign_lhs (use_stmt);
555 897 : if (TREE_CODE (use_lhs) == SSA_NAME
556 897 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs))
557 : return false;
558 :
559 897 : tree use_rhs = gimple_assign_rhs1 (use_stmt);
560 897 : if (lhs != use_rhs)
561 : return false;
562 :
563 897 : if (optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)))
564 : == CODE_FOR_nothing)
565 : return false;
566 :
567 605 : gimple *g;
568 605 : gimple_stmt_iterator gsi;
569 605 : tree var;
570 605 : int ibit = -1;
571 :
572 605 : if (rhs_code == BIT_NOT_EXPR)
573 : {
574 14 : g = convert_atomic_bit_not (fn, use_stmt, lhs, mask);
575 14 : if (!g)
576 : return false;
577 14 : use_stmt = g;
578 14 : ibit = 0;
579 : }
580 591 : else if (TREE_CODE (TREE_TYPE (use_lhs)) == BOOLEAN_TYPE)
581 : {
582 15 : tree and_mask;
583 15 : if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
584 : {
585 : /* MASK must be ~1. */
586 8 : if (!operand_equal_p (build_int_cst (TREE_TYPE (lhs),
587 : ~HOST_WIDE_INT_1),
588 : mask, 0))
589 : return false;
590 :
591 : /* Convert
592 : _1 = __atomic_fetch_and_* (ptr_6, ~1, _3);
593 : _4 = (_Bool) _1;
594 : to
595 : _1 = __atomic_fetch_and_* (ptr_6, ~1, _3);
596 : _5 = _1 & 1;
597 : _4 = (_Bool) _5;
598 : */
599 8 : and_mask = build_int_cst (TREE_TYPE (lhs), 1);
600 : }
601 : else
602 : {
603 7 : and_mask = build_int_cst (TREE_TYPE (lhs), 1);
604 7 : if (!operand_equal_p (and_mask, mask, 0))
605 : return false;
606 :
607 : /* Convert
608 : _1 = __atomic_fetch_or_* (ptr_6, 1, _3);
609 : _4 = (_Bool) _1;
610 : to
611 : _1 = __atomic_fetch_or_* (ptr_6, 1, _3);
612 : _5 = _1 & 1;
613 : _4 = (_Bool) _5;
614 : */
615 : }
616 15 : var = make_ssa_name (TREE_TYPE (use_rhs));
617 15 : replace_uses_by (use_rhs, var);
618 15 : g = gimple_build_assign (var, BIT_AND_EXPR, use_rhs,
619 : and_mask);
620 15 : gsi = gsi_for_stmt (use_stmt);
621 15 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
622 15 : use_stmt = g;
623 15 : ibit = 0;
624 : }
625 576 : else if (TYPE_PRECISION (TREE_TYPE (use_lhs))
626 576 : <= TYPE_PRECISION (TREE_TYPE (use_rhs)))
627 : {
628 574 : gimple *use_nop_stmt;
629 574 : if (!single_imm_use (use_lhs, &use_p, &use_nop_stmt)
630 574 : || (!is_gimple_assign (use_nop_stmt)
631 93 : && gimple_code (use_nop_stmt) != GIMPLE_COND))
632 446 : return false;
633 : /* Handle both
634 : _4 = _5 < 0;
635 : and
636 : if (_5 < 0)
637 : */
638 490 : tree use_nop_lhs = nullptr;
639 490 : rhs_code = ERROR_MARK;
640 490 : if (is_gimple_assign (use_nop_stmt))
641 : {
642 481 : use_nop_lhs = gimple_assign_lhs (use_nop_stmt);
643 481 : rhs_code = gimple_assign_rhs_code (use_nop_stmt);
644 : }
645 490 : if (!use_nop_lhs || rhs_code != BIT_AND_EXPR)
646 : {
647 : /* Also handle
648 : if (_5 < 0)
649 : */
650 396 : if (use_nop_lhs
651 387 : && TREE_CODE (use_nop_lhs) == SSA_NAME
652 447 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_nop_lhs))
653 : return false;
654 396 : if (use_nop_lhs && rhs_code == BIT_NOT_EXPR)
655 : {
656 : /* Handle
657 : _7 = ~_2;
658 : */
659 0 : g = convert_atomic_bit_not (fn, use_nop_stmt, lhs,
660 : mask);
661 0 : if (!g)
662 : return false;
663 : /* Convert
664 : _1 = __atomic_fetch_or_4 (ptr_6, 1, _3);
665 : _2 = (int) _1;
666 : _7 = ~_2;
667 : _5 = (_Bool) _7;
668 : to
669 : _1 = __atomic_fetch_or_4 (ptr_6, ~1, _3);
670 : _8 = _1 & 1;
671 : _5 = _8 == 0;
672 : and convert
673 : _1 = __atomic_fetch_and_4 (ptr_6, ~1, _3);
674 : _2 = (int) _1;
675 : _7 = ~_2;
676 : _5 = (_Bool) _7;
677 : to
678 : _1 = __atomic_fetch_and_4 (ptr_6, 1, _3);
679 : _8 = _1 & 1;
680 : _5 = _8 == 0;
681 : */
682 0 : gsi = gsi_for_stmt (use_stmt);
683 0 : gsi_remove (&gsi, true);
684 0 : use_stmt = g;
685 0 : ibit = 0;
686 : }
687 : else
688 : {
689 396 : tree cmp_rhs1, cmp_rhs2;
690 396 : if (use_nop_lhs)
691 : {
692 : /* Handle
693 : _4 = _5 < 0;
694 : */
695 387 : if (TREE_CODE (TREE_TYPE (use_nop_lhs))
696 : != BOOLEAN_TYPE)
697 446 : return false;
698 51 : cmp_rhs1 = gimple_assign_rhs1 (use_nop_stmt);
699 51 : cmp_rhs2 = gimple_assign_rhs2 (use_nop_stmt);
700 : }
701 : else
702 : {
703 : /* Handle
704 : if (_5 < 0)
705 : */
706 9 : rhs_code = gimple_cond_code (use_nop_stmt);
707 9 : cmp_rhs1 = gimple_cond_lhs (use_nop_stmt);
708 9 : cmp_rhs2 = gimple_cond_rhs (use_nop_stmt);
709 : }
710 60 : if (rhs_code != GE_EXPR && rhs_code != LT_EXPR)
711 : return false;
712 48 : if (use_lhs != cmp_rhs1)
713 : return false;
714 48 : if (!integer_zerop (cmp_rhs2))
715 : return false;
716 :
717 48 : tree and_mask;
718 :
719 48 : unsigned HOST_WIDE_INT bytes
720 48 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (use_rhs)));
721 48 : ibit = bytes * BITS_PER_UNIT - 1;
722 48 : unsigned HOST_WIDE_INT highest
723 48 : = HOST_WIDE_INT_1U << ibit;
724 :
725 48 : if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
726 : {
727 : /* Get the signed maximum of the USE_RHS type. */
728 19 : and_mask = build_int_cst (TREE_TYPE (use_rhs),
729 19 : highest - 1);
730 19 : if (!operand_equal_p (and_mask, mask, 0))
731 : return false;
732 :
733 : /* Convert
734 : _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3);
735 : _5 = (signed int) _1;
736 : _4 = _5 < 0 or _5 >= 0;
737 : to
738 : _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3);
739 : _6 = _1 & 0x80000000;
740 : _4 = _6 != 0 or _6 == 0;
741 : and convert
742 : _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3);
743 : _5 = (signed int) _1;
744 : if (_5 < 0 or _5 >= 0)
745 : to
746 : _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3);
747 : _6 = _1 & 0x80000000;
748 : if (_6 != 0 or _6 == 0)
749 : */
750 19 : and_mask = build_int_cst (TREE_TYPE (use_rhs),
751 19 : highest);
752 : }
753 : else
754 : {
755 : /* Get the signed minimum of the USE_RHS type. */
756 29 : and_mask = build_int_cst (TREE_TYPE (use_rhs),
757 29 : highest);
758 29 : if (!operand_equal_p (and_mask, mask, 0))
759 : return false;
760 :
761 : /* Convert
762 : _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3);
763 : _5 = (signed int) _1;
764 : _4 = _5 < 0 or _5 >= 0;
765 : to
766 : _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3);
767 : _6 = _1 & 0x80000000;
768 : _4 = _6 != 0 or _6 == 0;
769 : and convert
770 : _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3);
771 : _5 = (signed int) _1;
772 : if (_5 < 0 or _5 >= 0)
773 : to
774 : _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3);
775 : _6 = _1 & 0x80000000;
776 : if (_6 != 0 or _6 == 0)
777 : */
778 : }
779 36 : var = make_ssa_name (TREE_TYPE (use_rhs));
780 36 : gimple* use_stmt_removal = use_stmt;
781 36 : g = gimple_build_assign (var, BIT_AND_EXPR, use_rhs,
782 : and_mask);
783 36 : gsi = gsi_for_stmt (use_nop_stmt);
784 36 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
785 36 : use_stmt = g;
786 36 : rhs_code = rhs_code == GE_EXPR ? EQ_EXPR : NE_EXPR;
787 36 : tree const_zero = build_zero_cst (TREE_TYPE (use_rhs));
788 36 : if (use_nop_lhs)
789 27 : g = gimple_build_assign (use_nop_lhs, rhs_code,
790 : var, const_zero);
791 : else
792 9 : g = gimple_build_cond (rhs_code, var, const_zero,
793 : nullptr, nullptr);
794 36 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
795 36 : gsi = gsi_for_stmt (use_nop_stmt);
796 36 : gsi_remove (&gsi, true);
797 36 : gsi = gsi_for_stmt (use_stmt_removal);
798 36 : gsi_remove (&gsi, true);
799 : }
800 : }
801 : else
802 : {
803 94 : tree match_op[3];
804 94 : gimple *g;
805 94 : if (!gimple_nop_atomic_bit_test_and_p (use_nop_lhs,
806 : &match_op[0], NULL)
807 92 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (match_op[2])
808 92 : || !single_imm_use (match_op[2], &use_p, &g)
809 186 : || !is_gimple_assign (g))
810 2 : return false;
811 92 : mask = match_op[0];
812 92 : if (TREE_CODE (match_op[1]) == INTEGER_CST)
813 : {
814 48 : ibit = tree_log2 (match_op[1]);
815 48 : gcc_assert (ibit >= 0);
816 : }
817 : else
818 : {
819 44 : g = SSA_NAME_DEF_STMT (match_op[1]);
820 44 : gcc_assert (is_gimple_assign (g));
821 44 : bit = gimple_assign_rhs2 (g);
822 : }
823 : /* Convert
824 : _1 = __atomic_fetch_or_4 (ptr_6, mask, _3);
825 : _2 = (int) _1;
826 : _5 = _2 & mask;
827 : to
828 : _1 = __atomic_fetch_or_4 (ptr_6, mask, _3);
829 : _6 = _1 & mask;
830 : _5 = (int) _6;
831 : and convert
832 : _1 = ~mask_7;
833 : _2 = (unsigned int) _1;
834 : _3 = __atomic_fetch_and_4 (ptr_6, _2, 0);
835 : _4 = (int) _3;
836 : _5 = _4 & mask_7;
837 : to
838 : _1 = __atomic_fetch_and_* (ptr_6, ~mask_7, _3);
839 : _12 = _3 & mask_7;
840 : _5 = (int) _12;
841 :
842 : and Convert
843 : _1 = __atomic_fetch_and_4 (ptr_6, ~mask, _3);
844 : _2 = (short int) _1;
845 : _5 = _2 & mask;
846 : to
847 : _1 = __atomic_fetch_and_4 (ptr_6, ~mask, _3);
848 : _8 = _1 & mask;
849 : _5 = (short int) _8;
850 : */
851 92 : gimple_seq stmts = NULL;
852 92 : match_op[1] = gimple_convert (&stmts,
853 92 : TREE_TYPE (use_rhs),
854 : match_op[1]);
855 92 : var = gimple_build (&stmts, BIT_AND_EXPR,
856 92 : TREE_TYPE (use_rhs), use_rhs, match_op[1]);
857 92 : gsi = gsi_for_stmt (use_stmt);
858 92 : gsi_remove (&gsi, true);
859 92 : release_defs (use_stmt);
860 92 : use_stmt = gimple_seq_last_stmt (stmts);
861 92 : gsi = gsi_for_stmt (use_nop_stmt);
862 92 : gsi_insert_seq_before (&gsi, stmts, GSI_SAME_STMT);
863 92 : gimple_assign_set_rhs_with_ops (&gsi, CONVERT_EXPR, var);
864 92 : update_stmt (use_nop_stmt);
865 : }
866 : }
867 : else
868 : return false;
869 :
870 157 : if (!bit)
871 : {
872 113 : if (ibit < 0)
873 0 : gcc_unreachable ();
874 113 : bit = build_int_cst (TREE_TYPE (lhs), ibit);
875 : }
876 : }
877 292 : else if (optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)))
878 : == CODE_FOR_nothing)
879 : return false;
880 :
881 443 : tree use_lhs = gimple_assign_lhs (use_stmt);
882 443 : if (!use_lhs)
883 : return false;
884 :
885 443 : if (!bit)
886 : {
887 286 : if (TREE_CODE (mask) == INTEGER_CST)
888 : {
889 222 : if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
890 62 : mask = const_unop (BIT_NOT_EXPR, TREE_TYPE (mask), mask);
891 222 : mask = fold_convert (TREE_TYPE (lhs), mask);
892 222 : int ibit = tree_log2 (mask);
893 222 : if (ibit < 0)
894 16 : return false;
895 220 : bit = build_int_cst (TREE_TYPE (lhs), ibit);
896 : }
897 64 : else if (TREE_CODE (mask) == SSA_NAME)
898 : {
899 64 : gimple *g = SSA_NAME_DEF_STMT (mask);
900 64 : tree match_op;
901 64 : if (gimple_nop_convert (mask, &match_op, NULL))
902 : {
903 3 : mask = match_op;
904 3 : if (TREE_CODE (mask) != SSA_NAME)
905 7 : return false;
906 3 : g = SSA_NAME_DEF_STMT (mask);
907 : }
908 64 : if (!is_gimple_assign (g))
909 : return false;
910 :
911 62 : if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
912 : {
913 20 : if (gimple_assign_rhs_code (g) != BIT_NOT_EXPR)
914 : return false;
915 20 : mask = gimple_assign_rhs1 (g);
916 20 : if (TREE_CODE (mask) != SSA_NAME)
917 : return false;
918 20 : g = SSA_NAME_DEF_STMT (mask);
919 : }
920 :
921 62 : if (!is_gimple_assign (g)
922 57 : || gimple_assign_rhs_code (g) != LSHIFT_EXPR
923 119 : || !integer_onep (gimple_assign_rhs1 (g)))
924 : return false;
925 57 : bit = gimple_assign_rhs2 (g);
926 : }
927 : else
928 : return false;
929 :
930 277 : tree cmp_mask;
931 277 : if (gimple_assign_rhs1 (use_stmt) == lhs)
932 241 : cmp_mask = gimple_assign_rhs2 (use_stmt);
933 : else
934 : cmp_mask = gimple_assign_rhs1 (use_stmt);
935 :
936 277 : tree match_op;
937 277 : if (gimple_nop_convert (cmp_mask, &match_op, NULL))
938 1 : cmp_mask = match_op;
939 :
940 277 : if (!operand_equal_p (cmp_mask, mask, 0))
941 : return false;
942 : }
943 :
944 427 : bool use_bool = true;
945 427 : bool has_debug_uses = false;
946 427 : imm_use_iterator iter;
947 427 : gimple *g;
948 :
949 427 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs))
950 0 : use_bool = false;
951 629 : FOR_EACH_IMM_USE_STMT (g, iter, use_lhs)
952 : {
953 428 : enum tree_code code = ERROR_MARK;
954 428 : tree op0 = NULL_TREE, op1 = NULL_TREE;
955 428 : if (is_gimple_debug (g))
956 : {
957 1 : has_debug_uses = true;
958 1 : continue;
959 : }
960 427 : else if (is_gimple_assign (g))
961 385 : switch (gimple_assign_rhs_code (g))
962 : {
963 0 : case COND_EXPR:
964 0 : op1 = gimple_assign_rhs1 (g);
965 0 : code = TREE_CODE (op1);
966 0 : if (TREE_CODE_CLASS (code) != tcc_comparison)
967 : break;
968 0 : op0 = TREE_OPERAND (op1, 0);
969 0 : op1 = TREE_OPERAND (op1, 1);
970 0 : break;
971 173 : case EQ_EXPR:
972 173 : case NE_EXPR:
973 173 : code = gimple_assign_rhs_code (g);
974 173 : op0 = gimple_assign_rhs1 (g);
975 173 : op1 = gimple_assign_rhs2 (g);
976 173 : break;
977 : default:
978 : break;
979 : }
980 42 : else if (gimple_code (g) == GIMPLE_COND)
981 : {
982 28 : code = gimple_cond_code (g);
983 28 : op0 = gimple_cond_lhs (g);
984 28 : op1 = gimple_cond_rhs (g);
985 : }
986 :
987 201 : if ((code == EQ_EXPR || code == NE_EXPR)
988 201 : && op0 == use_lhs
989 402 : && integer_zerop (op1))
990 : {
991 201 : use_operand_p use_p;
992 201 : int n = 0;
993 402 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
994 201 : n++;
995 201 : if (n == 1)
996 201 : continue;
997 : }
998 :
999 : use_bool = false;
1000 : break;
1001 427 : }
1002 :
1003 427 : tree new_lhs = make_ssa_name (TREE_TYPE (lhs));
1004 427 : tree flag = build_int_cst (TREE_TYPE (lhs), use_bool);
1005 427 : if (has_model_arg)
1006 296 : g = gimple_build_call_internal (fn, 5, gimple_call_arg (call, 0),
1007 : bit, flag, gimple_call_arg (call, 2),
1008 : gimple_call_fn (call));
1009 : else
1010 131 : g = gimple_build_call_internal (fn, 4, gimple_call_arg (call, 0),
1011 : bit, flag, gimple_call_fn (call));
1012 427 : gimple_call_set_lhs (g, new_lhs);
1013 427 : gimple_set_location (g, gimple_location (call));
1014 427 : gimple_move_vops (g, call);
1015 427 : bool throws = stmt_can_throw_internal (cfun, call);
1016 427 : gimple_call_set_nothrow (as_a <gcall *> (g),
1017 427 : gimple_call_nothrow_p (as_a <gcall *> (call)));
1018 427 : gimple_stmt_iterator gsi = *gsip;
1019 427 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
1020 427 : edge e = NULL;
1021 427 : if (throws)
1022 : {
1023 75 : maybe_clean_or_replace_eh_stmt (call, g);
1024 75 : if (after || (use_bool && has_debug_uses))
1025 9 : e = find_fallthru_edge (gsi_bb (gsi)->succs);
1026 : }
1027 427 : if (after)
1028 : {
1029 : /* The internal function returns the value of the specified bit
1030 : before the atomic operation. If we are interested in the value
1031 : of the specified bit after the atomic operation (makes only sense
1032 : for xor, otherwise the bit content is compile time known),
1033 : we need to invert the bit. */
1034 55 : tree mask_convert = mask;
1035 55 : gimple_seq stmts = NULL;
1036 55 : if (!use_bool)
1037 43 : mask_convert = gimple_convert (&stmts, TREE_TYPE (lhs), mask);
1038 55 : new_lhs = gimple_build (&stmts, BIT_XOR_EXPR, TREE_TYPE (lhs), new_lhs,
1039 12 : use_bool ? build_int_cst (TREE_TYPE (lhs), 1)
1040 : : mask_convert);
1041 55 : if (throws)
1042 : {
1043 9 : gsi_insert_seq_on_edge_immediate (e, stmts);
1044 18 : gsi = gsi_for_stmt (gimple_seq_last (stmts));
1045 : }
1046 : else
1047 46 : gsi_insert_seq_after (&gsi, stmts, GSI_NEW_STMT);
1048 : }
1049 427 : if (use_bool && has_debug_uses)
1050 : {
1051 1 : tree temp = NULL_TREE;
1052 1 : if (!throws || after || single_pred_p (e->dest))
1053 : {
1054 1 : temp = build_debug_expr_decl (TREE_TYPE (lhs));
1055 1 : tree t = build2 (LSHIFT_EXPR, TREE_TYPE (lhs), new_lhs, bit);
1056 1 : g = gimple_build_debug_bind (temp, t, g);
1057 1 : if (throws && !after)
1058 : {
1059 0 : gsi = gsi_after_labels (e->dest);
1060 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1061 : }
1062 : else
1063 1 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
1064 : }
1065 3 : FOR_EACH_IMM_USE_STMT (g, iter, use_lhs)
1066 2 : if (is_gimple_debug (g))
1067 : {
1068 1 : use_operand_p use_p;
1069 1 : if (temp == NULL_TREE)
1070 0 : gimple_debug_bind_reset_value (g);
1071 : else
1072 2 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1073 1 : SET_USE (use_p, temp);
1074 1 : update_stmt (g);
1075 1 : }
1076 : }
1077 427 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_lhs)
1078 427 : = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs);
1079 427 : replace_uses_by (use_lhs, new_lhs);
1080 427 : gsi = gsi_for_stmt (use_stmt);
1081 427 : gsi_remove (&gsi, true);
1082 427 : release_defs (use_stmt);
1083 427 : gsi_remove (gsip, true);
1084 427 : release_ssa_name (lhs);
1085 427 : return true;
1086 : }
1087 :
1088 : /* Optimize
1089 : _4 = __atomic_add_fetch_* (ptr_6, arg_2, _3);
1090 : _5 = _4 == 0;
1091 : to
1092 : _4 = .ATOMIC_ADD_FETCH_CMP_0 (EQ_EXPR, ptr_6, arg_2, _3);
1093 : _5 = _4;
1094 : Similarly for __sync_add_and_fetch_* (without the ", _3" part
1095 : in there). */
1096 :
1097 : static bool
1098 8857 : optimize_atomic_op_fetch_cmp_0 (gimple_stmt_iterator *gsip,
1099 : enum internal_fn fn, bool has_model_arg)
1100 : {
1101 8857 : gimple *call = gsi_stmt (*gsip);
1102 8857 : tree lhs = gimple_call_lhs (call);
1103 8857 : use_operand_p use_p;
1104 8857 : gimple *use_stmt;
1105 :
1106 8857 : if (!flag_inline_atomics
1107 8857 : || !gimple_call_builtin_p (call, BUILT_IN_NORMAL)
1108 8857 : || !lhs
1109 6332 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs)
1110 6332 : || !single_imm_use (lhs, &use_p, &use_stmt)
1111 15015 : || !gimple_vdef (call))
1112 : return false;
1113 :
1114 6158 : optab optab;
1115 6158 : switch (fn)
1116 : {
1117 : case IFN_ATOMIC_ADD_FETCH_CMP_0:
1118 : optab = atomic_add_fetch_cmp_0_optab;
1119 : break;
1120 : case IFN_ATOMIC_SUB_FETCH_CMP_0:
1121 : optab = atomic_sub_fetch_cmp_0_optab;
1122 : break;
1123 : case IFN_ATOMIC_AND_FETCH_CMP_0:
1124 : optab = atomic_and_fetch_cmp_0_optab;
1125 : break;
1126 : case IFN_ATOMIC_OR_FETCH_CMP_0:
1127 : optab = atomic_or_fetch_cmp_0_optab;
1128 : break;
1129 : case IFN_ATOMIC_XOR_FETCH_CMP_0:
1130 : optab = atomic_xor_fetch_cmp_0_optab;
1131 : break;
1132 : default:
1133 : return false;
1134 : }
1135 :
1136 6158 : if (optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)))
1137 : == CODE_FOR_nothing)
1138 : return false;
1139 :
1140 6119 : tree use_lhs = lhs;
1141 6119 : if (gimple_assign_cast_p (use_stmt))
1142 : {
1143 965 : use_lhs = gimple_assign_lhs (use_stmt);
1144 965 : if (!tree_nop_conversion_p (TREE_TYPE (use_lhs), TREE_TYPE (lhs))
1145 943 : || (!INTEGRAL_TYPE_P (TREE_TYPE (use_lhs))
1146 91 : && !POINTER_TYPE_P (TREE_TYPE (use_lhs)))
1147 943 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs)
1148 1908 : || !single_imm_use (use_lhs, &use_p, &use_stmt))
1149 : return false;
1150 : }
1151 6028 : enum tree_code code = ERROR_MARK;
1152 6028 : tree op0 = NULL_TREE, op1 = NULL_TREE;
1153 6028 : if (is_gimple_assign (use_stmt))
1154 1286 : switch (gimple_assign_rhs_code (use_stmt))
1155 : {
1156 0 : case COND_EXPR:
1157 0 : op1 = gimple_assign_rhs1 (use_stmt);
1158 0 : code = TREE_CODE (op1);
1159 0 : if (TREE_CODE_CLASS (code) == tcc_comparison)
1160 : {
1161 0 : op0 = TREE_OPERAND (op1, 0);
1162 0 : op1 = TREE_OPERAND (op1, 1);
1163 : }
1164 : break;
1165 1286 : default:
1166 1286 : code = gimple_assign_rhs_code (use_stmt);
1167 1286 : if (TREE_CODE_CLASS (code) == tcc_comparison)
1168 : {
1169 839 : op0 = gimple_assign_rhs1 (use_stmt);
1170 839 : op1 = gimple_assign_rhs2 (use_stmt);
1171 : }
1172 : break;
1173 : }
1174 4742 : else if (gimple_code (use_stmt) == GIMPLE_COND)
1175 : {
1176 4227 : code = gimple_cond_code (use_stmt);
1177 4227 : op0 = gimple_cond_lhs (use_stmt);
1178 4227 : op1 = gimple_cond_rhs (use_stmt);
1179 : }
1180 :
1181 5513 : switch (code)
1182 : {
1183 243 : case LT_EXPR:
1184 243 : case LE_EXPR:
1185 243 : case GT_EXPR:
1186 243 : case GE_EXPR:
1187 486 : if (!INTEGRAL_TYPE_P (TREE_TYPE (use_lhs))
1188 243 : || TREE_CODE (TREE_TYPE (use_lhs)) == BOOLEAN_TYPE
1189 486 : || TYPE_UNSIGNED (TREE_TYPE (use_lhs)))
1190 : return false;
1191 : /* FALLTHRU */
1192 5066 : case EQ_EXPR:
1193 5066 : case NE_EXPR:
1194 5066 : if (op0 == use_lhs && integer_zerop (op1))
1195 : break;
1196 : return false;
1197 : default:
1198 : return false;
1199 : }
1200 :
1201 2148 : int encoded;
1202 2148 : switch (code)
1203 : {
1204 : /* Use special encoding of the operation. We want to also
1205 : encode the mode in the first argument and for neither EQ_EXPR
1206 : etc. nor EQ etc. we can rely it will fit into QImode. */
1207 : case EQ_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_EQ; break;
1208 : case NE_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_NE; break;
1209 : case LT_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_LT; break;
1210 : case LE_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_LE; break;
1211 : case GT_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_GT; break;
1212 : case GE_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_GE; break;
1213 : default: gcc_unreachable ();
1214 : }
1215 :
1216 2148 : tree new_lhs = make_ssa_name (boolean_type_node);
1217 2148 : gimple *g;
1218 2148 : tree flag = build_int_cst (TREE_TYPE (lhs), encoded);
1219 2148 : if (has_model_arg)
1220 1740 : g = gimple_build_call_internal (fn, 5, flag,
1221 : gimple_call_arg (call, 0),
1222 : gimple_call_arg (call, 1),
1223 : gimple_call_arg (call, 2),
1224 : gimple_call_fn (call));
1225 : else
1226 408 : g = gimple_build_call_internal (fn, 4, flag,
1227 : gimple_call_arg (call, 0),
1228 : gimple_call_arg (call, 1),
1229 : gimple_call_fn (call));
1230 2148 : gimple_call_set_lhs (g, new_lhs);
1231 2148 : gimple_set_location (g, gimple_location (call));
1232 2148 : gimple_move_vops (g, call);
1233 2148 : bool throws = stmt_can_throw_internal (cfun, call);
1234 2148 : gimple_call_set_nothrow (as_a <gcall *> (g),
1235 2148 : gimple_call_nothrow_p (as_a <gcall *> (call)));
1236 2148 : gimple_stmt_iterator gsi = *gsip;
1237 2148 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
1238 2148 : if (throws)
1239 0 : maybe_clean_or_replace_eh_stmt (call, g);
1240 2148 : if (is_gimple_assign (use_stmt))
1241 816 : switch (gimple_assign_rhs_code (use_stmt))
1242 : {
1243 0 : case COND_EXPR:
1244 0 : gimple_assign_set_rhs1 (use_stmt, new_lhs);
1245 0 : break;
1246 816 : default:
1247 816 : gsi = gsi_for_stmt (use_stmt);
1248 816 : if (tree ulhs = gimple_assign_lhs (use_stmt))
1249 816 : if (useless_type_conversion_p (TREE_TYPE (ulhs),
1250 : boolean_type_node))
1251 : {
1252 816 : gimple_assign_set_rhs_with_ops (&gsi, SSA_NAME, new_lhs);
1253 816 : break;
1254 : }
1255 0 : gimple_assign_set_rhs_with_ops (&gsi, NOP_EXPR, new_lhs);
1256 0 : break;
1257 : }
1258 1332 : else if (gimple_code (use_stmt) == GIMPLE_COND)
1259 : {
1260 1332 : gcond *use_cond = as_a <gcond *> (use_stmt);
1261 1332 : gimple_cond_set_code (use_cond, NE_EXPR);
1262 1332 : gimple_cond_set_lhs (use_cond, new_lhs);
1263 1332 : gimple_cond_set_rhs (use_cond, boolean_false_node);
1264 : }
1265 :
1266 2148 : update_stmt (use_stmt);
1267 2148 : if (use_lhs != lhs)
1268 : {
1269 234 : gsi = gsi_for_stmt (SSA_NAME_DEF_STMT (use_lhs));
1270 234 : gsi_remove (&gsi, true);
1271 234 : release_ssa_name (use_lhs);
1272 : }
1273 2148 : gsi_remove (gsip, true);
1274 2148 : release_ssa_name (lhs);
1275 2148 : return true;
1276 : }
1277 :
1278 : /* Process builtin CALL located at GSI.
1279 : Currently it is only fgr atomic functions optimizations from above. */
1280 : static void
1281 7073150 : gimple_isel_builtin_call (gcall *call, gimple_stmt_iterator *gsi)
1282 : {
1283 : /* Don't handle these in non optimization mode or optimize debug mode. */
1284 7073150 : if (!optimize || optimize_debug)
1285 : return;
1286 :
1287 5497571 : if (!gimple_call_builtin_p (call, BUILT_IN_NORMAL))
1288 : return;
1289 :
1290 1376242 : tree callee = gimple_call_fndecl (call);
1291 :
1292 1376242 : switch (DECL_FUNCTION_CODE (callee))
1293 : {
1294 : #define CASE_ATOMIC(NAME) \
1295 : case BUILT_IN_##NAME##_1: \
1296 : case BUILT_IN_##NAME##_2: \
1297 : case BUILT_IN_##NAME##_4: \
1298 : case BUILT_IN_##NAME##_8: \
1299 : case BUILT_IN_##NAME##_16
1300 : #define CASE_ATOMIC_CMP0(ATOMIC, SYNC) \
1301 : CASE_ATOMIC(ATOMIC_##ATOMIC): \
1302 : optimize_atomic_op_fetch_cmp_0 (gsi, \
1303 : IFN_ATOMIC_##ATOMIC##_CMP_0, \
1304 : true); \
1305 : break; \
1306 : CASE_ATOMIC(SYNC_##SYNC): \
1307 : optimize_atomic_op_fetch_cmp_0 (gsi, \
1308 : IFN_ATOMIC_##ATOMIC##_CMP_0, \
1309 : false); \
1310 : break;
1311 :
1312 :
1313 4183 : CASE_ATOMIC_CMP0(ADD_FETCH, ADD_AND_FETCH)
1314 2391 : CASE_ATOMIC_CMP0(SUB_FETCH, SUB_AND_FETCH)
1315 780 : CASE_ATOMIC_CMP0(AND_FETCH, AND_AND_FETCH)
1316 778 : CASE_ATOMIC_CMP0(OR_FETCH, OR_AND_FETCH)
1317 : #define CASE_ATOMIC_BIT_TEST_AND(ATOMIC, SYNC, FN, AFTER) \
1318 : CASE_ATOMIC(ATOMIC_##ATOMIC): \
1319 : optimize_atomic_bit_test_and (gsi, \
1320 : IFN_ATOMIC_BIT_TEST_AND_##FN, \
1321 : true, AFTER); \
1322 : break; \
1323 : CASE_ATOMIC(SYNC_##SYNC): \
1324 : optimize_atomic_bit_test_and (gsi, \
1325 : IFN_ATOMIC_BIT_TEST_AND_##FN, \
1326 : false, AFTER); \
1327 : break;
1328 1430 : CASE_ATOMIC_BIT_TEST_AND(FETCH_OR, FETCH_AND_OR, SET, false)
1329 1291 : CASE_ATOMIC_BIT_TEST_AND(FETCH_XOR, FETCH_AND_XOR, COMPLEMENT, false)
1330 1150 : CASE_ATOMIC_BIT_TEST_AND(FETCH_AND, FETCH_AND_AND, RESET, false)
1331 :
1332 580 : CASE_ATOMIC(ATOMIC_XOR_FETCH):
1333 580 : if (optimize_atomic_bit_test_and
1334 580 : (gsi, IFN_ATOMIC_BIT_TEST_AND_COMPLEMENT, true, true))
1335 : break;
1336 542 : optimize_atomic_op_fetch_cmp_0 (gsi,
1337 : IFN_ATOMIC_XOR_FETCH_CMP_0,
1338 : true);
1339 542 : break;
1340 200 : CASE_ATOMIC(SYNC_XOR_AND_FETCH):
1341 200 : if (optimize_atomic_bit_test_and
1342 200 : (gsi, IFN_ATOMIC_BIT_TEST_AND_COMPLEMENT, false, true))
1343 : break;
1344 183 : optimize_atomic_op_fetch_cmp_0 (gsi,
1345 : IFN_ATOMIC_XOR_FETCH_CMP_0,
1346 : false);
1347 183 : break;
1348 :
1349 55 : default:;
1350 : }
1351 : }
1352 :
1353 : /* Iterate all gimple statements and perform pre RTL expansion
1354 : GIMPLE massaging to improve instruction selection. */
1355 :
1356 : unsigned int
1357 1515903 : pass_gimple_isel::execute (struct function *fun)
1358 : {
1359 1515903 : gimple_stmt_iterator gsi;
1360 1515903 : basic_block bb;
1361 1515903 : bool cfg_changed = false;
1362 1515903 : if (optimize)
1363 1062676 : enable_ranger (fun);
1364 :
1365 17475900 : FOR_EACH_BB_FN (bb, fun)
1366 : {
1367 134415147 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1368 : {
1369 : /* Give the target first try at replacing the instruction. */
1370 102495153 : cfg_changed |= targetm.instruction_selection (fun, &gsi);
1371 :
1372 : /* Pre-expand VEC_COND_EXPRs to .VCOND* internal function
1373 : calls mapping to supported optabs. */
1374 102495153 : gimple *g = gimple_expand_vec_cond_expr (&gsi);
1375 102495153 : if (g != NULL)
1376 : {
1377 22602 : tree lhs = gimple_assign_lhs (gsi_stmt (gsi));
1378 22602 : gimple_set_lhs (g, lhs);
1379 22602 : gsi_replace (&gsi, g, false);
1380 : }
1381 :
1382 : /* Recognize .VEC_SET and .VEC_EXTRACT patterns. */
1383 102495153 : cfg_changed |= gimple_expand_vec_set_extract_expr (fun, &gsi);
1384 102495153 : if (gsi_end_p (gsi))
1385 : break;
1386 :
1387 102495153 : if (gcall *call = dyn_cast <gcall*>(*gsi))
1388 : {
1389 7073150 : gimple_isel_builtin_call (call, &gsi);
1390 7073150 : continue;
1391 : }
1392 95422003 : gassign *stmt = dyn_cast <gassign *> (*gsi);
1393 95422003 : if (!stmt)
1394 62664015 : continue;
1395 :
1396 32757988 : tree_code code = gimple_assign_rhs_code (stmt);
1397 32757988 : if (TREE_CODE_CLASS (code) == tcc_comparison)
1398 589297 : maybe_duplicate_comparison (stmt, bb);
1399 : }
1400 : }
1401 :
1402 1515903 : if (optimize)
1403 1062676 : disable_ranger (fun);
1404 1515903 : return cfg_changed ? TODO_cleanup_cfg : 0;
1405 : }
1406 :
1407 : } // anon namespace
1408 :
1409 : gimple_opt_pass *
1410 294196 : make_pass_gimple_isel (gcc::context *ctxt)
1411 : {
1412 294196 : return new pass_gimple_isel (ctxt);
1413 : }
1414 :
|
