Line data Source code
1 : /* Schedule GIMPLE vector statements.
2 : Copyright (C) 2020-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by the
8 : Free Software Foundation; either version 3, or (at your option) any
9 : later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "rtl.h"
25 : #include "tree.h"
26 : #include "gimple.h"
27 : #include "tree-pass.h"
28 : #include "ssa.h"
29 : #include "expmed.h"
30 : #include "optabs-tree.h"
31 : #include "tree-eh.h"
32 : #include "gimple-iterator.h"
33 : #include "gimplify-me.h"
34 : #include "gimplify.h"
35 : #include "tree-cfg.h"
36 : #include "bitmap.h"
37 : #include "tree-ssa-dce.h"
38 : #include "memmodel.h"
39 : #include "optabs.h"
40 : #include "gimple-fold.h"
41 : #include "internal-fn.h"
42 : #include "fold-const.h"
43 :
44 : /* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to
45 : internal function based on vector type of selected expansion.
46 :
47 : For vec_set:
48 :
49 : VIEW_CONVERT_EXPR<int[4]>(u)[_1] = i_4(D);
50 : =>
51 : _7 = u;
52 : _8 = .VEC_SET (_7, i_4(D), _1);
53 : u = _8;
54 :
55 : For vec_extract:
56 :
57 : _3 = VIEW_CONVERT_EXPR<intD.1[4]>(vD.2208)[idx_2(D)];
58 : =>
59 : _4 = vD.2208;
60 : _3 = .VEC_EXTRACT (_4, idx_2(D)); */
61 :
62 : static bool
63 96087188 : gimple_expand_vec_set_extract_expr (struct function *fun,
64 : gimple_stmt_iterator *gsi)
65 : {
66 96087188 : gcall *new_stmt = NULL;
67 96087188 : gassign *ass_stmt = NULL;
68 96087188 : bool cfg_changed = false;
69 :
70 : /* Only consider code == GIMPLE_ASSIGN. */
71 127669038 : gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi));
72 31591813 : if (!stmt)
73 : return false;
74 :
75 31591813 : bool is_extract = false;
76 :
77 31591813 : tree lhs = gimple_assign_lhs (stmt);
78 31591813 : tree rhs = gimple_assign_rhs1 (stmt);
79 31591813 : tree val, ref;
80 31591813 : if (TREE_CODE (lhs) == ARRAY_REF)
81 : {
82 : /* Assume it is a vec_set. */
83 : val = rhs;
84 : ref = lhs;
85 : }
86 30981220 : else if (TREE_CODE (rhs) == ARRAY_REF)
87 : {
88 : /* vec_extract. */
89 : is_extract = true;
90 : val = lhs;
91 : ref = rhs;
92 : }
93 : else
94 : return false;
95 :
96 1168511 : tree op0 = TREE_OPERAND (ref, 0);
97 32172 : if (TREE_CODE (op0) == VIEW_CONVERT_EXPR && DECL_P (TREE_OPERAND (op0, 0))
98 26822 : && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0)))
99 1188485 : && TYPE_MODE (TREE_TYPE (ref))
100 9987 : == TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0)))))
101 : {
102 9983 : tree pos = TREE_OPERAND (ref, 1);
103 :
104 9983 : tree view_op0 = TREE_OPERAND (op0, 0);
105 :
106 9983 : tree idx = TREE_OPERAND (ref, 1);
107 : // if index is a constant, then check the bounds
108 9983 : poly_uint64 idx_poly;
109 9983 : if (poly_int_tree_p (idx, &idx_poly))
110 : {
111 33 : poly_uint64 nelts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (view_op0));
112 33 : if (known_gt (idx_poly, nelts))
113 20 : return false;
114 : }
115 9963 : machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0));
116 9963 : machine_mode extract_mode = TYPE_MODE (TREE_TYPE (ref));
117 :
118 9963 : if ((auto_var_in_fn_p (view_op0, fun->decl)
119 1307 : || (VAR_P (view_op0) && DECL_HARD_REGISTER (view_op0)))
120 8666 : && !TREE_ADDRESSABLE (view_op0)
121 16894 : && ((!is_extract && can_vec_set_var_idx_p (outermode))
122 : || (is_extract
123 6790 : && can_vec_extract_var_idx_p (outermode, extract_mode))))
124 : {
125 123 : location_t loc = gimple_location (stmt);
126 123 : tree var_src = make_ssa_name (TREE_TYPE (view_op0));
127 :
128 123 : ass_stmt = gimple_build_assign (var_src, view_op0);
129 246 : gimple_set_vuse (ass_stmt, gimple_vuse (stmt));
130 123 : gimple_set_location (ass_stmt, loc);
131 123 : gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
132 :
133 123 : if (!is_extract)
134 : {
135 123 : tree var_dst = make_ssa_name (TREE_TYPE (view_op0));
136 :
137 123 : new_stmt = gimple_build_call_internal (IFN_VEC_SET, 3, var_src,
138 : val, pos);
139 :
140 123 : gimple_call_set_lhs (new_stmt, var_dst);
141 123 : gimple_set_location (new_stmt, loc);
142 123 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
143 :
144 123 : ass_stmt = gimple_build_assign (view_op0, var_dst);
145 123 : gimple_set_location (ass_stmt, loc);
146 123 : gimple_move_vops (ass_stmt, stmt);
147 123 : gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
148 :
149 123 : basic_block bb = gimple_bb (stmt);
150 123 : if (gsi_remove (gsi, true)
151 123 : && gimple_purge_dead_eh_edges (bb))
152 : cfg_changed = true;
153 123 : *gsi = gsi_for_stmt (ass_stmt);
154 : }
155 : else
156 : {
157 0 : new_stmt
158 0 : = gimple_build_call_internal (IFN_VEC_EXTRACT, 2, var_src, pos);
159 0 : gimple_call_set_lhs (new_stmt, lhs);
160 :
161 0 : gsi_replace (gsi, new_stmt, true);
162 0 : cfg_changed = true;
163 : }
164 : }
165 : }
166 :
167 : return cfg_changed;
168 : }
169 :
170 : /* Expand all VEC_COND_EXPR gimple assignments into calls to internal
171 : function based on type of selected expansion. */
172 :
173 : static gimple *
174 96087188 : gimple_expand_vec_cond_expr (gimple_stmt_iterator *gsi)
175 : {
176 96087188 : tree lhs, op0a = NULL_TREE;
177 96087188 : enum tree_code code;
178 96087188 : enum tree_code tcode;
179 :
180 : /* Only consider code == GIMPLE_ASSIGN. */
181 96087188 : gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi));
182 31600681 : if (!stmt)
183 : return NULL;
184 :
185 31600681 : code = gimple_assign_rhs_code (stmt);
186 31600681 : if (code != VEC_COND_EXPR)
187 : return NULL;
188 :
189 19221 : tree op0 = gimple_assign_rhs1 (stmt);
190 19221 : tree op1 = gimple_assign_rhs2 (stmt);
191 19221 : tree op2 = gimple_assign_rhs3 (stmt);
192 19221 : lhs = gimple_assign_lhs (stmt);
193 19221 : machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
194 :
195 : /* Lower mask typed, non-vector mode VEC_COND_EXPRs to bitwise operations.
196 : Those can end up generated by folding and at least for integer mode masks
197 : we cannot expect vcond expanders to exist. We lower a ? b : c
198 : to (b & a) | (c & ~a). */
199 38442 : if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (lhs))
200 19222 : && !VECTOR_MODE_P (mode))
201 : {
202 0 : gcc_assert (types_compatible_p (TREE_TYPE (op0), TREE_TYPE (op1)));
203 0 : gimple_seq stmts = NULL;
204 0 : tree type = TREE_TYPE (lhs);
205 0 : location_t loc = gimple_location (stmt);
206 0 : tree tem0 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op1, op0);
207 0 : tree tem1 = gimple_build (&stmts, loc, BIT_NOT_EXPR, type, op0);
208 0 : tree tem2 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op2, tem1);
209 0 : tree tem3 = gimple_build (&stmts, loc, BIT_IOR_EXPR, type, tem0, tem2);
210 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
211 0 : return gimple_build_assign (lhs, tem3);
212 : }
213 :
214 19221 : bool can_compute_op0 = true;
215 19221 : gcc_assert (!COMPARISON_CLASS_P (op0));
216 19221 : if (TREE_CODE (op0) == SSA_NAME)
217 : {
218 17941 : gassign *def_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op0));
219 17931 : if (def_stmt)
220 : {
221 17931 : tcode = gimple_assign_rhs_code (def_stmt);
222 17931 : op0a = gimple_assign_rhs1 (def_stmt);
223 :
224 17931 : tree op0_type = TREE_TYPE (op0);
225 17931 : tree op0a_type = TREE_TYPE (op0a);
226 17931 : if (TREE_CODE_CLASS (tcode) == tcc_comparison)
227 13397 : can_compute_op0 = expand_vec_cmp_expr_p (op0a_type, op0_type,
228 : tcode);
229 13397 : gcc_assert (can_compute_op0);
230 :
231 17931 : if (can_compute_op0
232 17931 : && TYPE_MODE (TREE_TYPE (lhs)) == TYPE_MODE (TREE_TYPE (op0)))
233 : {
234 : /* Assuming c = x CMP y. */
235 13134 : bool op1_minus_onep = integer_minus_onep (op1);
236 13134 : bool op2_zerop = integer_zerop (op2);
237 13134 : tree vtype = TREE_TYPE (lhs);
238 13134 : machine_mode vmode = TYPE_MODE (vtype);
239 : /* Try to fold r = c ? -1 : 0 to r = c. */
240 13134 : if (op1_minus_onep && op2_zerop)
241 : {
242 3536 : tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
243 3536 : return gimple_build_assign (lhs, conv_op);
244 : }
245 : /* Try to fold r = c ? -1 : z to r = c | z, or
246 : r = c ? c : z. */
247 9598 : if (op1_minus_onep)
248 : {
249 16 : tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
250 16 : tree new_op1 = make_ssa_name (vtype);
251 16 : gassign *new_stmt = gimple_build_assign (new_op1, conv_op);
252 16 : gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT);
253 16 : if (optab_handler (ior_optab, vmode) != CODE_FOR_nothing)
254 : /* r = c | z */
255 16 : return gimple_build_assign (lhs, BIT_IOR_EXPR, new_op1,
256 16 : op2);
257 : /* r = c ? c : z */
258 : op1 = new_op1;
259 : }
260 : /* Try to fold r = c ? z : 0 to r = c & z, or
261 : r = c ? z : c. */
262 9582 : else if (op2_zerop)
263 : {
264 6801 : tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
265 6801 : tree new_op2 = make_ssa_name (vtype);
266 6801 : gassign *new_stmt = gimple_build_assign (new_op2, conv_op);
267 6801 : gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT);
268 6801 : if (optab_handler (and_optab, vmode) != CODE_FOR_nothing)
269 : /* r = c | z */
270 6801 : return gimple_build_assign (lhs, BIT_AND_EXPR, new_op2,
271 6801 : op1);
272 : /* r = c ? z : c */
273 : op2 = new_op2;
274 : }
275 2781 : bool op1_zerop = integer_zerop (op1);
276 2781 : bool op2_minus_onep = integer_minus_onep (op2);
277 : /* Try to fold r = c ? 0 : z to r = .BIT_ANDN (z, c). */
278 2781 : if (op1_zerop
279 2781 : && (direct_internal_fn_supported_p (IFN_BIT_ANDN, vtype,
280 : OPTIMIZE_FOR_BOTH)))
281 : {
282 74 : tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
283 74 : tree new_op = make_ssa_name (vtype);
284 74 : gassign *new_stmt = gimple_build_assign (new_op, conv_op);
285 74 : gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT);
286 74 : return gimple_build_call_internal (IFN_BIT_ANDN, 2, op2,
287 74 : new_op);
288 : }
289 : /* Try to fold r = c ? z : -1 to r = .BIT_IORN (z, c). */
290 2707 : else if (op2_minus_onep
291 2707 : && (direct_internal_fn_supported_p (IFN_BIT_IORN, vtype,
292 : OPTIMIZE_FOR_BOTH)))
293 : {
294 0 : tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
295 0 : tree new_op = make_ssa_name (vtype);
296 0 : gassign *new_stmt = gimple_build_assign (new_op, conv_op);
297 0 : gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT);
298 0 : return gimple_build_call_internal (IFN_BIT_IORN, 2, op1,
299 0 : new_op);
300 : }
301 : }
302 : }
303 : }
304 :
305 8794 : gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
306 17588 : gcc_assert (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
307 : != CODE_FOR_nothing);
308 8794 : return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
309 : }
310 :
311 : /* Duplicate COND_EXPR condition defs of STMT located in BB when they are
312 : comparisons so RTL expansion with the help of TER
313 : can perform better if conversion. */
314 : static void
315 534477 : maybe_duplicate_comparison (gassign *stmt, basic_block bb)
316 : {
317 534477 : imm_use_iterator imm_iter;
318 534477 : use_operand_p use_p;
319 534477 : auto_vec<gassign *, 4> cond_exprs;
320 534477 : tree lhs = gimple_assign_lhs (stmt);
321 534477 : unsigned cnt = 0;
322 :
323 : /* This is should not be used for -O0 nor it is not useful
324 : when ter is turned off. */
325 534477 : if (!optimize || !flag_tree_ter)
326 : return;
327 :
328 1259967 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
329 : {
330 437839 : if (is_gimple_debug (USE_STMT (use_p)))
331 10345 : continue;
332 427494 : cnt++;
333 : /* Add the use statement if it was a cond_expr. */
334 427494 : if (gimple_bb (USE_STMT (use_p)) == bb
335 381293 : && is_gimple_assign (USE_STMT (use_p))
336 361985 : && gimple_assign_rhs_code (USE_STMT (use_p)) == COND_EXPR
337 443103 : && gimple_assign_rhs1_ptr (USE_STMT (use_p)) == use_p->use)
338 15199 : cond_exprs.safe_push (as_a <gassign *> (USE_STMT (use_p)));
339 411064 : }
340 :
341 : /* If the comparison has 0 or 1 uses, no reason to do anything. */
342 411064 : if (cnt <= 1)
343 : return;
344 :
345 : /* If we only use the expression inside cond_exprs in that BB, we don't
346 : need to duplicate for one of them so pop the top. */
347 21700 : if (cond_exprs.length () == cnt)
348 169 : cond_exprs.pop();
349 :
350 22131 : while (!cond_exprs.is_empty())
351 : {
352 431 : auto old_top = cond_exprs.pop();
353 431 : gassign *copy = as_a <gassign *> (gimple_copy (stmt));
354 431 : tree new_def = duplicate_ssa_name (lhs, copy);
355 431 : gimple_assign_set_lhs (copy, new_def);
356 431 : auto gsi2 = gsi_for_stmt (old_top);
357 431 : gsi_insert_before (&gsi2, copy, GSI_SAME_STMT);
358 431 : gimple_assign_set_rhs1 (old_top, new_def);
359 431 : update_stmt (old_top);
360 : }
361 534477 : }
362 :
363 : /* match.pd function to match atomic_bit_test_and pattern which
364 : has nop_convert:
365 : _1 = __atomic_fetch_or_4 (&v, 1, 0);
366 : _2 = (int) _1;
367 : _5 = _2 & 1;
368 : */
369 : extern bool gimple_nop_atomic_bit_test_and_p (tree, tree *,
370 : tree (*) (tree));
371 : extern bool gimple_nop_convert (tree, tree*, tree (*) (tree));
372 :
373 : namespace {
374 :
375 : const pass_data pass_data_gimple_isel =
376 : {
377 : GIMPLE_PASS, /* type */
378 : "isel", /* name */
379 : OPTGROUP_VEC, /* optinfo_flags */
380 : TV_NONE, /* tv_id */
381 : PROP_cfg, /* properties_required */
382 : 0, /* properties_provided */
383 : 0, /* properties_destroyed */
384 : 0, /* todo_flags_start */
385 : TODO_update_ssa, /* todo_flags_finish */
386 : };
387 :
388 : class pass_gimple_isel : public gimple_opt_pass
389 : {
390 : public:
391 285722 : pass_gimple_isel (gcc::context *ctxt)
392 571444 : : gimple_opt_pass (pass_data_gimple_isel, ctxt)
393 : {}
394 :
395 : /* opt_pass methods: */
396 1472150 : bool gate (function *) final override
397 : {
398 1472150 : return true;
399 : }
400 :
401 : unsigned int execute (function *fun) final override;
402 : }; // class pass_gimple_isel
403 :
404 :
405 :
406 : /* Convert
407 : _1 = __atomic_fetch_or_* (ptr_6, 1, _3);
408 : _7 = ~_1;
409 : _5 = (_Bool) _7;
410 : to
411 : _1 = __atomic_fetch_or_* (ptr_6, 1, _3);
412 : _8 = _1 & 1;
413 : _5 = _8 == 0;
414 : and convert
415 : _1 = __atomic_fetch_and_* (ptr_6, ~1, _3);
416 : _7 = ~_1;
417 : _4 = (_Bool) _7;
418 : to
419 : _1 = __atomic_fetch_and_* (ptr_6, ~1, _3);
420 : _8 = _1 & 1;
421 : _4 = (_Bool) _8;
422 :
423 : USE_STMT is the gimplt statement which uses the return value of
424 : __atomic_fetch_or_*. LHS is the return value of __atomic_fetch_or_*.
425 : MASK is the mask passed to __atomic_fetch_or_*.
426 : */
427 :
428 : static gimple *
429 14 : convert_atomic_bit_not (enum internal_fn fn, gimple *use_stmt,
430 : tree lhs, tree mask)
431 : {
432 14 : tree and_mask;
433 14 : if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
434 : {
435 : /* MASK must be ~1. */
436 8 : if (!operand_equal_p (build_int_cst (TREE_TYPE (lhs),
437 : ~HOST_WIDE_INT_1), mask, 0))
438 : return nullptr;
439 8 : and_mask = build_int_cst (TREE_TYPE (lhs), 1);
440 : }
441 : else
442 : {
443 : /* MASK must be 1. */
444 6 : if (!operand_equal_p (build_int_cst (TREE_TYPE (lhs), 1), mask, 0))
445 : return nullptr;
446 : and_mask = mask;
447 : }
448 :
449 14 : tree use_lhs = gimple_assign_lhs (use_stmt);
450 :
451 14 : use_operand_p use_p;
452 14 : gimple *use_not_stmt;
453 :
454 14 : if (!single_imm_use (use_lhs, &use_p, &use_not_stmt)
455 14 : || !is_gimple_assign (use_not_stmt))
456 : return nullptr;
457 :
458 14 : if (!CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (use_not_stmt)))
459 : return nullptr;
460 :
461 14 : tree use_not_lhs = gimple_assign_lhs (use_not_stmt);
462 14 : if (TREE_CODE (TREE_TYPE (use_not_lhs)) != BOOLEAN_TYPE)
463 : return nullptr;
464 :
465 14 : gimple_stmt_iterator gsi;
466 14 : tree var = make_ssa_name (TREE_TYPE (lhs));
467 : /* use_stmt need to be removed after use_nop_stmt,
468 : so use_lhs can be released. */
469 14 : gimple *use_stmt_removal = use_stmt;
470 14 : use_stmt = gimple_build_assign (var, BIT_AND_EXPR, lhs, and_mask);
471 14 : gsi = gsi_for_stmt (use_not_stmt);
472 14 : gsi_insert_before (&gsi, use_stmt, GSI_NEW_STMT);
473 14 : lhs = gimple_assign_lhs (use_not_stmt);
474 14 : gimple *g = gimple_build_assign (lhs, EQ_EXPR, var,
475 14 : build_zero_cst (TREE_TYPE (mask)));
476 14 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
477 14 : gsi = gsi_for_stmt (use_not_stmt);
478 14 : gsi_remove (&gsi, true);
479 14 : gsi = gsi_for_stmt (use_stmt_removal);
480 14 : gsi_remove (&gsi, true);
481 14 : return use_stmt;
482 : }
483 :
484 : /* Optimize
485 : mask_2 = 1 << cnt_1;
486 : _4 = __atomic_fetch_or_* (ptr_6, mask_2, _3);
487 : _5 = _4 & mask_2;
488 : to
489 : _4 = .ATOMIC_BIT_TEST_AND_SET (ptr_6, cnt_1, 0, _3);
490 : _5 = _4;
491 : If _5 is only used in _5 != 0 or _5 == 0 comparisons, 1
492 : is passed instead of 0, and the builtin just returns a zero
493 : or 1 value instead of the actual bit.
494 : Similarly for __sync_fetch_and_or_* (without the ", _3" part
495 : in there), and/or if mask_2 is a power of 2 constant.
496 : Similarly for xor instead of or, use ATOMIC_BIT_TEST_AND_COMPLEMENT
497 : in that case. And similarly for and instead of or, except that
498 : the second argument to the builtin needs to be one's complement
499 : of the mask instead of mask. */
500 :
501 : static bool
502 4624 : optimize_atomic_bit_test_and (gimple_stmt_iterator *gsip,
503 : enum internal_fn fn, bool has_model_arg,
504 : bool after)
505 : {
506 4624 : gimple *call = gsi_stmt (*gsip);
507 4624 : tree lhs = gimple_call_lhs (call);
508 4624 : use_operand_p use_p;
509 4624 : gimple *use_stmt;
510 4624 : tree mask;
511 4624 : optab optab;
512 :
513 4624 : if (!flag_inline_atomics
514 4624 : || optimize_debug
515 4624 : || !gimple_call_builtin_p (call, BUILT_IN_NORMAL)
516 4624 : || !lhs
517 2984 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs)
518 2984 : || !single_imm_use (lhs, &use_p, &use_stmt)
519 2954 : || !is_gimple_assign (use_stmt)
520 6356 : || !gimple_vdef (call))
521 2892 : return false;
522 :
523 1732 : switch (fn)
524 : {
525 : case IFN_ATOMIC_BIT_TEST_AND_SET:
526 : optab = atomic_bit_test_and_set_optab;
527 : break;
528 : case IFN_ATOMIC_BIT_TEST_AND_COMPLEMENT:
529 : optab = atomic_bit_test_and_complement_optab;
530 : break;
531 : case IFN_ATOMIC_BIT_TEST_AND_RESET:
532 : optab = atomic_bit_test_and_reset_optab;
533 : break;
534 : default:
535 : return false;
536 : }
537 :
538 1732 : tree bit = nullptr;
539 :
540 1732 : mask = gimple_call_arg (call, 1);
541 1732 : tree_code rhs_code = gimple_assign_rhs_code (use_stmt);
542 1732 : if (rhs_code != BIT_AND_EXPR)
543 : {
544 1440 : if (rhs_code != NOP_EXPR && rhs_code != BIT_NOT_EXPR)
545 1283 : return false;
546 :
547 869 : tree use_lhs = gimple_assign_lhs (use_stmt);
548 869 : if (TREE_CODE (use_lhs) == SSA_NAME
549 869 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs))
550 : return false;
551 :
552 869 : tree use_rhs = gimple_assign_rhs1 (use_stmt);
553 869 : if (lhs != use_rhs)
554 : return false;
555 :
556 869 : if (optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)))
557 : == CODE_FOR_nothing)
558 : return false;
559 :
560 605 : gimple *g;
561 605 : gimple_stmt_iterator gsi;
562 605 : tree var;
563 605 : int ibit = -1;
564 :
565 605 : if (rhs_code == BIT_NOT_EXPR)
566 : {
567 14 : g = convert_atomic_bit_not (fn, use_stmt, lhs, mask);
568 14 : if (!g)
569 : return false;
570 14 : use_stmt = g;
571 14 : ibit = 0;
572 : }
573 591 : else if (TREE_CODE (TREE_TYPE (use_lhs)) == BOOLEAN_TYPE)
574 : {
575 15 : tree and_mask;
576 15 : if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
577 : {
578 : /* MASK must be ~1. */
579 8 : if (!operand_equal_p (build_int_cst (TREE_TYPE (lhs),
580 : ~HOST_WIDE_INT_1),
581 : mask, 0))
582 : return false;
583 :
584 : /* Convert
585 : _1 = __atomic_fetch_and_* (ptr_6, ~1, _3);
586 : _4 = (_Bool) _1;
587 : to
588 : _1 = __atomic_fetch_and_* (ptr_6, ~1, _3);
589 : _5 = _1 & 1;
590 : _4 = (_Bool) _5;
591 : */
592 8 : and_mask = build_int_cst (TREE_TYPE (lhs), 1);
593 : }
594 : else
595 : {
596 7 : and_mask = build_int_cst (TREE_TYPE (lhs), 1);
597 7 : if (!operand_equal_p (and_mask, mask, 0))
598 : return false;
599 :
600 : /* Convert
601 : _1 = __atomic_fetch_or_* (ptr_6, 1, _3);
602 : _4 = (_Bool) _1;
603 : to
604 : _1 = __atomic_fetch_or_* (ptr_6, 1, _3);
605 : _5 = _1 & 1;
606 : _4 = (_Bool) _5;
607 : */
608 : }
609 15 : var = make_ssa_name (TREE_TYPE (use_rhs));
610 15 : replace_uses_by (use_rhs, var);
611 15 : g = gimple_build_assign (var, BIT_AND_EXPR, use_rhs,
612 : and_mask);
613 15 : gsi = gsi_for_stmt (use_stmt);
614 15 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
615 15 : use_stmt = g;
616 15 : ibit = 0;
617 : }
618 576 : else if (TYPE_PRECISION (TREE_TYPE (use_lhs))
619 576 : <= TYPE_PRECISION (TREE_TYPE (use_rhs)))
620 : {
621 574 : gimple *use_nop_stmt;
622 574 : if (!single_imm_use (use_lhs, &use_p, &use_nop_stmt)
623 574 : || (!is_gimple_assign (use_nop_stmt)
624 93 : && gimple_code (use_nop_stmt) != GIMPLE_COND))
625 446 : return false;
626 : /* Handle both
627 : _4 = _5 < 0;
628 : and
629 : if (_5 < 0)
630 : */
631 490 : tree use_nop_lhs = nullptr;
632 490 : rhs_code = ERROR_MARK;
633 490 : if (is_gimple_assign (use_nop_stmt))
634 : {
635 481 : use_nop_lhs = gimple_assign_lhs (use_nop_stmt);
636 481 : rhs_code = gimple_assign_rhs_code (use_nop_stmt);
637 : }
638 490 : if (!use_nop_lhs || rhs_code != BIT_AND_EXPR)
639 : {
640 : /* Also handle
641 : if (_5 < 0)
642 : */
643 396 : if (use_nop_lhs
644 387 : && TREE_CODE (use_nop_lhs) == SSA_NAME
645 447 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_nop_lhs))
646 : return false;
647 396 : if (use_nop_lhs && rhs_code == BIT_NOT_EXPR)
648 : {
649 : /* Handle
650 : _7 = ~_2;
651 : */
652 0 : g = convert_atomic_bit_not (fn, use_nop_stmt, lhs,
653 : mask);
654 0 : if (!g)
655 : return false;
656 : /* Convert
657 : _1 = __atomic_fetch_or_4 (ptr_6, 1, _3);
658 : _2 = (int) _1;
659 : _7 = ~_2;
660 : _5 = (_Bool) _7;
661 : to
662 : _1 = __atomic_fetch_or_4 (ptr_6, ~1, _3);
663 : _8 = _1 & 1;
664 : _5 = _8 == 0;
665 : and convert
666 : _1 = __atomic_fetch_and_4 (ptr_6, ~1, _3);
667 : _2 = (int) _1;
668 : _7 = ~_2;
669 : _5 = (_Bool) _7;
670 : to
671 : _1 = __atomic_fetch_and_4 (ptr_6, 1, _3);
672 : _8 = _1 & 1;
673 : _5 = _8 == 0;
674 : */
675 0 : gsi = gsi_for_stmt (use_stmt);
676 0 : gsi_remove (&gsi, true);
677 0 : use_stmt = g;
678 0 : ibit = 0;
679 : }
680 : else
681 : {
682 396 : tree cmp_rhs1, cmp_rhs2;
683 396 : if (use_nop_lhs)
684 : {
685 : /* Handle
686 : _4 = _5 < 0;
687 : */
688 387 : if (TREE_CODE (TREE_TYPE (use_nop_lhs))
689 : != BOOLEAN_TYPE)
690 446 : return false;
691 51 : cmp_rhs1 = gimple_assign_rhs1 (use_nop_stmt);
692 51 : cmp_rhs2 = gimple_assign_rhs2 (use_nop_stmt);
693 : }
694 : else
695 : {
696 : /* Handle
697 : if (_5 < 0)
698 : */
699 9 : rhs_code = gimple_cond_code (use_nop_stmt);
700 9 : cmp_rhs1 = gimple_cond_lhs (use_nop_stmt);
701 9 : cmp_rhs2 = gimple_cond_rhs (use_nop_stmt);
702 : }
703 60 : if (rhs_code != GE_EXPR && rhs_code != LT_EXPR)
704 : return false;
705 48 : if (use_lhs != cmp_rhs1)
706 : return false;
707 48 : if (!integer_zerop (cmp_rhs2))
708 : return false;
709 :
710 48 : tree and_mask;
711 :
712 48 : unsigned HOST_WIDE_INT bytes
713 48 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (use_rhs)));
714 48 : ibit = bytes * BITS_PER_UNIT - 1;
715 48 : unsigned HOST_WIDE_INT highest
716 48 : = HOST_WIDE_INT_1U << ibit;
717 :
718 48 : if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
719 : {
720 : /* Get the signed maximum of the USE_RHS type. */
721 19 : and_mask = build_int_cst (TREE_TYPE (use_rhs),
722 19 : highest - 1);
723 19 : if (!operand_equal_p (and_mask, mask, 0))
724 : return false;
725 :
726 : /* Convert
727 : _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3);
728 : _5 = (signed int) _1;
729 : _4 = _5 < 0 or _5 >= 0;
730 : to
731 : _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3);
732 : _6 = _1 & 0x80000000;
733 : _4 = _6 != 0 or _6 == 0;
734 : and convert
735 : _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3);
736 : _5 = (signed int) _1;
737 : if (_5 < 0 or _5 >= 0)
738 : to
739 : _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3);
740 : _6 = _1 & 0x80000000;
741 : if (_6 != 0 or _6 == 0)
742 : */
743 19 : and_mask = build_int_cst (TREE_TYPE (use_rhs),
744 19 : highest);
745 : }
746 : else
747 : {
748 : /* Get the signed minimum of the USE_RHS type. */
749 29 : and_mask = build_int_cst (TREE_TYPE (use_rhs),
750 29 : highest);
751 29 : if (!operand_equal_p (and_mask, mask, 0))
752 : return false;
753 :
754 : /* Convert
755 : _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3);
756 : _5 = (signed int) _1;
757 : _4 = _5 < 0 or _5 >= 0;
758 : to
759 : _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3);
760 : _6 = _1 & 0x80000000;
761 : _4 = _6 != 0 or _6 == 0;
762 : and convert
763 : _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3);
764 : _5 = (signed int) _1;
765 : if (_5 < 0 or _5 >= 0)
766 : to
767 : _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3);
768 : _6 = _1 & 0x80000000;
769 : if (_6 != 0 or _6 == 0)
770 : */
771 : }
772 36 : var = make_ssa_name (TREE_TYPE (use_rhs));
773 36 : gimple* use_stmt_removal = use_stmt;
774 36 : g = gimple_build_assign (var, BIT_AND_EXPR, use_rhs,
775 : and_mask);
776 36 : gsi = gsi_for_stmt (use_nop_stmt);
777 36 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
778 36 : use_stmt = g;
779 36 : rhs_code = rhs_code == GE_EXPR ? EQ_EXPR : NE_EXPR;
780 36 : tree const_zero = build_zero_cst (TREE_TYPE (use_rhs));
781 36 : if (use_nop_lhs)
782 27 : g = gimple_build_assign (use_nop_lhs, rhs_code,
783 : var, const_zero);
784 : else
785 9 : g = gimple_build_cond (rhs_code, var, const_zero,
786 : nullptr, nullptr);
787 36 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
788 36 : gsi = gsi_for_stmt (use_nop_stmt);
789 36 : gsi_remove (&gsi, true);
790 36 : gsi = gsi_for_stmt (use_stmt_removal);
791 36 : gsi_remove (&gsi, true);
792 : }
793 : }
794 : else
795 : {
796 94 : tree match_op[3];
797 94 : gimple *g;
798 94 : if (!gimple_nop_atomic_bit_test_and_p (use_nop_lhs,
799 : &match_op[0], NULL)
800 92 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (match_op[2])
801 92 : || !single_imm_use (match_op[2], &use_p, &g)
802 186 : || !is_gimple_assign (g))
803 2 : return false;
804 92 : mask = match_op[0];
805 92 : if (TREE_CODE (match_op[1]) == INTEGER_CST)
806 : {
807 48 : ibit = tree_log2 (match_op[1]);
808 48 : gcc_assert (ibit >= 0);
809 : }
810 : else
811 : {
812 44 : g = SSA_NAME_DEF_STMT (match_op[1]);
813 44 : gcc_assert (is_gimple_assign (g));
814 44 : bit = gimple_assign_rhs2 (g);
815 : }
816 : /* Convert
817 : _1 = __atomic_fetch_or_4 (ptr_6, mask, _3);
818 : _2 = (int) _1;
819 : _5 = _2 & mask;
820 : to
821 : _1 = __atomic_fetch_or_4 (ptr_6, mask, _3);
822 : _6 = _1 & mask;
823 : _5 = (int) _6;
824 : and convert
825 : _1 = ~mask_7;
826 : _2 = (unsigned int) _1;
827 : _3 = __atomic_fetch_and_4 (ptr_6, _2, 0);
828 : _4 = (int) _3;
829 : _5 = _4 & mask_7;
830 : to
831 : _1 = __atomic_fetch_and_* (ptr_6, ~mask_7, _3);
832 : _12 = _3 & mask_7;
833 : _5 = (int) _12;
834 :
835 : and Convert
836 : _1 = __atomic_fetch_and_4 (ptr_6, ~mask, _3);
837 : _2 = (short int) _1;
838 : _5 = _2 & mask;
839 : to
840 : _1 = __atomic_fetch_and_4 (ptr_6, ~mask, _3);
841 : _8 = _1 & mask;
842 : _5 = (short int) _8;
843 : */
844 92 : gimple_seq stmts = NULL;
845 92 : match_op[1] = gimple_convert (&stmts,
846 92 : TREE_TYPE (use_rhs),
847 : match_op[1]);
848 92 : var = gimple_build (&stmts, BIT_AND_EXPR,
849 92 : TREE_TYPE (use_rhs), use_rhs, match_op[1]);
850 92 : gsi = gsi_for_stmt (use_stmt);
851 92 : gsi_remove (&gsi, true);
852 92 : release_defs (use_stmt);
853 92 : use_stmt = gimple_seq_last_stmt (stmts);
854 92 : gsi = gsi_for_stmt (use_nop_stmt);
855 92 : gsi_insert_seq_before (&gsi, stmts, GSI_SAME_STMT);
856 92 : gimple_assign_set_rhs_with_ops (&gsi, CONVERT_EXPR, var);
857 92 : update_stmt (use_nop_stmt);
858 : }
859 : }
860 : else
861 : return false;
862 :
863 157 : if (!bit)
864 : {
865 113 : if (ibit < 0)
866 0 : gcc_unreachable ();
867 113 : bit = build_int_cst (TREE_TYPE (lhs), ibit);
868 : }
869 : }
870 292 : else if (optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)))
871 : == CODE_FOR_nothing)
872 : return false;
873 :
874 443 : tree use_lhs = gimple_assign_lhs (use_stmt);
875 443 : if (!use_lhs)
876 : return false;
877 :
878 443 : if (!bit)
879 : {
880 286 : if (TREE_CODE (mask) == INTEGER_CST)
881 : {
882 222 : if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
883 62 : mask = const_unop (BIT_NOT_EXPR, TREE_TYPE (mask), mask);
884 222 : mask = fold_convert (TREE_TYPE (lhs), mask);
885 222 : int ibit = tree_log2 (mask);
886 222 : if (ibit < 0)
887 16 : return false;
888 220 : bit = build_int_cst (TREE_TYPE (lhs), ibit);
889 : }
890 64 : else if (TREE_CODE (mask) == SSA_NAME)
891 : {
892 64 : gimple *g = SSA_NAME_DEF_STMT (mask);
893 64 : tree match_op;
894 64 : if (gimple_nop_convert (mask, &match_op, NULL))
895 : {
896 3 : mask = match_op;
897 3 : if (TREE_CODE (mask) != SSA_NAME)
898 7 : return false;
899 3 : g = SSA_NAME_DEF_STMT (mask);
900 : }
901 64 : if (!is_gimple_assign (g))
902 : return false;
903 :
904 62 : if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
905 : {
906 20 : if (gimple_assign_rhs_code (g) != BIT_NOT_EXPR)
907 : return false;
908 20 : mask = gimple_assign_rhs1 (g);
909 20 : if (TREE_CODE (mask) != SSA_NAME)
910 : return false;
911 20 : g = SSA_NAME_DEF_STMT (mask);
912 : }
913 :
914 62 : if (!is_gimple_assign (g)
915 57 : || gimple_assign_rhs_code (g) != LSHIFT_EXPR
916 119 : || !integer_onep (gimple_assign_rhs1 (g)))
917 5 : return false;
918 57 : bit = gimple_assign_rhs2 (g);
919 : }
920 : else
921 : return false;
922 :
923 277 : tree cmp_mask;
924 277 : if (gimple_assign_rhs1 (use_stmt) == lhs)
925 241 : cmp_mask = gimple_assign_rhs2 (use_stmt);
926 : else
927 : cmp_mask = gimple_assign_rhs1 (use_stmt);
928 :
929 277 : tree match_op;
930 277 : if (gimple_nop_convert (cmp_mask, &match_op, NULL))
931 1 : cmp_mask = match_op;
932 :
933 277 : if (!operand_equal_p (cmp_mask, mask, 0))
934 : return false;
935 : }
936 :
937 427 : bool use_bool = true;
938 427 : bool has_debug_uses = false;
939 427 : imm_use_iterator iter;
940 427 : gimple *g;
941 :
942 427 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs))
943 0 : use_bool = false;
944 1056 : FOR_EACH_IMM_USE_STMT (g, iter, use_lhs)
945 : {
946 428 : enum tree_code code = ERROR_MARK;
947 428 : tree op0 = NULL_TREE, op1 = NULL_TREE;
948 428 : if (is_gimple_debug (g))
949 : {
950 1 : has_debug_uses = true;
951 1 : continue;
952 : }
953 427 : else if (is_gimple_assign (g))
954 385 : switch (gimple_assign_rhs_code (g))
955 : {
956 0 : case COND_EXPR:
957 0 : op1 = gimple_assign_rhs1 (g);
958 0 : code = TREE_CODE (op1);
959 0 : if (TREE_CODE_CLASS (code) != tcc_comparison)
960 : break;
961 0 : op0 = TREE_OPERAND (op1, 0);
962 0 : op1 = TREE_OPERAND (op1, 1);
963 0 : break;
964 173 : case EQ_EXPR:
965 173 : case NE_EXPR:
966 173 : code = gimple_assign_rhs_code (g);
967 173 : op0 = gimple_assign_rhs1 (g);
968 173 : op1 = gimple_assign_rhs2 (g);
969 173 : break;
970 : default:
971 : break;
972 : }
973 42 : else if (gimple_code (g) == GIMPLE_COND)
974 : {
975 28 : code = gimple_cond_code (g);
976 28 : op0 = gimple_cond_lhs (g);
977 28 : op1 = gimple_cond_rhs (g);
978 : }
979 :
980 201 : if ((code == EQ_EXPR || code == NE_EXPR)
981 201 : && op0 == use_lhs
982 402 : && integer_zerop (op1))
983 : {
984 201 : use_operand_p use_p;
985 201 : int n = 0;
986 402 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
987 201 : n++;
988 201 : if (n == 1)
989 201 : continue;
990 : }
991 :
992 : use_bool = false;
993 : break;
994 427 : }
995 :
996 427 : tree new_lhs = make_ssa_name (TREE_TYPE (lhs));
997 427 : tree flag = build_int_cst (TREE_TYPE (lhs), use_bool);
998 427 : if (has_model_arg)
999 296 : g = gimple_build_call_internal (fn, 5, gimple_call_arg (call, 0),
1000 : bit, flag, gimple_call_arg (call, 2),
1001 : gimple_call_fn (call));
1002 : else
1003 131 : g = gimple_build_call_internal (fn, 4, gimple_call_arg (call, 0),
1004 : bit, flag, gimple_call_fn (call));
1005 427 : gimple_call_set_lhs (g, new_lhs);
1006 427 : gimple_set_location (g, gimple_location (call));
1007 427 : gimple_move_vops (g, call);
1008 427 : bool throws = stmt_can_throw_internal (cfun, call);
1009 427 : gimple_call_set_nothrow (as_a <gcall *> (g),
1010 427 : gimple_call_nothrow_p (as_a <gcall *> (call)));
1011 427 : gimple_stmt_iterator gsi = *gsip;
1012 427 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
1013 427 : edge e = NULL;
1014 427 : if (throws)
1015 : {
1016 75 : maybe_clean_or_replace_eh_stmt (call, g);
1017 75 : if (after || (use_bool && has_debug_uses))
1018 9 : e = find_fallthru_edge (gsi_bb (gsi)->succs);
1019 : }
1020 427 : if (after)
1021 : {
1022 : /* The internal function returns the value of the specified bit
1023 : before the atomic operation. If we are interested in the value
1024 : of the specified bit after the atomic operation (makes only sense
1025 : for xor, otherwise the bit content is compile time known),
1026 : we need to invert the bit. */
1027 55 : tree mask_convert = mask;
1028 55 : gimple_seq stmts = NULL;
1029 55 : if (!use_bool)
1030 43 : mask_convert = gimple_convert (&stmts, TREE_TYPE (lhs), mask);
1031 55 : new_lhs = gimple_build (&stmts, BIT_XOR_EXPR, TREE_TYPE (lhs), new_lhs,
1032 12 : use_bool ? build_int_cst (TREE_TYPE (lhs), 1)
1033 : : mask_convert);
1034 55 : if (throws)
1035 : {
1036 9 : gsi_insert_seq_on_edge_immediate (e, stmts);
1037 18 : gsi = gsi_for_stmt (gimple_seq_last (stmts));
1038 : }
1039 : else
1040 46 : gsi_insert_seq_after (&gsi, stmts, GSI_NEW_STMT);
1041 : }
1042 427 : if (use_bool && has_debug_uses)
1043 : {
1044 1 : tree temp = NULL_TREE;
1045 1 : if (!throws || after || single_pred_p (e->dest))
1046 : {
1047 1 : temp = build_debug_expr_decl (TREE_TYPE (lhs));
1048 1 : tree t = build2 (LSHIFT_EXPR, TREE_TYPE (lhs), new_lhs, bit);
1049 1 : g = gimple_build_debug_bind (temp, t, g);
1050 1 : if (throws && !after)
1051 : {
1052 0 : gsi = gsi_after_labels (e->dest);
1053 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1054 : }
1055 : else
1056 1 : gsi_insert_after (&gsi, g, GSI_NEW_STMT);
1057 : }
1058 4 : FOR_EACH_IMM_USE_STMT (g, iter, use_lhs)
1059 2 : if (is_gimple_debug (g))
1060 : {
1061 1 : use_operand_p use_p;
1062 1 : if (temp == NULL_TREE)
1063 0 : gimple_debug_bind_reset_value (g);
1064 : else
1065 3 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1066 1 : SET_USE (use_p, temp);
1067 1 : update_stmt (g);
1068 1 : }
1069 : }
1070 427 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_lhs)
1071 427 : = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs);
1072 427 : replace_uses_by (use_lhs, new_lhs);
1073 427 : gsi = gsi_for_stmt (use_stmt);
1074 427 : gsi_remove (&gsi, true);
1075 427 : release_defs (use_stmt);
1076 427 : gsi_remove (gsip, true);
1077 427 : release_ssa_name (lhs);
1078 427 : return true;
1079 : }
1080 :
1081 : /* Optimize
1082 : _4 = __atomic_add_fetch_* (ptr_6, arg_2, _3);
1083 : _5 = _4 == 0;
1084 : to
1085 : _4 = .ATOMIC_ADD_FETCH_CMP_0 (EQ_EXPR, ptr_6, arg_2, _3);
1086 : _5 = _4;
1087 : Similarly for __sync_add_and_fetch_* (without the ", _3" part
1088 : in there). */
1089 :
1090 : static bool
1091 8753 : optimize_atomic_op_fetch_cmp_0 (gimple_stmt_iterator *gsip,
1092 : enum internal_fn fn, bool has_model_arg)
1093 : {
1094 8753 : gimple *call = gsi_stmt (*gsip);
1095 8753 : tree lhs = gimple_call_lhs (call);
1096 8753 : use_operand_p use_p;
1097 8753 : gimple *use_stmt;
1098 :
1099 8753 : if (!flag_inline_atomics
1100 8753 : || !gimple_call_builtin_p (call, BUILT_IN_NORMAL)
1101 8753 : || !lhs
1102 6236 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs)
1103 6236 : || !single_imm_use (lhs, &use_p, &use_stmt)
1104 14815 : || !gimple_vdef (call))
1105 2691 : return false;
1106 :
1107 6062 : optab optab;
1108 6062 : switch (fn)
1109 : {
1110 : case IFN_ATOMIC_ADD_FETCH_CMP_0:
1111 : optab = atomic_add_fetch_cmp_0_optab;
1112 : break;
1113 : case IFN_ATOMIC_SUB_FETCH_CMP_0:
1114 : optab = atomic_sub_fetch_cmp_0_optab;
1115 : break;
1116 : case IFN_ATOMIC_AND_FETCH_CMP_0:
1117 : optab = atomic_and_fetch_cmp_0_optab;
1118 : break;
1119 : case IFN_ATOMIC_OR_FETCH_CMP_0:
1120 : optab = atomic_or_fetch_cmp_0_optab;
1121 : break;
1122 : case IFN_ATOMIC_XOR_FETCH_CMP_0:
1123 : optab = atomic_xor_fetch_cmp_0_optab;
1124 : break;
1125 : default:
1126 : return false;
1127 : }
1128 :
1129 6062 : if (optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)))
1130 : == CODE_FOR_nothing)
1131 : return false;
1132 :
1133 6044 : tree use_lhs = lhs;
1134 6044 : if (gimple_assign_cast_p (use_stmt))
1135 : {
1136 957 : use_lhs = gimple_assign_lhs (use_stmt);
1137 957 : if (!tree_nop_conversion_p (TREE_TYPE (use_lhs), TREE_TYPE (lhs))
1138 943 : || (!INTEGRAL_TYPE_P (TREE_TYPE (use_lhs))
1139 91 : && !POINTER_TYPE_P (TREE_TYPE (use_lhs)))
1140 943 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs)
1141 1900 : || !single_imm_use (use_lhs, &use_p, &use_stmt))
1142 83 : return false;
1143 : }
1144 5961 : enum tree_code code = ERROR_MARK;
1145 5961 : tree op0 = NULL_TREE, op1 = NULL_TREE;
1146 5961 : if (is_gimple_assign (use_stmt))
1147 1289 : switch (gimple_assign_rhs_code (use_stmt))
1148 : {
1149 0 : case COND_EXPR:
1150 0 : op1 = gimple_assign_rhs1 (use_stmt);
1151 0 : code = TREE_CODE (op1);
1152 0 : if (TREE_CODE_CLASS (code) == tcc_comparison)
1153 : {
1154 0 : op0 = TREE_OPERAND (op1, 0);
1155 0 : op1 = TREE_OPERAND (op1, 1);
1156 : }
1157 : break;
1158 1289 : default:
1159 1289 : code = gimple_assign_rhs_code (use_stmt);
1160 1289 : if (TREE_CODE_CLASS (code) == tcc_comparison)
1161 : {
1162 842 : op0 = gimple_assign_rhs1 (use_stmt);
1163 842 : op1 = gimple_assign_rhs2 (use_stmt);
1164 : }
1165 : break;
1166 : }
1167 4672 : else if (gimple_code (use_stmt) == GIMPLE_COND)
1168 : {
1169 4157 : code = gimple_cond_code (use_stmt);
1170 4157 : op0 = gimple_cond_lhs (use_stmt);
1171 4157 : op1 = gimple_cond_rhs (use_stmt);
1172 : }
1173 :
1174 5446 : switch (code)
1175 : {
1176 243 : case LT_EXPR:
1177 243 : case LE_EXPR:
1178 243 : case GT_EXPR:
1179 243 : case GE_EXPR:
1180 486 : if (!INTEGRAL_TYPE_P (TREE_TYPE (use_lhs))
1181 243 : || TREE_CODE (TREE_TYPE (use_lhs)) == BOOLEAN_TYPE
1182 486 : || TYPE_UNSIGNED (TREE_TYPE (use_lhs)))
1183 : return false;
1184 : /* FALLTHRU */
1185 4999 : case EQ_EXPR:
1186 4999 : case NE_EXPR:
1187 4999 : if (op0 == use_lhs && integer_zerop (op1))
1188 : break;
1189 : return false;
1190 : default:
1191 : return false;
1192 : }
1193 :
1194 2087 : int encoded;
1195 2087 : switch (code)
1196 : {
1197 : /* Use special encoding of the operation. We want to also
1198 : encode the mode in the first argument and for neither EQ_EXPR
1199 : etc. nor EQ etc. we can rely it will fit into QImode. */
1200 : case EQ_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_EQ; break;
1201 877 : case NE_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_NE; break;
1202 106 : case LT_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_LT; break;
1203 40 : case LE_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_LE; break;
1204 40 : case GT_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_GT; break;
1205 48 : case GE_EXPR: encoded = ATOMIC_OP_FETCH_CMP_0_GE; break;
1206 0 : default: gcc_unreachable ();
1207 : }
1208 :
1209 2087 : tree new_lhs = make_ssa_name (boolean_type_node);
1210 2087 : gimple *g;
1211 2087 : tree flag = build_int_cst (TREE_TYPE (lhs), encoded);
1212 2087 : if (has_model_arg)
1213 1679 : g = gimple_build_call_internal (fn, 5, flag,
1214 : gimple_call_arg (call, 0),
1215 : gimple_call_arg (call, 1),
1216 : gimple_call_arg (call, 2),
1217 : gimple_call_fn (call));
1218 : else
1219 408 : g = gimple_build_call_internal (fn, 4, flag,
1220 : gimple_call_arg (call, 0),
1221 : gimple_call_arg (call, 1),
1222 : gimple_call_fn (call));
1223 2087 : gimple_call_set_lhs (g, new_lhs);
1224 2087 : gimple_set_location (g, gimple_location (call));
1225 2087 : gimple_move_vops (g, call);
1226 2087 : bool throws = stmt_can_throw_internal (cfun, call);
1227 2087 : gimple_call_set_nothrow (as_a <gcall *> (g),
1228 2087 : gimple_call_nothrow_p (as_a <gcall *> (call)));
1229 2087 : gimple_stmt_iterator gsi = *gsip;
1230 2087 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
1231 2087 : if (throws)
1232 0 : maybe_clean_or_replace_eh_stmt (call, g);
1233 2087 : if (is_gimple_assign (use_stmt))
1234 816 : switch (gimple_assign_rhs_code (use_stmt))
1235 : {
1236 0 : case COND_EXPR:
1237 0 : gimple_assign_set_rhs1 (use_stmt, new_lhs);
1238 0 : break;
1239 816 : default:
1240 816 : gsi = gsi_for_stmt (use_stmt);
1241 816 : if (tree ulhs = gimple_assign_lhs (use_stmt))
1242 816 : if (useless_type_conversion_p (TREE_TYPE (ulhs),
1243 : boolean_type_node))
1244 : {
1245 816 : gimple_assign_set_rhs_with_ops (&gsi, SSA_NAME, new_lhs);
1246 816 : break;
1247 : }
1248 0 : gimple_assign_set_rhs_with_ops (&gsi, NOP_EXPR, new_lhs);
1249 0 : break;
1250 : }
1251 1271 : else if (gimple_code (use_stmt) == GIMPLE_COND)
1252 : {
1253 1271 : gcond *use_cond = as_a <gcond *> (use_stmt);
1254 1271 : gimple_cond_set_code (use_cond, NE_EXPR);
1255 1271 : gimple_cond_set_lhs (use_cond, new_lhs);
1256 1271 : gimple_cond_set_rhs (use_cond, boolean_false_node);
1257 : }
1258 :
1259 2087 : update_stmt (use_stmt);
1260 2087 : if (use_lhs != lhs)
1261 : {
1262 234 : gsi = gsi_for_stmt (SSA_NAME_DEF_STMT (use_lhs));
1263 234 : gsi_remove (&gsi, true);
1264 234 : release_ssa_name (use_lhs);
1265 : }
1266 2087 : gsi_remove (gsip, true);
1267 2087 : release_ssa_name (lhs);
1268 2087 : return true;
1269 : }
1270 :
1271 : /* Process builtin CALL located at GSI.
1272 : Currently it is only fgr atomic functions optimizations from above. */
1273 : static void
1274 6845747 : gimple_isel_builtin_call (gcall *call, gimple_stmt_iterator *gsi)
1275 : {
1276 : /* Don't handle these in non optimization mode or optimize debug mode. */
1277 6845747 : if (!optimize || optimize_debug)
1278 : return;
1279 :
1280 5327463 : if (!gimple_call_builtin_p (call, BUILT_IN_NORMAL))
1281 : return;
1282 :
1283 1344670 : tree callee = gimple_call_fndecl (call);
1284 :
1285 1344670 : switch (DECL_FUNCTION_CODE (callee))
1286 : {
1287 : #define CASE_ATOMIC(NAME) \
1288 : case BUILT_IN_##NAME##_1: \
1289 : case BUILT_IN_##NAME##_2: \
1290 : case BUILT_IN_##NAME##_4: \
1291 : case BUILT_IN_##NAME##_8: \
1292 : case BUILT_IN_##NAME##_16
1293 : #define CASE_ATOMIC_CMP0(ATOMIC, SYNC) \
1294 : CASE_ATOMIC(ATOMIC_##ATOMIC): \
1295 : optimize_atomic_op_fetch_cmp_0 (gsi, \
1296 : IFN_ATOMIC_##ATOMIC##_CMP_0, \
1297 : true); \
1298 : break; \
1299 : CASE_ATOMIC(SYNC_##SYNC): \
1300 : optimize_atomic_op_fetch_cmp_0 (gsi, \
1301 : IFN_ATOMIC_##ATOMIC##_CMP_0, \
1302 : false); \
1303 : break;
1304 :
1305 :
1306 4112 : CASE_ATOMIC_CMP0(ADD_FETCH, ADD_AND_FETCH)
1307 2381 : CASE_ATOMIC_CMP0(SUB_FETCH, SUB_AND_FETCH)
1308 773 : CASE_ATOMIC_CMP0(AND_FETCH, AND_AND_FETCH)
1309 769 : CASE_ATOMIC_CMP0(OR_FETCH, OR_AND_FETCH)
1310 : #define CASE_ATOMIC_BIT_TEST_AND(ATOMIC, SYNC, FN, AFTER) \
1311 : CASE_ATOMIC(ATOMIC_##ATOMIC): \
1312 : optimize_atomic_bit_test_and (gsi, \
1313 : IFN_ATOMIC_BIT_TEST_AND_##FN, \
1314 : true, AFTER); \
1315 : break; \
1316 : CASE_ATOMIC(SYNC_##SYNC): \
1317 : optimize_atomic_bit_test_and (gsi, \
1318 : IFN_ATOMIC_BIT_TEST_AND_##FN, \
1319 : false, AFTER); \
1320 : break;
1321 1424 : CASE_ATOMIC_BIT_TEST_AND(FETCH_OR, FETCH_AND_OR, SET, false)
1322 1284 : CASE_ATOMIC_BIT_TEST_AND(FETCH_XOR, FETCH_AND_XOR, COMPLEMENT, false)
1323 1143 : CASE_ATOMIC_BIT_TEST_AND(FETCH_AND, FETCH_AND_AND, RESET, false)
1324 :
1325 573 : CASE_ATOMIC(ATOMIC_XOR_FETCH):
1326 573 : if (optimize_atomic_bit_test_and
1327 573 : (gsi, IFN_ATOMIC_BIT_TEST_AND_COMPLEMENT, true, true))
1328 : break;
1329 535 : optimize_atomic_op_fetch_cmp_0 (gsi,
1330 : IFN_ATOMIC_XOR_FETCH_CMP_0,
1331 : true);
1332 535 : break;
1333 200 : CASE_ATOMIC(SYNC_XOR_AND_FETCH):
1334 200 : if (optimize_atomic_bit_test_and
1335 200 : (gsi, IFN_ATOMIC_BIT_TEST_AND_COMPLEMENT, false, true))
1336 : break;
1337 183 : optimize_atomic_op_fetch_cmp_0 (gsi,
1338 : IFN_ATOMIC_XOR_FETCH_CMP_0,
1339 : false);
1340 183 : break;
1341 :
1342 55 : default:;
1343 : }
1344 : }
1345 :
1346 : /* Iterate all gimple statements and perform pre RTL expansion
1347 : GIMPLE massaging to improve instruction selection. */
1348 :
1349 : unsigned int
1350 1472145 : pass_gimple_isel::execute (struct function *fun)
1351 : {
1352 1472145 : gimple_stmt_iterator gsi;
1353 1472145 : basic_block bb;
1354 1472145 : bool cfg_changed = false;
1355 :
1356 17107488 : FOR_EACH_BB_FN (bb, fun)
1357 : {
1358 127357874 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1359 : {
1360 : /* Give the target first try at replacing the instruction. */
1361 96087188 : cfg_changed |= targetm.instruction_selection (fun, &gsi);
1362 :
1363 : /* Pre-expand VEC_COND_EXPRs to .VCOND* internal function
1364 : calls mapping to supported optabs. */
1365 96087188 : gimple *g = gimple_expand_vec_cond_expr (&gsi);
1366 96087188 : if (g != NULL)
1367 : {
1368 19221 : tree lhs = gimple_assign_lhs (gsi_stmt (gsi));
1369 19221 : gimple_set_lhs (g, lhs);
1370 19221 : gsi_replace (&gsi, g, false);
1371 : }
1372 :
1373 : /* Recognize .VEC_SET and .VEC_EXTRACT patterns. */
1374 96087188 : cfg_changed |= gimple_expand_vec_set_extract_expr (fun, &gsi);
1375 96087188 : if (gsi_end_p (gsi))
1376 : break;
1377 :
1378 96087188 : if (gcall *call = dyn_cast <gcall*>(*gsi))
1379 : {
1380 6845747 : gimple_isel_builtin_call (call, &gsi);
1381 6845747 : continue;
1382 : }
1383 89241441 : gassign *stmt = dyn_cast <gassign *> (*gsi);
1384 89241441 : if (!stmt)
1385 57649628 : continue;
1386 :
1387 31591813 : tree_code code = gimple_assign_rhs_code (stmt);
1388 31591813 : if (TREE_CODE_CLASS (code) == tcc_comparison)
1389 534477 : maybe_duplicate_comparison (stmt, bb);
1390 : }
1391 : }
1392 :
1393 1472145 : return cfg_changed ? TODO_cleanup_cfg : 0;
1394 : }
1395 :
1396 : } // anon namespace
1397 :
1398 : gimple_opt_pass *
1399 285722 : make_pass_gimple_isel (gcc::context *ctxt)
1400 : {
1401 285722 : return new pass_gimple_isel (ctxt);
1402 : }
1403 :
|
