Line data Source code
1 : /* Lower complex number operations to scalar operations.
2 : Copyright (C) 2004-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by the
8 : Free Software Foundation; either version 3, or (at your option) any
9 : later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "cfghooks.h"
29 : #include "tree-pass.h"
30 : #include "ssa.h"
31 : #include "fold-const.h"
32 : #include "stor-layout.h"
33 : #include "tree-eh.h"
34 : #include "gimplify.h"
35 : #include "gimple-iterator.h"
36 : #include "gimplify-me.h"
37 : #include "tree-cfg.h"
38 : #include "tree-dfa.h"
39 : #include "tree-ssa.h"
40 : #include "tree-ssa-propagate.h"
41 : #include "tree-hasher.h"
42 : #include "cfgloop.h"
43 : #include "cfganal.h"
44 : #include "gimple-fold.h"
45 : #include "diagnostic-core.h"
46 : #include "case-cfn-macros.h"
47 : #include "builtins.h"
48 : #include "optabs-tree.h"
49 : #include "tree-ssa-dce.h"
50 :
51 : /* For each complex ssa name, a lattice value. We're interested in finding
52 : out whether a complex number is degenerate in some way, having only real
53 : or only complex parts. */
54 :
55 : enum
56 : {
57 : UNINITIALIZED = 0,
58 : ONLY_REAL = 1,
59 : ONLY_IMAG = 2,
60 : VARYING = 3
61 : };
62 :
63 : /* The type complex_lattice_t holds combinations of the above
64 : constants. */
65 : typedef int complex_lattice_t;
66 :
67 : #define PAIR(a, b) ((a) << 2 | (b))
68 :
69 8463 : class complex_propagate : public ssa_propagation_engine
70 : {
71 : enum ssa_prop_result visit_stmt (gimple *, edge *, tree *) final override;
72 : enum ssa_prop_result visit_phi (gphi *) final override;
73 : };
74 :
75 : static vec<complex_lattice_t> complex_lattice_values;
76 :
77 : /* For each complex variable, a pair of variables for the components exists in
78 : the hashtable. */
79 : static int_tree_htab_type *complex_variable_components;
80 :
81 : /* For each complex SSA_NAME, a pair of ssa names for the components. */
82 : static vec<tree> complex_ssa_name_components;
83 :
84 : /* Vector of PHI triplets (original complex PHI and corresponding real and
85 : imag PHIs if real and/or imag PHIs contain temporarily
86 : non-SSA_NAME/non-invariant args that need to be replaced by SSA_NAMEs. */
87 : static vec<gphi *> phis_to_revisit;
88 :
89 : /* BBs that need EH cleanup. */
90 : static bitmap need_eh_cleanup;
91 :
92 : /* SSA defs we should try to DCE. */
93 : static bitmap dce_worklist;
94 :
95 : /* Lookup UID in the complex_variable_components hashtable and return the
96 : associated tree. */
97 : static tree
98 35659 : cvc_lookup (unsigned int uid)
99 : {
100 35659 : struct int_tree_map in;
101 35659 : in.uid = uid;
102 35659 : return complex_variable_components->find_with_hash (in, uid).to;
103 : }
104 :
105 : /* Insert the pair UID, TO into the complex_variable_components hashtable. */
106 :
107 : static void
108 13059 : cvc_insert (unsigned int uid, tree to)
109 : {
110 13059 : int_tree_map h;
111 13059 : int_tree_map *loc;
112 :
113 13059 : h.uid = uid;
114 13059 : loc = complex_variable_components->find_slot_with_hash (h, uid, INSERT);
115 13059 : loc->uid = uid;
116 13059 : loc->to = to;
117 13059 : }
118 :
119 : /* Return true if T is not a zero constant. In the case of real values,
120 : we're only interested in +0.0. */
121 :
122 : static int
123 114322 : some_nonzerop (tree t)
124 : {
125 114322 : int zerop = false;
126 :
127 : /* Operations with real or imaginary part of a complex number zero
128 : cannot be treated the same as operations with a real or imaginary
129 : operand if we care about the signs of zeros in the result. */
130 114322 : if (TREE_CODE (t) == REAL_CST && !flag_signed_zeros)
131 2431 : zerop = real_identical (&TREE_REAL_CST (t), &dconst0);
132 111891 : else if (TREE_CODE (t) == FIXED_CST)
133 0 : zerop = fixed_zerop (t);
134 111891 : else if (TREE_CODE (t) == INTEGER_CST)
135 9242 : zerop = integer_zerop (t);
136 :
137 114322 : return !zerop;
138 : }
139 :
140 :
141 : /* Compute a lattice value from the components of a complex type REAL
142 : and IMAG. */
143 :
144 : static complex_lattice_t
145 57161 : find_lattice_value_parts (tree real, tree imag)
146 : {
147 57161 : int r, i;
148 57161 : complex_lattice_t ret;
149 :
150 57161 : r = some_nonzerop (real);
151 57161 : i = some_nonzerop (imag);
152 57161 : ret = r * ONLY_REAL + i * ONLY_IMAG;
153 :
154 : /* ??? On occasion we could do better than mapping 0+0i to real, but we
155 : certainly don't want to leave it UNINITIALIZED, which eventually gets
156 : mapped to VARYING. */
157 57161 : if (ret == UNINITIALIZED)
158 : ret = ONLY_REAL;
159 :
160 57161 : return ret;
161 : }
162 :
163 :
164 : /* Compute a lattice value from gimple_val T. */
165 :
166 : static complex_lattice_t
167 711575 : find_lattice_value (tree t)
168 : {
169 711575 : tree real, imag;
170 :
171 711575 : switch (TREE_CODE (t))
172 : {
173 678096 : case SSA_NAME:
174 678096 : return complex_lattice_values[SSA_NAME_VERSION (t)];
175 :
176 33479 : case COMPLEX_CST:
177 33479 : real = TREE_REALPART (t);
178 33479 : imag = TREE_IMAGPART (t);
179 33479 : break;
180 :
181 0 : default:
182 0 : gcc_unreachable ();
183 : }
184 :
185 33479 : return find_lattice_value_parts (real, imag);
186 : }
187 :
188 : /* Determine if LHS is something for which we're interested in seeing
189 : simulation results. */
190 :
191 : static bool
192 39967590 : is_complex_reg (tree lhs)
193 : {
194 39967590 : return TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE && is_gimple_reg (lhs);
195 : }
196 :
197 : /* Mark the incoming parameters to the function as VARYING. */
198 :
199 : static void
200 8463 : init_parameter_lattice_values (void)
201 : {
202 8463 : tree parm, ssa_name;
203 :
204 21005 : for (parm = DECL_ARGUMENTS (cfun->decl); parm ; parm = DECL_CHAIN (parm))
205 12542 : if (is_complex_reg (parm)
206 12542 : && (ssa_name = ssa_default_def (cfun, parm)) != NULL_TREE)
207 2389 : complex_lattice_values[SSA_NAME_VERSION (ssa_name)] = VARYING;
208 8463 : }
209 :
210 : /* Initialize simulation state for each statement. Return false if we
211 : found no statements we want to simulate, and thus there's nothing
212 : for the entire pass to do. */
213 :
214 : static bool
215 1518056 : init_dont_simulate_again (void)
216 : {
217 1518056 : basic_block bb;
218 1518056 : bool saw_a_complex_op = false;
219 :
220 14901180 : FOR_EACH_BB_FN (bb, cfun)
221 : {
222 18560087 : for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
223 5176963 : gsi_next (&gsi))
224 : {
225 5176963 : gphi *phi = gsi.phi ();
226 5176963 : prop_set_simulate_again (phi,
227 : is_complex_reg (gimple_phi_result (phi)));
228 : }
229 :
230 124965132 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
231 98198884 : gsi_next (&gsi))
232 : {
233 98198884 : gimple *stmt;
234 98198884 : tree op0, op1;
235 98198884 : bool sim_again_p;
236 :
237 98198884 : stmt = gsi_stmt (gsi);
238 98198884 : op0 = op1 = NULL_TREE;
239 :
240 : /* Most control-altering statements must be initially
241 : simulated, else we won't cover the entire cfg. */
242 98198884 : sim_again_p = stmt_ends_bb_p (stmt);
243 :
244 98198884 : switch (gimple_code (stmt))
245 : {
246 7245142 : case GIMPLE_CALL:
247 7245142 : if (gimple_call_lhs (stmt))
248 : {
249 2479227 : sim_again_p = is_complex_reg (gimple_call_lhs (stmt));
250 2479227 : switch (gimple_call_combined_fn (stmt))
251 : {
252 1145 : CASE_CFN_CABS:
253 : /* Expand cabs only if unsafe math and optimizing. */
254 1145 : if (optimize && flag_unsafe_math_optimizations)
255 7 : saw_a_complex_op = true;
256 : break;
257 : default:;
258 : }
259 : }
260 : break;
261 :
262 32194295 : case GIMPLE_ASSIGN:
263 32194295 : sim_again_p = is_complex_reg (gimple_assign_lhs (stmt));
264 32194295 : if (gimple_assign_rhs_code (stmt) == REALPART_EXPR
265 32194295 : || gimple_assign_rhs_code (stmt) == IMAGPART_EXPR)
266 234604 : op0 = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
267 : else
268 31959691 : op0 = gimple_assign_rhs1 (stmt);
269 32194295 : if (gimple_num_ops (stmt) > 2)
270 7501051 : op1 = gimple_assign_rhs2 (stmt);
271 : break;
272 :
273 5666734 : case GIMPLE_COND:
274 5666734 : op0 = gimple_cond_lhs (stmt);
275 5666734 : op1 = gimple_cond_rhs (stmt);
276 5666734 : break;
277 :
278 : default:
279 : break;
280 : }
281 :
282 98198884 : if (op0 || op1)
283 37861029 : switch (gimple_expr_code (stmt))
284 : {
285 9195897 : case EQ_EXPR:
286 9195897 : case NE_EXPR:
287 9195897 : case PLUS_EXPR:
288 9195897 : case MINUS_EXPR:
289 9195897 : case MULT_EXPR:
290 9195897 : case TRUNC_DIV_EXPR:
291 9195897 : case CEIL_DIV_EXPR:
292 9195897 : case FLOOR_DIV_EXPR:
293 9195897 : case ROUND_DIV_EXPR:
294 9195897 : case RDIV_EXPR:
295 9195897 : if (TREE_CODE (TREE_TYPE (op0)) == COMPLEX_TYPE
296 9195897 : || TREE_CODE (TREE_TYPE (op1)) == COMPLEX_TYPE)
297 : saw_a_complex_op = true;
298 : break;
299 :
300 54170 : case NEGATE_EXPR:
301 54170 : case CONJ_EXPR:
302 54170 : case PAREN_EXPR:
303 54170 : if (TREE_CODE (TREE_TYPE (op0)) == COMPLEX_TYPE)
304 98198884 : saw_a_complex_op = true;
305 : break;
306 :
307 234604 : case REALPART_EXPR:
308 234604 : case IMAGPART_EXPR:
309 : /* The total store transformation performed during
310 : gimplification creates such uninitialized loads
311 : and we need to lower the statement to be able
312 : to fix things up. */
313 234604 : if (TREE_CODE (op0) == SSA_NAME
314 234604 : && ssa_undefined_value_p (op0))
315 : saw_a_complex_op = true;
316 : break;
317 :
318 28376358 : default:
319 : /* When expand_complex_move would trigger make sure we
320 : perform lowering even when there is no actual complex
321 : operation. This helps consistency and vectorization. */
322 28376358 : if (TREE_CODE (TREE_TYPE (gimple_op (stmt, 0))) == COMPLEX_TYPE)
323 98198884 : saw_a_complex_op = true;
324 : break;
325 : }
326 :
327 98198884 : prop_set_simulate_again (stmt, sim_again_p);
328 : }
329 : }
330 :
331 1518056 : return saw_a_complex_op;
332 : }
333 :
334 :
335 : /* Evaluate statement STMT against the complex lattice defined above. */
336 :
337 : enum ssa_prop_result
338 312654 : complex_propagate::visit_stmt (gimple *stmt, edge *taken_edge_p ATTRIBUTE_UNUSED,
339 : tree *result_p)
340 : {
341 312654 : complex_lattice_t new_l, old_l, op1_l, op2_l;
342 312654 : unsigned int ver;
343 312654 : tree lhs;
344 :
345 312654 : lhs = gimple_get_lhs (stmt);
346 : /* Skip anything but GIMPLE_ASSIGN and GIMPLE_CALL with a lhs. */
347 312654 : if (!lhs)
348 : return SSA_PROP_VARYING;
349 :
350 : /* These conditions should be satisfied due to the initial filter
351 : set up in init_dont_simulate_again. */
352 108594 : gcc_assert (TREE_CODE (lhs) == SSA_NAME);
353 108594 : gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
354 :
355 108594 : *result_p = lhs;
356 108594 : ver = SSA_NAME_VERSION (lhs);
357 108594 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
358 : {
359 15 : complex_lattice_values[ver] = VARYING;
360 15 : return SSA_PROP_VARYING;
361 : }
362 :
363 108579 : old_l = complex_lattice_values[ver];
364 108579 : switch (gimple_expr_code (stmt))
365 : {
366 6042 : case SSA_NAME:
367 6042 : case COMPLEX_CST:
368 6042 : new_l = find_lattice_value (gimple_assign_rhs1 (stmt));
369 6042 : break;
370 :
371 23682 : case COMPLEX_EXPR:
372 23682 : new_l = find_lattice_value_parts (gimple_assign_rhs1 (stmt),
373 : gimple_assign_rhs2 (stmt));
374 23682 : break;
375 :
376 9706 : case PLUS_EXPR:
377 9706 : case MINUS_EXPR:
378 9706 : op1_l = find_lattice_value (gimple_assign_rhs1 (stmt));
379 9706 : op2_l = find_lattice_value (gimple_assign_rhs2 (stmt));
380 :
381 : /* We've set up the lattice values such that IOR neatly
382 : models addition. */
383 9706 : new_l = op1_l | op2_l;
384 9706 : break;
385 :
386 8480 : case MULT_EXPR:
387 8480 : case RDIV_EXPR:
388 8480 : case TRUNC_DIV_EXPR:
389 8480 : case CEIL_DIV_EXPR:
390 8480 : case FLOOR_DIV_EXPR:
391 8480 : case ROUND_DIV_EXPR:
392 8480 : op1_l = find_lattice_value (gimple_assign_rhs1 (stmt));
393 8480 : op2_l = find_lattice_value (gimple_assign_rhs2 (stmt));
394 :
395 : /* Obviously, if either varies, so does the result. */
396 8480 : if (op1_l == VARYING || op2_l == VARYING)
397 : new_l = VARYING;
398 : /* Don't prematurely promote variables if we've not yet seen
399 : their inputs. */
400 39 : else if (op1_l == UNINITIALIZED)
401 : new_l = op2_l;
402 33 : else if (op2_l == UNINITIALIZED)
403 : new_l = op1_l;
404 : else
405 : {
406 : /* At this point both numbers have only one component. If the
407 : numbers are of opposite kind, the result is imaginary,
408 : otherwise the result is real. The add/subtract translates
409 : the real/imag from/to 0/1; the ^ performs the comparison. */
410 33 : new_l = ((op1_l - ONLY_REAL) ^ (op2_l - ONLY_REAL)) + ONLY_REAL;
411 :
412 : /* Don't allow the lattice value to flip-flop indefinitely. */
413 33 : new_l |= old_l;
414 : }
415 : break;
416 :
417 1025 : case NEGATE_EXPR:
418 1025 : case PAREN_EXPR:
419 1025 : case CONJ_EXPR:
420 1025 : new_l = find_lattice_value (gimple_assign_rhs1 (stmt));
421 1025 : break;
422 :
423 : default:
424 : new_l = VARYING;
425 : break;
426 : }
427 :
428 : /* If nothing changed this round, let the propagator know. */
429 108579 : if (new_l == old_l)
430 : return SSA_PROP_NOT_INTERESTING;
431 :
432 108523 : complex_lattice_values[ver] = new_l;
433 108523 : return new_l == VARYING ? SSA_PROP_VARYING : SSA_PROP_INTERESTING;
434 : }
435 :
436 : /* Evaluate a PHI node against the complex lattice defined above. */
437 :
438 : enum ssa_prop_result
439 8039 : complex_propagate::visit_phi (gphi *phi)
440 : {
441 8039 : complex_lattice_t new_l, old_l;
442 8039 : unsigned int ver;
443 8039 : tree lhs;
444 8039 : int i;
445 :
446 8039 : lhs = gimple_phi_result (phi);
447 :
448 : /* This condition should be satisfied due to the initial filter
449 : set up in init_dont_simulate_again. */
450 8039 : gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
451 :
452 8039 : ver = SSA_NAME_VERSION (lhs);
453 8039 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
454 : {
455 12 : complex_lattice_values[ver] = VARYING;
456 12 : return SSA_PROP_VARYING;
457 : }
458 :
459 : /* We've set up the lattice values such that IOR neatly models PHI meet. */
460 8027 : new_l = UNINITIALIZED;
461 24497 : for (i = gimple_phi_num_args (phi) - 1; i >= 0; --i)
462 16470 : new_l |= find_lattice_value (gimple_phi_arg_def (phi, i));
463 :
464 8027 : old_l = complex_lattice_values[ver];
465 :
466 8027 : if (new_l == old_l)
467 : return SSA_PROP_NOT_INTERESTING;
468 :
469 7939 : complex_lattice_values[ver] = new_l;
470 7939 : return new_l == VARYING ? SSA_PROP_VARYING : SSA_PROP_INTERESTING;
471 : }
472 :
473 : /* Create one backing variable for a complex component of ORIG. */
474 :
475 : static tree
476 13059 : create_one_component_var (tree type, tree orig, const char *prefix,
477 : const char *suffix, enum tree_code code)
478 : {
479 13059 : tree r = create_tmp_var (type, prefix);
480 :
481 13059 : DECL_SOURCE_LOCATION (r) = DECL_SOURCE_LOCATION (orig);
482 13059 : DECL_ARTIFICIAL (r) = 1;
483 :
484 13059 : if (DECL_NAME (orig) && !DECL_IGNORED_P (orig))
485 : {
486 12996 : const char *name = IDENTIFIER_POINTER (DECL_NAME (orig));
487 12996 : name = ACONCAT ((name, suffix, NULL));
488 12996 : DECL_NAME (r) = get_identifier (name);
489 :
490 12996 : SET_DECL_DEBUG_EXPR (r, build1 (code, type, orig));
491 12996 : DECL_HAS_DEBUG_EXPR_P (r) = 1;
492 12996 : DECL_IGNORED_P (r) = 0;
493 12996 : copy_warning (r, orig);
494 : }
495 : else
496 : {
497 63 : DECL_IGNORED_P (r) = 1;
498 63 : suppress_warning (r);
499 : }
500 :
501 13059 : return r;
502 : }
503 :
504 : /* Retrieve a value for a complex component of VAR. */
505 :
506 : static tree
507 35659 : get_component_var (tree var, bool imag_p)
508 : {
509 35659 : size_t decl_index = DECL_UID (var) * 2 + imag_p;
510 35659 : tree ret = cvc_lookup (decl_index);
511 :
512 35659 : if (ret == NULL)
513 : {
514 19615 : ret = create_one_component_var (TREE_TYPE (TREE_TYPE (var)), var,
515 : imag_p ? "CI" : "CR",
516 : imag_p ? "$imag" : "$real",
517 : imag_p ? IMAGPART_EXPR : REALPART_EXPR);
518 13059 : cvc_insert (decl_index, ret);
519 : }
520 :
521 35659 : return ret;
522 : }
523 :
524 : /* Retrieve a value for a complex component of SSA_NAME. */
525 :
526 : static tree
527 340978 : get_component_ssa_name (tree ssa_name, bool imag_p)
528 : {
529 340978 : complex_lattice_t lattice = find_lattice_value (ssa_name);
530 340978 : size_t ssa_name_index;
531 340978 : tree ret;
532 :
533 511871 : if (lattice == (imag_p ? ONLY_REAL : ONLY_IMAG))
534 : {
535 1163 : tree inner_type = TREE_TYPE (TREE_TYPE (ssa_name));
536 1163 : if (SCALAR_FLOAT_TYPE_P (inner_type))
537 45 : return build_real (inner_type, dconst0);
538 : else
539 1118 : return build_int_cst (inner_type, 0);
540 : }
541 :
542 339815 : ssa_name_index = SSA_NAME_VERSION (ssa_name) * 2 + imag_p;
543 339815 : ret = complex_ssa_name_components[ssa_name_index];
544 339815 : if (ret == NULL)
545 : {
546 98066 : if (SSA_NAME_VAR (ssa_name))
547 26144 : ret = get_component_var (SSA_NAME_VAR (ssa_name), imag_p);
548 : else
549 71922 : ret = TREE_TYPE (TREE_TYPE (ssa_name));
550 98066 : ret = make_ssa_name (ret);
551 :
552 : /* Copy some properties from the original. In particular, whether it
553 : is used in an abnormal phi, and whether it's uninitialized. */
554 98066 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (ret)
555 98066 : = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (ssa_name);
556 98066 : if (SSA_NAME_IS_DEFAULT_DEF (ssa_name)
557 98066 : && VAR_P (SSA_NAME_VAR (ssa_name)))
558 : {
559 183 : SSA_NAME_DEF_STMT (ret) = SSA_NAME_DEF_STMT (ssa_name);
560 183 : set_ssa_default_def (cfun, SSA_NAME_VAR (ret), ret);
561 : }
562 :
563 98066 : complex_ssa_name_components[ssa_name_index] = ret;
564 : }
565 :
566 : return ret;
567 : }
568 :
569 : /* Set a value for a complex component of SSA_NAME, return a
570 : gimple_seq of stuff that needs doing. */
571 :
572 : static gimple_seq
573 221964 : set_component_ssa_name (tree ssa_name, bool imag_p, tree value)
574 : {
575 221964 : complex_lattice_t lattice = find_lattice_value (ssa_name);
576 221964 : size_t ssa_name_index;
577 221964 : tree comp;
578 221964 : gimple *last;
579 221964 : gimple_seq list;
580 :
581 : /* We know the value must be zero, else there's a bug in our lattice
582 : analysis. But the value may well be a variable known to contain
583 : zero. We should be safe ignoring it. */
584 332946 : if (lattice == (imag_p ? ONLY_REAL : ONLY_IMAG))
585 : return NULL;
586 :
587 : /* If we've already assigned an SSA_NAME to this component, then this
588 : means that our walk of the basic blocks found a use before the set.
589 : This is fine. Now we should create an initialization for the value
590 : we created earlier. */
591 221007 : ssa_name_index = SSA_NAME_VERSION (ssa_name) * 2 + imag_p;
592 221007 : comp = complex_ssa_name_components[ssa_name_index];
593 221007 : if (comp)
594 : ;
595 :
596 : /* If we've nothing assigned, and the value we're given is already stable,
597 : then install that as the value for this SSA_NAME. This preemptively
598 : copy-propagates the value, which avoids unnecessary memory allocation. */
599 214822 : else if (is_gimple_min_invariant (value)
600 224204 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (ssa_name))
601 : {
602 9371 : complex_ssa_name_components[ssa_name_index] = value;
603 9371 : return NULL;
604 : }
605 205451 : else if (TREE_CODE (value) == SSA_NAME
606 334175 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (ssa_name))
607 : {
608 : /* Replace an anonymous base value with the variable from cvc_lookup.
609 : This should result in better debug info. */
610 128705 : if (!SSA_NAME_IS_DEFAULT_DEF (value)
611 128333 : && SSA_NAME_VAR (ssa_name)
612 10670 : && (!SSA_NAME_VAR (value) || DECL_IGNORED_P (SSA_NAME_VAR (value)))
613 138685 : && !DECL_IGNORED_P (SSA_NAME_VAR (ssa_name)))
614 : {
615 9515 : comp = get_component_var (SSA_NAME_VAR (ssa_name), imag_p);
616 9515 : replace_ssa_name_symbol (value, comp);
617 : }
618 :
619 128705 : complex_ssa_name_components[ssa_name_index] = value;
620 128705 : return NULL;
621 : }
622 :
623 : /* Finally, we need to stabilize the result by installing the value into
624 : a new ssa name. */
625 : else
626 76746 : comp = get_component_ssa_name (ssa_name, imag_p);
627 :
628 : /* Do all the work to assign VALUE to COMP. */
629 82931 : list = NULL;
630 82931 : value = force_gimple_operand (value, &list, false, NULL);
631 82931 : last = gimple_build_assign (comp, value);
632 82931 : gimple_seq_add_stmt (&list, last);
633 82931 : gcc_assert (SSA_NAME_DEF_STMT (comp) == last);
634 :
635 82931 : return list;
636 : }
637 :
638 : /* Extract the real or imaginary part of a complex variable or constant.
639 : Make sure that it's a proper gimple_val and gimplify it if not.
640 : Emit any new code before gsi. */
641 :
642 : static tree
643 411694 : extract_component (gimple_stmt_iterator *gsi, tree t, bool imagpart_p,
644 : bool gimple_p, bool phiarg_p = false)
645 : {
646 411694 : switch (TREE_CODE (t))
647 : {
648 72232 : case COMPLEX_CST:
649 72232 : return imagpart_p ? TREE_IMAGPART (t) : TREE_REALPART (t);
650 :
651 0 : case COMPLEX_EXPR:
652 0 : gcc_unreachable ();
653 :
654 108 : case BIT_FIELD_REF:
655 108 : {
656 108 : tree inner_type = TREE_TYPE (TREE_TYPE (t));
657 108 : t = unshare_expr (t);
658 108 : TREE_TYPE (t) = inner_type;
659 108 : TREE_OPERAND (t, 1) = TYPE_SIZE (inner_type);
660 108 : if (imagpart_p)
661 54 : TREE_OPERAND (t, 2) = size_binop (PLUS_EXPR, TREE_OPERAND (t, 2),
662 : TYPE_SIZE (inner_type));
663 :
664 108 : if (gimple_p)
665 : {
666 108 : tree new_lhs = make_ssa_name (inner_type);
667 108 : gimple *new_load = gimple_build_assign (new_lhs, t);
668 108 : gsi_insert_before (gsi, new_load, GSI_SAME_STMT);
669 108 : t = new_lhs;
670 : }
671 : return t;
672 : }
673 :
674 5486 : case VIEW_CONVERT_EXPR:
675 : /* Getting the real/imag parts of a VCE of a ssa-name
676 : (or gimple invariant) requires to place the complex
677 : into a ssa name before getting the 2 parts.
678 : As `IMAGPART_EXPR<VIEW_CONVERT_EXPR<a_BN>>` is an invalid
679 : gimple. This will only show up when gimplifying it.
680 : Note this creates an extra copy. The call to
681 : force_gimple_operand_gsi would create one too. */
682 5486 : tree expr;
683 5486 : expr = TREE_OPERAND (t, 0);
684 5486 : if (TREE_CODE (expr) == SSA_NAME
685 5486 : || (is_gimple_min_invariant (expr)
686 30 : && TREE_CODE (expr) != STRING_CST))
687 : {
688 5486 : gcc_assert (gimple_p);
689 5486 : tree new_cplx = make_ssa_name (TREE_TYPE (t));
690 5486 : gimple *vce = gimple_build_assign (new_cplx, unshare_expr (t));
691 5486 : gsi_insert_before (gsi, vce, GSI_SAME_STMT);
692 :
693 5486 : tree new_lhs = make_ssa_name (TREE_TYPE (TREE_TYPE (t)));
694 8229 : t = build1 ((imagpart_p ? IMAGPART_EXPR : REALPART_EXPR),
695 5486 : TREE_TYPE (TREE_TYPE (t)), new_cplx);
696 5486 : gimple *new_ri = gimple_build_assign (new_lhs, t);
697 5486 : gsi_insert_before (gsi, new_ri, GSI_SAME_STMT);
698 5486 : t = new_lhs;
699 5486 : return t;
700 : }
701 : /* FALLTHRU */
702 84768 : case VAR_DECL:
703 84768 : case RESULT_DECL:
704 84768 : case PARM_DECL:
705 84768 : case COMPONENT_REF:
706 84768 : case ARRAY_REF:
707 84768 : case MEM_REF:
708 84768 : {
709 84768 : tree inner_type = TREE_TYPE (TREE_TYPE (t));
710 :
711 127295 : t = fold_build1 ((imagpart_p ? IMAGPART_EXPR : REALPART_EXPR),
712 : inner_type, unshare_expr (t));
713 :
714 84768 : if (gimple_p)
715 : {
716 46490 : tree new_lhs = make_ssa_name (inner_type);
717 46490 : gimple *new_load = gimple_build_assign (new_lhs, t);
718 46490 : gsi_insert_before (gsi, new_load, GSI_SAME_STMT);
719 46490 : t = new_lhs;
720 : }
721 :
722 : return t;
723 : }
724 :
725 249100 : case SSA_NAME:
726 249100 : t = get_component_ssa_name (t, imagpart_p);
727 249100 : if (TREE_CODE (t) == SSA_NAME && SSA_NAME_DEF_STMT (t) == NULL)
728 6857 : gcc_assert (phiarg_p);
729 : return t;
730 :
731 0 : default:
732 0 : gcc_unreachable ();
733 : }
734 : }
735 :
736 : /* Update the complex components of the ssa name on the lhs of STMT. */
737 :
738 : static void
739 108571 : update_complex_components (gimple_stmt_iterator *gsi, gimple *stmt, tree r,
740 : tree i)
741 : {
742 108571 : tree lhs;
743 108571 : gimple_seq list;
744 :
745 108571 : lhs = gimple_get_lhs (stmt);
746 :
747 108571 : list = set_component_ssa_name (lhs, false, r);
748 108571 : if (list)
749 39072 : gsi_insert_seq_after (gsi, list, GSI_CONTINUE_LINKING);
750 :
751 108571 : list = set_component_ssa_name (lhs, true, i);
752 108571 : if (list)
753 39039 : gsi_insert_seq_after (gsi, list, GSI_CONTINUE_LINKING);
754 108571 : }
755 :
756 : static void
757 2403 : update_complex_components_on_edge (edge e, tree lhs, tree r, tree i)
758 : {
759 2403 : gimple_seq list;
760 :
761 2403 : list = set_component_ssa_name (lhs, false, r);
762 2403 : if (list)
763 2403 : gsi_insert_seq_on_edge (e, list);
764 :
765 2403 : list = set_component_ssa_name (lhs, true, i);
766 2403 : if (list)
767 2403 : gsi_insert_seq_on_edge (e, list);
768 2403 : }
769 :
770 :
771 : /* Update an assignment to a complex variable in place. */
772 :
773 : static void
774 72864 : update_complex_assignment (gimple_stmt_iterator *gsi, tree r, tree i)
775 : {
776 72864 : gimple *old_stmt = gsi_stmt (*gsi);
777 72864 : gimple_assign_set_rhs_with_ops (gsi, COMPLEX_EXPR, r, i);
778 72864 : gimple *stmt = gsi_stmt (*gsi);
779 72864 : update_stmt (stmt);
780 72864 : if (maybe_clean_or_replace_eh_stmt (old_stmt, stmt))
781 8 : bitmap_set_bit (need_eh_cleanup, gimple_bb (stmt)->index);
782 72864 : if (optimize)
783 39081 : bitmap_set_bit (dce_worklist, SSA_NAME_VERSION (gimple_assign_lhs (stmt)));
784 :
785 72864 : update_complex_components (gsi, gsi_stmt (*gsi), r, i);
786 72864 : }
787 :
788 :
789 : /* Generate code at the entry point of the function to initialize the
790 : component variables for a complex parameter. */
791 :
792 : static void
793 8463 : update_parameter_components (void)
794 : {
795 8463 : edge entry_edge = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
796 8463 : tree parm;
797 :
798 21005 : for (parm = DECL_ARGUMENTS (cfun->decl); parm ; parm = DECL_CHAIN (parm))
799 : {
800 12542 : tree type = TREE_TYPE (parm);
801 12542 : tree ssa_name, r, i;
802 :
803 12542 : if (TREE_CODE (type) != COMPLEX_TYPE || !is_gimple_reg (parm))
804 10130 : continue;
805 :
806 2412 : type = TREE_TYPE (type);
807 2412 : ssa_name = ssa_default_def (cfun, parm);
808 2412 : if (!ssa_name)
809 23 : continue;
810 :
811 2389 : r = build1 (REALPART_EXPR, type, ssa_name);
812 2389 : i = build1 (IMAGPART_EXPR, type, ssa_name);
813 2389 : update_complex_components_on_edge (entry_edge, ssa_name, r, i);
814 : }
815 8463 : }
816 :
817 : /* Generate code to set the component variables of a complex variable
818 : to match the PHI statements in block BB. */
819 :
820 : static void
821 263188 : update_phi_components (basic_block bb)
822 : {
823 263188 : gphi_iterator gsi;
824 :
825 367751 : for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
826 : {
827 104563 : gphi *phi = gsi.phi ();
828 :
829 104563 : if (is_complex_reg (gimple_phi_result (phi)))
830 : {
831 7566 : gphi *p[2] = { NULL, NULL };
832 7566 : unsigned int i, j, n;
833 7566 : bool revisit_phi = false;
834 :
835 22698 : for (j = 0; j < 2; j++)
836 : {
837 15132 : tree l = get_component_ssa_name (gimple_phi_result (phi), j > 0);
838 15132 : if (TREE_CODE (l) == SSA_NAME)
839 14952 : p[j] = create_phi_node (l, bb);
840 : }
841 :
842 23113 : for (i = 0, n = gimple_phi_num_args (phi); i < n; ++i)
843 : {
844 15547 : tree comp, arg = gimple_phi_arg_def (phi, i);
845 62188 : for (j = 0; j < 2; j++)
846 31094 : if (p[j])
847 : {
848 30710 : comp = extract_component (NULL, arg, j > 0, false, true);
849 30710 : if (TREE_CODE (comp) == SSA_NAME
850 30710 : && SSA_NAME_DEF_STMT (comp) == NULL)
851 : {
852 : /* For the benefit of any gimple simplification during
853 : this pass that might walk SSA_NAME def stmts,
854 : don't add SSA_NAMEs without definitions into the
855 : PHI arguments, but put a decl in there instead
856 : temporarily, and revisit this PHI later on. */
857 6857 : if (SSA_NAME_VAR (comp))
858 : comp = SSA_NAME_VAR (comp);
859 : else
860 362 : comp = create_tmp_reg (TREE_TYPE (comp),
861 : get_name (comp));
862 : revisit_phi = true;
863 : }
864 30710 : SET_PHI_ARG_DEF (p[j], i, comp);
865 : }
866 : }
867 :
868 7566 : if (revisit_phi)
869 : {
870 3461 : phis_to_revisit.safe_push (phi);
871 3461 : phis_to_revisit.safe_push (p[0]);
872 3461 : phis_to_revisit.safe_push (p[1]);
873 : }
874 : }
875 : }
876 263188 : }
877 :
878 : /* Expand a complex move to scalars. */
879 :
880 : static void
881 139260 : expand_complex_move (gimple_stmt_iterator *gsi, tree type)
882 : {
883 139260 : tree inner_type = TREE_TYPE (type);
884 139260 : tree r, i, lhs, rhs;
885 139260 : gimple *stmt = gsi_stmt (*gsi);
886 :
887 139260 : if (is_gimple_assign (stmt))
888 : {
889 132937 : lhs = gimple_assign_lhs (stmt);
890 132937 : if (gimple_num_ops (stmt) == 2)
891 109250 : rhs = gimple_assign_rhs1 (stmt);
892 : else
893 : rhs = NULL_TREE;
894 : }
895 6323 : else if (is_gimple_call (stmt))
896 : {
897 6323 : lhs = gimple_call_lhs (stmt);
898 6323 : rhs = NULL_TREE;
899 : }
900 : else
901 0 : gcc_unreachable ();
902 :
903 139260 : if (TREE_CODE (lhs) == SSA_NAME)
904 : {
905 89368 : if (is_ctrl_altering_stmt (stmt))
906 : {
907 14 : edge e;
908 :
909 : /* The value is not assigned on the exception edges, so we need not
910 : concern ourselves there. We do need to update on the fallthru
911 : edge. Find it. */
912 14 : e = find_fallthru_edge (gsi_bb (*gsi)->succs);
913 14 : if (!e)
914 0 : gcc_unreachable ();
915 :
916 14 : r = build1 (REALPART_EXPR, inner_type, lhs);
917 14 : i = build1 (IMAGPART_EXPR, inner_type, lhs);
918 14 : update_complex_components_on_edge (e, lhs, r, i);
919 : }
920 89354 : else if (is_gimple_call (stmt)
921 89354 : || gimple_has_side_effects (stmt))
922 : {
923 33588 : r = build1 (REALPART_EXPR, inner_type, lhs);
924 33588 : i = build1 (IMAGPART_EXPR, inner_type, lhs);
925 33588 : update_complex_components (gsi, stmt, r, i);
926 : }
927 : else
928 : {
929 55766 : if (gimple_assign_rhs_code (stmt) != COMPLEX_EXPR)
930 : {
931 32079 : r = extract_component (gsi, rhs, 0, true);
932 32079 : i = extract_component (gsi, rhs, 1, true);
933 : }
934 : else
935 : {
936 23687 : r = gimple_assign_rhs1 (stmt);
937 23687 : i = gimple_assign_rhs2 (stmt);
938 : }
939 55766 : update_complex_assignment (gsi, r, i);
940 : }
941 : }
942 49892 : else if (rhs
943 49892 : && (TREE_CODE (rhs) == SSA_NAME || TREE_CODE (rhs) == COMPLEX_CST)
944 97226 : && !TREE_SIDE_EFFECTS (lhs))
945 : {
946 30683 : tree x;
947 30683 : gimple *t;
948 30683 : location_t loc;
949 :
950 30683 : loc = gimple_location (stmt);
951 30683 : r = extract_component (gsi, rhs, 0, false);
952 30683 : i = extract_component (gsi, rhs, 1, false);
953 :
954 30683 : x = build1 (REALPART_EXPR, inner_type, unshare_expr (lhs));
955 30683 : t = gimple_build_assign (x, r);
956 30683 : gimple_set_location (t, loc);
957 30683 : gsi_insert_before (gsi, t, GSI_SAME_STMT);
958 :
959 30683 : if (stmt == gsi_stmt (*gsi))
960 : {
961 30683 : x = build1 (IMAGPART_EXPR, inner_type, unshare_expr (lhs));
962 30683 : gimple_assign_set_lhs (stmt, x);
963 30683 : gimple_assign_set_rhs1 (stmt, i);
964 : }
965 : else
966 : {
967 0 : x = build1 (IMAGPART_EXPR, inner_type, unshare_expr (lhs));
968 0 : t = gimple_build_assign (x, i);
969 0 : gimple_set_location (t, loc);
970 0 : gsi_insert_before (gsi, t, GSI_SAME_STMT);
971 :
972 0 : stmt = gsi_stmt (*gsi);
973 0 : gcc_assert (gimple_code (stmt) == GIMPLE_RETURN);
974 0 : gimple_return_set_retval (as_a <greturn *> (stmt), lhs);
975 : }
976 :
977 30683 : update_stmt (stmt);
978 : }
979 139260 : }
980 :
981 : /* Expand complex addition to scalars:
982 : a + b = (ar + br) + i(ai + bi)
983 : a - b = (ar - br) + i(ai + bi)
984 : */
985 :
986 : static void
987 9705 : expand_complex_addition (gimple_stmt_iterator *gsi, tree inner_type,
988 : tree ar, tree ai, tree br, tree bi,
989 : enum tree_code code,
990 : complex_lattice_t al, complex_lattice_t bl)
991 : {
992 9705 : tree rr, ri;
993 9705 : gimple_seq stmts = NULL;
994 9705 : location_t loc = gimple_location (gsi_stmt (*gsi));
995 :
996 9705 : switch (PAIR (al, bl))
997 : {
998 82 : case PAIR (ONLY_REAL, ONLY_REAL):
999 82 : rr = gimple_build (&stmts, loc, code, inner_type, ar, br);
1000 82 : ri = ai;
1001 82 : break;
1002 :
1003 22 : case PAIR (ONLY_REAL, ONLY_IMAG):
1004 22 : rr = ar;
1005 22 : if (code == MINUS_EXPR)
1006 0 : ri = gimple_build (&stmts, loc, MINUS_EXPR, inner_type, ai, bi);
1007 : else
1008 : ri = bi;
1009 : break;
1010 :
1011 7 : case PAIR (ONLY_IMAG, ONLY_REAL):
1012 7 : if (code == MINUS_EXPR)
1013 6 : rr = gimple_build (&stmts, loc, MINUS_EXPR, inner_type, ar, br);
1014 : else
1015 : rr = br;
1016 : ri = ai;
1017 : break;
1018 :
1019 1 : case PAIR (ONLY_IMAG, ONLY_IMAG):
1020 1 : rr = ar;
1021 1 : ri = gimple_build (&stmts, loc, code, inner_type, ai, bi);
1022 1 : break;
1023 :
1024 1221 : case PAIR (VARYING, ONLY_REAL):
1025 1221 : rr = gimple_build (&stmts, loc, code, inner_type, ar, br);
1026 1221 : ri = ai;
1027 1221 : break;
1028 :
1029 3 : case PAIR (VARYING, ONLY_IMAG):
1030 3 : rr = ar;
1031 3 : ri = gimple_build (&stmts, loc, code, inner_type, ai, bi);
1032 3 : break;
1033 :
1034 10 : case PAIR (ONLY_REAL, VARYING):
1035 10 : if (code == MINUS_EXPR)
1036 1 : goto general;
1037 9 : rr = gimple_build (&stmts, loc, code, inner_type, ar, br);
1038 9 : ri = bi;
1039 9 : break;
1040 :
1041 0 : case PAIR (ONLY_IMAG, VARYING):
1042 0 : if (code == MINUS_EXPR)
1043 0 : goto general;
1044 0 : rr = br;
1045 0 : ri = gimple_build (&stmts, loc, code, inner_type, ai, bi);
1046 0 : break;
1047 :
1048 8360 : case PAIR (VARYING, VARYING):
1049 8360 : general:
1050 8360 : rr = gimple_build (&stmts, loc, code, inner_type, ar, br);
1051 : /* (a+ai) + (b+bi) -> (a+b)+(a+b)i
1052 : small optimization to remove one new statement. */
1053 8360 : if (operand_equal_p (ar, ai) && operand_equal_p (br, bi))
1054 : ri = rr;
1055 : else
1056 8309 : ri = gimple_build (&stmts, loc, code, inner_type, ai, bi);
1057 : break;
1058 :
1059 0 : default:
1060 0 : gcc_unreachable ();
1061 : }
1062 :
1063 9705 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1064 9705 : update_complex_assignment (gsi, rr, ri);
1065 9705 : }
1066 :
1067 : /* Expand a complex multiplication or division to a libcall to the c99
1068 : compliant routines. TYPE is the complex type of the operation.
1069 : If INPLACE_P replace the statement at GSI with
1070 : the libcall and return NULL_TREE. Else insert the call, assign its
1071 : result to an output variable and return that variable. If INPLACE_P
1072 : is true then the statement being replaced should be an assignment
1073 : statement. */
1074 :
1075 : static tree
1076 4324 : expand_complex_libcall (gimple_stmt_iterator *gsi, tree type, tree ar, tree ai,
1077 : tree br, tree bi, enum tree_code code, bool inplace_p)
1078 : {
1079 4324 : machine_mode mode;
1080 4324 : enum built_in_function bcode;
1081 4324 : tree fn, lhs;
1082 4324 : gcall *stmt;
1083 :
1084 4324 : mode = TYPE_MODE (type);
1085 4324 : gcc_assert (GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT);
1086 :
1087 4324 : if (code == MULT_EXPR)
1088 3618 : bcode = ((enum built_in_function)
1089 3618 : (BUILT_IN_COMPLEX_MUL_MIN + (mode - MIN_MODE_COMPLEX_FLOAT)));
1090 706 : else if (code == RDIV_EXPR)
1091 706 : bcode = ((enum built_in_function)
1092 706 : (BUILT_IN_COMPLEX_DIV_MIN + (mode - MIN_MODE_COMPLEX_FLOAT)));
1093 : else
1094 0 : gcc_unreachable ();
1095 4324 : fn = builtin_decl_explicit (bcode);
1096 4324 : stmt = gimple_build_call (fn, 4, ar, ai, br, bi);
1097 :
1098 4324 : if (inplace_p)
1099 : {
1100 2119 : gimple *old_stmt = gsi_stmt (*gsi);
1101 2119 : gimple_call_set_nothrow (stmt, !stmt_could_throw_p (cfun, old_stmt));
1102 2119 : lhs = gimple_assign_lhs (old_stmt);
1103 2119 : gimple_call_set_lhs (stmt, lhs);
1104 2119 : gsi_replace (gsi, stmt, true);
1105 :
1106 2119 : type = TREE_TYPE (type);
1107 2119 : if (stmt_can_throw_internal (cfun, stmt))
1108 : {
1109 12 : edge_iterator ei;
1110 12 : edge e;
1111 24 : FOR_EACH_EDGE (e, ei, gimple_bb (stmt)->succs)
1112 24 : if (!(e->flags & EDGE_EH))
1113 : break;
1114 12 : basic_block bb = split_edge (e);
1115 12 : gimple_stmt_iterator gsi2 = gsi_start_bb (bb);
1116 12 : update_complex_components (&gsi2, stmt,
1117 : build1 (REALPART_EXPR, type, lhs),
1118 : build1 (IMAGPART_EXPR, type, lhs));
1119 12 : return NULL_TREE;
1120 : }
1121 : else
1122 2107 : update_complex_components (gsi, stmt,
1123 : build1 (REALPART_EXPR, type, lhs),
1124 : build1 (IMAGPART_EXPR, type, lhs));
1125 2107 : SSA_NAME_DEF_STMT (lhs) = stmt;
1126 2107 : return NULL_TREE;
1127 : }
1128 :
1129 2205 : gimple_call_set_nothrow (stmt, true);
1130 2205 : lhs = make_ssa_name (type);
1131 2205 : gimple_call_set_lhs (stmt, lhs);
1132 2205 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
1133 :
1134 2205 : return lhs;
1135 : }
1136 :
1137 : /* Perform a complex multiplication on two complex constants A, B represented
1138 : by AR, AI, BR, BI of type TYPE.
1139 : The operation we want is: a * b = (ar*br - ai*bi) + i(ar*bi + br*ai).
1140 : Insert the GIMPLE statements into GSI. Store the real and imaginary
1141 : components of the result into RR and RI. */
1142 :
1143 : static void
1144 5951 : expand_complex_multiplication_components (gimple_seq *stmts, location_t loc,
1145 : tree type, tree ar, tree ai,
1146 : tree br, tree bi,
1147 : tree *rr, tree *ri)
1148 : {
1149 5951 : tree t1, t2, t3, t4;
1150 :
1151 5951 : t1 = gimple_build (stmts, loc, MULT_EXPR, type, ar, br);
1152 5951 : t2 = gimple_build (stmts, loc, MULT_EXPR, type, ai, bi);
1153 5951 : t3 = gimple_build (stmts, loc, MULT_EXPR, type, ar, bi);
1154 :
1155 : /* Avoid expanding redundant multiplication for the common
1156 : case of squaring a complex number. */
1157 5951 : if (ar == br && ai == bi)
1158 : t4 = t3;
1159 : else
1160 5824 : t4 = gimple_build (stmts, loc, MULT_EXPR, type, ai, br);
1161 :
1162 5951 : *rr = gimple_build (stmts, loc, MINUS_EXPR, type, t1, t2);
1163 5951 : *ri = gimple_build (stmts, loc, PLUS_EXPR, type, t3, t4);
1164 5951 : }
1165 :
1166 : /* Expand complex multiplication to scalars:
1167 : a * b = (ar*br - ai*bi) + i(ar*bi + br*ai)
1168 : */
1169 :
1170 : static void
1171 7418 : expand_complex_multiplication (gimple_stmt_iterator *gsi, tree type,
1172 : tree ar, tree ai, tree br, tree bi,
1173 : complex_lattice_t al, complex_lattice_t bl)
1174 : {
1175 7418 : tree rr, ri;
1176 7418 : tree inner_type = TREE_TYPE (type);
1177 7418 : location_t loc = gimple_location (gsi_stmt (*gsi));
1178 7418 : gimple_seq stmts = NULL;
1179 :
1180 7418 : if (al < bl)
1181 : {
1182 39 : complex_lattice_t tl;
1183 39 : rr = ar, ar = br, br = rr;
1184 39 : ri = ai, ai = bi, bi = ri;
1185 39 : tl = al, al = bl, bl = tl;
1186 : }
1187 :
1188 7418 : switch (PAIR (al, bl))
1189 : {
1190 0 : case PAIR (ONLY_REAL, ONLY_REAL):
1191 0 : rr = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ar, br);
1192 0 : ri = ai;
1193 0 : break;
1194 :
1195 28 : case PAIR (ONLY_IMAG, ONLY_REAL):
1196 28 : rr = ar;
1197 28 : if (TREE_CODE (ai) == REAL_CST
1198 28 : && real_identical (&TREE_REAL_CST (ai), &dconst1))
1199 0 : ri = br;
1200 : else
1201 28 : ri = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ai, br);
1202 : break;
1203 :
1204 0 : case PAIR (ONLY_IMAG, ONLY_IMAG):
1205 0 : rr = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ai, bi);
1206 0 : rr = gimple_build (&stmts, loc, NEGATE_EXPR, inner_type, rr);
1207 0 : ri = ar;
1208 0 : break;
1209 :
1210 19 : case PAIR (VARYING, ONLY_REAL):
1211 19 : rr = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ar, br);
1212 19 : ri = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ai, br);
1213 19 : break;
1214 :
1215 7 : case PAIR (VARYING, ONLY_IMAG):
1216 7 : rr = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ai, bi);
1217 7 : rr = gimple_build (&stmts, loc, NEGATE_EXPR, inner_type, rr);
1218 7 : ri = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ar, bi);
1219 7 : break;
1220 :
1221 7364 : case PAIR (VARYING, VARYING):
1222 7364 : if (flag_complex_method == 2 && SCALAR_FLOAT_TYPE_P (inner_type))
1223 : {
1224 : /* If optimizing for size or not at all just do a libcall.
1225 : Same if there are exception-handling edges or signaling NaNs. */
1226 2960 : if (optimize == 0 || optimize_bb_for_size_p (gsi_bb (*gsi))
1227 2213 : || stmt_can_throw_internal (cfun, gsi_stmt (*gsi))
1228 5833 : || flag_signaling_nans)
1229 : {
1230 1413 : expand_complex_libcall (gsi, type, ar, ai, br, bi,
1231 : MULT_EXPR, true);
1232 1413 : return;
1233 : }
1234 :
1235 2210 : if (!HONOR_NANS (inner_type))
1236 : {
1237 : /* If we are not worrying about NaNs expand to
1238 : (ar*br - ai*bi) + i(ar*bi + br*ai) directly. */
1239 5 : expand_complex_multiplication_components (&stmts, loc, inner_type,
1240 : ar, ai, br, bi,
1241 : &rr, &ri);
1242 5 : break;
1243 : }
1244 :
1245 : /* Else, expand x = a * b into
1246 : x = (ar*br - ai*bi) + i(ar*bi + br*ai);
1247 : if (isunordered (__real__ x, __imag__ x))
1248 : x = __muldc3 (a, b); */
1249 :
1250 2205 : tree tmpr, tmpi;
1251 2205 : expand_complex_multiplication_components (&stmts, loc,
1252 : inner_type, ar, ai,
1253 : br, bi, &tmpr, &tmpi);
1254 2205 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1255 2205 : stmts = NULL;
1256 :
1257 2205 : gimple *check
1258 2205 : = gimple_build_cond (UNORDERED_EXPR, tmpr, tmpi,
1259 : NULL_TREE, NULL_TREE);
1260 :
1261 2205 : basic_block orig_bb = gsi_bb (*gsi);
1262 : /* We want to keep track of the original complex multiplication
1263 : statement as we're going to modify it later in
1264 : update_complex_assignment. Make sure that insert_cond_bb leaves
1265 : that statement in the join block. */
1266 2205 : gsi_prev (gsi);
1267 2205 : basic_block cond_bb
1268 2205 : = insert_cond_bb (gsi_bb (*gsi), gsi_stmt (*gsi), check,
1269 : profile_probability::very_unlikely ());
1270 :
1271 2205 : gimple_stmt_iterator cond_bb_gsi = gsi_last_bb (cond_bb);
1272 2205 : gsi_insert_after (&cond_bb_gsi, gimple_build_nop (), GSI_NEW_STMT);
1273 :
1274 2205 : tree libcall_res
1275 2205 : = expand_complex_libcall (&cond_bb_gsi, type, ar, ai, br,
1276 : bi, MULT_EXPR, false);
1277 2205 : gimple_seq stmts2 = NULL;
1278 2205 : tree cond_real = gimple_build (&stmts2, loc, REALPART_EXPR,
1279 : inner_type, libcall_res);
1280 2205 : tree cond_imag = gimple_build (&stmts2, loc, IMAGPART_EXPR,
1281 : inner_type, libcall_res);
1282 2205 : gsi_insert_seq_before (&cond_bb_gsi, stmts2, GSI_SAME_STMT);
1283 :
1284 2205 : basic_block join_bb = single_succ_edge (cond_bb)->dest;
1285 2205 : *gsi = gsi_start_nondebug_after_labels_bb (join_bb);
1286 :
1287 : /* We have a conditional block with some assignments in cond_bb.
1288 : Wire up the PHIs to wrap up. */
1289 2205 : rr = make_ssa_name (inner_type);
1290 2205 : ri = make_ssa_name (inner_type);
1291 2205 : edge cond_to_join = single_succ_edge (cond_bb);
1292 2205 : edge orig_to_join = find_edge (orig_bb, join_bb);
1293 :
1294 2205 : gphi *real_phi = create_phi_node (rr, gsi_bb (*gsi));
1295 2205 : add_phi_arg (real_phi, cond_real, cond_to_join, UNKNOWN_LOCATION);
1296 2205 : add_phi_arg (real_phi, tmpr, orig_to_join, UNKNOWN_LOCATION);
1297 :
1298 2205 : gphi *imag_phi = create_phi_node (ri, gsi_bb (*gsi));
1299 2205 : add_phi_arg (imag_phi, cond_imag, cond_to_join, UNKNOWN_LOCATION);
1300 2205 : add_phi_arg (imag_phi, tmpi, orig_to_join, UNKNOWN_LOCATION);
1301 2205 : }
1302 : else
1303 : /* If we are not worrying about NaNs expand to
1304 : (ar*br - ai*bi) + i(ar*bi + br*ai) directly. */
1305 3741 : expand_complex_multiplication_components (&stmts, loc,
1306 : inner_type, ar, ai,
1307 : br, bi, &rr, &ri);
1308 : break;
1309 :
1310 0 : default:
1311 0 : gcc_unreachable ();
1312 : }
1313 :
1314 6005 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1315 6005 : update_complex_assignment (gsi, rr, ri);
1316 : }
1317 :
1318 : /* Keep this algorithm in sync with fold-const.cc:const_binop().
1319 :
1320 : Expand complex division to scalars, straightforward algorithm.
1321 : a / b = ((ar*br + ai*bi)/t) + i((ai*br - ar*bi)/t)
1322 : t = br*br + bi*bi
1323 : */
1324 :
1325 : static void
1326 22 : expand_complex_div_straight (gimple_stmt_iterator *gsi, tree inner_type,
1327 : tree ar, tree ai, tree br, tree bi,
1328 : enum tree_code code)
1329 : {
1330 22 : gimple_seq stmts = NULL;
1331 22 : location_t loc = gimple_location (gsi_stmt (*gsi));
1332 22 : tree rr, ri, div, t1, t2, t3;
1333 :
1334 22 : t1 = gimple_build (&stmts, loc, MULT_EXPR, inner_type, br, br);
1335 22 : t2 = gimple_build (&stmts, loc, MULT_EXPR, inner_type, bi, bi);
1336 22 : div = gimple_build (&stmts, loc, PLUS_EXPR, inner_type, t1, t2);
1337 :
1338 22 : t1 = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ar, br);
1339 22 : t2 = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ai, bi);
1340 22 : t3 = gimple_build (&stmts, loc, PLUS_EXPR, inner_type, t1, t2);
1341 22 : rr = gimple_build (&stmts, loc, code, inner_type, t3, div);
1342 :
1343 22 : t1 = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ai, br);
1344 22 : t2 = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ar, bi);
1345 22 : t3 = gimple_build (&stmts, loc, MINUS_EXPR, inner_type, t1, t2);
1346 22 : ri = gimple_build (&stmts, loc, code, inner_type, t3, div);
1347 :
1348 22 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1349 22 : update_complex_assignment (gsi, rr, ri);
1350 22 : }
1351 :
1352 : /* Keep this algorithm in sync with fold-const.cc:const_binop().
1353 :
1354 : Expand complex division to scalars, modified algorithm to minimize
1355 : overflow with wide input ranges. */
1356 :
1357 : static void
1358 297 : expand_complex_div_wide (gimple_stmt_iterator *gsi, tree inner_type,
1359 : tree ar, tree ai, tree br, tree bi,
1360 : enum tree_code code)
1361 : {
1362 297 : tree rr, ri, ratio, div, t1, t2, tr, ti, compare;
1363 297 : basic_block bb_cond, bb_true, bb_false, bb_join;
1364 297 : gimple *stmt;
1365 297 : gimple_seq stmts = NULL;
1366 297 : location_t loc = gimple_location (gsi_stmt (*gsi));
1367 :
1368 : /* Examine |br| < |bi|, and branch. */
1369 297 : t1 = gimple_build (&stmts, loc, ABS_EXPR, inner_type, br);
1370 297 : t2 = gimple_build (&stmts, loc, ABS_EXPR, inner_type, bi);
1371 297 : compare = gimple_build (&stmts, loc,
1372 : LT_EXPR, boolean_type_node, t1, t2);
1373 :
1374 297 : bb_cond = bb_true = bb_false = bb_join = NULL;
1375 297 : rr = ri = tr = ti = NULL;
1376 297 : if (TREE_CODE (compare) != INTEGER_CST)
1377 : {
1378 246 : edge e;
1379 246 : gimple *stmt;
1380 :
1381 246 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1382 246 : stmts = NULL;
1383 246 : stmt = gimple_build_cond (NE_EXPR, compare, boolean_false_node,
1384 : NULL_TREE, NULL_TREE);
1385 246 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
1386 :
1387 : /* Split the original block, and create the TRUE and FALSE blocks. */
1388 246 : e = split_block (gsi_bb (*gsi), stmt);
1389 246 : bb_cond = e->src;
1390 246 : bb_join = e->dest;
1391 246 : bb_true = create_empty_bb (bb_cond);
1392 246 : bb_false = create_empty_bb (bb_true);
1393 492 : bb_true->count = bb_false->count
1394 246 : = bb_cond->count.apply_probability (profile_probability::even ());
1395 :
1396 : /* Wire the blocks together. */
1397 246 : e->flags = EDGE_TRUE_VALUE;
1398 : /* TODO: With value profile we could add an histogram to determine real
1399 : branch outcome. */
1400 246 : e->probability = profile_probability::even ();
1401 246 : redirect_edge_succ (e, bb_true);
1402 246 : edge e2 = make_edge (bb_cond, bb_false, EDGE_FALSE_VALUE);
1403 246 : e2->probability = profile_probability::even ();
1404 246 : make_single_succ_edge (bb_true, bb_join, EDGE_FALLTHRU);
1405 246 : make_single_succ_edge (bb_false, bb_join, EDGE_FALLTHRU);
1406 246 : add_bb_to_loop (bb_true, bb_cond->loop_father);
1407 246 : add_bb_to_loop (bb_false, bb_cond->loop_father);
1408 :
1409 : /* Update dominance info. Note that bb_join's data was
1410 : updated by split_block. */
1411 246 : if (dom_info_available_p (CDI_DOMINATORS))
1412 : {
1413 240 : set_immediate_dominator (CDI_DOMINATORS, bb_true, bb_cond);
1414 240 : set_immediate_dominator (CDI_DOMINATORS, bb_false, bb_cond);
1415 : }
1416 :
1417 246 : rr = create_tmp_reg (inner_type);
1418 246 : ri = create_tmp_reg (inner_type);
1419 : }
1420 : else
1421 : {
1422 51 : gimple_seq_discard (stmts);
1423 51 : stmts = NULL;
1424 : }
1425 :
1426 : /* In the TRUE branch, we compute
1427 : ratio = br/bi;
1428 : div = (br * ratio) + bi;
1429 : tr = (ar * ratio) + ai;
1430 : ti = (ai * ratio) - ar;
1431 : tr = tr / div;
1432 : ti = ti / div; */
1433 297 : if (bb_true || integer_nonzerop (compare))
1434 : {
1435 260 : if (bb_true)
1436 : {
1437 246 : *gsi = gsi_last_bb (bb_true);
1438 246 : gsi_insert_after (gsi, gimple_build_nop (), GSI_NEW_STMT);
1439 : }
1440 :
1441 260 : ratio = gimple_build (&stmts, loc, code, inner_type, br, bi);
1442 :
1443 260 : t1 = gimple_build (&stmts, loc, MULT_EXPR, inner_type, br, ratio);
1444 260 : div = gimple_build (&stmts, loc, PLUS_EXPR, inner_type, t1, bi);
1445 :
1446 260 : t1 = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ar, ratio);
1447 260 : tr = gimple_build (&stmts, loc, PLUS_EXPR, inner_type, t1, ai);
1448 :
1449 260 : t1 = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ai, ratio);
1450 260 : ti = gimple_build (&stmts, loc, MINUS_EXPR, inner_type, t1, ar);
1451 :
1452 260 : tr = gimple_build (&stmts, loc, code, inner_type, tr, div);
1453 260 : ti = gimple_build (&stmts, loc, code, inner_type, ti, div);
1454 260 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1455 260 : stmts = NULL;
1456 :
1457 260 : if (bb_true)
1458 : {
1459 246 : stmt = gimple_build_assign (rr, tr);
1460 246 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
1461 246 : stmt = gimple_build_assign (ri, ti);
1462 246 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
1463 246 : gsi_remove (gsi, true);
1464 : }
1465 : }
1466 :
1467 : /* In the FALSE branch, we compute
1468 : ratio = d/c;
1469 : divisor = (d * ratio) + c;
1470 : tr = (b * ratio) + a;
1471 : ti = b - (a * ratio);
1472 : tr = tr / div;
1473 : ti = ti / div; */
1474 297 : if (bb_false || integer_zerop (compare))
1475 : {
1476 283 : if (bb_false)
1477 : {
1478 246 : *gsi = gsi_last_bb (bb_false);
1479 246 : gsi_insert_after (gsi, gimple_build_nop (), GSI_NEW_STMT);
1480 : }
1481 :
1482 283 : ratio = gimple_build (&stmts, loc, code, inner_type, bi, br);
1483 :
1484 283 : t1 = gimple_build (&stmts, loc, MULT_EXPR, inner_type, bi, ratio);
1485 283 : div = gimple_build (&stmts, loc, PLUS_EXPR, inner_type, t1, br);
1486 :
1487 283 : t1 = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ai, ratio);
1488 283 : tr = gimple_build (&stmts, loc, PLUS_EXPR, inner_type, t1, ar);
1489 :
1490 283 : t1 = gimple_build (&stmts, loc, MULT_EXPR, inner_type, ar, ratio);
1491 283 : ti = gimple_build (&stmts, loc, MINUS_EXPR, inner_type, ai, t1);
1492 :
1493 283 : tr = gimple_build (&stmts, loc, code, inner_type, tr, div);
1494 283 : ti = gimple_build (&stmts, loc, code, inner_type, ti, div);
1495 283 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1496 283 : stmts = NULL;
1497 :
1498 283 : if (bb_false)
1499 : {
1500 246 : stmt = gimple_build_assign (rr, tr);
1501 246 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
1502 246 : stmt = gimple_build_assign (ri, ti);
1503 246 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
1504 246 : gsi_remove (gsi, true);
1505 : }
1506 : }
1507 :
1508 297 : if (bb_join)
1509 492 : *gsi = gsi_start_bb (bb_join);
1510 : else
1511 : rr = tr, ri = ti;
1512 :
1513 297 : update_complex_assignment (gsi, rr, ri);
1514 297 : }
1515 :
1516 : /* Expand complex division to scalars. */
1517 :
1518 : static void
1519 1062 : expand_complex_division (gimple_stmt_iterator *gsi, tree type,
1520 : tree ar, tree ai, tree br, tree bi,
1521 : enum tree_code code,
1522 : complex_lattice_t al, complex_lattice_t bl)
1523 : {
1524 1062 : tree rr, ri;
1525 1062 : gimple_seq stmts = NULL;
1526 1062 : location_t loc = gimple_location (gsi_stmt (*gsi));
1527 :
1528 1062 : tree inner_type = TREE_TYPE (type);
1529 1062 : switch (PAIR (al, bl))
1530 : {
1531 5 : case PAIR (ONLY_REAL, ONLY_REAL):
1532 5 : rr = gimple_build (&stmts, loc, code, inner_type, ar, br);
1533 5 : ri = ai;
1534 5 : break;
1535 :
1536 0 : case PAIR (ONLY_REAL, ONLY_IMAG):
1537 0 : rr = ai;
1538 0 : ri = gimple_build (&stmts, loc, code, inner_type, ar, bi);
1539 0 : ri = gimple_build (&stmts, loc, NEGATE_EXPR, inner_type, ri);
1540 0 : break;
1541 :
1542 0 : case PAIR (ONLY_IMAG, ONLY_REAL):
1543 0 : rr = ar;
1544 0 : ri = gimple_build (&stmts, loc, code, inner_type, ai, br);
1545 0 : break;
1546 :
1547 0 : case PAIR (ONLY_IMAG, ONLY_IMAG):
1548 0 : rr = gimple_build (&stmts, loc, code, inner_type, ai, bi);
1549 0 : ri = ar;
1550 0 : break;
1551 :
1552 23 : case PAIR (VARYING, ONLY_REAL):
1553 23 : rr = gimple_build (&stmts, loc, code, inner_type, ar, br);
1554 23 : ri = gimple_build (&stmts, loc, code, inner_type, ai, br);
1555 23 : break;
1556 :
1557 9 : case PAIR (VARYING, ONLY_IMAG):
1558 9 : rr = gimple_build (&stmts, loc, code, inner_type, ai, bi);
1559 9 : ri = gimple_build (&stmts, loc, code, inner_type, ar, bi);
1560 9 : ri = gimple_build (&stmts, loc, NEGATE_EXPR, inner_type, ri);
1561 9 : break;
1562 :
1563 1025 : case PAIR (ONLY_REAL, VARYING):
1564 1025 : case PAIR (ONLY_IMAG, VARYING):
1565 1025 : case PAIR (VARYING, VARYING):
1566 1025 : switch (flag_complex_method)
1567 : {
1568 22 : case 0:
1569 : /* straightforward implementation of complex divide acceptable. */
1570 22 : expand_complex_div_straight (gsi, inner_type, ar, ai, br, bi, code);
1571 22 : break;
1572 :
1573 758 : case 2:
1574 758 : if (SCALAR_FLOAT_TYPE_P (inner_type))
1575 : {
1576 706 : expand_complex_libcall (gsi, type, ar, ai, br, bi, code, true);
1577 706 : break;
1578 : }
1579 : /* FALLTHRU */
1580 :
1581 297 : case 1:
1582 : /* wide ranges of inputs must work for complex divide. */
1583 297 : expand_complex_div_wide (gsi, inner_type, ar, ai, br, bi, code);
1584 297 : break;
1585 :
1586 0 : default:
1587 0 : gcc_unreachable ();
1588 : }
1589 1025 : return;
1590 :
1591 0 : default:
1592 0 : gcc_unreachable ();
1593 : }
1594 :
1595 37 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1596 37 : update_complex_assignment (gsi, rr, ri);
1597 : }
1598 :
1599 : /* Expand complex negation to scalars:
1600 : -a = (-ar) + i(-ai)
1601 : */
1602 :
1603 : static void
1604 72 : expand_complex_negation (gimple_stmt_iterator *gsi, tree inner_type,
1605 : tree ar, tree ai)
1606 : {
1607 72 : tree rr, ri;
1608 72 : gimple_seq stmts = NULL;
1609 72 : location_t loc = gimple_location (gsi_stmt (*gsi));
1610 :
1611 72 : rr = gimple_build (&stmts, loc, NEGATE_EXPR, inner_type, ar);
1612 72 : ri = gimple_build (&stmts, loc, NEGATE_EXPR, inner_type, ai);
1613 :
1614 72 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1615 72 : update_complex_assignment (gsi, rr, ri);
1616 72 : }
1617 :
1618 : /* Expand complex paren to scalars:
1619 : ((a)) = ((ar)) + i((ai))
1620 : */
1621 :
1622 : static void
1623 386 : expand_complex_paren (gimple_stmt_iterator *gsi, tree inner_type,
1624 : tree ar, tree ai)
1625 : {
1626 386 : tree rr, ri;
1627 386 : gimple_seq stmts = NULL;
1628 386 : location_t loc = gimple_location (gsi_stmt (*gsi));
1629 :
1630 386 : rr = gimple_build (&stmts, loc, PAREN_EXPR, inner_type, ar);
1631 386 : ri = gimple_build (&stmts, loc, PAREN_EXPR, inner_type, ai);
1632 :
1633 386 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1634 386 : update_complex_assignment (gsi, rr, ri);
1635 386 : }
1636 :
1637 : /* Expand complex conjugate to scalars:
1638 : ~a = (ar) + i(-ai)
1639 : */
1640 :
1641 : static void
1642 574 : expand_complex_conjugate (gimple_stmt_iterator *gsi, tree inner_type,
1643 : tree ar, tree ai)
1644 : {
1645 574 : tree ri;
1646 574 : gimple_seq stmts = NULL;
1647 574 : location_t loc = gimple_location (gsi_stmt (*gsi));
1648 :
1649 574 : ri = gimple_build (&stmts, loc, NEGATE_EXPR, inner_type, ai);
1650 :
1651 574 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1652 574 : update_complex_assignment (gsi, ar, ri);
1653 574 : }
1654 :
1655 : /* Expand complex comparison (EQ or NE only). */
1656 :
1657 : static void
1658 25750 : expand_complex_comparison (gimple_stmt_iterator *gsi, tree ar, tree ai,
1659 : tree br, tree bi, enum tree_code code)
1660 : {
1661 25750 : tree cr, ci, cc, type;
1662 25750 : gimple *stmt = gsi_stmt (*gsi);
1663 25750 : gimple_seq stmts = NULL;
1664 25750 : location_t loc = gimple_location (stmt);
1665 :
1666 25750 : cr = gimple_build (&stmts, loc, code, boolean_type_node, ar, br);
1667 25750 : ci = gimple_build (&stmts, loc, code, boolean_type_node, ai, bi);
1668 51065 : cc = gimple_build (&stmts, loc,
1669 : (code == EQ_EXPR ? BIT_AND_EXPR : BIT_IOR_EXPR),
1670 : boolean_type_node, cr, ci);
1671 25750 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1672 :
1673 25750 : switch (gimple_code (stmt))
1674 : {
1675 499 : case GIMPLE_ASSIGN:
1676 499 : type = TREE_TYPE (gimple_assign_lhs (stmt));
1677 499 : gimple_assign_set_rhs_from_tree (gsi, fold_convert (type, cc));
1678 499 : stmt = gsi_stmt (*gsi);
1679 499 : break;
1680 :
1681 25251 : case GIMPLE_COND:
1682 25251 : {
1683 25251 : gcond *cond_stmt = as_a <gcond *> (stmt);
1684 25251 : gimple_cond_set_code (cond_stmt, EQ_EXPR);
1685 25251 : gimple_cond_set_lhs (cond_stmt, cc);
1686 25251 : gimple_cond_set_rhs (cond_stmt, boolean_true_node);
1687 : }
1688 25251 : break;
1689 :
1690 0 : default:
1691 0 : gcc_unreachable ();
1692 : }
1693 :
1694 25750 : update_stmt (stmt);
1695 25750 : if (maybe_clean_eh_stmt (stmt))
1696 1 : bitmap_set_bit (need_eh_cleanup, gimple_bb (stmt)->index);
1697 25750 : }
1698 :
1699 : /* Expand inline asm that sets some complex SSA_NAMEs. */
1700 :
1701 : static void
1702 544 : expand_complex_asm (gimple_stmt_iterator *gsi)
1703 : {
1704 544 : gasm *stmt = as_a <gasm *> (gsi_stmt (*gsi));
1705 544 : unsigned int i;
1706 544 : bool diagnosed_p = false;
1707 :
1708 1558 : for (i = 0; i < gimple_asm_noutputs (stmt); ++i)
1709 : {
1710 470 : tree link = gimple_asm_output_op (stmt, i);
1711 470 : tree op = TREE_VALUE (link);
1712 470 : if (TREE_CODE (op) == SSA_NAME
1713 470 : && TREE_CODE (TREE_TYPE (op)) == COMPLEX_TYPE)
1714 : {
1715 8 : if (gimple_asm_nlabels (stmt) > 0)
1716 : {
1717 1 : if (!diagnosed_p)
1718 : {
1719 1 : sorry_at (gimple_location (stmt),
1720 : "%<asm goto%> with complex typed outputs");
1721 1 : diagnosed_p = true;
1722 : }
1723 : /* Make sure to not ICE later, see PR105165. */
1724 1 : tree zero = build_zero_cst (TREE_TYPE (TREE_TYPE (op)));
1725 1 : set_component_ssa_name (op, false, zero);
1726 1 : set_component_ssa_name (op, true, zero);
1727 1 : continue;
1728 1 : }
1729 7 : tree type = TREE_TYPE (op);
1730 7 : tree inner_type = TREE_TYPE (type);
1731 7 : tree r = build1 (REALPART_EXPR, inner_type, op);
1732 7 : tree i = build1 (IMAGPART_EXPR, inner_type, op);
1733 7 : gimple_seq list = set_component_ssa_name (op, false, r);
1734 :
1735 7 : if (list)
1736 7 : gsi_insert_seq_after (gsi, list, GSI_CONTINUE_LINKING);
1737 :
1738 7 : list = set_component_ssa_name (op, true, i);
1739 7 : if (list)
1740 7 : gsi_insert_seq_after (gsi, list, GSI_CONTINUE_LINKING);
1741 : }
1742 : }
1743 544 : }
1744 :
1745 :
1746 : /* ARG is the argument to a cabs builtin call in GSI from the
1747 : original OLD_STMT. Create a sequence of statements prior
1748 : to GSI that calculates sqrt(R*R + I*I), where R and
1749 : I are the real and imaginary components of ARG, respectively. */
1750 :
1751 : static void
1752 1131 : gimple_expand_builtin_cabs (gimple_stmt_iterator *gsi, gimple *old_stmt)
1753 : {
1754 1131 : tree real_part, imag_part, addend1, addend2, sum;
1755 1131 : tree arg = gimple_call_arg (old_stmt, 0);
1756 1131 : tree type = TREE_TYPE (TREE_TYPE (arg));
1757 1131 : machine_mode mode = TYPE_MODE (type);
1758 1131 : gimple *new_stmt;
1759 :
1760 1131 : tree lhs = gimple_call_lhs (old_stmt);
1761 :
1762 : /* If there is not a LHS, then just keep the statement around. */
1763 1131 : if (!lhs)
1764 1128 : return;
1765 :
1766 1130 : real_part = extract_component (gsi, arg, false, true);
1767 1130 : imag_part = extract_component (gsi, arg, true, true);
1768 1130 : location_t loc = gimple_location (old_stmt);
1769 :
1770 1130 : gimple_seq stmts = NULL;
1771 :
1772 : /* cabs(x+0i) -> abs(x).
1773 : cabs(0+xi) -> abs(x).
1774 : These 2 can be done even without unsafe math optimizations. */
1775 1130 : if (real_zerop (imag_part)
1776 1130 : || real_zerop (real_part))
1777 : {
1778 9 : tree other = real_zerop (imag_part) ? real_part : imag_part;
1779 9 : sum = gimple_build (&stmts, loc, ABS_EXPR, type, other);
1780 9 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1781 9 : new_stmt = gimple_build_assign (lhs, sum);
1782 9 : gimple_set_location (new_stmt, loc);
1783 9 : gsi_replace (gsi, new_stmt, true);
1784 9 : return;
1785 : }
1786 :
1787 1121 : if (!flag_unsafe_math_optimizations)
1788 : return;
1789 :
1790 : /* cabs(x+xi) -> fabs(x)*sqrt(2). */
1791 5 : if (operand_equal_p (real_part, imag_part))
1792 : {
1793 2 : tree sqrt2 = build_real_truncate (type, dconst_sqrt2 ());
1794 2 : sum = gimple_build (&stmts, loc, ABS_EXPR, type, real_part);
1795 2 : sum = gimple_build (&stmts, loc, MULT_EXPR, type, sum, sqrt2);
1796 2 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1797 2 : new_stmt = gimple_build_assign (lhs, sum);
1798 2 : gimple_set_location (new_stmt, loc);
1799 2 : gsi_replace (gsi, new_stmt, true);
1800 2 : return;
1801 : }
1802 :
1803 : /* cabs(a+bi) -> sqrt(a*a+b*b) if sqrt exists on the target
1804 : and optimizing for speed. */
1805 3 : tree sqrtfn = mathfn_built_in (type, BUILT_IN_SQRT);
1806 3 : if (!optimize_bb_for_speed_p (gimple_bb (old_stmt))
1807 3 : || !sqrtfn
1808 6 : || optab_handler (sqrt_optab, mode) == CODE_FOR_nothing)
1809 : return;
1810 :
1811 3 : addend1 = gimple_build (&stmts, loc, MULT_EXPR, type, real_part, real_part);
1812 3 : addend2 = gimple_build (&stmts, loc, MULT_EXPR, type, imag_part, imag_part);
1813 3 : sum = gimple_build (&stmts, loc, PLUS_EXPR, type, addend1, addend2);
1814 3 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
1815 :
1816 : /* Build the sqrt call. */
1817 3 : new_stmt = gimple_build_call (sqrtfn, 1, sum);
1818 3 : gimple_set_location (new_stmt, loc);
1819 3 : gimple_call_set_lhs (new_stmt, lhs);
1820 3 : gsi_replace (gsi, new_stmt, true);
1821 : }
1822 :
1823 : /* Process one statement. If we identify a complex operation, expand it. */
1824 :
1825 : static void
1826 1485813 : expand_complex_operations_1 (gimple_stmt_iterator *gsi)
1827 : {
1828 1485813 : gimple *stmt = gsi_stmt (*gsi);
1829 1485813 : tree type, inner_type, lhs;
1830 1485813 : tree ac, ar, ai, bc, br, bi;
1831 1485813 : complex_lattice_t al, bl;
1832 1485813 : enum tree_code code;
1833 1485813 : if (gimple_code (stmt) == GIMPLE_CALL)
1834 : {
1835 235453 : switch (gimple_call_combined_fn (stmt))
1836 : {
1837 1131 : CASE_CFN_CABS:
1838 1131 : gimple_expand_builtin_cabs (gsi, stmt);
1839 1131 : return;
1840 : default:;
1841 : }
1842 : }
1843 :
1844 1484682 : if (gimple_code (stmt) == GIMPLE_ASM)
1845 : {
1846 544 : expand_complex_asm (gsi);
1847 544 : return;
1848 : }
1849 :
1850 1484138 : lhs = gimple_get_lhs (stmt);
1851 1484138 : if (!lhs && gimple_code (stmt) != GIMPLE_COND)
1852 : return;
1853 :
1854 1026970 : type = TREE_TYPE (gimple_op (stmt, 0));
1855 1026970 : code = gimple_expr_code (stmt);
1856 :
1857 : /* Initial filter for operations we handle. */
1858 1026970 : switch (code)
1859 : {
1860 133330 : case PLUS_EXPR:
1861 133330 : case MINUS_EXPR:
1862 133330 : case MULT_EXPR:
1863 133330 : case TRUNC_DIV_EXPR:
1864 133330 : case CEIL_DIV_EXPR:
1865 133330 : case FLOOR_DIV_EXPR:
1866 133330 : case ROUND_DIV_EXPR:
1867 133330 : case RDIV_EXPR:
1868 133330 : case NEGATE_EXPR:
1869 133330 : case PAREN_EXPR:
1870 133330 : case CONJ_EXPR:
1871 133330 : if (TREE_CODE (type) != COMPLEX_TYPE)
1872 : return;
1873 19217 : inner_type = TREE_TYPE (type);
1874 19217 : break;
1875 :
1876 129664 : case EQ_EXPR:
1877 129664 : case NE_EXPR:
1878 : /* Note, both GIMPLE_ASSIGN and GIMPLE_COND may have an EQ_EXPR
1879 : subcode, so we need to access the operands using gimple_op. */
1880 129664 : inner_type = TREE_TYPE (gimple_op (stmt, 1));
1881 129664 : if (TREE_CODE (inner_type) != COMPLEX_TYPE)
1882 : return;
1883 : break;
1884 :
1885 763976 : default:
1886 763976 : {
1887 763976 : tree rhs;
1888 :
1889 : /* GIMPLE_COND may also fallthru here, but we do not need to
1890 : do anything with it. */
1891 763976 : if (gimple_code (stmt) == GIMPLE_COND)
1892 : return;
1893 :
1894 741524 : if (TREE_CODE (type) == COMPLEX_TYPE)
1895 139260 : expand_complex_move (gsi, type);
1896 602264 : else if (is_gimple_assign (stmt)
1897 504595 : && (gimple_assign_rhs_code (stmt) == REALPART_EXPR
1898 469703 : || gimple_assign_rhs_code (stmt) == IMAGPART_EXPR)
1899 671159 : && TREE_CODE (lhs) == SSA_NAME)
1900 : {
1901 68895 : rhs = gimple_assign_rhs1 (stmt);
1902 68895 : rhs = extract_component (gsi, TREE_OPERAND (rhs, 0),
1903 : gimple_assign_rhs_code (stmt)
1904 : == IMAGPART_EXPR,
1905 : false);
1906 68895 : gimple_assign_set_rhs_from_tree (gsi, rhs);
1907 68895 : stmt = gsi_stmt (*gsi);
1908 68895 : update_stmt (stmt);
1909 : }
1910 : }
1911 : return;
1912 : }
1913 :
1914 : /* Extract the components of the two complex values. Make sure and
1915 : handle the common case of the same value used twice specially. */
1916 44967 : if (is_gimple_assign (stmt))
1917 : {
1918 19716 : ac = gimple_assign_rhs1 (stmt);
1919 19716 : bc = (gimple_num_ops (stmt) > 2) ? gimple_assign_rhs2 (stmt) : NULL;
1920 : }
1921 : /* GIMPLE_CALL cannot get here. */
1922 : else
1923 : {
1924 25251 : ac = gimple_cond_lhs (stmt);
1925 25251 : bc = gimple_cond_rhs (stmt);
1926 : }
1927 :
1928 44967 : ar = extract_component (gsi, ac, false, true);
1929 44967 : ai = extract_component (gsi, ac, true, true);
1930 :
1931 44967 : if (ac == bc)
1932 : br = ar, bi = ai;
1933 44789 : else if (bc)
1934 : {
1935 43757 : br = extract_component (gsi, bc, 0, true);
1936 43757 : bi = extract_component (gsi, bc, 1, true);
1937 : }
1938 : else
1939 : br = bi = NULL_TREE;
1940 :
1941 44967 : al = find_lattice_value (ac);
1942 44967 : if (al == UNINITIALIZED)
1943 23 : al = VARYING;
1944 :
1945 44967 : if (TREE_CODE_CLASS (code) == tcc_unary)
1946 : bl = UNINITIALIZED;
1947 43935 : else if (ac == bc)
1948 : bl = al;
1949 : else
1950 : {
1951 43757 : bl = find_lattice_value (bc);
1952 43757 : if (bl == UNINITIALIZED)
1953 18 : bl = VARYING;
1954 : }
1955 :
1956 44967 : switch (code)
1957 : {
1958 9705 : case PLUS_EXPR:
1959 9705 : case MINUS_EXPR:
1960 9705 : expand_complex_addition (gsi, inner_type, ar, ai, br, bi, code, al, bl);
1961 9705 : break;
1962 :
1963 7418 : case MULT_EXPR:
1964 7418 : expand_complex_multiplication (gsi, type, ar, ai, br, bi, al, bl);
1965 7418 : break;
1966 :
1967 1062 : case TRUNC_DIV_EXPR:
1968 1062 : case CEIL_DIV_EXPR:
1969 1062 : case FLOOR_DIV_EXPR:
1970 1062 : case ROUND_DIV_EXPR:
1971 1062 : case RDIV_EXPR:
1972 1062 : expand_complex_division (gsi, type, ar, ai, br, bi, code, al, bl);
1973 1062 : break;
1974 :
1975 72 : case NEGATE_EXPR:
1976 72 : expand_complex_negation (gsi, inner_type, ar, ai);
1977 72 : break;
1978 :
1979 574 : case CONJ_EXPR:
1980 574 : expand_complex_conjugate (gsi, inner_type, ar, ai);
1981 574 : break;
1982 :
1983 25750 : case EQ_EXPR:
1984 25750 : case NE_EXPR:
1985 25750 : expand_complex_comparison (gsi, ar, ai, br, bi, code);
1986 25750 : break;
1987 :
1988 386 : case PAREN_EXPR:
1989 386 : expand_complex_paren (gsi, inner_type, ar, ai);
1990 386 : break;
1991 :
1992 : default:
1993 : gcc_unreachable ();
1994 : }
1995 : }
1996 :
1997 :
1998 : /* Entry point for complex operation lowering during optimization. */
1999 :
2000 : static unsigned int
2001 1518056 : tree_lower_complex (void)
2002 : {
2003 1518056 : gimple_stmt_iterator gsi;
2004 1518056 : basic_block bb;
2005 1518056 : int n_bbs, i;
2006 1518056 : int *rpo;
2007 :
2008 1518056 : if (!init_dont_simulate_again ())
2009 : return 0;
2010 :
2011 16926 : complex_lattice_values.create (num_ssa_names);
2012 16926 : complex_lattice_values.safe_grow_cleared (num_ssa_names, true);
2013 :
2014 8463 : init_parameter_lattice_values ();
2015 8463 : class complex_propagate complex_propagate;
2016 8463 : complex_propagate.ssa_propagate ();
2017 :
2018 8463 : need_eh_cleanup = BITMAP_ALLOC (NULL);
2019 8463 : if (optimize)
2020 5225 : dce_worklist = BITMAP_ALLOC (NULL);
2021 :
2022 8463 : complex_variable_components = new int_tree_htab_type (10);
2023 :
2024 16926 : complex_ssa_name_components.create (2 * num_ssa_names);
2025 16926 : complex_ssa_name_components.safe_grow_cleared (2 * num_ssa_names, true);
2026 :
2027 8463 : update_parameter_components ();
2028 :
2029 8463 : rpo = XNEWVEC (int, last_basic_block_for_fn (cfun));
2030 8463 : n_bbs = pre_and_rev_post_order_compute (NULL, rpo, false);
2031 280114 : for (i = 0; i < n_bbs; i++)
2032 : {
2033 263188 : bb = BASIC_BLOCK_FOR_FN (cfun, rpo[i]);
2034 263188 : if (!bb)
2035 0 : continue;
2036 263188 : update_phi_components (bb);
2037 2012189 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2038 1485813 : expand_complex_operations_1 (&gsi);
2039 : }
2040 :
2041 8463 : free (rpo);
2042 :
2043 8463 : if (!phis_to_revisit.is_empty ())
2044 : {
2045 : unsigned int n = phis_to_revisit.length ();
2046 3898 : for (unsigned int j = 0; j < n; j += 3)
2047 10383 : for (unsigned int k = 0; k < 2; k++)
2048 6922 : if (gphi *phi = phis_to_revisit[j + k + 1])
2049 : {
2050 6857 : unsigned int m = gimple_phi_num_args (phi);
2051 20571 : for (unsigned int l = 0; l < m; ++l)
2052 : {
2053 13714 : tree op = gimple_phi_arg_def (phi, l);
2054 20571 : if (TREE_CODE (op) == SSA_NAME
2055 13714 : || is_gimple_min_invariant (op))
2056 6857 : continue;
2057 6857 : tree arg = gimple_phi_arg_def (phis_to_revisit[j], l);
2058 6857 : op = extract_component (NULL, arg, k > 0, false, false);
2059 6857 : SET_PHI_ARG_DEF (phi, l, op);
2060 : }
2061 : }
2062 437 : phis_to_revisit.release ();
2063 : }
2064 :
2065 8463 : gsi_commit_edge_inserts ();
2066 :
2067 8463 : if (optimize)
2068 : {
2069 5225 : simple_dce_from_worklist (dce_worklist, need_eh_cleanup);
2070 5225 : BITMAP_FREE (dce_worklist);
2071 : }
2072 :
2073 8463 : unsigned todo
2074 8463 : = gimple_purge_all_dead_eh_edges (need_eh_cleanup) ? TODO_cleanup_cfg : 0;
2075 8463 : BITMAP_FREE (need_eh_cleanup);
2076 :
2077 8463 : delete complex_variable_components;
2078 8463 : complex_variable_components = NULL;
2079 8463 : complex_ssa_name_components.release ();
2080 8463 : complex_lattice_values.release ();
2081 8463 : return todo;
2082 8463 : }
2083 :
2084 : namespace {
2085 :
2086 : const pass_data pass_data_lower_complex =
2087 : {
2088 : GIMPLE_PASS, /* type */
2089 : "cplxlower", /* name */
2090 : OPTGROUP_NONE, /* optinfo_flags */
2091 : TV_NONE, /* tv_id */
2092 : PROP_ssa, /* properties_required */
2093 : PROP_gimple_lcx, /* properties_provided */
2094 : 0, /* properties_destroyed */
2095 : 0, /* todo_flags_start */
2096 : TODO_update_ssa, /* todo_flags_finish */
2097 : };
2098 :
2099 : class pass_lower_complex : public gimple_opt_pass
2100 : {
2101 : public:
2102 594360 : pass_lower_complex (gcc::context *ctxt)
2103 1188720 : : gimple_opt_pass (pass_data_lower_complex, ctxt)
2104 : {}
2105 :
2106 : /* opt_pass methods: */
2107 297180 : opt_pass * clone () final override { return new pass_lower_complex (m_ctxt); }
2108 1065442 : unsigned int execute (function *) final override
2109 : {
2110 1065442 : return tree_lower_complex ();
2111 : }
2112 :
2113 : }; // class pass_lower_complex
2114 :
2115 : } // anon namespace
2116 :
2117 : gimple_opt_pass *
2118 297180 : make_pass_lower_complex (gcc::context *ctxt)
2119 : {
2120 297180 : return new pass_lower_complex (ctxt);
2121 : }
2122 :
2123 :
2124 : namespace {
2125 :
2126 : const pass_data pass_data_lower_complex_O0 =
2127 : {
2128 : GIMPLE_PASS, /* type */
2129 : "cplxlower0", /* name */
2130 : OPTGROUP_NONE, /* optinfo_flags */
2131 : TV_NONE, /* tv_id */
2132 : PROP_cfg, /* properties_required */
2133 : PROP_gimple_lcx, /* properties_provided */
2134 : 0, /* properties_destroyed */
2135 : 0, /* todo_flags_start */
2136 : TODO_update_ssa, /* todo_flags_finish */
2137 : };
2138 :
2139 : class pass_lower_complex_O0 : public gimple_opt_pass
2140 : {
2141 : public:
2142 297180 : pass_lower_complex_O0 (gcc::context *ctxt)
2143 594360 : : gimple_opt_pass (pass_data_lower_complex_O0, ctxt)
2144 : {}
2145 :
2146 : /* opt_pass methods: */
2147 1517917 : bool gate (function *fun) final override
2148 : {
2149 : /* With errors, normal optimization passes are not run. If we don't
2150 : lower complex operations at all, rtl expansion will abort. */
2151 1517917 : return !(fun->curr_properties & PROP_gimple_lcx);
2152 : }
2153 :
2154 452614 : unsigned int execute (function *) final override
2155 : {
2156 452614 : return tree_lower_complex ();
2157 : }
2158 :
2159 : }; // class pass_lower_complex_O0
2160 :
2161 : } // anon namespace
2162 :
2163 : gimple_opt_pass *
2164 297180 : make_pass_lower_complex_O0 (gcc::context *ctxt)
2165 : {
2166 297180 : return new pass_lower_complex_O0 (ctxt);
2167 : }
|