Line data Source code
1 : /* Convert tree expression to rtl instructions, for GNU compiler.
2 : Copyright (C) 1988-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "predict.h"
29 : #include "memmodel.h"
30 : #include "tm_p.h"
31 : #include "ssa.h"
32 : #include "optabs.h"
33 : #include "expmed.h"
34 : #include "regs.h"
35 : #include "emit-rtl.h"
36 : #include "recog.h"
37 : #include "cgraph.h"
38 : #include "diagnostic.h"
39 : #include "alias.h"
40 : #include "fold-const.h"
41 : #include "stor-layout.h"
42 : #include "attribs.h"
43 : #include "varasm.h"
44 : #include "except.h"
45 : #include "insn-attr.h"
46 : #include "dojump.h"
47 : #include "explow.h"
48 : #include "calls.h"
49 : #include "stmt.h"
50 : /* Include expr.h after insn-config.h so we get HAVE_conditional_move. */
51 : #include "expr.h"
52 : #include "optabs-tree.h"
53 : #include "libfuncs.h"
54 : #include "reload.h"
55 : #include "langhooks.h"
56 : #include "common/common-target.h"
57 : #include "tree-dfa.h"
58 : #include "tree-ssa-live.h"
59 : #include "tree-outof-ssa.h"
60 : #include "tree-ssa-address.h"
61 : #include "builtins.h"
62 : #include "ccmp.h"
63 : #include "gimple-iterator.h"
64 : #include "gimple-fold.h"
65 : #include "rtx-vector-builder.h"
66 : #include "tree-pretty-print.h"
67 : #include "tree-eh.h"
68 : #include "flags.h"
69 : #include "internal-fn.h"
70 : #include "gimple-range.h"
71 :
72 :
73 : /* If this is nonzero, we do not bother generating VOLATILE
74 : around volatile memory references, and we are willing to
75 : output indirect addresses. If cse is to follow, we reject
76 : indirect addresses so a useful potential cse is generated;
77 : if it is used only once, instruction combination will produce
78 : the same indirect address eventually. */
79 : int cse_not_expected;
80 :
81 : /* Cache of the "extended" flag in the target's _BitInt description
82 : for use during expand. */
83 : int bitint_extended = -1;
84 :
85 : static bool block_move_libcall_safe_for_call_parm (void);
86 : static bool emit_block_move_via_pattern (rtx, rtx, rtx, unsigned, unsigned,
87 : HOST_WIDE_INT, unsigned HOST_WIDE_INT,
88 : unsigned HOST_WIDE_INT,
89 : unsigned HOST_WIDE_INT, bool);
90 : static void emit_block_move_via_loop (rtx, rtx, rtx, unsigned, int);
91 : static void emit_block_move_via_sized_loop (rtx, rtx, rtx, unsigned, unsigned);
92 : static void emit_block_move_via_oriented_loop (rtx, rtx, rtx, unsigned, unsigned);
93 : static rtx emit_block_cmp_via_loop (rtx, rtx, rtx, tree, rtx, bool,
94 : unsigned, unsigned);
95 : static rtx_insn *compress_float_constant (rtx, rtx);
96 : static rtx get_subtarget (rtx);
97 : static rtx store_field (rtx, poly_int64, poly_int64, poly_uint64, poly_uint64,
98 : machine_mode, tree, alias_set_type, bool, bool);
99 :
100 : static unsigned HOST_WIDE_INT highest_pow2_factor_for_target (const_tree, const_tree);
101 :
102 : static bool is_aligning_offset (const_tree, const_tree);
103 : static rtx reduce_to_bit_field_precision (rtx, rtx, tree);
104 : static rtx do_store_flag (const_sepops, rtx, machine_mode);
105 : #ifdef PUSH_ROUNDING
106 : static void emit_single_push_insn (machine_mode, rtx, tree);
107 : #endif
108 : static void do_tablejump (rtx, machine_mode, rtx, rtx, rtx,
109 : profile_probability);
110 : static rtx const_vector_from_tree (tree);
111 : static tree tree_expr_size (const_tree);
112 : static void convert_mode_scalar (rtx, rtx, int);
113 :
114 :
115 : /* This is run to set up which modes can be used
116 : directly in memory and to initialize the block move optab. It is run
117 : at the beginning of compilation and when the target is reinitialized. */
118 :
119 : void
120 220288 : init_expr_target (void)
121 : {
122 220288 : rtx pat;
123 220288 : int num_clobbers;
124 220288 : rtx mem, mem1;
125 220288 : rtx reg;
126 :
127 : /* Try indexing by frame ptr and try by stack ptr.
128 : It is known that on the Convex the stack ptr isn't a valid index.
129 : With luck, one or the other is valid on any machine. */
130 220288 : mem = gen_rtx_MEM (word_mode, stack_pointer_rtx);
131 220288 : mem1 = gen_rtx_MEM (word_mode, frame_pointer_rtx);
132 :
133 : /* A scratch register we can modify in-place below to avoid
134 : useless RTL allocations. */
135 220288 : reg = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
136 :
137 220288 : rtx_insn *insn = as_a<rtx_insn *> (rtx_alloc (INSN));
138 220288 : pat = gen_rtx_SET (NULL_RTX, NULL_RTX);
139 220288 : PATTERN (insn) = pat;
140 :
141 27536000 : for (machine_mode mode = VOIDmode; (int) mode < NUM_MACHINE_MODES;
142 27315712 : mode = (machine_mode) ((int) mode + 1))
143 : {
144 27315712 : int regno;
145 :
146 27315712 : direct_load[(int) mode] = direct_store[(int) mode] = 0;
147 27315712 : PUT_MODE (mem, mode);
148 27315712 : PUT_MODE (mem1, mode);
149 :
150 : /* See if there is some register that can be used in this mode and
151 : directly loaded or stored from memory. */
152 :
153 27315712 : if (mode != VOIDmode && mode != BLKmode)
154 1880153783 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER
155 1907028919 : && (direct_load[(int) mode] == 0 || direct_store[(int) mode] == 0);
156 : regno++)
157 : {
158 1880153783 : if (!targetm.hard_regno_mode_ok (regno, mode))
159 1753885390 : continue;
160 :
161 126268393 : set_mode_and_regno (reg, mode, regno);
162 :
163 126268393 : SET_SRC (pat) = mem;
164 126268393 : SET_DEST (pat) = reg;
165 126268393 : if (recog (pat, insn, &num_clobbers) >= 0)
166 7406565 : direct_load[(int) mode] = 1;
167 :
168 126268393 : SET_SRC (pat) = mem1;
169 126268393 : SET_DEST (pat) = reg;
170 126268393 : if (recog (pat, insn, &num_clobbers) >= 0)
171 7406565 : direct_load[(int) mode] = 1;
172 :
173 126268393 : SET_SRC (pat) = reg;
174 126268393 : SET_DEST (pat) = mem;
175 126268393 : if (recog (pat, insn, &num_clobbers) >= 0)
176 7406565 : direct_store[(int) mode] = 1;
177 :
178 126268393 : SET_SRC (pat) = reg;
179 126268393 : SET_DEST (pat) = mem1;
180 126268393 : if (recog (pat, insn, &num_clobbers) >= 0)
181 7406565 : direct_store[(int) mode] = 1;
182 : }
183 : }
184 :
185 225998 : mem = gen_rtx_MEM (VOIDmode, gen_raw_REG (Pmode, LAST_VIRTUAL_REGISTER + 1));
186 :
187 220288 : opt_scalar_float_mode mode_iter;
188 1542016 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_FLOAT)
189 : {
190 1321728 : scalar_float_mode mode = mode_iter.require ();
191 1321728 : scalar_float_mode srcmode;
192 4626048 : FOR_EACH_MODE_UNTIL (srcmode, mode)
193 : {
194 3304320 : enum insn_code ic;
195 :
196 3304320 : ic = can_extend_p (mode, srcmode, 0);
197 3304320 : if (ic == CODE_FOR_nothing)
198 2636493 : continue;
199 :
200 667827 : PUT_MODE (mem, srcmode);
201 :
202 667827 : if (insn_operand_matches (ic, 1, mem))
203 665659 : float_extend_from_mem[mode][srcmode] = true;
204 : }
205 : }
206 220288 : }
207 :
208 : /* This is run at the start of compiling a function. */
209 :
210 : void
211 1742740 : init_expr (void)
212 : {
213 1742740 : memset (&crtl->expr, 0, sizeof (crtl->expr));
214 1742740 : }
215 :
216 : /* Copy data from FROM to TO, where the machine modes are not the same.
217 : Both modes may be integer, or both may be floating, or both may be
218 : fixed-point.
219 : UNSIGNEDP should be nonzero if FROM is an unsigned type.
220 : This causes zero-extension instead of sign-extension. */
221 :
222 : void
223 2190530 : convert_move (rtx to, rtx from, int unsignedp)
224 : {
225 2190530 : machine_mode to_mode = GET_MODE (to);
226 2190530 : machine_mode from_mode = GET_MODE (from);
227 :
228 2190530 : gcc_assert (to_mode != BLKmode);
229 2190530 : gcc_assert (from_mode != BLKmode);
230 :
231 : /* If the source and destination are already the same, then there's
232 : nothing to do. */
233 2190530 : if (to == from)
234 2190530 : return;
235 :
236 : /* If FROM is a SUBREG that indicates that we have already done at least
237 : the required extension, strip it. We don't handle such SUBREGs as
238 : TO here. */
239 :
240 2190530 : scalar_int_mode to_int_mode;
241 2190530 : if (GET_CODE (from) == SUBREG
242 187117 : && SUBREG_PROMOTED_VAR_P (from)
243 2190537 : && is_a <scalar_int_mode> (to_mode, &to_int_mode)
244 2190537 : && (GET_MODE_PRECISION (subreg_promoted_mode (from))
245 7 : >= GET_MODE_PRECISION (to_int_mode))
246 2190537 : && SUBREG_CHECK_PROMOTED_SIGN (from, unsignedp))
247 : {
248 0 : scalar_int_mode int_orig_mode;
249 0 : scalar_int_mode int_inner_mode;
250 0 : machine_mode orig_mode = GET_MODE (from);
251 :
252 0 : from = gen_lowpart (to_int_mode, SUBREG_REG (from));
253 0 : from_mode = to_int_mode;
254 :
255 : /* Preserve SUBREG_PROMOTED_VAR_P if the new mode is wider than
256 : the original mode, but narrower than the inner mode. */
257 0 : if (GET_CODE (from) == SUBREG
258 0 : && is_a <scalar_int_mode> (orig_mode, &int_orig_mode)
259 0 : && GET_MODE_PRECISION (to_int_mode)
260 0 : > GET_MODE_PRECISION (int_orig_mode)
261 0 : && is_a <scalar_int_mode> (GET_MODE (SUBREG_REG (from)),
262 : &int_inner_mode)
263 0 : && GET_MODE_PRECISION (int_inner_mode)
264 0 : > GET_MODE_PRECISION (to_int_mode))
265 : {
266 0 : SUBREG_PROMOTED_VAR_P (from) = 1;
267 0 : SUBREG_PROMOTED_SET (from, unsignedp);
268 : }
269 : }
270 :
271 2190530 : gcc_assert (GET_CODE (to) != SUBREG || !SUBREG_PROMOTED_VAR_P (to));
272 :
273 2190530 : if (to_mode == from_mode
274 2135095 : || (from_mode == VOIDmode && CONSTANT_P (from)))
275 : {
276 55662 : emit_move_insn (to, from);
277 55662 : return;
278 : }
279 :
280 2134868 : if (VECTOR_MODE_P (to_mode) || VECTOR_MODE_P (from_mode))
281 : {
282 30344 : if (GET_MODE_UNIT_PRECISION (to_mode)
283 15172 : > GET_MODE_UNIT_PRECISION (from_mode))
284 : {
285 4956 : optab op = unsignedp ? zext_optab : sext_optab;
286 4956 : insn_code icode = convert_optab_handler (op, to_mode, from_mode);
287 4956 : if (icode != CODE_FOR_nothing)
288 : {
289 1130 : emit_unop_insn (icode, to, from,
290 : unsignedp ? ZERO_EXTEND : SIGN_EXTEND);
291 1130 : return;
292 : }
293 : }
294 :
295 28084 : if (GET_MODE_UNIT_PRECISION (to_mode)
296 14042 : < GET_MODE_UNIT_PRECISION (from_mode))
297 : {
298 2164 : insn_code icode = convert_optab_handler (trunc_optab,
299 : to_mode, from_mode);
300 2164 : if (icode != CODE_FOR_nothing)
301 : {
302 933 : emit_unop_insn (icode, to, from, TRUNCATE);
303 933 : return;
304 : }
305 : }
306 :
307 39327 : gcc_assert (known_eq (GET_MODE_BITSIZE (from_mode),
308 : GET_MODE_BITSIZE (to_mode)));
309 :
310 13109 : if (VECTOR_MODE_P (to_mode))
311 13109 : from = force_subreg (to_mode, from, GET_MODE (from), 0);
312 : else
313 0 : to = simplify_gen_subreg (from_mode, to, GET_MODE (to), 0);
314 :
315 13109 : emit_move_insn (to, from);
316 13109 : return;
317 : }
318 :
319 2119696 : if (GET_CODE (to) == CONCAT && GET_CODE (from) == CONCAT)
320 : {
321 0 : convert_move (XEXP (to, 0), XEXP (from, 0), unsignedp);
322 0 : convert_move (XEXP (to, 1), XEXP (from, 1), unsignedp);
323 0 : return;
324 : }
325 :
326 2119696 : convert_mode_scalar (to, from, unsignedp);
327 : }
328 :
329 : /* Like convert_move, but deals only with scalar modes. */
330 :
331 : static void
332 2119758 : convert_mode_scalar (rtx to, rtx from, int unsignedp)
333 : {
334 : /* Both modes should be scalar types. */
335 2119763 : scalar_mode from_mode = as_a <scalar_mode> (GET_MODE (from));
336 2119763 : scalar_mode to_mode = as_a <scalar_mode> (GET_MODE (to));
337 2119763 : bool to_real = SCALAR_FLOAT_MODE_P (to_mode);
338 2119763 : bool from_real = SCALAR_FLOAT_MODE_P (from_mode);
339 2119763 : enum insn_code code;
340 2119763 : rtx libcall;
341 :
342 2119763 : gcc_assert (to_real == from_real);
343 :
344 : /* rtx code for making an equivalent value. */
345 3229931 : enum rtx_code equiv_code = (unsignedp < 0 ? UNKNOWN
346 2119763 : : (unsignedp ? ZERO_EXTEND : SIGN_EXTEND));
347 :
348 2119763 : auto acceptable_same_precision_modes
349 580 : = [] (scalar_mode from_mode, scalar_mode to_mode) -> bool
350 : {
351 580 : if (DECIMAL_FLOAT_MODE_P (from_mode) != DECIMAL_FLOAT_MODE_P (to_mode))
352 : return true;
353 :
354 : /* arm_bfloat_half_format <-> ieee_half_format */
355 2 : if ((REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
356 1 : && REAL_MODE_FORMAT (to_mode) == &ieee_half_format)
357 2 : || (REAL_MODE_FORMAT (to_mode) == &arm_bfloat_half_format
358 1 : && REAL_MODE_FORMAT (from_mode) == &ieee_half_format))
359 : return true;
360 :
361 : /* ibm_extended_format <-> ieee_quad_format */
362 0 : if ((REAL_MODE_FORMAT (from_mode) == &ibm_extended_format
363 0 : && REAL_MODE_FORMAT (to_mode) == &ieee_quad_format)
364 0 : || (REAL_MODE_FORMAT (from_mode) == &ieee_quad_format
365 0 : && REAL_MODE_FORMAT (to_mode) == &ibm_extended_format))
366 0 : return true;
367 :
368 : return false;
369 : };
370 :
371 2119763 : if (to_real)
372 : {
373 189701 : rtx value;
374 189701 : rtx_insn *insns;
375 189701 : convert_optab tab;
376 :
377 189701 : gcc_assert ((GET_MODE_PRECISION (from_mode)
378 : != GET_MODE_PRECISION (to_mode))
379 : || acceptable_same_precision_modes (from_mode, to_mode));
380 :
381 189701 : if (GET_MODE_PRECISION (from_mode) == GET_MODE_PRECISION (to_mode))
382 : {
383 580 : if ((REAL_MODE_FORMAT (to_mode) == &arm_bfloat_half_format
384 1 : && REAL_MODE_FORMAT (from_mode) == &ieee_half_format)
385 580 : || (REAL_MODE_FORMAT (to_mode) == &ieee_quad_format
386 0 : && REAL_MODE_FORMAT (from_mode) == &ibm_extended_format))
387 : /* libgcc implements just __trunchfbf2, not __extendhfbf2;
388 : and __trunctfkf2, not __extendtfkf2. */
389 : tab = trunc_optab;
390 : else
391 : /* Conversion between decimal float and binary float, same
392 : size. */
393 579 : tab = DECIMAL_FLOAT_MODE_P (from_mode) ? trunc_optab : sext_optab;
394 : }
395 189121 : else if (GET_MODE_PRECISION (from_mode) < GET_MODE_PRECISION (to_mode))
396 : tab = sext_optab;
397 : else
398 18906 : tab = trunc_optab;
399 :
400 : /* Try converting directly if the insn is supported. */
401 :
402 189701 : code = convert_optab_handler (tab, to_mode, from_mode);
403 189701 : if (code != CODE_FOR_nothing)
404 : {
405 161733 : emit_unop_insn (code, to, from,
406 : tab == sext_optab ? FLOAT_EXTEND : FLOAT_TRUNCATE);
407 161733 : return;
408 : }
409 :
410 : #ifdef HAVE_SFmode
411 27968 : if (REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
412 27968 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
413 : {
414 3273 : if (GET_MODE_PRECISION (to_mode) > GET_MODE_PRECISION (SFmode))
415 : {
416 : /* To cut down on libgcc size, implement
417 : BFmode -> {DF,XF,TF}mode conversions by
418 : BFmode -> SFmode -> {DF,XF,TF}mode conversions. */
419 4 : rtx temp = gen_reg_rtx (SFmode);
420 4 : convert_mode_scalar (temp, from, unsignedp);
421 4 : convert_mode_scalar (to, temp, unsignedp);
422 4 : return;
423 : }
424 3269 : if (REAL_MODE_FORMAT (to_mode) == &ieee_half_format)
425 : {
426 : /* Similarly, implement BFmode -> HFmode as
427 : BFmode -> SFmode -> HFmode conversion where SFmode
428 : has superset of BFmode values. We don't need
429 : to handle sNaNs by raising exception and turning
430 : it into qNaN though, as that can be done in the
431 : SFmode -> HFmode conversion too. */
432 1 : rtx temp = gen_reg_rtx (SFmode);
433 1 : int save_flag_finite_math_only = flag_finite_math_only;
434 1 : flag_finite_math_only = true;
435 1 : convert_mode_scalar (temp, from, unsignedp);
436 1 : flag_finite_math_only = save_flag_finite_math_only;
437 1 : convert_mode_scalar (to, temp, unsignedp);
438 1 : return;
439 : }
440 3268 : if (to_mode == SFmode
441 3268 : && !HONOR_NANS (from_mode)
442 58 : && !HONOR_NANS (to_mode)
443 3326 : && optimize_insn_for_speed_p ())
444 : {
445 : /* If we don't expect sNaNs, for BFmode -> SFmode we can just
446 : shift the bits up. */
447 57 : machine_mode fromi_mode, toi_mode;
448 114 : if (int_mode_for_size (GET_MODE_BITSIZE (from_mode),
449 57 : 0).exists (&fromi_mode)
450 57 : && int_mode_for_size (GET_MODE_BITSIZE (to_mode),
451 0 : 0).exists (&toi_mode))
452 : {
453 57 : start_sequence ();
454 57 : rtx fromi = force_lowpart_subreg (fromi_mode, from,
455 : from_mode);
456 57 : rtx tof = NULL_RTX;
457 57 : if (fromi)
458 : {
459 57 : rtx toi;
460 57 : if (GET_MODE (fromi) == VOIDmode)
461 0 : toi = simplify_unary_operation (ZERO_EXTEND, toi_mode,
462 : fromi, fromi_mode);
463 : else
464 : {
465 57 : toi = gen_reg_rtx (toi_mode);
466 57 : convert_mode_scalar (toi, fromi, 1);
467 : }
468 57 : toi
469 114 : = maybe_expand_shift (LSHIFT_EXPR, toi_mode, toi,
470 57 : GET_MODE_PRECISION (to_mode)
471 57 : - GET_MODE_PRECISION (from_mode),
472 : NULL_RTX, 1);
473 57 : if (toi)
474 : {
475 57 : tof = force_lowpart_subreg (to_mode, toi, toi_mode);
476 57 : if (tof)
477 57 : emit_move_insn (to, tof);
478 : }
479 : }
480 57 : insns = end_sequence ();
481 57 : if (tof)
482 : {
483 57 : emit_insn (insns);
484 57 : return;
485 : }
486 : }
487 : }
488 : }
489 27906 : if (REAL_MODE_FORMAT (from_mode) == &ieee_single_format
490 4277 : && REAL_MODE_FORMAT (to_mode) == &arm_bfloat_half_format
491 1572 : && !HONOR_NANS (from_mode)
492 0 : && !HONOR_NANS (to_mode)
493 0 : && !flag_rounding_math
494 27906 : && optimize_insn_for_speed_p ())
495 : {
496 : /* If we don't expect qNaNs nor sNaNs and can assume rounding
497 : to nearest, we can expand the conversion inline as
498 : (fromi + 0x7fff + ((fromi >> 16) & 1)) >> 16. */
499 0 : machine_mode fromi_mode, toi_mode;
500 0 : if (int_mode_for_size (GET_MODE_BITSIZE (from_mode),
501 0 : 0).exists (&fromi_mode)
502 0 : && int_mode_for_size (GET_MODE_BITSIZE (to_mode),
503 0 : 0).exists (&toi_mode))
504 : {
505 0 : start_sequence ();
506 0 : rtx fromi = force_lowpart_subreg (fromi_mode, from, from_mode);
507 0 : rtx tof = NULL_RTX;
508 0 : do
509 : {
510 0 : if (!fromi)
511 : break;
512 0 : int shift = (GET_MODE_PRECISION (from_mode)
513 0 : - GET_MODE_PRECISION (to_mode));
514 0 : rtx temp1
515 0 : = maybe_expand_shift (RSHIFT_EXPR, fromi_mode, fromi,
516 : shift, NULL_RTX, 1);
517 0 : if (!temp1)
518 : break;
519 0 : rtx temp2
520 0 : = expand_binop (fromi_mode, and_optab, temp1, const1_rtx,
521 : NULL_RTX, 1, OPTAB_DIRECT);
522 0 : if (!temp2)
523 : break;
524 0 : rtx temp3
525 0 : = expand_binop (fromi_mode, add_optab, fromi,
526 : gen_int_mode ((HOST_WIDE_INT_1U
527 0 : << (shift - 1)) - 1,
528 : fromi_mode), NULL_RTX,
529 : 1, OPTAB_DIRECT);
530 0 : if (!temp3)
531 : break;
532 0 : rtx temp4
533 0 : = expand_binop (fromi_mode, add_optab, temp3, temp2,
534 : NULL_RTX, 1, OPTAB_DIRECT);
535 0 : if (!temp4)
536 : break;
537 0 : rtx temp5 = maybe_expand_shift (RSHIFT_EXPR, fromi_mode,
538 : temp4, shift, NULL_RTX, 1);
539 0 : if (!temp5)
540 : break;
541 0 : rtx temp6 = force_lowpart_subreg (toi_mode, temp5,
542 : fromi_mode);
543 0 : if (!temp6)
544 : break;
545 0 : tof = force_lowpart_subreg (to_mode, temp6, toi_mode);
546 0 : if (tof)
547 0 : emit_move_insn (to, tof);
548 : }
549 : while (0);
550 0 : insns = end_sequence ();
551 0 : if (tof)
552 : {
553 0 : emit_insn (insns);
554 0 : return;
555 : }
556 : }
557 : }
558 : #endif
559 :
560 : /* Otherwise use a libcall. */
561 27906 : libcall = convert_optab_libfunc (tab, to_mode, from_mode);
562 :
563 : /* Is this conversion implemented yet? */
564 27906 : gcc_assert (libcall);
565 :
566 27906 : start_sequence ();
567 27906 : value = emit_library_call_value (libcall, NULL_RTX, LCT_CONST, to_mode,
568 : from, from_mode);
569 27906 : insns = end_sequence ();
570 27906 : emit_libcall_block (insns, to, value,
571 6788 : tab == trunc_optab ? gen_rtx_FLOAT_TRUNCATE (to_mode,
572 : from)
573 21118 : : gen_rtx_FLOAT_EXTEND (to_mode, from));
574 27906 : return;
575 : }
576 :
577 : /* Handle pointer conversion. */ /* SPEE 900220. */
578 : /* If the target has a converter from FROM_MODE to TO_MODE, use it. */
579 1930062 : {
580 1930062 : convert_optab ctab;
581 :
582 1930062 : if (GET_MODE_PRECISION (from_mode) > GET_MODE_PRECISION (to_mode))
583 : ctab = trunc_optab;
584 1728691 : else if (unsignedp)
585 : ctab = zext_optab;
586 : else
587 834201 : ctab = sext_optab;
588 :
589 1930062 : if (convert_optab_handler (ctab, to_mode, from_mode)
590 : != CODE_FOR_nothing)
591 : {
592 1657921 : emit_unop_insn (convert_optab_handler (ctab, to_mode, from_mode),
593 : to, from, UNKNOWN);
594 1657921 : return;
595 : }
596 : }
597 :
598 : /* Targets are expected to provide conversion insns between PxImode and
599 : xImode for all MODE_PARTIAL_INT modes they use, but no others. */
600 272141 : if (GET_MODE_CLASS (to_mode) == MODE_PARTIAL_INT)
601 : {
602 0 : scalar_int_mode full_mode
603 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE (to_mode)).require ();
604 :
605 0 : gcc_assert (convert_optab_handler (trunc_optab, to_mode, full_mode)
606 : != CODE_FOR_nothing);
607 :
608 0 : if (full_mode != from_mode)
609 0 : from = convert_to_mode (full_mode, from, unsignedp);
610 0 : emit_unop_insn (convert_optab_handler (trunc_optab, to_mode, full_mode),
611 : to, from, UNKNOWN);
612 0 : return;
613 : }
614 272141 : if (GET_MODE_CLASS (from_mode) == MODE_PARTIAL_INT)
615 : {
616 0 : rtx new_from;
617 0 : scalar_int_mode full_mode
618 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE (from_mode)).require ();
619 0 : convert_optab ctab = unsignedp ? zext_optab : sext_optab;
620 0 : enum insn_code icode;
621 :
622 0 : icode = convert_optab_handler (ctab, full_mode, from_mode);
623 0 : gcc_assert (icode != CODE_FOR_nothing);
624 :
625 0 : if (to_mode == full_mode)
626 : {
627 0 : emit_unop_insn (icode, to, from, UNKNOWN);
628 0 : return;
629 : }
630 :
631 0 : new_from = gen_reg_rtx (full_mode);
632 0 : emit_unop_insn (icode, new_from, from, UNKNOWN);
633 :
634 : /* else proceed to integer conversions below. */
635 0 : from_mode = full_mode;
636 0 : from = new_from;
637 : }
638 :
639 : /* Make sure both are fixed-point modes or both are not. */
640 272141 : gcc_assert (ALL_SCALAR_FIXED_POINT_MODE_P (from_mode) ==
641 : ALL_SCALAR_FIXED_POINT_MODE_P (to_mode));
642 272141 : if (ALL_SCALAR_FIXED_POINT_MODE_P (from_mode))
643 : {
644 : /* If we widen from_mode to to_mode and they are in the same class,
645 : we won't saturate the result.
646 : Otherwise, always saturate the result to play safe. */
647 0 : if (GET_MODE_CLASS (from_mode) == GET_MODE_CLASS (to_mode)
648 0 : && GET_MODE_SIZE (from_mode) < GET_MODE_SIZE (to_mode))
649 0 : expand_fixed_convert (to, from, 0, 0);
650 : else
651 0 : expand_fixed_convert (to, from, 0, 1);
652 : return;
653 : }
654 :
655 : /* Now both modes are integers. */
656 :
657 : /* Handle expanding beyond a word. */
658 272141 : if (GET_MODE_PRECISION (from_mode) < GET_MODE_PRECISION (to_mode)
659 274842 : && GET_MODE_PRECISION (to_mode) > BITS_PER_WORD)
660 : {
661 70770 : rtx_insn *insns;
662 70770 : rtx lowpart;
663 70770 : rtx fill_value;
664 70770 : rtx lowfrom;
665 70770 : int i;
666 70770 : scalar_mode lowpart_mode;
667 141540 : int nwords = CEIL (GET_MODE_SIZE (to_mode), UNITS_PER_WORD);
668 :
669 : /* Try converting directly if the insn is supported. */
670 70770 : if ((code = can_extend_p (to_mode, from_mode, unsignedp))
671 : != CODE_FOR_nothing)
672 : {
673 : /* If FROM is a SUBREG, put it into a register. Do this
674 : so that we always generate the same set of insns for
675 : better cse'ing; if an intermediate assignment occurred,
676 : we won't be doing the operation directly on the SUBREG. */
677 0 : if (optimize > 0 && GET_CODE (from) == SUBREG)
678 0 : from = force_reg (from_mode, from);
679 0 : emit_unop_insn (code, to, from, equiv_code);
680 0 : return;
681 : }
682 : /* Next, try converting via full word. */
683 70770 : else if (GET_MODE_PRECISION (from_mode) < BITS_PER_WORD
684 70770 : && ((code = can_extend_p (to_mode, word_mode, unsignedp))
685 : != CODE_FOR_nothing))
686 : {
687 70631 : rtx word_to = gen_reg_rtx (word_mode);
688 70631 : if (REG_P (to))
689 : {
690 70558 : if (reg_overlap_mentioned_p (to, from))
691 0 : from = force_reg (from_mode, from);
692 70558 : emit_clobber (to);
693 : }
694 70631 : convert_move (word_to, from, unsignedp);
695 70631 : emit_unop_insn (code, to, word_to, equiv_code);
696 70631 : return;
697 : }
698 :
699 : /* No special multiword conversion insn; do it by hand. */
700 139 : start_sequence ();
701 :
702 : /* Since we will turn this into a no conflict block, we must ensure
703 : the source does not overlap the target so force it into an isolated
704 : register when maybe so. Likewise for any MEM input, since the
705 : conversion sequence might require several references to it and we
706 : must ensure we're getting the same value every time. */
707 :
708 139 : if (MEM_P (from) || reg_overlap_mentioned_p (to, from))
709 0 : from = force_reg (from_mode, from);
710 :
711 : /* Get a copy of FROM widened to a word, if necessary. */
712 278 : if (GET_MODE_PRECISION (from_mode) < BITS_PER_WORD)
713 0 : lowpart_mode = word_mode;
714 : else
715 : lowpart_mode = from_mode;
716 :
717 139 : lowfrom = convert_to_mode (lowpart_mode, from, unsignedp);
718 :
719 139 : lowpart = gen_lowpart (lowpart_mode, to);
720 139 : emit_move_insn (lowpart, lowfrom);
721 :
722 : /* Compute the value to put in each remaining word. */
723 139 : if (unsignedp)
724 117 : fill_value = const0_rtx;
725 : else
726 22 : fill_value = emit_store_flag_force (gen_reg_rtx (word_mode),
727 : LT, lowfrom, const0_rtx,
728 : lowpart_mode, 0, -1);
729 :
730 : /* Fill the remaining words. */
731 695 : for (i = GET_MODE_SIZE (lowpart_mode) / UNITS_PER_WORD; i < nwords; i++)
732 : {
733 278 : int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
734 278 : rtx subword = operand_subword (to, index, 1, to_mode);
735 :
736 278 : gcc_assert (subword);
737 :
738 278 : if (fill_value != subword)
739 278 : emit_move_insn (subword, fill_value);
740 : }
741 :
742 139 : insns = end_sequence ();
743 :
744 139 : emit_insn (insns);
745 139 : return;
746 : }
747 :
748 : /* Truncating multi-word to a word or less. */
749 201371 : if (GET_MODE_PRECISION (from_mode) > BITS_PER_WORD
750 201371 : && GET_MODE_PRECISION (to_mode) <= BITS_PER_WORD)
751 : {
752 75354 : if (!((MEM_P (from)
753 575 : && ! MEM_VOLATILE_P (from)
754 575 : && direct_load[(int) to_mode]
755 575 : && ! mode_dependent_address_p (XEXP (from, 0),
756 575 : MEM_ADDR_SPACE (from)))
757 46428 : || REG_P (from)
758 : || GET_CODE (from) == SUBREG))
759 0 : from = force_reg (from_mode, from);
760 47003 : convert_move (to, gen_lowpart (word_mode, from), 0);
761 47003 : return;
762 : }
763 :
764 : /* Now follow all the conversions between integers
765 : no more than a word long. */
766 :
767 : /* For truncation, usually we can just refer to FROM in a narrower mode. */
768 308736 : if (GET_MODE_BITSIZE (to_mode) < GET_MODE_BITSIZE (from_mode)
769 154368 : && TRULY_NOOP_TRUNCATION_MODES_P (to_mode, from_mode))
770 : {
771 166435 : if (!((MEM_P (from)
772 14154 : && ! MEM_VOLATILE_P (from)
773 14095 : && direct_load[(int) to_mode]
774 14095 : && ! mode_dependent_address_p (XEXP (from, 0),
775 14095 : MEM_ADDR_SPACE (from)))
776 140273 : || REG_P (from)
777 : || GET_CODE (from) == SUBREG))
778 923 : from = force_reg (from_mode, from);
779 129129 : if (REG_P (from) && REGNO (from) < FIRST_PSEUDO_REGISTER
780 154369 : && !targetm.hard_regno_mode_ok (REGNO (from), to_mode))
781 0 : from = copy_to_reg (from);
782 154368 : emit_move_insn (to, gen_lowpart (to_mode, from));
783 154368 : return;
784 : }
785 :
786 : /* Handle extension. */
787 0 : if (GET_MODE_PRECISION (to_mode) > GET_MODE_PRECISION (from_mode))
788 : {
789 : /* Convert directly if that works. */
790 0 : if ((code = can_extend_p (to_mode, from_mode, unsignedp))
791 : != CODE_FOR_nothing)
792 : {
793 0 : emit_unop_insn (code, to, from, equiv_code);
794 0 : return;
795 : }
796 : else
797 : {
798 0 : rtx tmp;
799 0 : int shift_amount;
800 :
801 : /* Search for a mode to convert via. */
802 0 : opt_scalar_mode intermediate_iter;
803 0 : FOR_EACH_MODE_FROM (intermediate_iter, from_mode)
804 : {
805 0 : scalar_mode intermediate = intermediate_iter.require ();
806 0 : if (((can_extend_p (to_mode, intermediate, unsignedp)
807 : != CODE_FOR_nothing)
808 0 : || (GET_MODE_SIZE (to_mode) < GET_MODE_SIZE (intermediate)
809 0 : && TRULY_NOOP_TRUNCATION_MODES_P (to_mode,
810 : intermediate)))
811 0 : && (can_extend_p (intermediate, from_mode, unsignedp)
812 : != CODE_FOR_nothing))
813 : {
814 0 : convert_move (to, convert_to_mode (intermediate, from,
815 : unsignedp), unsignedp);
816 0 : return;
817 : }
818 : }
819 :
820 : /* No suitable intermediate mode.
821 : Generate what we need with shifts. */
822 0 : shift_amount = (GET_MODE_PRECISION (to_mode)
823 0 : - GET_MODE_PRECISION (from_mode));
824 0 : from = gen_lowpart (to_mode, force_reg (from_mode, from));
825 0 : tmp = expand_shift (LSHIFT_EXPR, to_mode, from, shift_amount,
826 : to, unsignedp);
827 0 : tmp = expand_shift (RSHIFT_EXPR, to_mode, tmp, shift_amount,
828 : to, unsignedp);
829 0 : if (tmp != to)
830 0 : emit_move_insn (to, tmp);
831 : return;
832 : }
833 : }
834 :
835 : /* Support special truncate insns for certain modes. */
836 0 : if (convert_optab_handler (trunc_optab, to_mode,
837 : from_mode) != CODE_FOR_nothing)
838 : {
839 0 : emit_unop_insn (convert_optab_handler (trunc_optab, to_mode, from_mode),
840 : to, from, UNKNOWN);
841 0 : return;
842 : }
843 :
844 : /* Handle truncation of volatile memrefs, and so on;
845 : the things that couldn't be truncated directly,
846 : and for which there was no special instruction.
847 :
848 : ??? Code above formerly short-circuited this, for most integer
849 : mode pairs, with a force_reg in from_mode followed by a recursive
850 : call to this routine. Appears always to have been wrong. */
851 0 : if (GET_MODE_PRECISION (to_mode) < GET_MODE_PRECISION (from_mode))
852 : {
853 0 : rtx temp = force_reg (to_mode, gen_lowpart (to_mode, from));
854 0 : emit_move_insn (to, temp);
855 0 : return;
856 : }
857 :
858 : /* Mode combination is not recognized. */
859 0 : gcc_unreachable ();
860 : }
861 :
862 : /* Return an rtx for a value that would result
863 : from converting X to mode MODE.
864 : Both X and MODE may be floating, or both integer.
865 : UNSIGNEDP is nonzero if X is an unsigned value.
866 : This can be done by referring to a part of X in place
867 : or by copying to a new temporary with conversion. */
868 :
869 : rtx
870 2379183 : convert_to_mode (machine_mode mode, rtx x, int unsignedp)
871 : {
872 2379183 : return convert_modes (mode, VOIDmode, x, unsignedp);
873 : }
874 :
875 : /* Return an rtx for a value that would result
876 : from converting X from mode OLDMODE to mode MODE.
877 : Both modes may be floating, or both integer.
878 : UNSIGNEDP is nonzero if X is an unsigned value.
879 :
880 : This can be done by referring to a part of X in place
881 : or by copying to a new temporary with conversion.
882 :
883 : You can give VOIDmode for OLDMODE, if you are sure X has a nonvoid mode. */
884 :
885 : rtx
886 5228833 : convert_modes (machine_mode mode, machine_mode oldmode, rtx x, int unsignedp)
887 : {
888 5228833 : rtx temp;
889 5228833 : scalar_int_mode int_mode;
890 :
891 : /* If FROM is a SUBREG that indicates that we have already done at least
892 : the required extension, strip it. */
893 :
894 5228833 : if (GET_CODE (x) == SUBREG
895 141949 : && SUBREG_PROMOTED_VAR_P (x)
896 5228840 : && is_a <scalar_int_mode> (mode, &int_mode)
897 5228840 : && (GET_MODE_PRECISION (subreg_promoted_mode (x))
898 14 : >= GET_MODE_PRECISION (int_mode))
899 5228847 : && SUBREG_CHECK_PROMOTED_SIGN (x, unsignedp))
900 : {
901 7 : scalar_int_mode int_orig_mode;
902 7 : scalar_int_mode int_inner_mode;
903 7 : machine_mode orig_mode = GET_MODE (x);
904 7 : x = gen_lowpart (int_mode, SUBREG_REG (x));
905 :
906 : /* Preserve SUBREG_PROMOTED_VAR_P if the new mode is wider than
907 : the original mode, but narrower than the inner mode. */
908 7 : if (GET_CODE (x) == SUBREG
909 0 : && is_a <scalar_int_mode> (orig_mode, &int_orig_mode)
910 0 : && GET_MODE_PRECISION (int_mode)
911 0 : > GET_MODE_PRECISION (int_orig_mode)
912 0 : && is_a <scalar_int_mode> (GET_MODE (SUBREG_REG (x)),
913 : &int_inner_mode)
914 7 : && GET_MODE_PRECISION (int_inner_mode)
915 0 : > GET_MODE_PRECISION (int_mode))
916 : {
917 0 : SUBREG_PROMOTED_VAR_P (x) = 1;
918 0 : SUBREG_PROMOTED_SET (x, unsignedp);
919 : }
920 : }
921 :
922 5228833 : if (GET_MODE (x) != VOIDmode)
923 2868328 : oldmode = GET_MODE (x);
924 :
925 5228833 : if (mode == oldmode)
926 : return x;
927 :
928 3800821 : if (CONST_SCALAR_INT_P (x)
929 3800821 : && is_a <scalar_int_mode> (mode, &int_mode))
930 : {
931 : /* If the caller did not tell us the old mode, then there is not
932 : much to do with respect to canonicalization. We have to
933 : assume that all the bits are significant. */
934 2038857 : if (!is_a <scalar_int_mode> (oldmode))
935 1707442 : oldmode = MAX_MODE_INT;
936 2038857 : wide_int w = wide_int::from (rtx_mode_t (x, oldmode),
937 2038857 : GET_MODE_PRECISION (int_mode),
938 2689114 : unsignedp ? UNSIGNED : SIGNED);
939 2038857 : return immed_wide_int_const (w, int_mode);
940 2038857 : }
941 :
942 : /* We can do this with a gen_lowpart if both desired and current modes
943 : are integer, and this is either a constant integer, a register, or a
944 : non-volatile MEM. */
945 1761964 : scalar_int_mode int_oldmode;
946 1761964 : if (is_int_mode (mode, &int_mode)
947 1667237 : && is_int_mode (oldmode, &int_oldmode)
948 1667237 : && GET_MODE_PRECISION (int_mode) <= GET_MODE_PRECISION (int_oldmode)
949 455865 : && ((MEM_P (x) && !MEM_VOLATILE_P (x) && direct_load[(int) int_mode])
950 12944 : || CONST_POLY_INT_P (x)
951 442921 : || (REG_P (x)
952 408349 : && (!HARD_REGISTER_P (x)
953 1073 : || targetm.hard_regno_mode_ok (REGNO (x), int_mode))
954 421293 : && TRULY_NOOP_TRUNCATION_MODES_P (int_mode, GET_MODE (x)))))
955 421293 : return gen_lowpart (int_mode, x);
956 :
957 : /* Converting from integer constant into mode is always equivalent to an
958 : subreg operation. */
959 1340671 : if (VECTOR_MODE_P (mode) && GET_MODE (x) == VOIDmode)
960 : {
961 0 : gcc_assert (known_eq (GET_MODE_BITSIZE (mode),
962 : GET_MODE_BITSIZE (oldmode)));
963 0 : return force_subreg (mode, x, oldmode, 0);
964 : }
965 :
966 1340671 : temp = gen_reg_rtx (mode);
967 1340671 : convert_move (temp, x, unsignedp);
968 1340671 : return temp;
969 : }
970 :
971 : /* Variant of convert_modes for ABI parameter passing/return.
972 : Return an rtx for a value that would result from converting X from
973 : a floating point mode FMODE to wider integer mode MODE. */
974 :
975 : rtx
976 0 : convert_float_to_wider_int (machine_mode mode, machine_mode fmode, rtx x)
977 : {
978 0 : gcc_assert (SCALAR_INT_MODE_P (mode) && SCALAR_FLOAT_MODE_P (fmode));
979 0 : scalar_int_mode tmp_mode = int_mode_for_mode (fmode).require ();
980 0 : rtx tmp = force_reg (tmp_mode, gen_lowpart (tmp_mode, x));
981 0 : return convert_modes (mode, tmp_mode, tmp, 1);
982 : }
983 :
984 : /* Variant of convert_modes for ABI parameter passing/return.
985 : Return an rtx for a value that would result from converting X from
986 : an integer mode IMODE to a narrower floating point mode MODE. */
987 :
988 : rtx
989 0 : convert_wider_int_to_float (machine_mode mode, machine_mode imode, rtx x)
990 : {
991 0 : gcc_assert (SCALAR_FLOAT_MODE_P (mode) && SCALAR_INT_MODE_P (imode));
992 0 : scalar_int_mode tmp_mode = int_mode_for_mode (mode).require ();
993 0 : rtx tmp = force_reg (tmp_mode, gen_lowpart (tmp_mode, x));
994 0 : return gen_lowpart_SUBREG (mode, tmp);
995 : }
996 :
997 : /* Return the largest alignment we can use for doing a move (or store)
998 : of MAX_PIECES. ALIGN is the largest alignment we could use. */
999 :
1000 : static unsigned int
1001 2798324 : alignment_for_piecewise_move (unsigned int max_pieces, unsigned int align)
1002 : {
1003 2798324 : scalar_int_mode tmode
1004 2798324 : = int_mode_for_size (max_pieces * BITS_PER_UNIT, 0).require ();
1005 :
1006 2798324 : if (align >= GET_MODE_ALIGNMENT (tmode))
1007 2207299 : align = GET_MODE_ALIGNMENT (tmode);
1008 : else
1009 : {
1010 591025 : scalar_int_mode xmode = NARROWEST_INT_MODE;
1011 591025 : opt_scalar_int_mode mode_iter;
1012 3562585 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
1013 : {
1014 3556857 : tmode = mode_iter.require ();
1015 3556857 : if (GET_MODE_SIZE (tmode) > max_pieces
1016 3556857 : || targetm.slow_unaligned_access (tmode, align))
1017 : break;
1018 2971560 : xmode = tmode;
1019 : }
1020 :
1021 591025 : align = MAX (align, GET_MODE_ALIGNMENT (xmode));
1022 : }
1023 :
1024 2798324 : return align;
1025 : }
1026 :
1027 : /* Return true if we know how to implement OP using vectors of bytes. */
1028 : static bool
1029 5924346 : can_use_qi_vectors (by_pieces_operation op)
1030 : {
1031 5924346 : return (op == COMPARE_BY_PIECES
1032 0 : || op == SET_BY_PIECES
1033 0 : || op == CLEAR_BY_PIECES);
1034 : }
1035 :
1036 : /* Return true if optabs exists for the mode and certain by pieces
1037 : operations. */
1038 : static bool
1039 26220883 : by_pieces_mode_supported_p (fixed_size_mode mode, by_pieces_operation op)
1040 : {
1041 26220883 : if (optab_handler (mov_optab, mode) == CODE_FOR_nothing)
1042 : return false;
1043 :
1044 24847100 : if ((op == SET_BY_PIECES || op == CLEAR_BY_PIECES)
1045 695176 : && VECTOR_MODE_P (mode)
1046 25346442 : && optab_handler (vec_duplicate_optab, mode) == CODE_FOR_nothing)
1047 : return false;
1048 :
1049 24798308 : if (op == COMPARE_BY_PIECES
1050 24798308 : && !can_compare_p (EQ, mode, ccp_jump))
1051 : return false;
1052 :
1053 : return true;
1054 : }
1055 :
1056 : /* Return the widest mode that can be used to perform part of an
1057 : operation OP on SIZE bytes. Try to use QI vector modes where
1058 : possible. */
1059 : static fixed_size_mode
1060 5427415 : widest_fixed_size_mode_for_size (unsigned int size, by_pieces_operation op)
1061 : {
1062 5427415 : fixed_size_mode result = NARROWEST_INT_MODE;
1063 :
1064 5427415 : gcc_checking_assert (size > 1);
1065 :
1066 : /* Use QI vector only if size is wider than a WORD. */
1067 5427415 : if (can_use_qi_vectors (op))
1068 : {
1069 735900 : machine_mode mode;
1070 735900 : fixed_size_mode candidate;
1071 12150034 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_INT)
1072 12150034 : if (is_a<fixed_size_mode> (mode, &candidate)
1073 13258200 : && GET_MODE_SIZE (candidate) > UNITS_PER_WORD
1074 13075540 : && GET_MODE_INNER (candidate) == QImode)
1075 : {
1076 5722460 : if (GET_MODE_SIZE (candidate) >= size)
1077 : break;
1078 2125330 : if (by_pieces_mode_supported_p (candidate, op))
1079 11414134 : result = candidate;
1080 : }
1081 :
1082 735900 : if (result != NARROWEST_INT_MODE)
1083 477760 : return result;
1084 : }
1085 :
1086 4949655 : opt_scalar_int_mode tmode;
1087 4949655 : scalar_int_mode mode;
1088 39597240 : FOR_EACH_MODE_IN_CLASS (tmode, MODE_INT)
1089 : {
1090 34647585 : mode = tmode.require ();
1091 34647585 : if (GET_MODE_SIZE (mode) < size
1092 34647585 : && by_pieces_mode_supported_p (mode, op))
1093 24053832 : result = mode;
1094 : }
1095 :
1096 4949655 : return result;
1097 : }
1098 :
1099 : /* Determine whether an operation OP on LEN bytes with alignment ALIGN can
1100 : and should be performed piecewise. */
1101 :
1102 : static bool
1103 969929 : can_do_by_pieces (unsigned HOST_WIDE_INT len, unsigned int align,
1104 : enum by_pieces_operation op)
1105 : {
1106 969929 : return targetm.use_by_pieces_infrastructure_p (len, align, op,
1107 969929 : optimize_insn_for_speed_p ());
1108 : }
1109 :
1110 : /* Determine whether the LEN bytes can be moved by using several move
1111 : instructions. Return nonzero if a call to move_by_pieces should
1112 : succeed. */
1113 :
1114 : bool
1115 900209 : can_move_by_pieces (unsigned HOST_WIDE_INT len, unsigned int align)
1116 : {
1117 900209 : return can_do_by_pieces (len, align, MOVE_BY_PIECES);
1118 : }
1119 :
1120 : /* Return number of insns required to perform operation OP by pieces
1121 : for L bytes. ALIGN (in bits) is maximum alignment we can assume. */
1122 :
1123 : unsigned HOST_WIDE_INT
1124 1977211 : by_pieces_ninsns (unsigned HOST_WIDE_INT l, unsigned int align,
1125 : unsigned int max_size, by_pieces_operation op)
1126 : {
1127 1977211 : unsigned HOST_WIDE_INT n_insns = 0;
1128 1977211 : fixed_size_mode mode;
1129 :
1130 1977211 : if (targetm.overlap_op_by_pieces_p ())
1131 : {
1132 : /* NB: Round up L and ALIGN to the widest integer mode for
1133 : MAX_SIZE. */
1134 1977211 : mode = widest_fixed_size_mode_for_size (max_size, op);
1135 1977211 : gcc_assert (optab_handler (mov_optab, mode) != CODE_FOR_nothing);
1136 3954422 : unsigned HOST_WIDE_INT up = ROUND_UP (l, GET_MODE_SIZE (mode));
1137 1977211 : if (up > l)
1138 : l = up;
1139 1977211 : align = GET_MODE_ALIGNMENT (mode);
1140 : }
1141 :
1142 1977211 : align = alignment_for_piecewise_move (MOVE_MAX_PIECES, align);
1143 :
1144 5940161 : while (max_size > 1 && l > 0)
1145 : {
1146 1985739 : mode = widest_fixed_size_mode_for_size (max_size, op);
1147 1985739 : gcc_assert (optab_handler (mov_optab, mode) != CODE_FOR_nothing);
1148 :
1149 1985739 : unsigned int modesize = GET_MODE_SIZE (mode);
1150 :
1151 1985739 : if (align >= GET_MODE_ALIGNMENT (mode))
1152 : {
1153 1985739 : unsigned HOST_WIDE_INT n_pieces = l / modesize;
1154 1985739 : l %= modesize;
1155 1985739 : switch (op)
1156 : {
1157 1916019 : default:
1158 1916019 : n_insns += n_pieces;
1159 1916019 : break;
1160 :
1161 69720 : case COMPARE_BY_PIECES:
1162 69720 : int batch = targetm.compare_by_pieces_branch_ratio (mode);
1163 69720 : int batch_ops = 4 * batch - 1;
1164 69720 : unsigned HOST_WIDE_INT full = n_pieces / batch;
1165 69720 : n_insns += full * batch_ops;
1166 69720 : if (n_pieces % batch != 0)
1167 0 : n_insns++;
1168 : break;
1169 :
1170 : }
1171 : }
1172 1985739 : max_size = modesize;
1173 : }
1174 :
1175 1977211 : gcc_assert (!l);
1176 1977211 : return n_insns;
1177 : }
1178 :
1179 : /* Used when performing piecewise block operations, holds information
1180 : about one of the memory objects involved. The member functions
1181 : can be used to generate code for loading from the object and
1182 : updating the address when iterating. */
1183 :
1184 : class pieces_addr
1185 : {
1186 : /* The object being referenced, a MEM. Can be NULL_RTX to indicate
1187 : stack pushes. */
1188 : rtx m_obj;
1189 : /* The address of the object. Can differ from that seen in the
1190 : MEM rtx if we copied the address to a register. */
1191 : rtx m_addr;
1192 : /* Nonzero if the address on the object has an autoincrement already,
1193 : signifies whether that was an increment or decrement. */
1194 : signed char m_addr_inc;
1195 : /* Nonzero if we intend to use autoinc without the address already
1196 : having autoinc form. We will insert add insns around each memory
1197 : reference, expecting later passes to form autoinc addressing modes.
1198 : The only supported options are predecrement and postincrement. */
1199 : signed char m_explicit_inc;
1200 : /* True if we have either of the two possible cases of using
1201 : autoincrement. */
1202 : bool m_auto;
1203 : /* True if this is an address to be used for load operations rather
1204 : than stores. */
1205 : bool m_is_load;
1206 :
1207 : /* Optionally, a function to obtain constants for any given offset into
1208 : the objects, and data associated with it. */
1209 : by_pieces_constfn m_constfn;
1210 : void *m_cfndata;
1211 : public:
1212 : pieces_addr (rtx, bool, by_pieces_constfn, void *);
1213 : rtx adjust (fixed_size_mode, HOST_WIDE_INT, by_pieces_prev * = nullptr);
1214 : void increment_address (HOST_WIDE_INT);
1215 : void maybe_predec (HOST_WIDE_INT);
1216 : void maybe_postinc (HOST_WIDE_INT);
1217 : void decide_autoinc (machine_mode, bool, HOST_WIDE_INT);
1218 764267 : int get_addr_inc ()
1219 : {
1220 764267 : return m_addr_inc;
1221 : }
1222 : };
1223 :
1224 : /* Initialize a pieces_addr structure from an object OBJ. IS_LOAD is
1225 : true if the operation to be performed on this object is a load
1226 : rather than a store. For stores, OBJ can be NULL, in which case we
1227 : assume the operation is a stack push. For loads, the optional
1228 : CONSTFN and its associated CFNDATA can be used in place of the
1229 : memory load. */
1230 :
1231 1528534 : pieces_addr::pieces_addr (rtx obj, bool is_load, by_pieces_constfn constfn,
1232 : void *cfndata)
1233 1528534 : : m_obj (obj), m_is_load (is_load), m_constfn (constfn), m_cfndata (cfndata)
1234 : {
1235 1528534 : m_addr_inc = 0;
1236 1528534 : m_auto = false;
1237 1528534 : if (obj)
1238 : {
1239 1399133 : rtx addr = XEXP (obj, 0);
1240 1399133 : rtx_code code = GET_CODE (addr);
1241 1399133 : m_addr = addr;
1242 1399133 : bool dec = code == PRE_DEC || code == POST_DEC;
1243 1399133 : bool inc = code == PRE_INC || code == POST_INC;
1244 1399133 : m_auto = inc || dec;
1245 1399133 : if (m_auto)
1246 0 : m_addr_inc = dec ? -1 : 1;
1247 :
1248 : /* While we have always looked for these codes here, the code
1249 : implementing the memory operation has never handled them.
1250 : Support could be added later if necessary or beneficial. */
1251 1399133 : gcc_assert (code != PRE_INC && code != POST_DEC);
1252 : }
1253 : else
1254 : {
1255 129401 : m_addr = NULL_RTX;
1256 129401 : if (!is_load)
1257 : {
1258 54 : m_auto = true;
1259 54 : if (STACK_GROWS_DOWNWARD)
1260 54 : m_addr_inc = -1;
1261 : else
1262 : m_addr_inc = 1;
1263 : }
1264 : else
1265 129347 : gcc_assert (constfn != NULL);
1266 : }
1267 1528534 : m_explicit_inc = 0;
1268 1528534 : if (constfn)
1269 155654 : gcc_assert (is_load);
1270 1528534 : }
1271 :
1272 : /* Decide whether to use autoinc for an address involved in a memory op.
1273 : MODE is the mode of the accesses, REVERSE is true if we've decided to
1274 : perform the operation starting from the end, and LEN is the length of
1275 : the operation. Don't override an earlier decision to set m_auto. */
1276 :
1277 : void
1278 382082 : pieces_addr::decide_autoinc (machine_mode ARG_UNUSED (mode), bool reverse,
1279 : HOST_WIDE_INT len)
1280 : {
1281 382082 : if (m_auto || m_obj == NULL_RTX)
1282 : return;
1283 :
1284 348199 : bool use_predec = (m_is_load
1285 : ? USE_LOAD_PRE_DECREMENT (mode)
1286 : : USE_STORE_PRE_DECREMENT (mode));
1287 348199 : bool use_postinc = (m_is_load
1288 : ? USE_LOAD_POST_INCREMENT (mode)
1289 : : USE_STORE_POST_INCREMENT (mode));
1290 348199 : machine_mode addr_mode = get_address_mode (m_obj);
1291 :
1292 348199 : if (use_predec && reverse)
1293 : {
1294 : m_addr = copy_to_mode_reg (addr_mode,
1295 : plus_constant (addr_mode,
1296 : m_addr, len));
1297 : m_auto = true;
1298 : m_explicit_inc = -1;
1299 : }
1300 348199 : else if (use_postinc && !reverse)
1301 : {
1302 : m_addr = copy_to_mode_reg (addr_mode, m_addr);
1303 : m_auto = true;
1304 : m_explicit_inc = 1;
1305 : }
1306 348199 : else if (CONSTANT_P (m_addr))
1307 71963 : m_addr = copy_to_mode_reg (addr_mode, m_addr);
1308 : }
1309 :
1310 : /* Adjust the address to refer to the data at OFFSET in MODE. If we
1311 : are using autoincrement for this address, we don't add the offset,
1312 : but we still modify the MEM's properties. */
1313 :
1314 : rtx
1315 4182254 : pieces_addr::adjust (fixed_size_mode mode, HOST_WIDE_INT offset,
1316 : by_pieces_prev *prev)
1317 : {
1318 4182254 : if (m_constfn)
1319 : /* Pass the previous data to m_constfn. */
1320 389267 : return m_constfn (m_cfndata, prev, offset, mode);
1321 3792987 : if (m_obj == NULL_RTX)
1322 : return NULL_RTX;
1323 3792900 : if (m_auto)
1324 0 : return adjust_automodify_address (m_obj, mode, m_addr, offset);
1325 : else
1326 3792900 : return adjust_address (m_obj, mode, offset);
1327 : }
1328 :
1329 : /* Emit an add instruction to increment the address by SIZE. */
1330 :
1331 : void
1332 0 : pieces_addr::increment_address (HOST_WIDE_INT size)
1333 : {
1334 0 : rtx amount = gen_int_mode (size, GET_MODE (m_addr));
1335 0 : emit_insn (gen_add2_insn (m_addr, amount));
1336 0 : }
1337 :
1338 : /* If we are supposed to decrement the address after each access, emit code
1339 : to do so now. Increment by SIZE (which has should have the correct sign
1340 : already). */
1341 :
1342 : void
1343 4181386 : pieces_addr::maybe_predec (HOST_WIDE_INT size)
1344 : {
1345 4181386 : if (m_explicit_inc >= 0)
1346 4181386 : return;
1347 0 : gcc_assert (HAVE_PRE_DECREMENT);
1348 : increment_address (size);
1349 : }
1350 :
1351 : /* If we are supposed to decrement the address after each access, emit code
1352 : to do so now. Increment by SIZE. */
1353 :
1354 : void
1355 4181409 : pieces_addr::maybe_postinc (HOST_WIDE_INT size)
1356 : {
1357 4181409 : if (m_explicit_inc <= 0)
1358 4181409 : return;
1359 0 : gcc_assert (HAVE_POST_INCREMENT);
1360 : increment_address (size);
1361 : }
1362 :
1363 : /* This structure is used by do_op_by_pieces to describe the operation
1364 : to be performed. */
1365 :
1366 : class op_by_pieces_d
1367 : {
1368 : private:
1369 : fixed_size_mode get_usable_mode (fixed_size_mode, unsigned int);
1370 : fixed_size_mode smallest_fixed_size_mode_for_size (unsigned int);
1371 :
1372 : protected:
1373 : pieces_addr m_to, m_from;
1374 : /* Make m_len read-only so that smallest_fixed_size_mode_for_size can
1375 : use it to check the valid mode size. */
1376 : const unsigned HOST_WIDE_INT m_len;
1377 : HOST_WIDE_INT m_offset;
1378 : unsigned int m_align;
1379 : unsigned int m_max_size;
1380 : bool m_reverse;
1381 : /* True if this is a stack push. */
1382 : bool m_push;
1383 : /* True if targetm.overlap_op_by_pieces_p () returns true. */
1384 : bool m_overlap_op_by_pieces;
1385 : /* The type of operation that we're performing. */
1386 : by_pieces_operation m_op;
1387 :
1388 : /* Virtual functions, overridden by derived classes for the specific
1389 : operation. */
1390 : virtual void generate (rtx, rtx, machine_mode) = 0;
1391 : virtual bool prepare_mode (machine_mode, unsigned int) = 0;
1392 1159799 : virtual void finish_mode (machine_mode)
1393 : {
1394 1159799 : }
1395 :
1396 : public:
1397 : op_by_pieces_d (unsigned int, rtx, bool, rtx, bool, by_pieces_constfn,
1398 : void *, unsigned HOST_WIDE_INT, unsigned int, bool,
1399 : by_pieces_operation);
1400 : void run ();
1401 : };
1402 :
1403 : /* The constructor for an op_by_pieces_d structure. We require two
1404 : objects named TO and FROM, which are identified as loads or stores
1405 : by TO_LOAD and FROM_LOAD. If FROM is a load, the optional FROM_CFN
1406 : and its associated FROM_CFN_DATA can be used to replace loads with
1407 : constant values. MAX_PIECES describes the maximum number of bytes
1408 : at a time which can be moved efficiently. LEN describes the length
1409 : of the operation. */
1410 :
1411 764267 : op_by_pieces_d::op_by_pieces_d (unsigned int max_pieces, rtx to,
1412 : bool to_load, rtx from, bool from_load,
1413 : by_pieces_constfn from_cfn,
1414 : void *from_cfn_data,
1415 : unsigned HOST_WIDE_INT len,
1416 : unsigned int align, bool push,
1417 : by_pieces_operation op)
1418 764267 : : m_to (to, to_load, NULL, NULL),
1419 764267 : m_from (from, from_load, from_cfn, from_cfn_data),
1420 764267 : m_len (len), m_max_size (max_pieces + 1),
1421 764267 : m_push (push), m_op (op)
1422 : {
1423 764267 : int toi = m_to.get_addr_inc ();
1424 764267 : int fromi = m_from.get_addr_inc ();
1425 764267 : if (toi >= 0 && fromi >= 0)
1426 764213 : m_reverse = false;
1427 54 : else if (toi <= 0 && fromi <= 0)
1428 54 : m_reverse = true;
1429 : else
1430 0 : gcc_unreachable ();
1431 :
1432 764267 : m_offset = m_reverse ? len : 0;
1433 2798330 : align = MIN (to ? MEM_ALIGN (to) : align,
1434 : from ? MEM_ALIGN (from) : align);
1435 :
1436 : /* If copying requires more than two move insns,
1437 : copy addresses to registers (to make displacements shorter)
1438 : and use post-increment if available. */
1439 764267 : if (by_pieces_ninsns (len, align, m_max_size, MOVE_BY_PIECES) > 2)
1440 : {
1441 : /* Find the mode of the largest comparison. */
1442 191041 : fixed_size_mode mode
1443 191041 : = widest_fixed_size_mode_for_size (m_max_size, m_op);
1444 :
1445 191041 : m_from.decide_autoinc (mode, m_reverse, len);
1446 191041 : m_to.decide_autoinc (mode, m_reverse, len);
1447 : }
1448 :
1449 764277 : align = alignment_for_piecewise_move (MOVE_MAX_PIECES, align);
1450 764267 : m_align = align;
1451 :
1452 764267 : m_overlap_op_by_pieces = targetm.overlap_op_by_pieces_p ();
1453 764267 : }
1454 :
1455 : /* This function returns the largest usable integer mode for LEN bytes
1456 : whose size is no bigger than size of MODE. */
1457 :
1458 : fixed_size_mode
1459 1254705 : op_by_pieces_d::get_usable_mode (fixed_size_mode mode, unsigned int len)
1460 : {
1461 1555577 : unsigned int size;
1462 1856449 : do
1463 : {
1464 1555577 : size = GET_MODE_SIZE (mode);
1465 1555577 : if (len >= size && prepare_mode (mode, m_align))
1466 : break;
1467 : /* widest_fixed_size_mode_for_size checks SIZE > 1. */
1468 300872 : mode = widest_fixed_size_mode_for_size (size, m_op);
1469 : }
1470 : while (1);
1471 1254705 : return mode;
1472 : }
1473 :
1474 : /* Return the smallest integer or QI vector mode that is not narrower
1475 : than SIZE bytes. */
1476 :
1477 : fixed_size_mode
1478 496931 : op_by_pieces_d::smallest_fixed_size_mode_for_size (unsigned int size)
1479 : {
1480 : /* Use QI vector only for > size of WORD. */
1481 509149 : if (can_use_qi_vectors (m_op) && size > UNITS_PER_WORD)
1482 : {
1483 8235 : machine_mode mode;
1484 8235 : fixed_size_mode candidate;
1485 111442 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_INT)
1486 111442 : if (is_a<fixed_size_mode> (mode, &candidate)
1487 222884 : && GET_MODE_INNER (candidate) == QImode)
1488 : {
1489 : /* Don't return a mode wider than M_LEN. */
1490 103616 : if (GET_MODE_SIZE (candidate) > m_len)
1491 : break;
1492 :
1493 49871 : if (GET_MODE_SIZE (candidate) >= size
1494 49871 : && by_pieces_mode_supported_p (candidate, m_op))
1495 6298 : return candidate;
1496 : }
1497 : }
1498 :
1499 490633 : return smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
1500 : }
1501 :
1502 : /* This function contains the main loop used for expanding a block
1503 : operation. First move what we can in the largest integer mode,
1504 : then go to successively smaller modes. For every access, call
1505 : GENFUN with the two operands and the EXTRA_DATA. */
1506 :
1507 : void
1508 764267 : op_by_pieces_d::run ()
1509 : {
1510 764267 : if (m_len == 0)
1511 : return;
1512 :
1513 757774 : unsigned HOST_WIDE_INT length = m_len;
1514 :
1515 : /* widest_fixed_size_mode_for_size checks M_MAX_SIZE > 1. */
1516 757774 : fixed_size_mode mode
1517 757774 : = widest_fixed_size_mode_for_size (m_max_size, m_op);
1518 757774 : mode = get_usable_mode (mode, length);
1519 :
1520 757774 : by_pieces_prev to_prev = { nullptr, mode };
1521 757774 : by_pieces_prev from_prev = { nullptr, mode };
1522 :
1523 1254705 : do
1524 : {
1525 1254705 : unsigned int size = GET_MODE_SIZE (mode);
1526 1254705 : rtx to1 = NULL_RTX, from1;
1527 :
1528 3345398 : while (length >= size)
1529 : {
1530 2090693 : if (m_reverse)
1531 87 : m_offset -= size;
1532 :
1533 2090693 : to1 = m_to.adjust (mode, m_offset, &to_prev);
1534 2090693 : to_prev.data = to1;
1535 2090693 : to_prev.mode = mode;
1536 2090693 : from1 = m_from.adjust (mode, m_offset, &from_prev);
1537 2090693 : from_prev.data = from1;
1538 2090693 : from_prev.mode = mode;
1539 :
1540 2090693 : m_to.maybe_predec (-(HOST_WIDE_INT)size);
1541 2090693 : m_from.maybe_predec (-(HOST_WIDE_INT)size);
1542 :
1543 2090693 : generate (to1, from1, mode);
1544 :
1545 2090693 : m_to.maybe_postinc (size);
1546 2090693 : m_from.maybe_postinc (size);
1547 :
1548 2090693 : if (!m_reverse)
1549 2090606 : m_offset += size;
1550 :
1551 2090693 : length -= size;
1552 : }
1553 :
1554 1254705 : finish_mode (mode);
1555 :
1556 1254705 : if (length == 0)
1557 : return;
1558 :
1559 496931 : if (!m_push && m_overlap_op_by_pieces)
1560 : {
1561 : /* NB: Generate overlapping operations if it is not a stack
1562 : push since stack push must not overlap. Get the smallest
1563 : fixed size mode for M_LEN bytes. */
1564 496931 : mode = smallest_fixed_size_mode_for_size (length);
1565 993862 : mode = get_usable_mode (mode, GET_MODE_SIZE (mode));
1566 496931 : int gap = GET_MODE_SIZE (mode) - length;
1567 496931 : if (gap > 0)
1568 : {
1569 : /* If size of MODE > M_LEN, generate the last operation
1570 : in MODE for the remaining bytes with overlapping memory
1571 : from the previois operation. */
1572 49117 : if (m_reverse)
1573 0 : m_offset += gap;
1574 : else
1575 49117 : m_offset -= gap;
1576 49117 : length += gap;
1577 : }
1578 : }
1579 : else
1580 : {
1581 : /* widest_fixed_size_mode_for_size checks SIZE > 1. */
1582 0 : mode = widest_fixed_size_mode_for_size (size, m_op);
1583 0 : mode = get_usable_mode (mode, length);
1584 : }
1585 : }
1586 : while (1);
1587 : }
1588 :
1589 : /* Derived class from op_by_pieces_d, providing support for block move
1590 : operations. */
1591 :
1592 : #ifdef PUSH_ROUNDING
1593 : #define PUSHG_P(to) ((to) == nullptr)
1594 : #else
1595 : #define PUSHG_P(to) false
1596 : #endif
1597 :
1598 : class move_by_pieces_d : public op_by_pieces_d
1599 : {
1600 : insn_gen_fn m_gen_fun;
1601 : void generate (rtx, rtx, machine_mode) final override;
1602 : bool prepare_mode (machine_mode, unsigned int) final override;
1603 :
1604 : public:
1605 572014 : move_by_pieces_d (rtx to, rtx from, unsigned HOST_WIDE_INT len,
1606 : unsigned int align)
1607 572014 : : op_by_pieces_d (MOVE_MAX_PIECES, to, false, from, true, NULL,
1608 1144028 : NULL, len, align, PUSHG_P (to), MOVE_BY_PIECES)
1609 : {
1610 572014 : }
1611 : rtx finish_retmode (memop_ret);
1612 : };
1613 :
1614 : /* Return true if MODE can be used for a set of copies, given an
1615 : alignment ALIGN. Prepare whatever data is necessary for later
1616 : calls to generate. */
1617 :
1618 : bool
1619 953310 : move_by_pieces_d::prepare_mode (machine_mode mode, unsigned int align)
1620 : {
1621 953310 : insn_code icode = optab_handler (mov_optab, mode);
1622 953310 : m_gen_fun = GEN_FCN (icode);
1623 953310 : return icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode);
1624 : }
1625 :
1626 : /* A callback used when iterating for a compare_by_pieces_operation.
1627 : OP0 and OP1 are the values that have been loaded and should be
1628 : compared in MODE. If OP0 is NULL, this means we should generate a
1629 : push; otherwise EXTRA_DATA holds a pointer to a pointer to the insn
1630 : gen function that should be used to generate the mode. */
1631 :
1632 : void
1633 1655922 : move_by_pieces_d::generate (rtx op0, rtx op1,
1634 : machine_mode mode ATTRIBUTE_UNUSED)
1635 : {
1636 : #ifdef PUSH_ROUNDING
1637 1655922 : if (op0 == NULL_RTX)
1638 : {
1639 87 : emit_single_push_insn (mode, op1, NULL);
1640 87 : return;
1641 : }
1642 : #endif
1643 1655835 : emit_insn (m_gen_fun (op0, op1));
1644 : }
1645 :
1646 : /* Perform the final adjustment at the end of a string to obtain the
1647 : correct return value for the block operation.
1648 : Return value is based on RETMODE argument. */
1649 :
1650 : rtx
1651 0 : move_by_pieces_d::finish_retmode (memop_ret retmode)
1652 : {
1653 0 : gcc_assert (!m_reverse);
1654 0 : if (retmode == RETURN_END_MINUS_ONE)
1655 : {
1656 0 : m_to.maybe_postinc (-1);
1657 0 : --m_offset;
1658 : }
1659 0 : return m_to.adjust (QImode, m_offset);
1660 : }
1661 :
1662 : /* Generate several move instructions to copy LEN bytes from block FROM to
1663 : block TO. (These are MEM rtx's with BLKmode).
1664 :
1665 : If PUSH_ROUNDING is defined and TO is NULL, emit_single_push_insn is
1666 : used to push FROM to the stack.
1667 :
1668 : ALIGN is maximum stack alignment we can assume.
1669 :
1670 : Return value is based on RETMODE argument. */
1671 :
1672 : rtx
1673 572014 : move_by_pieces (rtx to, rtx from, unsigned HOST_WIDE_INT len,
1674 : unsigned int align, memop_ret retmode)
1675 : {
1676 : #ifndef PUSH_ROUNDING
1677 : if (to == NULL)
1678 : gcc_unreachable ();
1679 : #endif
1680 :
1681 572014 : move_by_pieces_d data (to, from, len, align);
1682 :
1683 572014 : data.run ();
1684 :
1685 572014 : if (retmode != RETURN_BEGIN)
1686 0 : return data.finish_retmode (retmode);
1687 : else
1688 : return to;
1689 : }
1690 :
1691 : /* Derived class from op_by_pieces_d, providing support for block move
1692 : operations. */
1693 :
1694 : class store_by_pieces_d : public op_by_pieces_d
1695 : {
1696 : insn_gen_fn m_gen_fun;
1697 :
1698 : void generate (rtx, rtx, machine_mode) final override;
1699 : bool prepare_mode (machine_mode, unsigned int) final override;
1700 :
1701 : public:
1702 129347 : store_by_pieces_d (rtx to, by_pieces_constfn cfn, void *cfn_data,
1703 : unsigned HOST_WIDE_INT len, unsigned int align,
1704 : by_pieces_operation op)
1705 129347 : : op_by_pieces_d (STORE_MAX_PIECES, to, false, NULL_RTX, true, cfn,
1706 258694 : cfn_data, len, align, false, op)
1707 : {
1708 129347 : }
1709 : rtx finish_retmode (memop_ret);
1710 : };
1711 :
1712 : /* Return true if MODE can be used for a set of stores, given an
1713 : alignment ALIGN. Prepare whatever data is necessary for later
1714 : calls to generate. */
1715 :
1716 : bool
1717 206489 : store_by_pieces_d::prepare_mode (machine_mode mode, unsigned int align)
1718 : {
1719 206489 : insn_code icode = optab_handler (mov_optab, mode);
1720 206489 : m_gen_fun = GEN_FCN (icode);
1721 206489 : return icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode);
1722 : }
1723 :
1724 : /* A callback used when iterating for a store_by_pieces_operation.
1725 : OP0 and OP1 are the values that have been loaded and should be
1726 : compared in MODE. If OP0 is NULL, this means we should generate a
1727 : push; otherwise EXTRA_DATA holds a pointer to a pointer to the insn
1728 : gen function that should be used to generate the mode. */
1729 :
1730 : void
1731 336612 : store_by_pieces_d::generate (rtx op0, rtx op1, machine_mode)
1732 : {
1733 336612 : emit_insn (m_gen_fun (op0, op1));
1734 336612 : }
1735 :
1736 : /* Perform the final adjustment at the end of a string to obtain the
1737 : correct return value for the block operation.
1738 : Return value is based on RETMODE argument. */
1739 :
1740 : rtx
1741 868 : store_by_pieces_d::finish_retmode (memop_ret retmode)
1742 : {
1743 868 : gcc_assert (!m_reverse);
1744 868 : if (retmode == RETURN_END_MINUS_ONE)
1745 : {
1746 23 : m_to.maybe_postinc (-1);
1747 23 : --m_offset;
1748 : }
1749 868 : return m_to.adjust (QImode, m_offset);
1750 : }
1751 :
1752 : /* Determine whether the LEN bytes generated by CONSTFUN can be
1753 : stored to memory using several move instructions. CONSTFUNDATA is
1754 : a pointer which will be passed as argument in every CONSTFUN call.
1755 : ALIGN is maximum alignment we can assume. MEMSETP is true if this is
1756 : a memset operation and false if it's a copy of a constant string.
1757 : Return true if a call to store_by_pieces should succeed. */
1758 :
1759 : bool
1760 94547 : can_store_by_pieces (unsigned HOST_WIDE_INT len,
1761 : by_pieces_constfn constfun,
1762 : void *constfundata, unsigned int align, bool memsetp)
1763 : {
1764 94547 : unsigned HOST_WIDE_INT l;
1765 94547 : unsigned int max_size;
1766 94547 : HOST_WIDE_INT offset = 0;
1767 94547 : enum insn_code icode;
1768 94547 : int reverse;
1769 : /* cst is set but not used if LEGITIMATE_CONSTANT doesn't use it. */
1770 94547 : rtx cst ATTRIBUTE_UNUSED;
1771 :
1772 94547 : if (len == 0)
1773 : return true;
1774 :
1775 144400 : if (!targetm.use_by_pieces_infrastructure_p (len, align,
1776 : memsetp
1777 : ? SET_BY_PIECES
1778 : : STORE_BY_PIECES,
1779 : optimize_insn_for_speed_p ()))
1780 : return false;
1781 :
1782 56846 : align = alignment_for_piecewise_move (STORE_MAX_PIECES, align);
1783 :
1784 : /* We would first store what we can in the largest integer mode, then go to
1785 : successively smaller modes. */
1786 :
1787 56846 : for (reverse = 0;
1788 113692 : reverse <= (HAVE_PRE_DECREMENT || HAVE_POST_DECREMENT);
1789 : reverse++)
1790 : {
1791 56846 : l = len;
1792 56872 : max_size = STORE_MAX_PIECES + 1;
1793 271624 : while (max_size > 1 && l > 0)
1794 : {
1795 214778 : auto op = memsetp ? SET_BY_PIECES : STORE_BY_PIECES;
1796 214778 : auto mode = widest_fixed_size_mode_for_size (max_size, op);
1797 :
1798 214778 : icode = optab_handler (mov_optab, mode);
1799 214778 : if (icode != CODE_FOR_nothing
1800 214778 : && align >= GET_MODE_ALIGNMENT (mode))
1801 : {
1802 214778 : unsigned int size = GET_MODE_SIZE (mode);
1803 :
1804 377263 : while (l >= size)
1805 : {
1806 162485 : if (reverse)
1807 : offset -= size;
1808 :
1809 162485 : cst = (*constfun) (constfundata, nullptr, offset, mode);
1810 : /* All CONST_VECTORs can be loaded for memset since
1811 : vec_duplicate_optab is a precondition to pick a
1812 : vector mode for the memset expander. */
1813 304108 : if (!((memsetp && VECTOR_MODE_P (mode))
1814 141623 : || targetm.legitimate_constant_p (mode, cst)))
1815 94547 : return false;
1816 :
1817 162485 : if (!reverse)
1818 162485 : offset += size;
1819 :
1820 162485 : l -= size;
1821 : }
1822 : }
1823 :
1824 429556 : max_size = GET_MODE_SIZE (mode);
1825 : }
1826 :
1827 : /* The code above should have handled everything. */
1828 56846 : gcc_assert (!l);
1829 : }
1830 :
1831 : return true;
1832 : }
1833 :
1834 : /* Generate several move instructions to store LEN bytes generated by
1835 : CONSTFUN to block TO. (A MEM rtx with BLKmode). CONSTFUNDATA is a
1836 : pointer which will be passed as argument in every CONSTFUN call.
1837 : ALIGN is maximum alignment we can assume. MEMSETP is true if this is
1838 : a memset operation and false if it's a copy of a constant string.
1839 : Return value is based on RETMODE argument. */
1840 :
1841 : rtx
1842 64071 : store_by_pieces (rtx to, unsigned HOST_WIDE_INT len,
1843 : by_pieces_constfn constfun,
1844 : void *constfundata, unsigned int align, bool memsetp,
1845 : memop_ret retmode)
1846 : {
1847 64071 : if (len == 0)
1848 : {
1849 7972 : gcc_assert (retmode != RETURN_END_MINUS_ONE);
1850 : return to;
1851 : }
1852 :
1853 101566 : gcc_assert (targetm.use_by_pieces_infrastructure_p
1854 : (len, align,
1855 : memsetp ? SET_BY_PIECES : STORE_BY_PIECES,
1856 : optimize_insn_for_speed_p ()));
1857 :
1858 56099 : store_by_pieces_d data (to, constfun, constfundata, len, align,
1859 56099 : memsetp ? SET_BY_PIECES : STORE_BY_PIECES);
1860 56099 : data.run ();
1861 :
1862 56099 : if (retmode != RETURN_BEGIN)
1863 868 : return data.finish_retmode (retmode);
1864 : else
1865 : return to;
1866 : }
1867 :
1868 : void
1869 73248 : clear_by_pieces (rtx to, unsigned HOST_WIDE_INT len, unsigned int align)
1870 : {
1871 73248 : if (len == 0)
1872 : return;
1873 :
1874 : /* Use builtin_memset_read_str to support vector mode broadcast. */
1875 73248 : char c = 0;
1876 73248 : store_by_pieces_d data (to, builtin_memset_read_str, &c, len, align,
1877 73248 : CLEAR_BY_PIECES);
1878 73248 : data.run ();
1879 : }
1880 :
1881 : /* Context used by compare_by_pieces_genfn. It stores the fail label
1882 : to jump to in case of miscomparison, and for branch ratios greater than 1,
1883 : it stores an accumulator and the current and maximum counts before
1884 : emitting another branch. */
1885 :
1886 : class compare_by_pieces_d : public op_by_pieces_d
1887 : {
1888 : rtx_code_label *m_fail_label;
1889 : rtx m_accumulator;
1890 : int m_count, m_batch;
1891 :
1892 : void generate (rtx, rtx, machine_mode) final override;
1893 : bool prepare_mode (machine_mode, unsigned int) final override;
1894 : void finish_mode (machine_mode) final override;
1895 :
1896 : public:
1897 62906 : compare_by_pieces_d (rtx op0, rtx op1, by_pieces_constfn op1_cfn,
1898 : void *op1_cfn_data, HOST_WIDE_INT len, int align,
1899 : rtx_code_label *fail_label)
1900 62906 : : op_by_pieces_d (COMPARE_MAX_PIECES, op0, true, op1, true, op1_cfn,
1901 125812 : op1_cfn_data, len, align, false, COMPARE_BY_PIECES)
1902 : {
1903 62906 : m_fail_label = fail_label;
1904 62906 : }
1905 : };
1906 :
1907 : /* A callback used when iterating for a compare_by_pieces_operation.
1908 : OP0 and OP1 are the values that have been loaded and should be
1909 : compared in MODE. DATA holds a pointer to the compare_by_pieces_data
1910 : context structure. */
1911 :
1912 : void
1913 98159 : compare_by_pieces_d::generate (rtx op0, rtx op1, machine_mode mode)
1914 : {
1915 98159 : if (m_batch > 1)
1916 : {
1917 0 : rtx temp = expand_binop (mode, sub_optab, op0, op1, NULL_RTX,
1918 : true, OPTAB_LIB_WIDEN);
1919 0 : if (m_count != 0)
1920 0 : temp = expand_binop (mode, ior_optab, m_accumulator, temp, temp,
1921 : true, OPTAB_LIB_WIDEN);
1922 0 : m_accumulator = temp;
1923 :
1924 0 : if (++m_count < m_batch)
1925 : return;
1926 :
1927 0 : m_count = 0;
1928 0 : op0 = m_accumulator;
1929 0 : op1 = const0_rtx;
1930 0 : m_accumulator = NULL_RTX;
1931 : }
1932 98159 : do_compare_rtx_and_jump (op0, op1, NE, true, mode, NULL_RTX, NULL,
1933 : m_fail_label, profile_probability::uninitialized ());
1934 : }
1935 :
1936 : /* Return true if MODE can be used for a set of moves and comparisons,
1937 : given an alignment ALIGN. Prepare whatever data is necessary for
1938 : later calls to generate. */
1939 :
1940 : bool
1941 94906 : compare_by_pieces_d::prepare_mode (machine_mode mode, unsigned int align)
1942 : {
1943 94906 : insn_code icode = optab_handler (mov_optab, mode);
1944 94906 : if (icode == CODE_FOR_nothing
1945 94906 : || align < GET_MODE_ALIGNMENT (mode)
1946 189812 : || !can_compare_p (EQ, mode, ccp_jump))
1947 : return false;
1948 94906 : m_batch = targetm.compare_by_pieces_branch_ratio (mode);
1949 94906 : if (m_batch < 0)
1950 : return false;
1951 94906 : m_accumulator = NULL_RTX;
1952 94906 : m_count = 0;
1953 94906 : return true;
1954 : }
1955 :
1956 : /* Called after expanding a series of comparisons in MODE. If we have
1957 : accumulated results for which we haven't emitted a branch yet, do
1958 : so now. */
1959 :
1960 : void
1961 94906 : compare_by_pieces_d::finish_mode (machine_mode mode)
1962 : {
1963 94906 : if (m_accumulator != NULL_RTX)
1964 0 : do_compare_rtx_and_jump (m_accumulator, const0_rtx, NE, true, mode,
1965 : NULL_RTX, NULL, m_fail_label,
1966 : profile_probability::uninitialized ());
1967 94906 : }
1968 :
1969 : /* Generate several move instructions to compare LEN bytes from blocks
1970 : ARG0 and ARG1. (These are MEM rtx's with BLKmode).
1971 :
1972 : If PUSH_ROUNDING is defined and TO is NULL, emit_single_push_insn is
1973 : used to push FROM to the stack.
1974 :
1975 : ALIGN is maximum stack alignment we can assume.
1976 :
1977 : Optionally, the caller can pass a constfn and associated data in A1_CFN
1978 : and A1_CFN_DATA. describing that the second operand being compared is a
1979 : known constant and how to obtain its data. */
1980 :
1981 : static rtx
1982 62906 : compare_by_pieces (rtx arg0, rtx arg1, unsigned HOST_WIDE_INT len,
1983 : rtx target, unsigned int align,
1984 : by_pieces_constfn a1_cfn, void *a1_cfn_data)
1985 : {
1986 62906 : rtx_code_label *fail_label = gen_label_rtx ();
1987 62906 : rtx_code_label *end_label = gen_label_rtx ();
1988 :
1989 62906 : if (target == NULL_RTX
1990 62906 : || !REG_P (target) || REGNO (target) < FIRST_PSEUDO_REGISTER)
1991 2 : target = gen_reg_rtx (TYPE_MODE (integer_type_node));
1992 :
1993 62906 : compare_by_pieces_d data (arg0, arg1, a1_cfn, a1_cfn_data, len, align,
1994 62906 : fail_label);
1995 :
1996 62906 : data.run ();
1997 :
1998 62906 : emit_move_insn (target, const0_rtx);
1999 62906 : emit_jump (end_label);
2000 62906 : emit_barrier ();
2001 62906 : emit_label (fail_label);
2002 62906 : emit_move_insn (target, const1_rtx);
2003 62906 : emit_label (end_label);
2004 :
2005 62906 : return target;
2006 : }
2007 :
2008 : /* Emit code to move a block Y to a block X. This may be done with
2009 : string-move instructions, with multiple scalar move instructions,
2010 : or with a library call.
2011 :
2012 : Both X and Y must be MEM rtx's (perhaps inside VOLATILE) with mode BLKmode.
2013 : SIZE is an rtx that says how long they are.
2014 : ALIGN is the maximum alignment we can assume they have.
2015 : METHOD describes what kind of copy this is, and what mechanisms may be used.
2016 : MIN_SIZE is the minimal size of block to move
2017 : MAX_SIZE is the maximal size of block to move, if it cannot be represented
2018 : in unsigned HOST_WIDE_INT, than it is mask of all ones.
2019 : CTZ_SIZE is the trailing-zeros count of SIZE; even a nonconstant SIZE is
2020 : known to be a multiple of 1<<CTZ_SIZE.
2021 :
2022 : Return the address of the new block, if memcpy is called and returns it,
2023 : 0 otherwise. */
2024 :
2025 : rtx
2026 673357 : emit_block_move_hints (rtx x, rtx y, rtx size, enum block_op_methods method,
2027 : unsigned int expected_align, HOST_WIDE_INT expected_size,
2028 : unsigned HOST_WIDE_INT min_size,
2029 : unsigned HOST_WIDE_INT max_size,
2030 : unsigned HOST_WIDE_INT probable_max_size,
2031 : bool bail_out_libcall, bool *is_move_done,
2032 : bool might_overlap, unsigned ctz_size)
2033 : {
2034 673357 : int may_use_call;
2035 673357 : rtx retval = 0;
2036 673357 : unsigned int align;
2037 :
2038 673357 : if (is_move_done)
2039 100156 : *is_move_done = true;
2040 :
2041 673357 : gcc_assert (size);
2042 673357 : if (CONST_INT_P (size) && INTVAL (size) == 0)
2043 : return 0;
2044 :
2045 673080 : switch (method)
2046 : {
2047 : case BLOCK_OP_NORMAL:
2048 : case BLOCK_OP_TAILCALL:
2049 : may_use_call = 1;
2050 : break;
2051 :
2052 269110 : case BLOCK_OP_CALL_PARM:
2053 269110 : may_use_call = block_move_libcall_safe_for_call_parm ();
2054 :
2055 : /* Make inhibit_defer_pop nonzero around the library call
2056 : to force it to pop the arguments right away. */
2057 269110 : NO_DEFER_POP;
2058 269110 : break;
2059 :
2060 354 : case BLOCK_OP_NO_LIBCALL:
2061 354 : may_use_call = 0;
2062 354 : break;
2063 :
2064 1075 : case BLOCK_OP_NO_LIBCALL_RET:
2065 1075 : may_use_call = -1;
2066 1075 : break;
2067 :
2068 0 : default:
2069 0 : gcc_unreachable ();
2070 : }
2071 :
2072 673080 : gcc_assert (MEM_P (x) && MEM_P (y));
2073 673165 : align = MIN (MEM_ALIGN (x), MEM_ALIGN (y));
2074 673080 : gcc_assert (align >= BITS_PER_UNIT);
2075 :
2076 : /* Make sure we've got BLKmode addresses; store_one_arg can decide that
2077 : block copy is more efficient for other large modes, e.g. DCmode. */
2078 673080 : x = adjust_address (x, BLKmode, 0);
2079 673080 : y = adjust_address (y, BLKmode, 0);
2080 :
2081 : /* If source and destination are the same, no need to copy anything. */
2082 673080 : if (rtx_equal_p (x, y)
2083 13 : && !MEM_VOLATILE_P (x)
2084 673093 : && !MEM_VOLATILE_P (y))
2085 : return 0;
2086 :
2087 : /* Set MEM_SIZE as appropriate for this block copy. The main place this
2088 : can be incorrect is coming from __builtin_memcpy. */
2089 673067 : poly_int64 const_size;
2090 673067 : if (poly_int_rtx_p (size, &const_size))
2091 : {
2092 592078 : x = shallow_copy_rtx (x);
2093 592078 : y = shallow_copy_rtx (y);
2094 592078 : set_mem_size (x, const_size);
2095 592078 : set_mem_size (y, const_size);
2096 : }
2097 :
2098 673067 : bool pieces_ok = CONST_INT_P (size)
2099 673067 : && can_move_by_pieces (INTVAL (size), align);
2100 673067 : bool pattern_ok = false;
2101 :
2102 673067 : if (!pieces_ok || might_overlap)
2103 : {
2104 108179 : pattern_ok
2105 108179 : = emit_block_move_via_pattern (x, y, size, align,
2106 : expected_align, expected_size,
2107 : min_size, max_size, probable_max_size,
2108 : might_overlap);
2109 108179 : if (!pattern_ok && might_overlap)
2110 : {
2111 : /* Do not try any of the other methods below as they are not safe
2112 : for overlapping moves. */
2113 12896 : *is_move_done = false;
2114 12896 : return retval;
2115 : }
2116 : }
2117 :
2118 95283 : bool dynamic_direction = false;
2119 95283 : if (!pattern_ok && !pieces_ok && may_use_call
2120 138232 : && (flag_inline_stringops & (might_overlap ? ILSOP_MEMMOVE : ILSOP_MEMCPY)))
2121 : {
2122 660171 : may_use_call = 0;
2123 660171 : dynamic_direction = might_overlap;
2124 : }
2125 :
2126 660171 : if (pattern_ok)
2127 : ;
2128 634004 : else if (pieces_ok)
2129 564888 : move_by_pieces (x, y, INTVAL (size), align, RETURN_BEGIN);
2130 69116 : else if (may_use_call && !might_overlap
2131 69065 : && ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (x))
2132 138181 : && ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (y)))
2133 : {
2134 69065 : if (bail_out_libcall)
2135 : {
2136 260 : if (is_move_done)
2137 260 : *is_move_done = false;
2138 : return retval;
2139 : }
2140 :
2141 68805 : if (may_use_call < 0)
2142 0 : return pc_rtx;
2143 :
2144 68805 : retval = emit_block_copy_via_libcall (x, y, size,
2145 : method == BLOCK_OP_TAILCALL);
2146 : }
2147 51 : else if (dynamic_direction)
2148 0 : emit_block_move_via_oriented_loop (x, y, size, align, ctz_size);
2149 51 : else if (might_overlap)
2150 0 : *is_move_done = false;
2151 : else
2152 51 : emit_block_move_via_sized_loop (x, y, size, align, ctz_size);
2153 :
2154 659911 : if (method == BLOCK_OP_CALL_PARM)
2155 269099 : OK_DEFER_POP;
2156 :
2157 : return retval;
2158 : }
2159 :
2160 : rtx
2161 573201 : emit_block_move (rtx x, rtx y, rtx size, enum block_op_methods method,
2162 : unsigned int ctz_size)
2163 : {
2164 573201 : unsigned HOST_WIDE_INT max, min = 0;
2165 573201 : if (GET_CODE (size) == CONST_INT)
2166 572981 : min = max = UINTVAL (size);
2167 : else
2168 220 : max = GET_MODE_MASK (GET_MODE (size));
2169 573201 : return emit_block_move_hints (x, y, size, method, 0, -1,
2170 : min, max, max,
2171 573201 : false, NULL, false, ctz_size);
2172 : }
2173 :
2174 : /* A subroutine of emit_block_move. Returns true if calling the
2175 : block move libcall will not clobber any parameters which may have
2176 : already been placed on the stack. */
2177 :
2178 : static bool
2179 269110 : block_move_libcall_safe_for_call_parm (void)
2180 : {
2181 269110 : tree fn;
2182 :
2183 : /* If arguments are pushed on the stack, then they're safe. */
2184 269110 : if (targetm.calls.push_argument (0))
2185 : return true;
2186 :
2187 : /* If registers go on the stack anyway, any argument is sure to clobber
2188 : an outgoing argument. */
2189 : #if defined (REG_PARM_STACK_SPACE)
2190 3 : fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
2191 : /* Avoid set but not used warning if *REG_PARM_STACK_SPACE doesn't
2192 : depend on its argument. */
2193 3 : (void) fn;
2194 3 : if (OUTGOING_REG_PARM_STACK_SPACE ((!fn ? NULL_TREE : TREE_TYPE (fn)))
2195 3 : && REG_PARM_STACK_SPACE (fn) != 0)
2196 : return false;
2197 : #endif
2198 :
2199 : /* If any argument goes in memory, then it might clobber an outgoing
2200 : argument. */
2201 3 : {
2202 3 : CUMULATIVE_ARGS args_so_far_v;
2203 3 : cumulative_args_t args_so_far;
2204 3 : tree arg;
2205 :
2206 3 : fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
2207 3 : INIT_CUMULATIVE_ARGS (args_so_far_v, TREE_TYPE (fn), NULL_RTX, 0, 3);
2208 3 : args_so_far = pack_cumulative_args (&args_so_far_v);
2209 :
2210 3 : arg = TYPE_ARG_TYPES (TREE_TYPE (fn));
2211 12 : for ( ; arg != void_list_node ; arg = TREE_CHAIN (arg))
2212 : {
2213 9 : machine_mode mode = TYPE_MODE (TREE_VALUE (arg));
2214 9 : function_arg_info arg_info (mode, /*named=*/true);
2215 9 : rtx tmp = targetm.calls.function_arg (args_so_far, arg_info);
2216 9 : if (!tmp || !REG_P (tmp))
2217 0 : return false;
2218 9 : if (targetm.calls.arg_partial_bytes (args_so_far, arg_info))
2219 : return false;
2220 9 : targetm.calls.function_arg_advance (args_so_far, arg_info);
2221 : }
2222 : }
2223 3 : return true;
2224 : }
2225 :
2226 : /* A subroutine of emit_block_move. Expand a cpymem or movmem pattern;
2227 : return true if successful.
2228 :
2229 : X is the destination of the copy or move.
2230 : Y is the source of the copy or move.
2231 : SIZE is the size of the block to be moved.
2232 :
2233 : MIGHT_OVERLAP indicates this originated with expansion of a
2234 : builtin_memmove() and the source and destination blocks may
2235 : overlap.
2236 : */
2237 :
2238 : static bool
2239 108179 : emit_block_move_via_pattern (rtx x, rtx y, rtx size, unsigned int align,
2240 : unsigned int expected_align,
2241 : HOST_WIDE_INT expected_size,
2242 : unsigned HOST_WIDE_INT min_size,
2243 : unsigned HOST_WIDE_INT max_size,
2244 : unsigned HOST_WIDE_INT probable_max_size,
2245 : bool might_overlap)
2246 : {
2247 108179 : if (expected_align < align)
2248 : expected_align = align;
2249 108179 : if (expected_size != -1)
2250 : {
2251 22 : if ((unsigned HOST_WIDE_INT)expected_size > probable_max_size)
2252 0 : expected_size = probable_max_size;
2253 22 : if ((unsigned HOST_WIDE_INT)expected_size < min_size)
2254 0 : expected_size = min_size;
2255 : }
2256 :
2257 : /* Since this is a move insn, we don't care about volatility. */
2258 108179 : temporary_volatile_ok v (true);
2259 :
2260 : /* Try the most limited insn first, because there's no point
2261 : including more than one in the machine description unless
2262 : the more limited one has some advantage. */
2263 :
2264 108179 : opt_scalar_int_mode mode_iter;
2265 738608 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
2266 : {
2267 656596 : scalar_int_mode mode = mode_iter.require ();
2268 656596 : enum insn_code code;
2269 656596 : if (might_overlap)
2270 104449 : code = direct_optab_handler (movmem_optab, mode);
2271 : else
2272 552147 : code = direct_optab_handler (cpymem_optab, mode);
2273 :
2274 656596 : if (code != CODE_FOR_nothing
2275 : /* We don't need MODE to be narrower than BITS_PER_HOST_WIDE_INT
2276 : here because if SIZE is less than the mode mask, as it is
2277 : returned by the macro, it will definitely be less than the
2278 : actual mode mask. Since SIZE is within the Pmode address
2279 : space, we limit MODE to Pmode. */
2280 656596 : && ((CONST_INT_P (size)
2281 42088 : && ((unsigned HOST_WIDE_INT) INTVAL (size)
2282 42088 : <= (GET_MODE_MASK (mode) >> 1)))
2283 147078 : || max_size <= (GET_MODE_MASK (mode) >> 1)
2284 183860 : || GET_MODE_BITSIZE (mode) >= GET_MODE_BITSIZE (Pmode)))
2285 : {
2286 125008 : class expand_operand ops[9];
2287 125008 : unsigned int nops;
2288 :
2289 : /* ??? When called via emit_block_move_for_call, it'd be
2290 : nice if there were some way to inform the backend, so
2291 : that it doesn't fail the expansion because it thinks
2292 : emitting the libcall would be more efficient. */
2293 125008 : nops = insn_data[(int) code].n_generator_args;
2294 125008 : gcc_assert (nops == 4 || nops == 6 || nops == 8 || nops == 9);
2295 :
2296 125008 : create_fixed_operand (&ops[0], x);
2297 125008 : create_fixed_operand (&ops[1], y);
2298 : /* The check above guarantees that this size conversion is valid. */
2299 125008 : create_convert_operand_to (&ops[2], size, mode, true);
2300 125008 : create_integer_operand (&ops[3], align / BITS_PER_UNIT);
2301 125008 : if (nops >= 6)
2302 : {
2303 125008 : create_integer_operand (&ops[4], expected_align / BITS_PER_UNIT);
2304 125008 : create_integer_operand (&ops[5], expected_size);
2305 : }
2306 125008 : if (nops >= 8)
2307 : {
2308 125008 : create_integer_operand (&ops[6], min_size);
2309 : /* If we cannot represent the maximal size,
2310 : make parameter NULL. */
2311 125008 : if ((HOST_WIDE_INT) max_size != -1)
2312 107046 : create_integer_operand (&ops[7], max_size);
2313 : else
2314 17962 : create_fixed_operand (&ops[7], NULL);
2315 : }
2316 125008 : if (nops == 9)
2317 : {
2318 : /* If we cannot represent the maximal size,
2319 : make parameter NULL. */
2320 125008 : if ((HOST_WIDE_INT) probable_max_size != -1)
2321 109134 : create_integer_operand (&ops[8], probable_max_size);
2322 : else
2323 15874 : create_fixed_operand (&ops[8], NULL);
2324 : }
2325 125008 : gcc_assert (min_size != max_size
2326 : || rtx_equal_p (ops[2].value, GEN_INT (min_size)));
2327 125008 : if (maybe_expand_insn (code, nops, ops))
2328 26167 : return true;
2329 : }
2330 : }
2331 :
2332 : return false;
2333 108179 : }
2334 :
2335 : /* Like emit_block_move_via_loop, but choose a suitable INCR based on
2336 : ALIGN and CTZ_SIZE. */
2337 :
2338 : static void
2339 51 : emit_block_move_via_sized_loop (rtx x, rtx y, rtx size,
2340 : unsigned int align,
2341 : unsigned int ctz_size)
2342 : {
2343 51 : int incr = align / BITS_PER_UNIT;
2344 :
2345 51 : if (CONST_INT_P (size))
2346 3 : ctz_size = MAX (ctz_size, (unsigned) wi::ctz (UINTVAL (size)));
2347 :
2348 51 : if (HOST_WIDE_INT_1U << ctz_size < (unsigned HOST_WIDE_INT) incr)
2349 0 : incr = HOST_WIDE_INT_1U << ctz_size;
2350 :
2351 51 : while (incr > 1 && !can_move_by_pieces (incr, align))
2352 0 : incr >>= 1;
2353 :
2354 51 : gcc_checking_assert (incr);
2355 :
2356 51 : return emit_block_move_via_loop (x, y, size, align, incr);
2357 : }
2358 :
2359 : /* Like emit_block_move_via_sized_loop, but besides choosing INCR so
2360 : as to ensure safe moves even in case of overlap, output dynamic
2361 : tests to choose between two loops, one moving downwards, another
2362 : moving upwards. */
2363 :
2364 : static void
2365 0 : emit_block_move_via_oriented_loop (rtx x, rtx y, rtx size,
2366 : unsigned int align,
2367 : unsigned int ctz_size)
2368 : {
2369 0 : int incr = align / BITS_PER_UNIT;
2370 :
2371 0 : if (CONST_INT_P (size))
2372 0 : ctz_size = MAX (ctz_size, (unsigned) wi::ctz (UINTVAL (size)));
2373 :
2374 0 : if (HOST_WIDE_INT_1U << ctz_size < (unsigned HOST_WIDE_INT) incr)
2375 0 : incr = HOST_WIDE_INT_1U << ctz_size;
2376 :
2377 0 : while (incr > 1 && !int_mode_for_size (incr, 0).exists ())
2378 0 : incr >>= 1;
2379 :
2380 0 : gcc_checking_assert (incr);
2381 :
2382 0 : rtx_code_label *upw_label, *end_label;
2383 0 : upw_label = gen_label_rtx ();
2384 0 : end_label = gen_label_rtx ();
2385 :
2386 0 : rtx x_addr = force_operand (XEXP (x, 0), NULL_RTX);
2387 0 : rtx y_addr = force_operand (XEXP (y, 0), NULL_RTX);
2388 0 : do_pending_stack_adjust ();
2389 :
2390 0 : machine_mode mode = GET_MODE (x_addr);
2391 0 : if (mode != GET_MODE (y_addr))
2392 : {
2393 0 : scalar_int_mode xmode
2394 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE (mode)).require ();
2395 0 : scalar_int_mode ymode
2396 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE
2397 0 : (GET_MODE (y_addr))).require ();
2398 0 : if (GET_MODE_BITSIZE (xmode) < GET_MODE_BITSIZE (ymode))
2399 : mode = ymode;
2400 : else
2401 0 : mode = xmode;
2402 :
2403 : #ifndef POINTERS_EXTEND_UNSIGNED
2404 : const int POINTERS_EXTEND_UNSIGNED = 1;
2405 : #endif
2406 0 : x_addr = convert_modes (mode, GET_MODE (x_addr), x_addr,
2407 : POINTERS_EXTEND_UNSIGNED);
2408 0 : y_addr = convert_modes (mode, GET_MODE (y_addr), y_addr,
2409 : POINTERS_EXTEND_UNSIGNED);
2410 : }
2411 :
2412 : /* Test for overlap: if (x >= y || x + size <= y) goto upw_label. */
2413 0 : emit_cmp_and_jump_insns (x_addr, y_addr, GEU, NULL_RTX, mode,
2414 : true, upw_label,
2415 0 : profile_probability::guessed_always ()
2416 : .apply_scale (5, 10));
2417 0 : rtx tmp = convert_modes (GET_MODE (x_addr), GET_MODE (size), size, true);
2418 0 : tmp = simplify_gen_binary (PLUS, GET_MODE (x_addr), x_addr, tmp);
2419 :
2420 0 : emit_cmp_and_jump_insns (tmp, y_addr, LEU, NULL_RTX, mode,
2421 : true, upw_label,
2422 0 : profile_probability::guessed_always ()
2423 : .apply_scale (8, 10));
2424 :
2425 0 : emit_block_move_via_loop (x, y, size, align, -incr);
2426 :
2427 0 : emit_jump (end_label);
2428 0 : emit_label (upw_label);
2429 :
2430 0 : emit_block_move_via_loop (x, y, size, align, incr);
2431 :
2432 0 : emit_label (end_label);
2433 0 : }
2434 :
2435 : /* A subroutine of emit_block_move. Copy the data via an explicit
2436 : loop. This is used only when libcalls are forbidden, or when
2437 : inlining is required. INCR is the block size to be copied in each
2438 : loop iteration. If it is negative, the absolute value is used, and
2439 : the block is copied backwards. INCR must be a power of two, an
2440 : exact divisor for SIZE and ALIGN, and imply a mode that can be
2441 : safely copied per iteration assuming no overlap. */
2442 :
2443 : static void
2444 51 : emit_block_move_via_loop (rtx x, rtx y, rtx size,
2445 : unsigned int align, int incr)
2446 : {
2447 51 : rtx_code_label *cmp_label, *top_label;
2448 51 : rtx iter, x_addr, y_addr, tmp;
2449 51 : machine_mode x_addr_mode = get_address_mode (x);
2450 51 : machine_mode y_addr_mode = get_address_mode (y);
2451 51 : machine_mode iter_mode;
2452 :
2453 51 : iter_mode = GET_MODE (size);
2454 51 : if (iter_mode == VOIDmode)
2455 3 : iter_mode = word_mode;
2456 :
2457 51 : top_label = gen_label_rtx ();
2458 51 : cmp_label = gen_label_rtx ();
2459 51 : iter = gen_reg_rtx (iter_mode);
2460 :
2461 51 : bool downwards = incr < 0;
2462 51 : rtx iter_init;
2463 51 : rtx_code iter_cond;
2464 51 : rtx iter_limit;
2465 51 : rtx iter_incr;
2466 51 : machine_mode move_mode;
2467 51 : if (downwards)
2468 : {
2469 0 : incr = -incr;
2470 0 : iter_init = size;
2471 0 : iter_cond = GEU;
2472 0 : iter_limit = const0_rtx;
2473 0 : iter_incr = GEN_INT (incr);
2474 : }
2475 : else
2476 : {
2477 51 : iter_init = const0_rtx;
2478 51 : iter_cond = LTU;
2479 51 : iter_limit = size;
2480 51 : iter_incr = GEN_INT (incr);
2481 : }
2482 51 : emit_move_insn (iter, iter_init);
2483 :
2484 51 : opt_scalar_int_mode int_move_mode
2485 51 : = int_mode_for_size (incr * BITS_PER_UNIT, 1);
2486 51 : if (!int_move_mode.exists (&move_mode)
2487 102 : || GET_MODE_BITSIZE (int_move_mode.require ()) != incr * BITS_PER_UNIT)
2488 : {
2489 0 : move_mode = BLKmode;
2490 0 : gcc_checking_assert (can_move_by_pieces (incr, align));
2491 : }
2492 :
2493 51 : x_addr = force_operand (XEXP (x, 0), NULL_RTX);
2494 51 : y_addr = force_operand (XEXP (y, 0), NULL_RTX);
2495 51 : do_pending_stack_adjust ();
2496 :
2497 51 : emit_jump (cmp_label);
2498 51 : emit_label (top_label);
2499 :
2500 51 : tmp = convert_modes (x_addr_mode, iter_mode, iter, true);
2501 51 : x_addr = simplify_gen_binary (PLUS, x_addr_mode, x_addr, tmp);
2502 :
2503 51 : if (x_addr_mode != y_addr_mode)
2504 0 : tmp = convert_modes (y_addr_mode, iter_mode, iter, true);
2505 51 : y_addr = simplify_gen_binary (PLUS, y_addr_mode, y_addr, tmp);
2506 :
2507 51 : x = change_address (x, move_mode, x_addr);
2508 51 : y = change_address (y, move_mode, y_addr);
2509 :
2510 51 : if (move_mode == BLKmode)
2511 : {
2512 0 : bool done;
2513 0 : emit_block_move_hints (x, y, iter_incr, BLOCK_OP_NO_LIBCALL,
2514 : align, incr, incr, incr, incr,
2515 : false, &done, false);
2516 0 : gcc_checking_assert (done);
2517 : }
2518 : else
2519 51 : emit_move_insn (x, y);
2520 :
2521 51 : if (downwards)
2522 0 : emit_label (cmp_label);
2523 :
2524 51 : tmp = expand_simple_binop (iter_mode, PLUS, iter, iter_incr, iter,
2525 : true, OPTAB_LIB_WIDEN);
2526 51 : if (tmp != iter)
2527 0 : emit_move_insn (iter, tmp);
2528 :
2529 51 : if (!downwards)
2530 51 : emit_label (cmp_label);
2531 :
2532 51 : emit_cmp_and_jump_insns (iter, iter_limit, iter_cond, NULL_RTX, iter_mode,
2533 : true, top_label,
2534 51 : profile_probability::guessed_always ()
2535 : .apply_scale (9, 10));
2536 51 : }
2537 :
2538 : /* Expand a call to memcpy or memmove or memcmp, and return the result.
2539 : TAILCALL is true if this is a tail call. */
2540 :
2541 : rtx
2542 68806 : emit_block_op_via_libcall (enum built_in_function fncode, rtx dst, rtx src,
2543 : rtx size, bool tailcall)
2544 : {
2545 68806 : rtx dst_addr, src_addr;
2546 68806 : tree call_expr, dst_tree, src_tree, size_tree;
2547 68806 : machine_mode size_mode;
2548 :
2549 : /* Since dst and src are passed to a libcall, mark the corresponding
2550 : tree EXPR as addressable. */
2551 68806 : tree dst_expr = MEM_EXPR (dst);
2552 68806 : tree src_expr = MEM_EXPR (src);
2553 68806 : if (dst_expr)
2554 62087 : mark_addressable (dst_expr);
2555 68806 : if (src_expr)
2556 67437 : mark_addressable (src_expr);
2557 :
2558 68806 : dst_addr = copy_addr_to_reg (XEXP (dst, 0));
2559 68806 : dst_addr = convert_memory_address (ptr_mode, dst_addr);
2560 68806 : dst_tree = make_tree (ptr_type_node, dst_addr);
2561 :
2562 68806 : src_addr = copy_addr_to_reg (XEXP (src, 0));
2563 68806 : src_addr = convert_memory_address (ptr_mode, src_addr);
2564 68806 : src_tree = make_tree (ptr_type_node, src_addr);
2565 :
2566 68806 : size_mode = TYPE_MODE (sizetype);
2567 68806 : size = convert_to_mode (size_mode, size, 1);
2568 68806 : size = copy_to_mode_reg (size_mode, size);
2569 68806 : size_tree = make_tree (sizetype, size);
2570 :
2571 : /* It is incorrect to use the libcall calling conventions for calls to
2572 : memcpy/memmove/memcmp because they can be provided by the user. */
2573 68806 : tree fn = builtin_decl_implicit (fncode);
2574 68806 : call_expr = build_call_expr (fn, 3, dst_tree, src_tree, size_tree);
2575 68806 : CALL_EXPR_TAILCALL (call_expr) = tailcall;
2576 :
2577 68806 : return expand_call (call_expr, NULL_RTX, false);
2578 : }
2579 :
2580 : /* Try to expand cmpstrn or cmpmem operation ICODE with the given operands.
2581 : ARG3_TYPE is the type of ARG3_RTX. Return the result rtx on success,
2582 : otherwise return null. */
2583 :
2584 : rtx
2585 172260 : expand_cmpstrn_or_cmpmem (insn_code icode, rtx target, rtx arg1_rtx,
2586 : rtx arg2_rtx, tree arg3_type, rtx arg3_rtx,
2587 : HOST_WIDE_INT align)
2588 : {
2589 172260 : machine_mode insn_mode = insn_data[icode].operand[0].mode;
2590 :
2591 172260 : if (target && (!REG_P (target) || HARD_REGISTER_P (target)))
2592 : target = NULL_RTX;
2593 :
2594 172260 : class expand_operand ops[5];
2595 172260 : create_output_operand (&ops[0], target, insn_mode);
2596 172260 : create_fixed_operand (&ops[1], arg1_rtx);
2597 172260 : create_fixed_operand (&ops[2], arg2_rtx);
2598 172260 : create_convert_operand_from (&ops[3], arg3_rtx, TYPE_MODE (arg3_type),
2599 172260 : TYPE_UNSIGNED (arg3_type));
2600 172260 : create_integer_operand (&ops[4], align);
2601 172260 : if (maybe_expand_insn (icode, 5, ops))
2602 5802 : return ops[0].value;
2603 : return NULL_RTX;
2604 : }
2605 :
2606 : /* Expand a block compare between X and Y with length LEN using the
2607 : cmpmem optab, placing the result in TARGET. LEN_TYPE is the type
2608 : of the expression that was used to calculate the length. ALIGN
2609 : gives the known minimum common alignment. */
2610 :
2611 : static rtx
2612 44332 : emit_block_cmp_via_cmpmem (rtx x, rtx y, rtx len, tree len_type, rtx target,
2613 : unsigned align)
2614 : {
2615 : /* Note: The cmpstrnsi pattern, if it exists, is not suitable for
2616 : implementing memcmp because it will stop if it encounters two
2617 : zero bytes. */
2618 44332 : insn_code icode = direct_optab_handler (cmpmem_optab, SImode);
2619 :
2620 44332 : if (icode == CODE_FOR_nothing)
2621 : return NULL_RTX;
2622 :
2623 44332 : return expand_cmpstrn_or_cmpmem (icode, target, x, y, len_type, len, align);
2624 : }
2625 :
2626 : /* Emit code to compare a block Y to a block X. This may be done with
2627 : string-compare instructions, with multiple scalar instructions,
2628 : or with a library call.
2629 :
2630 : Both X and Y must be MEM rtx's. LEN is an rtx that says how long
2631 : they are. LEN_TYPE is the type of the expression that was used to
2632 : calculate it, and CTZ_LEN is the known trailing-zeros count of LEN,
2633 : so LEN must be a multiple of 1<<CTZ_LEN even if it's not constant.
2634 :
2635 : If EQUALITY_ONLY is true, it means we don't have to return the tri-state
2636 : value of a normal memcmp call, instead we can just compare for equality.
2637 : If FORCE_LIBCALL is true, we should emit a call to memcmp rather than
2638 : returning NULL_RTX.
2639 :
2640 : Optionally, the caller can pass a constfn and associated data in Y_CFN
2641 : and Y_CFN_DATA. describing that the second operand being compared is a
2642 : known constant and how to obtain its data.
2643 : Return the result of the comparison, or NULL_RTX if we failed to
2644 : perform the operation. */
2645 :
2646 : rtx
2647 107237 : emit_block_cmp_hints (rtx x, rtx y, rtx len, tree len_type, rtx target,
2648 : bool equality_only, by_pieces_constfn y_cfn,
2649 : void *y_cfndata, unsigned ctz_len)
2650 : {
2651 107237 : rtx result = 0;
2652 :
2653 107237 : if (CONST_INT_P (len) && INTVAL (len) == 0)
2654 0 : return const0_rtx;
2655 :
2656 107237 : gcc_assert (MEM_P (x) && MEM_P (y));
2657 107237 : unsigned int align = MIN (MEM_ALIGN (x), MEM_ALIGN (y));
2658 107237 : gcc_assert (align >= BITS_PER_UNIT);
2659 :
2660 107237 : x = adjust_address (x, BLKmode, 0);
2661 107237 : y = adjust_address (y, BLKmode, 0);
2662 :
2663 107237 : if (equality_only
2664 91940 : && CONST_INT_P (len)
2665 176911 : && can_do_by_pieces (INTVAL (len), align, COMPARE_BY_PIECES))
2666 62905 : result = compare_by_pieces (x, y, INTVAL (len), target, align,
2667 : y_cfn, y_cfndata);
2668 : else
2669 44332 : result = emit_block_cmp_via_cmpmem (x, y, len, len_type, target, align);
2670 :
2671 107237 : if (!result && (flag_inline_stringops & ILSOP_MEMCMP))
2672 364 : result = emit_block_cmp_via_loop (x, y, len, len_type,
2673 : target, equality_only,
2674 : align, ctz_len);
2675 :
2676 : return result;
2677 : }
2678 :
2679 : /* Like emit_block_cmp_hints, but with known alignment and no support
2680 : for constats. Always expand to a loop with iterations that compare
2681 : blocks of the largest compare-by-pieces size that divides both len
2682 : and align, and then, if !EQUALITY_ONLY, identify the word and then
2683 : the unit that first differs to return the result. */
2684 :
2685 : rtx
2686 406 : emit_block_cmp_via_loop (rtx x, rtx y, rtx len, tree len_type, rtx target,
2687 : bool equality_only, unsigned align, unsigned ctz_len)
2688 : {
2689 406 : unsigned incr = align / BITS_PER_UNIT;
2690 :
2691 406 : if (CONST_INT_P (len))
2692 322 : ctz_len = MAX (ctz_len, (unsigned) wi::ctz (UINTVAL (len)));
2693 :
2694 406 : if (HOST_WIDE_INT_1U << ctz_len < (unsigned HOST_WIDE_INT) incr)
2695 148 : incr = HOST_WIDE_INT_1U << ctz_len;
2696 :
2697 : while (incr > 1
2698 410 : && !can_do_by_pieces (incr, align, COMPARE_BY_PIECES))
2699 4 : incr >>= 1;
2700 :
2701 406 : rtx_code_label *cmp_label, *top_label, *ne_label, *res_label;
2702 406 : rtx iter, x_addr, y_addr, tmp;
2703 406 : machine_mode x_addr_mode = get_address_mode (x);
2704 406 : machine_mode y_addr_mode = get_address_mode (y);
2705 406 : machine_mode iter_mode;
2706 :
2707 406 : iter_mode = GET_MODE (len);
2708 406 : if (iter_mode == VOIDmode)
2709 322 : iter_mode = word_mode;
2710 :
2711 406 : rtx iter_init = const0_rtx;
2712 406 : rtx_code iter_cond = LTU;
2713 406 : rtx_code entry_cond = GEU;
2714 406 : rtx iter_limit = len;
2715 406 : rtx iter_incr = GEN_INT (incr);
2716 406 : machine_mode cmp_mode;
2717 :
2718 : /* We can drop the loop back edge if we know there's exactly one
2719 : iteration. */
2720 406 : top_label = (!rtx_equal_p (len, iter_incr)
2721 406 : ? gen_label_rtx ()
2722 : : NULL);
2723 : /* We need not test before entering the loop if len is known
2724 : nonzero. ??? This could be even stricter, testing whether a
2725 : nonconstant LEN could possibly be zero. */
2726 406 : cmp_label = (!CONSTANT_P (len) || rtx_equal_p (len, iter_init)
2727 406 : ? gen_label_rtx ()
2728 : : NULL);
2729 406 : ne_label = gen_label_rtx ();
2730 406 : res_label = gen_label_rtx ();
2731 :
2732 406 : iter = gen_reg_rtx (iter_mode);
2733 406 : emit_move_insn (iter, iter_init);
2734 :
2735 406 : opt_scalar_int_mode int_cmp_mode
2736 406 : = int_mode_for_size (incr * BITS_PER_UNIT, 1);
2737 406 : if (!int_cmp_mode.exists (&cmp_mode)
2738 1216 : || GET_MODE_BITSIZE (int_cmp_mode.require ()) != incr * BITS_PER_UNIT
2739 405 : || !can_compare_p (NE, cmp_mode, ccp_jump))
2740 : {
2741 1 : cmp_mode = BLKmode;
2742 1 : gcc_checking_assert (incr != 1);
2743 : }
2744 :
2745 : /* Save the base addresses. */
2746 406 : x_addr = force_operand (XEXP (x, 0), NULL_RTX);
2747 406 : y_addr = force_operand (XEXP (y, 0), NULL_RTX);
2748 406 : do_pending_stack_adjust ();
2749 :
2750 406 : if (cmp_label)
2751 : {
2752 84 : if (top_label)
2753 84 : emit_jump (cmp_label);
2754 : else
2755 0 : emit_cmp_and_jump_insns (iter, iter_limit, entry_cond,
2756 : NULL_RTX, iter_mode,
2757 : true, cmp_label,
2758 0 : profile_probability::guessed_always ()
2759 : .apply_scale (1, 10));
2760 : }
2761 406 : if (top_label)
2762 389 : emit_label (top_label);
2763 :
2764 : /* Offset the base addresses by ITER. */
2765 406 : tmp = convert_modes (x_addr_mode, iter_mode, iter, true);
2766 406 : x_addr = simplify_gen_binary (PLUS, x_addr_mode, x_addr, tmp);
2767 :
2768 406 : if (x_addr_mode != y_addr_mode)
2769 0 : tmp = convert_modes (y_addr_mode, iter_mode, iter, true);
2770 406 : y_addr = simplify_gen_binary (PLUS, y_addr_mode, y_addr, tmp);
2771 :
2772 406 : x = change_address (x, cmp_mode, x_addr);
2773 406 : y = change_address (y, cmp_mode, y_addr);
2774 :
2775 : /* Compare one block. */
2776 406 : rtx part_res;
2777 406 : if (cmp_mode == BLKmode)
2778 1 : part_res = compare_by_pieces (x, y, incr, target, align, 0, 0);
2779 : else
2780 405 : part_res = expand_binop (cmp_mode, sub_optab, x, y, NULL_RTX,
2781 : true, OPTAB_LIB_WIDEN);
2782 :
2783 : /* Stop if we found a difference. */
2784 812 : emit_cmp_and_jump_insns (part_res, GEN_INT (0), NE, NULL_RTX,
2785 406 : GET_MODE (part_res), true, ne_label,
2786 406 : profile_probability::guessed_always ()
2787 : .apply_scale (1, 10));
2788 :
2789 : /* Increment ITER. */
2790 406 : tmp = expand_simple_binop (iter_mode, PLUS, iter, iter_incr, iter,
2791 : true, OPTAB_LIB_WIDEN);
2792 406 : if (tmp != iter)
2793 0 : emit_move_insn (iter, tmp);
2794 :
2795 406 : if (cmp_label)
2796 84 : emit_label (cmp_label);
2797 : /* Loop until we reach the limit. */
2798 :
2799 406 : if (top_label)
2800 389 : emit_cmp_and_jump_insns (iter, iter_limit, iter_cond, NULL_RTX, iter_mode,
2801 : true, top_label,
2802 778 : profile_probability::guessed_always ()
2803 : .apply_scale (9, 10));
2804 :
2805 : /* We got to the end without differences, so the result is zero. */
2806 406 : if (target == NULL_RTX
2807 406 : || !REG_P (target) || REGNO (target) < FIRST_PSEUDO_REGISTER)
2808 49 : target = gen_reg_rtx (TYPE_MODE (integer_type_node));
2809 :
2810 406 : emit_move_insn (target, const0_rtx);
2811 406 : emit_jump (res_label);
2812 :
2813 406 : emit_label (ne_label);
2814 :
2815 : /* Return nonzero, or pinpoint the difference to return the expected
2816 : result for non-equality tests. */
2817 406 : if (equality_only)
2818 0 : emit_move_insn (target, const1_rtx);
2819 : else
2820 : {
2821 406 : if (incr > UNITS_PER_WORD)
2822 : /* ??? Re-compare the block found to be different one word at a
2823 : time. */
2824 5 : part_res = emit_block_cmp_via_loop (x, y, GEN_INT (incr), len_type,
2825 : target, equality_only,
2826 : BITS_PER_WORD, 0);
2827 401 : else if (incr > 1)
2828 : /* ??? Re-compare the block found to be different one byte at a
2829 : time. We could do better using part_res, and being careful
2830 : about endianness. */
2831 37 : part_res = emit_block_cmp_via_loop (x, y, GEN_INT (incr), len_type,
2832 : target, equality_only,
2833 : BITS_PER_UNIT, 0);
2834 1092 : else if (known_gt (GET_MODE_BITSIZE (GET_MODE (target)),
2835 : GET_MODE_BITSIZE (cmp_mode)))
2836 364 : part_res = expand_binop (GET_MODE (target), sub_optab, x, y, target,
2837 : true, OPTAB_LIB_WIDEN);
2838 : else
2839 : {
2840 : /* In the odd chance target is QImode, we can't count on
2841 : widening subtract to capture the result of the unsigned
2842 : compares. */
2843 0 : rtx_code_label *ltu_label;
2844 0 : ltu_label = gen_label_rtx ();
2845 0 : emit_cmp_and_jump_insns (x, y, LTU, NULL_RTX,
2846 : cmp_mode, true, ltu_label,
2847 0 : profile_probability::guessed_always ()
2848 : .apply_scale (5, 10));
2849 :
2850 0 : emit_move_insn (target, const1_rtx);
2851 0 : emit_jump (res_label);
2852 :
2853 0 : emit_label (ltu_label);
2854 0 : emit_move_insn (target, constm1_rtx);
2855 0 : part_res = target;
2856 : }
2857 :
2858 406 : if (target != part_res)
2859 0 : convert_move (target, part_res, false);
2860 : }
2861 :
2862 406 : emit_label (res_label);
2863 :
2864 406 : return target;
2865 : }
2866 :
2867 :
2868 : /* Copy all or part of a value X into registers starting at REGNO.
2869 : The number of registers to be filled is NREGS. */
2870 :
2871 : void
2872 434 : move_block_to_reg (int regno, rtx x, int nregs, machine_mode mode)
2873 : {
2874 434 : if (nregs == 0)
2875 : return;
2876 :
2877 420 : if (CONSTANT_P (x) && !targetm.legitimate_constant_p (mode, x))
2878 0 : x = validize_mem (force_const_mem (mode, x));
2879 :
2880 : /* See if the machine can do this with a load multiple insn. */
2881 420 : if (targetm.have_load_multiple ())
2882 : {
2883 0 : rtx_insn *last = get_last_insn ();
2884 0 : rtx first = gen_rtx_REG (word_mode, regno);
2885 0 : if (rtx_insn *pat = targetm.gen_load_multiple (first, x,
2886 : GEN_INT (nregs)))
2887 : {
2888 0 : emit_insn (pat);
2889 0 : return;
2890 : }
2891 : else
2892 0 : delete_insns_since (last);
2893 : }
2894 :
2895 848 : for (int i = 0; i < nregs; i++)
2896 428 : emit_move_insn (gen_rtx_REG (word_mode, regno + i),
2897 428 : operand_subword_force (x, i, mode));
2898 : }
2899 :
2900 : /* Copy all or part of a BLKmode value X out of registers starting at REGNO.
2901 : The number of registers to be filled is NREGS. */
2902 :
2903 : void
2904 1216 : move_block_from_reg (int regno, rtx x, int nregs)
2905 : {
2906 1216 : if (nregs == 0)
2907 : return;
2908 :
2909 : /* See if the machine can do this with a store multiple insn. */
2910 1216 : if (targetm.have_store_multiple ())
2911 : {
2912 0 : rtx_insn *last = get_last_insn ();
2913 0 : rtx first = gen_rtx_REG (word_mode, regno);
2914 0 : if (rtx_insn *pat = targetm.gen_store_multiple (x, first,
2915 : GEN_INT (nregs)))
2916 : {
2917 0 : emit_insn (pat);
2918 0 : return;
2919 : }
2920 : else
2921 0 : delete_insns_since (last);
2922 : }
2923 :
2924 2436 : for (int i = 0; i < nregs; i++)
2925 : {
2926 1220 : rtx tem = operand_subword (x, i, 1, BLKmode);
2927 :
2928 1220 : gcc_assert (tem);
2929 :
2930 1220 : emit_move_insn (tem, gen_rtx_REG (word_mode, regno + i));
2931 : }
2932 : }
2933 :
2934 : /* Generate a PARALLEL rtx for a new non-consecutive group of registers from
2935 : ORIG, where ORIG is a non-consecutive group of registers represented by
2936 : a PARALLEL. The clone is identical to the original except in that the
2937 : original set of registers is replaced by a new set of pseudo registers.
2938 : The new set has the same modes as the original set. */
2939 :
2940 : rtx
2941 3306 : gen_group_rtx (rtx orig)
2942 : {
2943 3306 : int i, length;
2944 3306 : rtx *tmps;
2945 :
2946 3306 : gcc_assert (GET_CODE (orig) == PARALLEL);
2947 :
2948 3306 : length = XVECLEN (orig, 0);
2949 3306 : tmps = XALLOCAVEC (rtx, length);
2950 :
2951 : /* Skip a NULL entry in first slot. */
2952 3306 : i = XEXP (XVECEXP (orig, 0, 0), 0) ? 0 : 1;
2953 :
2954 3306 : if (i)
2955 0 : tmps[0] = 0;
2956 :
2957 8055 : for (; i < length; i++)
2958 : {
2959 4749 : machine_mode mode = GET_MODE (XEXP (XVECEXP (orig, 0, i), 0));
2960 4749 : rtx offset = XEXP (XVECEXP (orig, 0, i), 1);
2961 :
2962 4749 : tmps[i] = gen_rtx_EXPR_LIST (VOIDmode, gen_reg_rtx (mode), offset);
2963 : }
2964 :
2965 3306 : return gen_rtx_PARALLEL (GET_MODE (orig), gen_rtvec_v (length, tmps));
2966 : }
2967 :
2968 : /* A subroutine of emit_group_load. Arguments as for emit_group_load,
2969 : except that values are placed in TMPS[i], and must later be moved
2970 : into corresponding XEXP (XVECEXP (DST, 0, i), 0) element. */
2971 :
2972 : static void
2973 298060 : emit_group_load_1 (rtx *tmps, rtx dst, rtx orig_src, tree type,
2974 : poly_int64 ssize)
2975 : {
2976 298063 : rtx src;
2977 298063 : int start, i;
2978 298063 : machine_mode m = GET_MODE (orig_src);
2979 :
2980 298063 : gcc_assert (GET_CODE (dst) == PARALLEL);
2981 :
2982 298063 : if (m != VOIDmode
2983 279199 : && !SCALAR_INT_MODE_P (m)
2984 19422 : && !MEM_P (orig_src)
2985 5708 : && GET_CODE (orig_src) != CONCAT)
2986 : {
2987 3 : scalar_int_mode imode;
2988 3 : if (int_mode_for_mode (GET_MODE (orig_src)).exists (&imode))
2989 : {
2990 3 : src = gen_reg_rtx (imode);
2991 3 : emit_move_insn (gen_lowpart (GET_MODE (orig_src), src), orig_src);
2992 : }
2993 : else
2994 : {
2995 0 : src = assign_stack_temp (GET_MODE (orig_src), ssize);
2996 0 : emit_move_insn (src, orig_src);
2997 : }
2998 3 : emit_group_load_1 (tmps, dst, src, type, ssize);
2999 : return;
3000 : }
3001 :
3002 : /* Check for a NULL entry, used to indicate that the parameter goes
3003 : both on the stack and in registers. */
3004 298060 : if (XEXP (XVECEXP (dst, 0, 0), 0))
3005 : start = 0;
3006 : else
3007 0 : start = 1;
3008 :
3009 : /* Process the pieces. */
3010 882550 : for (i = start; i < XVECLEN (dst, 0); i++)
3011 : {
3012 584490 : machine_mode mode = GET_MODE (XEXP (XVECEXP (dst, 0, i), 0));
3013 584490 : poly_int64 bytepos = rtx_to_poly_int64 (XEXP (XVECEXP (dst, 0, i), 1));
3014 1168980 : poly_int64 bytelen = GET_MODE_SIZE (mode);
3015 584490 : poly_int64 shift = 0;
3016 :
3017 : /* Handle trailing fragments that run over the size of the struct.
3018 : It's the target's responsibility to make sure that the fragment
3019 : cannot be strictly smaller in some cases and strictly larger
3020 : in others. */
3021 584490 : gcc_checking_assert (ordered_p (bytepos + bytelen, ssize));
3022 584490 : if (known_size_p (ssize) && maybe_gt (bytepos + bytelen, ssize))
3023 : {
3024 : /* Arrange to shift the fragment to where it belongs.
3025 : extract_bit_field loads to the lsb of the reg. */
3026 : if (
3027 : #ifdef BLOCK_REG_PADDING
3028 : BLOCK_REG_PADDING (GET_MODE (orig_src), type, i == start)
3029 : == (BYTES_BIG_ENDIAN ? PAD_UPWARD : PAD_DOWNWARD)
3030 : #else
3031 : BYTES_BIG_ENDIAN
3032 : #endif
3033 : )
3034 : shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
3035 1829 : bytelen = ssize - bytepos;
3036 1829 : gcc_assert (maybe_gt (bytelen, 0));
3037 : }
3038 :
3039 : /* If we won't be loading directly from memory, protect the real source
3040 : from strange tricks we might play; but make sure that the source can
3041 : be loaded directly into the destination. */
3042 584490 : src = orig_src;
3043 584490 : if (!MEM_P (orig_src)
3044 482909 : && (!REG_P (orig_src) || HARD_REGISTER_P (orig_src))
3045 46373 : && GET_CODE (orig_src) != CONCAT
3046 622218 : && !CONSTANT_P (orig_src))
3047 : {
3048 0 : gcc_assert (GET_MODE (orig_src) != VOIDmode);
3049 0 : src = force_reg (GET_MODE (orig_src), orig_src);
3050 : }
3051 :
3052 : /* Optimize the access just a bit. */
3053 584490 : if (MEM_P (src)
3054 101581 : && (! targetm.slow_unaligned_access (mode, MEM_ALIGN (src))
3055 0 : || MEM_ALIGN (src) >= GET_MODE_ALIGNMENT (mode))
3056 203162 : && multiple_p (bytepos * BITS_PER_UNIT, GET_MODE_ALIGNMENT (mode))
3057 686071 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3058 : {
3059 99771 : tmps[i] = gen_reg_rtx (mode);
3060 99771 : emit_move_insn (tmps[i], adjust_address (src, mode, bytepos));
3061 : }
3062 484719 : else if (COMPLEX_MODE_P (mode)
3063 0 : && GET_MODE (src) == mode
3064 484719 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3065 : /* Let emit_move_complex do the bulk of the work. */
3066 0 : tmps[i] = src;
3067 484719 : else if (SCALAR_INT_MODE_P (mode)
3068 478212 : && COMPLEX_MODE_P (GET_MODE (src))
3069 9396 : && known_eq (GET_MODE_SIZE (mode),
3070 : GET_MODE_SIZE (GET_MODE (src)))
3071 487465 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3072 : {
3073 : /* When passing a complex value in an integer mode of the same
3074 : size, explicitly construct (highpart<<isize)+lowpart to
3075 : avoid spilling to memory before reload. */
3076 2746 : rtx tmp = read_complex_part (src, !BYTES_BIG_ENDIAN);
3077 2746 : scalar_int_mode imode = int_mode_for_mode (GET_MODE (tmp)).require();
3078 2746 : tmp = gen_lowpart (imode, tmp);
3079 2746 : tmp = simplify_gen_unary (ZERO_EXTEND, mode, tmp, imode);
3080 2746 : rtx result = force_reg (mode, tmp);
3081 5492 : result = expand_shift (LSHIFT_EXPR, mode, result,
3082 2746 : GET_MODE_BITSIZE (imode), NULL_RTX, 1);
3083 2746 : tmp = read_complex_part (src, BYTES_BIG_ENDIAN);
3084 2746 : tmp = gen_lowpart (imode, tmp);
3085 2746 : tmp = simplify_gen_unary (ZERO_EXTEND, mode, tmp, imode);
3086 2746 : result = simplify_gen_binary (PLUS, mode, result, tmp);
3087 2746 : tmps[i] = force_reg (mode, result);
3088 : }
3089 481973 : else if (GET_CODE (src) == CONCAT)
3090 : {
3091 11798 : poly_int64 slen = GET_MODE_SIZE (GET_MODE (src));
3092 11798 : poly_int64 slen0 = GET_MODE_SIZE (GET_MODE (XEXP (src, 0)));
3093 5899 : unsigned int elt;
3094 5899 : poly_int64 subpos;
3095 :
3096 5899 : if (can_div_trunc_p (bytepos, slen0, &elt, &subpos)
3097 5899 : && known_le (subpos + bytelen, slen0))
3098 : {
3099 : /* The following assumes that the concatenated objects all
3100 : have the same size. In this case, a simple calculation
3101 : can be used to determine the object and the bit field
3102 : to be extracted. */
3103 5880 : tmps[i] = XEXP (src, elt);
3104 5880 : if (maybe_ne (subpos, 0)
3105 5880 : || maybe_ne (subpos + bytelen, slen0)
3106 11760 : || (!CONSTANT_P (tmps[i])
3107 5880 : && (!REG_P (tmps[i]) || GET_MODE (tmps[i]) != mode)))
3108 0 : tmps[i] = extract_bit_field (tmps[i], bytelen * BITS_PER_UNIT,
3109 0 : subpos * BITS_PER_UNIT,
3110 : 1, NULL_RTX, mode, mode, false,
3111 : NULL);
3112 : }
3113 : else
3114 : {
3115 19 : rtx mem;
3116 :
3117 19 : gcc_assert (known_eq (bytepos, 0));
3118 19 : mem = assign_stack_temp (GET_MODE (src), slen);
3119 19 : emit_move_insn (mem, src);
3120 19 : tmps[i] = extract_bit_field (mem, bytelen * BITS_PER_UNIT,
3121 : 0, 1, NULL_RTX, mode, mode, false,
3122 : NULL);
3123 : }
3124 : }
3125 476074 : else if (CONSTANT_P (src) && GET_MODE (dst) != BLKmode
3126 37728 : && XVECLEN (dst, 0) > 1)
3127 37728 : tmps[i] = force_subreg (mode, src, GET_MODE (dst), bytepos);
3128 438346 : else if (CONSTANT_P (src))
3129 : {
3130 0 : if (known_eq (bytelen, ssize))
3131 0 : tmps[i] = src;
3132 : else
3133 : {
3134 : rtx first, second;
3135 :
3136 : /* TODO: const_wide_int can have sizes other than this... */
3137 0 : gcc_assert (known_eq (2 * bytelen, ssize));
3138 0 : split_double (src, &first, &second);
3139 0 : if (i)
3140 0 : tmps[i] = second;
3141 : else
3142 0 : tmps[i] = first;
3143 : }
3144 : }
3145 438346 : else if (REG_P (src) && GET_MODE (src) == mode)
3146 0 : tmps[i] = src;
3147 : else
3148 438346 : tmps[i] = extract_bit_field (src, bytelen * BITS_PER_UNIT,
3149 876692 : bytepos * BITS_PER_UNIT, 1, NULL_RTX,
3150 : mode, mode, false, NULL);
3151 :
3152 584490 : if (maybe_ne (shift, 0))
3153 : tmps[i] = expand_shift (LSHIFT_EXPR, mode, tmps[i],
3154 : shift, tmps[i], 0);
3155 : }
3156 : }
3157 :
3158 : /* Emit code to move a block SRC of type TYPE to a block DST,
3159 : where DST is non-consecutive registers represented by a PARALLEL.
3160 : SSIZE represents the total size of block ORIG_SRC in bytes, or -1
3161 : if not known. */
3162 :
3163 : void
3164 10992 : emit_group_load (rtx dst, rtx src, tree type, poly_int64 ssize)
3165 : {
3166 10992 : rtx *tmps;
3167 10992 : int i;
3168 :
3169 10992 : tmps = XALLOCAVEC (rtx, XVECLEN (dst, 0));
3170 10992 : emit_group_load_1 (tmps, dst, src, type, ssize);
3171 :
3172 : /* Copy the extracted pieces into the proper (probable) hard regs. */
3173 41621 : for (i = 0; i < XVECLEN (dst, 0); i++)
3174 : {
3175 19637 : rtx d = XEXP (XVECEXP (dst, 0, i), 0);
3176 19637 : if (d == NULL)
3177 0 : continue;
3178 19637 : emit_move_insn (d, tmps[i]);
3179 : }
3180 10992 : }
3181 :
3182 : /* Similar, but load SRC into new pseudos in a format that looks like
3183 : PARALLEL. This can later be fed to emit_group_move to get things
3184 : in the right place. */
3185 :
3186 : rtx
3187 287068 : emit_group_load_into_temps (rtx parallel, rtx src, tree type, poly_int64 ssize)
3188 : {
3189 287068 : rtvec vec;
3190 287068 : int i;
3191 :
3192 287068 : vec = rtvec_alloc (XVECLEN (parallel, 0));
3193 287068 : emit_group_load_1 (&RTVEC_ELT (vec, 0), parallel, src, type, ssize);
3194 :
3195 : /* Convert the vector to look just like the original PARALLEL, except
3196 : with the computed values. */
3197 1138989 : for (i = 0; i < XVECLEN (parallel, 0); i++)
3198 : {
3199 564853 : rtx e = XVECEXP (parallel, 0, i);
3200 564853 : rtx d = XEXP (e, 0);
3201 :
3202 564853 : if (d)
3203 : {
3204 564853 : d = force_reg (GET_MODE (d), RTVEC_ELT (vec, i));
3205 564853 : e = alloc_EXPR_LIST (REG_NOTE_KIND (e), d, XEXP (e, 1));
3206 : }
3207 564853 : RTVEC_ELT (vec, i) = e;
3208 : }
3209 :
3210 287068 : return gen_rtx_PARALLEL (GET_MODE (parallel), vec);
3211 : }
3212 :
3213 : /* Emit code to move a block SRC to block DST, where SRC and DST are
3214 : non-consecutive groups of registers, each represented by a PARALLEL. */
3215 :
3216 : void
3217 290374 : emit_group_move (rtx dst, rtx src)
3218 : {
3219 290374 : int i;
3220 :
3221 290374 : gcc_assert (GET_CODE (src) == PARALLEL
3222 : && GET_CODE (dst) == PARALLEL
3223 : && XVECLEN (src, 0) == XVECLEN (dst, 0));
3224 :
3225 : /* Skip first entry if NULL. */
3226 859976 : for (i = XEXP (XVECEXP (src, 0, 0), 0) ? 0 : 1; i < XVECLEN (src, 0); i++)
3227 569602 : emit_move_insn (XEXP (XVECEXP (dst, 0, i), 0),
3228 569602 : XEXP (XVECEXP (src, 0, i), 0));
3229 290374 : }
3230 :
3231 : /* Move a group of registers represented by a PARALLEL into pseudos. */
3232 :
3233 : rtx
3234 5819 : emit_group_move_into_temps (rtx src)
3235 : {
3236 5819 : rtvec vec = rtvec_alloc (XVECLEN (src, 0));
3237 5819 : int i;
3238 :
3239 19512 : for (i = 0; i < XVECLEN (src, 0); i++)
3240 : {
3241 7874 : rtx e = XVECEXP (src, 0, i);
3242 7874 : rtx d = XEXP (e, 0);
3243 :
3244 7874 : if (d)
3245 7874 : e = alloc_EXPR_LIST (REG_NOTE_KIND (e), copy_to_reg (d), XEXP (e, 1));
3246 7874 : RTVEC_ELT (vec, i) = e;
3247 : }
3248 :
3249 5819 : return gen_rtx_PARALLEL (GET_MODE (src), vec);
3250 : }
3251 :
3252 : /* Emit code to move a block SRC to a block ORIG_DST of type TYPE,
3253 : where SRC is non-consecutive registers represented by a PARALLEL.
3254 : SSIZE represents the total size of block ORIG_DST, or -1 if not
3255 : known. */
3256 :
3257 : void
3258 65813 : emit_group_store (rtx orig_dst, rtx src, tree type ATTRIBUTE_UNUSED,
3259 : poly_int64 ssize)
3260 : {
3261 65813 : rtx *tmps, dst;
3262 65813 : int start, finish, i;
3263 65813 : machine_mode m = GET_MODE (orig_dst);
3264 :
3265 65813 : gcc_assert (GET_CODE (src) == PARALLEL);
3266 :
3267 65813 : if (!SCALAR_INT_MODE_P (m)
3268 14579 : && !MEM_P (orig_dst) && GET_CODE (orig_dst) != CONCAT)
3269 : {
3270 0 : scalar_int_mode imode;
3271 0 : if (int_mode_for_mode (GET_MODE (orig_dst)).exists (&imode))
3272 : {
3273 0 : dst = gen_reg_rtx (imode);
3274 0 : emit_group_store (dst, src, type, ssize);
3275 0 : dst = gen_lowpart (GET_MODE (orig_dst), dst);
3276 : }
3277 : else
3278 : {
3279 0 : dst = assign_stack_temp (GET_MODE (orig_dst), ssize);
3280 0 : emit_group_store (dst, src, type, ssize);
3281 : }
3282 0 : emit_move_insn (orig_dst, dst);
3283 0 : return;
3284 : }
3285 :
3286 : /* Check for a NULL entry, used to indicate that the parameter goes
3287 : both on the stack and in registers. */
3288 65813 : if (XEXP (XVECEXP (src, 0, 0), 0))
3289 : start = 0;
3290 : else
3291 0 : start = 1;
3292 65813 : finish = XVECLEN (src, 0);
3293 :
3294 65813 : tmps = XALLOCAVEC (rtx, finish);
3295 :
3296 : /* Copy the (probable) hard regs into pseudos. */
3297 189090 : for (i = start; i < finish; i++)
3298 : {
3299 123277 : rtx reg = XEXP (XVECEXP (src, 0, i), 0);
3300 123277 : if (!REG_P (reg) || REGNO (reg) < FIRST_PSEUDO_REGISTER)
3301 : {
3302 115403 : tmps[i] = gen_reg_rtx (GET_MODE (reg));
3303 115403 : emit_move_insn (tmps[i], reg);
3304 : }
3305 : else
3306 7874 : tmps[i] = reg;
3307 : }
3308 :
3309 : /* If we won't be storing directly into memory, protect the real destination
3310 : from strange tricks we might play. */
3311 65813 : dst = orig_dst;
3312 65813 : if (GET_CODE (dst) == PARALLEL)
3313 : {
3314 0 : rtx temp;
3315 :
3316 : /* We can get a PARALLEL dst if there is a conditional expression in
3317 : a return statement. In that case, the dst and src are the same,
3318 : so no action is necessary. */
3319 0 : if (rtx_equal_p (dst, src))
3320 : return;
3321 :
3322 : /* It is unclear if we can ever reach here, but we may as well handle
3323 : it. Allocate a temporary, and split this into a store/load to/from
3324 : the temporary. */
3325 0 : temp = assign_stack_temp (GET_MODE (dst), ssize);
3326 0 : emit_group_store (temp, src, type, ssize);
3327 0 : emit_group_load (dst, temp, type, ssize);
3328 0 : return;
3329 : }
3330 65813 : else if (!MEM_P (dst) && GET_CODE (dst) != CONCAT)
3331 : {
3332 50600 : machine_mode outer = GET_MODE (dst);
3333 50600 : machine_mode inner;
3334 50600 : poly_int64 bytepos;
3335 50600 : bool done = false;
3336 50600 : rtx temp;
3337 :
3338 50600 : if (!REG_P (dst) || REGNO (dst) < FIRST_PSEUDO_REGISTER)
3339 0 : dst = gen_reg_rtx (outer);
3340 :
3341 : /* Make life a bit easier for combine: if the first element of the
3342 : vector is the low part of the destination mode, use a paradoxical
3343 : subreg to initialize the destination. */
3344 50600 : if (start < finish)
3345 : {
3346 50600 : inner = GET_MODE (tmps[start]);
3347 50600 : bytepos = subreg_lowpart_offset (inner, outer);
3348 50600 : if (known_eq (rtx_to_poly_int64 (XEXP (XVECEXP (src, 0, start), 1)),
3349 : bytepos))
3350 : {
3351 50576 : temp = force_subreg (outer, tmps[start], inner, 0);
3352 50576 : if (temp)
3353 : {
3354 49908 : emit_move_insn (dst, temp);
3355 49908 : done = true;
3356 49908 : start++;
3357 : }
3358 : }
3359 : }
3360 :
3361 : /* If the first element wasn't the low part, try the last. */
3362 49908 : if (!done
3363 692 : && start < finish - 1)
3364 : {
3365 625 : inner = GET_MODE (tmps[finish - 1]);
3366 625 : bytepos = subreg_lowpart_offset (inner, outer);
3367 625 : if (known_eq (rtx_to_poly_int64 (XEXP (XVECEXP (src, 0,
3368 : finish - 1), 1)),
3369 : bytepos))
3370 : {
3371 0 : temp = force_subreg (outer, tmps[finish - 1], inner, 0);
3372 0 : if (temp)
3373 : {
3374 0 : emit_move_insn (dst, temp);
3375 0 : done = true;
3376 0 : finish--;
3377 : }
3378 : }
3379 : }
3380 :
3381 : /* Otherwise, simply initialize the result to zero. */
3382 49908 : if (!done)
3383 692 : emit_move_insn (dst, CONST0_RTX (outer));
3384 : }
3385 :
3386 : /* Process the pieces. */
3387 135612 : for (i = start; i < finish; i++)
3388 : {
3389 73369 : poly_int64 bytepos = rtx_to_poly_int64 (XEXP (XVECEXP (src, 0, i), 1));
3390 73369 : machine_mode mode = GET_MODE (tmps[i]);
3391 146738 : poly_int64 bytelen = GET_MODE_SIZE (mode);
3392 73369 : poly_uint64 adj_bytelen;
3393 73369 : rtx dest = dst;
3394 :
3395 : /* Handle trailing fragments that run over the size of the struct.
3396 : It's the target's responsibility to make sure that the fragment
3397 : cannot be strictly smaller in some cases and strictly larger
3398 : in others. */
3399 73369 : gcc_checking_assert (ordered_p (bytepos + bytelen, ssize));
3400 145791 : if (known_size_p (ssize) && maybe_gt (bytepos + bytelen, ssize))
3401 947 : adj_bytelen = ssize - bytepos;
3402 : else
3403 73369 : adj_bytelen = bytelen;
3404 :
3405 : /* Deal with destination CONCATs by either storing into one of the parts
3406 : or doing a copy after storing into a register or stack temporary. */
3407 73369 : if (GET_CODE (dst) == CONCAT)
3408 : {
3409 28984 : if (known_le (bytepos + adj_bytelen,
3410 : GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)))))
3411 : dest = XEXP (dst, 0);
3412 :
3413 18062 : else if (known_ge (bytepos, GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)))))
3414 : {
3415 10922 : bytepos -= GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)));
3416 5461 : dest = XEXP (dst, 1);
3417 : }
3418 :
3419 : else
3420 : {
3421 3570 : machine_mode dest_mode = GET_MODE (dest);
3422 3570 : machine_mode tmp_mode = GET_MODE (tmps[i]);
3423 3570 : scalar_int_mode dest_imode;
3424 :
3425 3570 : gcc_assert (known_eq (bytepos, 0) && XVECLEN (src, 0));
3426 :
3427 : /* If the source is a single scalar integer register, and the
3428 : destination has a complex mode for which a same-sized integer
3429 : mode exists, then we can take the left-justified part of the
3430 : source in the complex mode. */
3431 3570 : if (finish == start + 1
3432 3570 : && REG_P (tmps[i])
3433 3570 : && SCALAR_INT_MODE_P (tmp_mode)
3434 3570 : && COMPLEX_MODE_P (dest_mode)
3435 7140 : && int_mode_for_mode (dest_mode).exists (&dest_imode))
3436 : {
3437 3570 : const scalar_int_mode tmp_imode
3438 3570 : = as_a <scalar_int_mode> (tmp_mode);
3439 :
3440 7140 : if (GET_MODE_BITSIZE (dest_imode)
3441 3570 : < GET_MODE_BITSIZE (tmp_imode))
3442 : {
3443 60 : dest = gen_reg_rtx (dest_imode);
3444 60 : if (BYTES_BIG_ENDIAN)
3445 : tmps[i] = expand_shift (RSHIFT_EXPR, tmp_mode, tmps[i],
3446 : GET_MODE_BITSIZE (tmp_imode)
3447 : - GET_MODE_BITSIZE (dest_imode),
3448 : NULL_RTX, 1);
3449 60 : emit_move_insn (dest, gen_lowpart (dest_imode, tmps[i]));
3450 60 : dst = gen_lowpart (dest_mode, dest);
3451 : }
3452 : else
3453 3510 : dst = gen_lowpart (dest_mode, tmps[i]);
3454 : }
3455 :
3456 : /* Otherwise spill the source onto the stack using the more
3457 : aligned of the two modes. */
3458 0 : else if (GET_MODE_ALIGNMENT (dest_mode)
3459 0 : >= GET_MODE_ALIGNMENT (tmp_mode))
3460 : {
3461 0 : dest = assign_stack_temp (dest_mode,
3462 0 : GET_MODE_SIZE (dest_mode));
3463 0 : emit_move_insn (adjust_address (dest, tmp_mode, bytepos),
3464 : tmps[i]);
3465 0 : dst = dest;
3466 : }
3467 :
3468 : else
3469 : {
3470 0 : dest = assign_stack_temp (tmp_mode,
3471 0 : GET_MODE_SIZE (tmp_mode));
3472 0 : emit_move_insn (dest, tmps[i]);
3473 0 : dst = adjust_address (dest, dest_mode, bytepos);
3474 : }
3475 :
3476 : break;
3477 : }
3478 : }
3479 :
3480 : /* Handle trailing fragments that run over the size of the struct. */
3481 69799 : if (known_size_p (ssize) && maybe_gt (bytepos + bytelen, ssize))
3482 : {
3483 : /* store_bit_field always takes its value from the lsb.
3484 : Move the fragment to the lsb if it's not already there. */
3485 887 : if (
3486 : #ifdef BLOCK_REG_PADDING
3487 : BLOCK_REG_PADDING (GET_MODE (orig_dst), type, i == start)
3488 : == (BYTES_BIG_ENDIAN ? PAD_UPWARD : PAD_DOWNWARD)
3489 : #else
3490 : BYTES_BIG_ENDIAN
3491 : #endif
3492 : )
3493 : {
3494 : poly_int64 shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
3495 : tmps[i] = expand_shift (RSHIFT_EXPR, mode, tmps[i],
3496 : shift, tmps[i], 0);
3497 : }
3498 :
3499 : /* Make sure not to write past the end of the struct. */
3500 887 : store_bit_field (dest,
3501 887 : adj_bytelen * BITS_PER_UNIT, bytepos * BITS_PER_UNIT,
3502 3548 : bytepos * BITS_PER_UNIT, ssize * BITS_PER_UNIT - 1,
3503 : VOIDmode, tmps[i], false, false);
3504 : }
3505 :
3506 : /* Optimize the access just a bit. */
3507 68912 : else if (MEM_P (dest)
3508 7598 : && (!targetm.slow_unaligned_access (mode, MEM_ALIGN (dest))
3509 0 : || MEM_ALIGN (dest) >= GET_MODE_ALIGNMENT (mode))
3510 15196 : && multiple_p (bytepos * BITS_PER_UNIT,
3511 : GET_MODE_ALIGNMENT (mode))
3512 76510 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3513 7598 : emit_move_insn (adjust_address (dest, mode, bytepos), tmps[i]);
3514 :
3515 : else
3516 122628 : store_bit_field (dest, bytelen * BITS_PER_UNIT, bytepos * BITS_PER_UNIT,
3517 : 0, 0, mode, tmps[i], false, false);
3518 : }
3519 :
3520 : /* Copy from the pseudo into the (probable) hard reg. */
3521 65813 : if (orig_dst != dst)
3522 3570 : emit_move_insn (orig_dst, dst);
3523 : }
3524 :
3525 : /* Return a form of X that does not use a PARALLEL. TYPE is the type
3526 : of the value stored in X. */
3527 :
3528 : rtx
3529 335942 : maybe_emit_group_store (rtx x, tree type)
3530 : {
3531 335942 : machine_mode mode = TYPE_MODE (type);
3532 335942 : gcc_checking_assert (GET_MODE (x) == VOIDmode || GET_MODE (x) == mode);
3533 335942 : if (GET_CODE (x) == PARALLEL)
3534 : {
3535 0 : rtx result = gen_reg_rtx (mode);
3536 0 : emit_group_store (result, x, type, int_size_in_bytes (type));
3537 0 : return result;
3538 : }
3539 : return x;
3540 : }
3541 :
3542 : /* Copy a BLKmode object of TYPE out of a register SRCREG into TARGET.
3543 :
3544 : This is used on targets that return BLKmode values in registers. */
3545 :
3546 : static void
3547 256 : copy_blkmode_from_reg (rtx target, rtx srcreg, tree type)
3548 : {
3549 256 : unsigned HOST_WIDE_INT bytes = int_size_in_bytes (type);
3550 256 : rtx src = NULL, dst = NULL;
3551 256 : unsigned HOST_WIDE_INT bitsize = MIN (TYPE_ALIGN (type), BITS_PER_WORD);
3552 256 : unsigned HOST_WIDE_INT bitpos, xbitpos, padding_correction = 0;
3553 : /* No current ABI uses variable-sized modes to pass a BLKmnode type. */
3554 256 : fixed_size_mode mode = as_a <fixed_size_mode> (GET_MODE (srcreg));
3555 256 : fixed_size_mode tmode = as_a <fixed_size_mode> (GET_MODE (target));
3556 256 : fixed_size_mode copy_mode;
3557 :
3558 : /* BLKmode registers created in the back-end shouldn't have survived. */
3559 256 : gcc_assert (mode != BLKmode);
3560 :
3561 : /* If the structure doesn't take up a whole number of words, see whether
3562 : SRCREG is padded on the left or on the right. If it's on the left,
3563 : set PADDING_CORRECTION to the number of bits to skip.
3564 :
3565 : In most ABIs, the structure will be returned at the least end of
3566 : the register, which translates to right padding on little-endian
3567 : targets and left padding on big-endian targets. The opposite
3568 : holds if the structure is returned at the most significant
3569 : end of the register. */
3570 256 : if (bytes % UNITS_PER_WORD != 0
3571 256 : && (targetm.calls.return_in_msb (type)
3572 225 : ? !BYTES_BIG_ENDIAN
3573 : : BYTES_BIG_ENDIAN))
3574 0 : padding_correction
3575 0 : = (BITS_PER_WORD - ((bytes % UNITS_PER_WORD) * BITS_PER_UNIT));
3576 :
3577 : /* We can use a single move if we have an exact mode for the size. */
3578 256 : else if (MEM_P (target)
3579 256 : && (!targetm.slow_unaligned_access (mode, MEM_ALIGN (target))
3580 0 : || MEM_ALIGN (target) >= GET_MODE_ALIGNMENT (mode))
3581 512 : && bytes == GET_MODE_SIZE (mode))
3582 : {
3583 38 : emit_move_insn (adjust_address (target, mode, 0), srcreg);
3584 38 : return;
3585 : }
3586 :
3587 : /* And if we additionally have the same mode for a register. */
3588 218 : else if (REG_P (target)
3589 0 : && GET_MODE (target) == mode
3590 218 : && bytes == GET_MODE_SIZE (mode))
3591 : {
3592 0 : emit_move_insn (target, srcreg);
3593 0 : return;
3594 : }
3595 :
3596 : /* This code assumes srcreg is at least a full word. If it isn't, copy it
3597 : into a new pseudo which is a full word. */
3598 436 : if (GET_MODE_SIZE (mode) < UNITS_PER_WORD)
3599 : {
3600 77 : srcreg = convert_to_mode (word_mode, srcreg, TYPE_UNSIGNED (type));
3601 77 : mode = word_mode;
3602 : }
3603 :
3604 : /* Copy the structure BITSIZE bits at a time. If the target lives in
3605 : memory, take care of not reading/writing past its end by selecting
3606 : a copy mode suited to BITSIZE. This should always be possible given
3607 : how it is computed.
3608 :
3609 : If the target lives in register, make sure not to select a copy mode
3610 : larger than the mode of the register.
3611 :
3612 : We could probably emit more efficient code for machines which do not use
3613 : strict alignment, but it doesn't seem worth the effort at the current
3614 : time. */
3615 :
3616 218 : copy_mode = word_mode;
3617 218 : if (MEM_P (target))
3618 : {
3619 218 : opt_scalar_int_mode mem_mode = int_mode_for_size (bitsize, 1);
3620 218 : if (mem_mode.exists ())
3621 218 : copy_mode = mem_mode.require ();
3622 : }
3623 0 : else if (REG_P (target) && GET_MODE_BITSIZE (tmode) < BITS_PER_WORD)
3624 : copy_mode = tmode;
3625 :
3626 218 : for (bitpos = 0, xbitpos = padding_correction;
3627 1068 : bitpos < bytes * BITS_PER_UNIT;
3628 850 : bitpos += bitsize, xbitpos += bitsize)
3629 : {
3630 : /* We need a new source operand each time xbitpos is on a
3631 : word boundary and when xbitpos == padding_correction
3632 : (the first time through). */
3633 850 : if (xbitpos % BITS_PER_WORD == 0 || xbitpos == padding_correction)
3634 218 : src = operand_subword_force (srcreg, xbitpos / BITS_PER_WORD, mode);
3635 :
3636 : /* We need a new destination operand each time bitpos is on
3637 : a word boundary. */
3638 850 : if (REG_P (target) && GET_MODE_BITSIZE (tmode) < BITS_PER_WORD)
3639 : dst = target;
3640 850 : else if (bitpos % BITS_PER_WORD == 0)
3641 218 : dst = operand_subword (target, bitpos / BITS_PER_WORD, 1, tmode);
3642 :
3643 : /* Use xbitpos for the source extraction (right justified) and
3644 : bitpos for the destination store (left justified). */
3645 850 : store_bit_field (dst, bitsize, bitpos % BITS_PER_WORD, 0, 0, copy_mode,
3646 850 : extract_bit_field (src, bitsize,
3647 850 : xbitpos % BITS_PER_WORD, 1,
3648 : NULL_RTX, copy_mode, copy_mode,
3649 : false, NULL),
3650 : false, false);
3651 : }
3652 : }
3653 :
3654 : /* Copy BLKmode value SRC into a register of mode MODE_IN. Return the
3655 : register if it contains any data, otherwise return null.
3656 :
3657 : This is used on targets that return BLKmode values in registers. */
3658 :
3659 : rtx
3660 3688 : copy_blkmode_to_reg (machine_mode mode_in, tree src)
3661 : {
3662 3688 : int i, n_regs;
3663 3688 : unsigned HOST_WIDE_INT bitpos, xbitpos, padding_correction = 0, bytes;
3664 3688 : unsigned int bitsize;
3665 3688 : rtx *dst_words, dst, x, src_word = NULL_RTX, dst_word = NULL_RTX;
3666 : /* No current ABI uses variable-sized modes to pass a BLKmnode type. */
3667 3688 : fixed_size_mode mode = as_a <fixed_size_mode> (mode_in);
3668 3688 : fixed_size_mode dst_mode;
3669 3688 : scalar_int_mode min_mode;
3670 :
3671 3688 : gcc_assert (TYPE_MODE (TREE_TYPE (src)) == BLKmode);
3672 :
3673 3688 : x = expand_normal (src);
3674 :
3675 3688 : bytes = arg_int_size_in_bytes (TREE_TYPE (src));
3676 3688 : if (bytes == 0)
3677 : return NULL_RTX;
3678 :
3679 : /* If the structure doesn't take up a whole number of words, see
3680 : whether the register value should be padded on the left or on
3681 : the right. Set PADDING_CORRECTION to the number of padding
3682 : bits needed on the left side.
3683 :
3684 : In most ABIs, the structure will be returned at the least end of
3685 : the register, which translates to right padding on little-endian
3686 : targets and left padding on big-endian targets. The opposite
3687 : holds if the structure is returned at the most significant
3688 : end of the register. */
3689 1224 : if (bytes % UNITS_PER_WORD != 0
3690 1224 : && (targetm.calls.return_in_msb (TREE_TYPE (src))
3691 1188 : ? !BYTES_BIG_ENDIAN
3692 : : BYTES_BIG_ENDIAN))
3693 0 : padding_correction = (BITS_PER_WORD - ((bytes % UNITS_PER_WORD)
3694 0 : * BITS_PER_UNIT));
3695 :
3696 1224 : n_regs = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
3697 1224 : dst_words = XALLOCAVEC (rtx, n_regs);
3698 1224 : bitsize = MIN (TYPE_ALIGN (TREE_TYPE (src)), BITS_PER_WORD);
3699 1224 : min_mode = smallest_int_mode_for_size (bitsize).require ();
3700 :
3701 : /* Copy the structure BITSIZE bits at a time. */
3702 1224 : for (bitpos = 0, xbitpos = padding_correction;
3703 3974 : bitpos < bytes * BITS_PER_UNIT;
3704 2750 : bitpos += bitsize, xbitpos += bitsize)
3705 : {
3706 : /* We need a new destination pseudo each time xbitpos is
3707 : on a word boundary and when xbitpos == padding_correction
3708 : (the first time through). */
3709 2750 : if (xbitpos % BITS_PER_WORD == 0
3710 1523 : || xbitpos == padding_correction)
3711 : {
3712 : /* Generate an appropriate register. */
3713 1227 : dst_word = gen_reg_rtx (word_mode);
3714 1227 : dst_words[xbitpos / BITS_PER_WORD] = dst_word;
3715 :
3716 : /* Clear the destination before we move anything into it. */
3717 1227 : emit_move_insn (dst_word, CONST0_RTX (word_mode));
3718 : }
3719 :
3720 : /* Find the largest integer mode that can be used to copy all or as
3721 : many bits as possible of the structure if the target supports larger
3722 : copies. There are too many corner cases here w.r.t to alignments on
3723 : the read/writes. So if there is any padding just use single byte
3724 : operations. */
3725 2750 : opt_scalar_int_mode mode_iter;
3726 2750 : if (padding_correction == 0 && !STRICT_ALIGNMENT)
3727 : {
3728 7488 : FOR_EACH_MODE_FROM (mode_iter, min_mode)
3729 : {
3730 7488 : unsigned int msize = GET_MODE_BITSIZE (mode_iter.require ());
3731 7488 : if (msize <= ((bytes * BITS_PER_UNIT) - bitpos)
3732 4741 : && msize <= BITS_PER_WORD)
3733 4738 : bitsize = msize;
3734 : else
3735 : break;
3736 : }
3737 : }
3738 :
3739 : /* We need a new source operand each time bitpos is on a word
3740 : boundary. */
3741 2750 : if (bitpos % BITS_PER_WORD == 0)
3742 1227 : src_word = operand_subword_force (x, bitpos / BITS_PER_WORD, BLKmode);
3743 :
3744 : /* Use bitpos for the source extraction (left justified) and
3745 : xbitpos for the destination store (right justified). */
3746 5500 : store_bit_field (dst_word, bitsize, xbitpos % BITS_PER_WORD,
3747 : 0, 0, word_mode,
3748 2750 : extract_bit_field (src_word, bitsize,
3749 2750 : bitpos % BITS_PER_WORD, 1,
3750 : NULL_RTX, word_mode, word_mode,
3751 : false, NULL),
3752 : false, false);
3753 : }
3754 :
3755 1224 : if (mode == BLKmode)
3756 : {
3757 : /* Find the smallest integer mode large enough to hold the
3758 : entire structure. */
3759 0 : opt_scalar_int_mode mode_iter;
3760 0 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
3761 0 : if (GET_MODE_SIZE (mode_iter.require ()) >= bytes)
3762 : break;
3763 :
3764 : /* A suitable mode should have been found. */
3765 0 : mode = mode_iter.require ();
3766 : }
3767 :
3768 3672 : if (GET_MODE_SIZE (mode) < GET_MODE_SIZE (word_mode))
3769 : dst_mode = word_mode;
3770 : else
3771 498 : dst_mode = mode;
3772 1224 : dst = gen_reg_rtx (dst_mode);
3773 :
3774 3675 : for (i = 0; i < n_regs; i++)
3775 1227 : emit_move_insn (operand_subword (dst, i, 0, dst_mode), dst_words[i]);
3776 :
3777 1224 : if (mode != dst_mode)
3778 726 : dst = gen_lowpart (mode, dst);
3779 :
3780 : return dst;
3781 : }
3782 :
3783 : /* Add a USE expression for REG to the (possibly empty) list pointed
3784 : to by CALL_FUSAGE. REG must denote a hard register. */
3785 :
3786 : void
3787 11769963 : use_reg_mode (rtx *call_fusage, rtx reg, machine_mode mode)
3788 : {
3789 11769963 : gcc_assert (REG_P (reg));
3790 :
3791 11769963 : if (!HARD_REGISTER_P (reg))
3792 : return;
3793 :
3794 11769963 : *call_fusage
3795 11769963 : = gen_rtx_EXPR_LIST (mode, gen_rtx_USE (VOIDmode, reg), *call_fusage);
3796 : }
3797 :
3798 : /* Add a CLOBBER expression for REG to the (possibly empty) list pointed
3799 : to by CALL_FUSAGE. REG must denote a hard register. */
3800 :
3801 : void
3802 0 : clobber_reg_mode (rtx *call_fusage, rtx reg, machine_mode mode)
3803 : {
3804 0 : gcc_assert (REG_P (reg) && REGNO (reg) < FIRST_PSEUDO_REGISTER);
3805 :
3806 0 : *call_fusage
3807 0 : = gen_rtx_EXPR_LIST (mode, gen_rtx_CLOBBER (VOIDmode, reg), *call_fusage);
3808 0 : }
3809 :
3810 : /* Add USE expressions to *CALL_FUSAGE for each of NREGS consecutive regs,
3811 : starting at REGNO. All of these registers must be hard registers. */
3812 :
3813 : void
3814 1609 : use_regs (rtx *call_fusage, int regno, int nregs)
3815 : {
3816 1609 : int i;
3817 :
3818 1609 : gcc_assert (regno + nregs <= FIRST_PSEUDO_REGISTER);
3819 :
3820 3226 : for (i = 0; i < nregs; i++)
3821 1617 : use_reg (call_fusage, regno_reg_rtx[regno + i]);
3822 1609 : }
3823 :
3824 : /* Add USE expressions to *CALL_FUSAGE for each REG contained in the
3825 : PARALLEL REGS. This is for calls that pass values in multiple
3826 : non-contiguous locations. The Irix 6 ABI has examples of this. */
3827 :
3828 : void
3829 293801 : use_group_regs (rtx *call_fusage, rtx regs)
3830 : {
3831 293801 : int i;
3832 :
3833 872120 : for (i = 0; i < XVECLEN (regs, 0); i++)
3834 : {
3835 578319 : rtx reg = XEXP (XVECEXP (regs, 0, i), 0);
3836 :
3837 : /* A NULL entry means the parameter goes both on the stack and in
3838 : registers. This can also be a MEM for targets that pass values
3839 : partially on the stack and partially in registers. */
3840 578319 : if (reg != 0 && REG_P (reg))
3841 578319 : use_reg (call_fusage, reg);
3842 : }
3843 293801 : }
3844 :
3845 : /* Return the defining gimple statement for SSA_NAME NAME if it is an
3846 : assignment and the code of the expression on the RHS is CODE. Return
3847 : NULL otherwise. */
3848 :
3849 : static gimple *
3850 12210146 : get_def_for_expr (tree name, enum tree_code code)
3851 : {
3852 12210146 : gimple *def_stmt;
3853 :
3854 12210146 : if (TREE_CODE (name) != SSA_NAME)
3855 : return NULL;
3856 :
3857 9521276 : def_stmt = get_gimple_for_ssa_name (name);
3858 9521276 : if (!def_stmt
3859 2042641 : || !is_gimple_assign (def_stmt)
3860 11563397 : || gimple_assign_rhs_code (def_stmt) != code)
3861 9488788 : return NULL;
3862 :
3863 : return def_stmt;
3864 : }
3865 :
3866 : /* Return the defining gimple statement for SSA_NAME NAME if it is an
3867 : assignment and the class of the expression on the RHS is CLASS. Return
3868 : NULL otherwise. */
3869 :
3870 : static gimple *
3871 17074 : get_def_for_expr_class (tree name, enum tree_code_class tclass)
3872 : {
3873 17074 : gimple *def_stmt;
3874 :
3875 17074 : if (TREE_CODE (name) != SSA_NAME)
3876 : return NULL;
3877 :
3878 17074 : def_stmt = get_gimple_for_ssa_name (name);
3879 17074 : if (!def_stmt
3880 14949 : || !is_gimple_assign (def_stmt)
3881 32023 : || TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) != tclass)
3882 2262 : return NULL;
3883 :
3884 : return def_stmt;
3885 : }
3886 :
3887 : /* Write zeros through the storage of OBJECT. If OBJECT has BLKmode, SIZE is
3888 : its length in bytes. */
3889 :
3890 : rtx
3891 160812 : clear_storage_hints (rtx object, rtx size, enum block_op_methods method,
3892 : unsigned int expected_align, HOST_WIDE_INT expected_size,
3893 : unsigned HOST_WIDE_INT min_size,
3894 : unsigned HOST_WIDE_INT max_size,
3895 : unsigned HOST_WIDE_INT probable_max_size,
3896 : unsigned ctz_size)
3897 : {
3898 160812 : machine_mode mode = GET_MODE (object);
3899 160812 : unsigned int align;
3900 :
3901 160812 : gcc_assert (method == BLOCK_OP_NORMAL || method == BLOCK_OP_TAILCALL);
3902 :
3903 : /* If OBJECT is not BLKmode and SIZE is the same size as its mode,
3904 : just move a zero. Otherwise, do this a piece at a time. */
3905 160812 : poly_int64 size_val;
3906 160812 : if (mode != BLKmode
3907 58897 : && poly_int_rtx_p (size, &size_val)
3908 219709 : && known_eq (size_val, GET_MODE_SIZE (mode)))
3909 : {
3910 58897 : rtx zero = CONST0_RTX (mode);
3911 58897 : if (zero != NULL)
3912 : {
3913 58897 : emit_move_insn (object, zero);
3914 58897 : return NULL;
3915 : }
3916 :
3917 0 : if (COMPLEX_MODE_P (mode))
3918 : {
3919 0 : zero = CONST0_RTX (GET_MODE_INNER (mode));
3920 0 : if (zero != NULL)
3921 : {
3922 0 : write_complex_part (object, zero, 0, true);
3923 0 : write_complex_part (object, zero, 1, false);
3924 0 : return NULL;
3925 : }
3926 : }
3927 : }
3928 :
3929 101915 : if (size == const0_rtx)
3930 : return NULL;
3931 :
3932 101912 : align = MEM_ALIGN (object);
3933 :
3934 101912 : if (CONST_INT_P (size)
3935 101912 : && targetm.use_by_pieces_infrastructure_p (INTVAL (size), align,
3936 : CLEAR_BY_PIECES,
3937 : optimize_insn_for_speed_p ()))
3938 73248 : clear_by_pieces (object, INTVAL (size), align);
3939 28664 : else if (set_storage_via_setmem (object, size, const0_rtx, align,
3940 : expected_align, expected_size,
3941 : min_size, max_size, probable_max_size))
3942 : ;
3943 16251 : else if (try_store_by_multiple_pieces (object, size, ctz_size,
3944 : min_size, max_size,
3945 : NULL_RTX, 0, align))
3946 : ;
3947 15987 : else if (ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (object)))
3948 15987 : return set_storage_via_libcall (object, size, const0_rtx,
3949 15987 : method == BLOCK_OP_TAILCALL);
3950 : else
3951 0 : gcc_unreachable ();
3952 :
3953 : return NULL;
3954 : }
3955 :
3956 : rtx
3957 113308 : clear_storage (rtx object, rtx size, enum block_op_methods method)
3958 : {
3959 113308 : unsigned HOST_WIDE_INT max, min = 0;
3960 113308 : if (GET_CODE (size) == CONST_INT)
3961 113308 : min = max = UINTVAL (size);
3962 : else
3963 0 : max = GET_MODE_MASK (GET_MODE (size));
3964 113308 : return clear_storage_hints (object, size, method, 0, -1, min, max, max, 0);
3965 : }
3966 :
3967 :
3968 : /* A subroutine of clear_storage. Expand a call to memset.
3969 : Return the return value of memset, 0 otherwise. */
3970 :
3971 : rtx
3972 15988 : set_storage_via_libcall (rtx object, rtx size, rtx val, bool tailcall)
3973 : {
3974 15988 : tree call_expr, fn, object_tree, size_tree, val_tree;
3975 15988 : machine_mode size_mode;
3976 :
3977 15988 : object = copy_addr_to_reg (XEXP (object, 0));
3978 15988 : object_tree = make_tree (ptr_type_node, object);
3979 :
3980 15988 : if (!CONST_INT_P (val))
3981 0 : val = convert_to_mode (TYPE_MODE (integer_type_node), val, 1);
3982 15988 : val_tree = make_tree (integer_type_node, val);
3983 :
3984 15988 : size_mode = TYPE_MODE (sizetype);
3985 15988 : size = convert_to_mode (size_mode, size, 1);
3986 15988 : size = copy_to_mode_reg (size_mode, size);
3987 15988 : size_tree = make_tree (sizetype, size);
3988 :
3989 : /* It is incorrect to use the libcall calling conventions for calls to
3990 : memset because it can be provided by the user. */
3991 15988 : fn = builtin_decl_implicit (BUILT_IN_MEMSET);
3992 15988 : call_expr = build_call_expr (fn, 3, object_tree, val_tree, size_tree);
3993 15988 : CALL_EXPR_TAILCALL (call_expr) = tailcall;
3994 :
3995 15988 : return expand_call (call_expr, NULL_RTX, false);
3996 : }
3997 :
3998 : /* Expand a setmem pattern; return true if successful. */
3999 :
4000 : bool
4001 37422 : set_storage_via_setmem (rtx object, rtx size, rtx val, unsigned int align,
4002 : unsigned int expected_align, HOST_WIDE_INT expected_size,
4003 : unsigned HOST_WIDE_INT min_size,
4004 : unsigned HOST_WIDE_INT max_size,
4005 : unsigned HOST_WIDE_INT probable_max_size)
4006 : {
4007 : /* Try the most limited insn first, because there's no point
4008 : including more than one in the machine description unless
4009 : the more limited one has some advantage. */
4010 :
4011 37422 : if (expected_align < align)
4012 : expected_align = align;
4013 37422 : if (expected_size != -1)
4014 : {
4015 8 : if ((unsigned HOST_WIDE_INT)expected_size > max_size)
4016 0 : expected_size = max_size;
4017 8 : if ((unsigned HOST_WIDE_INT)expected_size < min_size)
4018 0 : expected_size = min_size;
4019 : }
4020 :
4021 37422 : opt_scalar_int_mode mode_iter;
4022 226645 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
4023 : {
4024 203918 : scalar_int_mode mode = mode_iter.require ();
4025 203918 : enum insn_code code = direct_optab_handler (setmem_optab, mode);
4026 :
4027 203918 : if (code != CODE_FOR_nothing
4028 : /* We don't need MODE to be narrower than BITS_PER_HOST_WIDE_INT
4029 : here because if SIZE is less than the mode mask, as it is
4030 : returned by the macro, it will definitely be less than the
4031 : actual mode mask. Since SIZE is within the Pmode address
4032 : space, we limit MODE to Pmode. */
4033 203918 : && ((CONST_INT_P (size)
4034 32223 : && ((unsigned HOST_WIDE_INT) INTVAL (size)
4035 32223 : <= (GET_MODE_MASK (mode) >> 1)))
4036 27037 : || max_size <= (GET_MODE_MASK (mode) >> 1)
4037 32462 : || GET_MODE_BITSIZE (mode) >= GET_MODE_BITSIZE (Pmode)))
4038 : {
4039 48434 : class expand_operand ops[9];
4040 48434 : unsigned int nops;
4041 :
4042 48434 : nops = insn_data[(int) code].n_generator_args;
4043 48434 : gcc_assert (nops == 4 || nops == 6 || nops == 8 || nops == 9);
4044 :
4045 48434 : create_fixed_operand (&ops[0], object);
4046 : /* The check above guarantees that this size conversion is valid. */
4047 48434 : create_convert_operand_to (&ops[1], size, mode, true);
4048 48434 : create_convert_operand_from (&ops[2], val, byte_mode, true);
4049 48434 : create_integer_operand (&ops[3], align / BITS_PER_UNIT);
4050 48434 : if (nops >= 6)
4051 : {
4052 48434 : create_integer_operand (&ops[4], expected_align / BITS_PER_UNIT);
4053 48434 : create_integer_operand (&ops[5], expected_size);
4054 : }
4055 48434 : if (nops >= 8)
4056 : {
4057 48434 : create_integer_operand (&ops[6], min_size);
4058 : /* If we cannot represent the maximal size,
4059 : make parameter NULL. */
4060 48434 : if ((HOST_WIDE_INT) max_size != -1)
4061 45101 : create_integer_operand (&ops[7], max_size);
4062 : else
4063 3333 : create_fixed_operand (&ops[7], NULL);
4064 : }
4065 48434 : if (nops == 9)
4066 : {
4067 : /* If we cannot represent the maximal size,
4068 : make parameter NULL. */
4069 48434 : if ((HOST_WIDE_INT) probable_max_size != -1)
4070 45251 : create_integer_operand (&ops[8], probable_max_size);
4071 : else
4072 3183 : create_fixed_operand (&ops[8], NULL);
4073 : }
4074 48434 : gcc_assert (min_size != max_size
4075 : || rtx_equal_p (ops[1].value, GEN_INT (min_size)));
4076 48434 : if (maybe_expand_insn (code, nops, ops))
4077 14695 : return true;
4078 : }
4079 : }
4080 :
4081 : return false;
4082 : }
4083 :
4084 :
4085 : /* Write to one of the components of the complex value CPLX. Write VAL to
4086 : the real part if IMAG_P is false, and the imaginary part if its true.
4087 : If UNDEFINED_P then the value in CPLX is currently undefined. */
4088 :
4089 : void
4090 575761 : write_complex_part (rtx cplx, rtx val, bool imag_p, bool undefined_p)
4091 : {
4092 575761 : machine_mode cmode;
4093 575761 : scalar_mode imode;
4094 575761 : unsigned ibitsize;
4095 :
4096 575761 : if (GET_CODE (cplx) == CONCAT)
4097 : {
4098 517317 : emit_move_insn (XEXP (cplx, imag_p), val);
4099 517317 : return;
4100 : }
4101 :
4102 58444 : cmode = GET_MODE (cplx);
4103 58444 : imode = GET_MODE_INNER (cmode);
4104 58444 : ibitsize = GET_MODE_BITSIZE (imode);
4105 :
4106 : /* For MEMs simplify_gen_subreg may generate an invalid new address
4107 : because, e.g., the original address is considered mode-dependent
4108 : by the target, which restricts simplify_subreg from invoking
4109 : adjust_address_nv. Instead of preparing fallback support for an
4110 : invalid address, we call adjust_address_nv directly. */
4111 58444 : if (MEM_P (cplx))
4112 : {
4113 69087 : emit_move_insn (adjust_address_nv (cplx, imode,
4114 : imag_p ? GET_MODE_SIZE (imode) : 0),
4115 : val);
4116 46058 : return;
4117 : }
4118 :
4119 : /* If the sub-object is at least word sized, then we know that subregging
4120 : will work. This special case is important, since store_bit_field
4121 : wants to operate on integer modes, and there's rarely an OImode to
4122 : correspond to TCmode. */
4123 12386 : if (ibitsize >= BITS_PER_WORD
4124 : /* For hard regs we have exact predicates. Assume we can split
4125 : the original object if it spans an even number of hard regs.
4126 : This special case is important for SCmode on 64-bit platforms
4127 : where the natural size of floating-point regs is 32-bit. */
4128 12386 : || (REG_P (cplx)
4129 7324 : && REGNO (cplx) < FIRST_PSEUDO_REGISTER
4130 7122 : && REG_NREGS (cplx) % 2 == 0))
4131 : {
4132 5078 : rtx part = simplify_gen_subreg (imode, cplx, cmode,
4133 7617 : imag_p ? GET_MODE_SIZE (imode) : 0);
4134 5078 : if (part)
4135 : {
4136 5078 : emit_move_insn (part, val);
4137 5078 : return;
4138 : }
4139 : else
4140 : /* simplify_gen_subreg may fail for sub-word MEMs. */
4141 0 : gcc_assert (MEM_P (cplx) && ibitsize < BITS_PER_WORD);
4142 : }
4143 :
4144 10962 : store_bit_field (cplx, ibitsize, imag_p ? ibitsize : 0, 0, 0, imode, val,
4145 : false, undefined_p);
4146 : }
4147 :
4148 : /* Extract one of the components of the complex value CPLX. Extract the
4149 : real part if IMAG_P is false, and the imaginary part if it's true. */
4150 :
4151 : rtx
4152 562935 : read_complex_part (rtx cplx, bool imag_p)
4153 : {
4154 562935 : machine_mode cmode;
4155 562935 : scalar_mode imode;
4156 562935 : unsigned ibitsize;
4157 :
4158 562935 : if (GET_CODE (cplx) == CONCAT)
4159 348306 : return XEXP (cplx, imag_p);
4160 :
4161 214629 : cmode = GET_MODE (cplx);
4162 214629 : imode = GET_MODE_INNER (cmode);
4163 214629 : ibitsize = GET_MODE_BITSIZE (imode);
4164 :
4165 : /* Special case reads from complex constants that got spilled to memory. */
4166 214629 : if (MEM_P (cplx) && GET_CODE (XEXP (cplx, 0)) == SYMBOL_REF)
4167 : {
4168 37945 : tree decl = SYMBOL_REF_DECL (XEXP (cplx, 0));
4169 37945 : if (decl && TREE_CODE (decl) == COMPLEX_CST)
4170 : {
4171 0 : tree part = imag_p ? TREE_IMAGPART (decl) : TREE_REALPART (decl);
4172 0 : if (CONSTANT_CLASS_P (part))
4173 0 : return expand_expr (part, NULL_RTX, imode, EXPAND_NORMAL);
4174 : }
4175 : }
4176 :
4177 : /* For MEMs simplify_gen_subreg may generate an invalid new address
4178 : because, e.g., the original address is considered mode-dependent
4179 : by the target, which restricts simplify_subreg from invoking
4180 : adjust_address_nv. Instead of preparing fallback support for an
4181 : invalid address, we call adjust_address_nv directly. */
4182 214629 : if (MEM_P (cplx))
4183 237094 : return adjust_address_nv (cplx, imode,
4184 : imag_p ? GET_MODE_SIZE (imode) : 0);
4185 :
4186 : /* If the sub-object is at least word sized, then we know that subregging
4187 : will work. This special case is important, since extract_bit_field
4188 : wants to operate on integer modes, and there's rarely an OImode to
4189 : correspond to TCmode. */
4190 56392 : if (ibitsize >= BITS_PER_WORD
4191 : /* For hard regs we have exact predicates. Assume we can split
4192 : the original object if it spans an even number of hard regs.
4193 : This special case is important for SCmode on 64-bit platforms
4194 : where the natural size of floating-point regs is 32-bit. */
4195 56392 : || (REG_P (cplx)
4196 35434 : && REGNO (cplx) < FIRST_PSEUDO_REGISTER
4197 294 : && REG_NREGS (cplx) % 2 == 0))
4198 : {
4199 10297 : rtx ret = simplify_gen_subreg (imode, cplx, cmode,
4200 15445 : imag_p ? GET_MODE_SIZE (imode) : 0);
4201 10297 : if (ret)
4202 : return ret;
4203 : }
4204 :
4205 69140 : return extract_bit_field (cplx, ibitsize, imag_p ? ibitsize : 0,
4206 : true, NULL_RTX, imode, imode, false, NULL);
4207 : }
4208 :
4209 : /* A subroutine of emit_move_insn_1. Yet another lowpart generator.
4210 : NEW_MODE and OLD_MODE are the same size. Return NULL if X cannot be
4211 : represented in NEW_MODE. If FORCE is true, this will never happen, as
4212 : we'll force-create a SUBREG if needed. */
4213 :
4214 : static rtx
4215 276883 : emit_move_change_mode (machine_mode new_mode,
4216 : machine_mode old_mode, rtx x, bool force)
4217 : {
4218 276883 : rtx ret;
4219 :
4220 276883 : if (push_operand (x, GET_MODE (x)))
4221 : {
4222 1318 : ret = gen_rtx_MEM (new_mode, XEXP (x, 0));
4223 1318 : MEM_COPY_ATTRIBUTES (ret, x);
4224 : }
4225 275565 : else if (MEM_P (x))
4226 : {
4227 : /* We don't have to worry about changing the address since the
4228 : size in bytes is supposed to be the same. */
4229 52533 : if (reload_in_progress)
4230 : {
4231 : /* Copy the MEM to change the mode and move any
4232 : substitutions from the old MEM to the new one. */
4233 0 : ret = adjust_address_nv (x, new_mode, 0);
4234 0 : copy_replacements (x, ret);
4235 : }
4236 : else
4237 52533 : ret = adjust_address (x, new_mode, 0);
4238 : }
4239 : else
4240 : {
4241 : /* Note that we do want simplify_subreg's behavior of validating
4242 : that the new mode is ok for a hard register. If we were to use
4243 : simplify_gen_subreg, we would create the subreg, but would
4244 : probably run into the target not being able to implement it. */
4245 : /* Except, of course, when FORCE is true, when this is exactly what
4246 : we want. Which is needed for CCmodes on some targets. */
4247 223032 : if (force)
4248 223031 : ret = simplify_gen_subreg (new_mode, x, old_mode, 0);
4249 : else
4250 1 : ret = simplify_subreg (new_mode, x, old_mode, 0);
4251 : }
4252 :
4253 276883 : return ret;
4254 : }
4255 :
4256 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X using
4257 : an integer mode of the same size as MODE. Returns the instruction
4258 : emitted, or NULL if such a move could not be generated. */
4259 :
4260 : static rtx_insn *
4261 138442 : emit_move_via_integer (machine_mode mode, rtx x, rtx y, bool force)
4262 : {
4263 138442 : scalar_int_mode imode;
4264 138442 : enum insn_code code;
4265 :
4266 : /* There must exist a mode of the exact size we require. */
4267 138442 : if (!int_mode_for_mode (mode).exists (&imode))
4268 0 : return NULL;
4269 :
4270 : /* The target must support moves in this mode. */
4271 138442 : code = optab_handler (mov_optab, imode);
4272 138442 : if (code == CODE_FOR_nothing)
4273 : return NULL;
4274 :
4275 138442 : x = emit_move_change_mode (imode, mode, x, force);
4276 138442 : if (x == NULL_RTX)
4277 : return NULL;
4278 138441 : y = emit_move_change_mode (imode, mode, y, force);
4279 138441 : if (y == NULL_RTX)
4280 : return NULL;
4281 138441 : return emit_insn (GEN_FCN (code) (x, y));
4282 : }
4283 :
4284 : /* A subroutine of emit_move_insn_1. X is a push_operand in MODE.
4285 : Return an equivalent MEM that does not use an auto-increment. */
4286 :
4287 : rtx
4288 3638 : emit_move_resolve_push (machine_mode mode, rtx x)
4289 : {
4290 3638 : enum rtx_code code = GET_CODE (XEXP (x, 0));
4291 3638 : rtx temp;
4292 :
4293 7276 : poly_int64 adjust = GET_MODE_SIZE (mode);
4294 : #ifdef PUSH_ROUNDING
4295 3638 : adjust = PUSH_ROUNDING (adjust);
4296 : #endif
4297 3638 : if (code == PRE_DEC || code == POST_DEC)
4298 3242 : adjust = -adjust;
4299 396 : else if (code == PRE_MODIFY || code == POST_MODIFY)
4300 : {
4301 396 : rtx expr = XEXP (XEXP (x, 0), 1);
4302 :
4303 396 : gcc_assert (GET_CODE (expr) == PLUS || GET_CODE (expr) == MINUS);
4304 396 : poly_int64 val = rtx_to_poly_int64 (XEXP (expr, 1));
4305 396 : if (GET_CODE (expr) == MINUS)
4306 0 : val = -val;
4307 396 : gcc_assert (known_eq (adjust, val) || known_eq (adjust, -val));
4308 : adjust = val;
4309 : }
4310 :
4311 : /* Do not use anti_adjust_stack, since we don't want to update
4312 : stack_pointer_delta. */
4313 3642 : temp = expand_simple_binop (Pmode, PLUS, stack_pointer_rtx,
4314 3638 : gen_int_mode (adjust, Pmode), stack_pointer_rtx,
4315 : 0, OPTAB_LIB_WIDEN);
4316 3638 : if (temp != stack_pointer_rtx)
4317 0 : emit_move_insn (stack_pointer_rtx, temp);
4318 :
4319 3638 : switch (code)
4320 : {
4321 3638 : case PRE_INC:
4322 3638 : case PRE_DEC:
4323 3638 : case PRE_MODIFY:
4324 3638 : temp = stack_pointer_rtx;
4325 3638 : break;
4326 : case POST_INC:
4327 : case POST_DEC:
4328 : case POST_MODIFY:
4329 0 : temp = plus_constant (Pmode, stack_pointer_rtx, -adjust);
4330 0 : break;
4331 0 : default:
4332 0 : gcc_unreachable ();
4333 : }
4334 :
4335 3638 : return replace_equiv_address (x, temp);
4336 : }
4337 :
4338 : /* A subroutine of emit_move_complex. Generate a move from Y into X.
4339 : X is known to satisfy push_operand, and MODE is known to be complex.
4340 : Returns the last instruction emitted. */
4341 :
4342 : rtx_insn *
4343 5646 : emit_move_complex_push (machine_mode mode, rtx x, rtx y)
4344 : {
4345 5646 : scalar_mode submode = GET_MODE_INNER (mode);
4346 5646 : bool imag_first;
4347 :
4348 : #ifdef PUSH_ROUNDING
4349 11292 : poly_int64 submodesize = GET_MODE_SIZE (submode);
4350 :
4351 : /* In case we output to the stack, but the size is smaller than the
4352 : machine can push exactly, we need to use move instructions. */
4353 5646 : if (maybe_ne (PUSH_ROUNDING (submodesize), submodesize))
4354 : {
4355 718 : x = emit_move_resolve_push (mode, x);
4356 718 : return emit_move_insn (x, y);
4357 : }
4358 : #endif
4359 :
4360 : /* Note that the real part always precedes the imag part in memory
4361 : regardless of machine's endianness. */
4362 4928 : switch (GET_CODE (XEXP (x, 0)))
4363 : {
4364 : case PRE_DEC:
4365 : case POST_DEC:
4366 : imag_first = true;
4367 : break;
4368 0 : case PRE_INC:
4369 0 : case POST_INC:
4370 0 : imag_first = false;
4371 0 : break;
4372 0 : default:
4373 0 : gcc_unreachable ();
4374 : }
4375 :
4376 4928 : emit_move_insn (gen_rtx_MEM (submode, XEXP (x, 0)),
4377 : read_complex_part (y, imag_first));
4378 4928 : return emit_move_insn (gen_rtx_MEM (submode, XEXP (x, 0)),
4379 4928 : read_complex_part (y, !imag_first));
4380 : }
4381 :
4382 : /* A subroutine of emit_move_complex. Perform the move from Y to X
4383 : via two moves of the parts. Returns the last instruction emitted. */
4384 :
4385 : rtx_insn *
4386 88867 : emit_move_complex_parts (rtx x, rtx y)
4387 : {
4388 : /* Show the output dies here. This is necessary for SUBREGs
4389 : of pseudos since we cannot track their lifetimes correctly;
4390 : hard regs shouldn't appear here except as return values. */
4391 88867 : if (!reload_completed && !reload_in_progress
4392 177734 : && REG_P (x) && !reg_overlap_mentioned_p (x, y))
4393 6186 : emit_clobber (x);
4394 :
4395 88867 : write_complex_part (x, read_complex_part (y, false), false, true);
4396 88867 : write_complex_part (x, read_complex_part (y, true), true, false);
4397 :
4398 88867 : return get_last_insn ();
4399 : }
4400 :
4401 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X.
4402 : MODE is known to be complex. Returns the last instruction emitted. */
4403 :
4404 : static rtx_insn *
4405 111404 : emit_move_complex (machine_mode mode, rtx x, rtx y)
4406 : {
4407 111404 : bool try_int;
4408 :
4409 : /* Need to take special care for pushes, to maintain proper ordering
4410 : of the data, and possibly extra padding. */
4411 111404 : if (push_operand (x, mode))
4412 5038 : return emit_move_complex_push (mode, x, y);
4413 :
4414 : /* See if we can coerce the target into moving both values at once, except
4415 : for floating point where we favor moving as parts if this is easy. */
4416 106366 : if (GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT
4417 63031 : && optab_handler (mov_optab, GET_MODE_INNER (mode)) != CODE_FOR_nothing
4418 63031 : && !(REG_P (x)
4419 2263 : && HARD_REGISTER_P (x)
4420 597 : && REG_NREGS (x) == 1)
4421 169397 : && !(REG_P (y)
4422 3336 : && HARD_REGISTER_P (y)
4423 1668 : && REG_NREGS (y) == 1))
4424 : try_int = false;
4425 : /* Not possible if the values are inherently not adjacent. */
4426 43335 : else if (GET_CODE (x) == CONCAT || GET_CODE (y) == CONCAT)
4427 : try_int = false;
4428 : /* Is possible if both are registers (or subregs of registers). */
4429 18312 : else if (register_operand (x, mode) && register_operand (y, mode))
4430 : try_int = true;
4431 : /* If one of the operands is a memory, and alignment constraints
4432 : are friendly enough, we may be able to do combined memory operations.
4433 : We do not attempt this if Y is a constant because that combination is
4434 : usually better with the by-parts thing below. */
4435 635 : else if ((MEM_P (x) ? !CONSTANT_P (y) : MEM_P (y))
4436 : && (!STRICT_ALIGNMENT
4437 : || get_mode_alignment (mode) == BIGGEST_ALIGNMENT))
4438 : try_int = true;
4439 : else
4440 : try_int = false;
4441 :
4442 : if (try_int)
4443 : {
4444 18312 : rtx_insn *ret;
4445 :
4446 : /* For memory to memory moves, optimal behavior can be had with the
4447 : existing block move logic. But use normal expansion if optimizing
4448 : for size. */
4449 18312 : if (MEM_P (x) && MEM_P (y))
4450 : {
4451 726 : emit_block_move (x, y, gen_int_mode (GET_MODE_SIZE (mode), Pmode),
4452 360 : (optimize_insn_for_speed_p()
4453 : ? BLOCK_OP_NO_LIBCALL : BLOCK_OP_NORMAL));
4454 360 : return get_last_insn ();
4455 : }
4456 :
4457 17952 : ret = emit_move_via_integer (mode, x, y, true);
4458 17952 : if (ret)
4459 : return ret;
4460 : }
4461 :
4462 88054 : return emit_move_complex_parts (x, y);
4463 : }
4464 :
4465 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X.
4466 : MODE is known to be MODE_CC. Returns the last instruction emitted. */
4467 :
4468 : static rtx_insn *
4469 0 : emit_move_ccmode (machine_mode mode, rtx x, rtx y)
4470 : {
4471 0 : rtx_insn *ret;
4472 :
4473 : /* Assume all MODE_CC modes are equivalent; if we have movcc, use it. */
4474 0 : if (mode != CCmode)
4475 : {
4476 0 : enum insn_code code = optab_handler (mov_optab, CCmode);
4477 0 : if (code != CODE_FOR_nothing)
4478 : {
4479 0 : x = emit_move_change_mode (CCmode, mode, x, true);
4480 0 : y = emit_move_change_mode (CCmode, mode, y, true);
4481 0 : return emit_insn (GEN_FCN (code) (x, y));
4482 : }
4483 : }
4484 :
4485 : /* Otherwise, find the MODE_INT mode of the same width. */
4486 0 : ret = emit_move_via_integer (mode, x, y, false);
4487 0 : gcc_assert (ret != NULL);
4488 : return ret;
4489 : }
4490 :
4491 : /* Return true if word I of OP lies entirely in the
4492 : undefined bits of a paradoxical subreg. */
4493 :
4494 : static bool
4495 2 : undefined_operand_subword_p (const_rtx op, int i)
4496 : {
4497 2 : if (GET_CODE (op) != SUBREG)
4498 : return false;
4499 0 : machine_mode innermostmode = GET_MODE (SUBREG_REG (op));
4500 0 : poly_int64 offset = i * UNITS_PER_WORD + subreg_memory_offset (op);
4501 0 : return (known_ge (offset, GET_MODE_SIZE (innermostmode))
4502 0 : || known_le (offset, -UNITS_PER_WORD));
4503 : }
4504 :
4505 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X.
4506 : MODE is any multi-word or full-word mode that lacks a move_insn
4507 : pattern. Note that you will get better code if you define such
4508 : patterns, even if they must turn into multiple assembler instructions. */
4509 :
4510 : static rtx_insn *
4511 1 : emit_move_multi_word (machine_mode mode, rtx x, rtx y)
4512 : {
4513 1 : rtx_insn *last_insn = 0;
4514 1 : rtx_insn *seq;
4515 1 : rtx inner;
4516 1 : bool need_clobber;
4517 1 : int i, mode_size;
4518 :
4519 : /* This function can only handle cases where the number of words is
4520 : known at compile time. */
4521 1 : mode_size = GET_MODE_SIZE (mode).to_constant ();
4522 1 : gcc_assert (mode_size >= UNITS_PER_WORD);
4523 :
4524 : /* If X is a push on the stack, do the push now and replace
4525 : X with a reference to the stack pointer. */
4526 1 : if (push_operand (x, mode))
4527 0 : x = emit_move_resolve_push (mode, x);
4528 :
4529 : /* If we are in reload, see if either operand is a MEM whose address
4530 : is scheduled for replacement. */
4531 0 : if (reload_in_progress && MEM_P (x)
4532 1 : && (inner = find_replacement (&XEXP (x, 0))) != XEXP (x, 0))
4533 0 : x = replace_equiv_address_nv (x, inner);
4534 0 : if (reload_in_progress && MEM_P (y)
4535 1 : && (inner = find_replacement (&XEXP (y, 0))) != XEXP (y, 0))
4536 0 : y = replace_equiv_address_nv (y, inner);
4537 :
4538 1 : start_sequence ();
4539 :
4540 1 : need_clobber = false;
4541 3 : for (i = 0; i < CEIL (mode_size, UNITS_PER_WORD); i++)
4542 : {
4543 : /* Do not generate code for a move if it would go entirely
4544 : to the non-existing bits of a paradoxical subreg. */
4545 1 : if (undefined_operand_subword_p (x, i))
4546 0 : continue;
4547 :
4548 1 : rtx xpart = operand_subword (x, i, 1, mode);
4549 1 : rtx ypart;
4550 :
4551 : /* Do not generate code for a move if it would come entirely
4552 : from the undefined bits of a paradoxical subreg. */
4553 1 : if (undefined_operand_subword_p (y, i))
4554 0 : continue;
4555 :
4556 1 : ypart = operand_subword (y, i, 1, mode);
4557 :
4558 : /* If we can't get a part of Y, put Y into memory if it is a
4559 : constant. Otherwise, force it into a register. Then we must
4560 : be able to get a part of Y. */
4561 1 : if (ypart == 0 && CONSTANT_P (y))
4562 : {
4563 0 : y = use_anchored_address (force_const_mem (mode, y));
4564 0 : ypart = operand_subword (y, i, 1, mode);
4565 : }
4566 0 : else if (ypart == 0)
4567 0 : ypart = operand_subword_force (y, i, mode);
4568 :
4569 1 : gcc_assert (xpart && ypart);
4570 :
4571 1 : need_clobber |= (GET_CODE (xpart) == SUBREG);
4572 :
4573 1 : last_insn = emit_move_insn (xpart, ypart);
4574 : }
4575 :
4576 1 : seq = end_sequence ();
4577 :
4578 : /* Show the output dies here. This is necessary for SUBREGs
4579 : of pseudos since we cannot track their lifetimes correctly;
4580 : hard regs shouldn't appear here except as return values.
4581 : We never want to emit such a clobber after reload. */
4582 1 : if (x != y
4583 1 : && ! (reload_in_progress || reload_completed)
4584 1 : && need_clobber != 0)
4585 1 : emit_clobber (x);
4586 :
4587 1 : emit_insn (seq);
4588 :
4589 1 : return last_insn;
4590 : }
4591 :
4592 : /* Low level part of emit_move_insn.
4593 : Called just like emit_move_insn, but assumes X and Y
4594 : are basically valid. */
4595 :
4596 : rtx_insn *
4597 78178448 : emit_move_insn_1 (rtx x, rtx y)
4598 : {
4599 78178448 : machine_mode mode = GET_MODE (x);
4600 78178448 : enum insn_code code;
4601 :
4602 78178448 : gcc_assert ((unsigned int) mode < (unsigned int) MAX_MACHINE_MODE);
4603 :
4604 78178448 : code = optab_handler (mov_optab, mode);
4605 78178448 : if (code != CODE_FOR_nothing)
4606 77946554 : return emit_insn (GEN_FCN (code) (x, y));
4607 :
4608 : /* Expand complex moves by moving real part and imag part. */
4609 231894 : if (COMPLEX_MODE_P (mode))
4610 111404 : return emit_move_complex (mode, x, y);
4611 :
4612 : if (GET_MODE_CLASS (mode) == MODE_DECIMAL_FLOAT
4613 : || ALL_FIXED_POINT_MODE_P (mode))
4614 : {
4615 120489 : rtx_insn *result = emit_move_via_integer (mode, x, y, true);
4616 :
4617 : /* If we can't find an integer mode, use multi words. */
4618 120489 : if (result)
4619 : return result;
4620 : else
4621 0 : return emit_move_multi_word (mode, x, y);
4622 : }
4623 :
4624 : if (GET_MODE_CLASS (mode) == MODE_CC)
4625 0 : return emit_move_ccmode (mode, x, y);
4626 :
4627 : /* Try using a move pattern for the corresponding integer mode. This is
4628 : only safe when simplify_subreg can convert MODE constants into integer
4629 : constants. At present, it can only do this reliably if the value
4630 : fits within a HOST_WIDE_INT. */
4631 1 : if (!CONSTANT_P (y)
4632 2 : || known_le (GET_MODE_BITSIZE (mode), HOST_BITS_PER_WIDE_INT))
4633 : {
4634 1 : rtx_insn *ret = emit_move_via_integer (mode, x, y, lra_in_progress);
4635 :
4636 1 : if (ret)
4637 : {
4638 0 : if (! lra_in_progress || recog (PATTERN (ret), ret, 0) >= 0)
4639 : return ret;
4640 : }
4641 : }
4642 :
4643 1 : return emit_move_multi_word (mode, x, y);
4644 : }
4645 :
4646 : /* Generate code to copy Y into X.
4647 : Both Y and X must have the same mode, except that
4648 : Y can be a constant with VOIDmode.
4649 : This mode cannot be BLKmode; use emit_block_move for that.
4650 :
4651 : Return the last instruction emitted. */
4652 :
4653 : rtx_insn *
4654 72132855 : emit_move_insn (rtx x, rtx y)
4655 : {
4656 72132855 : machine_mode mode = GET_MODE (x);
4657 72132855 : rtx y_cst = NULL_RTX;
4658 72132855 : rtx_insn *last_insn;
4659 72132855 : rtx set;
4660 :
4661 72132855 : gcc_assert (mode != BLKmode
4662 : && (GET_MODE (y) == mode || GET_MODE (y) == VOIDmode));
4663 :
4664 : /* If we have a copy that looks like one of the following patterns:
4665 : (set (subreg:M1 (reg:M2 ...)) (subreg:M1 (reg:M2 ...)))
4666 : (set (subreg:M1 (reg:M2 ...)) (mem:M1 ADDR))
4667 : (set (mem:M1 ADDR) (subreg:M1 (reg:M2 ...)))
4668 : (set (subreg:M1 (reg:M2 ...)) (constant C))
4669 : where mode M1 is equal in size to M2, try to detect whether the
4670 : mode change involves an implicit round trip through memory.
4671 : If so, see if we can avoid that by removing the subregs and
4672 : doing the move in mode M2 instead. */
4673 :
4674 72132855 : rtx x_inner = NULL_RTX;
4675 72132855 : rtx y_inner = NULL_RTX;
4676 :
4677 74944120 : auto candidate_subreg_p = [&](rtx subreg) {
4678 2811265 : return (REG_P (SUBREG_REG (subreg))
4679 8431971 : && known_eq (GET_MODE_SIZE (GET_MODE (SUBREG_REG (subreg))),
4680 : GET_MODE_SIZE (GET_MODE (subreg)))
4681 3191718 : && optab_handler (mov_optab, GET_MODE (SUBREG_REG (subreg)))
4682 2811265 : != CODE_FOR_nothing);
4683 : };
4684 :
4685 72139083 : auto candidate_mem_p = [&](machine_mode innermode, rtx mem) {
4686 6228 : return (!targetm.can_change_mode_class (innermode, GET_MODE (mem), ALL_REGS)
4687 6228 : && !push_operand (mem, GET_MODE (mem))
4688 : /* Not a candidate if innermode requires too much alignment. */
4689 12396 : && (MEM_ALIGN (mem) >= GET_MODE_ALIGNMENT (innermode)
4690 232 : || targetm.slow_unaligned_access (GET_MODE (mem),
4691 116 : MEM_ALIGN (mem))
4692 232 : || !targetm.slow_unaligned_access (innermode,
4693 116 : MEM_ALIGN (mem))));
4694 : };
4695 :
4696 72132855 : if (SUBREG_P (x) && candidate_subreg_p (x))
4697 16736 : x_inner = SUBREG_REG (x);
4698 :
4699 72132855 : if (SUBREG_P (y) && candidate_subreg_p (y))
4700 327234 : y_inner = SUBREG_REG (y);
4701 :
4702 72132855 : if (x_inner != NULL_RTX
4703 72132855 : && y_inner != NULL_RTX
4704 3090 : && GET_MODE (x_inner) == GET_MODE (y_inner)
4705 72135515 : && !targetm.can_change_mode_class (GET_MODE (x_inner), mode, ALL_REGS))
4706 : {
4707 2660 : x = x_inner;
4708 2660 : y = y_inner;
4709 2660 : mode = GET_MODE (x_inner);
4710 : }
4711 72130195 : else if (x_inner != NULL_RTX
4712 14076 : && MEM_P (y)
4713 72130519 : && candidate_mem_p (GET_MODE (x_inner), y))
4714 : {
4715 324 : x = x_inner;
4716 324 : y = adjust_address (y, GET_MODE (x_inner), 0);
4717 324 : mode = GET_MODE (x_inner);
4718 : }
4719 72129871 : else if (y_inner != NULL_RTX
4720 324574 : && MEM_P (x)
4721 72135775 : && candidate_mem_p (GET_MODE (y_inner), x))
4722 : {
4723 5844 : x = adjust_address (x, GET_MODE (y_inner), 0);
4724 5844 : y = y_inner;
4725 5844 : mode = GET_MODE (y_inner);
4726 : }
4727 72124027 : else if (x_inner != NULL_RTX
4728 13752 : && CONSTANT_P (y)
4729 476 : && !targetm.can_change_mode_class (GET_MODE (x_inner),
4730 : mode, ALL_REGS)
4731 72124503 : && (y_inner = simplify_subreg (GET_MODE (x_inner), y, mode, 0)))
4732 : {
4733 476 : x = x_inner;
4734 476 : y = y_inner;
4735 476 : mode = GET_MODE (x_inner);
4736 : }
4737 :
4738 72132855 : if (CONSTANT_P (y))
4739 : {
4740 16898705 : if (optimize
4741 13167544 : && SCALAR_FLOAT_MODE_P (GET_MODE (x))
4742 17677307 : && (last_insn = compress_float_constant (x, y)))
4743 : return last_insn;
4744 :
4745 16829270 : y_cst = y;
4746 :
4747 16829270 : if (!targetm.legitimate_constant_p (mode, y))
4748 : {
4749 520133 : y = force_const_mem (mode, y);
4750 :
4751 : /* If the target's cannot_force_const_mem prevented the spill,
4752 : assume that the target's move expanders will also take care
4753 : of the non-legitimate constant. */
4754 520133 : if (!y)
4755 : y = y_cst;
4756 : else
4757 500128 : y = use_anchored_address (y);
4758 : }
4759 : }
4760 :
4761 : /* If X or Y are memory references, verify that their addresses are valid
4762 : for the machine. */
4763 72063420 : if (MEM_P (x)
4764 88396352 : && (! memory_address_addr_space_p (GET_MODE (x), XEXP (x, 0),
4765 14070048 : MEM_ADDR_SPACE (x))
4766 2263058 : && ! push_operand (x, GET_MODE (x))))
4767 174 : x = validize_mem (x);
4768 :
4769 72063420 : if (MEM_P (y)
4770 89936063 : && ! memory_address_addr_space_p (GET_MODE (y), XEXP (y, 0),
4771 17872643 : MEM_ADDR_SPACE (y)))
4772 8764 : y = validize_mem (y);
4773 :
4774 72063420 : gcc_assert (mode != BLKmode);
4775 :
4776 72063420 : last_insn = emit_move_insn_1 (x, y);
4777 :
4778 16829270 : if (y_cst && REG_P (x)
4779 12353626 : && (set = single_set (last_insn)) != NULL_RTX
4780 12353626 : && SET_DEST (set) == x
4781 84402564 : && ! rtx_equal_p (y_cst, SET_SRC (set)))
4782 1281488 : set_unique_reg_note (last_insn, REG_EQUAL, copy_rtx (y_cst));
4783 :
4784 : return last_insn;
4785 : }
4786 :
4787 : /* Generate the body of an instruction to copy Y into X.
4788 : It may be a list of insns, if one insn isn't enough. */
4789 :
4790 : rtx_insn *
4791 6114948 : gen_move_insn (rtx x, rtx y)
4792 : {
4793 6114948 : start_sequence ();
4794 6114948 : emit_move_insn_1 (x, y);
4795 6114948 : return end_sequence ();
4796 : }
4797 :
4798 : /* If Y is representable exactly in a narrower mode, and the target can
4799 : perform the extension directly from constant or memory, then emit the
4800 : move as an extension. */
4801 :
4802 : static rtx_insn *
4803 778602 : compress_float_constant (rtx x, rtx y)
4804 : {
4805 778602 : machine_mode dstmode = GET_MODE (x);
4806 778602 : machine_mode orig_srcmode = GET_MODE (y);
4807 778602 : machine_mode srcmode;
4808 778602 : const REAL_VALUE_TYPE *r;
4809 778602 : int oldcost, newcost;
4810 778602 : bool speed = optimize_insn_for_speed_p ();
4811 :
4812 778602 : r = CONST_DOUBLE_REAL_VALUE (y);
4813 :
4814 778602 : if (targetm.legitimate_constant_p (dstmode, y))
4815 776441 : oldcost = set_src_cost (y, orig_srcmode, speed);
4816 : else
4817 2161 : oldcost = set_src_cost (force_const_mem (dstmode, y), dstmode, speed);
4818 :
4819 2758125 : FOR_EACH_MODE_UNTIL (srcmode, orig_srcmode)
4820 : {
4821 2048958 : enum insn_code ic;
4822 2048958 : rtx trunc_y;
4823 2048958 : rtx_insn *last_insn;
4824 :
4825 : /* Skip if the target can't extend this way. */
4826 2048958 : ic = can_extend_p (dstmode, srcmode, 0);
4827 2048958 : if (ic == CODE_FOR_nothing)
4828 1620334 : continue;
4829 :
4830 : /* Skip if the narrowed value isn't exact. */
4831 428624 : if (! exact_real_truncate (srcmode, r))
4832 54050 : continue;
4833 :
4834 374574 : trunc_y = const_double_from_real_value (*r, srcmode);
4835 :
4836 374574 : if (targetm.legitimate_constant_p (srcmode, trunc_y))
4837 : {
4838 : /* Skip if the target needs extra instructions to perform
4839 : the extension. */
4840 370632 : if (!insn_operand_matches (ic, 1, trunc_y))
4841 4930 : continue;
4842 : /* This is valid, but may not be cheaper than the original. */
4843 365702 : newcost = set_src_cost (gen_rtx_FLOAT_EXTEND (dstmode, trunc_y),
4844 : dstmode, speed);
4845 365702 : if (oldcost < newcost)
4846 296267 : continue;
4847 : }
4848 3942 : else if (float_extend_from_mem[dstmode][srcmode])
4849 : {
4850 0 : trunc_y = force_const_mem (srcmode, trunc_y);
4851 : /* This is valid, but may not be cheaper than the original. */
4852 0 : newcost = set_src_cost (gen_rtx_FLOAT_EXTEND (dstmode, trunc_y),
4853 : dstmode, speed);
4854 0 : if (oldcost < newcost)
4855 0 : continue;
4856 0 : trunc_y = validize_mem (trunc_y);
4857 : }
4858 : else
4859 3942 : continue;
4860 :
4861 : /* For CSE's benefit, force the compressed constant pool entry
4862 : into a new pseudo. This constant may be used in different modes,
4863 : and if not, combine will put things back together for us. */
4864 69435 : trunc_y = force_reg (srcmode, trunc_y);
4865 :
4866 : /* If x is a hard register, perform the extension into a pseudo,
4867 : so that e.g. stack realignment code is aware of it. */
4868 69435 : rtx target = x;
4869 69435 : if (REG_P (x) && HARD_REGISTER_P (x))
4870 1 : target = gen_reg_rtx (dstmode);
4871 :
4872 69435 : emit_unop_insn (ic, target, trunc_y, UNKNOWN);
4873 69435 : last_insn = get_last_insn ();
4874 :
4875 69435 : if (REG_P (target))
4876 58703 : set_unique_reg_note (last_insn, REG_EQUAL, y);
4877 :
4878 69435 : if (target != x)
4879 1 : return emit_move_insn (x, target);
4880 : return last_insn;
4881 : }
4882 :
4883 : return NULL;
4884 : }
4885 :
4886 : /* Pushing data onto the stack. */
4887 :
4888 : /* Push a block of length SIZE (perhaps variable)
4889 : and return an rtx to address the beginning of the block.
4890 : The value may be virtual_outgoing_args_rtx.
4891 :
4892 : EXTRA is the number of bytes of padding to push in addition to SIZE.
4893 : BELOW nonzero means this padding comes at low addresses;
4894 : otherwise, the padding comes at high addresses. */
4895 :
4896 : rtx
4897 269068 : push_block (rtx size, poly_int64 extra, int below)
4898 : {
4899 269068 : rtx temp;
4900 :
4901 334061 : size = convert_modes (Pmode, ptr_mode, size, 1);
4902 269068 : if (CONSTANT_P (size))
4903 334061 : anti_adjust_stack (plus_constant (Pmode, size, extra));
4904 0 : else if (REG_P (size) && known_eq (extra, 0))
4905 0 : anti_adjust_stack (size);
4906 : else
4907 : {
4908 0 : temp = copy_to_mode_reg (Pmode, size);
4909 0 : if (maybe_ne (extra, 0))
4910 0 : temp = expand_binop (Pmode, add_optab, temp,
4911 0 : gen_int_mode (extra, Pmode),
4912 : temp, 0, OPTAB_LIB_WIDEN);
4913 0 : anti_adjust_stack (temp);
4914 : }
4915 :
4916 269068 : if (STACK_GROWS_DOWNWARD)
4917 : {
4918 269068 : temp = virtual_outgoing_args_rtx;
4919 269068 : if (maybe_ne (extra, 0) && below)
4920 0 : temp = plus_constant (Pmode, temp, extra);
4921 : }
4922 : else
4923 : {
4924 : poly_int64 csize;
4925 : if (poly_int_rtx_p (size, &csize))
4926 : temp = plus_constant (Pmode, virtual_outgoing_args_rtx,
4927 : -csize - (below ? 0 : extra));
4928 : else if (maybe_ne (extra, 0) && !below)
4929 : temp = gen_rtx_PLUS (Pmode, virtual_outgoing_args_rtx,
4930 : negate_rtx (Pmode, plus_constant (Pmode, size,
4931 : extra)));
4932 : else
4933 : temp = gen_rtx_PLUS (Pmode, virtual_outgoing_args_rtx,
4934 : negate_rtx (Pmode, size));
4935 : }
4936 :
4937 269068 : return memory_address (NARROWEST_INT_MODE, temp);
4938 : }
4939 :
4940 : /* A utility routine that returns the base of an auto-inc memory, or NULL. */
4941 :
4942 : static rtx
4943 2575304 : mem_autoinc_base (rtx mem)
4944 : {
4945 0 : if (MEM_P (mem))
4946 : {
4947 1572068 : rtx addr = XEXP (mem, 0);
4948 1572068 : if (GET_RTX_CLASS (GET_CODE (addr)) == RTX_AUTOINC)
4949 1031602 : return XEXP (addr, 0);
4950 : }
4951 : return NULL;
4952 : }
4953 :
4954 : /* A utility routine used here, in reload, and in try_split. The insns
4955 : after PREV up to and including LAST are known to adjust the stack,
4956 : with a final value of END_ARGS_SIZE. Iterate backward from LAST
4957 : placing notes as appropriate. PREV may be NULL, indicating the
4958 : entire insn sequence prior to LAST should be scanned.
4959 :
4960 : The set of allowed stack pointer modifications is small:
4961 : (1) One or more auto-inc style memory references (aka pushes),
4962 : (2) One or more addition/subtraction with the SP as destination,
4963 : (3) A single move insn with the SP as destination,
4964 : (4) A call_pop insn,
4965 : (5) Noreturn call insns if !ACCUMULATE_OUTGOING_ARGS.
4966 :
4967 : Insns in the sequence that do not modify the SP are ignored,
4968 : except for noreturn calls.
4969 :
4970 : The return value is the amount of adjustment that can be trivially
4971 : verified, via immediate operand or auto-inc. If the adjustment
4972 : cannot be trivially extracted, the return value is HOST_WIDE_INT_MIN. */
4973 :
4974 : poly_int64
4975 4625705 : find_args_size_adjust (rtx_insn *insn)
4976 : {
4977 4625705 : rtx dest, set, pat;
4978 4625705 : int i;
4979 :
4980 4625705 : pat = PATTERN (insn);
4981 4625705 : set = NULL;
4982 :
4983 : /* Look for a call_pop pattern. */
4984 4625705 : if (CALL_P (insn))
4985 : {
4986 : /* We have to allow non-call_pop patterns for the case
4987 : of emit_single_push_insn of a TLS address. */
4988 2981385 : if (GET_CODE (pat) != PARALLEL)
4989 2966564 : return 0;
4990 :
4991 : /* All call_pop have a stack pointer adjust in the parallel.
4992 : The call itself is always first, and the stack adjust is
4993 : usually last, so search from the end. */
4994 14821 : for (i = XVECLEN (pat, 0) - 1; i > 0; --i)
4995 : {
4996 14821 : set = XVECEXP (pat, 0, i);
4997 14821 : if (GET_CODE (set) != SET)
4998 0 : continue;
4999 14821 : dest = SET_DEST (set);
5000 14821 : if (dest == stack_pointer_rtx)
5001 : break;
5002 : }
5003 : /* We'd better have found the stack pointer adjust. */
5004 14821 : if (i == 0)
5005 0 : return 0;
5006 : /* Fall through to process the extracted SET and DEST
5007 : as if it was a standalone insn. */
5008 : }
5009 1644320 : else if (GET_CODE (pat) == SET)
5010 : set = pat;
5011 257403 : else if ((set = single_set (insn)) != NULL)
5012 : ;
5013 47798 : else if (GET_CODE (pat) == PARALLEL)
5014 : {
5015 : /* ??? Some older ports use a parallel with a stack adjust
5016 : and a store for a PUSH_ROUNDING pattern, rather than a
5017 : PRE/POST_MODIFY rtx. Don't force them to update yet... */
5018 : /* ??? See h8300 and m68k, pushqi1. */
5019 0 : for (i = XVECLEN (pat, 0) - 1; i >= 0; --i)
5020 : {
5021 0 : set = XVECEXP (pat, 0, i);
5022 0 : if (GET_CODE (set) != SET)
5023 0 : continue;
5024 0 : dest = SET_DEST (set);
5025 0 : if (dest == stack_pointer_rtx)
5026 : break;
5027 :
5028 : /* We do not expect an auto-inc of the sp in the parallel. */
5029 0 : gcc_checking_assert (mem_autoinc_base (dest) != stack_pointer_rtx);
5030 0 : gcc_checking_assert (mem_autoinc_base (SET_SRC (set))
5031 : != stack_pointer_rtx);
5032 : }
5033 0 : if (i < 0)
5034 0 : return 0;
5035 : }
5036 : else
5037 47798 : return 0;
5038 :
5039 1611343 : dest = SET_DEST (set);
5040 :
5041 : /* Look for direct modifications of the stack pointer. */
5042 1611343 : if (REG_P (dest) && REGNO (dest) == STACK_POINTER_REGNUM)
5043 : {
5044 : /* Look for a trivial adjustment, otherwise assume nothing. */
5045 : /* Note that the SPU restore_stack_block pattern refers to
5046 : the stack pointer in V4SImode. Consider that non-trivial. */
5047 323691 : poly_int64 offset;
5048 323691 : if (SCALAR_INT_MODE_P (GET_MODE (dest))
5049 323691 : && strip_offset (SET_SRC (set), &offset) == stack_pointer_rtx)
5050 321220 : return offset;
5051 : /* ??? Reload can generate no-op moves, which will be cleaned
5052 : up later. Recognize it and continue searching. */
5053 2471 : else if (rtx_equal_p (dest, SET_SRC (set)))
5054 0 : return 0;
5055 : else
5056 2471 : return HOST_WIDE_INT_MIN;
5057 : }
5058 : else
5059 : {
5060 1287652 : rtx mem, addr;
5061 :
5062 : /* Otherwise only think about autoinc patterns. */
5063 2327326 : if (mem_autoinc_base (dest) == stack_pointer_rtx)
5064 : {
5065 921439 : mem = dest;
5066 1274984 : gcc_checking_assert (mem_autoinc_base (SET_SRC (set))
5067 : != stack_pointer_rtx);
5068 : }
5069 545062 : else if (mem_autoinc_base (SET_SRC (set)) == stack_pointer_rtx)
5070 : mem = SET_SRC (set);
5071 : else
5072 256050 : return 0;
5073 :
5074 1031602 : addr = XEXP (mem, 0);
5075 1031602 : switch (GET_CODE (addr))
5076 : {
5077 110163 : case PRE_INC:
5078 110163 : case POST_INC:
5079 220326 : return GET_MODE_SIZE (GET_MODE (mem));
5080 903324 : case PRE_DEC:
5081 903324 : case POST_DEC:
5082 1806648 : return -GET_MODE_SIZE (GET_MODE (mem));
5083 18115 : case PRE_MODIFY:
5084 18115 : case POST_MODIFY:
5085 18115 : addr = XEXP (addr, 1);
5086 18115 : gcc_assert (GET_CODE (addr) == PLUS);
5087 18115 : gcc_assert (XEXP (addr, 0) == stack_pointer_rtx);
5088 18115 : return rtx_to_poly_int64 (XEXP (addr, 1));
5089 0 : default:
5090 0 : gcc_unreachable ();
5091 : }
5092 : }
5093 : }
5094 :
5095 : poly_int64
5096 665384 : fixup_args_size_notes (rtx_insn *prev, rtx_insn *last,
5097 : poly_int64 end_args_size)
5098 : {
5099 665384 : poly_int64 args_size = end_args_size;
5100 665384 : bool saw_unknown = false;
5101 665384 : rtx_insn *insn;
5102 :
5103 1898531 : for (insn = last; insn != prev; insn = PREV_INSN (insn))
5104 : {
5105 1233147 : if (!NONDEBUG_INSN_P (insn))
5106 0 : continue;
5107 :
5108 : /* We might have existing REG_ARGS_SIZE notes, e.g. when pushing
5109 : a call argument containing a TLS address that itself requires
5110 : a call to __tls_get_addr. The handling of stack_pointer_delta
5111 : in emit_single_push_insn is supposed to ensure that any such
5112 : notes are already correct. */
5113 1233147 : rtx note = find_reg_note (insn, REG_ARGS_SIZE, NULL_RTX);
5114 1233147 : gcc_assert (!note || known_eq (args_size, get_args_size (note)));
5115 :
5116 1233147 : poly_int64 this_delta = find_args_size_adjust (insn);
5117 1233147 : if (known_eq (this_delta, 0))
5118 : {
5119 603441 : if (!CALL_P (insn)
5120 9 : || ACCUMULATE_OUTGOING_ARGS
5121 301734 : || find_reg_note (insn, REG_NORETURN, NULL_RTX) == NULL_RTX)
5122 301716 : continue;
5123 : }
5124 :
5125 931431 : gcc_assert (!saw_unknown);
5126 931431 : if (known_eq (this_delta, HOST_WIDE_INT_MIN))
5127 2444 : saw_unknown = true;
5128 :
5129 931431 : if (!note)
5130 931431 : add_args_size_note (insn, args_size);
5131 931431 : if (STACK_GROWS_DOWNWARD)
5132 1862862 : this_delta = -poly_uint64 (this_delta);
5133 :
5134 931431 : if (saw_unknown)
5135 : args_size = HOST_WIDE_INT_MIN;
5136 : else
5137 1233147 : args_size -= this_delta;
5138 : }
5139 :
5140 665384 : return args_size;
5141 : }
5142 :
5143 : #ifdef PUSH_ROUNDING
5144 : /* Emit single push insn. */
5145 :
5146 : static void
5147 1848454 : emit_single_push_insn_1 (machine_mode mode, rtx x, tree type)
5148 : {
5149 1848454 : rtx dest_addr;
5150 3696908 : poly_int64 rounded_size = PUSH_ROUNDING (GET_MODE_SIZE (mode));
5151 1848454 : rtx dest;
5152 1848454 : enum insn_code icode;
5153 :
5154 : /* If there is push pattern, use it. Otherwise try old way of throwing
5155 : MEM representing push operation to move expander. */
5156 1848454 : icode = optab_handler (push_optab, mode);
5157 1848454 : if (icode != CODE_FOR_nothing)
5158 : {
5159 0 : class expand_operand ops[1];
5160 :
5161 0 : create_input_operand (&ops[0], x, mode);
5162 0 : if (maybe_expand_insn (icode, 1, ops))
5163 0 : return;
5164 : }
5165 3696908 : if (known_eq (GET_MODE_SIZE (mode), rounded_size))
5166 3190824 : dest_addr = gen_rtx_fmt_e (STACK_PUSH_CODE, Pmode, stack_pointer_rtx);
5167 : /* If we are to pad downward, adjust the stack pointer first and
5168 : then store X into the stack location using an offset. This is
5169 : because emit_move_insn does not know how to pad; it does not have
5170 : access to type. */
5171 105000 : else if (targetm.calls.function_arg_padding (mode, type) == PAD_DOWNWARD)
5172 : {
5173 0 : emit_move_insn (stack_pointer_rtx,
5174 0 : expand_binop (Pmode,
5175 : STACK_GROWS_DOWNWARD ? sub_optab
5176 : : add_optab,
5177 : stack_pointer_rtx,
5178 0 : gen_int_mode (rounded_size, Pmode),
5179 : NULL_RTX, 0, OPTAB_LIB_WIDEN));
5180 :
5181 0 : poly_int64 offset = rounded_size - GET_MODE_SIZE (mode);
5182 0 : if (STACK_GROWS_DOWNWARD && STACK_PUSH_CODE == POST_DEC)
5183 : /* We have already decremented the stack pointer, so get the
5184 : previous value. */
5185 : offset += rounded_size;
5186 :
5187 0 : if (!STACK_GROWS_DOWNWARD && STACK_PUSH_CODE == POST_INC)
5188 : /* We have already incremented the stack pointer, so get the
5189 : previous value. */
5190 : offset -= rounded_size;
5191 :
5192 0 : dest_addr = plus_constant (Pmode, stack_pointer_rtx, offset);
5193 : }
5194 : else
5195 : {
5196 : if (STACK_GROWS_DOWNWARD)
5197 : /* ??? This seems wrong if STACK_PUSH_CODE == POST_DEC. */
5198 105574 : dest_addr = plus_constant (Pmode, stack_pointer_rtx, -rounded_size);
5199 : else
5200 : /* ??? This seems wrong if STACK_PUSH_CODE == POST_INC. */
5201 : dest_addr = plus_constant (Pmode, stack_pointer_rtx, rounded_size);
5202 :
5203 105574 : dest_addr = gen_rtx_PRE_MODIFY (Pmode, stack_pointer_rtx, dest_addr);
5204 : }
5205 :
5206 1848454 : dest = gen_rtx_MEM (mode, dest_addr);
5207 :
5208 1848454 : if (type != 0)
5209 : {
5210 1848367 : set_mem_attributes (dest, type, 1);
5211 :
5212 1848367 : if (cfun->tail_call_marked)
5213 : /* Function incoming arguments may overlap with sibling call
5214 : outgoing arguments and we cannot allow reordering of reads
5215 : from function arguments with stores to outgoing arguments
5216 : of sibling calls. */
5217 143343 : set_mem_alias_set (dest, 0);
5218 : }
5219 1848454 : emit_move_insn (dest, x);
5220 : }
5221 :
5222 : /* Emit and annotate a single push insn. */
5223 :
5224 : static void
5225 1848454 : emit_single_push_insn (machine_mode mode, rtx x, tree type)
5226 : {
5227 1848454 : poly_int64 delta, old_delta = stack_pointer_delta;
5228 1848454 : rtx_insn *prev = get_last_insn ();
5229 1848454 : rtx_insn *last;
5230 :
5231 1848454 : emit_single_push_insn_1 (mode, x, type);
5232 :
5233 : /* Adjust stack_pointer_delta to describe the situation after the push
5234 : we just performed. Note that we must do this after the push rather
5235 : than before the push in case calculating X needs pushes and pops of
5236 : its own (e.g. if calling __tls_get_addr). The REG_ARGS_SIZE notes
5237 : for such pushes and pops must not include the effect of the future
5238 : push of X. */
5239 3696908 : stack_pointer_delta += PUSH_ROUNDING (GET_MODE_SIZE (mode));
5240 :
5241 1848454 : last = get_last_insn ();
5242 :
5243 : /* Notice the common case where we emitted exactly one insn. */
5244 1848454 : if (PREV_INSN (last) == prev)
5245 : {
5246 1737319 : add_args_size_note (last, stack_pointer_delta);
5247 1737319 : return;
5248 : }
5249 :
5250 111135 : delta = fixup_args_size_notes (prev, last, stack_pointer_delta);
5251 111135 : gcc_assert (known_eq (delta, HOST_WIDE_INT_MIN)
5252 : || known_eq (delta, old_delta));
5253 : }
5254 : #endif
5255 :
5256 : /* If reading SIZE bytes from X will end up reading from
5257 : Y return the number of bytes that overlap. Return -1
5258 : if there is no overlap or -2 if we can't determine
5259 : (for example when X and Y have different base registers). */
5260 :
5261 : static int
5262 0 : memory_load_overlap (rtx x, rtx y, HOST_WIDE_INT size)
5263 : {
5264 0 : rtx tmp = plus_constant (Pmode, x, size);
5265 0 : rtx sub = simplify_gen_binary (MINUS, Pmode, tmp, y);
5266 :
5267 0 : if (!CONST_INT_P (sub))
5268 : return -2;
5269 :
5270 0 : HOST_WIDE_INT val = INTVAL (sub);
5271 :
5272 0 : return IN_RANGE (val, 1, size) ? val : -1;
5273 : }
5274 :
5275 : /* Generate code to push X onto the stack, assuming it has mode MODE and
5276 : type TYPE.
5277 : MODE is redundant except when X is a CONST_INT (since they don't
5278 : carry mode info).
5279 : SIZE is an rtx for the size of data to be copied (in bytes),
5280 : needed only if X is BLKmode.
5281 : Return true if successful. May return false if asked to push a
5282 : partial argument during a sibcall optimization (as specified by
5283 : SIBCALL_P) and the incoming and outgoing pointers cannot be shown
5284 : to not overlap.
5285 :
5286 : ALIGN (in bits) is maximum alignment we can assume.
5287 :
5288 : If PARTIAL and REG are both nonzero, then copy that many of the first
5289 : bytes of X into registers starting with REG, and push the rest of X.
5290 : The amount of space pushed is decreased by PARTIAL bytes.
5291 : REG must be a hard register in this case.
5292 : If REG is zero but PARTIAL is not, take any all others actions for an
5293 : argument partially in registers, but do not actually load any
5294 : registers.
5295 :
5296 : EXTRA is the amount in bytes of extra space to leave next to this arg.
5297 : This is ignored if an argument block has already been allocated.
5298 :
5299 : On a machine that lacks real push insns, ARGS_ADDR is the address of
5300 : the bottom of the argument block for this call. We use indexing off there
5301 : to store the arg. On machines with push insns, ARGS_ADDR is 0 when a
5302 : argument block has not been preallocated.
5303 :
5304 : ARGS_SO_FAR is the size of args previously pushed for this call.
5305 :
5306 : REG_PARM_STACK_SPACE is nonzero if functions require stack space
5307 : for arguments passed in registers. If nonzero, it will be the number
5308 : of bytes required. */
5309 :
5310 : bool
5311 2172879 : emit_push_insn (rtx x, machine_mode mode, tree type, rtx size,
5312 : unsigned int align, int partial, rtx reg, poly_int64 extra,
5313 : rtx args_addr, rtx args_so_far, int reg_parm_stack_space,
5314 : rtx alignment_pad, bool sibcall_p)
5315 : {
5316 2172879 : rtx xinner;
5317 2172879 : pad_direction stack_direction
5318 : = STACK_GROWS_DOWNWARD ? PAD_DOWNWARD : PAD_UPWARD;
5319 :
5320 : /* Decide where to pad the argument: PAD_DOWNWARD for below,
5321 : PAD_UPWARD for above, or PAD_NONE for don't pad it.
5322 : Default is below for small data on big-endian machines; else above. */
5323 2172879 : pad_direction where_pad = targetm.calls.function_arg_padding (mode, type);
5324 :
5325 : /* Invert direction if stack is post-decrement.
5326 : FIXME: why? */
5327 2172879 : if (STACK_PUSH_CODE == POST_DEC)
5328 : if (where_pad != PAD_NONE)
5329 : where_pad = (where_pad == PAD_DOWNWARD ? PAD_UPWARD : PAD_DOWNWARD);
5330 :
5331 2172879 : xinner = x;
5332 :
5333 2172879 : int nregs = partial / UNITS_PER_WORD;
5334 2172879 : rtx *tmp_regs = NULL;
5335 2172879 : int overlapping = 0;
5336 :
5337 2172879 : if (mode == BLKmode
5338 : || (STRICT_ALIGNMENT && align < GET_MODE_ALIGNMENT (mode)))
5339 : {
5340 : /* Copy a block into the stack, entirely or partially. */
5341 :
5342 269167 : rtx temp;
5343 269167 : int used;
5344 269167 : int offset;
5345 269167 : int skip;
5346 :
5347 269167 : offset = partial % (PARM_BOUNDARY / BITS_PER_UNIT);
5348 269167 : used = partial - offset;
5349 :
5350 269167 : if (mode != BLKmode)
5351 : {
5352 : /* A value is to be stored in an insufficiently aligned
5353 : stack slot; copy via a suitably aligned slot if
5354 : necessary. */
5355 : size = gen_int_mode (GET_MODE_SIZE (mode), Pmode);
5356 : if (!MEM_P (xinner))
5357 : {
5358 : temp = assign_temp (type, 1, 1);
5359 : emit_move_insn (temp, xinner);
5360 : xinner = temp;
5361 : }
5362 : }
5363 :
5364 269167 : gcc_assert (size);
5365 :
5366 : /* USED is now the # of bytes we need not copy to the stack
5367 : because registers will take care of them. */
5368 :
5369 269167 : if (partial != 0)
5370 0 : xinner = adjust_address (xinner, BLKmode, used);
5371 :
5372 : /* If the partial register-part of the arg counts in its stack size,
5373 : skip the part of stack space corresponding to the registers.
5374 : Otherwise, start copying to the beginning of the stack space,
5375 : by setting SKIP to 0. */
5376 269167 : skip = (reg_parm_stack_space == 0) ? 0 : used;
5377 :
5378 : #ifdef PUSH_ROUNDING
5379 : /* NB: Let the backend known the number of bytes to push and
5380 : decide if push insns should be generated. */
5381 269167 : unsigned int push_size;
5382 269167 : if (CONST_INT_P (size))
5383 269167 : push_size = INTVAL (size);
5384 : else
5385 : push_size = 0;
5386 :
5387 : /* Do it with several push insns if that doesn't take lots of insns
5388 : and if there is no difficulty with push insns that skip bytes
5389 : on the stack for alignment purposes. */
5390 269167 : if (args_addr == 0
5391 269113 : && targetm.calls.push_argument (push_size)
5392 3783 : && CONST_INT_P (size)
5393 3783 : && skip == 0
5394 3783 : && MEM_ALIGN (xinner) >= align
5395 2950 : && can_move_by_pieces ((unsigned) INTVAL (size) - used, align)
5396 : /* Here we avoid the case of a structure whose weak alignment
5397 : forces many pushes of a small amount of data,
5398 : and such small pushes do rounding that causes trouble. */
5399 2950 : && ((!targetm.slow_unaligned_access (word_mode, align))
5400 0 : || align >= BIGGEST_ALIGNMENT
5401 0 : || known_eq (PUSH_ROUNDING (align / BITS_PER_UNIT),
5402 : align / BITS_PER_UNIT))
5403 272117 : && known_eq (PUSH_ROUNDING (INTVAL (size)), INTVAL (size)))
5404 : {
5405 : /* Push padding now if padding above and stack grows down,
5406 : or if padding below and stack grows up.
5407 : But if space already allocated, this has already been done. */
5408 54 : if (maybe_ne (extra, 0)
5409 : && args_addr == 0
5410 0 : && where_pad != PAD_NONE
5411 54 : && where_pad != stack_direction)
5412 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5413 :
5414 54 : move_by_pieces (NULL, xinner, INTVAL (size) - used, align,
5415 : RETURN_BEGIN);
5416 : }
5417 : else
5418 : #endif /* PUSH_ROUNDING */
5419 : {
5420 269113 : rtx target;
5421 :
5422 : /* Otherwise make space on the stack and copy the data
5423 : to the address of that space. */
5424 :
5425 : /* Deduct words put into registers from the size we must copy. */
5426 269113 : if (partial != 0)
5427 : {
5428 0 : if (CONST_INT_P (size))
5429 0 : size = GEN_INT (INTVAL (size) - used);
5430 : else
5431 0 : size = expand_binop (GET_MODE (size), sub_optab, size,
5432 0 : gen_int_mode (used, GET_MODE (size)),
5433 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
5434 : }
5435 :
5436 : /* Get the address of the stack space.
5437 : In this case, we do not deal with EXTRA separately.
5438 : A single stack adjust will do. */
5439 269113 : poly_int64 const_args_so_far;
5440 269113 : if (! args_addr)
5441 : {
5442 269059 : temp = push_block (size, extra, where_pad == PAD_DOWNWARD);
5443 269059 : extra = 0;
5444 : }
5445 54 : else if (poly_int_rtx_p (args_so_far, &const_args_so_far))
5446 72 : temp = memory_address (BLKmode,
5447 : plus_constant (Pmode, args_addr,
5448 : skip + const_args_so_far));
5449 : else
5450 0 : temp = memory_address (BLKmode,
5451 : plus_constant (Pmode,
5452 : gen_rtx_PLUS (Pmode,
5453 : args_addr,
5454 : args_so_far),
5455 : skip));
5456 :
5457 269113 : if (!ACCUMULATE_OUTGOING_ARGS)
5458 : {
5459 : /* If the source is referenced relative to the stack pointer,
5460 : copy it to another register to stabilize it. We do not need
5461 : to do this if we know that we won't be changing sp. */
5462 :
5463 269113 : if (reg_mentioned_p (virtual_stack_dynamic_rtx, temp)
5464 269113 : || reg_mentioned_p (virtual_outgoing_args_rtx, temp))
5465 269059 : temp = copy_to_reg (temp);
5466 : }
5467 :
5468 269113 : target = gen_rtx_MEM (BLKmode, temp);
5469 :
5470 : /* We do *not* set_mem_attributes here, because incoming arguments
5471 : may overlap with sibling call outgoing arguments and we cannot
5472 : allow reordering of reads from function arguments with stores
5473 : to outgoing arguments of sibling calls. We do, however, want
5474 : to record the alignment of the stack slot. */
5475 : /* ALIGN may well be better aligned than TYPE, e.g. due to
5476 : PARM_BOUNDARY. Assume the caller isn't lying. */
5477 269113 : set_mem_align (target, align);
5478 :
5479 : /* If part should go in registers and pushing to that part would
5480 : overwrite some of the values that need to go into regs, load the
5481 : overlapping values into temporary pseudos to be moved into the hard
5482 : regs at the end after the stack pushing has completed.
5483 : We cannot load them directly into the hard regs here because
5484 : they can be clobbered by the block move expansions.
5485 : See PR 65358. */
5486 :
5487 269113 : if (partial > 0 && reg != 0 && mode == BLKmode
5488 0 : && GET_CODE (reg) != PARALLEL)
5489 : {
5490 0 : overlapping = memory_load_overlap (XEXP (x, 0), temp, partial);
5491 0 : if (overlapping > 0)
5492 : {
5493 0 : gcc_assert (overlapping % UNITS_PER_WORD == 0);
5494 0 : overlapping /= UNITS_PER_WORD;
5495 :
5496 0 : tmp_regs = XALLOCAVEC (rtx, overlapping);
5497 :
5498 0 : for (int i = 0; i < overlapping; i++)
5499 0 : tmp_regs[i] = gen_reg_rtx (word_mode);
5500 :
5501 0 : for (int i = 0; i < overlapping; i++)
5502 0 : emit_move_insn (tmp_regs[i],
5503 0 : operand_subword_force (target, i, mode));
5504 : }
5505 0 : else if (overlapping == -1)
5506 : overlapping = 0;
5507 : /* Could not determine whether there is overlap.
5508 : Fail the sibcall. */
5509 : else
5510 : {
5511 0 : overlapping = 0;
5512 0 : if (sibcall_p)
5513 2172879 : return false;
5514 : }
5515 : }
5516 :
5517 : /* If source is a constant VAR_DECL with a simple constructor,
5518 : store the constructor to the stack instead of moving it. */
5519 269113 : const_tree decl;
5520 269113 : HOST_WIDE_INT sz;
5521 269113 : if (partial == 0
5522 269113 : && MEM_P (xinner)
5523 269113 : && SYMBOL_REF_P (XEXP (xinner, 0))
5524 82598 : && (decl = SYMBOL_REF_DECL (XEXP (xinner, 0))) != NULL_TREE
5525 82598 : && VAR_P (decl)
5526 82598 : && TREE_READONLY (decl)
5527 5237 : && !TREE_SIDE_EFFECTS (decl)
5528 5237 : && immediate_const_ctor_p (DECL_INITIAL (decl), 2)
5529 9 : && (sz = int_expr_size (DECL_INITIAL (decl))) > 0
5530 9 : && CONST_INT_P (size)
5531 269122 : && INTVAL (size) == sz)
5532 3 : store_constructor (DECL_INITIAL (decl), target, 0, sz, false);
5533 : else
5534 269110 : emit_block_move (target, xinner, size, BLOCK_OP_CALL_PARM);
5535 : }
5536 : }
5537 1903712 : else if (partial > 0)
5538 : {
5539 : /* Scalar partly in registers. This case is only supported
5540 : for fixed-wdth modes. */
5541 0 : int num_words = GET_MODE_SIZE (mode).to_constant ();
5542 0 : num_words /= UNITS_PER_WORD;
5543 0 : int i;
5544 0 : int not_stack;
5545 : /* # bytes of start of argument
5546 : that we must make space for but need not store. */
5547 0 : int offset = partial % (PARM_BOUNDARY / BITS_PER_UNIT);
5548 0 : int args_offset = INTVAL (args_so_far);
5549 0 : int skip;
5550 :
5551 : /* Push padding now if padding above and stack grows down,
5552 : or if padding below and stack grows up.
5553 : But if space already allocated, this has already been done. */
5554 0 : if (maybe_ne (extra, 0)
5555 0 : && args_addr == 0
5556 0 : && where_pad != PAD_NONE
5557 0 : && where_pad != stack_direction)
5558 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5559 :
5560 : /* If we make space by pushing it, we might as well push
5561 : the real data. Otherwise, we can leave OFFSET nonzero
5562 : and leave the space uninitialized. */
5563 0 : if (args_addr == 0)
5564 0 : offset = 0;
5565 :
5566 : /* Now NOT_STACK gets the number of words that we don't need to
5567 : allocate on the stack. Convert OFFSET to words too. */
5568 0 : not_stack = (partial - offset) / UNITS_PER_WORD;
5569 0 : offset /= UNITS_PER_WORD;
5570 :
5571 : /* If the partial register-part of the arg counts in its stack size,
5572 : skip the part of stack space corresponding to the registers.
5573 : Otherwise, start copying to the beginning of the stack space,
5574 : by setting SKIP to 0. */
5575 0 : skip = (reg_parm_stack_space == 0) ? 0 : not_stack;
5576 :
5577 0 : if (CONSTANT_P (x) && !targetm.legitimate_constant_p (mode, x))
5578 0 : x = validize_mem (force_const_mem (mode, x));
5579 :
5580 : /* If X is a hard register in a non-integer mode, copy it into a pseudo;
5581 : SUBREGs of such registers are not allowed. */
5582 0 : if ((REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER
5583 0 : && GET_MODE_CLASS (GET_MODE (x)) != MODE_INT))
5584 0 : x = copy_to_reg (x);
5585 :
5586 : /* Loop over all the words allocated on the stack for this arg. */
5587 : /* We can do it by words, because any scalar bigger than a word
5588 : has a size a multiple of a word. */
5589 0 : tree word_mode_type = lang_hooks.types.type_for_mode (word_mode, 1);
5590 0 : for (i = num_words - 1; i >= not_stack; i--)
5591 0 : if (i >= not_stack + offset)
5592 0 : if (!emit_push_insn (operand_subword_force (x, i, mode),
5593 : word_mode, word_mode_type, NULL_RTX, align, 0,
5594 : NULL_RTX, 0, args_addr,
5595 0 : GEN_INT (args_offset + ((i - not_stack + skip)
5596 : * UNITS_PER_WORD)),
5597 : reg_parm_stack_space, alignment_pad, sibcall_p))
5598 2172879 : return false;
5599 : }
5600 : else
5601 : {
5602 1903712 : rtx addr;
5603 1903712 : rtx dest;
5604 :
5605 : /* Push padding now if padding above and stack grows down,
5606 : or if padding below and stack grows up.
5607 : But if space already allocated, this has already been done. */
5608 1903712 : if (maybe_ne (extra, 0)
5609 0 : && args_addr == 0
5610 0 : && where_pad != PAD_NONE
5611 1903712 : && where_pad != stack_direction)
5612 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5613 :
5614 : #ifdef PUSH_ROUNDING
5615 1903712 : if (args_addr == 0 && targetm.calls.push_argument (0))
5616 1848367 : emit_single_push_insn (mode, x, type);
5617 : else
5618 : #endif
5619 : {
5620 63586 : addr = simplify_gen_binary (PLUS, Pmode, args_addr, args_so_far);
5621 55345 : dest = gen_rtx_MEM (mode, memory_address (mode, addr));
5622 :
5623 : /* We do *not* set_mem_attributes here, because incoming arguments
5624 : may overlap with sibling call outgoing arguments and we cannot
5625 : allow reordering of reads from function arguments with stores
5626 : to outgoing arguments of sibling calls. We do, however, want
5627 : to record the alignment of the stack slot. */
5628 : /* ALIGN may well be better aligned than TYPE, e.g. due to
5629 : PARM_BOUNDARY. Assume the caller isn't lying. */
5630 55345 : set_mem_align (dest, align);
5631 :
5632 55345 : emit_move_insn (dest, x);
5633 : }
5634 : }
5635 :
5636 : /* Move the partial arguments into the registers and any overlapping
5637 : values that we moved into the pseudos in tmp_regs. */
5638 2172879 : if (partial > 0 && reg != 0)
5639 : {
5640 : /* Handle calls that pass values in multiple non-contiguous locations.
5641 : The Irix 6 ABI has examples of this. */
5642 0 : if (GET_CODE (reg) == PARALLEL)
5643 0 : emit_group_load (reg, x, type, -1);
5644 : else
5645 : {
5646 0 : gcc_assert (partial % UNITS_PER_WORD == 0);
5647 0 : move_block_to_reg (REGNO (reg), x, nregs - overlapping, mode);
5648 :
5649 0 : for (int i = 0; i < overlapping; i++)
5650 0 : emit_move_insn (gen_rtx_REG (word_mode, REGNO (reg)
5651 0 : + nregs - overlapping + i),
5652 0 : tmp_regs[i]);
5653 :
5654 : }
5655 : }
5656 :
5657 2172879 : if (maybe_ne (extra, 0) && args_addr == 0 && where_pad == stack_direction)
5658 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5659 :
5660 2172879 : if (alignment_pad && args_addr == 0)
5661 2117480 : anti_adjust_stack (alignment_pad);
5662 :
5663 : return true;
5664 : }
5665 :
5666 : /* Return X if X can be used as a subtarget in a sequence of arithmetic
5667 : operations. */
5668 :
5669 : static rtx
5670 206921987 : get_subtarget (rtx x)
5671 : {
5672 206921987 : return (optimize
5673 52833308 : || x == 0
5674 : /* Only registers can be subtargets. */
5675 17689330 : || !REG_P (x)
5676 : /* Don't use hard regs to avoid extending their life. */
5677 12009496 : || REGNO (x) < FIRST_PSEUDO_REGISTER
5678 206921987 : ? 0 : x);
5679 : }
5680 :
5681 : /* A subroutine of expand_assignment. Optimize FIELD op= VAL, where
5682 : FIELD is a bitfield. Returns true if the optimization was successful,
5683 : and there's nothing else to do. */
5684 :
5685 : static bool
5686 4836250 : optimize_bitfield_assignment_op (poly_uint64 pbitsize,
5687 : poly_uint64 pbitpos,
5688 : poly_uint64 pbitregion_start,
5689 : poly_uint64 pbitregion_end,
5690 : machine_mode mode1, rtx str_rtx,
5691 : tree to, tree src, bool reverse)
5692 : {
5693 : /* str_mode is not guaranteed to be a scalar type. */
5694 4836250 : machine_mode str_mode = GET_MODE (str_rtx);
5695 4836250 : unsigned int str_bitsize;
5696 4836250 : tree op0, op1;
5697 4836250 : rtx value, result;
5698 4836250 : optab binop;
5699 4836250 : gimple *srcstmt;
5700 4836250 : enum tree_code code;
5701 :
5702 4836250 : unsigned HOST_WIDE_INT bitsize, bitpos, bitregion_start, bitregion_end;
5703 4836250 : if (mode1 != VOIDmode
5704 67445 : || !pbitsize.is_constant (&bitsize)
5705 67445 : || !pbitpos.is_constant (&bitpos)
5706 67445 : || !pbitregion_start.is_constant (&bitregion_start)
5707 67445 : || !pbitregion_end.is_constant (&bitregion_end)
5708 68123 : || bitsize >= BITS_PER_WORD
5709 67174 : || !GET_MODE_BITSIZE (str_mode).is_constant (&str_bitsize)
5710 67852 : || str_bitsize > BITS_PER_WORD
5711 61779 : || TREE_SIDE_EFFECTS (to)
5712 4897900 : || TREE_THIS_VOLATILE (to))
5713 : return false;
5714 :
5715 61650 : STRIP_NOPS (src);
5716 61650 : if (TREE_CODE (src) != SSA_NAME)
5717 : return false;
5718 20346 : if (TREE_CODE (TREE_TYPE (src)) != INTEGER_TYPE)
5719 : return false;
5720 :
5721 18683 : srcstmt = get_gimple_for_ssa_name (src);
5722 18683 : if (!srcstmt
5723 4978 : || !is_gimple_assign (srcstmt)
5724 23661 : || TREE_CODE_CLASS (gimple_assign_rhs_code (srcstmt)) != tcc_binary)
5725 : return false;
5726 :
5727 846 : code = gimple_assign_rhs_code (srcstmt);
5728 :
5729 846 : op0 = gimple_assign_rhs1 (srcstmt);
5730 :
5731 : /* If OP0 is an SSA_NAME, then we want to walk the use-def chain
5732 : to find its initialization. Hopefully the initialization will
5733 : be from a bitfield load. */
5734 846 : if (TREE_CODE (op0) == SSA_NAME)
5735 : {
5736 846 : gimple *op0stmt = get_gimple_for_ssa_name (op0);
5737 :
5738 : /* We want to eventually have OP0 be the same as TO, which
5739 : should be a bitfield. */
5740 846 : if (!op0stmt
5741 776 : || !is_gimple_assign (op0stmt)
5742 1622 : || gimple_assign_rhs_code (op0stmt) != TREE_CODE (to))
5743 : return false;
5744 548 : op0 = gimple_assign_rhs1 (op0stmt);
5745 : }
5746 :
5747 548 : op1 = gimple_assign_rhs2 (srcstmt);
5748 :
5749 548 : if (!operand_equal_p (to, op0, 0))
5750 : return false;
5751 :
5752 463 : if (MEM_P (str_rtx))
5753 : {
5754 422 : unsigned HOST_WIDE_INT offset1;
5755 :
5756 422 : if (str_bitsize == 0 || str_bitsize > BITS_PER_WORD)
5757 82 : str_bitsize = BITS_PER_WORD;
5758 :
5759 422 : scalar_int_mode best_mode;
5760 422 : if (!get_best_mode (bitsize, bitpos, bitregion_start, bitregion_end,
5761 422 : MEM_ALIGN (str_rtx), str_bitsize, false, &best_mode))
5762 0 : return false;
5763 422 : str_mode = best_mode;
5764 422 : str_bitsize = GET_MODE_BITSIZE (best_mode);
5765 :
5766 422 : offset1 = bitpos;
5767 422 : bitpos %= str_bitsize;
5768 422 : offset1 = (offset1 - bitpos) / BITS_PER_UNIT;
5769 422 : str_rtx = adjust_address (str_rtx, str_mode, offset1);
5770 : }
5771 41 : else if (!REG_P (str_rtx) && GET_CODE (str_rtx) != SUBREG)
5772 : return false;
5773 :
5774 : /* If the bit field covers the whole REG/MEM, store_field
5775 : will likely generate better code. */
5776 463 : if (bitsize >= str_bitsize)
5777 : return false;
5778 :
5779 : /* We can't handle fields split across multiple entities. */
5780 463 : if (bitpos + bitsize > str_bitsize)
5781 : return false;
5782 :
5783 463 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
5784 0 : bitpos = str_bitsize - bitpos - bitsize;
5785 :
5786 463 : switch (code)
5787 : {
5788 140 : case PLUS_EXPR:
5789 140 : case MINUS_EXPR:
5790 : /* For now, just optimize the case of the topmost bitfield
5791 : where we don't need to do any masking and also
5792 : 1 bit bitfields where xor can be used.
5793 : We might win by one instruction for the other bitfields
5794 : too if insv/extv instructions aren't used, so that
5795 : can be added later. */
5796 140 : if ((reverse || bitpos + bitsize != str_bitsize)
5797 96 : && (bitsize != 1 || TREE_CODE (op1) != INTEGER_CST))
5798 : break;
5799 :
5800 69 : value = expand_expr (op1, NULL_RTX, str_mode, EXPAND_NORMAL);
5801 69 : value = convert_modes (str_mode,
5802 69 : TYPE_MODE (TREE_TYPE (op1)), value,
5803 69 : TYPE_UNSIGNED (TREE_TYPE (op1)));
5804 :
5805 : /* We may be accessing data outside the field, which means
5806 : we can alias adjacent data. */
5807 69 : if (MEM_P (str_rtx))
5808 : {
5809 69 : str_rtx = shallow_copy_rtx (str_rtx);
5810 69 : set_mem_alias_set (str_rtx, 0);
5811 69 : set_mem_expr (str_rtx, 0);
5812 : }
5813 :
5814 69 : if (bitsize == 1 && (reverse || bitpos + bitsize != str_bitsize))
5815 : {
5816 25 : value = expand_and (str_mode, value, const1_rtx, NULL);
5817 25 : binop = xor_optab;
5818 : }
5819 : else
5820 44 : binop = code == PLUS_EXPR ? add_optab : sub_optab;
5821 :
5822 69 : value = expand_shift (LSHIFT_EXPR, str_mode, value, bitpos, NULL_RTX, 1);
5823 69 : if (reverse)
5824 0 : value = flip_storage_order (str_mode, value);
5825 69 : result = expand_binop (str_mode, binop, str_rtx,
5826 : value, str_rtx, 1, OPTAB_WIDEN);
5827 69 : if (result != str_rtx)
5828 0 : emit_move_insn (str_rtx, result);
5829 : return true;
5830 :
5831 255 : case BIT_IOR_EXPR:
5832 255 : case BIT_XOR_EXPR:
5833 255 : if (TREE_CODE (op1) != INTEGER_CST)
5834 : break;
5835 97 : value = expand_expr (op1, NULL_RTX, str_mode, EXPAND_NORMAL);
5836 97 : value = convert_modes (str_mode,
5837 97 : TYPE_MODE (TREE_TYPE (op1)), value,
5838 97 : TYPE_UNSIGNED (TREE_TYPE (op1)));
5839 :
5840 : /* We may be accessing data outside the field, which means
5841 : we can alias adjacent data. */
5842 97 : if (MEM_P (str_rtx))
5843 : {
5844 61 : str_rtx = shallow_copy_rtx (str_rtx);
5845 61 : set_mem_alias_set (str_rtx, 0);
5846 61 : set_mem_expr (str_rtx, 0);
5847 : }
5848 :
5849 97 : binop = code == BIT_IOR_EXPR ? ior_optab : xor_optab;
5850 97 : if (bitpos + bitsize != str_bitsize)
5851 : {
5852 55 : rtx mask = gen_int_mode ((HOST_WIDE_INT_1U << bitsize) - 1,
5853 : str_mode);
5854 55 : value = expand_and (str_mode, value, mask, NULL_RTX);
5855 : }
5856 97 : value = expand_shift (LSHIFT_EXPR, str_mode, value, bitpos, NULL_RTX, 1);
5857 97 : if (reverse)
5858 0 : value = flip_storage_order (str_mode, value);
5859 97 : result = expand_binop (str_mode, binop, str_rtx,
5860 : value, str_rtx, 1, OPTAB_WIDEN);
5861 97 : if (result != str_rtx)
5862 0 : emit_move_insn (str_rtx, result);
5863 : return true;
5864 :
5865 : default:
5866 : break;
5867 : }
5868 :
5869 : return false;
5870 : }
5871 :
5872 : /* In the C++ memory model, consecutive bit fields in a structure are
5873 : considered one memory location.
5874 :
5875 : Given a COMPONENT_REF EXP at position (BITPOS, OFFSET), this function
5876 : returns the bit range of consecutive bits in which this COMPONENT_REF
5877 : belongs. The values are returned in *BITSTART and *BITEND. *BITPOS
5878 : and *OFFSET may be adjusted in the process.
5879 :
5880 : If the access does not need to be restricted, 0 is returned in both
5881 : *BITSTART and *BITEND. */
5882 :
5883 : void
5884 941637 : get_bit_range (poly_uint64 *bitstart, poly_uint64 *bitend, tree exp,
5885 : poly_int64 *bitpos, tree *offset)
5886 : {
5887 941637 : poly_int64 bitoffset;
5888 941637 : tree field, repr;
5889 :
5890 941637 : gcc_assert (TREE_CODE (exp) == COMPONENT_REF);
5891 :
5892 941637 : field = TREE_OPERAND (exp, 1);
5893 941637 : repr = DECL_BIT_FIELD_REPRESENTATIVE (field);
5894 : /* If we do not have a DECL_BIT_FIELD_REPRESENTATIVE there is no
5895 : need to limit the range we can access. */
5896 941637 : if (!repr)
5897 : {
5898 65 : *bitstart = *bitend = 0;
5899 65 : return;
5900 : }
5901 :
5902 : /* If we have a DECL_BIT_FIELD_REPRESENTATIVE but the enclosing record is
5903 : part of a larger bit field, then the representative does not serve any
5904 : useful purpose. This can occur in Ada. */
5905 941572 : if (handled_component_p (TREE_OPERAND (exp, 0)))
5906 : {
5907 509341 : machine_mode rmode;
5908 509341 : poly_int64 rbitsize, rbitpos;
5909 509341 : tree roffset;
5910 509341 : int unsignedp, reversep, volatilep = 0;
5911 509341 : get_inner_reference (TREE_OPERAND (exp, 0), &rbitsize, &rbitpos,
5912 : &roffset, &rmode, &unsignedp, &reversep,
5913 : &volatilep);
5914 1018682 : if (!multiple_p (rbitpos, BITS_PER_UNIT))
5915 : {
5916 0 : *bitstart = *bitend = 0;
5917 0 : return;
5918 : }
5919 : }
5920 :
5921 : /* Compute the adjustment to bitpos from the offset of the field
5922 : relative to the representative. DECL_FIELD_OFFSET of field and
5923 : repr are the same by construction if they are not constants,
5924 : see finish_bitfield_layout. */
5925 941572 : poly_uint64 field_offset, repr_offset;
5926 941572 : if (poly_int_tree_p (DECL_FIELD_OFFSET (field), &field_offset)
5927 1883139 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
5928 941567 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
5929 : else
5930 : bitoffset = 0;
5931 941572 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (field))
5932 941572 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
5933 :
5934 : /* If the adjustment is larger than bitpos, we would have a negative bit
5935 : position for the lower bound and this may wreak havoc later. Adjust
5936 : offset and bitpos to make the lower bound non-negative in that case. */
5937 941572 : if (maybe_gt (bitoffset, *bitpos))
5938 : {
5939 9 : poly_int64 adjust_bits = upper_bound (bitoffset, *bitpos) - *bitpos;
5940 9 : poly_int64 adjust_bytes = exact_div (adjust_bits, BITS_PER_UNIT);
5941 :
5942 9 : *bitpos += adjust_bits;
5943 9 : if (*offset == NULL_TREE)
5944 5 : *offset = size_int (-adjust_bytes);
5945 : else
5946 4 : *offset = size_binop (MINUS_EXPR, *offset, size_int (adjust_bytes));
5947 9 : *bitstart = 0;
5948 : }
5949 : else
5950 941563 : *bitstart = *bitpos - bitoffset;
5951 :
5952 941572 : *bitend = *bitstart + tree_to_poly_uint64 (DECL_SIZE (repr)) - 1;
5953 : }
5954 :
5955 : /* Returns true if BASE is a DECL that does not reside in memory and
5956 : has non-BLKmode. DECL_RTL must not be a MEM; if
5957 : DECL_RTL was not set yet, return false. */
5958 :
5959 : bool
5960 5579308 : non_mem_decl_p (tree base)
5961 : {
5962 5579308 : if (!DECL_P (base)
5963 5578910 : || TREE_ADDRESSABLE (base)
5964 8052450 : || DECL_MODE (base) == BLKmode)
5965 : return false;
5966 :
5967 1603558 : if (!DECL_RTL_SET_P (base))
5968 : return false;
5969 :
5970 1484602 : return (!MEM_P (DECL_RTL (base)));
5971 : }
5972 :
5973 : /* Returns true if REF refers to an object that does not
5974 : reside in memory and has non-BLKmode. */
5975 :
5976 : bool
5977 12232474 : mem_ref_refers_to_non_mem_p (tree ref)
5978 : {
5979 12232474 : tree base;
5980 :
5981 12232474 : if (TREE_CODE (ref) == MEM_REF
5982 12232474 : || TREE_CODE (ref) == TARGET_MEM_REF)
5983 : {
5984 10696739 : tree addr = TREE_OPERAND (ref, 0);
5985 :
5986 10696739 : if (TREE_CODE (addr) != ADDR_EXPR)
5987 : return false;
5988 :
5989 4017749 : base = TREE_OPERAND (addr, 0);
5990 : }
5991 : else
5992 : base = ref;
5993 :
5994 5553484 : return non_mem_decl_p (base);
5995 : }
5996 :
5997 : /* Helper function of expand_assignment. Check if storing field of
5998 : size BITSIZE at position BITPOS overlaps with the most significant
5999 : bit of TO_RTX, known to be SUBREG_PROMOTED_VAR_P.
6000 : Updating this field requires an explicit extension. */
6001 : static bool
6002 0 : store_field_updates_msb_p (poly_int64 bitpos, poly_int64 bitsize, rtx to_rtx)
6003 : {
6004 0 : return (BYTES_BIG_ENDIAN
6005 : ? maybe_le (bitpos, 0)
6006 0 : : maybe_ge (bitpos + bitsize, GET_MODE_BITSIZE (GET_MODE (to_rtx))));
6007 : }
6008 :
6009 : /* Expand an assignment that stores the value of FROM into TO. If NONTEMPORAL
6010 : is true, try generating a nontemporal store. */
6011 :
6012 : void
6013 18817305 : expand_assignment (tree to, tree from, bool nontemporal)
6014 : {
6015 18817305 : rtx to_rtx = 0;
6016 18817305 : rtx result;
6017 18817305 : machine_mode mode;
6018 18817305 : unsigned int align;
6019 18817305 : enum insn_code icode;
6020 :
6021 : /* Don't crash if the lhs of the assignment was erroneous. */
6022 18817305 : if (TREE_CODE (to) == ERROR_MARK)
6023 : {
6024 0 : expand_normal (from);
6025 0 : return;
6026 : }
6027 :
6028 : /* Optimize away no-op moves without side-effects. */
6029 18817305 : if (operand_equal_p (to, from, 0))
6030 : return;
6031 :
6032 : /* Handle misaligned stores. */
6033 18817149 : mode = TYPE_MODE (TREE_TYPE (to));
6034 18817149 : if ((TREE_CODE (to) == MEM_REF
6035 18817149 : || TREE_CODE (to) == TARGET_MEM_REF
6036 16567587 : || DECL_P (to))
6037 4116448 : && mode != BLKmode
6038 3645988 : && !mem_ref_refers_to_non_mem_p (to)
6039 6534070 : && ((align = get_object_alignment (to))
6040 3267035 : < GET_MODE_ALIGNMENT (mode))
6041 19250240 : && (((icode = optab_handler (movmisalign_optab, mode))
6042 : != CODE_FOR_nothing)
6043 97563 : || targetm.slow_unaligned_access (mode, align)))
6044 : {
6045 335528 : rtx reg, mem;
6046 :
6047 335528 : reg = expand_expr (from, NULL_RTX, VOIDmode, EXPAND_NORMAL);
6048 : /* Handle PARALLEL. */
6049 335528 : reg = maybe_emit_group_store (reg, TREE_TYPE (from));
6050 335528 : reg = force_not_mem (reg);
6051 335528 : mem = expand_expr (to, NULL_RTX, VOIDmode, EXPAND_WRITE);
6052 335528 : if (TREE_CODE (to) == MEM_REF && REF_REVERSE_STORAGE_ORDER (to))
6053 0 : reg = flip_storage_order (mode, reg);
6054 :
6055 335528 : if (icode != CODE_FOR_nothing)
6056 : {
6057 335528 : class expand_operand ops[2];
6058 :
6059 335528 : create_fixed_operand (&ops[0], mem);
6060 335528 : create_input_operand (&ops[1], reg, mode);
6061 : /* The movmisalign<mode> pattern cannot fail, else the assignment
6062 : would silently be omitted. */
6063 335528 : expand_insn (icode, 2, ops);
6064 : }
6065 : else
6066 0 : store_bit_field (mem, GET_MODE_BITSIZE (mode), 0, 0, 0, mode, reg,
6067 : false, false);
6068 : return;
6069 : }
6070 :
6071 : /* Assignment of a structure component needs special treatment
6072 : if the structure component's rtx is not simply a MEM.
6073 : Assignment of an array element at a constant index, and assignment of
6074 : an array element in an unaligned packed structure field, has the same
6075 : problem. Same for (partially) storing into a non-memory object. */
6076 18481621 : if (handled_component_p (to)
6077 13879243 : || (TREE_CODE (to) == MEM_REF
6078 1679035 : && (REF_REVERSE_STORAGE_ORDER (to)
6079 1678948 : || mem_ref_refers_to_non_mem_p (to)))
6080 13767635 : || TREE_CODE (TREE_TYPE (to)) == ARRAY_TYPE)
6081 : {
6082 4836349 : machine_mode mode1;
6083 4836349 : poly_int64 bitsize, bitpos;
6084 4836349 : poly_uint64 bitregion_start = 0;
6085 4836349 : poly_uint64 bitregion_end = 0;
6086 4836349 : tree offset;
6087 4836349 : int unsignedp, reversep, volatilep = 0;
6088 4836349 : tree tem;
6089 :
6090 4836349 : push_temp_slots ();
6091 4836349 : tem = get_inner_reference (to, &bitsize, &bitpos, &offset, &mode1,
6092 : &unsignedp, &reversep, &volatilep);
6093 :
6094 : /* Make sure bitpos is not negative, it can wreak havoc later. */
6095 4836349 : if (maybe_lt (bitpos, 0))
6096 : {
6097 199 : gcc_assert (offset == NULL_TREE);
6098 199 : offset = size_int (bits_to_bytes_round_down (bitpos));
6099 199 : bitpos = num_trailing_bits (bitpos);
6100 : }
6101 :
6102 4836349 : if (TREE_CODE (to) == COMPONENT_REF
6103 4836349 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (to, 1)))
6104 85632 : get_bit_range (&bitregion_start, &bitregion_end, to, &bitpos, &offset);
6105 : /* The C++ memory model naturally applies to byte-aligned fields.
6106 : However, if we do not have a DECL_BIT_FIELD_TYPE but BITPOS or
6107 : BITSIZE are not byte-aligned, there is no need to limit the range
6108 : we can access. This can occur with packed structures in Ada. */
6109 4750717 : else if (maybe_gt (bitsize, 0)
6110 4750702 : && multiple_p (bitsize, BITS_PER_UNIT)
6111 9501164 : && multiple_p (bitpos, BITS_PER_UNIT))
6112 : {
6113 4750447 : bitregion_start = bitpos;
6114 4750447 : bitregion_end = bitpos + bitsize - 1;
6115 : }
6116 :
6117 4836349 : to_rtx = expand_expr (tem, NULL_RTX, VOIDmode, EXPAND_WRITE);
6118 :
6119 : /* If the field has a mode, we want to access it in the
6120 : field's mode, not the computed mode.
6121 : If a MEM has VOIDmode (external with incomplete type),
6122 : use BLKmode for it instead. */
6123 4836349 : if (MEM_P (to_rtx))
6124 : {
6125 4146423 : if (mode1 != VOIDmode)
6126 4079697 : to_rtx = adjust_address (to_rtx, mode1, 0);
6127 66726 : else if (GET_MODE (to_rtx) == VOIDmode)
6128 0 : to_rtx = adjust_address (to_rtx, BLKmode, 0);
6129 : }
6130 :
6131 4836349 : rtx stemp = NULL_RTX, old_to_rtx = NULL_RTX;
6132 4836349 : if (offset != 0)
6133 : {
6134 179955 : machine_mode address_mode;
6135 179955 : rtx offset_rtx;
6136 :
6137 179955 : if (!MEM_P (to_rtx))
6138 : {
6139 : /* We can get constant negative offsets into arrays with broken
6140 : user code. Translate this to a trap instead of ICEing. */
6141 5 : if (TREE_CODE (offset) == INTEGER_CST)
6142 : {
6143 2 : expand_builtin_trap ();
6144 2 : to_rtx = gen_rtx_MEM (BLKmode, const0_rtx);
6145 : }
6146 : /* Else spill for variable offset to the destination. We expect
6147 : to run into this only for hard registers. */
6148 : else
6149 : {
6150 3 : gcc_assert (VAR_P (tem) && DECL_HARD_REGISTER (tem));
6151 3 : stemp = assign_stack_temp (GET_MODE (to_rtx),
6152 6 : GET_MODE_SIZE (GET_MODE (to_rtx)));
6153 3 : emit_move_insn (stemp, to_rtx);
6154 3 : old_to_rtx = to_rtx;
6155 3 : to_rtx = stemp;
6156 : }
6157 : }
6158 :
6159 179955 : offset_rtx = expand_expr (offset, NULL_RTX, VOIDmode, EXPAND_SUM);
6160 179955 : address_mode = get_address_mode (to_rtx);
6161 179955 : if (GET_MODE (offset_rtx) != address_mode)
6162 : {
6163 : /* We cannot be sure that the RTL in offset_rtx is valid outside
6164 : of a memory address context, so force it into a register
6165 : before attempting to convert it to the desired mode. */
6166 224 : offset_rtx = force_operand (offset_rtx, NULL_RTX);
6167 224 : offset_rtx = convert_to_mode (address_mode, offset_rtx, 0);
6168 : }
6169 :
6170 : /* If we have an expression in OFFSET_RTX and a non-zero
6171 : byte offset in BITPOS, adding the byte offset before the
6172 : OFFSET_RTX results in better intermediate code, which makes
6173 : later rtl optimization passes perform better.
6174 :
6175 : We prefer intermediate code like this:
6176 :
6177 : r124:DI=r123:DI+0x18
6178 : [r124:DI]=r121:DI
6179 :
6180 : ... instead of ...
6181 :
6182 : r124:DI=r123:DI+0x10
6183 : [r124:DI+0x8]=r121:DI
6184 :
6185 : This is only done for aligned data values, as these can
6186 : be expected to result in single move instructions. */
6187 179955 : poly_int64 bytepos;
6188 179955 : if (mode1 != VOIDmode
6189 179849 : && maybe_ne (bitpos, 0)
6190 48050 : && maybe_gt (bitsize, 0)
6191 227873 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
6192 227873 : && multiple_p (bitpos, bitsize)
6193 95836 : && multiple_p (bitsize, GET_MODE_ALIGNMENT (mode1))
6194 227873 : && MEM_ALIGN (to_rtx) >= GET_MODE_ALIGNMENT (mode1))
6195 : {
6196 47669 : to_rtx = adjust_address (to_rtx, mode1, bytepos);
6197 47669 : bitregion_start = 0;
6198 47669 : if (known_ge (bitregion_end, poly_uint64 (bitpos)))
6199 47669 : bitregion_end -= bitpos;
6200 47669 : bitpos = 0;
6201 : }
6202 :
6203 179955 : to_rtx = offset_address (to_rtx, offset_rtx,
6204 : highest_pow2_factor_for_target (to,
6205 : offset));
6206 : }
6207 :
6208 : /* No action is needed if the target is not a memory and the field
6209 : lies completely outside that target. This can occur if the source
6210 : code contains an out-of-bounds access to a small array. */
6211 4836349 : if (!MEM_P (to_rtx)
6212 689921 : && GET_MODE (to_rtx) != BLKmode
6213 5526270 : && known_ge (bitpos, GET_MODE_PRECISION (GET_MODE (to_rtx))))
6214 : {
6215 3 : expand_normal (from);
6216 3 : result = NULL;
6217 : }
6218 : /* Handle expand_expr of a complex value returning a CONCAT. */
6219 4836346 : else if (GET_CODE (to_rtx) == CONCAT)
6220 : {
6221 90 : machine_mode to_mode = GET_MODE (to_rtx);
6222 90 : gcc_checking_assert (COMPLEX_MODE_P (to_mode));
6223 180 : poly_int64 mode_bitsize = GET_MODE_BITSIZE (to_mode);
6224 90 : unsigned short inner_bitsize = GET_MODE_UNIT_BITSIZE (to_mode);
6225 90 : if (TYPE_MODE (TREE_TYPE (from)) == to_mode
6226 0 : && known_eq (bitpos, 0)
6227 90 : && known_eq (bitsize, mode_bitsize))
6228 0 : result = store_expr (from, to_rtx, false, nontemporal, reversep);
6229 90 : else if (TYPE_MODE (TREE_TYPE (from)) == GET_MODE_INNER (to_mode)
6230 38 : && known_eq (bitsize, inner_bitsize)
6231 128 : && (known_eq (bitpos, 0)
6232 15 : || known_eq (bitpos, inner_bitsize)))
6233 38 : result = store_expr (from, XEXP (to_rtx, maybe_ne (bitpos, 0)),
6234 : false, nontemporal, reversep);
6235 52 : else if (known_le (bitpos + bitsize, inner_bitsize))
6236 5 : result = store_field (XEXP (to_rtx, 0), bitsize, bitpos,
6237 : bitregion_start, bitregion_end,
6238 : mode1, from, get_alias_set (to),
6239 : nontemporal, reversep);
6240 47 : else if (known_ge (bitpos, inner_bitsize))
6241 3 : result = store_field (XEXP (to_rtx, 1), bitsize,
6242 : bitpos - inner_bitsize,
6243 : bitregion_start, bitregion_end,
6244 : mode1, from, get_alias_set (to),
6245 : nontemporal, reversep);
6246 44 : else if (known_eq (bitpos, 0) && known_eq (bitsize, mode_bitsize))
6247 : {
6248 37 : result = expand_normal (from);
6249 37 : if (GET_CODE (result) == CONCAT)
6250 : {
6251 0 : to_mode = GET_MODE_INNER (to_mode);
6252 0 : machine_mode from_mode = GET_MODE_INNER (GET_MODE (result));
6253 0 : rtx from_real
6254 0 : = force_subreg (to_mode, XEXP (result, 0), from_mode, 0);
6255 0 : rtx from_imag
6256 0 : = force_subreg (to_mode, XEXP (result, 1), from_mode, 0);
6257 0 : if (!from_real || !from_imag)
6258 0 : goto concat_store_slow;
6259 0 : emit_move_insn (XEXP (to_rtx, 0), from_real);
6260 0 : emit_move_insn (XEXP (to_rtx, 1), from_imag);
6261 : }
6262 : else
6263 : {
6264 37 : machine_mode from_mode
6265 37 : = GET_MODE (result) == VOIDmode
6266 37 : ? TYPE_MODE (TREE_TYPE (from))
6267 37 : : GET_MODE (result);
6268 37 : rtx from_rtx;
6269 37 : if (MEM_P (result))
6270 1 : from_rtx = change_address (result, to_mode, NULL_RTX);
6271 : else
6272 36 : from_rtx = force_subreg (to_mode, result, from_mode, 0);
6273 37 : if (from_rtx)
6274 : {
6275 37 : emit_move_insn (XEXP (to_rtx, 0),
6276 : read_complex_part (from_rtx, false));
6277 37 : emit_move_insn (XEXP (to_rtx, 1),
6278 : read_complex_part (from_rtx, true));
6279 : }
6280 : else
6281 : {
6282 0 : to_mode = GET_MODE_INNER (to_mode);
6283 0 : rtx from_real
6284 0 : = force_subreg (to_mode, result, from_mode, 0);
6285 0 : rtx from_imag
6286 0 : = force_subreg (to_mode, result, from_mode,
6287 0 : GET_MODE_SIZE (to_mode));
6288 0 : if (!from_real || !from_imag)
6289 0 : goto concat_store_slow;
6290 0 : emit_move_insn (XEXP (to_rtx, 0), from_real);
6291 0 : emit_move_insn (XEXP (to_rtx, 1), from_imag);
6292 : }
6293 : }
6294 : }
6295 : else
6296 : {
6297 7 : concat_store_slow:;
6298 7 : rtx temp = assign_stack_temp (GET_MODE (to_rtx),
6299 14 : GET_MODE_SIZE (GET_MODE (to_rtx)));
6300 7 : write_complex_part (temp, XEXP (to_rtx, 0), false, true);
6301 7 : write_complex_part (temp, XEXP (to_rtx, 1), true, false);
6302 7 : result = store_field (temp, bitsize, bitpos,
6303 : bitregion_start, bitregion_end,
6304 : mode1, from, get_alias_set (to),
6305 : nontemporal, reversep);
6306 7 : emit_move_insn (XEXP (to_rtx, 0), read_complex_part (temp, false));
6307 7 : emit_move_insn (XEXP (to_rtx, 1), read_complex_part (temp, true));
6308 : }
6309 : }
6310 : /* For calls to functions returning variable length structures, if TO_RTX
6311 : is not a MEM, go through a MEM because we must not create temporaries
6312 : of the VLA type. */
6313 4836256 : else if (!MEM_P (to_rtx)
6314 689828 : && TREE_CODE (from) == CALL_EXPR
6315 244 : && COMPLETE_TYPE_P (TREE_TYPE (from))
6316 4836500 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (from))) != INTEGER_CST)
6317 : {
6318 6 : rtx temp = assign_stack_temp (GET_MODE (to_rtx),
6319 12 : GET_MODE_SIZE (GET_MODE (to_rtx)));
6320 6 : result = store_field (temp, bitsize, bitpos, bitregion_start,
6321 : bitregion_end, mode1, from, get_alias_set (to),
6322 : nontemporal, reversep);
6323 6 : emit_move_insn (to_rtx, temp);
6324 : }
6325 : else
6326 : {
6327 4836250 : if (MEM_P (to_rtx))
6328 : {
6329 : /* If the field is at offset zero, we could have been given the
6330 : DECL_RTX of the parent struct. Don't munge it. */
6331 4146428 : to_rtx = shallow_copy_rtx (to_rtx);
6332 4146428 : set_mem_attributes_minus_bitpos (to_rtx, to, 0, bitpos, true);
6333 4146428 : if (volatilep)
6334 8776 : MEM_VOLATILE_P (to_rtx) = 1;
6335 : }
6336 :
6337 4836250 : gcc_checking_assert (known_ge (bitpos, 0));
6338 9672500 : if (optimize_bitfield_assignment_op (bitsize, bitpos,
6339 : bitregion_start, bitregion_end,
6340 : mode1, to_rtx, to, from,
6341 : reversep))
6342 : result = NULL;
6343 4836084 : else if (SUBREG_P (to_rtx)
6344 4836084 : && SUBREG_PROMOTED_VAR_P (to_rtx))
6345 : {
6346 : /* If to_rtx is a promoted subreg, we need to zero or sign
6347 : extend the value afterwards. */
6348 0 : if (TREE_CODE (to) == MEM_REF
6349 0 : && TYPE_MODE (TREE_TYPE (from)) != BLKmode
6350 0 : && !REF_REVERSE_STORAGE_ORDER (to)
6351 0 : && known_eq (bitpos, 0)
6352 0 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (to_rtx))))
6353 0 : result = store_expr (from, to_rtx, 0, nontemporal, false);
6354 : /* Check if the field overlaps the MSB, requiring extension. */
6355 0 : else if (store_field_updates_msb_p (bitpos, bitsize, to_rtx))
6356 : {
6357 0 : scalar_int_mode imode = subreg_unpromoted_mode (to_rtx);
6358 0 : scalar_int_mode omode = subreg_promoted_mode (to_rtx);
6359 0 : rtx to_rtx1 = lowpart_subreg (imode, SUBREG_REG (to_rtx),
6360 : omode);
6361 0 : result = store_field (to_rtx1, bitsize, bitpos,
6362 : bitregion_start, bitregion_end,
6363 : mode1, from, get_alias_set (to),
6364 : nontemporal, reversep);
6365 : /* If the target usually keeps IMODE appropriately
6366 : extended in OMODE it's unsafe to refer to it using
6367 : a SUBREG whilst this invariant doesn't hold. */
6368 0 : if (targetm.mode_rep_extended (imode, omode) != UNKNOWN)
6369 0 : to_rtx1 = simplify_gen_unary (TRUNCATE, imode,
6370 : SUBREG_REG (to_rtx), omode);
6371 0 : convert_move (SUBREG_REG (to_rtx), to_rtx1,
6372 0 : SUBREG_PROMOTED_SIGN (to_rtx));
6373 : }
6374 : else
6375 0 : result = store_field (to_rtx, bitsize, bitpos,
6376 : bitregion_start, bitregion_end,
6377 : mode1, from, get_alias_set (to),
6378 : nontemporal, reversep);
6379 : }
6380 : else
6381 4836084 : result = store_field (to_rtx, bitsize, bitpos,
6382 : bitregion_start, bitregion_end,
6383 : mode1, from, get_alias_set (to),
6384 : nontemporal, reversep);
6385 : /* Move the temporary storage back to the non-MEM_P. */
6386 4836250 : if (stemp)
6387 3 : emit_move_insn (old_to_rtx, stemp);
6388 : }
6389 :
6390 4836349 : if (result)
6391 832377 : preserve_temp_slots (result);
6392 4836349 : pop_temp_slots ();
6393 4836349 : return;
6394 : }
6395 :
6396 : /* If the rhs is a function call and its value is not an aggregate,
6397 : call the function before we start to compute the lhs.
6398 : This is needed for correct code for cases such as
6399 : val = setjmp (buf) on machines where reference to val
6400 : requires loading up part of an address in a separate insn.
6401 :
6402 : Don't do this if TO is a VAR_DECL or PARM_DECL whose DECL_RTL is REG
6403 : since it might be a promoted variable where the zero- or sign- extension
6404 : needs to be done. Handling this in the normal way is safe because no
6405 : computation is done before the call. The same is true for SSA names. */
6406 2388369 : if (TREE_CODE (from) == CALL_EXPR && ! aggregate_value_p (from, from)
6407 2187728 : && COMPLETE_TYPE_P (TREE_TYPE (from))
6408 2187728 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (from))) == INTEGER_CST
6409 15833000 : && ! (((VAR_P (to)
6410 : || TREE_CODE (to) == PARM_DECL
6411 : || TREE_CODE (to) == RESULT_DECL)
6412 123564 : && REG_P (DECL_RTL (to)))
6413 2074431 : || TREE_CODE (to) == SSA_NAME))
6414 : {
6415 10602 : rtx value;
6416 :
6417 10602 : push_temp_slots ();
6418 10602 : value = expand_normal (from);
6419 :
6420 10602 : if (to_rtx == 0)
6421 10602 : to_rtx = expand_expr (to, NULL_RTX, VOIDmode, EXPAND_WRITE);
6422 :
6423 : /* Handle calls that return values in multiple non-contiguous locations.
6424 : The Irix 6 ABI has examples of this. */
6425 10602 : if (GET_CODE (to_rtx) == PARALLEL)
6426 : {
6427 0 : if (GET_CODE (value) == PARALLEL)
6428 0 : emit_group_move (to_rtx, value);
6429 : else
6430 0 : emit_group_load (to_rtx, value, TREE_TYPE (from),
6431 0 : int_size_in_bytes (TREE_TYPE (from)));
6432 : }
6433 10602 : else if (GET_CODE (value) == PARALLEL)
6434 1957 : emit_group_store (to_rtx, value, TREE_TYPE (from),
6435 1957 : int_size_in_bytes (TREE_TYPE (from)));
6436 8645 : else if (GET_MODE (to_rtx) == BLKmode)
6437 : {
6438 : /* Handle calls that return BLKmode values in registers. */
6439 254 : if (REG_P (value))
6440 254 : copy_blkmode_from_reg (to_rtx, value, TREE_TYPE (from));
6441 : else
6442 0 : emit_block_move (to_rtx, value, expr_size (from), BLOCK_OP_NORMAL);
6443 : }
6444 : else
6445 : {
6446 8391 : if (POINTER_TYPE_P (TREE_TYPE (to)))
6447 0 : value = convert_memory_address_addr_space
6448 0 : (as_a <scalar_int_mode> (GET_MODE (to_rtx)), value,
6449 0 : TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (to))));
6450 :
6451 8391 : emit_move_insn (to_rtx, value);
6452 : }
6453 :
6454 10602 : preserve_temp_slots (to_rtx);
6455 10602 : pop_temp_slots ();
6456 10602 : return;
6457 : }
6458 :
6459 : /* Ordinary treatment. Expand TO to get a REG or MEM rtx. */
6460 13634670 : to_rtx = expand_expr (to, NULL_RTX, VOIDmode, EXPAND_WRITE);
6461 :
6462 : /* Don't move directly into a return register. */
6463 13634670 : if (TREE_CODE (to) == RESULT_DECL
6464 29320 : && (REG_P (to_rtx) || GET_CODE (to_rtx) == PARALLEL))
6465 : {
6466 0 : rtx temp;
6467 :
6468 0 : push_temp_slots ();
6469 :
6470 : /* If the source is itself a return value, it still is in a pseudo at
6471 : this point so we can move it back to the return register directly. */
6472 0 : if (REG_P (to_rtx)
6473 0 : && TYPE_MODE (TREE_TYPE (from)) == BLKmode
6474 0 : && TREE_CODE (from) != CALL_EXPR)
6475 0 : temp = copy_blkmode_to_reg (GET_MODE (to_rtx), from);
6476 : else
6477 0 : temp = expand_expr (from, NULL_RTX, GET_MODE (to_rtx), EXPAND_NORMAL);
6478 :
6479 : /* Handle calls that return values in multiple non-contiguous locations.
6480 : The Irix 6 ABI has examples of this. */
6481 0 : if (GET_CODE (to_rtx) == PARALLEL)
6482 : {
6483 0 : if (GET_CODE (temp) == PARALLEL)
6484 0 : emit_group_move (to_rtx, temp);
6485 : else
6486 0 : emit_group_load (to_rtx, temp, TREE_TYPE (from),
6487 0 : int_size_in_bytes (TREE_TYPE (from)));
6488 : }
6489 0 : else if (temp)
6490 0 : emit_move_insn (to_rtx, temp);
6491 :
6492 0 : preserve_temp_slots (to_rtx);
6493 0 : pop_temp_slots ();
6494 0 : return;
6495 : }
6496 :
6497 : /* In case we are returning the contents of an object which overlaps
6498 : the place the value is being stored, use a safe function when copying
6499 : a value through a pointer into a structure value return block. */
6500 13634670 : if (TREE_CODE (to) == RESULT_DECL
6501 29320 : && TREE_CODE (from) == INDIRECT_REF
6502 0 : && ADDR_SPACE_GENERIC_P
6503 : (TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (from, 0)))))
6504 0 : && refs_may_alias_p (to, from)
6505 0 : && cfun->returns_struct
6506 13634670 : && !cfun->returns_pcc_struct)
6507 : {
6508 0 : rtx from_rtx, size;
6509 :
6510 0 : push_temp_slots ();
6511 0 : size = expr_size (from);
6512 0 : from_rtx = expand_normal (from);
6513 :
6514 0 : emit_block_move_via_libcall (XEXP (to_rtx, 0), XEXP (from_rtx, 0), size);
6515 :
6516 0 : preserve_temp_slots (to_rtx);
6517 0 : pop_temp_slots ();
6518 0 : return;
6519 : }
6520 :
6521 : /* Compute FROM and store the value in the rtx we got. */
6522 :
6523 13634670 : push_temp_slots ();
6524 13634670 : result = store_expr (from, to_rtx, 0, nontemporal, false);
6525 13634668 : preserve_temp_slots (result);
6526 13634668 : pop_temp_slots ();
6527 13634668 : return;
6528 : }
6529 :
6530 : /* Emits nontemporal store insn that moves FROM to TO. Returns true if this
6531 : succeeded, false otherwise. */
6532 :
6533 : bool
6534 17 : emit_storent_insn (rtx to, rtx from)
6535 : {
6536 17 : class expand_operand ops[2];
6537 17 : machine_mode mode = GET_MODE (to);
6538 17 : enum insn_code code = optab_handler (storent_optab, mode);
6539 :
6540 17 : if (code == CODE_FOR_nothing)
6541 : return false;
6542 :
6543 17 : create_fixed_operand (&ops[0], to);
6544 17 : create_input_operand (&ops[1], from, mode);
6545 17 : return maybe_expand_insn (code, 2, ops);
6546 : }
6547 :
6548 : /* Helper function for store_expr storing of STRING_CST. */
6549 :
6550 : static rtx
6551 110272 : string_cst_read_str (void *data, void *, HOST_WIDE_INT offset,
6552 : fixed_size_mode mode)
6553 : {
6554 110272 : tree str = (tree) data;
6555 :
6556 110272 : gcc_assert (offset >= 0);
6557 110272 : if (offset >= TREE_STRING_LENGTH (str))
6558 27304 : return const0_rtx;
6559 :
6560 82968 : if ((unsigned HOST_WIDE_INT) offset + GET_MODE_SIZE (mode)
6561 82968 : > (unsigned HOST_WIDE_INT) TREE_STRING_LENGTH (str))
6562 : {
6563 7626 : char *p = XALLOCAVEC (char, GET_MODE_SIZE (mode));
6564 7626 : size_t l = TREE_STRING_LENGTH (str) - offset;
6565 7626 : memcpy (p, TREE_STRING_POINTER (str) + offset, l);
6566 7626 : memset (p + l, '\0', GET_MODE_SIZE (mode) - l);
6567 7626 : return c_readstr (p, mode, false);
6568 : }
6569 :
6570 75342 : return c_readstr (TREE_STRING_POINTER (str) + offset, mode, false);
6571 : }
6572 :
6573 : /* Helper function for store_expr storing of RAW_DATA_CST. */
6574 :
6575 : static rtx
6576 36 : raw_data_cst_read_str (void *data, void *, HOST_WIDE_INT offset,
6577 : fixed_size_mode mode)
6578 : {
6579 36 : tree cst = (tree) data;
6580 :
6581 36 : gcc_assert (offset >= 0);
6582 36 : if (offset >= RAW_DATA_LENGTH (cst))
6583 0 : return const0_rtx;
6584 :
6585 36 : if ((unsigned HOST_WIDE_INT) offset + GET_MODE_SIZE (mode)
6586 36 : > (unsigned HOST_WIDE_INT) RAW_DATA_LENGTH (cst))
6587 : {
6588 0 : char *p = XALLOCAVEC (char, GET_MODE_SIZE (mode));
6589 0 : size_t l = RAW_DATA_LENGTH (cst) - offset;
6590 0 : memcpy (p, RAW_DATA_POINTER (cst) + offset, l);
6591 0 : memset (p + l, '\0', GET_MODE_SIZE (mode) - l);
6592 0 : return c_readstr (p, mode, false);
6593 : }
6594 :
6595 36 : return c_readstr (RAW_DATA_POINTER (cst) + offset, mode, false);
6596 : }
6597 :
6598 : /* Generate code for computing expression EXP,
6599 : and storing the value into TARGET.
6600 :
6601 : If the mode is BLKmode then we may return TARGET itself.
6602 : It turns out that in BLKmode it doesn't cause a problem.
6603 : because C has no operators that could combine two different
6604 : assignments into the same BLKmode object with different values
6605 : with no sequence point. Will other languages need this to
6606 : be more thorough?
6607 :
6608 : If CALL_PARAM_P is nonzero, this is a store into a call param on the
6609 : stack, and block moves may need to be treated specially.
6610 :
6611 : If NONTEMPORAL is true, try using a nontemporal store instruction.
6612 :
6613 : If REVERSE is true, the store is to be done in reverse order. */
6614 :
6615 : rtx
6616 17679527 : store_expr (tree exp, rtx target, int call_param_p,
6617 : bool nontemporal, bool reverse)
6618 : {
6619 17679527 : rtx temp;
6620 17679527 : rtx alt_rtl = NULL_RTX;
6621 17679527 : location_t loc = curr_insn_location ();
6622 17679527 : bool shortened_string_cst = false;
6623 :
6624 17679527 : if (VOID_TYPE_P (TREE_TYPE (exp)))
6625 : {
6626 : /* C++ can generate ?: expressions with a throw expression in one
6627 : branch and an rvalue in the other. Here, we resolve attempts to
6628 : store the throw expression's nonexistent result. */
6629 0 : gcc_assert (!call_param_p);
6630 0 : expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
6631 0 : return NULL_RTX;
6632 : }
6633 17679527 : if (TREE_CODE (exp) == COMPOUND_EXPR)
6634 : {
6635 : /* Perform first part of compound expression, then assign from second
6636 : part. */
6637 0 : expand_expr (TREE_OPERAND (exp, 0), const0_rtx, VOIDmode,
6638 : call_param_p ? EXPAND_STACK_PARM : EXPAND_NORMAL);
6639 0 : return store_expr (TREE_OPERAND (exp, 1), target,
6640 0 : call_param_p, nontemporal, reverse);
6641 : }
6642 17679527 : else if (TREE_CODE (exp) == COND_EXPR && GET_MODE (target) == BLKmode)
6643 : {
6644 : /* For conditional expression, get safe form of the target. Then
6645 : test the condition, doing the appropriate assignment on either
6646 : side. This avoids the creation of unnecessary temporaries.
6647 : For non-BLKmode, it is more efficient not to do this. */
6648 :
6649 0 : rtx_code_label *lab1 = gen_label_rtx (), *lab2 = gen_label_rtx ();
6650 :
6651 0 : do_pending_stack_adjust ();
6652 0 : NO_DEFER_POP;
6653 0 : jumpifnot (TREE_OPERAND (exp, 0), lab1,
6654 : profile_probability::uninitialized ());
6655 0 : store_expr (TREE_OPERAND (exp, 1), target, call_param_p,
6656 : nontemporal, reverse);
6657 0 : emit_jump_insn (targetm.gen_jump (lab2));
6658 0 : emit_barrier ();
6659 0 : emit_label (lab1);
6660 0 : store_expr (TREE_OPERAND (exp, 2), target, call_param_p,
6661 : nontemporal, reverse);
6662 0 : emit_label (lab2);
6663 0 : OK_DEFER_POP;
6664 :
6665 0 : return NULL_RTX;
6666 : }
6667 17679527 : else if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
6668 : /* If this is a scalar in a register that is stored in a wider mode
6669 : than the declared mode, compute the result into its declared mode
6670 : and then convert to the wider mode. Our value is the computed
6671 : expression. */
6672 : {
6673 7 : rtx inner_target = 0;
6674 7 : scalar_int_mode outer_mode = subreg_unpromoted_mode (target);
6675 7 : scalar_int_mode inner_mode = subreg_promoted_mode (target);
6676 :
6677 : /* We can do the conversion inside EXP, which will often result
6678 : in some optimizations. Do the conversion in two steps: first
6679 : change the signedness, if needed, then the extend. But don't
6680 : do this if the type of EXP is a subtype of something else
6681 : since then the conversion might involve more than just
6682 : converting modes. */
6683 14 : if (INTEGRAL_TYPE_P (TREE_TYPE (exp))
6684 0 : && TREE_TYPE (TREE_TYPE (exp)) == 0
6685 7 : && GET_MODE_PRECISION (outer_mode)
6686 0 : == TYPE_PRECISION (TREE_TYPE (exp)))
6687 : {
6688 0 : if (!SUBREG_CHECK_PROMOTED_SIGN (target,
6689 : TYPE_UNSIGNED (TREE_TYPE (exp))))
6690 : {
6691 : /* Some types, e.g. Fortran's logical*4, won't have a signed
6692 : version, so use the mode instead. */
6693 0 : tree ntype
6694 : = (signed_or_unsigned_type_for
6695 0 : (SUBREG_PROMOTED_SIGN (target), TREE_TYPE (exp)));
6696 0 : if (ntype == NULL)
6697 0 : ntype = lang_hooks.types.type_for_mode
6698 0 : (TYPE_MODE (TREE_TYPE (exp)),
6699 0 : SUBREG_PROMOTED_SIGN (target));
6700 :
6701 0 : exp = fold_convert_loc (loc, ntype, exp);
6702 : }
6703 :
6704 0 : exp = fold_convert_loc (loc, lang_hooks.types.type_for_mode
6705 0 : (inner_mode, SUBREG_PROMOTED_SIGN (target)),
6706 : exp);
6707 :
6708 0 : inner_target = SUBREG_REG (target);
6709 : }
6710 :
6711 7 : temp = expand_expr (exp, inner_target, VOIDmode,
6712 : call_param_p ? EXPAND_STACK_PARM : EXPAND_NORMAL);
6713 :
6714 :
6715 : /* If TEMP is a VOIDmode constant, use convert_modes to make
6716 : sure that we properly convert it. */
6717 7 : if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode)
6718 : {
6719 0 : temp = convert_modes (outer_mode, TYPE_MODE (TREE_TYPE (exp)),
6720 0 : temp, SUBREG_PROMOTED_SIGN (target));
6721 0 : temp = convert_modes (inner_mode, outer_mode, temp,
6722 0 : SUBREG_PROMOTED_SIGN (target));
6723 0 : }
6724 7 : else if (!SCALAR_INT_MODE_P (GET_MODE (temp)))
6725 0 : temp = convert_modes (outer_mode, TYPE_MODE (TREE_TYPE (exp)),
6726 0 : temp, SUBREG_PROMOTED_SIGN (target));
6727 :
6728 21 : convert_move (SUBREG_REG (target), temp,
6729 7 : SUBREG_PROMOTED_SIGN (target));
6730 :
6731 7 : return NULL_RTX;
6732 : }
6733 17679520 : else if ((TREE_CODE (exp) == STRING_CST
6734 17669555 : || (TREE_CODE (exp) == MEM_REF
6735 1362114 : && TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR
6736 446001 : && TREE_CODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
6737 : == STRING_CST
6738 6608 : && integer_zerop (TREE_OPERAND (exp, 1))))
6739 16573 : && !nontemporal && !call_param_p
6740 17696093 : && MEM_P (target))
6741 : {
6742 : /* Optimize initialization of an array with a STRING_CST. */
6743 16555 : HOST_WIDE_INT exp_len, str_copy_len;
6744 16555 : rtx dest_mem;
6745 16555 : tree str = TREE_CODE (exp) == STRING_CST
6746 16555 : ? exp : TREE_OPERAND (TREE_OPERAND (exp, 0), 0);
6747 :
6748 16555 : exp_len = int_expr_size (exp);
6749 16555 : if (exp_len <= 0)
6750 0 : goto normal_expr;
6751 :
6752 16555 : if (TREE_STRING_LENGTH (str) <= 0)
6753 0 : goto normal_expr;
6754 :
6755 33110 : if (can_store_by_pieces (exp_len, string_cst_read_str, (void *) str,
6756 16555 : MEM_ALIGN (target), false))
6757 : {
6758 16138 : store_by_pieces (target, exp_len, string_cst_read_str, (void *) str,
6759 16138 : MEM_ALIGN (target), false, RETURN_BEGIN);
6760 16138 : return NULL_RTX;
6761 : }
6762 :
6763 417 : str_copy_len = TREE_STRING_LENGTH (str);
6764 :
6765 : /* Trailing NUL bytes in EXP will be handled by the call to
6766 : clear_storage, which is more efficient than copying them from
6767 : the STRING_CST, so trim those from STR_COPY_LEN. */
6768 773 : while (str_copy_len)
6769 : {
6770 721 : if (TREE_STRING_POINTER (str)[str_copy_len - 1])
6771 : break;
6772 356 : str_copy_len--;
6773 : }
6774 :
6775 417 : if ((STORE_MAX_PIECES & (STORE_MAX_PIECES - 1)) == 0)
6776 : {
6777 417 : str_copy_len += STORE_MAX_PIECES - 1;
6778 417 : str_copy_len &= ~(STORE_MAX_PIECES - 1);
6779 : }
6780 417 : if (str_copy_len >= exp_len)
6781 136 : goto normal_expr;
6782 :
6783 562 : if (!can_store_by_pieces (str_copy_len, string_cst_read_str,
6784 281 : (void *) str, MEM_ALIGN (target), false))
6785 2 : goto normal_expr;
6786 :
6787 279 : dest_mem = store_by_pieces (target, str_copy_len, string_cst_read_str,
6788 279 : (void *) str, MEM_ALIGN (target), false,
6789 : RETURN_END);
6790 279 : clear_storage (adjust_address_1 (dest_mem, BLKmode, 0, 1, 1, 0,
6791 279 : exp_len - str_copy_len),
6792 : GEN_INT (exp_len - str_copy_len), BLOCK_OP_NORMAL);
6793 279 : return NULL_RTX;
6794 : }
6795 : else
6796 : {
6797 17663103 : rtx tmp_target;
6798 :
6799 17663103 : normal_expr:
6800 : /* If we want to use a nontemporal or a reverse order store, force the
6801 : value into a register first. */
6802 17663103 : tmp_target = nontemporal || reverse ? NULL_RTX : target;
6803 17663103 : tree rexp = exp;
6804 17663103 : if (TREE_CODE (exp) == STRING_CST
6805 20 : && tmp_target == target
6806 20 : && GET_MODE (target) == BLKmode
6807 17663123 : && TYPE_MODE (TREE_TYPE (exp)) == BLKmode)
6808 : {
6809 20 : rtx size = expr_size (exp);
6810 20 : if (CONST_INT_P (size)
6811 20 : && size != const0_rtx
6812 40 : && (UINTVAL (size)
6813 20 : > ((unsigned HOST_WIDE_INT) TREE_STRING_LENGTH (exp) + 32)))
6814 : {
6815 : /* If the STRING_CST has much larger array type than
6816 : TREE_STRING_LENGTH, only emit the TREE_STRING_LENGTH part of
6817 : it into the rodata section as the code later on will use
6818 : memset zero for the remainder anyway. See PR95052. */
6819 2 : tmp_target = NULL_RTX;
6820 2 : rexp = copy_node (exp);
6821 2 : tree index
6822 2 : = build_index_type (size_int (TREE_STRING_LENGTH (exp) - 1));
6823 2 : TREE_TYPE (rexp) = build_array_type (TREE_TYPE (TREE_TYPE (exp)),
6824 : index);
6825 2 : shortened_string_cst = true;
6826 : }
6827 : }
6828 35326206 : temp = expand_expr_real (rexp, tmp_target, GET_MODE (target),
6829 : (call_param_p
6830 : ? EXPAND_STACK_PARM : EXPAND_NORMAL),
6831 : &alt_rtl, false);
6832 17663101 : if (shortened_string_cst)
6833 : {
6834 2 : gcc_assert (MEM_P (temp));
6835 2 : temp = change_address (temp, BLKmode, NULL_RTX);
6836 : }
6837 : }
6838 :
6839 : /* If TEMP is a VOIDmode constant and the mode of the type of EXP is not
6840 : the same as that of TARGET, adjust the constant. This is needed, for
6841 : example, in case it is a CONST_DOUBLE or CONST_WIDE_INT and we want
6842 : only a word-sized value. */
6843 3631707 : if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode
6844 2626068 : && TREE_CODE (exp) != ERROR_MARK
6845 20289169 : && GET_MODE (target) != TYPE_MODE (TREE_TYPE (exp)))
6846 : {
6847 0 : gcc_assert (!shortened_string_cst);
6848 0 : if (GET_MODE_CLASS (GET_MODE (target))
6849 0 : != GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (exp)))
6850 0 : && known_eq (GET_MODE_BITSIZE (GET_MODE (target)),
6851 : GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (exp)))))
6852 : {
6853 0 : rtx t = simplify_gen_subreg (GET_MODE (target), temp,
6854 0 : TYPE_MODE (TREE_TYPE (exp)), 0);
6855 0 : if (t)
6856 0 : temp = t;
6857 : }
6858 0 : if (GET_MODE (temp) == VOIDmode)
6859 0 : temp = convert_modes (GET_MODE (target), TYPE_MODE (TREE_TYPE (exp)),
6860 0 : temp, TYPE_UNSIGNED (TREE_TYPE (exp)));
6861 : }
6862 :
6863 : /* If value was not generated in the target, store it there.
6864 : Convert the value to TARGET's type first if necessary and emit the
6865 : pending incrementations that have been queued when expanding EXP.
6866 : Note that we cannot emit the whole queue blindly because this will
6867 : effectively disable the POST_INC optimization later.
6868 :
6869 : If TEMP and TARGET compare equal according to rtx_equal_p, but
6870 : one or both of them are volatile memory refs, we have to distinguish
6871 : two cases:
6872 : - expand_expr has used TARGET. In this case, we must not generate
6873 : another copy. This can be detected by TARGET being equal according
6874 : to == .
6875 : - expand_expr has not used TARGET - that means that the source just
6876 : happens to have the same RTX form. Since temp will have been created
6877 : by expand_expr, it will compare unequal according to == .
6878 : We must generate a copy in this case, to reach the correct number
6879 : of volatile memory references. */
6880 :
6881 17663101 : if ((! rtx_equal_p (temp, target)
6882 3151793 : || (temp != target && (side_effects_p (temp)
6883 46976 : || side_effects_p (target)
6884 46976 : || (MEM_P (temp)
6885 46565 : && !mems_same_for_tbaa_p (temp, target)))))
6886 14511312 : && TREE_CODE (exp) != ERROR_MARK
6887 : /* If store_expr stores a DECL whose DECL_RTL(exp) == TARGET,
6888 : but TARGET is not valid memory reference, TEMP will differ
6889 : from TARGET although it is really the same location. */
6890 14511312 : && !(alt_rtl
6891 1979735 : && rtx_equal_p (alt_rtl, target)
6892 1 : && !side_effects_p (alt_rtl)
6893 0 : && !side_effects_p (target))
6894 : /* If there's nothing to copy, don't bother. Don't call
6895 : expr_size unless necessary, because some front-ends (C++)
6896 : expr_size-hook must not be given objects that are not
6897 : supposed to be bit-copied or bit-initialized. */
6898 32174413 : && expr_size (exp) != const0_rtx)
6899 : {
6900 14511312 : if (GET_MODE (temp) != GET_MODE (target) && GET_MODE (temp) != VOIDmode)
6901 : {
6902 1353 : gcc_assert (!shortened_string_cst);
6903 1353 : if (GET_MODE (target) == BLKmode)
6904 : {
6905 : /* Handle calls that return BLKmode values in registers. */
6906 2 : if (REG_P (temp) && TREE_CODE (exp) == CALL_EXPR)
6907 2 : copy_blkmode_from_reg (target, temp, TREE_TYPE (exp));
6908 : else
6909 0 : store_bit_field (target,
6910 0 : rtx_to_poly_int64 (expr_size (exp))
6911 0 : * BITS_PER_UNIT,
6912 : 0, 0, 0, GET_MODE (temp), temp, reverse,
6913 : false);
6914 : }
6915 : else
6916 1351 : convert_move (target, temp, TYPE_UNSIGNED (TREE_TYPE (exp)));
6917 : }
6918 :
6919 14509959 : else if (GET_MODE (temp) == BLKmode && TREE_CODE (exp) == STRING_CST)
6920 : {
6921 : /* Handle copying a string constant into an array. The string
6922 : constant may be shorter than the array. So copy just the string's
6923 : actual length, and clear the rest. First get the size of the data
6924 : type of the string, which is actually the size of the target. */
6925 20 : rtx size = expr_size (exp);
6926 :
6927 20 : if (CONST_INT_P (size)
6928 20 : && INTVAL (size) < TREE_STRING_LENGTH (exp))
6929 0 : emit_block_move (target, temp, size,
6930 : (call_param_p
6931 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
6932 : else
6933 : {
6934 20 : machine_mode pointer_mode
6935 20 : = targetm.addr_space.pointer_mode (MEM_ADDR_SPACE (target));
6936 20 : machine_mode address_mode = get_address_mode (target);
6937 :
6938 : /* Compute the size of the data to copy from the string. */
6939 20 : tree copy_size
6940 20 : = size_binop_loc (loc, MIN_EXPR,
6941 : make_tree (sizetype, size),
6942 20 : size_int (TREE_STRING_LENGTH (exp)));
6943 20 : rtx copy_size_rtx
6944 20 : = expand_expr (copy_size, NULL_RTX, VOIDmode,
6945 : (call_param_p
6946 : ? EXPAND_STACK_PARM : EXPAND_NORMAL));
6947 20 : rtx_code_label *label = 0;
6948 :
6949 : /* Copy that much. */
6950 60 : copy_size_rtx = convert_to_mode (pointer_mode, copy_size_rtx,
6951 20 : TYPE_UNSIGNED (sizetype));
6952 40 : emit_block_move (target, temp, copy_size_rtx,
6953 : (call_param_p
6954 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
6955 :
6956 : /* Figure out how much is left in TARGET that we have to clear.
6957 : Do all calculations in pointer_mode. */
6958 20 : poly_int64 const_copy_size;
6959 20 : if (poly_int_rtx_p (copy_size_rtx, &const_copy_size))
6960 : {
6961 20 : size = plus_constant (address_mode, size, -const_copy_size);
6962 20 : target = adjust_address (target, BLKmode, const_copy_size);
6963 : }
6964 : else
6965 : {
6966 0 : size = expand_binop (TYPE_MODE (sizetype), sub_optab, size,
6967 : copy_size_rtx, NULL_RTX, 0,
6968 : OPTAB_LIB_WIDEN);
6969 :
6970 0 : if (GET_MODE (copy_size_rtx) != address_mode)
6971 0 : copy_size_rtx = convert_to_mode (address_mode,
6972 : copy_size_rtx,
6973 0 : TYPE_UNSIGNED (sizetype));
6974 :
6975 0 : target = offset_address (target, copy_size_rtx,
6976 : highest_pow2_factor (copy_size));
6977 0 : label = gen_label_rtx ();
6978 0 : emit_cmp_and_jump_insns (size, const0_rtx, LT, NULL_RTX,
6979 0 : GET_MODE (size), 0, label);
6980 : }
6981 :
6982 20 : if (size != const0_rtx)
6983 2 : clear_storage (target, size, BLOCK_OP_NORMAL);
6984 :
6985 20 : if (label)
6986 0 : emit_label (label);
6987 : }
6988 : }
6989 14509939 : else if (shortened_string_cst)
6990 0 : gcc_unreachable ();
6991 : /* Handle calls that return values in multiple non-contiguous locations.
6992 : The Irix 6 ABI has examples of this. */
6993 14509939 : else if (GET_CODE (target) == PARALLEL)
6994 : {
6995 0 : if (GET_CODE (temp) == PARALLEL)
6996 0 : emit_group_move (target, temp);
6997 : else
6998 0 : emit_group_load (target, temp, TREE_TYPE (exp),
6999 0 : int_size_in_bytes (TREE_TYPE (exp)));
7000 : }
7001 14509939 : else if (GET_CODE (temp) == PARALLEL)
7002 0 : emit_group_store (target, temp, TREE_TYPE (exp),
7003 0 : int_size_in_bytes (TREE_TYPE (exp)));
7004 14509939 : else if (GET_MODE (temp) == BLKmode)
7005 605140 : emit_block_move (target, temp, expr_size (exp),
7006 : (call_param_p
7007 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
7008 : /* If we emit a nontemporal store, there is nothing else to do. */
7009 14207369 : else if (nontemporal && emit_storent_insn (target, temp))
7010 : ;
7011 : else
7012 : {
7013 14207352 : if (reverse)
7014 734 : temp = flip_storage_order (GET_MODE (target), temp);
7015 14207352 : temp = force_operand (temp, target);
7016 14207352 : if (temp != target)
7017 14205898 : emit_move_insn (target, temp);
7018 : }
7019 : }
7020 : else
7021 3151789 : gcc_assert (!shortened_string_cst);
7022 :
7023 : return NULL_RTX;
7024 : }
7025 :
7026 : /* Return true if field F of structure TYPE is a flexible array. */
7027 :
7028 : static bool
7029 3952258 : flexible_array_member_p (const_tree f, const_tree type)
7030 : {
7031 3952258 : const_tree tf;
7032 :
7033 3952258 : tf = TREE_TYPE (f);
7034 3952258 : return (DECL_CHAIN (f) == NULL
7035 1082737 : && TREE_CODE (tf) == ARRAY_TYPE
7036 2437 : && TYPE_DOMAIN (tf)
7037 2437 : && TYPE_MIN_VALUE (TYPE_DOMAIN (tf))
7038 2437 : && integer_zerop (TYPE_MIN_VALUE (TYPE_DOMAIN (tf)))
7039 2437 : && !TYPE_MAX_VALUE (TYPE_DOMAIN (tf))
7040 3952297 : && int_size_in_bytes (type) >= 0);
7041 : }
7042 :
7043 : /* If FOR_CTOR_P, return the number of top-level elements that a constructor
7044 : must have in order for it to completely initialize a value of type TYPE.
7045 : Return -1 if the number isn't known.
7046 :
7047 : If !FOR_CTOR_P, return an estimate of the number of scalars in TYPE. */
7048 :
7049 : static HOST_WIDE_INT
7050 3733794 : count_type_elements (const_tree type, bool for_ctor_p)
7051 : {
7052 3735147 : switch (TREE_CODE (type))
7053 : {
7054 155438 : case ARRAY_TYPE:
7055 155438 : {
7056 155438 : tree nelts_minus_one;
7057 :
7058 155438 : nelts_minus_one = array_type_nelts_minus_one (type);
7059 155438 : if (nelts_minus_one && tree_fits_uhwi_p (nelts_minus_one))
7060 : {
7061 155427 : unsigned HOST_WIDE_INT n;
7062 :
7063 155427 : n = tree_to_uhwi (nelts_minus_one) + 1;
7064 155427 : if (n == 0 || for_ctor_p)
7065 154074 : return n;
7066 : else
7067 1353 : return n * count_type_elements (TREE_TYPE (type), false);
7068 : }
7069 11 : return for_ctor_p ? -1 : 1;
7070 : }
7071 :
7072 1827076 : case RECORD_TYPE:
7073 1827076 : {
7074 1827076 : unsigned HOST_WIDE_INT n;
7075 1827076 : tree f;
7076 :
7077 1827076 : n = 0;
7078 17426263 : for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
7079 15599187 : if (TREE_CODE (f) == FIELD_DECL)
7080 : {
7081 4416463 : if (!for_ctor_p)
7082 464205 : n += count_type_elements (TREE_TYPE (f), false);
7083 3952258 : else if (!flexible_array_member_p (f, type))
7084 : /* Don't count flexible arrays, which are not supposed
7085 : to be initialized. */
7086 3952219 : n += 1;
7087 : }
7088 :
7089 1827076 : return n;
7090 : }
7091 :
7092 3393 : case UNION_TYPE:
7093 3393 : case QUAL_UNION_TYPE:
7094 3393 : {
7095 3393 : tree f;
7096 3393 : HOST_WIDE_INT n, m;
7097 :
7098 3393 : gcc_assert (!for_ctor_p);
7099 : /* Estimate the number of scalars in each field and pick the
7100 : maximum. Other estimates would do instead; the idea is simply
7101 : to make sure that the estimate is not sensitive to the ordering
7102 : of the fields. */
7103 3393 : n = 1;
7104 77132 : for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
7105 73739 : if (TREE_CODE (f) == FIELD_DECL)
7106 : {
7107 69219 : m = count_type_elements (TREE_TYPE (f), false);
7108 : /* If the field doesn't span the whole union, add an extra
7109 : scalar for the rest. */
7110 69219 : if (simple_cst_equal (TYPE_SIZE (TREE_TYPE (f)),
7111 69219 : TYPE_SIZE (type)) != 1)
7112 49335 : m++;
7113 69219 : if (n < m)
7114 73739 : n = m;
7115 : }
7116 : return n;
7117 : }
7118 :
7119 : case COMPLEX_TYPE:
7120 : return 2;
7121 :
7122 422 : case VECTOR_TYPE:
7123 422 : {
7124 422 : unsigned HOST_WIDE_INT nelts;
7125 422 : if (TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts))
7126 422 : return nelts;
7127 : else
7128 : return -1;
7129 : }
7130 :
7131 : case INTEGER_TYPE:
7132 : case REAL_TYPE:
7133 : case FIXED_POINT_TYPE:
7134 : case ENUMERAL_TYPE:
7135 : case BOOLEAN_TYPE:
7136 : case POINTER_TYPE:
7137 : case OFFSET_TYPE:
7138 : case REFERENCE_TYPE:
7139 : /* This could represent the C++ std::meta::info type. */
7140 : case LANG_TYPE:
7141 : case NULLPTR_TYPE:
7142 : case OPAQUE_TYPE:
7143 : case BITINT_TYPE:
7144 : return 1;
7145 :
7146 0 : case ERROR_MARK:
7147 0 : return 0;
7148 :
7149 0 : case VOID_TYPE:
7150 0 : case METHOD_TYPE:
7151 0 : case FUNCTION_TYPE:
7152 0 : default:
7153 0 : gcc_unreachable ();
7154 : }
7155 : }
7156 :
7157 : /* Helper for categorize_ctor_elements. Identical interface. */
7158 :
7159 : static bool
7160 1553447 : categorize_ctor_elements_1 (const_tree ctor, HOST_WIDE_INT *p_nz_elts,
7161 : HOST_WIDE_INT *p_unique_nz_elts,
7162 : HOST_WIDE_INT *p_init_elts,
7163 : ctor_completeness *p_complete)
7164 : {
7165 1553447 : unsigned HOST_WIDE_INT idx;
7166 1553447 : HOST_WIDE_INT nz_elts, unique_nz_elts, init_elts, num_fields;
7167 1553447 : tree value, purpose, elt_type;
7168 :
7169 : /* Whether CTOR is a valid constant initializer, in accordance with what
7170 : initializer_constant_valid_p does. If inferred from the constructor
7171 : elements, true until proven otherwise. */
7172 1553447 : bool const_from_elts_p = constructor_static_from_elts_p (ctor);
7173 1553447 : bool const_p = const_from_elts_p ? true : TREE_STATIC (ctor);
7174 :
7175 1553447 : nz_elts = 0;
7176 1553447 : unique_nz_elts = 0;
7177 1553447 : init_elts = 0;
7178 1553447 : num_fields = 0;
7179 1553447 : elt_type = NULL_TREE;
7180 :
7181 5883548 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (ctor), idx, purpose, value)
7182 : {
7183 4330101 : HOST_WIDE_INT mult = 1;
7184 :
7185 4330101 : if (purpose && TREE_CODE (purpose) == RANGE_EXPR)
7186 : {
7187 855 : tree lo_index = TREE_OPERAND (purpose, 0);
7188 855 : tree hi_index = TREE_OPERAND (purpose, 1);
7189 :
7190 855 : if (tree_fits_uhwi_p (lo_index) && tree_fits_uhwi_p (hi_index))
7191 855 : mult = (tree_to_uhwi (hi_index)
7192 855 : - tree_to_uhwi (lo_index) + 1);
7193 : }
7194 4330101 : num_fields += mult;
7195 4330101 : elt_type = TREE_TYPE (value);
7196 :
7197 4330101 : switch (TREE_CODE (value))
7198 : {
7199 546176 : case CONSTRUCTOR:
7200 546176 : {
7201 546176 : HOST_WIDE_INT nz = 0, unz = 0, ic = 0;
7202 :
7203 546176 : bool const_elt_p = categorize_ctor_elements_1 (value, &nz, &unz,
7204 : &ic, p_complete);
7205 :
7206 546176 : nz_elts += mult * nz;
7207 546176 : unique_nz_elts += unz;
7208 546176 : init_elts += mult * ic;
7209 :
7210 546176 : if (const_from_elts_p && const_p)
7211 332909 : const_p = const_elt_p;
7212 : }
7213 546176 : break;
7214 :
7215 2315739 : case INTEGER_CST:
7216 2315739 : case REAL_CST:
7217 2315739 : case FIXED_CST:
7218 2315739 : if (!initializer_zerop (value))
7219 : {
7220 1692865 : nz_elts += mult;
7221 1692865 : unique_nz_elts++;
7222 : }
7223 2315739 : init_elts += mult;
7224 2315739 : break;
7225 :
7226 3772 : case STRING_CST:
7227 3772 : {
7228 3772 : HOST_WIDE_INT len = TREE_STRING_LENGTH (value);
7229 3772 : nz_elts += mult * len;
7230 3772 : unique_nz_elts += len;
7231 : /* Count implicit trailing zeros as well, so that the density
7232 : heuristic can prefer clearing to copying a mostly-zero
7233 : aggregate from read-only data. */
7234 3772 : if (tree_fits_shwi_p (TYPE_SIZE_UNIT (elt_type)))
7235 3772 : len = MAX (len, tree_to_shwi (TYPE_SIZE_UNIT (elt_type)));
7236 3772 : init_elts += mult * len;
7237 : }
7238 3772 : break;
7239 :
7240 166 : case RAW_DATA_CST:
7241 166 : nz_elts += mult * RAW_DATA_LENGTH (value);
7242 166 : unique_nz_elts += RAW_DATA_LENGTH (value);
7243 166 : init_elts += mult * RAW_DATA_LENGTH (value);
7244 166 : num_fields += mult * (RAW_DATA_LENGTH (value) - 1);
7245 166 : break;
7246 :
7247 3234 : case COMPLEX_CST:
7248 3234 : if (!initializer_zerop (TREE_REALPART (value)))
7249 : {
7250 2630 : nz_elts += mult;
7251 2630 : unique_nz_elts++;
7252 : }
7253 3234 : if (!initializer_zerop (TREE_IMAGPART (value)))
7254 : {
7255 2272 : nz_elts += mult;
7256 2272 : unique_nz_elts++;
7257 : }
7258 3234 : init_elts += 2 * mult;
7259 3234 : break;
7260 :
7261 6420 : case VECTOR_CST:
7262 6420 : {
7263 6420 : unsigned int nunits
7264 : = constant_lower_bound
7265 6420 : (TYPE_VECTOR_SUBPARTS (TREE_TYPE (value)));
7266 39931 : for (unsigned int i = 0; i < nunits; ++i)
7267 : {
7268 33511 : tree v = VECTOR_CST_ELT (value, i);
7269 33511 : if (!initializer_zerop (v))
7270 : {
7271 19057 : nz_elts += mult;
7272 19057 : unique_nz_elts++;
7273 : }
7274 33511 : init_elts += mult;
7275 : }
7276 : }
7277 : break;
7278 :
7279 1454594 : default:
7280 1454594 : {
7281 1454594 : HOST_WIDE_INT tc = count_type_elements (elt_type, false);
7282 1454594 : nz_elts += mult * tc;
7283 1454594 : unique_nz_elts += tc;
7284 1454594 : init_elts += mult * tc;
7285 :
7286 1454594 : if (const_from_elts_p && const_p)
7287 600161 : const_p
7288 600161 : = initializer_constant_valid_p (value,
7289 : elt_type,
7290 600161 : TYPE_REVERSE_STORAGE_ORDER
7291 : (TREE_TYPE (ctor)))
7292 : != NULL_TREE;
7293 : }
7294 : break;
7295 : }
7296 : }
7297 :
7298 1553447 : if (!p_complete->sparse && !complete_ctor_at_level_p (TREE_TYPE (ctor),
7299 : num_fields, elt_type))
7300 188571 : p_complete->sparse = true;
7301 1364876 : else if (TREE_CODE (TREE_TYPE (ctor)) == UNION_TYPE
7302 1364876 : || TREE_CODE (TREE_TYPE (ctor)) == QUAL_UNION_TYPE)
7303 : {
7304 12750 : if (!p_complete->sparse
7305 8244 : && CONSTRUCTOR_ZERO_PADDING_BITS (ctor)
7306 22871 : && (num_fields
7307 5062 : ? simple_cst_equal (TYPE_SIZE (TREE_TYPE (ctor)),
7308 5059 : TYPE_SIZE (elt_type)) != 1
7309 3 : : type_has_padding_at_level_p (TREE_TYPE (ctor))))
7310 3367 : p_complete->sparse = true;
7311 4877 : else if (!p_complete->sparse && !p_complete->padded_union
7312 19129 : && (num_fields
7313 4873 : ? simple_cst_equal (TYPE_SIZE (TREE_TYPE (ctor)),
7314 4873 : TYPE_SIZE (elt_type)) != 1
7315 0 : : type_has_padding_at_level_p (TREE_TYPE (ctor))))
7316 644 : p_complete->padded_union = true;
7317 : }
7318 1352126 : else if (!p_complete->sparse
7319 1233692 : && (CONSTRUCTOR_ZERO_PADDING_BITS (ctor)
7320 1214257 : || flag_zero_init_padding_bits == ZERO_INIT_PADDING_BITS_ALL)
7321 1371590 : && type_has_padding_at_level_p (TREE_TYPE (ctor)))
7322 2497 : p_complete->sparse = true;
7323 1231195 : else if (!p_complete->sparse && !p_complete->padded_non_union
7324 2534282 : && type_has_padding_at_level_p (TREE_TYPE (ctor)))
7325 15623 : p_complete->padded_non_union = true;
7326 :
7327 1553447 : *p_nz_elts += nz_elts;
7328 1553447 : *p_unique_nz_elts += unique_nz_elts;
7329 1553447 : *p_init_elts += init_elts;
7330 :
7331 1553447 : return const_p;
7332 : }
7333 :
7334 : /* Examine CTOR to discover:
7335 : * how many scalar fields are set to nonzero values,
7336 : and place it in *P_NZ_ELTS;
7337 : * the same, but counting RANGE_EXPRs as multiplier of 1 instead of
7338 : high - low + 1 (this can be useful for callers to determine ctors
7339 : that could be cheaply initialized with - perhaps nested - loops
7340 : compared to copied from huge read-only data),
7341 : and place it in *P_UNIQUE_NZ_ELTS;
7342 : * how many scalar fields in total are in CTOR,
7343 : and place it in *P_ELT_COUNT.
7344 : * whether the constructor is complete -- in the sense that every
7345 : meaningful byte is explicitly given a value --
7346 : and place it in *P_COMPLETE:
7347 :
7348 : Return whether or not CTOR is a valid static constant initializer, the same
7349 : as "initializer_constant_valid_p (CTOR, TREE_TYPE (CTOR)) != 0". */
7350 :
7351 : bool
7352 1007271 : categorize_ctor_elements (const_tree ctor, HOST_WIDE_INT *p_nz_elts,
7353 : HOST_WIDE_INT *p_unique_nz_elts,
7354 : HOST_WIDE_INT *p_init_elts,
7355 : ctor_completeness *p_complete)
7356 : {
7357 1007271 : *p_nz_elts = 0;
7358 1007271 : *p_unique_nz_elts = 0;
7359 1007271 : *p_init_elts = 0;
7360 1007271 : *p_complete = {};
7361 :
7362 1007271 : return categorize_ctor_elements_1 (ctor, p_nz_elts, p_unique_nz_elts,
7363 1007271 : p_init_elts, p_complete);
7364 : }
7365 :
7366 : /* Return true if constructor CTOR is simple enough to be materialized
7367 : in an integer mode register. Limit the size to WORDS words, which
7368 : is 1 by default. */
7369 :
7370 : bool
7371 24514 : immediate_const_ctor_p (const_tree ctor, unsigned int words)
7372 : {
7373 : /* Allow function to be called with a VAR_DECL's DECL_INITIAL. */
7374 24514 : if (!ctor || TREE_CODE (ctor) != CONSTRUCTOR)
7375 : return false;
7376 :
7377 2773 : return TREE_CONSTANT (ctor)
7378 2773 : && !TREE_ADDRESSABLE (ctor)
7379 2715 : && CONSTRUCTOR_NELTS (ctor)
7380 2689 : && TREE_CODE (TREE_TYPE (ctor)) != ARRAY_TYPE
7381 634 : && int_expr_size (ctor) <= words * UNITS_PER_WORD
7382 2962 : && initializer_constant_valid_for_bitfield_p (ctor);
7383 : }
7384 :
7385 : /* TYPE is initialized by a constructor with NUM_ELTS elements, the last
7386 : of which had type LAST_TYPE. Each element was itself a complete
7387 : initializer, in the sense that every meaningful byte was explicitly
7388 : given a value. Return true if the same is true for the constructor
7389 : as a whole. */
7390 :
7391 : bool
7392 1755244 : complete_ctor_at_level_p (const_tree type, HOST_WIDE_INT num_elts,
7393 : const_tree last_type)
7394 : {
7395 1755244 : if (TREE_CODE (type) == UNION_TYPE || TREE_CODE (type) == QUAL_UNION_TYPE)
7396 : {
7397 9468 : if (num_elts == 0)
7398 : {
7399 24 : if (flag_zero_init_padding_bits >= ZERO_INIT_PADDING_BITS_UNIONS)
7400 : return false;
7401 :
7402 : /* If the CONSTRUCTOR doesn't have any elts, it is
7403 : incomplete if the union has at least one field. */
7404 327 : for (tree f = TYPE_FIELDS (type); f; f = DECL_CHAIN (f))
7405 324 : if (TREE_CODE (f) == FIELD_DECL)
7406 : return false;
7407 :
7408 : return true;
7409 : }
7410 :
7411 9444 : gcc_assert (num_elts == 1 && last_type);
7412 :
7413 9444 : if (flag_zero_init_padding_bits >= ZERO_INIT_PADDING_BITS_UNIONS)
7414 : /* ??? We could look at each element of the union, and find the
7415 : largest element. Which would avoid comparing the size of the
7416 : initialized element against any tail padding in the union.
7417 : Doesn't seem worth the effort... */
7418 102 : return simple_cst_equal (TYPE_SIZE (type), TYPE_SIZE (last_type)) == 1;
7419 :
7420 : return true;
7421 : }
7422 :
7423 1745776 : return count_type_elements (type, true) == num_elts;
7424 : }
7425 :
7426 : /* Return true if EXP contains mostly (3/4) zeros. */
7427 :
7428 : static bool
7429 369310 : mostly_zeros_p (const_tree exp)
7430 : {
7431 369310 : if (TREE_CODE (exp) == CONSTRUCTOR)
7432 : {
7433 197 : HOST_WIDE_INT nz_elts, unz_elts, init_elts;
7434 197 : ctor_completeness complete_p;
7435 :
7436 197 : categorize_ctor_elements (exp, &nz_elts, &unz_elts, &init_elts,
7437 : &complete_p);
7438 197 : return complete_p.sparse || nz_elts < init_elts / 4;
7439 : }
7440 :
7441 369113 : return initializer_zerop (exp);
7442 : }
7443 :
7444 : /* Return true if EXP contains all zeros. */
7445 :
7446 : static bool
7447 1151 : all_zeros_p (const_tree exp)
7448 : {
7449 1151 : if (TREE_CODE (exp) == CONSTRUCTOR)
7450 : {
7451 1151 : HOST_WIDE_INT nz_elts, unz_elts, init_elts;
7452 1151 : ctor_completeness complete_p;
7453 :
7454 1151 : categorize_ctor_elements (exp, &nz_elts, &unz_elts, &init_elts,
7455 : &complete_p);
7456 1151 : return nz_elts == 0;
7457 : }
7458 :
7459 0 : return initializer_zerop (exp);
7460 : }
7461 :
7462 : /* Helper function for store_constructor.
7463 : TARGET, BITSIZE, BITPOS, MODE, EXP are as for store_field.
7464 : CLEARED is as for store_constructor.
7465 : ALIAS_SET is the alias set to use for any stores.
7466 : If REVERSE is true, the store is to be done in reverse order.
7467 :
7468 : This provides a recursive shortcut back to store_constructor when it isn't
7469 : necessary to go through store_field. This is so that we can pass through
7470 : the cleared field to let store_constructor know that we may not have to
7471 : clear a substructure if the outer structure has already been cleared. */
7472 :
7473 : static void
7474 37006 : store_constructor_field (rtx target, poly_uint64 bitsize, poly_int64 bitpos,
7475 : poly_uint64 bitregion_start,
7476 : poly_uint64 bitregion_end,
7477 : machine_mode mode,
7478 : tree exp, int cleared,
7479 : alias_set_type alias_set, bool reverse)
7480 : {
7481 37006 : poly_int64 bytepos;
7482 37006 : poly_uint64 bytesize;
7483 37006 : if (TREE_CODE (exp) == CONSTRUCTOR
7484 : /* We can only call store_constructor recursively if the size and
7485 : bit position are on a byte boundary. */
7486 37006 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
7487 82 : && maybe_ne (bitsize, 0U)
7488 36924 : && multiple_p (bitsize, BITS_PER_UNIT, &bytesize)
7489 : /* If we have a nonzero bitpos for a register target, then we just
7490 : let store_field do the bitfield handling. This is unlikely to
7491 : generate unnecessary clear instructions anyways. */
7492 37088 : && (known_eq (bitpos, 0) || MEM_P (target)))
7493 : {
7494 82 : if (MEM_P (target))
7495 : {
7496 40 : machine_mode target_mode = GET_MODE (target);
7497 40 : if (target_mode != BLKmode
7498 40 : && !multiple_p (bitpos, GET_MODE_ALIGNMENT (target_mode)))
7499 : target_mode = BLKmode;
7500 40 : target = adjust_address (target, target_mode, bytepos);
7501 : }
7502 :
7503 :
7504 : /* Update the alias set, if required. */
7505 40 : if (MEM_P (target) && ! MEM_KEEP_ALIAS_SET_P (target)
7506 122 : && MEM_ALIAS_SET (target) != 0)
7507 : {
7508 36 : target = copy_rtx (target);
7509 36 : set_mem_alias_set (target, alias_set);
7510 : }
7511 :
7512 82 : store_constructor (exp, target, cleared, bytesize, reverse);
7513 : }
7514 : else
7515 36924 : store_field (target, bitsize, bitpos, bitregion_start, bitregion_end, mode,
7516 : exp, alias_set, false, reverse);
7517 37006 : }
7518 :
7519 :
7520 : /* Returns the number of FIELD_DECLs in TYPE. */
7521 :
7522 : static int
7523 70395 : fields_length (const_tree type)
7524 : {
7525 70395 : tree t = TYPE_FIELDS (type);
7526 70395 : int count = 0;
7527 :
7528 962356 : for (; t; t = DECL_CHAIN (t))
7529 891961 : if (TREE_CODE (t) == FIELD_DECL)
7530 305968 : ++count;
7531 :
7532 70395 : return count;
7533 : }
7534 :
7535 : /* Store the value of constructor EXP into the rtx TARGET.
7536 : TARGET is either a REG or a MEM; we know it cannot conflict, since
7537 : safe_from_p has been called.
7538 : CLEARED is true if TARGET is known to have been zero'd.
7539 : If the constructor EXP has a vector type then elements of TARGET for which
7540 : there is no corresponding element in EXP are zero'd. For a variable-length
7541 : vector type, only elements up to the minimum number of subparts of the type
7542 : are explicitly zero'd; any elements beyond that are implicitly zero.
7543 : SIZE is the number of bytes of TARGET we are allowed to modify: this
7544 : may not be the same as the size of EXP if we are assigning to a field
7545 : which has been packed to exclude padding bits.
7546 : If REVERSE is true, the store is to be done in reverse order. */
7547 :
7548 : void
7549 251460 : store_constructor (tree exp, rtx target, int cleared, poly_int64 size,
7550 : bool reverse)
7551 : {
7552 251460 : tree type = TREE_TYPE (exp);
7553 251460 : HOST_WIDE_INT exp_size = int_size_in_bytes (type);
7554 251460 : poly_int64 bitregion_end = known_gt (size, 0) ? size * BITS_PER_UNIT - 1 : 0;
7555 :
7556 251460 : switch (TREE_CODE (type))
7557 : {
7558 71275 : case RECORD_TYPE:
7559 71275 : case UNION_TYPE:
7560 71275 : case QUAL_UNION_TYPE:
7561 71275 : {
7562 71275 : unsigned HOST_WIDE_INT idx;
7563 71275 : tree field, value;
7564 :
7565 : /* The storage order is specified for every aggregate type. */
7566 71275 : reverse = TYPE_REVERSE_STORAGE_ORDER (type);
7567 :
7568 : /* If size is zero or the target is already cleared, do nothing. */
7569 71275 : if (known_eq (size, 0) || cleared)
7570 : cleared = 1;
7571 : /* We either clear the aggregate or indicate the value is dead. */
7572 71275 : else if ((TREE_CODE (type) == UNION_TYPE
7573 71275 : || TREE_CODE (type) == QUAL_UNION_TYPE)
7574 71970 : && ! CONSTRUCTOR_ELTS (exp))
7575 : /* If the constructor is empty, clear the union. */
7576 : {
7577 646 : clear_storage (target, expr_size (exp), BLOCK_OP_NORMAL);
7578 646 : cleared = 1;
7579 : }
7580 :
7581 : /* If we are building a static constructor into a register,
7582 : set the initial value as zero so we can fold the value into
7583 : a constant. But if more than one register is involved,
7584 : this probably loses. */
7585 4307 : else if (REG_P (target) && TREE_STATIC (exp)
7586 71219 : && known_le (GET_MODE_SIZE (GET_MODE (target)),
7587 : REGMODE_NATURAL_SIZE (GET_MODE (target))))
7588 : {
7589 234 : emit_move_insn (target, CONST0_RTX (GET_MODE (target)));
7590 234 : cleared = 1;
7591 : }
7592 :
7593 : /* If the constructor has fewer fields than the structure or
7594 : if we are initializing the structure to mostly zeros, clear
7595 : the whole structure first. Don't do this if TARGET is a
7596 : register whose mode size isn't equal to SIZE since
7597 : clear_storage can't handle this case. */
7598 70395 : else if (known_size_p (size)
7599 70564 : && (((int) CONSTRUCTOR_NELTS (exp) != fields_length (type))
7600 156 : || mostly_zeros_p (exp))
7601 140638 : && (!REG_P (target)
7602 8136 : || known_eq (GET_MODE_SIZE (GET_MODE (target)), size)))
7603 : {
7604 80978 : clear_storage (target, gen_int_mode (size, Pmode),
7605 : BLOCK_OP_NORMAL);
7606 70243 : cleared = 1;
7607 : }
7608 :
7609 71275 : if (REG_P (target) && !cleared)
7610 5 : emit_clobber (target);
7611 :
7612 : /* Store each element of the constructor into the
7613 : corresponding field of TARGET. */
7614 71621 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (exp), idx, field, value)
7615 : {
7616 346 : machine_mode mode;
7617 346 : HOST_WIDE_INT bitsize;
7618 346 : HOST_WIDE_INT bitpos = 0;
7619 346 : tree offset;
7620 346 : rtx to_rtx = target;
7621 :
7622 : /* Just ignore missing fields. We cleared the whole
7623 : structure, above, if any fields are missing. */
7624 346 : if (field == 0)
7625 346 : continue;
7626 :
7627 346 : if (cleared && initializer_zerop (value))
7628 17 : continue;
7629 :
7630 : /* Variable sized arrays are ignored. */
7631 329 : tree decl_size = DECL_SIZE (field);
7632 329 : if (!decl_size)
7633 4 : continue;
7634 :
7635 325 : if (tree_fits_uhwi_p (decl_size))
7636 325 : bitsize = tree_to_uhwi (decl_size);
7637 : else
7638 0 : gcc_unreachable ();
7639 :
7640 325 : mode = DECL_MODE (field);
7641 325 : if (DECL_BIT_FIELD (field))
7642 24 : mode = VOIDmode;
7643 :
7644 325 : offset = DECL_FIELD_OFFSET (field);
7645 325 : if (tree_fits_shwi_p (offset)
7646 325 : && tree_fits_shwi_p (bit_position (field)))
7647 : {
7648 325 : bitpos = int_bit_position (field);
7649 325 : offset = NULL_TREE;
7650 : }
7651 : else
7652 0 : gcc_unreachable ();
7653 :
7654 : /* If this initializes a field that is smaller than a
7655 : word, at the start of a word, try to widen it to a full
7656 : word. This special case allows us to output C++ member
7657 : function initializations in a form that the optimizers
7658 : can understand. */
7659 325 : if (WORD_REGISTER_OPERATIONS
7660 : && REG_P (target)
7661 : && bitsize < BITS_PER_WORD
7662 : && bitpos % BITS_PER_WORD == 0
7663 : && GET_MODE_CLASS (mode) == MODE_INT
7664 : && TREE_CODE (value) == INTEGER_CST
7665 : && exp_size >= 0
7666 : && bitpos + BITS_PER_WORD <= exp_size * BITS_PER_UNIT)
7667 : {
7668 : type = TREE_TYPE (value);
7669 :
7670 : if (TYPE_PRECISION (type) < BITS_PER_WORD)
7671 : {
7672 : type = lang_hooks.types.type_for_mode
7673 : (word_mode, TYPE_UNSIGNED (type));
7674 : value = fold_convert (type, value);
7675 : /* Make sure the bits beyond the original bitsize are zero
7676 : so that we can correctly avoid extra zeroing stores in
7677 : later constructor elements. */
7678 : tree bitsize_mask
7679 : = wide_int_to_tree (type, wi::mask (bitsize, false,
7680 : BITS_PER_WORD));
7681 : value = fold_build2 (BIT_AND_EXPR, type, value, bitsize_mask);
7682 : }
7683 :
7684 : if (BYTES_BIG_ENDIAN)
7685 : value
7686 : = fold_build2 (LSHIFT_EXPR, type, value,
7687 : build_int_cst (type,
7688 : BITS_PER_WORD - bitsize));
7689 : bitsize = BITS_PER_WORD;
7690 : mode = word_mode;
7691 : }
7692 :
7693 207 : if (MEM_P (to_rtx) && !MEM_KEEP_ALIAS_SET_P (to_rtx)
7694 532 : && DECL_NONADDRESSABLE_P (field))
7695 : {
7696 0 : to_rtx = copy_rtx (to_rtx);
7697 0 : MEM_KEEP_ALIAS_SET_P (to_rtx) = 1;
7698 : }
7699 :
7700 325 : store_constructor_field (to_rtx, bitsize, bitpos,
7701 325 : 0, bitregion_end, mode,
7702 : value, cleared,
7703 325 : get_alias_set (TREE_TYPE (field)),
7704 : reverse);
7705 : }
7706 : break;
7707 : }
7708 39581 : case ARRAY_TYPE:
7709 39581 : {
7710 39581 : tree value, index;
7711 39581 : unsigned HOST_WIDE_INT i, j = 0;
7712 39581 : bool need_to_clear;
7713 39581 : tree domain;
7714 39581 : tree elttype = TREE_TYPE (type);
7715 39581 : bool const_bounds_p;
7716 39581 : unsigned HOST_WIDE_INT minelt = 0;
7717 39581 : unsigned HOST_WIDE_INT maxelt = 0;
7718 :
7719 : /* The storage order is specified for every aggregate type. */
7720 39581 : reverse = TYPE_REVERSE_STORAGE_ORDER (type);
7721 :
7722 39581 : domain = TYPE_DOMAIN (type);
7723 39581 : const_bounds_p = (TYPE_MIN_VALUE (domain)
7724 39581 : && TYPE_MAX_VALUE (domain)
7725 39581 : && tree_fits_uhwi_p (TYPE_MIN_VALUE (domain))
7726 79162 : && tree_fits_uhwi_p (TYPE_MAX_VALUE (domain)));
7727 :
7728 : /* If we have constant bounds for the range of the type, get them. */
7729 39581 : if (const_bounds_p)
7730 : {
7731 39581 : minelt = tree_to_uhwi (TYPE_MIN_VALUE (domain));
7732 39581 : maxelt = tree_to_uhwi (TYPE_MAX_VALUE (domain));
7733 : }
7734 :
7735 : /* If the constructor has fewer elements than the array, clear
7736 : the whole array first. Similarly if this is static
7737 : constructor of a non-BLKmode object. */
7738 39581 : if (cleared)
7739 : need_to_clear = false;
7740 39535 : else if (REG_P (target) && TREE_STATIC (exp))
7741 : need_to_clear = true;
7742 : else
7743 : {
7744 39428 : unsigned HOST_WIDE_INT idx;
7745 39428 : unsigned HOST_WIDE_INT count = 0, zero_count = 0;
7746 39428 : need_to_clear = ! const_bounds_p;
7747 :
7748 : /* This loop is a more accurate version of the loop in
7749 : mostly_zeros_p (it handles RANGE_EXPR in an index). It
7750 : is also needed to check for missing elements. */
7751 39436 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (exp), idx, index, value)
7752 : {
7753 8 : unsigned HOST_WIDE_INT this_node_count;
7754 :
7755 8 : if (need_to_clear)
7756 : break;
7757 :
7758 8 : if (index != NULL_TREE && TREE_CODE (index) == RANGE_EXPR)
7759 : {
7760 0 : tree lo_index = TREE_OPERAND (index, 0);
7761 0 : tree hi_index = TREE_OPERAND (index, 1);
7762 :
7763 0 : if (! tree_fits_uhwi_p (lo_index)
7764 0 : || ! tree_fits_uhwi_p (hi_index))
7765 : {
7766 : need_to_clear = true;
7767 : break;
7768 : }
7769 :
7770 0 : this_node_count = (tree_to_uhwi (hi_index)
7771 0 : - tree_to_uhwi (lo_index) + 1);
7772 0 : }
7773 8 : else if (TREE_CODE (value) == RAW_DATA_CST)
7774 0 : this_node_count = RAW_DATA_LENGTH (value);
7775 : else
7776 : this_node_count = 1;
7777 :
7778 8 : count += this_node_count;
7779 8 : if (mostly_zeros_p (value))
7780 0 : zero_count += this_node_count;
7781 : }
7782 :
7783 : /* Clear the entire array first if there are any missing
7784 : elements, or if the incidence of zero elements is >=
7785 : 75%. */
7786 39428 : if (! need_to_clear
7787 39428 : && (count < maxelt - minelt + 1
7788 1 : || 4 * zero_count >= 3 * count))
7789 : need_to_clear = true;
7790 : }
7791 :
7792 39534 : if (need_to_clear && maybe_gt (size, 0))
7793 : {
7794 39534 : if (REG_P (target))
7795 1672 : emit_move_insn (target, CONST0_RTX (GET_MODE (target)));
7796 : else
7797 42638 : clear_storage (target, gen_int_mode (size, Pmode),
7798 : BLOCK_OP_NORMAL);
7799 : cleared = 1;
7800 : }
7801 :
7802 39581 : if (!cleared && REG_P (target))
7803 : /* Inform later passes that the old value is dead. */
7804 0 : emit_clobber (target);
7805 :
7806 : /* Store each element of the constructor into the
7807 : corresponding element of TARGET, determined by counting the
7808 : elements. */
7809 40697 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (exp), i, index, value)
7810 : {
7811 1116 : machine_mode mode;
7812 1116 : poly_int64 bitsize;
7813 1116 : HOST_WIDE_INT bitpos;
7814 1116 : rtx xtarget = target;
7815 :
7816 1116 : if (cleared && initializer_zerop (value))
7817 : {
7818 672 : if (TREE_CODE (value) == RAW_DATA_CST)
7819 0 : j += RAW_DATA_LENGTH (value) - 1;
7820 674 : continue;
7821 : }
7822 :
7823 444 : mode = TYPE_MODE (elttype);
7824 444 : if (mode != BLKmode)
7825 888 : bitsize = GET_MODE_BITSIZE (mode);
7826 0 : else if (!poly_int_tree_p (TYPE_SIZE (elttype), &bitsize))
7827 0 : bitsize = -1;
7828 :
7829 444 : if (index != NULL_TREE && TREE_CODE (index) == RANGE_EXPR)
7830 : {
7831 0 : tree lo_index = TREE_OPERAND (index, 0);
7832 0 : tree hi_index = TREE_OPERAND (index, 1);
7833 0 : rtx index_r;
7834 0 : unsigned HOST_WIDE_INT lo, hi, count;
7835 0 : tree offset;
7836 :
7837 0 : gcc_assert (TREE_CODE (value) != RAW_DATA_CST);
7838 :
7839 : /* If the range is constant and "small", unroll the loop. */
7840 0 : if (const_bounds_p
7841 0 : && tree_fits_uhwi_p (lo_index)
7842 0 : && tree_fits_uhwi_p (hi_index)
7843 0 : && (lo = tree_to_uhwi (lo_index),
7844 0 : hi = tree_to_uhwi (hi_index),
7845 0 : count = hi - lo + 1,
7846 0 : (!MEM_P (target)
7847 0 : || count <= 2
7848 0 : || (tree_fits_uhwi_p (TYPE_SIZE (elttype))
7849 0 : && (tree_to_uhwi (TYPE_SIZE (elttype)) * count
7850 : <= 40 * 8)))))
7851 : {
7852 0 : lo -= minelt; hi -= minelt;
7853 0 : for (; lo <= hi; lo++)
7854 : {
7855 0 : bitpos = lo * tree_to_uhwi (TYPE_SIZE (elttype));
7856 :
7857 0 : if (MEM_P (target)
7858 0 : && !MEM_KEEP_ALIAS_SET_P (target)
7859 0 : && TREE_CODE (type) == ARRAY_TYPE
7860 0 : && TYPE_NONALIASED_COMPONENT (type))
7861 : {
7862 0 : target = copy_rtx (target);
7863 0 : MEM_KEEP_ALIAS_SET_P (target) = 1;
7864 : }
7865 :
7866 0 : store_constructor_field
7867 0 : (target, bitsize, bitpos, 0, bitregion_end,
7868 : mode, value, cleared,
7869 : get_alias_set (elttype), reverse);
7870 : }
7871 : }
7872 : else
7873 : {
7874 0 : rtx_code_label *loop_start = gen_label_rtx ();
7875 0 : rtx_code_label *loop_end = gen_label_rtx ();
7876 0 : tree exit_cond;
7877 :
7878 0 : expand_normal (hi_index);
7879 :
7880 0 : index = build_decl (EXPR_LOCATION (exp),
7881 : VAR_DECL, NULL_TREE, domain);
7882 0 : index_r = gen_reg_rtx (promote_decl_mode (index, NULL));
7883 0 : SET_DECL_RTL (index, index_r);
7884 0 : store_expr (lo_index, index_r, 0, false, reverse);
7885 :
7886 : /* Build the head of the loop. */
7887 0 : do_pending_stack_adjust ();
7888 0 : emit_label (loop_start);
7889 :
7890 : /* Assign value to element index. */
7891 0 : offset = fold_build2 (MINUS_EXPR,
7892 : TREE_TYPE (index),
7893 : index,
7894 : TYPE_MIN_VALUE (domain));
7895 :
7896 0 : offset = size_binop (MULT_EXPR,
7897 : fold_convert (sizetype, offset),
7898 : TYPE_SIZE_UNIT (elttype));
7899 :
7900 0 : xtarget = offset_address (target,
7901 : expand_normal (offset),
7902 : highest_pow2_factor (offset));
7903 0 : xtarget = adjust_address (xtarget, mode, 0);
7904 0 : if (TREE_CODE (value) == CONSTRUCTOR)
7905 0 : store_constructor (value, xtarget, cleared,
7906 : exact_div (bitsize, BITS_PER_UNIT),
7907 : reverse);
7908 : else
7909 0 : store_expr (value, xtarget, 0, false, reverse);
7910 :
7911 : /* Generate a conditional jump to exit the loop. */
7912 0 : exit_cond = build2 (LT_EXPR, integer_type_node,
7913 : index, hi_index);
7914 0 : jumpif (exit_cond, loop_end,
7915 : profile_probability::uninitialized ());
7916 :
7917 : /* Update the loop counter, and jump to the head of
7918 : the loop. */
7919 0 : expand_assignment (index,
7920 0 : build2 (PLUS_EXPR, TREE_TYPE (index),
7921 : index, integer_one_node),
7922 : false);
7923 :
7924 0 : emit_jump (loop_start);
7925 :
7926 : /* Build the end of the loop. */
7927 0 : emit_label (loop_end);
7928 : }
7929 : }
7930 444 : else if ((index && !tree_fits_uhwi_p (index))
7931 444 : || !tree_fits_uhwi_p (TYPE_SIZE (elttype)))
7932 : {
7933 0 : tree offset;
7934 :
7935 0 : gcc_assert (TREE_CODE (value) != RAW_DATA_CST);
7936 0 : if (index)
7937 0 : offset = fold_build2 (MINUS_EXPR,
7938 : TREE_TYPE (index),
7939 : index,
7940 : TYPE_MIN_VALUE (domain));
7941 : else
7942 0 : offset = size_int (i + j);
7943 :
7944 0 : offset = size_binop (MULT_EXPR,
7945 : fold_convert (sizetype, offset),
7946 : TYPE_SIZE_UNIT (elttype));
7947 0 : xtarget = offset_address (target,
7948 : expand_normal (offset),
7949 : highest_pow2_factor (offset));
7950 0 : xtarget = adjust_address (xtarget, mode, 0);
7951 0 : store_expr (value, xtarget, 0, false, reverse);
7952 : }
7953 : else
7954 : {
7955 444 : if (index)
7956 888 : bitpos = ((tree_to_uhwi (index) - minelt)
7957 444 : * tree_to_uhwi (TYPE_SIZE (elttype)));
7958 : else
7959 0 : bitpos = ((i + j) * tree_to_uhwi (TYPE_SIZE (elttype)));
7960 :
7961 12 : if (MEM_P (target) && !MEM_KEEP_ALIAS_SET_P (target)
7962 12 : && TREE_CODE (type) == ARRAY_TYPE
7963 456 : && TYPE_NONALIASED_COMPONENT (type))
7964 : {
7965 0 : target = copy_rtx (target);
7966 0 : MEM_KEEP_ALIAS_SET_P (target) = 1;
7967 : }
7968 444 : if (TREE_CODE (value) != RAW_DATA_CST)
7969 440 : store_constructor_field (target, bitsize, bitpos, 0,
7970 440 : bitregion_end, mode, value,
7971 : cleared, get_alias_set (elttype),
7972 : reverse);
7973 : else
7974 : {
7975 4 : j += RAW_DATA_LENGTH (value) - 1;
7976 4 : gcc_assert (known_eq (bitsize, BITS_PER_UNIT));
7977 4 : rtx to_rtx = adjust_address (target, mode,
7978 : bitpos / BITS_PER_UNIT);
7979 :
7980 4 : if (to_rtx == target)
7981 0 : to_rtx = copy_rtx (to_rtx);
7982 :
7983 4 : if (!MEM_KEEP_ALIAS_SET_P (to_rtx)
7984 8 : && MEM_ALIAS_SET (to_rtx) != 0)
7985 0 : set_mem_alias_set (to_rtx, get_alias_set (elttype));
7986 :
7987 8 : if (can_store_by_pieces (RAW_DATA_LENGTH (value),
7988 : raw_data_cst_read_str,
7989 : (void *) value,
7990 4 : MEM_ALIGN (target), false))
7991 : {
7992 4 : store_by_pieces (target, RAW_DATA_LENGTH (value),
7993 : raw_data_cst_read_str, (void *) value,
7994 2 : MEM_ALIGN (target), false,
7995 : RETURN_BEGIN);
7996 2 : continue;
7997 : }
7998 :
7999 2 : elttype
8000 2 : = build_array_type_nelts (TREE_TYPE (value),
8001 2 : RAW_DATA_LENGTH (value));
8002 2 : tree ctor = build_constructor_single (elttype, NULL_TREE,
8003 : value);
8004 2 : ctor = tree_output_constant_def (ctor);
8005 2 : mode = TYPE_MODE (type);
8006 2 : store_constructor_field (target,
8007 2 : bitsize * RAW_DATA_LENGTH (value),
8008 2 : bitpos, 0, bitregion_end, mode,
8009 : ctor, cleared,
8010 : get_alias_set (elttype), reverse);
8011 : }
8012 : }
8013 : }
8014 : break;
8015 : }
8016 :
8017 140604 : case VECTOR_TYPE:
8018 140604 : {
8019 140604 : unsigned HOST_WIDE_INT idx;
8020 140604 : constructor_elt *ce;
8021 140604 : bool need_to_clear;
8022 140604 : insn_code icode = CODE_FOR_nothing;
8023 140604 : tree elt;
8024 140604 : tree elttype = TREE_TYPE (type);
8025 140604 : int elt_size = vector_element_bits (type);
8026 140604 : machine_mode eltmode = TYPE_MODE (elttype);
8027 140604 : poly_int64 bitsize;
8028 140604 : poly_int64 bitpos;
8029 140604 : rtvec vector = NULL;
8030 140604 : poly_uint64 n_elts;
8031 140604 : unsigned HOST_WIDE_INT const_n_elts;
8032 140604 : alias_set_type alias;
8033 140604 : bool vec_vec_init_p = false;
8034 140604 : machine_mode mode = GET_MODE (target);
8035 :
8036 140604 : gcc_assert (eltmode != BLKmode);
8037 :
8038 : /* Try using vec_duplicate_optab for uniform vectors. */
8039 140604 : icode = optab_handler (vec_duplicate_optab, mode);
8040 140604 : if (!TREE_SIDE_EFFECTS (exp)
8041 140604 : && VECTOR_MODE_P (mode)
8042 137039 : && icode != CODE_FOR_nothing
8043 : /* If the vec_duplicate target pattern does not specify an element
8044 : mode check that eltmode is the normal inner mode of the
8045 : requested vector mode. But if the target allows eltmode
8046 : explicitly go ahead and use it. */
8047 93796 : && (eltmode == GET_MODE_INNER (mode)
8048 0 : || insn_data[icode].operand[1].mode == eltmode)
8049 93796 : && (elt = uniform_vector_p (exp))
8050 158339 : && !VECTOR_TYPE_P (TREE_TYPE (elt)))
8051 : {
8052 17735 : class expand_operand ops[2];
8053 17735 : create_output_operand (&ops[0], target, mode);
8054 17735 : create_input_operand (&ops[1], expand_normal (elt), eltmode);
8055 17735 : expand_insn (icode, 2, ops);
8056 17735 : if (!rtx_equal_p (target, ops[0].value))
8057 0 : emit_move_insn (target, ops[0].value);
8058 17735 : break;
8059 : }
8060 : /* Use sign-extension for uniform boolean vectors with
8061 : integer modes and single-bit mask entries.
8062 : Effectively "vec_duplicate" for bitmasks. */
8063 122869 : if (elt_size == 1
8064 146 : && !TREE_SIDE_EFFECTS (exp)
8065 146 : && VECTOR_BOOLEAN_TYPE_P (type)
8066 146 : && SCALAR_INT_MODE_P (TYPE_MODE (type))
8067 146 : && (elt = uniform_vector_p (exp))
8068 143 : && !VECTOR_TYPE_P (TREE_TYPE (elt))
8069 122869 : && !BYTES_BIG_ENDIAN)
8070 : {
8071 143 : rtx op0 = force_reg (TYPE_MODE (TREE_TYPE (elt)),
8072 : expand_normal (elt));
8073 143 : rtx tmp = gen_reg_rtx (mode);
8074 143 : convert_move (tmp, op0, 0);
8075 :
8076 : /* Ensure no excess bits are set.
8077 : GCN needs this for nunits < 64.
8078 : x86 needs this for nunits < 8. */
8079 143 : unsigned int nunits = TYPE_VECTOR_SUBPARTS (type).to_constant ();
8080 143 : if (maybe_ne (GET_MODE_PRECISION (mode), nunits))
8081 2 : tmp = expand_binop (mode, and_optab, tmp,
8082 2 : GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
8083 : target, true, OPTAB_WIDEN);
8084 143 : if (tmp != target)
8085 141 : emit_move_insn (target, tmp);
8086 : break;
8087 : }
8088 :
8089 122726 : n_elts = TYPE_VECTOR_SUBPARTS (type);
8090 122726 : if (REG_P (target)
8091 119508 : && VECTOR_MODE_P (mode))
8092 : {
8093 119304 : const_n_elts = 0;
8094 119304 : if (CONSTRUCTOR_NELTS (exp)
8095 119304 : && (TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (exp, 0)->value))
8096 : == VECTOR_TYPE))
8097 : {
8098 1475 : tree etype = TREE_TYPE (CONSTRUCTOR_ELT (exp, 0)->value);
8099 1475 : gcc_assert (known_eq (CONSTRUCTOR_NELTS (exp)
8100 : * TYPE_VECTOR_SUBPARTS (etype),
8101 : n_elts));
8102 :
8103 4425 : icode = convert_optab_handler (vec_init_optab, mode,
8104 1475 : TYPE_MODE (etype));
8105 1475 : const_n_elts = CONSTRUCTOR_NELTS (exp);
8106 1475 : vec_vec_init_p = icode != CODE_FOR_nothing;
8107 : }
8108 235658 : else if (exact_div (n_elts, GET_MODE_NUNITS (eltmode))
8109 117829 : .is_constant (&const_n_elts))
8110 : {
8111 : /* For a non-const type vector, we check it is made up of
8112 : similarly non-const type vectors. */
8113 117829 : icode = convert_optab_handler (vec_init_optab, mode, eltmode);
8114 : }
8115 : else
8116 : {
8117 : /* Handle variable-length vector types. */
8118 : icode = convert_optab_handler (vec_init_optab, mode, eltmode);
8119 : const_n_elts = constant_lower_bound (n_elts);
8120 : }
8121 :
8122 119304 : if (const_n_elts && icode != CODE_FOR_nothing)
8123 : {
8124 118554 : vector = rtvec_alloc (const_n_elts);
8125 562166 : for (unsigned int k = 0; k < const_n_elts; k++)
8126 325058 : RTVEC_ELT (vector, k) = CONST0_RTX (eltmode);
8127 : }
8128 : }
8129 : else
8130 : gcc_assert (n_elts.is_constant ());
8131 :
8132 : /* Compute the size of the elements in the CTOR. It differs
8133 : from the size of the vector type elements only when the
8134 : CTOR elements are vectors themselves. */
8135 122726 : tree val_type = (CONSTRUCTOR_NELTS (exp) != 0
8136 122726 : ? TREE_TYPE (CONSTRUCTOR_ELT (exp, 0)->value)
8137 122726 : : elttype);
8138 122726 : if (VECTOR_TYPE_P (val_type))
8139 1825 : bitsize = tree_to_poly_uint64 (TYPE_SIZE (val_type));
8140 : else
8141 120901 : bitsize = elt_size;
8142 :
8143 : /* If the constructor has fewer elements than the vector,
8144 : clear the whole array first. Similarly if this is static
8145 : constructor of a non-BLKmode object. */
8146 122726 : if (cleared)
8147 : need_to_clear = false;
8148 122726 : else if (REG_P (target) && TREE_STATIC (exp))
8149 : need_to_clear = true;
8150 : else
8151 : {
8152 : poly_uint64 count = 0, zero_count = 0;
8153 : tree value;
8154 :
8155 491831 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), idx, value)
8156 : {
8157 369105 : poly_int64 n_elts_here = exact_div (bitsize, elt_size);
8158 369105 : count += n_elts_here;
8159 369105 : if (mostly_zeros_p (value))
8160 369105 : zero_count += n_elts_here;
8161 : }
8162 :
8163 : /* Clear the entire vector first if there are any missing elements,
8164 : or if the incidence of zero elements is >= 75%. */
8165 122726 : need_to_clear = (maybe_lt (count, n_elts)
8166 122726 : || maybe_gt (4 * zero_count, 3 * count));
8167 : }
8168 :
8169 1355 : if (need_to_clear && maybe_gt (size, 0) && !vector)
8170 : {
8171 765 : if (REG_P (target))
8172 29 : emit_move_insn (target, CONST0_RTX (mode));
8173 : else
8174 736 : clear_storage (target, gen_int_mode (size, Pmode),
8175 : BLOCK_OP_NORMAL);
8176 : cleared = 1;
8177 : }
8178 :
8179 : /* Inform later passes that the old value is dead. */
8180 122726 : if (!cleared && !vector && REG_P (target) && maybe_gt (n_elts, 1u))
8181 : {
8182 598 : emit_move_insn (target, CONST0_RTX (mode));
8183 598 : cleared = 1;
8184 : }
8185 :
8186 122726 : if (MEM_P (target))
8187 3218 : alias = MEM_ALIAS_SET (target);
8188 : else
8189 119508 : alias = get_alias_set (elttype);
8190 :
8191 : /* Store each element of the constructor into the corresponding
8192 : element of TARGET, determined by counting the elements. */
8193 122726 : HOST_WIDE_INT chunk_size = 0;
8194 122726 : bool chunk_multiple_p = constant_multiple_p (bitsize, elt_size,
8195 : &chunk_size);
8196 122726 : gcc_assert (chunk_multiple_p || vec_vec_init_p);
8197 :
8198 491831 : for (idx = 0; vec_safe_iterate (CONSTRUCTOR_ELTS (exp), idx, &ce);
8199 : idx++)
8200 : {
8201 369105 : HOST_WIDE_INT eltpos;
8202 369105 : tree value = ce->value;
8203 :
8204 369105 : if (cleared && initializer_zerop (value))
8205 8893 : continue;
8206 :
8207 360212 : if (ce->index)
8208 51196 : eltpos = tree_to_uhwi (ce->index);
8209 : else
8210 309016 : eltpos = idx * chunk_size;
8211 :
8212 360212 : if (vector)
8213 : {
8214 323973 : if (vec_vec_init_p)
8215 : {
8216 2846 : gcc_assert (ce->index == NULL_TREE);
8217 2846 : gcc_assert (TREE_CODE (TREE_TYPE (value)) == VECTOR_TYPE);
8218 2846 : eltpos = idx;
8219 : }
8220 : else
8221 321127 : gcc_assert (TREE_CODE (TREE_TYPE (value)) != VECTOR_TYPE);
8222 323973 : RTVEC_ELT (vector, eltpos) = expand_normal (value);
8223 : }
8224 : else
8225 : {
8226 36239 : machine_mode value_mode
8227 36239 : = (TREE_CODE (TREE_TYPE (value)) == VECTOR_TYPE
8228 36239 : ? TYPE_MODE (TREE_TYPE (value)) : eltmode);
8229 36239 : bitpos = eltpos * elt_size;
8230 36239 : store_constructor_field (target, bitsize, bitpos, 0,
8231 36239 : bitregion_end, value_mode,
8232 : value, cleared, alias, reverse);
8233 : }
8234 : }
8235 :
8236 122726 : if (vector)
8237 118554 : emit_insn (GEN_FCN (icode) (target,
8238 : gen_rtx_PARALLEL (mode, vector)));
8239 : break;
8240 : }
8241 :
8242 0 : default:
8243 0 : gcc_unreachable ();
8244 : }
8245 251460 : }
8246 :
8247 : /* Store the value of EXP (an expression tree)
8248 : into a subfield of TARGET which has mode MODE and occupies
8249 : BITSIZE bits, starting BITPOS bits from the start of TARGET.
8250 : If MODE is VOIDmode, it means that we are storing into a bit-field.
8251 :
8252 : BITREGION_START is bitpos of the first bitfield in this region.
8253 : BITREGION_END is the bitpos of the ending bitfield in this region.
8254 : These two fields are 0, if the C++ memory model does not apply,
8255 : or we are not interested in keeping track of bitfield regions.
8256 :
8257 : Always return const0_rtx unless we have something particular to
8258 : return.
8259 :
8260 : ALIAS_SET is the alias set for the destination. This value will
8261 : (in general) be different from that for TARGET, since TARGET is a
8262 : reference to the containing structure.
8263 :
8264 : If NONTEMPORAL is true, try generating a nontemporal store.
8265 :
8266 : If REVERSE is true, the store is to be done in reverse order. */
8267 :
8268 : static rtx
8269 4873029 : store_field (rtx target, poly_int64 bitsize, poly_int64 bitpos,
8270 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
8271 : machine_mode mode, tree exp,
8272 : alias_set_type alias_set, bool nontemporal, bool reverse)
8273 : {
8274 4873029 : if (TREE_CODE (exp) == ERROR_MARK)
8275 0 : return const0_rtx;
8276 :
8277 : /* If we have nothing to store, do nothing unless the expression has
8278 : side-effects. Don't do that for zero sized addressable lhs of
8279 : calls. */
8280 4873029 : if (known_eq (bitsize, 0)
8281 4873029 : && (!TREE_ADDRESSABLE (TREE_TYPE (exp))
8282 3 : || TREE_CODE (exp) != CALL_EXPR))
8283 0 : return expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
8284 :
8285 4873029 : if (GET_CODE (target) == CONCAT)
8286 : {
8287 : /* We're storing into a struct containing a single __complex. */
8288 :
8289 0 : gcc_assert (known_eq (bitpos, 0));
8290 0 : return store_expr (exp, target, 0, nontemporal, reverse);
8291 : }
8292 :
8293 : /* If the structure is in a register or if the component
8294 : is a bit field, we cannot use addressing to access it.
8295 : Use bit-field techniques or SUBREG to store in it. */
8296 :
8297 4873029 : poly_int64 decl_bitsize;
8298 4873029 : if (mode == VOIDmode
8299 4805726 : || (mode != BLKmode && ! direct_store[(int) mode]
8300 5311 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_INT
8301 5309 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_FLOAT)
8302 4800446 : || REG_P (target)
8303 4109065 : || GET_CODE (target) == SUBREG
8304 : /* If the field isn't aligned enough to store as an ordinary memref,
8305 : store it as a bit field. */
8306 4109065 : || (mode != BLKmode
8307 3990330 : && ((((MEM_ALIGN (target) < GET_MODE_ALIGNMENT (mode))
8308 7890259 : || !multiple_p (bitpos, GET_MODE_ALIGNMENT (mode)))
8309 47916 : && targetm.slow_unaligned_access (mode, MEM_ALIGN (target)))
8310 3990330 : || !multiple_p (bitpos, BITS_PER_UNIT)))
8311 4109065 : || (known_size_p (bitsize)
8312 4109053 : && mode != BLKmode
8313 3990330 : && maybe_gt (GET_MODE_BITSIZE (mode), bitsize))
8314 : /* If the RHS and field are a constant size and the size of the
8315 : RHS isn't the same size as the bitfield, we must use bitfield
8316 : operations. */
8317 4109056 : || (known_size_p (bitsize)
8318 4109044 : && poly_int_tree_p (TYPE_SIZE (TREE_TYPE (exp)))
8319 4873320 : && maybe_ne (wi::to_poly_offset (TYPE_SIZE (TREE_TYPE (exp))),
8320 : bitsize)
8321 : /* Except for initialization of full bytes from a CONSTRUCTOR, which
8322 : we will handle specially below. */
8323 42 : && !(TREE_CODE (exp) == CONSTRUCTOR
8324 11 : && multiple_p (bitsize, BITS_PER_UNIT))
8325 : /* And except for bitwise copying of TREE_ADDRESSABLE types,
8326 : where the FIELD_DECL has the right bitsize, but TREE_TYPE (exp)
8327 : includes some extra padding. store_expr / expand_expr will in
8328 : that case call get_inner_reference that will have the bitsize
8329 : we check here and thus the block move will not clobber the
8330 : padding that shouldn't be clobbered. In the future we could
8331 : replace the TREE_ADDRESSABLE check with a check that
8332 : get_base_address needs to live in memory. */
8333 31 : && (!TREE_ADDRESSABLE (TREE_TYPE (exp))
8334 9 : || TREE_CODE (exp) != COMPONENT_REF
8335 6 : || !multiple_p (bitsize, BITS_PER_UNIT)
8336 6 : || !multiple_p (bitpos, BITS_PER_UNIT)
8337 6 : || !poly_int_tree_p (DECL_SIZE (TREE_OPERAND (exp, 1)),
8338 : &decl_bitsize)
8339 6 : || maybe_ne (decl_bitsize, bitsize))
8340 : /* A call with an addressable return type and return-slot
8341 : optimization must not need bitfield operations but we must
8342 : pass down the original target. */
8343 25 : && (TREE_CODE (exp) != CALL_EXPR
8344 6 : || !TREE_ADDRESSABLE (TREE_TYPE (exp))
8345 0 : || !CALL_EXPR_RETURN_SLOT_OPT (exp)))
8346 : /* If we are expanding a MEM_REF of a non-BLKmode non-addressable
8347 : decl we must use bitfield operations. */
8348 8982060 : || (known_size_p (bitsize)
8349 4109019 : && TREE_CODE (exp) == MEM_REF
8350 32950 : && TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR
8351 26969 : && DECL_P (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
8352 20393 : && !TREE_ADDRESSABLE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
8353 5670 : && DECL_MODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0)) != BLKmode))
8354 : {
8355 764276 : rtx temp;
8356 764276 : gimple *nop_def;
8357 :
8358 : /* If EXP is a NOP_EXPR of precision less than its mode, then that
8359 : implies a mask operation. If the precision is the same size as
8360 : the field we're storing into, that mask is redundant. This is
8361 : particularly common with bit field assignments generated by the
8362 : C front end. */
8363 764276 : nop_def = get_def_for_expr (exp, NOP_EXPR);
8364 764276 : if (nop_def)
8365 : {
8366 5472 : tree type = TREE_TYPE (exp);
8367 5472 : if (INTEGRAL_TYPE_P (type)
8368 5359 : && maybe_ne (TYPE_PRECISION (type),
8369 10718 : GET_MODE_BITSIZE (TYPE_MODE (type)))
8370 8780 : && known_eq (bitsize, TYPE_PRECISION (type)))
8371 : {
8372 3308 : tree op = gimple_assign_rhs1 (nop_def);
8373 3308 : type = TREE_TYPE (op);
8374 3308 : if (INTEGRAL_TYPE_P (type)
8375 3308 : && known_ge (TYPE_PRECISION (type), bitsize))
8376 : exp = op;
8377 : }
8378 : }
8379 :
8380 764276 : temp = expand_normal (exp);
8381 :
8382 : /* We don't support variable-sized BLKmode bitfields, since our
8383 : handling of BLKmode is bound up with the ability to break
8384 : things into words. */
8385 764276 : gcc_assert (mode != BLKmode || bitsize.is_constant ());
8386 :
8387 : /* Handle calls that return values in multiple non-contiguous locations.
8388 : The Irix 6 ABI has examples of this. */
8389 764276 : if (GET_CODE (temp) == PARALLEL)
8390 : {
8391 6 : HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));
8392 6 : machine_mode temp_mode = GET_MODE (temp);
8393 6 : if (temp_mode == BLKmode || temp_mode == VOIDmode)
8394 6 : temp_mode
8395 6 : = smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
8396 6 : rtx temp_target = gen_reg_rtx (temp_mode);
8397 6 : emit_group_store (temp_target, temp, TREE_TYPE (exp), size);
8398 6 : temp = temp_target;
8399 : }
8400 :
8401 : /* Handle calls that return BLKmode values in registers. */
8402 764270 : else if (mode == BLKmode && REG_P (temp) && TREE_CODE (exp) == CALL_EXPR)
8403 : {
8404 0 : rtx temp_target = gen_reg_rtx (GET_MODE (temp));
8405 0 : copy_blkmode_from_reg (temp_target, temp, TREE_TYPE (exp));
8406 0 : temp = temp_target;
8407 : }
8408 :
8409 : /* If the value has aggregate type and an integral mode then, if BITSIZE
8410 : is narrower than this mode and this is for big-endian data, we first
8411 : need to put the value into the low-order bits for store_bit_field,
8412 : except when MODE is BLKmode and BITSIZE larger than the word size
8413 : (see the handling of fields larger than a word in store_bit_field).
8414 : Moreover, the field may be not aligned on a byte boundary; in this
8415 : case, if it has reverse storage order, it needs to be accessed as a
8416 : scalar field with reverse storage order and we must first put the
8417 : value into target order. */
8418 764276 : scalar_int_mode temp_mode;
8419 1526436 : if (AGGREGATE_TYPE_P (TREE_TYPE (exp))
8420 765053 : && is_int_mode (GET_MODE (temp), &temp_mode))
8421 : {
8422 2538 : HOST_WIDE_INT size = GET_MODE_BITSIZE (temp_mode);
8423 :
8424 2538 : reverse = TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (exp));
8425 :
8426 2538 : if (reverse)
8427 0 : temp = flip_storage_order (temp_mode, temp);
8428 :
8429 2538 : gcc_checking_assert (known_le (bitsize, size));
8430 2538 : if (maybe_lt (bitsize, size)
8431 2538 : && reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN
8432 : /* Use of to_constant for BLKmode was checked above. */
8433 : && !(mode == BLKmode && bitsize.to_constant () > BITS_PER_WORD))
8434 0 : temp = expand_shift (RSHIFT_EXPR, temp_mode, temp,
8435 : size - bitsize, NULL_RTX, 1);
8436 : }
8437 :
8438 : /* Unless MODE is VOIDmode or BLKmode, convert TEMP to MODE. */
8439 696973 : if (mode != VOIDmode && mode != BLKmode
8440 1460944 : && mode != TYPE_MODE (TREE_TYPE (exp)))
8441 4 : temp = convert_modes (mode, TYPE_MODE (TREE_TYPE (exp)), temp, 1);
8442 :
8443 : /* If the mode of TEMP and TARGET is BLKmode, both must be in memory
8444 : and BITPOS must be aligned on a byte boundary. If so, we simply do
8445 : a block copy. Likewise for a BLKmode-like TARGET. */
8446 764276 : if (GET_MODE (temp) == BLKmode
8447 764276 : && (GET_MODE (target) == BLKmode
8448 202 : || (MEM_P (target)
8449 0 : && GET_MODE_CLASS (GET_MODE (target)) == MODE_INT
8450 0 : && multiple_p (bitpos, BITS_PER_UNIT)
8451 0 : && multiple_p (bitsize, BITS_PER_UNIT))))
8452 : {
8453 97 : gcc_assert (MEM_P (target) && MEM_P (temp));
8454 97 : poly_int64 bytepos = exact_div (bitpos, BITS_PER_UNIT);
8455 97 : poly_int64 bytesize = bits_to_bytes_round_up (bitsize);
8456 :
8457 97 : target = adjust_address (target, VOIDmode, bytepos);
8458 97 : emit_block_move (target, temp,
8459 97 : gen_int_mode (bytesize, Pmode),
8460 : BLOCK_OP_NORMAL);
8461 :
8462 97 : return const0_rtx;
8463 : }
8464 :
8465 : /* If the mode of TEMP is still BLKmode and BITSIZE not larger than the
8466 : word size, we need to load the value (see again store_bit_field). */
8467 764197 : if (GET_MODE (temp) == BLKmode && known_le (bitsize, BITS_PER_WORD))
8468 : {
8469 81 : temp_mode = smallest_int_mode_for_size (bitsize).require ();
8470 81 : temp = extract_bit_field (temp, bitsize, 0, 1, NULL_RTX, temp_mode,
8471 : temp_mode, false, NULL);
8472 : }
8473 :
8474 : /* Store the value in the bitfield. */
8475 764179 : gcc_checking_assert (known_ge (bitpos, 0));
8476 764179 : store_bit_field (target, bitsize, bitpos,
8477 : bitregion_start, bitregion_end,
8478 : mode, temp, reverse, false);
8479 :
8480 764179 : return const0_rtx;
8481 : }
8482 : else
8483 : {
8484 : /* Now build a reference to just the desired component. */
8485 4108753 : rtx to_rtx = adjust_address (target, mode,
8486 : exact_div (bitpos, BITS_PER_UNIT));
8487 :
8488 4108753 : if (to_rtx == target)
8489 1310179 : to_rtx = copy_rtx (to_rtx);
8490 :
8491 7939620 : if (!MEM_KEEP_ALIAS_SET_P (to_rtx) && MEM_ALIAS_SET (to_rtx) != 0)
8492 3571704 : set_mem_alias_set (to_rtx, alias_set);
8493 :
8494 : /* Above we avoided using bitfield operations for storing a CONSTRUCTOR
8495 : into a target smaller than its type; handle that case now. */
8496 4108753 : if (TREE_CODE (exp) == CONSTRUCTOR && known_size_p (bitsize))
8497 : {
8498 70677 : poly_int64 bytesize = exact_div (bitsize, BITS_PER_UNIT);
8499 70677 : store_constructor (exp, to_rtx, 0, bytesize, reverse);
8500 70677 : return to_rtx;
8501 : }
8502 :
8503 4038076 : return store_expr (exp, to_rtx, 0, nontemporal, reverse);
8504 : }
8505 : }
8506 :
8507 : /* Given an expression EXP that may be a COMPONENT_REF, a BIT_FIELD_REF,
8508 : an ARRAY_REF, or an ARRAY_RANGE_REF, look for nested operations of these
8509 : codes and find the ultimate containing object, which we return.
8510 :
8511 : We set *PBITSIZE to the size in bits that we want, *PBITPOS to the
8512 : bit position, *PUNSIGNEDP to the signedness and *PREVERSEP to the
8513 : storage order of the field.
8514 : If the position of the field is variable, we store a tree
8515 : giving the variable offset (in units) in *POFFSET.
8516 : This offset is in addition to the bit position.
8517 : If the position is not variable, we store 0 in *POFFSET.
8518 :
8519 : If any of the extraction expressions is volatile,
8520 : we store 1 in *PVOLATILEP. Otherwise we don't change that.
8521 :
8522 : If the field is a non-BLKmode bit-field, *PMODE is set to VOIDmode.
8523 : Otherwise, it is a mode that can be used to access the field.
8524 :
8525 : If the field describes a variable-sized object, *PMODE is set to
8526 : BLKmode and *PBITSIZE is set to -1. An access cannot be made in
8527 : this case, but the address of the object can be found. */
8528 :
8529 : tree
8530 299546275 : get_inner_reference (tree exp, poly_int64 *pbitsize,
8531 : poly_int64 *pbitpos, tree *poffset,
8532 : machine_mode *pmode, int *punsignedp,
8533 : int *preversep, int *pvolatilep)
8534 : {
8535 299546275 : tree size_tree = 0;
8536 299546275 : machine_mode mode = VOIDmode;
8537 299546275 : bool blkmode_bitfield = false;
8538 299546275 : tree offset = size_zero_node;
8539 299546275 : poly_offset_int bit_offset = 0;
8540 :
8541 : /* First get the mode, signedness, storage order and size. We do this from
8542 : just the outermost expression. */
8543 299546275 : *pbitsize = -1;
8544 299546275 : if (TREE_CODE (exp) == COMPONENT_REF)
8545 : {
8546 128562533 : tree field = TREE_OPERAND (exp, 1);
8547 128562533 : size_tree = DECL_SIZE (field);
8548 128562533 : if (flag_strict_volatile_bitfields > 0
8549 58 : && TREE_THIS_VOLATILE (exp)
8550 40 : && DECL_BIT_FIELD_TYPE (field)
8551 128562555 : && DECL_MODE (field) != BLKmode)
8552 : /* Volatile bitfields should be accessed in the mode of the
8553 : field's type, not the mode computed based on the bit
8554 : size. */
8555 22 : mode = TYPE_MODE (DECL_BIT_FIELD_TYPE (field));
8556 128562511 : else if (!DECL_BIT_FIELD (field))
8557 : {
8558 127211208 : mode = DECL_MODE (field);
8559 : /* For vector fields re-check the target flags, as DECL_MODE
8560 : could have been set with different target flags than
8561 : the current function has. */
8562 127211208 : if (VECTOR_TYPE_P (TREE_TYPE (field))
8563 127211208 : && VECTOR_MODE_P (TYPE_MODE_RAW (TREE_TYPE (field))))
8564 380168 : mode = TYPE_MODE (TREE_TYPE (field));
8565 : }
8566 1351303 : else if (DECL_MODE (field) == BLKmode)
8567 622 : blkmode_bitfield = true;
8568 :
8569 128562533 : *punsignedp = DECL_UNSIGNED (field);
8570 : }
8571 170983742 : else if (TREE_CODE (exp) == BIT_FIELD_REF)
8572 : {
8573 584230 : size_tree = TREE_OPERAND (exp, 1);
8574 1168053 : *punsignedp = (! INTEGRAL_TYPE_P (TREE_TYPE (exp))
8575 1016908 : || TYPE_UNSIGNED (TREE_TYPE (exp)));
8576 :
8577 : /* For vector element types with the correct size of access or for
8578 : vector typed accesses use the mode of the access type. */
8579 584230 : if ((TREE_CODE (TREE_TYPE (TREE_OPERAND (exp, 0))) == VECTOR_TYPE
8580 428705 : && TREE_TYPE (exp) == TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0)))
8581 382598 : && tree_int_cst_equal (size_tree, TYPE_SIZE (TREE_TYPE (exp))))
8582 630372 : || VECTOR_TYPE_P (TREE_TYPE (exp)))
8583 406384 : mode = TYPE_MODE (TREE_TYPE (exp));
8584 : }
8585 : else
8586 : {
8587 170399512 : mode = TYPE_MODE (TREE_TYPE (exp));
8588 170399512 : *punsignedp = TYPE_UNSIGNED (TREE_TYPE (exp));
8589 :
8590 170399512 : if (mode == BLKmode)
8591 75784760 : size_tree = TYPE_SIZE (TREE_TYPE (exp));
8592 : else
8593 189229504 : *pbitsize = GET_MODE_BITSIZE (mode);
8594 : }
8595 :
8596 299546275 : if (size_tree != 0)
8597 : {
8598 204100038 : if (!poly_int_tree_p (size_tree, pbitsize))
8599 490444 : mode = BLKmode, *pbitsize = -1;
8600 : }
8601 :
8602 299546275 : *preversep = reverse_storage_order_for_component_p (exp);
8603 :
8604 : /* Compute cumulative bit-offset for nested component-refs and array-refs,
8605 : and find the ultimate containing object. */
8606 764441659 : while (1)
8607 : {
8608 531993967 : switch (TREE_CODE (exp))
8609 : {
8610 584230 : case BIT_FIELD_REF:
8611 584230 : bit_offset += wi::to_poly_offset (TREE_OPERAND (exp, 2));
8612 584230 : break;
8613 :
8614 195821827 : case COMPONENT_REF:
8615 195821827 : {
8616 195821827 : tree field = TREE_OPERAND (exp, 1);
8617 195821827 : tree this_offset = component_ref_field_offset (exp);
8618 :
8619 : /* If this field hasn't been filled in yet, don't go past it.
8620 : This should only happen when folding expressions made during
8621 : type construction. */
8622 195821827 : if (this_offset == 0)
8623 : break;
8624 :
8625 195821827 : offset = size_binop (PLUS_EXPR, offset, this_offset);
8626 195821827 : bit_offset += wi::to_poly_offset (DECL_FIELD_BIT_OFFSET (field));
8627 :
8628 : /* ??? Right now we don't do anything with DECL_OFFSET_ALIGN. */
8629 : }
8630 195821827 : break;
8631 :
8632 32721772 : case ARRAY_REF:
8633 32721772 : case ARRAY_RANGE_REF:
8634 32721772 : {
8635 32721772 : tree index = TREE_OPERAND (exp, 1);
8636 32721772 : tree low_bound = array_ref_low_bound (exp);
8637 32721772 : tree unit_size = array_ref_element_size (exp);
8638 :
8639 : /* We assume all arrays have sizes that are a multiple of a byte.
8640 : First subtract the lower bound, if any, in the type of the
8641 : index, then convert to sizetype and multiply by the size of
8642 : the array element. */
8643 32721772 : if (! integer_zerop (low_bound))
8644 976701 : index = fold_build2 (MINUS_EXPR, TREE_TYPE (index),
8645 : index, low_bound);
8646 :
8647 32721772 : offset = size_binop (PLUS_EXPR, offset,
8648 : size_binop (MULT_EXPR,
8649 : fold_convert (sizetype, index),
8650 : unit_size));
8651 : }
8652 32721772 : break;
8653 :
8654 : case REALPART_EXPR:
8655 : break;
8656 :
8657 : case IMAGPART_EXPR:
8658 232447692 : bit_offset += *pbitsize;
8659 : break;
8660 :
8661 : case VIEW_CONVERT_EXPR:
8662 : break;
8663 :
8664 94245230 : case MEM_REF:
8665 : /* Hand back the decl for MEM[&decl, off]. */
8666 94245230 : if (TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR)
8667 : {
8668 22161461 : tree off = TREE_OPERAND (exp, 1);
8669 22161461 : if (!integer_zerop (off))
8670 : {
8671 12607172 : poly_offset_int boff = mem_ref_offset (exp);
8672 12607172 : boff <<= LOG2_BITS_PER_UNIT;
8673 12607172 : bit_offset += boff;
8674 : }
8675 22161461 : exp = TREE_OPERAND (TREE_OPERAND (exp, 0), 0);
8676 : }
8677 94245230 : goto done;
8678 :
8679 205301045 : default:
8680 205301045 : goto done;
8681 : }
8682 :
8683 : /* If any reference in the chain is volatile, the effect is volatile. */
8684 232447692 : if (TREE_THIS_VOLATILE (exp))
8685 568134 : *pvolatilep = 1;
8686 :
8687 232447692 : exp = TREE_OPERAND (exp, 0);
8688 232447692 : }
8689 299546275 : done:
8690 :
8691 : /* If OFFSET is constant, see if we can return the whole thing as a
8692 : constant bit position. Make sure to handle overflow during
8693 : this conversion. */
8694 299546275 : if (poly_int_tree_p (offset))
8695 : {
8696 285809840 : poly_offset_int tem = wi::sext (wi::to_poly_offset (offset),
8697 285809840 : TYPE_PRECISION (sizetype));
8698 285809840 : tem <<= LOG2_BITS_PER_UNIT;
8699 285809840 : tem += bit_offset;
8700 285809840 : if (tem.to_shwi (pbitpos))
8701 285808370 : *poffset = offset = NULL_TREE;
8702 : }
8703 :
8704 : /* Otherwise, split it up. */
8705 285809840 : if (offset)
8706 : {
8707 : /* Avoid returning a negative bitpos as this may wreak havoc later. */
8708 13737905 : if (!bit_offset.to_shwi (pbitpos) || maybe_lt (*pbitpos, 0))
8709 : {
8710 309 : *pbitpos = num_trailing_bits (bit_offset.force_shwi ());
8711 309 : poly_offset_int bytes = bits_to_bytes_round_down (bit_offset);
8712 309 : offset = size_binop (PLUS_EXPR, offset,
8713 : build_int_cst (sizetype, bytes.force_shwi ()));
8714 : }
8715 :
8716 13737905 : *poffset = offset;
8717 : }
8718 :
8719 : /* We can use BLKmode for a byte-aligned BLKmode bitfield. */
8720 299546275 : if (mode == VOIDmode
8721 5856796 : && blkmode_bitfield
8722 622 : && multiple_p (*pbitpos, BITS_PER_UNIT)
8723 299546761 : && multiple_p (*pbitsize, BITS_PER_UNIT))
8724 5 : *pmode = BLKmode;
8725 : else
8726 299546270 : *pmode = mode;
8727 :
8728 299546275 : return exp;
8729 : }
8730 :
8731 : /* Alignment in bits the TARGET of an assignment may be assumed to have. */
8732 :
8733 : static unsigned HOST_WIDE_INT
8734 516995 : target_align (const_tree target)
8735 : {
8736 : /* We might have a chain of nested references with intermediate misaligning
8737 : bitfields components, so need to recurse to find out. */
8738 :
8739 516995 : unsigned HOST_WIDE_INT this_align, outer_align;
8740 :
8741 516995 : switch (TREE_CODE (target))
8742 : {
8743 : case BIT_FIELD_REF:
8744 : return 1;
8745 :
8746 141173 : case COMPONENT_REF:
8747 141173 : this_align = DECL_ALIGN (TREE_OPERAND (target, 1));
8748 141173 : outer_align = target_align (TREE_OPERAND (target, 0));
8749 141173 : return MIN (this_align, outer_align);
8750 :
8751 191165 : case ARRAY_REF:
8752 191165 : case ARRAY_RANGE_REF:
8753 191165 : this_align = TYPE_ALIGN (TREE_TYPE (target));
8754 191165 : outer_align = target_align (TREE_OPERAND (target, 0));
8755 191165 : return MIN (this_align, outer_align);
8756 :
8757 4702 : CASE_CONVERT:
8758 4702 : case NON_LVALUE_EXPR:
8759 4702 : case VIEW_CONVERT_EXPR:
8760 4702 : this_align = TYPE_ALIGN (TREE_TYPE (target));
8761 4702 : outer_align = target_align (TREE_OPERAND (target, 0));
8762 4702 : return MAX (this_align, outer_align);
8763 :
8764 179951 : default:
8765 179951 : return TYPE_ALIGN (TREE_TYPE (target));
8766 : }
8767 : }
8768 :
8769 :
8770 : /* Given an rtx VALUE that may contain additions and multiplications, return
8771 : an equivalent value that just refers to a register, memory, or constant.
8772 : This is done by generating instructions to perform the arithmetic and
8773 : returning a pseudo-register containing the value.
8774 :
8775 : The returned value may be a REG, SUBREG, MEM or constant. */
8776 :
8777 : rtx
8778 32148999 : force_operand (rtx value, rtx target)
8779 : {
8780 32148999 : rtx op1, op2;
8781 : /* Use subtarget as the target for operand 0 of a binary operation. */
8782 32148999 : rtx subtarget = get_subtarget (target);
8783 32148999 : enum rtx_code code = GET_CODE (value);
8784 :
8785 : /* Check for subreg applied to an expression produced by loop optimizer. */
8786 32148999 : if (code == SUBREG
8787 311658 : && !REG_P (SUBREG_REG (value))
8788 169 : && !MEM_P (SUBREG_REG (value)))
8789 : {
8790 169 : value
8791 169 : = simplify_gen_subreg (GET_MODE (value),
8792 169 : force_reg (GET_MODE (SUBREG_REG (value)),
8793 : force_operand (SUBREG_REG (value),
8794 : NULL_RTX)),
8795 169 : GET_MODE (SUBREG_REG (value)),
8796 169 : SUBREG_BYTE (value));
8797 169 : code = GET_CODE (value);
8798 : }
8799 :
8800 : /* Check for a PIC address load. */
8801 32148999 : if ((code == PLUS || code == MINUS)
8802 3867689 : && XEXP (value, 0) == pic_offset_table_rtx
8803 1932 : && (GET_CODE (XEXP (value, 1)) == SYMBOL_REF
8804 1932 : || GET_CODE (XEXP (value, 1)) == LABEL_REF
8805 1932 : || GET_CODE (XEXP (value, 1)) == CONST))
8806 : {
8807 189 : if (!subtarget)
8808 189 : subtarget = gen_reg_rtx (GET_MODE (value));
8809 189 : emit_move_insn (subtarget, value);
8810 189 : return subtarget;
8811 : }
8812 :
8813 32148810 : if (ARITHMETIC_P (value))
8814 : {
8815 3990185 : op2 = XEXP (value, 1);
8816 3990185 : if (!CONSTANT_P (op2) && !(REG_P (op2) && op2 != subtarget))
8817 3990185 : subtarget = 0;
8818 3990185 : if (code == MINUS && CONST_INT_P (op2))
8819 : {
8820 0 : code = PLUS;
8821 0 : op2 = negate_rtx (GET_MODE (value), op2);
8822 : }
8823 :
8824 : /* Check for an addition with OP2 a constant integer and our first
8825 : operand a PLUS of a virtual register and something else. In that
8826 : case, we want to emit the sum of the virtual register and the
8827 : constant first and then add the other value. This allows virtual
8828 : register instantiation to simply modify the constant rather than
8829 : creating another one around this addition. */
8830 3805588 : if (code == PLUS && CONST_INT_P (op2)
8831 3426256 : && GET_CODE (XEXP (value, 0)) == PLUS
8832 65429 : && REG_P (XEXP (XEXP (value, 0), 0))
8833 3997892 : && VIRTUAL_REGISTER_P (XEXP (XEXP (value, 0), 0)))
8834 : {
8835 1178 : rtx temp = expand_simple_binop (GET_MODE (value), code,
8836 : XEXP (XEXP (value, 0), 0), op2,
8837 : subtarget, 0, OPTAB_LIB_WIDEN);
8838 1178 : return expand_simple_binop (GET_MODE (value), code, temp,
8839 1178 : force_operand (XEXP (XEXP (value,
8840 : 0), 1), 0),
8841 1178 : target, 0, OPTAB_LIB_WIDEN);
8842 : }
8843 :
8844 3989007 : op1 = force_operand (XEXP (value, 0), subtarget);
8845 3989007 : op2 = force_operand (op2, NULL_RTX);
8846 3989007 : switch (code)
8847 : {
8848 98457 : case MULT:
8849 98457 : return expand_mult (GET_MODE (value), op1, op2, target, 1);
8850 506 : case DIV:
8851 506 : if (!INTEGRAL_MODE_P (GET_MODE (value)))
8852 506 : return expand_simple_binop (GET_MODE (value), code, op1, op2,
8853 506 : target, 1, OPTAB_LIB_WIDEN);
8854 : else
8855 0 : return expand_divmod (0,
8856 : FLOAT_MODE_P (GET_MODE (value))
8857 : ? RDIV_EXPR : TRUNC_DIV_EXPR,
8858 0 : GET_MODE (value), op1, op2, target, 0);
8859 0 : case MOD:
8860 0 : return expand_divmod (1, TRUNC_MOD_EXPR, GET_MODE (value), op1, op2,
8861 0 : target, 0);
8862 348 : case UDIV:
8863 348 : return expand_divmod (0, TRUNC_DIV_EXPR, GET_MODE (value), op1, op2,
8864 348 : target, 1);
8865 0 : case UMOD:
8866 0 : return expand_divmod (1, TRUNC_MOD_EXPR, GET_MODE (value), op1, op2,
8867 0 : target, 1);
8868 186 : case ASHIFTRT:
8869 186 : return expand_simple_binop (GET_MODE (value), code, op1, op2,
8870 186 : target, 0, OPTAB_LIB_WIDEN);
8871 3889510 : default:
8872 3889510 : return expand_simple_binop (GET_MODE (value), code, op1, op2,
8873 3889510 : target, 1, OPTAB_LIB_WIDEN);
8874 : }
8875 : }
8876 28158625 : if (UNARY_P (value))
8877 : {
8878 25975 : if (!target)
8879 11491 : target = gen_reg_rtx (GET_MODE (value));
8880 : /* FIX or UNSIGNED_FIX with integral mode has unspecified rounding,
8881 : while FIX with floating point mode rounds toward zero. So, some
8882 : targets use expressions like (fix:SI (fix:DF (reg:DF ...)))
8883 : to express rounding toward zero during the conversion to int.
8884 : expand_fix isn't able to handle that, it can only handle
8885 : FIX/UNSIGNED_FIX from floating point mode to integral one. */
8886 25975 : if ((code == FIX || code == UNSIGNED_FIX)
8887 8 : && GET_CODE (XEXP (value, 0)) == FIX
8888 0 : && (GET_MODE (XEXP (value, 0))
8889 0 : == GET_MODE (XEXP (XEXP (value, 0), 0))))
8890 0 : op1 = force_operand (XEXP (XEXP (value, 0), 0), NULL_RTX);
8891 : else
8892 25975 : op1 = force_operand (XEXP (value, 0), NULL_RTX);
8893 25975 : switch (code)
8894 : {
8895 11696 : case ZERO_EXTEND:
8896 11696 : case SIGN_EXTEND:
8897 11696 : case TRUNCATE:
8898 11696 : case FLOAT_EXTEND:
8899 11696 : case FLOAT_TRUNCATE:
8900 11696 : convert_move (target, op1, code == ZERO_EXTEND);
8901 11696 : return target;
8902 :
8903 8 : case FIX:
8904 8 : case UNSIGNED_FIX:
8905 8 : expand_fix (target, op1, code == UNSIGNED_FIX);
8906 8 : return target;
8907 :
8908 72 : case FLOAT:
8909 72 : case UNSIGNED_FLOAT:
8910 72 : expand_float (target, op1, code == UNSIGNED_FLOAT);
8911 72 : return target;
8912 :
8913 14199 : default:
8914 14199 : return expand_simple_unop (GET_MODE (value), code, op1, target, 0);
8915 : }
8916 : }
8917 :
8918 : #ifdef INSN_SCHEDULING
8919 : /* On machines that have insn scheduling, we want all memory reference to be
8920 : explicit, so we need to deal with such paradoxical SUBREGs. */
8921 28132650 : if (paradoxical_subreg_p (value) && MEM_P (SUBREG_REG (value)))
8922 0 : value
8923 0 : = simplify_gen_subreg (GET_MODE (value),
8924 0 : force_reg (GET_MODE (SUBREG_REG (value)),
8925 : force_operand (SUBREG_REG (value),
8926 : NULL_RTX)),
8927 : GET_MODE (SUBREG_REG (value)),
8928 0 : SUBREG_BYTE (value));
8929 : #endif
8930 :
8931 : return value;
8932 : }
8933 :
8934 : /* Subroutine of expand_expr: return true iff there is no way that
8935 : EXP can reference X, which is being modified. TOP_P is nonzero if this
8936 : call is going to be used to determine whether we need a temporary
8937 : for EXP, as opposed to a recursive call to this function.
8938 :
8939 : It is always safe for this routine to return false since it merely
8940 : searches for optimization opportunities. */
8941 :
8942 : bool
8943 8967944 : safe_from_p (const_rtx x, tree exp, int top_p)
8944 : {
8945 8967960 : rtx exp_rtl = 0;
8946 8967960 : int i, nops;
8947 :
8948 8967960 : if (x == 0
8949 : /* If EXP has varying size, we MUST use a target since we currently
8950 : have no way of allocating temporaries of variable size
8951 : (except for arrays that have TYPE_ARRAY_MAX_SIZE set).
8952 : So we assume here that something at a higher level has prevented a
8953 : clash. This is somewhat bogus, but the best we can do. Only
8954 : do this when X is BLKmode and when we are at the top level. */
8955 2026414 : || (top_p && TREE_TYPE (exp) != 0 && COMPLETE_TYPE_P (TREE_TYPE (exp))
8956 1888471 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (exp))) != INTEGER_CST
8957 0 : && (TREE_CODE (TREE_TYPE (exp)) != ARRAY_TYPE
8958 0 : || TYPE_ARRAY_MAX_SIZE (TREE_TYPE (exp)) == NULL_TREE
8959 0 : || TREE_CODE (TYPE_ARRAY_MAX_SIZE (TREE_TYPE (exp)))
8960 : != INTEGER_CST)
8961 0 : && GET_MODE (x) == BLKmode)
8962 : /* If X is in the outgoing argument area, it is always safe. */
8963 10994374 : || (MEM_P (x)
8964 182764 : && (XEXP (x, 0) == virtual_outgoing_args_rtx
8965 182764 : || (GET_CODE (XEXP (x, 0)) == PLUS
8966 133906 : && XEXP (XEXP (x, 0), 0) == virtual_outgoing_args_rtx))))
8967 : return true;
8968 :
8969 : /* If this is a subreg of a hard register, declare it unsafe, otherwise,
8970 : find the underlying pseudo. */
8971 2026414 : if (GET_CODE (x) == SUBREG)
8972 : {
8973 0 : x = SUBREG_REG (x);
8974 0 : if (REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER)
8975 : return false;
8976 : }
8977 :
8978 : /* Now look at our tree code and possibly recurse. */
8979 2026414 : switch (TREE_CODE_CLASS (TREE_CODE (exp)))
8980 : {
8981 602 : case tcc_declaration:
8982 602 : exp_rtl = DECL_RTL_IF_SET (exp);
8983 : break;
8984 :
8985 : case tcc_constant:
8986 : return true;
8987 :
8988 874424 : case tcc_exceptional:
8989 874424 : if (TREE_CODE (exp) == TREE_LIST)
8990 : {
8991 0 : while (1)
8992 : {
8993 0 : if (TREE_VALUE (exp) && !safe_from_p (x, TREE_VALUE (exp), 0))
8994 : return false;
8995 0 : exp = TREE_CHAIN (exp);
8996 0 : if (!exp)
8997 : return true;
8998 0 : if (TREE_CODE (exp) != TREE_LIST)
8999 : return safe_from_p (x, exp, 0);
9000 : }
9001 : }
9002 874424 : else if (TREE_CODE (exp) == CONSTRUCTOR)
9003 : {
9004 : constructor_elt *ce;
9005 : unsigned HOST_WIDE_INT idx;
9006 :
9007 8265515 : FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (exp), idx, ce)
9008 6206 : if ((ce->index != NULL_TREE && !safe_from_p (x, ce->index, 0))
9009 137516 : || !safe_from_p (x, ce->value, 0))
9010 : return false;
9011 : return true;
9012 : }
9013 728137 : else if (TREE_CODE (exp) == ERROR_MARK)
9014 : return true; /* An already-visited SAVE_EXPR? */
9015 : else
9016 : return false;
9017 :
9018 0 : case tcc_statement:
9019 : /* The only case we look at here is the DECL_INITIAL inside a
9020 : DECL_EXPR. */
9021 0 : return (TREE_CODE (exp) != DECL_EXPR
9022 0 : || TREE_CODE (DECL_EXPR_DECL (exp)) != VAR_DECL
9023 0 : || !DECL_INITIAL (DECL_EXPR_DECL (exp))
9024 0 : || safe_from_p (x, DECL_INITIAL (DECL_EXPR_DECL (exp)), 0));
9025 :
9026 0 : case tcc_binary:
9027 0 : case tcc_comparison:
9028 0 : if (!safe_from_p (x, TREE_OPERAND (exp, 1), 0))
9029 : return false;
9030 : /* Fall through. */
9031 :
9032 16 : case tcc_unary:
9033 16 : return safe_from_p (x, TREE_OPERAND (exp, 0), 0);
9034 :
9035 473 : case tcc_expression:
9036 473 : case tcc_reference:
9037 473 : case tcc_vl_exp:
9038 : /* Now do code-specific tests. EXP_RTL is set to any rtx we find in
9039 : the expression. If it is set, we conflict iff we are that rtx or
9040 : both are in memory. Otherwise, we check all operands of the
9041 : expression recursively. */
9042 :
9043 473 : switch (TREE_CODE (exp))
9044 : {
9045 385 : case ADDR_EXPR:
9046 : /* If the operand is static or we are static, we can't conflict.
9047 : Likewise if we don't conflict with the operand at all. */
9048 385 : if (staticp (TREE_OPERAND (exp, 0))
9049 235 : || TREE_STATIC (exp)
9050 620 : || safe_from_p (x, TREE_OPERAND (exp, 0), 0))
9051 : return true;
9052 :
9053 : /* Otherwise, the only way this can conflict is if we are taking
9054 : the address of a DECL a that address if part of X, which is
9055 : very rare. */
9056 0 : exp = TREE_OPERAND (exp, 0);
9057 0 : if (DECL_P (exp))
9058 : {
9059 0 : if (!DECL_RTL_SET_P (exp)
9060 0 : || !MEM_P (DECL_RTL (exp)))
9061 : return false;
9062 : else
9063 0 : exp_rtl = XEXP (DECL_RTL (exp), 0);
9064 : }
9065 : break;
9066 :
9067 73 : case MEM_REF:
9068 73 : if (MEM_P (x)
9069 73 : && alias_sets_conflict_p (MEM_ALIAS_SET (x),
9070 : get_alias_set (exp)))
9071 : return false;
9072 : break;
9073 :
9074 0 : case CALL_EXPR:
9075 : /* Assume that the call will clobber all hard registers and
9076 : all of memory. */
9077 0 : if ((REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER)
9078 0 : || MEM_P (x))
9079 : return false;
9080 : break;
9081 :
9082 0 : case WITH_CLEANUP_EXPR:
9083 0 : case CLEANUP_POINT_EXPR:
9084 : /* Lowered by gimplify.cc. */
9085 0 : gcc_unreachable ();
9086 :
9087 0 : case SAVE_EXPR:
9088 0 : return safe_from_p (x, TREE_OPERAND (exp, 0), 0);
9089 :
9090 : default:
9091 : break;
9092 : }
9093 :
9094 : /* If we have an rtx, we do not need to scan our operands. */
9095 0 : if (exp_rtl)
9096 : break;
9097 :
9098 88 : nops = TREE_OPERAND_LENGTH (exp);
9099 371 : for (i = 0; i < nops; i++)
9100 195 : if (TREE_OPERAND (exp, i) != 0
9101 195 : && ! safe_from_p (x, TREE_OPERAND (exp, i), 0))
9102 : return false;
9103 :
9104 : break;
9105 :
9106 0 : case tcc_type:
9107 : /* Should never get a type here. */
9108 0 : gcc_unreachable ();
9109 : }
9110 :
9111 : /* If we have an rtl, find any enclosed object. Then see if we conflict
9112 : with it. */
9113 394 : if (exp_rtl)
9114 : {
9115 306 : if (GET_CODE (exp_rtl) == SUBREG)
9116 : {
9117 0 : exp_rtl = SUBREG_REG (exp_rtl);
9118 0 : if (REG_P (exp_rtl)
9119 0 : && REGNO (exp_rtl) < FIRST_PSEUDO_REGISTER)
9120 : return false;
9121 : }
9122 :
9123 : /* If the rtl is X, then it is not safe. Otherwise, it is unless both
9124 : are memory and they conflict. */
9125 306 : return ! (rtx_equal_p (x, exp_rtl)
9126 306 : || (MEM_P (x) && MEM_P (exp_rtl)
9127 4 : && true_dependence (exp_rtl, VOIDmode, x)));
9128 : }
9129 :
9130 : /* If we reach here, it is safe. */
9131 : return true;
9132 : }
9133 :
9134 :
9135 : /* Return the highest power of two that EXP is known to be a multiple of.
9136 : This is used in updating alignment of MEMs in array references. */
9137 :
9138 : unsigned HOST_WIDE_INT
9139 33710016 : highest_pow2_factor (const_tree exp)
9140 : {
9141 33710016 : unsigned HOST_WIDE_INT ret;
9142 33710016 : int trailing_zeros = tree_ctz (exp);
9143 33710016 : if (trailing_zeros >= HOST_BITS_PER_WIDE_INT)
9144 23731900 : return BIGGEST_ALIGNMENT;
9145 9978116 : ret = HOST_WIDE_INT_1U << trailing_zeros;
9146 19488935 : if (ret > BIGGEST_ALIGNMENT)
9147 6632909 : return BIGGEST_ALIGNMENT;
9148 : return ret;
9149 : }
9150 :
9151 : /* Similar, except that the alignment requirements of TARGET are
9152 : taken into account. Assume it is at least as aligned as its
9153 : type, unless it is a COMPONENT_REF in which case the layout of
9154 : the structure gives the alignment. */
9155 :
9156 : static unsigned HOST_WIDE_INT
9157 179955 : highest_pow2_factor_for_target (const_tree target, const_tree exp)
9158 : {
9159 179955 : unsigned HOST_WIDE_INT talign = target_align (target) / BITS_PER_UNIT;
9160 179955 : unsigned HOST_WIDE_INT factor = highest_pow2_factor (exp);
9161 :
9162 179955 : return MAX (factor, talign);
9163 : }
9164 :
9165 : /* Convert the tree comparison code TCODE to the rtl one where the
9166 : signedness is UNSIGNEDP. */
9167 :
9168 : enum rtx_code
9169 21721 : convert_tree_comp_to_rtx (enum tree_code tcode, int unsignedp)
9170 : {
9171 21721 : enum rtx_code code;
9172 21721 : switch (tcode)
9173 : {
9174 : case EQ_EXPR:
9175 : code = EQ;
9176 : break;
9177 2477 : case NE_EXPR:
9178 2477 : code = NE;
9179 2477 : break;
9180 4050 : case LT_EXPR:
9181 4050 : code = unsignedp ? LTU : LT;
9182 : break;
9183 3096 : case LE_EXPR:
9184 3096 : code = unsignedp ? LEU : LE;
9185 : break;
9186 3241 : case GT_EXPR:
9187 3241 : code = unsignedp ? GTU : GT;
9188 : break;
9189 4345 : case GE_EXPR:
9190 4345 : code = unsignedp ? GEU : GE;
9191 : break;
9192 35 : case UNORDERED_EXPR:
9193 35 : code = UNORDERED;
9194 35 : break;
9195 35 : case ORDERED_EXPR:
9196 35 : code = ORDERED;
9197 35 : break;
9198 10 : case UNLT_EXPR:
9199 10 : code = UNLT;
9200 10 : break;
9201 19 : case UNLE_EXPR:
9202 19 : code = UNLE;
9203 19 : break;
9204 10 : case UNGT_EXPR:
9205 10 : code = UNGT;
9206 10 : break;
9207 11 : case UNGE_EXPR:
9208 11 : code = UNGE;
9209 11 : break;
9210 4 : case UNEQ_EXPR:
9211 4 : code = UNEQ;
9212 4 : break;
9213 4 : case LTGT_EXPR:
9214 4 : code = LTGT;
9215 4 : break;
9216 :
9217 0 : default:
9218 0 : gcc_unreachable ();
9219 : }
9220 21721 : return code;
9221 : }
9222 :
9223 : /* Subroutine of expand_expr. Expand the two operands of a binary
9224 : expression EXP0 and EXP1 placing the results in OP0 and OP1.
9225 : The value may be stored in TARGET if TARGET is nonzero. The
9226 : MODIFIER argument is as documented by expand_expr. */
9227 :
9228 : void
9229 8345929 : expand_operands (tree exp0, tree exp1, rtx target, rtx *op0, rtx *op1,
9230 : enum expand_modifier modifier)
9231 : {
9232 8345929 : if (! safe_from_p (target, exp1, 1))
9233 604126 : target = 0;
9234 8345929 : if (operand_equal_p (exp0, exp1, 0))
9235 : {
9236 46543 : *op0 = expand_expr (exp0, target, VOIDmode, modifier);
9237 46543 : *op1 = copy_rtx (*op0);
9238 : }
9239 : else
9240 : {
9241 8299386 : *op0 = expand_expr (exp0, target, VOIDmode, modifier);
9242 8299386 : *op1 = expand_expr (exp1, NULL_RTX, VOIDmode, modifier);
9243 : }
9244 8345929 : }
9245 :
9246 :
9247 : /* Return a MEM that contains constant EXP. DEFER is as for
9248 : output_constant_def and MODIFIER is as for expand_expr. */
9249 :
9250 : static rtx
9251 2992079 : expand_expr_constant (tree exp, int defer, enum expand_modifier modifier)
9252 : {
9253 2992079 : rtx mem;
9254 :
9255 2992079 : mem = output_constant_def (exp, defer);
9256 2992079 : if (modifier != EXPAND_INITIALIZER)
9257 1911110 : mem = use_anchored_address (mem);
9258 2992079 : return mem;
9259 : }
9260 :
9261 : /* A subroutine of expand_expr_addr_expr. Evaluate the address of EXP.
9262 : The TARGET, TMODE and MODIFIER arguments are as for expand_expr. */
9263 :
9264 : static rtx
9265 14682251 : expand_expr_addr_expr_1 (tree exp, rtx target, scalar_int_mode tmode,
9266 : enum expand_modifier modifier, addr_space_t as)
9267 : {
9268 14682251 : rtx result, subtarget;
9269 14682251 : tree inner, offset;
9270 14682251 : poly_int64 bitsize, bitpos;
9271 14682251 : int unsignedp, reversep, volatilep = 0;
9272 14682251 : machine_mode mode1;
9273 :
9274 : /* If we are taking the address of a constant and are at the top level,
9275 : we have to use output_constant_def since we can't call force_const_mem
9276 : at top level. */
9277 : /* ??? This should be considered a front-end bug. We should not be
9278 : generating ADDR_EXPR of something that isn't an LVALUE. The only
9279 : exception here is STRING_CST. */
9280 14682251 : if (CONSTANT_CLASS_P (exp))
9281 : {
9282 2725585 : result = XEXP (expand_expr_constant (exp, 0, modifier), 0);
9283 2725585 : if (modifier < EXPAND_SUM)
9284 1819009 : result = force_operand (result, target);
9285 : return result;
9286 : }
9287 :
9288 : /* Everything must be something allowed by is_gimple_addressable. */
9289 11956666 : switch (TREE_CODE (exp))
9290 : {
9291 46 : case INDIRECT_REF:
9292 : /* This case will happen via recursion for &a->b. */
9293 46 : return expand_expr (TREE_OPERAND (exp, 0), target, tmode, modifier);
9294 :
9295 590865 : case MEM_REF:
9296 590865 : {
9297 590865 : tree tem = TREE_OPERAND (exp, 0);
9298 590865 : if (!integer_zerop (TREE_OPERAND (exp, 1)))
9299 314553 : tem = fold_build_pointer_plus (tem, TREE_OPERAND (exp, 1));
9300 590865 : return expand_expr (tem, target, tmode, modifier);
9301 : }
9302 :
9303 1149 : case TARGET_MEM_REF:
9304 1149 : return addr_for_mem_ref (exp, as, true);
9305 :
9306 60375 : case CONST_DECL:
9307 : /* Expand the initializer like constants above. */
9308 60375 : result = XEXP (expand_expr_constant (DECL_INITIAL (exp),
9309 : 0, modifier), 0);
9310 60375 : if (modifier < EXPAND_SUM)
9311 60369 : result = force_operand (result, target);
9312 : return result;
9313 :
9314 209 : case REALPART_EXPR:
9315 : /* The real part of the complex number is always first, therefore
9316 : the address is the same as the address of the parent object. */
9317 209 : offset = 0;
9318 209 : bitpos = 0;
9319 209 : inner = TREE_OPERAND (exp, 0);
9320 209 : break;
9321 :
9322 126 : case IMAGPART_EXPR:
9323 : /* The imaginary part of the complex number is always second.
9324 : The expression is therefore always offset by the size of the
9325 : scalar type. */
9326 126 : offset = 0;
9327 252 : bitpos = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (exp)));
9328 126 : inner = TREE_OPERAND (exp, 0);
9329 126 : break;
9330 :
9331 13 : case COMPOUND_LITERAL_EXPR:
9332 : /* Allow COMPOUND_LITERAL_EXPR in initializers or coming from
9333 : initializers, if e.g. rtl_for_decl_init is called on DECL_INITIAL
9334 : with COMPOUND_LITERAL_EXPRs in it, or ARRAY_REF on a const static
9335 : array with address of COMPOUND_LITERAL_EXPR in DECL_INITIAL;
9336 : the initializers aren't gimplified. */
9337 13 : if (COMPOUND_LITERAL_EXPR_DECL (exp)
9338 13 : && is_global_var (COMPOUND_LITERAL_EXPR_DECL (exp)))
9339 13 : return expand_expr_addr_expr_1 (COMPOUND_LITERAL_EXPR_DECL (exp),
9340 13 : target, tmode, modifier, as);
9341 : /* FALLTHRU */
9342 11303883 : default:
9343 : /* If the object is a DECL, then expand it for its rtl. Don't bypass
9344 : expand_expr, as that can have various side effects; LABEL_DECLs for
9345 : example, may not have their DECL_RTL set yet. Expand the rtl of
9346 : CONSTRUCTORs too, which should yield a memory reference for the
9347 : constructor's contents. Assume language specific tree nodes can
9348 : be expanded in some interesting way. */
9349 11303883 : gcc_assert (TREE_CODE (exp) < LAST_AND_UNUSED_TREE_CODE);
9350 11303883 : if (DECL_P (exp)
9351 1353905 : || TREE_CODE (exp) == CONSTRUCTOR
9352 1353905 : || TREE_CODE (exp) == COMPOUND_LITERAL_EXPR)
9353 : {
9354 16379393 : result = expand_expr (exp, target, tmode,
9355 : modifier == EXPAND_INITIALIZER
9356 : ? EXPAND_INITIALIZER : EXPAND_CONST_ADDRESS);
9357 :
9358 : /* If the DECL isn't in memory, then the DECL wasn't properly
9359 : marked TREE_ADDRESSABLE, which will be either a front-end
9360 : or a tree optimizer bug. */
9361 :
9362 9949978 : gcc_assert (MEM_P (result));
9363 9949976 : result = XEXP (result, 0);
9364 :
9365 : /* ??? Is this needed anymore? */
9366 9949976 : if (DECL_P (exp))
9367 9949976 : TREE_USED (exp) = 1;
9368 :
9369 9949976 : if (modifier != EXPAND_INITIALIZER
9370 : && modifier != EXPAND_CONST_ADDRESS
9371 9949976 : && modifier != EXPAND_SUM)
9372 4361832 : result = force_operand (result, target);
9373 : return result;
9374 : }
9375 :
9376 : /* Pass FALSE as the last argument to get_inner_reference although
9377 : we are expanding to RTL. The rationale is that we know how to
9378 : handle "aligning nodes" here: we can just bypass them because
9379 : they won't change the final object whose address will be returned
9380 : (they actually exist only for that purpose). */
9381 1353905 : inner = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
9382 : &unsignedp, &reversep, &volatilep);
9383 1353905 : break;
9384 : }
9385 :
9386 : /* We must have made progress. */
9387 1354240 : gcc_assert (inner != exp);
9388 :
9389 1354240 : subtarget = offset || maybe_ne (bitpos, 0) ? NULL_RTX : target;
9390 : /* For VIEW_CONVERT_EXPR, where the outer alignment is bigger than
9391 : inner alignment, force the inner to be sufficiently aligned. */
9392 1354240 : if (CONSTANT_CLASS_P (inner)
9393 1354240 : && TYPE_ALIGN (TREE_TYPE (inner)) < TYPE_ALIGN (TREE_TYPE (exp)))
9394 : {
9395 0 : inner = copy_node (inner);
9396 0 : TREE_TYPE (inner) = copy_node (TREE_TYPE (inner));
9397 0 : SET_TYPE_ALIGN (TREE_TYPE (inner), TYPE_ALIGN (TREE_TYPE (exp)));
9398 0 : TYPE_USER_ALIGN (TREE_TYPE (inner)) = 1;
9399 : }
9400 1354240 : result = expand_expr_addr_expr_1 (inner, subtarget, tmode, modifier, as);
9401 :
9402 1354240 : if (offset)
9403 : {
9404 52874 : rtx tmp;
9405 :
9406 52874 : if (modifier != EXPAND_NORMAL)
9407 1702 : result = force_operand (result, NULL);
9408 54576 : tmp = expand_expr (offset, NULL_RTX, tmode,
9409 : modifier == EXPAND_INITIALIZER
9410 : ? EXPAND_INITIALIZER : EXPAND_NORMAL);
9411 :
9412 : /* expand_expr is allowed to return an object in a mode other
9413 : than TMODE. If it did, we need to convert. */
9414 52874 : if (GET_MODE (tmp) != VOIDmode && tmode != GET_MODE (tmp))
9415 0 : tmp = convert_modes (tmode, GET_MODE (tmp),
9416 0 : tmp, TYPE_UNSIGNED (TREE_TYPE (offset)));
9417 52874 : result = convert_memory_address_addr_space (tmode, result, as);
9418 52874 : tmp = convert_memory_address_addr_space (tmode, tmp, as);
9419 :
9420 52874 : if (modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER)
9421 1702 : result = simplify_gen_binary (PLUS, tmode, result, tmp);
9422 : else
9423 : {
9424 51172 : subtarget = maybe_ne (bitpos, 0) ? NULL_RTX : target;
9425 51172 : result = expand_simple_binop (tmode, PLUS, result, tmp, subtarget,
9426 : 1, OPTAB_LIB_WIDEN);
9427 : }
9428 : }
9429 :
9430 1354240 : if (maybe_ne (bitpos, 0))
9431 : {
9432 : /* Someone beforehand should have rejected taking the address
9433 : of an object that isn't byte-aligned. */
9434 644848 : poly_int64 bytepos = exact_div (bitpos, BITS_PER_UNIT);
9435 644848 : result = convert_memory_address_addr_space (tmode, result, as);
9436 644848 : result = plus_constant (tmode, result, bytepos);
9437 644848 : if (modifier < EXPAND_SUM)
9438 607271 : result = force_operand (result, target);
9439 : }
9440 :
9441 : return result;
9442 : }
9443 :
9444 : /* A subroutine of expand_expr. Evaluate EXP, which is an ADDR_EXPR.
9445 : The TARGET, TMODE and MODIFIER arguments are as for expand_expr. */
9446 :
9447 : static rtx
9448 13327996 : expand_expr_addr_expr (tree exp, rtx target, machine_mode tmode,
9449 : enum expand_modifier modifier)
9450 : {
9451 13327996 : addr_space_t as = ADDR_SPACE_GENERIC;
9452 13327996 : scalar_int_mode address_mode = Pmode;
9453 13327996 : scalar_int_mode pointer_mode = ptr_mode;
9454 13327996 : machine_mode rmode;
9455 13327996 : rtx result;
9456 :
9457 : /* Target mode of VOIDmode says "whatever's natural". */
9458 13327996 : if (tmode == VOIDmode)
9459 11364948 : tmode = TYPE_MODE (TREE_TYPE (exp));
9460 :
9461 13327996 : if (POINTER_TYPE_P (TREE_TYPE (exp)))
9462 : {
9463 13327996 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)));
9464 13327996 : address_mode = targetm.addr_space.address_mode (as);
9465 13327996 : pointer_mode = targetm.addr_space.pointer_mode (as);
9466 : }
9467 :
9468 : /* We can get called with some Weird Things if the user does silliness
9469 : like "(short) &a". In that case, convert_memory_address won't do
9470 : the right thing, so ignore the given target mode. */
9471 13327996 : scalar_int_mode new_tmode = (tmode == pointer_mode
9472 13327996 : ? pointer_mode
9473 13327996 : : address_mode);
9474 :
9475 13327996 : result = expand_expr_addr_expr_1 (TREE_OPERAND (exp, 0), target,
9476 : new_tmode, modifier, as);
9477 :
9478 : /* Despite expand_expr claims concerning ignoring TMODE when not
9479 : strictly convenient, stuff breaks if we don't honor it. Note
9480 : that combined with the above, we only do this for pointer modes. */
9481 13327994 : rmode = GET_MODE (result);
9482 13327994 : if (rmode == VOIDmode)
9483 6 : rmode = new_tmode;
9484 13327994 : if (rmode != new_tmode)
9485 74 : result = convert_memory_address_addr_space (new_tmode, result, as);
9486 :
9487 13327994 : return result;
9488 : }
9489 :
9490 : /* Generate code for computing CONSTRUCTOR EXP.
9491 : An rtx for the computed value is returned. If AVOID_TEMP_MEM
9492 : is TRUE, instead of creating a temporary variable in memory
9493 : NULL is returned and the caller needs to handle it differently. */
9494 :
9495 : static rtx
9496 181816 : expand_constructor (tree exp, rtx target, enum expand_modifier modifier,
9497 : bool avoid_temp_mem)
9498 : {
9499 181816 : tree type = TREE_TYPE (exp);
9500 181816 : machine_mode mode = TYPE_MODE (type);
9501 :
9502 : /* Try to avoid creating a temporary at all. This is possible
9503 : if all of the initializer is zero.
9504 : FIXME: try to handle all [0..255] initializers we can handle
9505 : with memset. */
9506 181816 : if (TREE_STATIC (exp)
9507 2684 : && !TREE_ADDRESSABLE (exp)
9508 2684 : && target != 0 && mode == BLKmode
9509 182967 : && all_zeros_p (exp))
9510 : {
9511 1142 : clear_storage (target, expr_size (exp), BLOCK_OP_NORMAL);
9512 1142 : return target;
9513 : }
9514 :
9515 : /* All elts simple constants => refer to a constant in memory. But
9516 : if this is a non-BLKmode mode, let it store a field at a time
9517 : since that should make a CONST_INT, CONST_WIDE_INT or
9518 : CONST_DOUBLE when we fold. Likewise, if we have a target we can
9519 : use, it is best to store directly into the target unless the type
9520 : is large enough that memcpy will be used. If we are making an
9521 : initializer and all operands are constant, put it in memory as
9522 : well.
9523 :
9524 : FIXME: Avoid trying to fill vector constructors piece-meal.
9525 : Output them with output_constant_def below unless we're sure
9526 : they're zeros. This should go away when vector initializers
9527 : are treated like VECTOR_CST instead of arrays. */
9528 180674 : if ((TREE_STATIC (exp)
9529 1542 : && ((mode == BLKmode
9530 46 : && ! (target != 0 && safe_from_p (target, exp, 1)))
9531 1505 : || TREE_ADDRESSABLE (exp)
9532 1505 : || (tree_fits_uhwi_p (TYPE_SIZE_UNIT (type))
9533 1505 : && (! can_move_by_pieces
9534 1505 : (tree_to_uhwi (TYPE_SIZE_UNIT (type)),
9535 1505 : TYPE_ALIGN (type)))
9536 4 : && ! mostly_zeros_p (exp))))
9537 182177 : || ((modifier == EXPAND_INITIALIZER || modifier == EXPAND_CONST_ADDRESS)
9538 0 : && TREE_CONSTANT (exp)))
9539 : {
9540 39 : rtx constructor;
9541 :
9542 39 : if (avoid_temp_mem)
9543 : return NULL_RTX;
9544 :
9545 37 : constructor = expand_expr_constant (exp, 1, modifier);
9546 :
9547 37 : if (modifier != EXPAND_CONST_ADDRESS
9548 : && modifier != EXPAND_INITIALIZER
9549 37 : && modifier != EXPAND_SUM)
9550 37 : constructor = validize_mem (constructor);
9551 :
9552 : return constructor;
9553 : }
9554 :
9555 : /* If the CTOR is available in static storage and not mostly
9556 : zeros and we can move it by pieces prefer to do so since
9557 : that's usually more efficient than performing a series of
9558 : stores from immediates. */
9559 180635 : if (avoid_temp_mem
9560 97 : && TREE_STATIC (exp)
9561 38 : && TREE_CONSTANT (exp)
9562 38 : && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type))
9563 38 : && can_move_by_pieces (tree_to_uhwi (TYPE_SIZE_UNIT (type)),
9564 38 : TYPE_ALIGN (type))
9565 180672 : && ! mostly_zeros_p (exp))
9566 : return NULL_RTX;
9567 :
9568 : /* Handle calls that pass values in multiple non-contiguous
9569 : locations. The Irix 6 ABI has examples of this. */
9570 146234 : if (target == 0 || ! safe_from_p (target, exp, 1)
9571 39333 : || GET_CODE (target) == PARALLEL || modifier == EXPAND_STACK_PARM
9572 : /* Also make a temporary if the store is to volatile memory, to
9573 : avoid individual accesses to aggregate members. */
9574 219926 : || (GET_CODE (target) == MEM
9575 34744 : && MEM_VOLATILE_P (target)
9576 134 : && !TREE_ADDRESSABLE (TREE_TYPE (exp))))
9577 : {
9578 141412 : if (avoid_temp_mem)
9579 : return NULL_RTX;
9580 :
9581 141401 : target = assign_temp (type, TREE_ADDRESSABLE (exp), 1);
9582 : }
9583 :
9584 180594 : store_constructor (exp, target, 0, int_expr_size (exp), false);
9585 180594 : return target;
9586 : }
9587 :
9588 :
9589 : /* expand_expr: generate code for computing expression EXP.
9590 : An rtx for the computed value is returned. The value is never null.
9591 : In the case of a void EXP, const0_rtx is returned.
9592 :
9593 : The value may be stored in TARGET if TARGET is nonzero.
9594 : TARGET is just a suggestion; callers must assume that
9595 : the rtx returned may not be the same as TARGET.
9596 :
9597 : If TARGET is CONST0_RTX, it means that the value will be ignored.
9598 :
9599 : If TMODE is not VOIDmode, it suggests generating the
9600 : result in mode TMODE. But this is done only when convenient.
9601 : Otherwise, TMODE is ignored and the value generated in its natural mode.
9602 : TMODE is just a suggestion; callers must assume that
9603 : the rtx returned may not have mode TMODE.
9604 :
9605 : Note that TARGET may have neither TMODE nor MODE. In that case, it
9606 : probably will not be used.
9607 :
9608 : If MODIFIER is EXPAND_SUM then when EXP is an addition
9609 : we can return an rtx of the form (MULT (REG ...) (CONST_INT ...))
9610 : or a nest of (PLUS ...) and (MINUS ...) where the terms are
9611 : products as above, or REG or MEM, or constant.
9612 : Ordinarily in such cases we would output mul or add instructions
9613 : and then return a pseudo reg containing the sum.
9614 :
9615 : EXPAND_INITIALIZER is much like EXPAND_SUM except that
9616 : it also marks a label as absolutely required (it can't be dead).
9617 : It also makes a ZERO_EXTEND or SIGN_EXTEND instead of emitting extend insns.
9618 : This is used for outputting expressions used in initializers.
9619 :
9620 : EXPAND_CONST_ADDRESS says that it is okay to return a MEM
9621 : with a constant address even if that address is not normally legitimate.
9622 : EXPAND_INITIALIZER and EXPAND_SUM also have this effect.
9623 :
9624 : EXPAND_STACK_PARM is used when expanding to a TARGET on the stack for
9625 : a call parameter. Such targets require special care as we haven't yet
9626 : marked TARGET so that it's safe from being trashed by libcalls. We
9627 : don't want to use TARGET for anything but the final result;
9628 : Intermediate values must go elsewhere. Additionally, calls to
9629 : emit_block_move will be flagged with BLOCK_OP_CALL_PARM.
9630 :
9631 : If EXP is a VAR_DECL whose DECL_RTL was a MEM with an invalid
9632 : address, and ALT_RTL is non-NULL, then *ALT_RTL is set to the
9633 : DECL_RTL of the VAR_DECL. *ALT_RTL is also set if EXP is a
9634 : COMPOUND_EXPR whose second argument is such a VAR_DECL, and so on
9635 : recursively.
9636 : If the result can be stored at TARGET, and ALT_RTL is non-NULL,
9637 : then *ALT_RTL is set to TARGET (before legitimziation).
9638 :
9639 : If INNER_REFERENCE_P is true, we are expanding an inner reference.
9640 : In this case, we don't adjust a returned MEM rtx that wouldn't be
9641 : sufficiently aligned for its mode; instead, it's up to the caller
9642 : to deal with it afterwards. This is used to make sure that unaligned
9643 : base objects for which out-of-bounds accesses are supported, for
9644 : example record types with trailing arrays, aren't realigned behind
9645 : the back of the caller.
9646 : The normal operating mode is to pass FALSE for this parameter. */
9647 :
9648 : rtx
9649 160474715 : expand_expr_real (tree exp, rtx target, machine_mode tmode,
9650 : enum expand_modifier modifier, rtx *alt_rtl,
9651 : bool inner_reference_p)
9652 : {
9653 160474715 : rtx ret;
9654 :
9655 : /* Handle ERROR_MARK before anybody tries to access its type. */
9656 160474715 : if (TREE_CODE (exp) == ERROR_MARK
9657 160474715 : || (TREE_CODE (TREE_TYPE (exp)) == ERROR_MARK))
9658 : {
9659 0 : ret = CONST0_RTX (tmode);
9660 0 : return ret ? ret : const0_rtx;
9661 : }
9662 :
9663 160474715 : ret = expand_expr_real_1 (exp, target, tmode, modifier, alt_rtl,
9664 : inner_reference_p);
9665 160474715 : return ret;
9666 : }
9667 :
9668 : /* Try to expand the conditional expression which is represented by
9669 : TREEOP0 ? TREEOP1 : TREEOP2 using conditional moves. If it succeeds
9670 : return the rtl reg which represents the result. Otherwise return
9671 : NULL_RTX. */
9672 :
9673 : static rtx
9674 17770 : expand_cond_expr_using_cmove (tree treeop0 ATTRIBUTE_UNUSED,
9675 : tree treeop1 ATTRIBUTE_UNUSED,
9676 : tree treeop2 ATTRIBUTE_UNUSED)
9677 : {
9678 17770 : rtx insn;
9679 17770 : rtx op00, op01, op1, op2;
9680 17770 : enum rtx_code comparison_code;
9681 17770 : machine_mode comparison_mode;
9682 17770 : gimple *srcstmt;
9683 17770 : rtx temp;
9684 17770 : tree type = TREE_TYPE (treeop1);
9685 17770 : int unsignedp = TYPE_UNSIGNED (type);
9686 17770 : machine_mode mode = TYPE_MODE (type);
9687 17770 : machine_mode orig_mode = mode;
9688 17770 : static bool expanding_cond_expr_using_cmove = false;
9689 :
9690 : /* Conditional move expansion can end up TERing two operands which,
9691 : when recursively hitting conditional expressions can result in
9692 : exponential behavior if the cmove expansion ultimatively fails.
9693 : It's hardly profitable to TER a cmove into a cmove so avoid doing
9694 : that by failing early if we end up recursing. */
9695 17770 : if (expanding_cond_expr_using_cmove)
9696 : return NULL_RTX;
9697 :
9698 : /* If we cannot do a conditional move on the mode, try doing it
9699 : with the promoted mode. */
9700 17525 : if (!can_conditionally_move_p (mode))
9701 : {
9702 288 : mode = promote_mode (type, mode, &unsignedp);
9703 288 : if (!can_conditionally_move_p (mode))
9704 : return NULL_RTX;
9705 0 : temp = assign_temp (type, 0, 0); /* Use promoted mode for temp. */
9706 : }
9707 : else
9708 17237 : temp = assign_temp (type, 0, 1);
9709 :
9710 17237 : expanding_cond_expr_using_cmove = true;
9711 17237 : start_sequence ();
9712 17237 : expand_operands (treeop1, treeop2,
9713 : mode == orig_mode ? temp : NULL_RTX, &op1, &op2,
9714 : EXPAND_NORMAL);
9715 :
9716 17237 : if (TREE_CODE (treeop0) == SSA_NAME
9717 17074 : && (srcstmt = get_def_for_expr_class (treeop0, tcc_comparison))
9718 32049 : && !VECTOR_TYPE_P (TREE_TYPE (gimple_assign_rhs1 (srcstmt))))
9719 : {
9720 14812 : type = TREE_TYPE (gimple_assign_rhs1 (srcstmt));
9721 14812 : enum tree_code cmpcode = gimple_assign_rhs_code (srcstmt);
9722 14812 : op00 = expand_normal (gimple_assign_rhs1 (srcstmt));
9723 14812 : op01 = expand_normal (gimple_assign_rhs2 (srcstmt));
9724 14812 : comparison_mode = TYPE_MODE (type);
9725 14812 : unsignedp = TYPE_UNSIGNED (type);
9726 14812 : comparison_code = convert_tree_comp_to_rtx (cmpcode, unsignedp);
9727 : }
9728 2425 : else if (COMPARISON_CLASS_P (treeop0)
9729 2425 : && !VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (treeop0, 0))))
9730 : {
9731 163 : type = TREE_TYPE (TREE_OPERAND (treeop0, 0));
9732 163 : enum tree_code cmpcode = TREE_CODE (treeop0);
9733 163 : op00 = expand_normal (TREE_OPERAND (treeop0, 0));
9734 163 : op01 = expand_normal (TREE_OPERAND (treeop0, 1));
9735 163 : unsignedp = TYPE_UNSIGNED (type);
9736 163 : comparison_mode = TYPE_MODE (type);
9737 163 : comparison_code = convert_tree_comp_to_rtx (cmpcode, unsignedp);
9738 : }
9739 : else
9740 : {
9741 2262 : op00 = expand_normal (treeop0);
9742 2262 : op01 = const0_rtx;
9743 2262 : comparison_code = NE;
9744 2262 : comparison_mode = GET_MODE (op00);
9745 2262 : if (comparison_mode == VOIDmode)
9746 0 : comparison_mode = TYPE_MODE (TREE_TYPE (treeop0));
9747 : }
9748 17237 : expanding_cond_expr_using_cmove = false;
9749 :
9750 17237 : if (GET_MODE (op1) != mode)
9751 2365 : op1 = gen_lowpart (mode, op1);
9752 :
9753 17237 : if (GET_MODE (op2) != mode)
9754 9180 : op2 = gen_lowpart (mode, op2);
9755 :
9756 : /* Try to emit the conditional move. */
9757 17237 : insn = emit_conditional_move (temp,
9758 : { comparison_code, op00, op01,
9759 : comparison_mode },
9760 : op1, op2, mode,
9761 : unsignedp);
9762 :
9763 : /* If we could do the conditional move, emit the sequence,
9764 : and return. */
9765 17237 : if (insn)
9766 : {
9767 14706 : rtx_insn *seq = end_sequence ();
9768 14706 : emit_insn (seq);
9769 14706 : return convert_modes (orig_mode, mode, temp, 0);
9770 : }
9771 :
9772 : /* Otherwise discard the sequence and fall back to code with
9773 : branches. */
9774 2531 : end_sequence ();
9775 2531 : return NULL_RTX;
9776 : }
9777 :
9778 : /* A helper function for expand_expr_real_2 to be used with a
9779 : misaligned mem_ref TEMP. Assume an unsigned type if UNSIGNEDP
9780 : is nonzero, with alignment ALIGN in bits.
9781 : Store the value at TARGET if possible (if TARGET is nonzero).
9782 : Regardless of TARGET, we return the rtx for where the value is placed.
9783 : If the result can be stored at TARGET, and ALT_RTL is non-NULL,
9784 : then *ALT_RTL is set to TARGET (before legitimziation). */
9785 :
9786 : static rtx
9787 203401 : expand_misaligned_mem_ref (rtx temp, machine_mode mode, int unsignedp,
9788 : unsigned int align, rtx target, rtx *alt_rtl)
9789 : {
9790 203401 : enum insn_code icode;
9791 :
9792 203401 : if ((icode = optab_handler (movmisalign_optab, mode))
9793 : != CODE_FOR_nothing)
9794 : {
9795 127049 : class expand_operand ops[2];
9796 :
9797 : /* We've already validated the memory, and we're creating a
9798 : new pseudo destination. The predicates really can't fail,
9799 : nor can the generator. */
9800 127049 : create_output_operand (&ops[0], NULL_RTX, mode);
9801 127049 : create_fixed_operand (&ops[1], temp);
9802 127049 : expand_insn (icode, 2, ops);
9803 127049 : temp = ops[0].value;
9804 : }
9805 76352 : else if (targetm.slow_unaligned_access (mode, align))
9806 0 : temp = extract_bit_field (temp, GET_MODE_BITSIZE (mode),
9807 : 0, unsignedp, target,
9808 : mode, mode, false, alt_rtl);
9809 203401 : return temp;
9810 : }
9811 :
9812 : /* Return true if OP is known to be either LOWER or LOWER + 1, with one
9813 : value a positive power of two. */
9814 :
9815 : static bool
9816 60841 : near_pow2_divisor_range_p (tree op, wide_int &lower)
9817 : {
9818 60841 : if (TREE_CODE (op) != SSA_NAME)
9819 : return false;
9820 :
9821 32157 : int_range_max range;
9822 32157 : range_query *query = get_range_query (cfun);
9823 32157 : if (!query->range_of_expr (range, op, currently_expanding_gimple_stmt)
9824 32157 : || range.num_pairs () != 1)
9825 : return false;
9826 :
9827 30151 : lower = range.lower_bound ();
9828 30151 : wide_int upper = lower + 1;
9829 60302 : return (range.upper_bound () == upper
9830 99 : && wi::gt_p (lower, 0, TYPE_SIGN (TREE_TYPE (op)))
9831 30354 : && (wi::popcount (lower) == 1 || wi::popcount (upper) == 1));
9832 32157 : }
9833 :
9834 : /* Helper function of expand_expr_2, expand a division or modulo.
9835 : op0 and op1 should be already expanded treeop0 and treeop1, using
9836 : expand_operands. */
9837 :
9838 : static rtx
9839 169738 : expand_expr_divmod (tree_code code, machine_mode mode, tree treeop0,
9840 : tree treeop1, rtx op0, rtx op1, rtx target, int unsignedp)
9841 : {
9842 339476 : bool mod_p = (code == TRUNC_MOD_EXPR || code == FLOOR_MOD_EXPR
9843 169738 : || code == CEIL_MOD_EXPR || code == ROUND_MOD_EXPR);
9844 169738 : bool speed_p = optimize_insn_for_speed_p ();
9845 :
9846 169738 : scalar_int_mode int_mode;
9847 169738 : wide_int lower;
9848 : /* Split x / y when y is one of two neighboring constants and the target can
9849 : select between the constant divisions cheaply. */
9850 169738 : if (code == TRUNC_DIV_EXPR
9851 235741 : && is_a <scalar_int_mode> (mode, &int_mode)
9852 66076 : && speed_p
9853 63462 : && can_conditionally_move_p (int_mode)
9854 230579 : && near_pow2_divisor_range_p (treeop1, lower))
9855 : {
9856 73 : signop sgn = TYPE_SIGN (TREE_TYPE (treeop1));
9857 73 : unsigned int prec = GET_MODE_PRECISION (int_mode);
9858 73 : wide_int upper = lower + 1;
9859 73 : rtx op_lower
9860 73 : = immed_wide_int_const (wide_int::from (lower, prec, sgn), int_mode);
9861 73 : rtx op_upper
9862 73 : = immed_wide_int_const (wide_int::from (upper, prec, sgn), int_mode);
9863 :
9864 73 : do_pending_stack_adjust ();
9865 73 : start_sequence ();
9866 73 : rtx q_lower = expand_divmod (0, TRUNC_DIV_EXPR, mode, op0, op_lower,
9867 : NULL_RTX, unsignedp);
9868 73 : rtx q_upper = expand_divmod (0, TRUNC_DIV_EXPR, mode, op0, op_upper,
9869 : NULL_RTX, unsignedp);
9870 73 : rtx split_ret
9871 73 : = emit_conditional_move (target, { EQ, op1, op_lower, int_mode },
9872 : q_lower, q_upper, int_mode, unsignedp);
9873 73 : rtx_insn *split_insns = end_sequence ();
9874 :
9875 : /* Cost the unsplit form as a single DIV/UDIV. */
9876 73 : rtx div_rtx = gen_rtx_fmt_ee (unsignedp ? UDIV : DIV, int_mode, op0, op1);
9877 73 : unsigned div_cost = set_src_cost (div_rtx, int_mode, speed_p);
9878 73 : if (split_ret && seq_cost (split_insns, speed_p) < div_cost)
9879 : {
9880 73 : emit_insn (split_insns);
9881 73 : return split_ret;
9882 : }
9883 73 : }
9884 :
9885 169665 : if (SCALAR_INT_MODE_P (mode)
9886 169665 : && optimize >= 2
9887 142319 : && get_range_pos_neg (treeop0, currently_expanding_gimple_stmt) == 1
9888 212841 : && get_range_pos_neg (treeop1, currently_expanding_gimple_stmt) == 1)
9889 : {
9890 : /* If both arguments are known to be positive when interpreted
9891 : as signed, we can expand it as both signed and unsigned
9892 : division or modulo. Choose the cheaper sequence in that case. */
9893 13499 : do_pending_stack_adjust ();
9894 13499 : start_sequence ();
9895 13499 : rtx uns_ret = expand_divmod (mod_p, code, mode, op0, op1, target, 1);
9896 13499 : rtx_insn *uns_insns = end_sequence ();
9897 13499 : start_sequence ();
9898 13499 : rtx sgn_ret = expand_divmod (mod_p, code, mode, op0, op1, target, 0);
9899 13499 : rtx_insn *sgn_insns = end_sequence ();
9900 13499 : unsigned uns_cost = seq_cost (uns_insns, speed_p);
9901 13499 : unsigned sgn_cost = seq_cost (sgn_insns, speed_p);
9902 13499 : bool was_tie = false;
9903 :
9904 : /* If costs are the same then use as tie breaker the other other
9905 : factor. */
9906 13499 : if (uns_cost == sgn_cost)
9907 : {
9908 3540 : uns_cost = seq_cost (uns_insns, !speed_p);
9909 3540 : sgn_cost = seq_cost (sgn_insns, !speed_p);
9910 3540 : was_tie = true;
9911 : }
9912 :
9913 13499 : if (dump_file && (dump_flags & TDF_DETAILS))
9914 0 : fprintf (dump_file, ";; positive division:%s unsigned cost: %u; "
9915 : "signed cost: %u\n",
9916 : was_tie ? " (needed tie breaker)" : "", uns_cost, sgn_cost);
9917 :
9918 13499 : if (uns_cost < sgn_cost || (uns_cost == sgn_cost && unsignedp))
9919 : {
9920 10787 : emit_insn (uns_insns);
9921 10787 : return uns_ret;
9922 : }
9923 2712 : emit_insn (sgn_insns);
9924 2712 : return sgn_ret;
9925 : }
9926 156166 : return expand_divmod (mod_p, code, mode, op0, op1, target, unsignedp);
9927 169738 : }
9928 :
9929 : /* Return true if EXP has a range of values [0..1], false
9930 : otherwise. This works for constants and ssa names, calling back into the ranger. */
9931 : static bool
9932 1529034 : expr_has_boolean_range (tree exp, gimple *stmt)
9933 : {
9934 : /* An integral type with a single bit of precision. */
9935 3058055 : if (INTEGRAL_TYPE_P (TREE_TYPE (exp))
9936 1529034 : && TYPE_UNSIGNED (TREE_TYPE (exp))
9937 2543100 : && TYPE_PRECISION (TREE_TYPE (exp)) == 1)
9938 : return true;
9939 :
9940 : /* Signed 1 bit integers are not boolean ranges. */
9941 3058055 : if (!INTEGRAL_TYPE_P (TREE_TYPE (exp))
9942 3058055 : || TYPE_PRECISION (TREE_TYPE (exp)) <= 1)
9943 : return false;
9944 :
9945 1528990 : if (TREE_CODE (exp) == SSA_NAME)
9946 910275 : return ssa_name_has_boolean_range (exp, stmt);
9947 618715 : if (TREE_CODE (exp) == INTEGER_CST)
9948 547685 : return wi::leu_p (wi::to_wide (exp), 1);
9949 : return false;
9950 : }
9951 :
9952 : rtx
9953 13701202 : expand_expr_real_2 (const_sepops ops, rtx target, machine_mode tmode,
9954 : enum expand_modifier modifier)
9955 : {
9956 13701202 : rtx op0, op1, op2, temp;
9957 13701202 : rtx_code_label *lab;
9958 13701202 : tree type;
9959 13701202 : int unsignedp;
9960 13701202 : machine_mode mode;
9961 13701202 : scalar_int_mode int_mode;
9962 13701202 : enum tree_code code = ops->code;
9963 13701202 : optab this_optab;
9964 13701202 : rtx subtarget, original_target;
9965 13701202 : int ignore;
9966 13701202 : bool reduce_bit_field;
9967 13701202 : location_t loc = ops->location;
9968 13701202 : tree treeop0, treeop1, treeop2;
9969 : #define REDUCE_BIT_FIELD(expr) (reduce_bit_field \
9970 : ? reduce_to_bit_field_precision ((expr), \
9971 : target, \
9972 : type) \
9973 : : (expr))
9974 :
9975 13701202 : type = ops->type;
9976 13701202 : mode = TYPE_MODE (type);
9977 13701202 : unsignedp = TYPE_UNSIGNED (type);
9978 :
9979 13701202 : treeop0 = ops->op0;
9980 13701202 : treeop1 = ops->op1;
9981 13701202 : treeop2 = ops->op2;
9982 :
9983 : /* We should be called only on simple (binary or unary) expressions,
9984 : exactly those that are valid in gimple expressions that aren't
9985 : GIMPLE_SINGLE_RHS (or invalid). */
9986 13701202 : gcc_assert (get_gimple_rhs_class (code) == GIMPLE_UNARY_RHS
9987 : || get_gimple_rhs_class (code) == GIMPLE_BINARY_RHS
9988 : || get_gimple_rhs_class (code) == GIMPLE_TERNARY_RHS);
9989 :
9990 27402404 : ignore = (target == const0_rtx
9991 13701202 : || ((CONVERT_EXPR_CODE_P (code)
9992 9856194 : || code == COND_EXPR || code == VIEW_CONVERT_EXPR)
9993 3862778 : && TREE_CODE (type) == VOID_TYPE));
9994 :
9995 : /* We should be called only if we need the result. */
9996 0 : gcc_assert (!ignore);
9997 :
9998 : /* An operation in what may be a bit-field type needs the
9999 : result to be reduced to the precision of the bit-field type,
10000 : which is narrower than that of the type's mode. */
10001 28213417 : reduce_bit_field = (INTEGRAL_TYPE_P (type)
10002 13701202 : && !type_has_mode_precision_p (type));
10003 :
10004 811013 : if (reduce_bit_field
10005 811013 : && (modifier == EXPAND_STACK_PARM
10006 810488 : || (target && GET_MODE (target) != mode)))
10007 496524 : target = 0;
10008 :
10009 : /* Use subtarget as the target for operand 0 of a binary operation. */
10010 13701202 : subtarget = get_subtarget (target);
10011 13701202 : original_target = target;
10012 :
10013 13701202 : switch (code)
10014 : {
10015 3849196 : case NON_LVALUE_EXPR:
10016 3849196 : case PAREN_EXPR:
10017 3849196 : CASE_CONVERT:
10018 3849196 : if (treeop0 == error_mark_node)
10019 0 : return const0_rtx;
10020 :
10021 3849196 : if (TREE_CODE (type) == UNION_TYPE)
10022 : {
10023 0 : tree valtype = TREE_TYPE (treeop0);
10024 :
10025 : /* If both input and output are BLKmode, this conversion isn't doing
10026 : anything except possibly changing memory attribute. */
10027 0 : if (mode == BLKmode && TYPE_MODE (valtype) == BLKmode)
10028 : {
10029 0 : rtx result = expand_expr (treeop0, target, tmode,
10030 : modifier);
10031 :
10032 0 : result = copy_rtx (result);
10033 0 : set_mem_attributes (result, type, 0);
10034 0 : return result;
10035 : }
10036 :
10037 0 : if (target == 0)
10038 : {
10039 0 : if (TYPE_MODE (type) != BLKmode)
10040 0 : target = gen_reg_rtx (TYPE_MODE (type));
10041 : else
10042 0 : target = assign_temp (type, 1, 1);
10043 : }
10044 :
10045 0 : if (MEM_P (target))
10046 : /* Store data into beginning of memory target. */
10047 0 : store_expr (treeop0,
10048 0 : adjust_address (target, TYPE_MODE (valtype), 0),
10049 : modifier == EXPAND_STACK_PARM,
10050 0 : false, TYPE_REVERSE_STORAGE_ORDER (type));
10051 :
10052 : else
10053 : {
10054 0 : gcc_assert (REG_P (target)
10055 : && !TYPE_REVERSE_STORAGE_ORDER (type));
10056 :
10057 : /* Store this field into a union of the proper type. */
10058 0 : poly_uint64 op0_size
10059 0 : = tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (treeop0)));
10060 0 : poly_uint64 union_size = GET_MODE_BITSIZE (mode);
10061 0 : store_field (target,
10062 : /* The conversion must be constructed so that
10063 : we know at compile time how many bits
10064 : to preserve. */
10065 0 : ordered_min (op0_size, union_size),
10066 0 : 0, 0, 0, TYPE_MODE (valtype), treeop0, 0,
10067 : false, false);
10068 : }
10069 :
10070 : /* Return the entire union. */
10071 : return target;
10072 : }
10073 :
10074 3849196 : if (mode == TYPE_MODE (TREE_TYPE (treeop0)))
10075 : {
10076 2286145 : op0 = expand_expr (treeop0, target, VOIDmode,
10077 : modifier);
10078 :
10079 2286145 : return REDUCE_BIT_FIELD (op0);
10080 : }
10081 :
10082 1927958 : op0 = expand_expr (treeop0, NULL_RTX, mode,
10083 : modifier == EXPAND_SUM ? EXPAND_NORMAL : modifier);
10084 1563051 : if (GET_MODE (op0) == mode)
10085 : ;
10086 :
10087 : /* If OP0 is a constant, just convert it into the proper mode. */
10088 1519728 : else if (CONSTANT_P (op0))
10089 : {
10090 902 : tree inner_type = TREE_TYPE (treeop0);
10091 902 : machine_mode inner_mode = GET_MODE (op0);
10092 :
10093 902 : if (inner_mode == VOIDmode)
10094 15 : inner_mode = TYPE_MODE (inner_type);
10095 :
10096 902 : if (modifier == EXPAND_INITIALIZER)
10097 0 : op0 = force_lowpart_subreg (mode, op0, inner_mode);
10098 : else
10099 902 : op0 = convert_modes (mode, inner_mode, op0,
10100 902 : TYPE_UNSIGNED (inner_type));
10101 : }
10102 :
10103 1518826 : else if (modifier == EXPAND_INITIALIZER)
10104 24 : op0 = gen_rtx_fmt_e (TYPE_UNSIGNED (TREE_TYPE (treeop0))
10105 : ? ZERO_EXTEND : SIGN_EXTEND, mode, op0);
10106 :
10107 1518814 : else if (SCALAR_INT_MODE_P (GET_MODE (op0))
10108 1343072 : && optimize >= 2
10109 788897 : && SCALAR_INT_MODE_P (mode)
10110 788897 : && INTEGRAL_TYPE_P (TREE_TYPE (treeop0))
10111 788729 : && (GET_MODE_SIZE (as_a <scalar_int_mode> (mode))
10112 1577458 : > GET_MODE_SIZE (as_a <scalar_int_mode> (GET_MODE (op0))))
10113 2071554 : && get_range_pos_neg (treeop0,
10114 : currently_expanding_gimple_stmt) == 1)
10115 : {
10116 : /* If argument is known to be positive when interpreted
10117 : as signed, we can expand it as both sign and zero
10118 : extension. Choose the cheaper sequence in that case. */
10119 140497 : bool speed_p = optimize_insn_for_speed_p ();
10120 140497 : rtx uns_ret = NULL_RTX, sgn_ret = NULL_RTX;
10121 140497 : do_pending_stack_adjust ();
10122 140497 : start_sequence ();
10123 140497 : if (target == NULL_RTX)
10124 116833 : uns_ret = convert_to_mode (mode, op0, 1);
10125 : else
10126 23664 : convert_move (target, op0, 1);
10127 140497 : rtx_insn *uns_insns = end_sequence ();
10128 140497 : start_sequence ();
10129 140497 : if (target == NULL_RTX)
10130 116833 : sgn_ret = convert_to_mode (mode, op0, 0);
10131 : else
10132 23664 : convert_move (target, op0, 0);
10133 140497 : rtx_insn *sgn_insns = end_sequence ();
10134 140497 : unsigned uns_cost = seq_cost (uns_insns, speed_p);
10135 140497 : unsigned sgn_cost = seq_cost (sgn_insns, speed_p);
10136 140497 : bool was_tie = false;
10137 :
10138 : /* If costs are the same then use as tie breaker the other other
10139 : factor. */
10140 140497 : if (uns_cost == sgn_cost)
10141 : {
10142 36533 : uns_cost = seq_cost (uns_insns, !speed_p);
10143 36533 : sgn_cost = seq_cost (sgn_insns, !speed_p);
10144 36533 : was_tie = true;
10145 : }
10146 :
10147 140497 : if (dump_file && (dump_flags & TDF_DETAILS))
10148 0 : fprintf (dump_file, ";; positive extension:%s unsigned cost: %u; "
10149 : "signed cost: %u\n",
10150 : was_tie ? " (needed tie breaker)" : "",
10151 : uns_cost, sgn_cost);
10152 140497 : if (uns_cost < sgn_cost
10153 140497 : || (uns_cost == sgn_cost && TYPE_UNSIGNED (TREE_TYPE (treeop0))))
10154 : {
10155 126299 : emit_insn (uns_insns);
10156 126299 : sgn_ret = uns_ret;
10157 : }
10158 : else
10159 14198 : emit_insn (sgn_insns);
10160 140497 : if (target == NULL_RTX)
10161 116833 : op0 = sgn_ret;
10162 : else
10163 23664 : op0 = target;
10164 : }
10165 1378317 : else if (target == 0)
10166 863561 : op0 = convert_to_mode (mode, op0, TYPE_UNSIGNED (TREE_TYPE (treeop0)));
10167 : else
10168 : {
10169 514756 : convert_move (target, op0, TYPE_UNSIGNED (TREE_TYPE (treeop0)));
10170 514756 : op0 = target;
10171 : }
10172 :
10173 1563051 : return REDUCE_BIT_FIELD (op0);
10174 :
10175 0 : case ADDR_SPACE_CONVERT_EXPR:
10176 0 : {
10177 0 : tree treeop0_type = TREE_TYPE (treeop0);
10178 :
10179 0 : gcc_assert (POINTER_TYPE_P (type));
10180 0 : gcc_assert (POINTER_TYPE_P (treeop0_type));
10181 :
10182 0 : addr_space_t as_to = TYPE_ADDR_SPACE (TREE_TYPE (type));
10183 0 : addr_space_t as_from = TYPE_ADDR_SPACE (TREE_TYPE (treeop0_type));
10184 :
10185 : /* Conversions between pointers to the same address space should
10186 : have been implemented via CONVERT_EXPR / NOP_EXPR. */
10187 0 : gcc_assert (as_to != as_from);
10188 :
10189 0 : op0 = expand_expr (treeop0, NULL_RTX, VOIDmode, modifier);
10190 :
10191 : /* Ask target code to handle conversion between pointers
10192 : to overlapping address spaces. */
10193 0 : if (targetm.addr_space.subset_p (as_to, as_from)
10194 0 : || targetm.addr_space.subset_p (as_from, as_to))
10195 : {
10196 0 : op0 = targetm.addr_space.convert (op0, treeop0_type, type);
10197 : }
10198 : else
10199 : {
10200 : /* For disjoint address spaces, converting anything but a null
10201 : pointer invokes undefined behavior. We truncate or extend the
10202 : value as if we'd converted via integers, which handles 0 as
10203 : required, and all others as the programmer likely expects. */
10204 : #ifndef POINTERS_EXTEND_UNSIGNED
10205 : const int POINTERS_EXTEND_UNSIGNED = 1;
10206 : #endif
10207 0 : op0 = convert_modes (mode, TYPE_MODE (treeop0_type),
10208 : op0, POINTERS_EXTEND_UNSIGNED);
10209 : }
10210 0 : gcc_assert (op0);
10211 : return op0;
10212 : }
10213 :
10214 1391883 : case POINTER_PLUS_EXPR:
10215 : /* Even though the sizetype mode and the pointer's mode can be different
10216 : expand is able to handle this correctly and get the correct result out
10217 : of the PLUS_EXPR code. */
10218 : /* Make sure to sign-extend the sizetype offset in a POINTER_PLUS_EXPR
10219 : if sizetype precision is smaller than pointer precision. */
10220 1391883 : if (TYPE_PRECISION (sizetype) < TYPE_PRECISION (type))
10221 0 : treeop1 = fold_convert_loc (loc, type,
10222 : fold_convert_loc (loc, ssizetype,
10223 : treeop1));
10224 : /* If sizetype precision is larger than pointer precision, truncate the
10225 : offset to have matching modes. */
10226 1391883 : else if (TYPE_PRECISION (sizetype) > TYPE_PRECISION (type))
10227 0 : treeop1 = fold_convert_loc (loc, type, treeop1);
10228 : /* FALLTHRU */
10229 :
10230 5438260 : case PLUS_EXPR:
10231 : /* If we are adding a constant, a VAR_DECL that is sp, fp, or ap, and
10232 : something else, make sure we add the register to the constant and
10233 : then to the other thing. This case can occur during strength
10234 : reduction and doing it this way will produce better code if the
10235 : frame pointer or argument pointer is eliminated.
10236 :
10237 : fold-const.cc will ensure that the constant is always in the inner
10238 : PLUS_EXPR, so the only case we need to do anything about is if
10239 : sp, ap, or fp is our second argument, in which case we must swap
10240 : the innermost first argument and our second argument. */
10241 :
10242 5438260 : if (TREE_CODE (treeop0) == PLUS_EXPR
10243 6600 : && TREE_CODE (TREE_OPERAND (treeop0, 1)) == INTEGER_CST
10244 0 : && VAR_P (treeop1)
10245 5438260 : && (DECL_RTL (treeop1) == frame_pointer_rtx
10246 0 : || DECL_RTL (treeop1) == stack_pointer_rtx
10247 0 : || DECL_RTL (treeop1) == arg_pointer_rtx))
10248 : {
10249 0 : gcc_unreachable ();
10250 : }
10251 :
10252 : /* If the result is to be ptr_mode and we are adding an integer to
10253 : something, we might be forming a constant. So try to use
10254 : plus_constant. If it produces a sum and we can't accept it,
10255 : use force_operand. This allows P = &ARR[const] to generate
10256 : efficient code on machines where a SYMBOL_REF is not a valid
10257 : address.
10258 :
10259 : If this is an EXPAND_SUM call, always return the sum. */
10260 5438260 : if (modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER
10261 5438260 : || (mode == ptr_mode && (unsignedp || ! flag_trapv)))
10262 : {
10263 3564096 : if (modifier == EXPAND_STACK_PARM)
10264 18888 : target = 0;
10265 3564096 : if (TREE_CODE (treeop0) == INTEGER_CST
10266 3565103 : && HWI_COMPUTABLE_MODE_P (mode)
10267 3565107 : && TREE_CONSTANT (treeop1))
10268 : {
10269 4 : rtx constant_part;
10270 4 : HOST_WIDE_INT wc;
10271 4 : machine_mode wmode = TYPE_MODE (TREE_TYPE (treeop1));
10272 :
10273 4 : op1 = expand_expr (treeop1, subtarget, VOIDmode,
10274 : EXPAND_SUM);
10275 : /* Use wi::shwi to ensure that the constant is
10276 : truncated according to the mode of OP1, then sign extended
10277 : to a HOST_WIDE_INT. Using the constant directly can result
10278 : in non-canonical RTL in a 64x32 cross compile. */
10279 4 : wc = TREE_INT_CST_LOW (treeop0);
10280 4 : constant_part =
10281 4 : immed_wide_int_const (wi::shwi (wc, wmode), wmode);
10282 4 : op1 = plus_constant (mode, op1, INTVAL (constant_part));
10283 4 : if (modifier != EXPAND_SUM && modifier != EXPAND_INITIALIZER)
10284 1 : op1 = force_operand (op1, target);
10285 4 : return REDUCE_BIT_FIELD (op1);
10286 : }
10287 :
10288 3564092 : else if (TREE_CODE (treeop1) == INTEGER_CST
10289 7563378 : && HWI_COMPUTABLE_MODE_P (mode)
10290 5976668 : && TREE_CONSTANT (treeop0))
10291 : {
10292 287454 : rtx constant_part;
10293 287454 : HOST_WIDE_INT wc;
10294 287454 : machine_mode wmode = TYPE_MODE (TREE_TYPE (treeop0));
10295 :
10296 502300 : op0 = expand_expr (treeop0, subtarget, VOIDmode,
10297 : (modifier == EXPAND_INITIALIZER
10298 : ? EXPAND_INITIALIZER : EXPAND_SUM));
10299 287454 : if (! CONSTANT_P (op0))
10300 : {
10301 6 : op1 = expand_expr (treeop1, NULL_RTX,
10302 : VOIDmode, modifier);
10303 : /* Return a PLUS if modifier says it's OK. */
10304 6 : if (modifier == EXPAND_SUM
10305 : || modifier == EXPAND_INITIALIZER)
10306 6 : return simplify_gen_binary (PLUS, mode, op0, op1);
10307 0 : goto binop2;
10308 : }
10309 : /* Use wi::shwi to ensure that the constant is
10310 : truncated according to the mode of OP1, then sign extended
10311 : to a HOST_WIDE_INT. Using the constant directly can result
10312 : in non-canonical RTL in a 64x32 cross compile. */
10313 287448 : wc = TREE_INT_CST_LOW (treeop1);
10314 287448 : constant_part
10315 287448 : = immed_wide_int_const (wi::shwi (wc, wmode), wmode);
10316 287448 : op0 = plus_constant (mode, op0, INTVAL (constant_part));
10317 287448 : if (modifier != EXPAND_SUM && modifier != EXPAND_INITIALIZER)
10318 214012 : op0 = force_operand (op0, target);
10319 287448 : return REDUCE_BIT_FIELD (op0);
10320 : }
10321 : }
10322 :
10323 : /* Use TER to expand pointer addition of a negated value
10324 : as pointer subtraction. */
10325 9216953 : if ((POINTER_TYPE_P (TREE_TYPE (treeop0))
10326 4046364 : || (TREE_CODE (TREE_TYPE (treeop0)) == VECTOR_TYPE
10327 80005 : && POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (treeop0)))))
10328 1104438 : && TREE_CODE (treeop1) == SSA_NAME
10329 6298972 : && TYPE_MODE (TREE_TYPE (treeop0))
10330 574085 : == TYPE_MODE (TREE_TYPE (treeop1)))
10331 : {
10332 574085 : gimple *def = get_def_for_expr (treeop1, NEGATE_EXPR);
10333 574085 : if (def)
10334 : {
10335 2065 : treeop1 = gimple_assign_rhs1 (def);
10336 2065 : code = MINUS_EXPR;
10337 2065 : goto do_minus;
10338 : }
10339 : }
10340 :
10341 : /* No sense saving up arithmetic to be done
10342 : if it's all in the wrong mode to form part of an address.
10343 : And force_operand won't know whether to sign-extend or
10344 : zero-extend. */
10345 5148737 : if (modifier != EXPAND_INITIALIZER
10346 5148737 : && (modifier != EXPAND_SUM || mode != ptr_mode))
10347 : {
10348 4566924 : expand_operands (treeop0, treeop1,
10349 : subtarget, &op0, &op1, modifier);
10350 4566924 : if (op0 == const0_rtx)
10351 8670 : return op1;
10352 4558254 : if (op1 == const0_rtx)
10353 : return op0;
10354 4558003 : goto binop2;
10355 : }
10356 :
10357 581813 : expand_operands (treeop0, treeop1,
10358 : subtarget, &op0, &op1, modifier);
10359 581813 : return REDUCE_BIT_FIELD (simplify_gen_binary (PLUS, mode, op0, op1));
10360 :
10361 546197 : case MINUS_EXPR:
10362 546197 : case POINTER_DIFF_EXPR:
10363 546197 : do_minus:
10364 : /* For initializers, we are allowed to return a MINUS of two
10365 : symbolic constants. Here we handle all cases when both operands
10366 : are constant. */
10367 : /* Handle difference of two symbolic constants,
10368 : for the sake of an initializer. */
10369 546197 : if ((modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER)
10370 2529 : && really_constant_p (treeop0)
10371 546704 : && really_constant_p (treeop1))
10372 : {
10373 68 : expand_operands (treeop0, treeop1,
10374 : NULL_RTX, &op0, &op1, modifier);
10375 68 : return simplify_gen_binary (MINUS, mode, op0, op1);
10376 : }
10377 :
10378 : /* No sense saving up arithmetic to be done
10379 : if it's all in the wrong mode to form part of an address.
10380 : And force_operand won't know whether to sign-extend or
10381 : zero-extend. */
10382 546129 : if (modifier != EXPAND_INITIALIZER
10383 546129 : && (modifier != EXPAND_SUM || mode != ptr_mode))
10384 543668 : goto binop;
10385 :
10386 2461 : expand_operands (treeop0, treeop1,
10387 : subtarget, &op0, &op1, modifier);
10388 :
10389 : /* Convert A - const to A + (-const). */
10390 2461 : if (CONST_INT_P (op1))
10391 : {
10392 0 : op1 = negate_rtx (mode, op1);
10393 0 : return REDUCE_BIT_FIELD (simplify_gen_binary (PLUS, mode, op0, op1));
10394 : }
10395 :
10396 2461 : goto binop2;
10397 :
10398 0 : case WIDEN_MULT_PLUS_EXPR:
10399 0 : case WIDEN_MULT_MINUS_EXPR:
10400 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
10401 0 : op2 = expand_normal (treeop2);
10402 0 : target = expand_widen_pattern_expr (ops, op0, op1, op2,
10403 : target, unsignedp);
10404 0 : return target;
10405 :
10406 24107 : case WIDEN_MULT_EXPR:
10407 : /* If first operand is constant, swap them.
10408 : Thus the following special case checks need only
10409 : check the second operand. */
10410 24107 : if (TREE_CODE (treeop0) == INTEGER_CST)
10411 368 : std::swap (treeop0, treeop1);
10412 :
10413 : /* First, check if we have a multiplication of one signed and one
10414 : unsigned operand. */
10415 24107 : if (TREE_CODE (treeop1) != INTEGER_CST
10416 24107 : && (TYPE_UNSIGNED (TREE_TYPE (treeop0))
10417 18634 : != TYPE_UNSIGNED (TREE_TYPE (treeop1))))
10418 : {
10419 0 : machine_mode innermode = TYPE_MODE (TREE_TYPE (treeop0));
10420 0 : this_optab = usmul_widen_optab;
10421 0 : if (find_widening_optab_handler (this_optab, mode, innermode)
10422 : != CODE_FOR_nothing)
10423 : {
10424 0 : if (TYPE_UNSIGNED (TREE_TYPE (treeop0)))
10425 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1,
10426 : EXPAND_NORMAL);
10427 : else
10428 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op1, &op0,
10429 : EXPAND_NORMAL);
10430 : /* op0 and op1 might still be constant, despite the above
10431 : != INTEGER_CST check. Handle it. */
10432 0 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
10433 : {
10434 0 : op0 = convert_modes (mode, innermode, op0, true);
10435 0 : op1 = convert_modes (mode, innermode, op1, false);
10436 0 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1,
10437 : target, unsignedp));
10438 : }
10439 0 : goto binop3;
10440 : }
10441 : }
10442 : /* Check for a multiplication with matching signedness. */
10443 24107 : else if ((TREE_CODE (treeop1) == INTEGER_CST
10444 5473 : && int_fits_type_p (treeop1, TREE_TYPE (treeop0)))
10445 24111 : || (TYPE_UNSIGNED (TREE_TYPE (treeop1))
10446 18638 : == TYPE_UNSIGNED (TREE_TYPE (treeop0))))
10447 : {
10448 24107 : tree op0type = TREE_TYPE (treeop0);
10449 24107 : machine_mode innermode = TYPE_MODE (op0type);
10450 24107 : bool zextend_p = TYPE_UNSIGNED (op0type);
10451 24107 : optab other_optab = zextend_p ? smul_widen_optab : umul_widen_optab;
10452 2584 : this_optab = zextend_p ? umul_widen_optab : smul_widen_optab;
10453 :
10454 24107 : if (TREE_CODE (treeop0) != INTEGER_CST)
10455 : {
10456 24107 : if (find_widening_optab_handler (this_optab, mode, innermode)
10457 : != CODE_FOR_nothing)
10458 : {
10459 24107 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1,
10460 : EXPAND_NORMAL);
10461 : /* op0 and op1 might still be constant, despite the above
10462 : != INTEGER_CST check. Handle it. */
10463 24107 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
10464 : {
10465 0 : widen_mult_const:
10466 0 : op0 = convert_modes (mode, innermode, op0, zextend_p);
10467 0 : op1
10468 0 : = convert_modes (mode, innermode, op1,
10469 0 : TYPE_UNSIGNED (TREE_TYPE (treeop1)));
10470 0 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1,
10471 : target,
10472 : unsignedp));
10473 : }
10474 24107 : temp = expand_widening_mult (mode, op0, op1, target,
10475 : unsignedp, this_optab);
10476 24107 : return REDUCE_BIT_FIELD (temp);
10477 : }
10478 0 : if (find_widening_optab_handler (other_optab, mode, innermode)
10479 : != CODE_FOR_nothing
10480 0 : && innermode == word_mode)
10481 : {
10482 0 : rtx htem, hipart;
10483 0 : op0 = expand_normal (treeop0);
10484 0 : op1 = expand_normal (treeop1);
10485 : /* op0 and op1 might be constants, despite the above
10486 : != INTEGER_CST check. Handle it. */
10487 0 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
10488 0 : goto widen_mult_const;
10489 0 : temp = expand_binop (mode, other_optab, op0, op1, target,
10490 : unsignedp, OPTAB_LIB_WIDEN);
10491 0 : hipart = gen_highpart (word_mode, temp);
10492 0 : htem = expand_mult_highpart_adjust (word_mode, hipart,
10493 : op0, op1, hipart,
10494 : zextend_p);
10495 0 : if (htem != hipart)
10496 0 : emit_move_insn (hipart, htem);
10497 0 : return REDUCE_BIT_FIELD (temp);
10498 : }
10499 : }
10500 : }
10501 0 : treeop0 = fold_build1 (CONVERT_EXPR, type, treeop0);
10502 0 : treeop1 = fold_build1 (CONVERT_EXPR, type, treeop1);
10503 0 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10504 0 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1, target, unsignedp));
10505 :
10506 1431970 : case MULT_EXPR:
10507 : /* If this is a fixed-point operation, then we cannot use the code
10508 : below because "expand_mult" doesn't support sat/no-sat fixed-point
10509 : multiplications. */
10510 1431970 : if (ALL_FIXED_POINT_MODE_P (mode))
10511 0 : goto binop;
10512 :
10513 : /* If first operand is constant, swap them.
10514 : Thus the following special case checks need only
10515 : check the second operand. */
10516 1431970 : if (TREE_CODE (treeop0) == INTEGER_CST)
10517 686 : std::swap (treeop0, treeop1);
10518 :
10519 : /* Attempt to return something suitable for generating an
10520 : indexed address, for machines that support that. */
10521 :
10522 539640 : if (modifier == EXPAND_SUM && mode == ptr_mode
10523 1971610 : && tree_fits_shwi_p (treeop1))
10524 : {
10525 532410 : tree exp1 = treeop1;
10526 :
10527 532410 : op0 = expand_expr (treeop0, subtarget, VOIDmode,
10528 : EXPAND_SUM);
10529 :
10530 532410 : if (!REG_P (op0))
10531 245295 : op0 = force_operand (op0, NULL_RTX);
10532 532410 : if (!REG_P (op0))
10533 2312 : op0 = copy_to_mode_reg (mode, op0);
10534 :
10535 532410 : op1 = gen_int_mode (tree_to_shwi (exp1),
10536 532410 : TYPE_MODE (TREE_TYPE (exp1)));
10537 532410 : return REDUCE_BIT_FIELD (gen_rtx_MULT (mode, op0, op1));
10538 : }
10539 :
10540 899560 : if (modifier == EXPAND_STACK_PARM)
10541 2897 : target = 0;
10542 :
10543 899560 : if (SCALAR_INT_MODE_P (mode) && optimize >= 2)
10544 : {
10545 544876 : gimple *def_stmt0 = get_def_for_expr (treeop0, TRUNC_DIV_EXPR);
10546 544876 : gimple *def_stmt1 = get_def_for_expr (treeop1, TRUNC_DIV_EXPR);
10547 544876 : if (def_stmt0
10548 544876 : && !operand_equal_p (treeop1, gimple_assign_rhs2 (def_stmt0), 0))
10549 : def_stmt0 = NULL;
10550 544876 : if (def_stmt1
10551 544876 : && !operand_equal_p (treeop0, gimple_assign_rhs2 (def_stmt1), 0))
10552 : def_stmt1 = NULL;
10553 :
10554 544876 : if (def_stmt0 || def_stmt1)
10555 : {
10556 : /* X / Y * Y can be expanded as X - X % Y too.
10557 : Choose the cheaper sequence of those two. */
10558 194 : if (def_stmt0)
10559 194 : treeop0 = gimple_assign_rhs1 (def_stmt0);
10560 : else
10561 : {
10562 0 : treeop1 = treeop0;
10563 0 : treeop0 = gimple_assign_rhs1 (def_stmt1);
10564 : }
10565 194 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1,
10566 : EXPAND_NORMAL);
10567 194 : bool speed_p = optimize_insn_for_speed_p ();
10568 194 : do_pending_stack_adjust ();
10569 194 : start_sequence ();
10570 194 : rtx divmul_ret
10571 194 : = expand_expr_divmod (TRUNC_DIV_EXPR, mode, treeop0, treeop1,
10572 : op0, op1, NULL_RTX, unsignedp);
10573 194 : divmul_ret = expand_mult (mode, divmul_ret, op1, target,
10574 : unsignedp);
10575 194 : rtx_insn *divmul_insns = end_sequence ();
10576 194 : start_sequence ();
10577 194 : rtx modsub_ret
10578 194 : = expand_expr_divmod (TRUNC_MOD_EXPR, mode, treeop0, treeop1,
10579 : op0, op1, NULL_RTX, unsignedp);
10580 194 : this_optab = optab_for_tree_code (MINUS_EXPR, type,
10581 : optab_default);
10582 194 : modsub_ret = expand_binop (mode, this_optab, op0, modsub_ret,
10583 : target, unsignedp, OPTAB_LIB_WIDEN);
10584 194 : rtx_insn *modsub_insns = end_sequence ();
10585 194 : unsigned divmul_cost = seq_cost (divmul_insns, speed_p);
10586 194 : unsigned modsub_cost = seq_cost (modsub_insns, speed_p);
10587 : /* If costs are the same then use as tie breaker the other other
10588 : factor. */
10589 194 : if (divmul_cost == modsub_cost)
10590 : {
10591 5 : divmul_cost = seq_cost (divmul_insns, !speed_p);
10592 5 : modsub_cost = seq_cost (modsub_insns, !speed_p);
10593 : }
10594 :
10595 194 : if (divmul_cost <= modsub_cost)
10596 : {
10597 121 : emit_insn (divmul_insns);
10598 121 : return REDUCE_BIT_FIELD (divmul_ret);
10599 : }
10600 73 : emit_insn (modsub_insns);
10601 73 : return REDUCE_BIT_FIELD (modsub_ret);
10602 : }
10603 : }
10604 :
10605 899366 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10606 :
10607 : /* Expand X*Y as X&-Y when Y must be zero or one. */
10608 899366 : if (SCALAR_INT_MODE_P (mode))
10609 : {
10610 764517 : bool bit0_p = expr_has_boolean_range (treeop0, currently_expanding_gimple_stmt);
10611 764517 : bool bit1_p = expr_has_boolean_range (treeop1, currently_expanding_gimple_stmt);
10612 :
10613 : /* Expand X*Y as X&Y when both X and Y must be zero or one. */
10614 764517 : if (bit0_p && bit1_p)
10615 0 : return REDUCE_BIT_FIELD (expand_and (mode, op0, op1, target));
10616 :
10617 764517 : if (bit0_p || bit1_p)
10618 : {
10619 3127 : bool speed = optimize_insn_for_speed_p ();
10620 3127 : int cost = add_cost (speed, mode) + neg_cost (speed, mode);
10621 3127 : struct algorithm algorithm;
10622 3127 : enum mult_variant variant;
10623 3127 : if (CONST_INT_P (op1)
10624 3127 : ? !choose_mult_variant (mode, INTVAL (op1),
10625 : &algorithm, &variant, cost)
10626 616 : : cost < mul_cost (speed, mode))
10627 : {
10628 1321 : temp = bit0_p ? expand_and (mode, negate_rtx (mode, op0),
10629 : op1, target)
10630 260 : : expand_and (mode, op0,
10631 : negate_rtx (mode, op1),
10632 : target);
10633 1321 : return REDUCE_BIT_FIELD (temp);
10634 : }
10635 : }
10636 : }
10637 :
10638 898045 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1, target, unsignedp));
10639 :
10640 169350 : case TRUNC_MOD_EXPR:
10641 169350 : case FLOOR_MOD_EXPR:
10642 169350 : case CEIL_MOD_EXPR:
10643 169350 : case ROUND_MOD_EXPR:
10644 :
10645 169350 : case TRUNC_DIV_EXPR:
10646 169350 : case FLOOR_DIV_EXPR:
10647 169350 : case CEIL_DIV_EXPR:
10648 169350 : case ROUND_DIV_EXPR:
10649 169350 : case EXACT_DIV_EXPR:
10650 : /* If this is a fixed-point operation, then we cannot use the code
10651 : below because "expand_divmod" doesn't support sat/no-sat fixed-point
10652 : divisions. */
10653 169350 : if (ALL_FIXED_POINT_MODE_P (mode))
10654 0 : goto binop;
10655 :
10656 169350 : if (modifier == EXPAND_STACK_PARM)
10657 306 : target = 0;
10658 : /* Possible optimization: compute the dividend with EXPAND_SUM
10659 : then if the divisor is constant can optimize the case
10660 : where some terms of the dividend have coeffs divisible by it. */
10661 169350 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10662 169350 : return expand_expr_divmod (code, mode, treeop0, treeop1, op0, op1,
10663 169350 : target, unsignedp);
10664 :
10665 31877 : case RDIV_EXPR:
10666 31877 : goto binop;
10667 :
10668 4149 : case MULT_HIGHPART_EXPR:
10669 4149 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10670 4149 : temp = expand_mult_highpart (mode, op0, op1, target, unsignedp);
10671 4149 : gcc_assert (temp);
10672 : return temp;
10673 :
10674 0 : case FIXED_CONVERT_EXPR:
10675 0 : op0 = expand_normal (treeop0);
10676 0 : if (target == 0 || modifier == EXPAND_STACK_PARM)
10677 0 : target = gen_reg_rtx (mode);
10678 :
10679 0 : if ((TREE_CODE (TREE_TYPE (treeop0)) == INTEGER_TYPE
10680 0 : && TYPE_UNSIGNED (TREE_TYPE (treeop0)))
10681 0 : || (TREE_CODE (type) == INTEGER_TYPE && TYPE_UNSIGNED (type)))
10682 0 : expand_fixed_convert (target, op0, 1, TYPE_SATURATING (type));
10683 : else
10684 0 : expand_fixed_convert (target, op0, 0, TYPE_SATURATING (type));
10685 : return target;
10686 :
10687 48487 : case FIX_TRUNC_EXPR:
10688 48487 : op0 = expand_normal (treeop0);
10689 48487 : if (target == 0 || modifier == EXPAND_STACK_PARM)
10690 3444 : target = gen_reg_rtx (mode);
10691 48487 : expand_fix (target, op0, unsignedp);
10692 48487 : return target;
10693 :
10694 134313 : case FLOAT_EXPR:
10695 134313 : op0 = expand_normal (treeop0);
10696 134313 : if (target == 0 || modifier == EXPAND_STACK_PARM)
10697 63890 : target = gen_reg_rtx (mode);
10698 : /* expand_float can't figure out what to do if FROM has VOIDmode.
10699 : So give it the correct mode. With -O, cse will optimize this. */
10700 134313 : if (GET_MODE (op0) == VOIDmode)
10701 109 : op0 = copy_to_mode_reg (TYPE_MODE (TREE_TYPE (treeop0)),
10702 : op0);
10703 268626 : expand_float (target, op0,
10704 134313 : TYPE_UNSIGNED (TREE_TYPE (treeop0)));
10705 134313 : return target;
10706 :
10707 59089 : case NEGATE_EXPR:
10708 59089 : op0 = expand_expr (treeop0, subtarget,
10709 : VOIDmode, EXPAND_NORMAL);
10710 59089 : if (modifier == EXPAND_STACK_PARM)
10711 241 : target = 0;
10712 59089 : temp = expand_unop (mode,
10713 : optab_for_tree_code (NEGATE_EXPR, type,
10714 : optab_default),
10715 : op0, target, 0);
10716 59089 : gcc_assert (temp);
10717 59089 : return REDUCE_BIT_FIELD (temp);
10718 :
10719 28400 : case ABS_EXPR:
10720 28400 : case ABSU_EXPR:
10721 28400 : op0 = expand_expr (treeop0, subtarget,
10722 : VOIDmode, EXPAND_NORMAL);
10723 28400 : if (modifier == EXPAND_STACK_PARM)
10724 63 : target = 0;
10725 :
10726 : /* ABS_EXPR is not valid for complex arguments. */
10727 28400 : gcc_assert (GET_MODE_CLASS (mode) != MODE_COMPLEX_INT
10728 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_FLOAT);
10729 :
10730 : /* Unsigned abs is simply the operand. Testing here means we don't
10731 : risk generating incorrect code below. */
10732 28400 : if (TYPE_UNSIGNED (TREE_TYPE (treeop0)))
10733 : return op0;
10734 :
10735 28400 : return expand_abs (mode, op0, target, unsignedp,
10736 56800 : safe_from_p (target, treeop0, 1));
10737 :
10738 136712 : case MAX_EXPR:
10739 136712 : case MIN_EXPR:
10740 136712 : target = original_target;
10741 136712 : if (target == 0
10742 136712 : || modifier == EXPAND_STACK_PARM
10743 78262 : || (MEM_P (target) && MEM_VOLATILE_P (target))
10744 78262 : || GET_MODE (target) != mode
10745 214974 : || (REG_P (target)
10746 71560 : && REGNO (target) < FIRST_PSEUDO_REGISTER))
10747 58450 : target = gen_reg_rtx (mode);
10748 136712 : expand_operands (treeop0, treeop1,
10749 : target, &op0, &op1, EXPAND_NORMAL);
10750 :
10751 : /* First try to do it with a special MIN or MAX instruction.
10752 : If that does not win, use a conditional jump to select the proper
10753 : value. */
10754 136712 : this_optab = optab_for_tree_code (code, type, optab_default);
10755 136712 : temp = expand_binop (mode, this_optab, op0, op1, target, unsignedp,
10756 : OPTAB_WIDEN);
10757 136712 : if (temp != 0)
10758 : return temp;
10759 :
10760 122 : if (VECTOR_TYPE_P (type))
10761 0 : gcc_unreachable ();
10762 :
10763 : /* At this point, a MEM target is no longer useful; we will get better
10764 : code without it. */
10765 :
10766 122 : if (! REG_P (target))
10767 1 : target = gen_reg_rtx (mode);
10768 :
10769 : /* If op1 was placed in target, swap op0 and op1. */
10770 122 : if (target != op0 && target == op1)
10771 0 : std::swap (op0, op1);
10772 :
10773 : /* We generate better code and avoid problems with op1 mentioning
10774 : target by forcing op1 into a pseudo if it isn't a constant. */
10775 122 : if (! CONSTANT_P (op1))
10776 42 : op1 = force_reg (mode, op1);
10777 :
10778 122 : {
10779 122 : enum rtx_code comparison_code;
10780 122 : rtx cmpop1 = op1;
10781 :
10782 122 : if (code == MAX_EXPR)
10783 62 : comparison_code = unsignedp ? GEU : GE;
10784 : else
10785 60 : comparison_code = unsignedp ? LEU : LE;
10786 :
10787 : /* Canonicalize to comparisons against 0. */
10788 122 : if (op1 == const1_rtx)
10789 : {
10790 : /* Converting (a >= 1 ? a : 1) into (a > 0 ? a : 1)
10791 : or (a != 0 ? a : 1) for unsigned.
10792 : For MIN we are safe converting (a <= 1 ? a : 1)
10793 : into (a <= 0 ? a : 1) */
10794 0 : cmpop1 = const0_rtx;
10795 0 : if (code == MAX_EXPR)
10796 0 : comparison_code = unsignedp ? NE : GT;
10797 : }
10798 122 : if (op1 == constm1_rtx && !unsignedp)
10799 : {
10800 : /* Converting (a >= -1 ? a : -1) into (a >= 0 ? a : -1)
10801 : and (a <= -1 ? a : -1) into (a < 0 ? a : -1) */
10802 0 : cmpop1 = const0_rtx;
10803 0 : if (code == MIN_EXPR)
10804 0 : comparison_code = LT;
10805 : }
10806 :
10807 : /* Use a conditional move if possible. */
10808 122 : if (can_conditionally_move_p (mode))
10809 : {
10810 75 : rtx insn;
10811 :
10812 75 : start_sequence ();
10813 :
10814 : /* Try to emit the conditional move. */
10815 75 : insn = emit_conditional_move (target,
10816 : { comparison_code,
10817 : op0, cmpop1, mode },
10818 : op0, op1, mode,
10819 : unsignedp);
10820 :
10821 : /* If we could do the conditional move, emit the sequence,
10822 : and return. */
10823 75 : if (insn)
10824 : {
10825 41 : rtx_insn *seq = end_sequence ();
10826 41 : emit_insn (seq);
10827 41 : return target;
10828 : }
10829 :
10830 : /* Otherwise discard the sequence and fall back to code with
10831 : branches. */
10832 34 : end_sequence ();
10833 : }
10834 :
10835 81 : if (target != op0)
10836 52 : emit_move_insn (target, op0);
10837 :
10838 81 : lab = gen_label_rtx ();
10839 81 : do_compare_rtx_and_jump (target, cmpop1, comparison_code,
10840 : unsignedp, mode, NULL_RTX, NULL, lab,
10841 : profile_probability::uninitialized ());
10842 : }
10843 81 : emit_move_insn (target, op1);
10844 81 : emit_label (lab);
10845 81 : return target;
10846 :
10847 61236 : case BIT_NOT_EXPR:
10848 61236 : op0 = expand_expr (treeop0, subtarget,
10849 : VOIDmode, EXPAND_NORMAL);
10850 61236 : if (modifier == EXPAND_STACK_PARM)
10851 90 : target = 0;
10852 : /* In case we have to reduce the result to bitfield precision
10853 : for unsigned bitfield expand this as XOR with a proper constant
10854 : instead. */
10855 61236 : if (reduce_bit_field && TYPE_UNSIGNED (type))
10856 : {
10857 22098 : int_mode = SCALAR_INT_TYPE_MODE (type);
10858 22098 : wide_int mask = wi::mask (TYPE_PRECISION (type),
10859 44196 : false, GET_MODE_PRECISION (int_mode));
10860 :
10861 44196 : temp = expand_binop (int_mode, xor_optab, op0,
10862 44196 : immed_wide_int_const (mask, int_mode),
10863 : target, 1, OPTAB_LIB_WIDEN);
10864 22098 : }
10865 : else
10866 39138 : temp = expand_unop (mode, one_cmpl_optab, op0, target, 1);
10867 61236 : gcc_assert (temp);
10868 : return temp;
10869 :
10870 : /* ??? Can optimize bitwise operations with one arg constant.
10871 : Can optimize (a bitwise1 n) bitwise2 (a bitwise3 b)
10872 : and (a bitwise1 b) bitwise2 b (etc)
10873 : but that is probably not worth while. */
10874 :
10875 392078 : case BIT_AND_EXPR:
10876 392078 : goto binop;
10877 :
10878 225484 : case BIT_IOR_EXPR:
10879 225484 : case BIT_XOR_EXPR:
10880 225484 : if ((get_nonzero_bits (treeop0) & get_nonzero_bits (treeop1)) == 0)
10881 : {
10882 18570 : expand_operands (treeop0, treeop1,
10883 : subtarget, &op0, &op1, EXPAND_NORMAL);
10884 18570 : this_optab = aop_optab;
10885 18570 : goto binop3;
10886 : }
10887 206914 : goto binop;
10888 :
10889 8655 : case LROTATE_EXPR:
10890 8655 : case RROTATE_EXPR:
10891 8655 : gcc_assert (VECTOR_MODE_P (TYPE_MODE (type))
10892 : || type_has_mode_precision_p (type));
10893 : /* fall through */
10894 :
10895 307178 : case LSHIFT_EXPR:
10896 307178 : case RSHIFT_EXPR:
10897 307178 : {
10898 : /* If this is a fixed-point operation, then we cannot use the code
10899 : below because "expand_shift" doesn't support sat/no-sat fixed-point
10900 : shifts. */
10901 307178 : if (ALL_FIXED_POINT_MODE_P (mode))
10902 0 : goto binop;
10903 :
10904 307178 : if (! safe_from_p (subtarget, treeop1, 1))
10905 4885 : subtarget = 0;
10906 307178 : if (modifier == EXPAND_STACK_PARM)
10907 2586 : target = 0;
10908 307178 : op0 = expand_expr (treeop0, subtarget,
10909 : VOIDmode, EXPAND_NORMAL);
10910 :
10911 : /* Left shift optimization when shifting across word_size boundary.
10912 :
10913 : If mode == GET_MODE_WIDER_MODE (word_mode), then normally
10914 : there isn't native instruction to support this wide mode
10915 : left shift. Given below scenario:
10916 :
10917 : Type A = (Type) B << C
10918 :
10919 : |< T >|
10920 : | dest_high | dest_low |
10921 :
10922 : | word_size |
10923 :
10924 : If the shift amount C caused we shift B to across the word
10925 : size boundary, i.e part of B shifted into high half of
10926 : destination register, and part of B remains in the low
10927 : half, then GCC will use the following left shift expand
10928 : logic:
10929 :
10930 : 1. Initialize dest_low to B.
10931 : 2. Initialize every bit of dest_high to the sign bit of B.
10932 : 3. Logic left shift dest_low by C bit to finalize dest_low.
10933 : The value of dest_low before this shift is kept in a temp D.
10934 : 4. Logic left shift dest_high by C.
10935 : 5. Logic right shift D by (word_size - C).
10936 : 6. Or the result of 4 and 5 to finalize dest_high.
10937 :
10938 : While, by checking gimple statements, if operand B is
10939 : coming from signed extension, then we can simplify above
10940 : expand logic into:
10941 :
10942 : 1. dest_high = src_low >> (word_size - C).
10943 : 2. dest_low = src_low << C.
10944 :
10945 : We can use one arithmetic right shift to finish all the
10946 : purpose of steps 2, 4, 5, 6, thus we reduce the steps
10947 : needed from 6 into 2.
10948 :
10949 : The case is similar for zero extension, except that we
10950 : initialize dest_high to zero rather than copies of the sign
10951 : bit from B. Furthermore, we need to use a logical right shift
10952 : in this case.
10953 :
10954 : The choice of sign-extension versus zero-extension is
10955 : determined entirely by whether or not B is signed and is
10956 : independent of the current setting of unsignedp. */
10957 :
10958 307178 : temp = NULL_RTX;
10959 307178 : if (code == LSHIFT_EXPR
10960 307178 : && target
10961 33416 : && REG_P (target)
10962 30457 : && GET_MODE_2XWIDER_MODE (word_mode).exists (&int_mode)
10963 30457 : && mode == int_mode
10964 1518 : && TREE_CONSTANT (treeop1)
10965 308029 : && TREE_CODE (treeop0) == SSA_NAME)
10966 : {
10967 851 : gimple *def = SSA_NAME_DEF_STMT (treeop0);
10968 851 : if (is_gimple_assign (def)
10969 851 : && gimple_assign_rhs_code (def) == NOP_EXPR)
10970 : {
10971 337 : scalar_int_mode rmode = SCALAR_INT_TYPE_MODE
10972 : (TREE_TYPE (gimple_assign_rhs1 (def)));
10973 :
10974 674 : if (GET_MODE_SIZE (rmode) < GET_MODE_SIZE (int_mode)
10975 628 : && TREE_INT_CST_LOW (treeop1) < GET_MODE_BITSIZE (word_mode)
10976 483 : && ((TREE_INT_CST_LOW (treeop1) + GET_MODE_BITSIZE (rmode))
10977 73 : >= GET_MODE_BITSIZE (word_mode)))
10978 : {
10979 69 : rtx_insn *seq, *seq_old;
10980 69 : poly_uint64 high_off = subreg_highpart_offset (word_mode,
10981 : int_mode);
10982 69 : bool extend_unsigned
10983 69 : = TYPE_UNSIGNED (TREE_TYPE (gimple_assign_rhs1 (def)));
10984 69 : rtx low = lowpart_subreg (word_mode, op0, int_mode);
10985 69 : rtx dest_low = lowpart_subreg (word_mode, target, int_mode);
10986 69 : rtx dest_high = simplify_gen_subreg (word_mode, target,
10987 : int_mode, high_off);
10988 69 : HOST_WIDE_INT ramount = (BITS_PER_WORD
10989 69 : - TREE_INT_CST_LOW (treeop1));
10990 69 : tree rshift = build_int_cst (TREE_TYPE (treeop1), ramount);
10991 :
10992 69 : start_sequence ();
10993 : /* dest_high = src_low >> (word_size - C). */
10994 69 : temp = expand_variable_shift (RSHIFT_EXPR, word_mode, low,
10995 : rshift, dest_high,
10996 : extend_unsigned);
10997 69 : if (temp != dest_high)
10998 0 : emit_move_insn (dest_high, temp);
10999 :
11000 : /* dest_low = src_low << C. */
11001 69 : temp = expand_variable_shift (LSHIFT_EXPR, word_mode, low,
11002 : treeop1, dest_low, unsignedp);
11003 69 : if (temp != dest_low)
11004 1 : emit_move_insn (dest_low, temp);
11005 :
11006 69 : seq = end_sequence ();
11007 69 : temp = target ;
11008 :
11009 69 : if (have_insn_for (ASHIFT, int_mode))
11010 : {
11011 69 : bool speed_p = optimize_insn_for_speed_p ();
11012 69 : start_sequence ();
11013 69 : rtx ret_old = expand_variable_shift (code, int_mode,
11014 : op0, treeop1,
11015 : target,
11016 : unsignedp);
11017 :
11018 69 : seq_old = end_sequence ();
11019 138 : if (seq_cost (seq, speed_p)
11020 69 : >= seq_cost (seq_old, speed_p))
11021 : {
11022 69 : seq = seq_old;
11023 69 : temp = ret_old;
11024 : }
11025 : }
11026 69 : emit_insn (seq);
11027 : }
11028 : }
11029 : }
11030 :
11031 69 : if (temp == NULL_RTX)
11032 307109 : temp = expand_variable_shift (code, mode, op0, treeop1, target,
11033 : unsignedp);
11034 307178 : if (code == LSHIFT_EXPR)
11035 80739 : temp = REDUCE_BIT_FIELD (temp);
11036 : return temp;
11037 : }
11038 :
11039 : /* Could determine the answer when only additive constants differ. Also,
11040 : the addition of one can be handled by changing the condition. */
11041 631945 : case LT_EXPR:
11042 631945 : case LE_EXPR:
11043 631945 : case GT_EXPR:
11044 631945 : case GE_EXPR:
11045 631945 : case EQ_EXPR:
11046 631945 : case NE_EXPR:
11047 631945 : case UNORDERED_EXPR:
11048 631945 : case ORDERED_EXPR:
11049 631945 : case UNLT_EXPR:
11050 631945 : case UNLE_EXPR:
11051 631945 : case UNGT_EXPR:
11052 631945 : case UNGE_EXPR:
11053 631945 : case UNEQ_EXPR:
11054 631945 : case LTGT_EXPR:
11055 631945 : {
11056 1034540 : temp = do_store_flag (ops,
11057 : modifier != EXPAND_STACK_PARM ? target : NULL_RTX,
11058 : tmode != VOIDmode ? tmode : mode);
11059 631945 : if (temp)
11060 : return temp;
11061 :
11062 : /* Use a compare and a jump for BLKmode comparisons, or for function
11063 : type comparisons is have_canonicalize_funcptr_for_compare. */
11064 :
11065 0 : if ((target == 0
11066 0 : || modifier == EXPAND_STACK_PARM
11067 0 : || ! safe_from_p (target, treeop0, 1)
11068 0 : || ! safe_from_p (target, treeop1, 1)
11069 : /* Make sure we don't have a hard reg (such as function's return
11070 : value) live across basic blocks, if not optimizing. */
11071 0 : || (!optimize && REG_P (target)
11072 0 : && REGNO (target) < FIRST_PSEUDO_REGISTER)))
11073 0 : target = gen_reg_rtx (tmode != VOIDmode ? tmode : mode);
11074 :
11075 0 : emit_move_insn (target, const0_rtx);
11076 :
11077 0 : rtx_code_label *lab1 = gen_label_rtx ();
11078 0 : jumpifnot_1 (code, treeop0, treeop1, lab1,
11079 : profile_probability::uninitialized ());
11080 :
11081 0 : if (TYPE_PRECISION (type) == 1 && !TYPE_UNSIGNED (type))
11082 0 : emit_move_insn (target, constm1_rtx);
11083 : else
11084 0 : emit_move_insn (target, const1_rtx);
11085 :
11086 0 : emit_label (lab1);
11087 0 : return target;
11088 : }
11089 55632 : case COMPLEX_EXPR:
11090 : /* Get the rtx code of the operands. */
11091 55632 : op0 = expand_normal (treeop0);
11092 55632 : op1 = expand_normal (treeop1);
11093 :
11094 55632 : if (!target)
11095 3108 : target = gen_reg_rtx (TYPE_MODE (type));
11096 : else
11097 : /* If target overlaps with op1, then either we need to force
11098 : op1 into a pseudo (if target also overlaps with op0),
11099 : or write the complex parts in reverse order. */
11100 52524 : switch (GET_CODE (target))
11101 : {
11102 49765 : case CONCAT:
11103 49765 : if (reg_overlap_mentioned_p (XEXP (target, 0), op1))
11104 : {
11105 0 : if (reg_overlap_mentioned_p (XEXP (target, 1), op0))
11106 : {
11107 0 : complex_expr_force_op1:
11108 1726 : temp = gen_reg_rtx (GET_MODE_INNER (GET_MODE (target)));
11109 863 : emit_move_insn (temp, op1);
11110 863 : op1 = temp;
11111 863 : break;
11112 : }
11113 0 : complex_expr_swap_order:
11114 : /* Move the imaginary (op1) and real (op0) parts to their
11115 : location. */
11116 1 : write_complex_part (target, op1, true, true);
11117 1 : write_complex_part (target, op0, false, false);
11118 :
11119 1 : return target;
11120 : }
11121 : break;
11122 2759 : case MEM:
11123 5518 : temp = adjust_address_nv (target,
11124 : GET_MODE_INNER (GET_MODE (target)), 0);
11125 2759 : if (reg_overlap_mentioned_p (temp, op1))
11126 : {
11127 864 : scalar_mode imode = GET_MODE_INNER (GET_MODE (target));
11128 1728 : temp = adjust_address_nv (target, imode,
11129 : GET_MODE_SIZE (imode));
11130 864 : if (reg_overlap_mentioned_p (temp, op0))
11131 863 : goto complex_expr_force_op1;
11132 1 : goto complex_expr_swap_order;
11133 : }
11134 : break;
11135 0 : default:
11136 0 : if (reg_overlap_mentioned_p (target, op1))
11137 : {
11138 0 : if (reg_overlap_mentioned_p (target, op0))
11139 0 : goto complex_expr_force_op1;
11140 0 : goto complex_expr_swap_order;
11141 : }
11142 : break;
11143 : }
11144 :
11145 : /* Move the real (op0) and imaginary (op1) parts to their location. */
11146 55631 : write_complex_part (target, op0, false, true);
11147 55631 : write_complex_part (target, op1, true, false);
11148 :
11149 55631 : return target;
11150 :
11151 0 : case WIDEN_SUM_EXPR:
11152 0 : {
11153 0 : tree oprnd0 = treeop0;
11154 0 : tree oprnd1 = treeop1;
11155 :
11156 0 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11157 0 : target = expand_widen_pattern_expr (ops, op0, NULL_RTX, op1,
11158 : target, unsignedp);
11159 0 : return target;
11160 : }
11161 :
11162 17783 : case VEC_UNPACK_HI_EXPR:
11163 17783 : case VEC_UNPACK_LO_EXPR:
11164 17783 : case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
11165 17783 : case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
11166 17783 : {
11167 17783 : op0 = expand_normal (treeop0);
11168 17783 : temp = expand_widen_pattern_expr (ops, op0, NULL_RTX, NULL_RTX,
11169 : target, unsignedp);
11170 17783 : gcc_assert (temp);
11171 : return temp;
11172 : }
11173 :
11174 1836 : case VEC_UNPACK_FLOAT_HI_EXPR:
11175 1836 : case VEC_UNPACK_FLOAT_LO_EXPR:
11176 1836 : {
11177 1836 : op0 = expand_normal (treeop0);
11178 : /* The signedness is determined from input operand. */
11179 1836 : temp = expand_widen_pattern_expr
11180 3672 : (ops, op0, NULL_RTX, NULL_RTX,
11181 1836 : target, TYPE_UNSIGNED (TREE_TYPE (treeop0)));
11182 :
11183 1836 : gcc_assert (temp);
11184 : return temp;
11185 : }
11186 :
11187 1019 : case VEC_WIDEN_MULT_HI_EXPR:
11188 1019 : case VEC_WIDEN_MULT_LO_EXPR:
11189 1019 : case VEC_WIDEN_MULT_EVEN_EXPR:
11190 1019 : case VEC_WIDEN_MULT_ODD_EXPR:
11191 1019 : case VEC_WIDEN_LSHIFT_HI_EXPR:
11192 1019 : case VEC_WIDEN_LSHIFT_LO_EXPR:
11193 1019 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11194 1019 : target = expand_widen_pattern_expr (ops, op0, op1, NULL_RTX,
11195 : target, unsignedp);
11196 1019 : gcc_assert (target);
11197 : return target;
11198 :
11199 371 : case VEC_PACK_SAT_EXPR:
11200 371 : case VEC_PACK_FIX_TRUNC_EXPR:
11201 371 : mode = TYPE_MODE (TREE_TYPE (treeop0));
11202 371 : subtarget = NULL_RTX;
11203 371 : goto binop;
11204 :
11205 11822 : case VEC_PACK_TRUNC_EXPR:
11206 11822 : if (VECTOR_BOOLEAN_TYPE_P (type)
11207 2575 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (treeop0))
11208 2575 : && mode == TYPE_MODE (TREE_TYPE (treeop0))
11209 12528 : && SCALAR_INT_MODE_P (mode))
11210 : {
11211 706 : class expand_operand eops[4];
11212 706 : machine_mode imode = TYPE_MODE (TREE_TYPE (treeop0));
11213 706 : expand_operands (treeop0, treeop1,
11214 : subtarget, &op0, &op1, EXPAND_NORMAL);
11215 706 : this_optab = vec_pack_sbool_trunc_optab;
11216 706 : enum insn_code icode = optab_handler (this_optab, imode);
11217 706 : create_output_operand (&eops[0], target, mode);
11218 706 : create_convert_operand_from (&eops[1], op0, imode, false);
11219 706 : create_convert_operand_from (&eops[2], op1, imode, false);
11220 706 : temp = GEN_INT (TYPE_VECTOR_SUBPARTS (type).to_constant ());
11221 706 : create_input_operand (&eops[3], temp, imode);
11222 706 : expand_insn (icode, 4, eops);
11223 706 : return eops[0].value;
11224 : }
11225 11116 : mode = TYPE_MODE (TREE_TYPE (treeop0));
11226 11116 : subtarget = NULL_RTX;
11227 11116 : goto binop;
11228 :
11229 27 : case VEC_PACK_FLOAT_EXPR:
11230 27 : mode = TYPE_MODE (TREE_TYPE (treeop0));
11231 27 : expand_operands (treeop0, treeop1,
11232 : subtarget, &op0, &op1, EXPAND_NORMAL);
11233 27 : this_optab = optab_for_tree_code (code, TREE_TYPE (treeop0),
11234 : optab_default);
11235 81 : target = expand_binop (mode, this_optab, op0, op1, target,
11236 27 : TYPE_UNSIGNED (TREE_TYPE (treeop0)),
11237 : OPTAB_LIB_WIDEN);
11238 27 : gcc_assert (target);
11239 : return target;
11240 :
11241 75395 : case VEC_PERM_EXPR:
11242 75395 : {
11243 75395 : expand_operands (treeop0, treeop1, target, &op0, &op1, EXPAND_NORMAL);
11244 75395 : vec_perm_builder sel;
11245 75395 : if (TREE_CODE (treeop2) == VECTOR_CST
11246 75395 : && tree_to_vec_perm_builder (&sel, treeop2))
11247 : {
11248 75377 : machine_mode sel_mode = TYPE_MODE (TREE_TYPE (treeop2));
11249 75377 : temp = expand_vec_perm_const (mode, op0, op1, sel,
11250 : sel_mode, target);
11251 : }
11252 : else
11253 : {
11254 18 : op2 = expand_normal (treeop2);
11255 18 : temp = expand_vec_perm_var (mode, op0, op1, op2, target);
11256 : }
11257 75395 : gcc_assert (temp);
11258 75395 : return temp;
11259 75395 : }
11260 :
11261 441 : case DOT_PROD_EXPR:
11262 441 : {
11263 441 : tree oprnd0 = treeop0;
11264 441 : tree oprnd1 = treeop1;
11265 441 : tree oprnd2 = treeop2;
11266 :
11267 441 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11268 441 : op2 = expand_normal (oprnd2);
11269 441 : target = expand_widen_pattern_expr (ops, op0, op1, op2,
11270 : target, unsignedp);
11271 441 : return target;
11272 : }
11273 :
11274 119 : case SAD_EXPR:
11275 119 : {
11276 119 : tree oprnd0 = treeop0;
11277 119 : tree oprnd1 = treeop1;
11278 119 : tree oprnd2 = treeop2;
11279 :
11280 119 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11281 119 : op2 = expand_normal (oprnd2);
11282 119 : target = expand_widen_pattern_expr (ops, op0, op1, op2,
11283 : target, unsignedp);
11284 119 : return target;
11285 : }
11286 :
11287 0 : case REALIGN_LOAD_EXPR:
11288 0 : {
11289 0 : tree oprnd0 = treeop0;
11290 0 : tree oprnd1 = treeop1;
11291 0 : tree oprnd2 = treeop2;
11292 :
11293 0 : this_optab = optab_for_tree_code (code, type, optab_default);
11294 0 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11295 0 : op2 = expand_normal (oprnd2);
11296 0 : temp = expand_ternary_op (mode, this_optab, op0, op1, op2,
11297 : target, unsignedp);
11298 0 : gcc_assert (temp);
11299 : return temp;
11300 : }
11301 :
11302 17770 : case COND_EXPR:
11303 17770 : {
11304 : /* A COND_EXPR with its type being VOID_TYPE represents a
11305 : conditional jump and is handled in
11306 : expand_gimple_cond_expr. */
11307 17770 : gcc_assert (!VOID_TYPE_P (type));
11308 :
11309 : /* Note that COND_EXPRs whose type is a structure or union
11310 : are required to be constructed to contain assignments of
11311 : a temporary variable, so that we can evaluate them here
11312 : for side effect only. If type is void, we must do likewise. */
11313 :
11314 17770 : gcc_assert (!TREE_ADDRESSABLE (type)
11315 : && !ignore
11316 : && TREE_TYPE (treeop1) != void_type_node
11317 : && TREE_TYPE (treeop2) != void_type_node);
11318 :
11319 17770 : temp = expand_cond_expr_using_cmove (treeop0, treeop1, treeop2);
11320 17770 : if (temp)
11321 : return temp;
11322 :
11323 : /* If we are not to produce a result, we have no target. Otherwise,
11324 : if a target was specified use it; it will not be used as an
11325 : intermediate target unless it is safe. If no target, use a
11326 : temporary. */
11327 :
11328 3064 : if (modifier != EXPAND_STACK_PARM
11329 3064 : && original_target
11330 2267 : && safe_from_p (original_target, treeop0, 1)
11331 0 : && GET_MODE (original_target) == mode
11332 3064 : && !MEM_P (original_target))
11333 : temp = original_target;
11334 : else
11335 3064 : temp = assign_temp (type, 0, 1);
11336 :
11337 3064 : do_pending_stack_adjust ();
11338 3064 : NO_DEFER_POP;
11339 3064 : rtx_code_label *lab0 = gen_label_rtx ();
11340 3064 : rtx_code_label *lab1 = gen_label_rtx ();
11341 3064 : jumpifnot (treeop0, lab0,
11342 : profile_probability::uninitialized ());
11343 3064 : store_expr (treeop1, temp,
11344 : modifier == EXPAND_STACK_PARM,
11345 : false, false);
11346 :
11347 3064 : emit_jump_insn (targetm.gen_jump (lab1));
11348 3064 : emit_barrier ();
11349 3064 : emit_label (lab0);
11350 3064 : store_expr (treeop2, temp,
11351 : modifier == EXPAND_STACK_PARM,
11352 : false, false);
11353 :
11354 3064 : emit_label (lab1);
11355 3064 : OK_DEFER_POP;
11356 3064 : return temp;
11357 : }
11358 :
11359 0 : case VEC_DUPLICATE_EXPR:
11360 0 : op0 = expand_expr (treeop0, NULL_RTX, VOIDmode, modifier);
11361 0 : target = expand_vector_broadcast (mode, op0);
11362 0 : gcc_assert (target);
11363 : return target;
11364 :
11365 0 : case VEC_SERIES_EXPR:
11366 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, modifier);
11367 0 : return expand_vec_series_expr (mode, op0, op1, target);
11368 :
11369 1024 : case BIT_INSERT_EXPR:
11370 1024 : {
11371 1024 : unsigned bitpos = tree_to_uhwi (treeop2);
11372 1024 : unsigned bitsize;
11373 1024 : if (INTEGRAL_TYPE_P (TREE_TYPE (treeop1)))
11374 675 : bitsize = TYPE_PRECISION (TREE_TYPE (treeop1));
11375 : else
11376 349 : bitsize = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (treeop1)));
11377 1024 : op0 = expand_normal (treeop0);
11378 1024 : op1 = expand_normal (treeop1);
11379 1024 : rtx dst = gen_reg_rtx (mode);
11380 1024 : emit_move_insn (dst, op0);
11381 1024 : store_bit_field (dst, bitsize, bitpos, 0, 0,
11382 1024 : TYPE_MODE (TREE_TYPE (treeop1)), op1, false, false);
11383 1024 : return dst;
11384 : }
11385 :
11386 0 : default:
11387 0 : gcc_unreachable ();
11388 : }
11389 :
11390 : /* Here to do an ordinary binary operator. */
11391 1186024 : binop:
11392 1186024 : expand_operands (treeop0, treeop1,
11393 : subtarget, &op0, &op1, EXPAND_NORMAL);
11394 5746488 : binop2:
11395 5746488 : this_optab = optab_for_tree_code (code, type, optab_default);
11396 5765058 : binop3:
11397 5765058 : if (modifier == EXPAND_STACK_PARM)
11398 27291 : target = 0;
11399 5765058 : temp = expand_binop (mode, this_optab, op0, op1, target,
11400 : unsignedp, OPTAB_LIB_WIDEN);
11401 5765058 : gcc_assert (temp);
11402 : /* Bitwise operations do not need bitfield reduction as we expect their
11403 : operands being properly truncated. */
11404 5765058 : if (code == BIT_XOR_EXPR
11405 : || code == BIT_AND_EXPR
11406 5765058 : || code == BIT_IOR_EXPR)
11407 : return temp;
11408 5147496 : return REDUCE_BIT_FIELD (temp);
11409 : }
11410 : #undef REDUCE_BIT_FIELD
11411 :
11412 :
11413 : /* Return TRUE if expression STMT is suitable for replacement.
11414 : Never consider memory loads as replaceable, because those don't ever lead
11415 : into constant expressions. */
11416 :
11417 : static bool
11418 8 : stmt_is_replaceable_p (gimple *stmt)
11419 : {
11420 8 : if (ssa_is_replaceable_p (stmt))
11421 : {
11422 : /* Don't move around loads. */
11423 7 : if (!gimple_assign_single_p (stmt)
11424 7 : || is_gimple_val (gimple_assign_rhs1 (stmt)))
11425 6 : return true;
11426 : }
11427 : return false;
11428 : }
11429 :
11430 : /* A subroutine of expand_expr_real_1. Expand gimple assignment G,
11431 : which is known to set an SSA_NAME result. The other arguments are
11432 : as for expand_expr_real_1. */
11433 :
11434 : rtx
11435 15299962 : expand_expr_real_gassign (gassign *g, rtx target, machine_mode tmode,
11436 : enum expand_modifier modifier, rtx *alt_rtl,
11437 : bool inner_reference_p)
11438 : {
11439 15299962 : separate_ops ops;
11440 15299962 : rtx r;
11441 15299962 : location_t saved_loc = curr_insn_location ();
11442 15299962 : auto loc = gimple_location (g);
11443 15299962 : if (loc != UNKNOWN_LOCATION)
11444 12306515 : set_curr_insn_location (loc);
11445 15299962 : tree lhs = gimple_assign_lhs (g);
11446 15299962 : ops.code = gimple_assign_rhs_code (g);
11447 15299962 : ops.type = TREE_TYPE (lhs);
11448 15299962 : switch (get_gimple_rhs_class (ops.code))
11449 : {
11450 94586 : case GIMPLE_TERNARY_RHS:
11451 189172 : ops.op2 = gimple_assign_rhs3 (g);
11452 : /* Fallthru */
11453 8210796 : case GIMPLE_BINARY_RHS:
11454 8210796 : ops.op1 = gimple_assign_rhs2 (g);
11455 :
11456 : /* Try to expand conditional compare. */
11457 8210796 : if (targetm.have_ccmp ())
11458 : {
11459 164994 : gcc_checking_assert (targetm.gen_ccmp_next != NULL);
11460 164994 : r = expand_ccmp_expr (g, TYPE_MODE (ops.type));
11461 164994 : if (r)
11462 : break;
11463 : }
11464 : /* Fallthru */
11465 11798844 : case GIMPLE_UNARY_RHS:
11466 11798844 : ops.op0 = gimple_assign_rhs1 (g);
11467 11798844 : ops.location = loc;
11468 11798844 : r = expand_expr_real_2 (&ops, target, tmode, modifier);
11469 11798844 : break;
11470 3501098 : case GIMPLE_SINGLE_RHS:
11471 3501098 : {
11472 3501098 : r = expand_expr_real (gimple_assign_rhs1 (g), target,
11473 : tmode, modifier, alt_rtl,
11474 : inner_reference_p);
11475 3501098 : break;
11476 : }
11477 0 : default:
11478 0 : gcc_unreachable ();
11479 : }
11480 15299962 : set_curr_insn_location (saved_loc);
11481 15299962 : if (REG_P (r) && !REG_EXPR (r))
11482 3975749 : set_reg_attrs_for_decl_rtl (lhs, r);
11483 15299962 : return r;
11484 : }
11485 :
11486 : /* A subroutine of expand_expr_real_1. Attempt to VIEW_CONVERT_EXPR
11487 : the complex expression OP0 to the vector mode MODE. Store the
11488 : result at TARGET if possible (if TARGET is nonzero). Returns
11489 : NULL_RTX on failure. */
11490 : static rtx
11491 6655 : try_expand_complex_as_vector (machine_mode mode, rtx op0, rtx target)
11492 : {
11493 6655 : if (COMPLEX_MODE_P (GET_MODE (op0))
11494 1388 : && VECTOR_MODE_P (mode)
11495 26 : && known_eq (GET_MODE_NUNITS (mode), 2)
11496 6681 : && GET_MODE_INNER (mode) == GET_MODE_INNER (GET_MODE (op0)))
11497 : {
11498 13 : enum insn_code icode = convert_optab_handler (vec_init_optab, mode,
11499 13 : GET_MODE_INNER (mode));
11500 13 : if (icode != CODE_FOR_nothing)
11501 : {
11502 13 : if (!target || !REG_P (target))
11503 0 : target = gen_reg_rtx (mode);
11504 13 : rtx rpart = read_complex_part (op0, false);
11505 13 : rtx ipart = read_complex_part (op0, true);
11506 13 : if (!REG_P (rpart) && !CONSTANT_P (rpart))
11507 0 : rpart = force_reg (GET_MODE_INNER (mode), rpart);
11508 13 : if (!REG_P (ipart) && !CONSTANT_P (ipart))
11509 0 : ipart = force_reg (GET_MODE_INNER (mode), ipart);
11510 13 : rtvec vec = rtvec_alloc (2);
11511 13 : RTVEC_ELT (vec, 0) = rpart;
11512 13 : RTVEC_ELT (vec, 1) = ipart;
11513 13 : rtx par = gen_rtx_PARALLEL (mode, vec);
11514 13 : rtx_insn *insn = GEN_FCN (icode) (target, par);
11515 13 : if (insn)
11516 : {
11517 13 : emit_insn (insn);
11518 13 : return target;
11519 : }
11520 : }
11521 : }
11522 : return NULL_RTX;
11523 : }
11524 :
11525 : rtx
11526 160476392 : expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
11527 : enum expand_modifier modifier, rtx *alt_rtl,
11528 : bool inner_reference_p)
11529 : {
11530 160476392 : rtx op0, op1, temp, decl_rtl;
11531 160476392 : tree type;
11532 160476392 : int unsignedp;
11533 160476392 : machine_mode mode, dmode;
11534 160476392 : enum tree_code code = TREE_CODE (exp);
11535 160476392 : rtx subtarget, original_target;
11536 160476392 : int ignore;
11537 160476392 : bool reduce_bit_field;
11538 160476392 : location_t loc = EXPR_LOCATION (exp);
11539 160476392 : struct separate_ops ops;
11540 160476392 : tree treeop0, treeop1, treeop2;
11541 160476392 : tree ssa_name = NULL_TREE;
11542 160476392 : gimple *g;
11543 :
11544 : /* EXPAND_NORMAL is the only modifier that guarantees that a memory
11545 : reference describes a load. Use store semantics otherwise. */
11546 160476392 : const bool may_store_p = modifier != EXPAND_NORMAL;
11547 :
11548 : /* Some ABIs define padding bits in _BitInt uninitialized. Normally, RTL
11549 : expansion sign/zero extends integral types with less than mode precision
11550 : when reading from bit-fields and after arithmetic operations (see
11551 : REDUCE_BIT_FIELD in expand_expr_real_2) and on subsequent loads relies
11552 : on those extensions to have been already performed, but because of the
11553 : above for _BitInt they need to be sign/zero extended when reading from
11554 : locations that could be exposed to ABI boundaries (when loading from
11555 : objects in memory, or function arguments, return value). Because we
11556 : internally extend after arithmetic operations, we can avoid doing that
11557 : when reading from SSA_NAMEs of vars. */
11558 : #define EXTEND_BITINT(expr) \
11559 : ((BITINT_TYPE_P (type) \
11560 : && !bitint_extended \
11561 : && reduce_bit_field \
11562 : && mode != BLKmode \
11563 : && modifier != EXPAND_MEMORY \
11564 : && modifier != EXPAND_WRITE \
11565 : && modifier != EXPAND_INITIALIZER \
11566 : && modifier != EXPAND_CONST_ADDRESS) \
11567 : ? reduce_to_bit_field_precision ((expr), NULL_RTX, type) : (expr))
11568 :
11569 160476392 : type = TREE_TYPE (exp);
11570 160476392 : mode = TYPE_MODE (type);
11571 160476392 : unsignedp = TYPE_UNSIGNED (type);
11572 160476392 : if (BITINT_TYPE_P (type) && bitint_extended == -1)
11573 : {
11574 9265 : struct bitint_info info;
11575 9265 : bool ok = targetm.c.bitint_type_info (TYPE_PRECISION (type), &info);
11576 9265 : gcc_assert (ok);
11577 9265 : bitint_extended = info.extended;
11578 : }
11579 :
11580 160476392 : treeop0 = treeop1 = treeop2 = NULL_TREE;
11581 160476392 : if (!VL_EXP_CLASS_P (exp))
11582 153582756 : switch (TREE_CODE_LENGTH (code))
11583 : {
11584 5641827 : default:
11585 5641827 : case 3: treeop2 = TREE_OPERAND (exp, 2); /* FALLTHRU */
11586 13837311 : case 2: treeop1 = TREE_OPERAND (exp, 1); /* FALLTHRU */
11587 28496214 : case 1: treeop0 = TREE_OPERAND (exp, 0); /* FALLTHRU */
11588 : case 0: break;
11589 : }
11590 160476392 : ops.code = code;
11591 160476392 : ops.type = type;
11592 160476392 : ops.op0 = treeop0;
11593 160476392 : ops.op1 = treeop1;
11594 160476392 : ops.op2 = treeop2;
11595 160476392 : ops.location = loc;
11596 :
11597 320952784 : ignore = (target == const0_rtx
11598 160476392 : || ((CONVERT_EXPR_CODE_P (code)
11599 155311634 : || code == COND_EXPR || code == VIEW_CONVERT_EXPR)
11600 958884 : && TREE_CODE (type) == VOID_TYPE));
11601 :
11602 : /* An operation in what may be a bit-field type needs the
11603 : result to be reduced to the precision of the bit-field type,
11604 : which is narrower than that of the type's mode. */
11605 320952547 : reduce_bit_field = (!ignore
11606 155923048 : && INTEGRAL_TYPE_P (type)
11607 82085973 : && !type_has_mode_precision_p (type));
11608 :
11609 : /* If we are going to ignore this result, we need only do something
11610 : if there is a side-effect somewhere in the expression. If there
11611 : is, short-circuit the most common cases here. Note that we must
11612 : not call expand_expr with anything but const0_rtx in case this
11613 : is an initial expansion of a size that contains a PLACEHOLDER_EXPR. */
11614 :
11615 : if (ignore)
11616 : {
11617 4553344 : if (! TREE_SIDE_EFFECTS (exp))
11618 : return const0_rtx;
11619 :
11620 : /* Ensure we reference a volatile object even if value is ignored, but
11621 : don't do this if all we are doing is taking its address. */
11622 4553107 : if (TREE_THIS_VOLATILE (exp)
11623 0 : && TREE_CODE (exp) != FUNCTION_DECL
11624 0 : && mode != VOIDmode && mode != BLKmode
11625 0 : && modifier != EXPAND_CONST_ADDRESS)
11626 : {
11627 0 : temp = expand_expr (exp, NULL_RTX, VOIDmode, modifier);
11628 0 : if (MEM_P (temp))
11629 0 : copy_to_reg (temp);
11630 0 : return const0_rtx;
11631 : }
11632 :
11633 4553107 : if (TREE_CODE_CLASS (code) == tcc_unary
11634 : || code == BIT_FIELD_REF
11635 4553107 : || code == COMPONENT_REF
11636 4553107 : || code == INDIRECT_REF)
11637 0 : return expand_expr (treeop0, const0_rtx, VOIDmode,
11638 0 : modifier);
11639 :
11640 4553107 : else if (TREE_CODE_CLASS (code) == tcc_binary
11641 4553107 : || TREE_CODE_CLASS (code) == tcc_comparison
11642 4553107 : || code == ARRAY_REF || code == ARRAY_RANGE_REF)
11643 : {
11644 0 : expand_expr (treeop0, const0_rtx, VOIDmode, modifier);
11645 0 : expand_expr (treeop1, const0_rtx, VOIDmode, modifier);
11646 0 : return const0_rtx;
11647 : }
11648 :
11649 : target = 0;
11650 : }
11651 :
11652 160476155 : if (reduce_bit_field && modifier == EXPAND_STACK_PARM)
11653 39639 : target = 0;
11654 :
11655 : /* Use subtarget as the target for operand 0 of a binary operation. */
11656 160476155 : subtarget = get_subtarget (target);
11657 160476155 : original_target = target;
11658 :
11659 160476155 : switch (code)
11660 : {
11661 10884 : case LABEL_DECL:
11662 10884 : {
11663 10884 : tree function = decl_function_context (exp);
11664 :
11665 10884 : temp = label_rtx (exp);
11666 14828 : temp = gen_rtx_LABEL_REF (Pmode, temp);
11667 :
11668 10884 : if (function != current_function_decl
11669 1330 : && function != 0)
11670 1330 : LABEL_REF_NONLOCAL_P (temp) = 1;
11671 :
11672 10884 : temp = gen_rtx_MEM (FUNCTION_MODE, temp);
11673 10884 : return temp;
11674 : }
11675 :
11676 57858667 : case SSA_NAME:
11677 : /* ??? ivopts calls expander, without any preparation from
11678 : out-of-ssa. So fake instructions as if this was an access to the
11679 : base variable. This unnecessarily allocates a pseudo, see how we can
11680 : reuse it, if partition base vars have it set already. */
11681 57858667 : if (!currently_expanding_to_rtl)
11682 : {
11683 0 : tree var = SSA_NAME_VAR (exp);
11684 0 : if (var && DECL_RTL_SET_P (var))
11685 0 : return DECL_RTL (var);
11686 0 : return gen_raw_REG (TYPE_MODE (TREE_TYPE (exp)),
11687 0 : LAST_VIRTUAL_REGISTER + 1);
11688 : }
11689 :
11690 57858667 : g = get_gimple_for_ssa_name (exp);
11691 : /* For EXPAND_INITIALIZER try harder to get something simpler. */
11692 57858667 : if (g == NULL
11693 57858667 : && modifier == EXPAND_INITIALIZER
11694 26 : && !SSA_NAME_IS_DEFAULT_DEF (exp)
11695 11 : && (optimize || !SSA_NAME_VAR (exp)
11696 1 : || DECL_IGNORED_P (SSA_NAME_VAR (exp)))
11697 10 : && is_gimple_assign (SSA_NAME_DEF_STMT (exp))
11698 57858675 : && stmt_is_replaceable_p (SSA_NAME_DEF_STMT (exp)))
11699 6 : g = SSA_NAME_DEF_STMT (exp);
11700 57858667 : if (safe_is_a <gassign *> (g))
11701 9228102 : return expand_expr_real_gassign (as_a<gassign *> (g), target, tmode,
11702 9228102 : modifier, alt_rtl, inner_reference_p);
11703 48630565 : else if (safe_is_a <gcall *> (g))
11704 : {
11705 : /* ??? internal call expansion doesn't follow the usual API
11706 : of returning the destination RTX and being passed a desired
11707 : target. */
11708 68732 : if (modifier == EXPAND_WRITE)
11709 34370 : return DECL_RTL (SSA_NAME_VAR (exp));
11710 34362 : rtx dest = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
11711 34362 : tree tmplhs = make_tree (TREE_TYPE (exp), dest);
11712 34362 : tree var_or_id = SSA_NAME_VAR (exp);
11713 : if (!var_or_id)
11714 24157 : var_or_id = SSA_NAME_IDENTIFIER (exp);
11715 34362 : SET_SSA_NAME_VAR_OR_IDENTIFIER (exp, tmplhs);
11716 34362 : expand_internal_call (as_a <gcall *> (g));
11717 34362 : SET_SSA_NAME_VAR_OR_IDENTIFIER (exp, var_or_id);
11718 34362 : return dest;
11719 : }
11720 :
11721 48561833 : ssa_name = exp;
11722 48561833 : decl_rtl = get_rtx_for_ssa_name (ssa_name);
11723 48561833 : exp = SSA_NAME_VAR (ssa_name);
11724 : /* Optimize and avoid to EXTEND_BITINIT doing anything if it is an
11725 : SSA_NAME computed within the current function. In such case the
11726 : value have been already extended before. While if it is a function
11727 : parameter, result or some memory location, we need to be prepared
11728 : for some other compiler leaving the bits uninitialized. */
11729 19082123 : if (!exp || VAR_P (exp))
11730 : reduce_bit_field = false;
11731 48561833 : goto expand_decl_rtl;
11732 :
11733 19991410 : case VAR_DECL:
11734 : /* Allow accel compiler to handle variables that require special
11735 : treatment, e.g. if they have been modified in some way earlier in
11736 : compilation by the adjust_private_decl OpenACC hook. */
11737 19991410 : if (flag_openacc && targetm.goacc.expand_var_decl)
11738 : {
11739 0 : temp = targetm.goacc.expand_var_decl (exp);
11740 0 : if (temp)
11741 : return temp;
11742 : }
11743 : /* Expand const VAR_DECLs with CONSTRUCTOR initializers that
11744 : have scalar integer modes to a reg via store_constructor. */
11745 19991410 : if (TREE_READONLY (exp)
11746 3390809 : && !TREE_SIDE_EFFECTS (exp)
11747 3368401 : && (modifier == EXPAND_NORMAL || modifier == EXPAND_STACK_PARM)
11748 19261 : && immediate_const_ctor_p (DECL_INITIAL (exp))
11749 163 : && SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (exp)))
11750 127 : && crtl->emit.regno_pointer_align_length
11751 19991537 : && !target)
11752 : {
11753 88 : target = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
11754 88 : store_constructor (DECL_INITIAL (exp), target, 0,
11755 88 : int_expr_size (DECL_INITIAL (exp)), false);
11756 88 : return target;
11757 : }
11758 : /* ... fall through ... */
11759 :
11760 20541525 : case PARM_DECL:
11761 : /* If a static var's type was incomplete when the decl was written,
11762 : but the type is complete now, lay out the decl now. */
11763 20541525 : if (DECL_SIZE (exp) == 0
11764 45923 : && COMPLETE_OR_UNBOUND_ARRAY_TYPE_P (TREE_TYPE (exp))
11765 20587336 : && (TREE_STATIC (exp) || DECL_EXTERNAL (exp)))
11766 45811 : layout_decl (exp, 0);
11767 :
11768 : /* fall through */
11769 :
11770 22011415 : case FUNCTION_DECL:
11771 22011415 : case RESULT_DECL:
11772 22011415 : decl_rtl = DECL_RTL (exp);
11773 70573248 : expand_decl_rtl:
11774 70573248 : gcc_assert (decl_rtl);
11775 :
11776 : /* DECL_MODE might change when TYPE_MODE depends on attribute target
11777 : settings for VECTOR_TYPE_P that might switch for the function. */
11778 70573248 : if (currently_expanding_to_rtl
11779 66884107 : && code == VAR_DECL && MEM_P (decl_rtl)
11780 85523404 : && VECTOR_TYPE_P (type) && exp && DECL_MODE (exp) != mode)
11781 61 : decl_rtl = change_address (decl_rtl, TYPE_MODE (type), 0);
11782 : else
11783 70573187 : decl_rtl = copy_rtx (decl_rtl);
11784 :
11785 70573248 : if (exp && MEM_P (decl_rtl))
11786 22923496 : MEM_NOTRAP_P (decl_rtl)
11787 45846992 : = !(may_store_p
11788 22923496 : ? lhs_could_trap_p (exp) : tree_could_trap_p (exp));
11789 :
11790 : /* Record writes to register variables. */
11791 70573248 : if (modifier == EXPAND_WRITE
11792 22151989 : && REG_P (decl_rtl)
11793 86779265 : && HARD_REGISTER_P (decl_rtl))
11794 2027 : add_to_hard_reg_set (&crtl->asm_clobbers,
11795 2027 : GET_MODE (decl_rtl), REGNO (decl_rtl));
11796 :
11797 : /* Ensure variable marked as used even if it doesn't go through
11798 : a parser. If it hasn't be used yet, write out an external
11799 : definition. */
11800 70573248 : if (exp)
11801 41093538 : TREE_USED (exp) = 1;
11802 :
11803 : /* Show we haven't gotten RTL for this yet. */
11804 111666786 : temp = 0;
11805 :
11806 : /* Variables inherited from containing functions should have
11807 : been lowered by this point. */
11808 41093538 : if (exp)
11809 : {
11810 41093538 : tree context = decl_function_context (exp);
11811 41093538 : gcc_assert (SCOPE_FILE_SCOPE_P (context)
11812 : || context == current_function_decl
11813 : || TREE_STATIC (exp)
11814 : || DECL_EXTERNAL (exp)
11815 : /* ??? C++ creates functions that are not
11816 : TREE_STATIC. */
11817 : || TREE_CODE (exp) == FUNCTION_DECL);
11818 : }
11819 :
11820 : /* This is the case of an array whose size is to be determined
11821 : from its initializer, while the initializer is still being parsed.
11822 : ??? We aren't parsing while expanding anymore. */
11823 :
11824 70573248 : if (MEM_P (decl_rtl) && REG_P (XEXP (decl_rtl, 0)))
11825 400868 : temp = validize_mem (decl_rtl);
11826 :
11827 : /* If DECL_RTL is memory, we are in the normal case and the
11828 : address is not valid, get the address into a register. */
11829 :
11830 70172380 : else if (MEM_P (decl_rtl) && modifier != EXPAND_INITIALIZER)
11831 : {
11832 19020301 : if (alt_rtl)
11833 2026232 : *alt_rtl = decl_rtl;
11834 19020301 : decl_rtl = use_anchored_address (decl_rtl);
11835 19020301 : if (modifier != EXPAND_CONST_ADDRESS
11836 19020301 : && modifier != EXPAND_SUM
11837 31450091 : && !memory_address_addr_space_p (exp ? DECL_MODE (exp)
11838 17427 : : GET_MODE (decl_rtl),
11839 : XEXP (decl_rtl, 0),
11840 12429790 : MEM_ADDR_SPACE (decl_rtl)))
11841 137273 : temp = replace_equiv_address (decl_rtl,
11842 : copy_rtx (XEXP (decl_rtl, 0)));
11843 : }
11844 :
11845 : /* If we got something, return it. But first, set the alignment
11846 : if the address is a register. */
11847 19421169 : if (temp != 0)
11848 : {
11849 538141 : if (exp && MEM_P (temp) && REG_P (XEXP (temp, 0)))
11850 506192 : mark_reg_pointer (XEXP (temp, 0), DECL_ALIGN (exp));
11851 : }
11852 70035107 : else if (MEM_P (decl_rtl))
11853 : temp = decl_rtl;
11854 :
11855 48138517 : if (temp != 0)
11856 : {
11857 22940933 : if (MEM_P (temp)
11858 : && modifier != EXPAND_WRITE
11859 22940933 : && modifier != EXPAND_MEMORY
11860 : && modifier != EXPAND_INITIALIZER
11861 16933243 : && modifier != EXPAND_CONST_ADDRESS
11862 6994144 : && modifier != EXPAND_SUM
11863 6994144 : && !inner_reference_p
11864 4565563 : && mode != BLKmode
11865 27138411 : && MEM_ALIGN (temp) < GET_MODE_ALIGNMENT (mode))
11866 20286 : temp = expand_misaligned_mem_ref (temp, mode, unsignedp,
11867 20286 : MEM_ALIGN (temp), NULL_RTX, NULL);
11868 :
11869 22940933 : return EXTEND_BITINT (temp);
11870 : }
11871 :
11872 47632315 : if (exp)
11873 18170042 : dmode = DECL_MODE (exp);
11874 : else
11875 29462273 : dmode = TYPE_MODE (TREE_TYPE (ssa_name));
11876 :
11877 : /* If the mode of DECL_RTL does not match that of the decl,
11878 : there are two cases: we are dealing with a BLKmode value
11879 : that is returned in a register, or we are dealing with
11880 : a promoted value. In the latter case, return a SUBREG
11881 : of the wanted mode, but mark it so that we know that it
11882 : was already extended. */
11883 47632315 : if (REG_P (decl_rtl)
11884 47095897 : && dmode != BLKmode
11885 47095897 : && GET_MODE (decl_rtl) != dmode)
11886 : {
11887 84 : machine_mode pmode;
11888 :
11889 : /* Get the signedness to be used for this variable. Ensure we get
11890 : the same mode we got when the variable was declared. */
11891 84 : if (code != SSA_NAME)
11892 0 : pmode = promote_decl_mode (exp, &unsignedp);
11893 84 : else if ((g = SSA_NAME_DEF_STMT (ssa_name))
11894 84 : && gimple_code (g) == GIMPLE_CALL
11895 86 : && !gimple_call_internal_p (g))
11896 2 : pmode = promote_function_mode (type, mode, &unsignedp,
11897 2 : gimple_call_fntype (g),
11898 : 2);
11899 : else
11900 82 : pmode = promote_ssa_mode (ssa_name, &unsignedp);
11901 84 : gcc_assert (GET_MODE (decl_rtl) == pmode);
11902 :
11903 : /* Some ABIs require scalar floating point modes to be passed
11904 : in a wider scalar integer mode. We need to explicitly
11905 : truncate to an integer mode of the correct precision before
11906 : using a SUBREG to reinterpret as a floating point value. */
11907 84 : if (SCALAR_FLOAT_MODE_P (mode)
11908 0 : && SCALAR_INT_MODE_P (pmode)
11909 84 : && known_lt (GET_MODE_SIZE (mode), GET_MODE_SIZE (pmode)))
11910 0 : return convert_wider_int_to_float (mode, pmode, decl_rtl);
11911 :
11912 84 : temp = gen_lowpart_SUBREG (mode, decl_rtl);
11913 84 : SUBREG_PROMOTED_VAR_P (temp) = 1;
11914 84 : SUBREG_PROMOTED_SET (temp, unsignedp);
11915 84 : return EXTEND_BITINT (temp);
11916 : }
11917 :
11918 47632231 : return EXTEND_BITINT (decl_rtl);
11919 :
11920 43365386 : case INTEGER_CST:
11921 43365386 : {
11922 43365386 : if (BITINT_TYPE_P (type))
11923 : {
11924 11892 : unsigned int prec = TYPE_PRECISION (type);
11925 11892 : struct bitint_info info;
11926 11892 : bool ok = targetm.c.bitint_type_info (prec, &info);
11927 11892 : gcc_assert (ok);
11928 11892 : scalar_int_mode limb_mode
11929 11892 : = as_a <scalar_int_mode> (info.limb_mode);
11930 11892 : unsigned int limb_prec = GET_MODE_PRECISION (limb_mode);
11931 19405 : if (prec > limb_prec && prec > MAX_FIXED_MODE_SIZE)
11932 : {
11933 : /* Emit large/huge _BitInt INTEGER_CSTs into memory. */
11934 5167 : exp = tree_output_constant_def (exp);
11935 5167 : return expand_expr (exp, target, VOIDmode, modifier);
11936 : }
11937 : }
11938 :
11939 : /* Given that TYPE_PRECISION (type) is not always equal to
11940 : GET_MODE_PRECISION (TYPE_MODE (type)), we need to extend from
11941 : the former to the latter according to the signedness of the
11942 : type. */
11943 43360219 : scalar_int_mode int_mode = SCALAR_INT_TYPE_MODE (type);
11944 43360219 : temp = immed_wide_int_const
11945 43360219 : (wi::to_wide (exp, GET_MODE_PRECISION (int_mode)), int_mode);
11946 43360219 : return temp;
11947 : }
11948 :
11949 579036 : case VECTOR_CST:
11950 579036 : {
11951 579036 : tree tmp = NULL_TREE;
11952 579036 : if (VECTOR_MODE_P (mode))
11953 576167 : return const_vector_from_tree (exp);
11954 2869 : scalar_int_mode int_mode;
11955 2869 : if (is_int_mode (mode, &int_mode))
11956 : {
11957 926 : tree type_for_mode = lang_hooks.types.type_for_mode (int_mode, 1);
11958 926 : if (type_for_mode)
11959 926 : tmp = fold_unary_loc (loc, VIEW_CONVERT_EXPR,
11960 : type_for_mode, exp);
11961 : }
11962 926 : if (!tmp)
11963 : {
11964 1943 : vec<constructor_elt, va_gc> *v;
11965 : /* Constructors need to be fixed-length. FIXME. */
11966 1943 : unsigned int nunits = VECTOR_CST_NELTS (exp).to_constant ();
11967 1943 : vec_alloc (v, nunits);
11968 31617 : for (unsigned int i = 0; i < nunits; ++i)
11969 27731 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, VECTOR_CST_ELT (exp, i));
11970 1943 : tmp = build_constructor (type, v);
11971 : }
11972 2869 : return expand_expr (tmp, ignore ? const0_rtx : target,
11973 2869 : tmode, modifier);
11974 : }
11975 :
11976 136 : case CONST_DECL:
11977 136 : if (modifier == EXPAND_WRITE)
11978 : {
11979 : /* Writing into CONST_DECL is always invalid, but handle it
11980 : gracefully. */
11981 2 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (exp));
11982 2 : scalar_int_mode address_mode = targetm.addr_space.address_mode (as);
11983 2 : op0 = expand_expr_addr_expr_1 (exp, NULL_RTX, address_mode,
11984 : EXPAND_NORMAL, as);
11985 2 : op0 = memory_address_addr_space (mode, op0, as);
11986 2 : temp = gen_rtx_MEM (mode, op0);
11987 2 : set_mem_addr_space (temp, as);
11988 2 : return temp;
11989 : }
11990 134 : return expand_expr (DECL_INITIAL (exp), target, VOIDmode, modifier);
11991 :
11992 872893 : case REAL_CST:
11993 : /* If optimized, generate immediate CONST_DOUBLE
11994 : which will be turned into memory by reload if necessary.
11995 :
11996 : We used to force a register so that loop.c could see it. But
11997 : this does not allow gen_* patterns to perform optimizations with
11998 : the constants. It also produces two insns in cases like "x = 1.0;".
11999 : On most machines, floating-point constants are not permitted in
12000 : many insns, so we'd end up copying it to a register in any case.
12001 :
12002 : Now, we do the copying in expand_binop, if appropriate. */
12003 872893 : return const_double_from_real_value (TREE_REAL_CST (exp),
12004 1745786 : TYPE_MODE (TREE_TYPE (exp)));
12005 :
12006 0 : case FIXED_CST:
12007 0 : return CONST_FIXED_FROM_FIXED_VALUE (TREE_FIXED_CST (exp),
12008 : TYPE_MODE (TREE_TYPE (exp)));
12009 :
12010 18408 : case COMPLEX_CST:
12011 : /* Handle evaluating a complex constant in a CONCAT target. */
12012 18408 : if (original_target && GET_CODE (original_target) == CONCAT)
12013 : {
12014 231 : rtx rtarg, itarg;
12015 :
12016 231 : mode = TYPE_MODE (TREE_TYPE (TREE_TYPE (exp)));
12017 231 : rtarg = XEXP (original_target, 0);
12018 231 : itarg = XEXP (original_target, 1);
12019 :
12020 : /* Move the real and imaginary parts separately. */
12021 231 : op0 = expand_expr (TREE_REALPART (exp), rtarg, mode, EXPAND_NORMAL);
12022 231 : op1 = expand_expr (TREE_IMAGPART (exp), itarg, mode, EXPAND_NORMAL);
12023 :
12024 231 : if (op0 != rtarg)
12025 231 : emit_move_insn (rtarg, op0);
12026 231 : if (op1 != itarg)
12027 231 : emit_move_insn (itarg, op1);
12028 :
12029 : return original_target;
12030 : }
12031 :
12032 : /* fall through */
12033 :
12034 206082 : case STRING_CST:
12035 206082 : temp = expand_expr_constant (exp, 1, modifier);
12036 :
12037 : /* temp contains a constant address.
12038 : On RISC machines where a constant address isn't valid,
12039 : make some insns to get that address into a register. */
12040 206082 : if (modifier != EXPAND_CONST_ADDRESS
12041 : && modifier != EXPAND_INITIALIZER
12042 206082 : && modifier != EXPAND_SUM
12043 224628 : && ! memory_address_addr_space_p (mode, XEXP (temp, 0),
12044 18546 : MEM_ADDR_SPACE (temp)))
12045 12 : return replace_equiv_address (temp,
12046 12 : copy_rtx (XEXP (temp, 0)));
12047 : return temp;
12048 :
12049 0 : case POLY_INT_CST:
12050 0 : return immed_wide_int_const (poly_int_cst_value (exp), mode);
12051 :
12052 1677 : case SAVE_EXPR:
12053 1677 : {
12054 1677 : tree val = treeop0;
12055 1677 : rtx ret = expand_expr_real_1 (val, target, tmode, modifier, alt_rtl,
12056 : inner_reference_p);
12057 :
12058 1677 : if (!SAVE_EXPR_RESOLVED_P (exp))
12059 : {
12060 : /* We can indeed still hit this case, typically via builtin
12061 : expanders calling save_expr immediately before expanding
12062 : something. Assume this means that we only have to deal
12063 : with non-BLKmode values. */
12064 1630 : gcc_assert (GET_MODE (ret) != BLKmode);
12065 :
12066 1630 : val = build_decl (curr_insn_location (),
12067 1630 : VAR_DECL, NULL, TREE_TYPE (exp));
12068 1630 : DECL_ARTIFICIAL (val) = 1;
12069 1630 : DECL_IGNORED_P (val) = 1;
12070 1630 : treeop0 = val;
12071 1630 : TREE_OPERAND (exp, 0) = treeop0;
12072 1630 : SAVE_EXPR_RESOLVED_P (exp) = 1;
12073 :
12074 1630 : if (!CONSTANT_P (ret))
12075 1630 : ret = copy_to_reg (ret);
12076 1630 : SET_DECL_RTL (val, ret);
12077 : }
12078 :
12079 : return ret;
12080 : }
12081 :
12082 :
12083 181716 : case CONSTRUCTOR:
12084 : /* If we don't need the result, just ensure we evaluate any
12085 : subexpressions. */
12086 181716 : if (ignore)
12087 : {
12088 : unsigned HOST_WIDE_INT idx;
12089 : tree value;
12090 :
12091 0 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), idx, value)
12092 0 : expand_expr (value, const0_rtx, VOIDmode, EXPAND_NORMAL);
12093 :
12094 0 : return const0_rtx;
12095 : }
12096 :
12097 181716 : return expand_constructor (exp, target, modifier, false);
12098 :
12099 862004 : case TARGET_MEM_REF:
12100 862004 : {
12101 862004 : addr_space_t as
12102 862004 : = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
12103 862004 : unsigned int align;
12104 :
12105 862004 : op0 = addr_for_mem_ref (exp, as, true);
12106 862004 : op0 = memory_address_addr_space (mode, op0, as);
12107 862004 : temp = gen_rtx_MEM (mode, op0);
12108 862004 : set_mem_attributes (temp, exp, 0, may_store_p);
12109 862004 : set_mem_addr_space (temp, as);
12110 862004 : align = get_object_alignment (exp);
12111 862004 : if (modifier != EXPAND_WRITE
12112 862004 : && modifier != EXPAND_MEMORY
12113 595214 : && mode != BLKmode
12114 1451167 : && align < GET_MODE_ALIGNMENT (mode))
12115 53968 : temp = expand_misaligned_mem_ref (temp, mode, unsignedp,
12116 : align, NULL_RTX, NULL);
12117 862004 : return EXTEND_BITINT (temp);
12118 : }
12119 :
12120 6879882 : case MEM_REF:
12121 6879882 : {
12122 6879882 : const bool reverse = REF_REVERSE_STORAGE_ORDER (exp);
12123 6879882 : addr_space_t as
12124 6879882 : = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
12125 6879882 : machine_mode address_mode;
12126 6879882 : tree base = TREE_OPERAND (exp, 0);
12127 6879882 : gimple *def_stmt;
12128 6879882 : unsigned align;
12129 : /* Handle expansion of non-aliased memory with non-BLKmode. That
12130 : might end up in a register. */
12131 6879882 : if (mem_ref_refers_to_non_mem_p (exp))
12132 : {
12133 81639 : poly_int64 offset = mem_ref_offset (exp).force_shwi ();
12134 81639 : base = TREE_OPERAND (base, 0);
12135 81639 : poly_uint64 type_size;
12136 81639 : if (known_eq (offset, 0)
12137 46549 : && !reverse
12138 46549 : && poly_int_tree_p (TYPE_SIZE (type), &type_size)
12139 174737 : && known_eq (GET_MODE_BITSIZE (DECL_MODE (base)), type_size))
12140 18907 : return expand_expr (build1 (VIEW_CONVERT_EXPR, type, base),
12141 18907 : target, tmode, modifier);
12142 62732 : unsigned align;
12143 62732 : if (TYPE_MODE (type) == BLKmode || maybe_lt (offset, 0))
12144 : {
12145 218 : temp = assign_stack_temp (DECL_MODE (base),
12146 436 : GET_MODE_SIZE (DECL_MODE (base)));
12147 218 : store_expr (base, temp, 0, false, false);
12148 218 : temp = adjust_address (temp, TYPE_MODE (type), offset);
12149 218 : if (TYPE_MODE (type) == BLKmode)
12150 197 : set_mem_size (temp, int_size_in_bytes (type));
12151 : /* When the original ref was misaligned so will be the
12152 : access to the stack temporary. Not all targets handle
12153 : this correctly, some will ICE in sanity checking.
12154 : Handle this by doing bitfield extraction when necessary. */
12155 42 : else if ((align = get_object_alignment (exp))
12156 21 : < GET_MODE_ALIGNMENT (TYPE_MODE (type)))
12157 3 : temp
12158 3 : = expand_misaligned_mem_ref (temp, TYPE_MODE (type),
12159 : unsignedp, align,
12160 : modifier == EXPAND_STACK_PARM
12161 : ? NULL_RTX : target, NULL);
12162 : return temp;
12163 : }
12164 : /* When the access is fully outside of the underlying object
12165 : expand the offset as zero. This avoids out-of-bound
12166 : BIT_FIELD_REFs and generates smaller code for these cases
12167 : with UB. */
12168 62514 : type_size = tree_to_poly_uint64 (TYPE_SIZE_UNIT (type));
12169 125028 : if (!ranges_maybe_overlap_p (offset, type_size, 0,
12170 125028 : GET_MODE_SIZE (DECL_MODE (base))))
12171 37 : offset = 0;
12172 62514 : exp = build3 (BIT_FIELD_REF, type, base, TYPE_SIZE (type),
12173 62514 : bitsize_int (offset * BITS_PER_UNIT));
12174 62514 : REF_REVERSE_STORAGE_ORDER (exp) = reverse;
12175 62514 : return expand_expr (exp, target, tmode, modifier);
12176 : }
12177 6798243 : address_mode = targetm.addr_space.address_mode (as);
12178 6798243 : if ((def_stmt = get_def_for_expr (base, BIT_AND_EXPR)))
12179 : {
12180 44 : tree mask = gimple_assign_rhs2 (def_stmt);
12181 44 : base = build2 (BIT_AND_EXPR, TREE_TYPE (base),
12182 : gimple_assign_rhs1 (def_stmt), mask);
12183 44 : TREE_OPERAND (exp, 0) = base;
12184 : }
12185 6798243 : align = get_object_alignment (exp);
12186 6798243 : op0 = expand_expr (base, NULL_RTX, VOIDmode, EXPAND_SUM);
12187 6798243 : op0 = memory_address_addr_space (mode, op0, as);
12188 6798243 : if (!integer_zerop (TREE_OPERAND (exp, 1)))
12189 : {
12190 2071930 : rtx off = immed_wide_int_const (mem_ref_offset (exp), address_mode);
12191 2071930 : op0 = simplify_gen_binary (PLUS, address_mode, op0, off);
12192 2071930 : op0 = memory_address_addr_space (mode, op0, as);
12193 : }
12194 6798243 : temp = gen_rtx_MEM (mode, op0);
12195 6798243 : set_mem_attributes (temp, exp, 0, may_store_p);
12196 6798243 : set_mem_addr_space (temp, as);
12197 6798243 : if (TREE_THIS_VOLATILE (exp))
12198 12475 : MEM_VOLATILE_P (temp) = 1;
12199 6798243 : if (modifier == EXPAND_WRITE || modifier == EXPAND_MEMORY)
12200 : return temp;
12201 4031822 : if (!inner_reference_p
12202 2068853 : && mode != BLKmode
12203 6038784 : && align < GET_MODE_ALIGNMENT (mode))
12204 134503 : temp = expand_misaligned_mem_ref (temp, mode, unsignedp, align,
12205 : modifier == EXPAND_STACK_PARM
12206 : ? NULL_RTX : target, alt_rtl);
12207 4031822 : if (reverse)
12208 16 : temp = flip_storage_order (mode, temp);
12209 4031822 : return EXTEND_BITINT (temp);
12210 : }
12211 :
12212 631953 : case ARRAY_REF:
12213 :
12214 631953 : {
12215 631953 : tree array = treeop0;
12216 631953 : tree index = treeop1;
12217 631953 : tree init;
12218 :
12219 : /* Fold an expression like: "foo"[2].
12220 : This is not done in fold so it won't happen inside &.
12221 : Don't fold if this is for wide characters since it's too
12222 : difficult to do correctly and this is a very rare case. */
12223 :
12224 631953 : if (modifier != EXPAND_CONST_ADDRESS
12225 631953 : && modifier != EXPAND_INITIALIZER
12226 631953 : && modifier != EXPAND_MEMORY)
12227 : {
12228 631848 : tree t = fold_read_from_constant_string (exp);
12229 :
12230 631848 : if (t)
12231 0 : return expand_expr (t, target, tmode, modifier);
12232 : }
12233 :
12234 : /* If this is a constant index into a constant array,
12235 : just get the value from the array. Handle both the cases when
12236 : we have an explicit constructor and when our operand is a variable
12237 : that was declared const. */
12238 :
12239 631953 : if (modifier != EXPAND_CONST_ADDRESS
12240 : && modifier != EXPAND_INITIALIZER
12241 : && modifier != EXPAND_MEMORY
12242 631848 : && TREE_CODE (array) == CONSTRUCTOR
12243 0 : && ! TREE_SIDE_EFFECTS (array)
12244 631953 : && TREE_CODE (index) == INTEGER_CST)
12245 : {
12246 : unsigned HOST_WIDE_INT ix;
12247 : tree field, value;
12248 :
12249 0 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (array), ix,
12250 : field, value)
12251 0 : if (tree_int_cst_equal (field, index))
12252 : {
12253 0 : if (!TREE_SIDE_EFFECTS (value)
12254 0 : && TREE_CODE (value) != RAW_DATA_CST)
12255 0 : return expand_expr (fold (value), target, tmode, modifier);
12256 : break;
12257 : }
12258 : }
12259 :
12260 631953 : else if (optimize >= 1
12261 : && modifier != EXPAND_CONST_ADDRESS
12262 361507 : && modifier != EXPAND_INITIALIZER
12263 361507 : && modifier != EXPAND_MEMORY
12264 361406 : && TREE_READONLY (array) && ! TREE_SIDE_EFFECTS (array)
12265 9272 : && TREE_CODE (index) == INTEGER_CST
12266 2148 : && (VAR_P (array) || TREE_CODE (array) == CONST_DECL)
12267 632102 : && (init = ctor_for_folding (array)) != error_mark_node)
12268 : {
12269 108 : if (init == NULL_TREE)
12270 : {
12271 5 : tree value = build_zero_cst (type);
12272 5 : if (TREE_CODE (value) == CONSTRUCTOR)
12273 : {
12274 : /* If VALUE is a CONSTRUCTOR, this optimization is only
12275 : useful if this doesn't store the CONSTRUCTOR into
12276 : memory. If it does, it is more efficient to just
12277 : load the data from the array directly. */
12278 5 : rtx ret = expand_constructor (value, target,
12279 : modifier, true);
12280 5 : if (ret == NULL_RTX)
12281 : value = NULL_TREE;
12282 : }
12283 :
12284 : if (value)
12285 0 : return expand_expr (value, target, tmode, modifier);
12286 : }
12287 103 : else if (TREE_CODE (init) == CONSTRUCTOR)
12288 : {
12289 : unsigned HOST_WIDE_INT ix;
12290 : tree field, value;
12291 :
12292 168 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (init), ix,
12293 : field, value)
12294 166 : if (tree_int_cst_equal (field, index))
12295 : {
12296 100 : if (TREE_SIDE_EFFECTS (value)
12297 100 : || TREE_CODE (value) == RAW_DATA_CST)
12298 : break;
12299 :
12300 100 : if (TREE_CODE (value) == CONSTRUCTOR)
12301 : {
12302 : /* If VALUE is a CONSTRUCTOR, this
12303 : optimization is only useful if
12304 : this doesn't store the CONSTRUCTOR
12305 : into memory. If it does, it is more
12306 : efficient to just load the data from
12307 : the array directly. */
12308 95 : rtx ret = expand_constructor (value, target,
12309 : modifier, true);
12310 95 : if (ret == NULL_RTX)
12311 : break;
12312 : }
12313 :
12314 62 : return expand_expr (fold (value), target, tmode,
12315 62 : modifier);
12316 : }
12317 : }
12318 1 : else if (TREE_CODE (init) == STRING_CST)
12319 : {
12320 1 : tree low_bound = array_ref_low_bound (exp);
12321 1 : tree index1 = fold_convert_loc (loc, sizetype, treeop1);
12322 :
12323 : /* Optimize the special case of a zero lower bound.
12324 :
12325 : We convert the lower bound to sizetype to avoid problems
12326 : with constant folding. E.g. suppose the lower bound is
12327 : 1 and its mode is QI. Without the conversion
12328 : (ARRAY + (INDEX - (unsigned char)1))
12329 : becomes
12330 : (ARRAY + (-(unsigned char)1) + INDEX)
12331 : which becomes
12332 : (ARRAY + 255 + INDEX). Oops! */
12333 1 : if (!integer_zerop (low_bound))
12334 0 : index1 = size_diffop_loc (loc, index1,
12335 : fold_convert_loc (loc, sizetype,
12336 : low_bound));
12337 :
12338 1 : if (tree_fits_uhwi_p (index1)
12339 2 : && compare_tree_int (index1, TREE_STRING_LENGTH (init)) < 0)
12340 : {
12341 0 : tree char_type = TREE_TYPE (TREE_TYPE (init));
12342 0 : scalar_int_mode char_mode;
12343 :
12344 631891 : if (is_int_mode (TYPE_MODE (char_type), &char_mode)
12345 0 : && GET_MODE_SIZE (char_mode) == 1)
12346 0 : return gen_int_mode (TREE_STRING_POINTER (init)
12347 0 : [TREE_INT_CST_LOW (index1)],
12348 : char_mode);
12349 : }
12350 : }
12351 : }
12352 : }
12353 631891 : goto normal_inner_ref;
12354 :
12355 3818384 : case COMPONENT_REF:
12356 3818384 : gcc_assert (TREE_CODE (treeop0) != CONSTRUCTOR);
12357 : /* Fall through. */
12358 4692253 : case BIT_FIELD_REF:
12359 4692253 : case ARRAY_RANGE_REF:
12360 3818384 : normal_inner_ref:
12361 4692253 : {
12362 4692253 : machine_mode mode1, mode2;
12363 4692253 : poly_int64 bitsize, bitpos, bytepos;
12364 4692253 : tree offset;
12365 4692253 : int reversep, volatilep = 0;
12366 4692253 : tree tem
12367 4692253 : = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
12368 : &unsignedp, &reversep, &volatilep);
12369 4692253 : rtx orig_op0, memloc;
12370 4692253 : bool clear_mem_expr = false;
12371 4692253 : bool must_force_mem;
12372 :
12373 : /* If we got back the original object, something is wrong. Perhaps
12374 : we are evaluating an expression too early. In any event, don't
12375 : infinitely recurse. */
12376 4692253 : gcc_assert (tem != exp);
12377 :
12378 : /* Make sure bitpos is not negative, this can wreak havoc later. */
12379 4692253 : if (maybe_lt (bitpos, 0))
12380 : {
12381 249 : gcc_checking_assert (offset == NULL_TREE);
12382 249 : offset = size_int (bits_to_bytes_round_down (bitpos));
12383 249 : bitpos = num_trailing_bits (bitpos);
12384 : }
12385 :
12386 : /* If we have either an offset, a BLKmode result, or a reference
12387 : outside the underlying object, we must force it to memory.
12388 : Such a case can occur in Ada if we have unchecked conversion
12389 : of an expression from a scalar type to an aggregate type or
12390 : for an ARRAY_RANGE_REF whose type is BLKmode, or if we were
12391 : passed a partially uninitialized object or a view-conversion
12392 : to a larger size. */
12393 9384506 : must_force_mem = offset != NULL_TREE
12394 4411395 : || mode1 == BLKmode
12395 9031327 : || (mode == BLKmode
12396 0 : && !int_mode_for_size (bitsize, 1).exists ());
12397 :
12398 531382 : const enum expand_modifier tem_modifier
12399 : = must_force_mem
12400 : ? EXPAND_MEMORY
12401 4339074 : : modifier == EXPAND_SUM ? EXPAND_NORMAL : modifier;
12402 :
12403 : /* If TEM's type is a union of variable size, pass TARGET to the inner
12404 : computation, since it will need a temporary and TARGET is known
12405 : to have to do. This occurs in unchecked conversion in Ada. */
12406 4692253 : const rtx tem_target
12407 4692253 : = TREE_CODE (TREE_TYPE (tem)) == UNION_TYPE
12408 36789 : && COMPLETE_TYPE_P (TREE_TYPE (tem))
12409 36784 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (tem))) != INTEGER_CST
12410 0 : && modifier != EXPAND_STACK_PARM
12411 4692253 : ? target
12412 : : NULL_RTX;
12413 :
12414 9384506 : orig_op0 = op0
12415 4692253 : = expand_expr_real (tem, tem_target, VOIDmode, tem_modifier, NULL,
12416 : true);
12417 :
12418 : /* If the field has a mode, we want to access it in the
12419 : field's mode, not the computed mode.
12420 : If a MEM has VOIDmode (external with incomplete type),
12421 : use BLKmode for it instead. */
12422 4692253 : if (MEM_P (op0))
12423 : {
12424 4368441 : if (mode1 != VOIDmode)
12425 4209999 : op0 = adjust_address (op0, mode1, 0);
12426 158442 : else if (GET_MODE (op0) == VOIDmode)
12427 0 : op0 = adjust_address (op0, BLKmode, 0);
12428 : }
12429 :
12430 4692253 : mode2
12431 4692253 : = CONSTANT_P (op0) ? TYPE_MODE (TREE_TYPE (tem)) : GET_MODE (op0);
12432 :
12433 : /* See above for the rationale. */
12434 9384506 : if (maybe_gt (bitpos + bitsize, GET_MODE_BITSIZE (mode2)))
12435 2981988 : must_force_mem = true;
12436 :
12437 : /* Handle CONCAT first. */
12438 4692253 : if (GET_CODE (op0) == CONCAT && !must_force_mem)
12439 : {
12440 114 : if (known_eq (bitpos, 0)
12441 206 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0)))
12442 98 : && COMPLEX_MODE_P (mode1)
12443 93 : && COMPLEX_MODE_P (GET_MODE (op0))
12444 393 : && (GET_MODE_PRECISION (GET_MODE_INNER (mode1))
12445 186 : == GET_MODE_PRECISION (GET_MODE_INNER (GET_MODE (op0)))))
12446 : {
12447 93 : if (reversep)
12448 0 : op0 = flip_storage_order (GET_MODE (op0), op0);
12449 93 : if (mode1 != GET_MODE (op0))
12450 : {
12451 : rtx parts[2];
12452 0 : for (int i = 0; i < 2; i++)
12453 : {
12454 0 : rtx op = read_complex_part (op0, i != 0);
12455 0 : if (GET_CODE (op) == SUBREG)
12456 0 : op = force_reg (GET_MODE (op), op);
12457 0 : temp = gen_lowpart_common (GET_MODE_INNER (mode1), op);
12458 0 : if (temp)
12459 : op = temp;
12460 : else
12461 : {
12462 0 : if (!REG_P (op) && !MEM_P (op))
12463 0 : op = force_reg (GET_MODE (op), op);
12464 0 : op = gen_lowpart (GET_MODE_INNER (mode1), op);
12465 : }
12466 0 : parts[i] = op;
12467 : }
12468 0 : op0 = gen_rtx_CONCAT (mode1, parts[0], parts[1]);
12469 : }
12470 : return op0;
12471 : }
12472 21 : if (known_eq (bitpos, 0)
12473 20 : && known_eq (bitsize,
12474 : GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 0))))
12475 23 : && maybe_ne (bitsize, 0))
12476 : {
12477 : op0 = XEXP (op0, 0);
12478 : mode2 = GET_MODE (op0);
12479 : }
12480 38 : else if (known_eq (bitpos,
12481 : GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 0))))
12482 4 : && known_eq (bitsize,
12483 : GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 1))))
12484 0 : && maybe_ne (bitpos, 0)
12485 19 : && maybe_ne (bitsize, 0))
12486 : {
12487 0 : op0 = XEXP (op0, 1);
12488 0 : bitpos = 0;
12489 0 : mode2 = GET_MODE (op0);
12490 : }
12491 : else
12492 : /* Otherwise force into memory. */
12493 : must_force_mem = true;
12494 : }
12495 :
12496 : /* If this is a constant, put it in a register if it is a legitimate
12497 : constant and we don't need a memory reference. */
12498 4692160 : if (CONSTANT_P (op0)
12499 33 : && mode2 != BLKmode
12500 33 : && targetm.legitimate_constant_p (mode2, op0)
12501 4692181 : && !must_force_mem)
12502 21 : op0 = force_reg (mode2, op0);
12503 :
12504 : /* Otherwise, if this is a constant, try to force it to the constant
12505 : pool. Note that back-ends, e.g. MIPS, may refuse to do so if it
12506 : is a legitimate constant. */
12507 4692139 : else if (CONSTANT_P (op0) && (memloc = force_const_mem (mode2, op0)))
12508 12 : op0 = validize_mem (memloc);
12509 :
12510 : /* Otherwise, if this is a constant or the object is not in memory
12511 : and need be, put it there. */
12512 4692127 : else if (CONSTANT_P (op0) || (!MEM_P (op0) && must_force_mem))
12513 : {
12514 1095 : machine_mode tem_mode = TYPE_MODE (TREE_TYPE (tem));
12515 1095 : poly_int64 size;
12516 1095 : if (!poly_int_tree_p (TYPE_SIZE_UNIT (TREE_TYPE (tem)), &size))
12517 0 : size = max_int_size_in_bytes (TREE_TYPE (tem));
12518 1095 : unsigned int align = TREE_CODE (tem) == SSA_NAME
12519 1095 : ? TYPE_ALIGN (TREE_TYPE (tem))
12520 1095 : : get_object_alignment (tem);
12521 1095 : if (STRICT_ALIGNMENT)
12522 : {
12523 : /* For STRICT_ALIGNMENT targets, when we force the operand to
12524 : memory, we may need to increase the alignment to meet the
12525 : expectation in later RTL lowering passes. The increased
12526 : alignment is capped by MAX_SUPPORTED_STACK_ALIGNMENT. */
12527 : if (tem_mode != BLKmode)
12528 : align = MAX (align, GET_MODE_ALIGNMENT (tem_mode));
12529 : else
12530 : align = MAX (align, TYPE_ALIGN (TREE_TYPE (tem)));
12531 : align = MIN (align, (unsigned) MAX_SUPPORTED_STACK_ALIGNMENT);
12532 : }
12533 1095 : memloc = assign_stack_local (tem_mode, size, align);
12534 1095 : emit_move_insn (memloc, op0);
12535 1095 : op0 = memloc;
12536 1095 : clear_mem_expr = true;
12537 : }
12538 :
12539 4692160 : if (offset)
12540 : {
12541 280858 : machine_mode address_mode;
12542 280858 : rtx offset_rtx = expand_expr (offset, NULL_RTX, VOIDmode,
12543 : EXPAND_SUM);
12544 :
12545 280858 : gcc_assert (MEM_P (op0));
12546 :
12547 280858 : address_mode = get_address_mode (op0);
12548 280858 : if (GET_MODE (offset_rtx) != address_mode)
12549 : {
12550 : /* We cannot be sure that the RTL in offset_rtx is valid outside
12551 : of a memory address context, so force it into a register
12552 : before attempting to convert it to the desired mode. */
12553 424 : offset_rtx = force_operand (offset_rtx, NULL_RTX);
12554 424 : offset_rtx = convert_to_mode (address_mode, offset_rtx, 0);
12555 : }
12556 :
12557 : /* See the comment in expand_assignment for the rationale. */
12558 280858 : if (mode1 != VOIDmode
12559 280642 : && maybe_ne (bitpos, 0)
12560 74972 : && maybe_gt (bitsize, 0)
12561 355291 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
12562 355291 : && multiple_p (bitpos, bitsize)
12563 148866 : && multiple_p (bitsize, GET_MODE_ALIGNMENT (mode1))
12564 355291 : && MEM_ALIGN (op0) >= GET_MODE_ALIGNMENT (mode1))
12565 : {
12566 74075 : op0 = adjust_address (op0, mode1, bytepos);
12567 74075 : bitpos = 0;
12568 : }
12569 :
12570 280858 : op0 = offset_address (op0, offset_rtx,
12571 : highest_pow2_factor (offset));
12572 : }
12573 :
12574 : /* If OFFSET is making OP0 more aligned than BIGGEST_ALIGNMENT,
12575 : record its alignment as BIGGEST_ALIGNMENT. */
12576 4692160 : if (MEM_P (op0)
12577 4369548 : && known_eq (bitpos, 0)
12578 1473074 : && offset != 0
12579 4971971 : && is_aligning_offset (offset, tem))
12580 0 : set_mem_align (op0, BIGGEST_ALIGNMENT);
12581 :
12582 : /* Don't forget about volatility even if this is a bitfield. */
12583 4692160 : if (MEM_P (op0) && volatilep && ! MEM_VOLATILE_P (op0))
12584 : {
12585 514 : if (op0 == orig_op0)
12586 158 : op0 = copy_rtx (op0);
12587 :
12588 514 : MEM_VOLATILE_P (op0) = 1;
12589 : }
12590 :
12591 4692160 : if (MEM_P (op0) && TREE_CODE (tem) == FUNCTION_DECL)
12592 : {
12593 6 : if (op0 == orig_op0)
12594 0 : op0 = copy_rtx (op0);
12595 :
12596 6 : set_mem_align (op0, BITS_PER_UNIT);
12597 : }
12598 :
12599 : /* In cases where an aligned union has an unaligned object
12600 : as a field, we might be extracting a BLKmode value from
12601 : an integer-mode (e.g., SImode) object. Handle this case
12602 : by doing the extract into an object as wide as the field
12603 : (which we know to be the width of a basic mode), then
12604 : storing into memory, and changing the mode to BLKmode. */
12605 4692160 : if (mode1 == VOIDmode
12606 4459104 : || REG_P (op0) || GET_CODE (op0) == SUBREG
12607 4211072 : || (mode1 != BLKmode && ! direct_load[(int) mode1]
12608 26739 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_INT
12609 22564 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_FLOAT
12610 : && modifier != EXPAND_CONST_ADDRESS
12611 15389 : && modifier != EXPAND_INITIALIZER
12612 15389 : && modifier != EXPAND_MEMORY)
12613 : /* If the bitfield is volatile and the bitsize
12614 : is narrower than the access size of the bitfield,
12615 : we need to extract bitfields from the access. */
12616 4195683 : || (volatilep && TREE_CODE (exp) == COMPONENT_REF
12617 1497 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (exp, 1))
12618 10 : && mode1 != BLKmode
12619 20 : && maybe_lt (bitsize, GET_MODE_SIZE (mode1) * BITS_PER_UNIT))
12620 : /* If the field isn't aligned enough to fetch as a memref,
12621 : fetch it as a bit field. */
12622 4195674 : || (mode1 != BLKmode
12623 4122481 : && (((MEM_P (op0)
12624 4122481 : ? MEM_ALIGN (op0) < GET_MODE_ALIGNMENT (mode1)
12625 8099599 : || !multiple_p (bitpos, GET_MODE_ALIGNMENT (mode1))
12626 0 : : TYPE_ALIGN (TREE_TYPE (tem)) < GET_MODE_ALIGNMENT (mode)
12627 0 : || !multiple_p (bitpos, GET_MODE_ALIGNMENT (mode)))
12628 75941 : && modifier != EXPAND_MEMORY
12629 75941 : && ((modifier == EXPAND_CONST_ADDRESS
12630 75941 : || modifier == EXPAND_INITIALIZER)
12631 75941 : ? STRICT_ALIGNMENT
12632 75941 : : targetm.slow_unaligned_access (mode1,
12633 75941 : MEM_ALIGN (op0))))
12634 4122481 : || !multiple_p (bitpos, BITS_PER_UNIT)))
12635 : /* If the type and the field are a constant size and the
12636 : size of the type isn't the same size as the bitfield,
12637 : we must use bitfield operations. */
12638 13010315 : || (known_size_p (bitsize)
12639 4195674 : && TYPE_SIZE (TREE_TYPE (exp))
12640 4195674 : && poly_int_tree_p (TYPE_SIZE (TREE_TYPE (exp)))
12641 4692160 : && maybe_ne (wi::to_poly_offset (TYPE_SIZE (TREE_TYPE (exp))),
12642 : bitsize)))
12643 : {
12644 496524 : machine_mode ext_mode = mode;
12645 :
12646 496524 : if (ext_mode == BLKmode
12647 496524 : && ! (target != 0 && MEM_P (op0)
12648 22 : && MEM_P (target)
12649 22 : && multiple_p (bitpos, BITS_PER_UNIT)))
12650 12 : ext_mode = int_mode_for_size (bitsize, 1).else_blk ();
12651 :
12652 496524 : if (ext_mode == BLKmode)
12653 : {
12654 34 : if (target == 0)
12655 12 : target = assign_temp (type, 1, 1);
12656 :
12657 : /* ??? Unlike the similar test a few lines below, this one is
12658 : very likely obsolete. */
12659 34 : if (known_eq (bitsize, 0))
12660 : return target;
12661 :
12662 : /* In this case, BITPOS must start at a byte boundary and
12663 : TARGET, if specified, must be a MEM. */
12664 34 : gcc_assert (MEM_P (op0)
12665 : && (!target || MEM_P (target)));
12666 :
12667 34 : bytepos = exact_div (bitpos, BITS_PER_UNIT);
12668 34 : poly_int64 bytesize = bits_to_bytes_round_up (bitsize);
12669 68 : emit_block_move (target,
12670 : adjust_address (op0, VOIDmode, bytepos),
12671 34 : gen_int_mode (bytesize, Pmode),
12672 : (modifier == EXPAND_STACK_PARM
12673 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
12674 :
12675 34 : return target;
12676 : }
12677 :
12678 : /* If we have nothing to extract, the result will be 0 for targets
12679 : with SHIFT_COUNT_TRUNCATED == 0 and garbage otherwise. Always
12680 : return 0 for the sake of consistency, as reading a zero-sized
12681 : bitfield is valid in Ada and the value is fully specified. */
12682 496490 : if (known_eq (bitsize, 0))
12683 0 : return const0_rtx;
12684 :
12685 496490 : op0 = validize_mem (op0);
12686 :
12687 496490 : if (MEM_P (op0) && REG_P (XEXP (op0, 0)))
12688 50221 : mark_reg_pointer (XEXP (op0, 0), MEM_ALIGN (op0));
12689 :
12690 : /* If the result has aggregate type and the extraction is done in
12691 : an integral mode, then the field may be not aligned on a byte
12692 : boundary; in this case, if it has reverse storage order, it
12693 : needs to be extracted as a scalar field with reverse storage
12694 : order and put back into memory order afterwards. */
12695 496490 : if (AGGREGATE_TYPE_P (type)
12696 5205 : && GET_MODE_CLASS (ext_mode) == MODE_INT)
12697 5123 : reversep = TYPE_REVERSE_STORAGE_ORDER (type);
12698 :
12699 496490 : gcc_checking_assert (known_ge (bitpos, 0));
12700 999414 : op0 = extract_bit_field (op0, bitsize, bitpos, unsignedp,
12701 : (modifier == EXPAND_STACK_PARM
12702 : ? NULL_RTX : target),
12703 : ext_mode, ext_mode, reversep, alt_rtl);
12704 :
12705 : /* If the result has aggregate type and the mode of OP0 is an
12706 : integral mode then, if BITSIZE is narrower than this mode
12707 : and this is for big-endian data, we must put the field
12708 : into the high-order bits. And we must also put it back
12709 : into memory order if it has been previously reversed. */
12710 496490 : scalar_int_mode op0_mode;
12711 496490 : if (AGGREGATE_TYPE_P (type)
12712 496490 : && is_int_mode (GET_MODE (op0), &op0_mode))
12713 : {
12714 5123 : HOST_WIDE_INT size = GET_MODE_BITSIZE (op0_mode);
12715 :
12716 5123 : gcc_checking_assert (known_le (bitsize, size));
12717 5123 : if (maybe_lt (bitsize, size)
12718 5123 : && reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
12719 0 : op0 = expand_shift (LSHIFT_EXPR, op0_mode, op0,
12720 : size - bitsize, op0, 1);
12721 :
12722 5123 : if (reversep)
12723 0 : op0 = flip_storage_order (op0_mode, op0);
12724 : }
12725 :
12726 : /* If the result type is BLKmode, store the data into a temporary
12727 : of the appropriate type, but with the mode corresponding to the
12728 : mode for the data we have (op0's mode). */
12729 496490 : if (mode == BLKmode)
12730 : {
12731 0 : rtx new_rtx
12732 0 : = assign_stack_temp_for_type (ext_mode,
12733 0 : GET_MODE_BITSIZE (ext_mode),
12734 : type);
12735 0 : emit_move_insn (new_rtx, op0);
12736 0 : op0 = copy_rtx (new_rtx);
12737 0 : PUT_MODE (op0, BLKmode);
12738 : }
12739 :
12740 : return op0;
12741 : }
12742 :
12743 : /* If the result is BLKmode, use that to access the object
12744 : now as well. */
12745 4195636 : if (mode == BLKmode)
12746 73159 : mode1 = BLKmode;
12747 :
12748 : /* Get a reference to just this component. */
12749 4195636 : bytepos = bits_to_bytes_round_down (bitpos);
12750 4195636 : if (modifier == EXPAND_CONST_ADDRESS
12751 4195636 : || modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER)
12752 169085 : op0 = adjust_address_nv (op0, mode1, bytepos);
12753 : else
12754 4026551 : op0 = adjust_address (op0, mode1, bytepos);
12755 :
12756 4195636 : if (op0 == orig_op0)
12757 128713 : op0 = copy_rtx (op0);
12758 :
12759 : /* Don't set memory attributes if the base expression is
12760 : SSA_NAME that got expanded as a MEM or a CONSTANT. In that case,
12761 : we should just honor its original memory attributes. */
12762 4195636 : if (!(TREE_CODE (tem) == SSA_NAME
12763 9527 : && (MEM_P (orig_op0) || CONSTANT_P (orig_op0))))
12764 4186109 : set_mem_attributes (op0, exp, 0, may_store_p);
12765 :
12766 4195636 : if (REG_P (XEXP (op0, 0)))
12767 675093 : mark_reg_pointer (XEXP (op0, 0), MEM_ALIGN (op0));
12768 :
12769 : /* If op0 is a temporary because the original expressions was forced
12770 : to memory, clear MEM_EXPR so that the original expression cannot
12771 : be marked as addressable through MEM_EXPR of the temporary. */
12772 4195636 : if (clear_mem_expr)
12773 1061 : set_mem_expr (op0, NULL_TREE);
12774 :
12775 4195636 : MEM_VOLATILE_P (op0) |= volatilep;
12776 :
12777 4195636 : if (reversep
12778 : && modifier != EXPAND_MEMORY
12779 497 : && modifier != EXPAND_WRITE)
12780 497 : op0 = flip_storage_order (mode1, op0);
12781 :
12782 4195636 : op0 = EXTEND_BITINT (op0);
12783 :
12784 4195636 : if (mode == mode1 || mode1 == BLKmode || mode1 == tmode
12785 : || modifier == EXPAND_CONST_ADDRESS
12786 0 : || modifier == EXPAND_INITIALIZER)
12787 : return op0;
12788 :
12789 0 : if (target == 0)
12790 0 : target = gen_reg_rtx (tmode != VOIDmode ? tmode : mode);
12791 :
12792 0 : convert_move (target, op0, unsignedp);
12793 0 : return target;
12794 : }
12795 :
12796 87345 : case OBJ_TYPE_REF:
12797 87345 : return expand_expr (OBJ_TYPE_REF_EXPR (exp), target, tmode, modifier);
12798 :
12799 6893407 : case CALL_EXPR:
12800 : /* All valid uses of __builtin_va_arg_pack () are removed during
12801 : inlining. */
12802 6893407 : if (CALL_EXPR_VA_ARG_PACK (exp))
12803 6 : error ("invalid use of %<__builtin_va_arg_pack ()%>");
12804 6893407 : {
12805 6893407 : tree fndecl = get_callee_fndecl (exp), attr;
12806 :
12807 6893407 : if (fndecl
12808 : /* Don't diagnose the error attribute in thunks, those are
12809 : artificially created. */
12810 6705089 : && !CALL_FROM_THUNK_P (exp)
12811 13594851 : && (attr = lookup_attribute ("error",
12812 6701444 : DECL_ATTRIBUTES (fndecl))) != NULL)
12813 : {
12814 5 : const char *ident = lang_hooks.decl_printable_name (fndecl, 1);
12815 10 : error ("call to %qs declared with attribute error: %s",
12816 : identifier_to_locale (ident),
12817 5 : TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr))));
12818 : }
12819 6893407 : if (fndecl
12820 : /* Don't diagnose the warning attribute in thunks, those are
12821 : artificially created. */
12822 6705089 : && !CALL_FROM_THUNK_P (exp)
12823 13594851 : && (attr = lookup_attribute ("warning",
12824 6701444 : DECL_ATTRIBUTES (fndecl))) != NULL)
12825 : {
12826 21 : const char *ident = lang_hooks.decl_printable_name (fndecl, 1);
12827 42 : warning_at (EXPR_LOCATION (exp),
12828 21 : OPT_Wattribute_warning,
12829 : "call to %qs declared with attribute warning: %s",
12830 : identifier_to_locale (ident),
12831 21 : TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr))));
12832 : }
12833 :
12834 : /* Check for a built-in function. */
12835 6893407 : if (fndecl && fndecl_built_in_p (fndecl))
12836 : {
12837 2036546 : gcc_assert (DECL_BUILT_IN_CLASS (fndecl) != BUILT_IN_FRONTEND);
12838 2036546 : return expand_builtin (exp, target, subtarget, tmode, ignore);
12839 : }
12840 : }
12841 4856861 : temp = expand_call (exp, target, ignore);
12842 4856859 : return EXTEND_BITINT (temp);
12843 :
12844 347307 : case VIEW_CONVERT_EXPR:
12845 347307 : op0 = NULL_RTX;
12846 :
12847 : /* If we are converting to BLKmode, try to avoid an intermediate
12848 : temporary by fetching an inner memory reference. */
12849 347307 : if (mode == BLKmode
12850 74063 : && poly_int_tree_p (TYPE_SIZE (type))
12851 74063 : && TYPE_MODE (TREE_TYPE (treeop0)) != BLKmode
12852 347307 : && handled_component_p (treeop0))
12853 : {
12854 0 : machine_mode mode1;
12855 0 : poly_int64 bitsize, bitpos, bytepos;
12856 0 : tree offset;
12857 0 : int reversep, volatilep = 0;
12858 0 : tree tem
12859 0 : = get_inner_reference (treeop0, &bitsize, &bitpos, &offset, &mode1,
12860 : &unsignedp, &reversep, &volatilep);
12861 :
12862 : /* ??? We should work harder and deal with non-zero offsets. */
12863 0 : if (!offset
12864 0 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
12865 0 : && !reversep
12866 0 : && known_size_p (bitsize)
12867 0 : && known_eq (wi::to_poly_offset (TYPE_SIZE (type)), bitsize))
12868 : {
12869 : /* See the normal_inner_ref case for the rationale. */
12870 0 : rtx orig_op0
12871 0 : = expand_expr_real (tem,
12872 0 : (TREE_CODE (TREE_TYPE (tem)) == UNION_TYPE
12873 0 : && (TREE_CODE (TYPE_SIZE (TREE_TYPE (tem)))
12874 : != INTEGER_CST)
12875 0 : && modifier != EXPAND_STACK_PARM
12876 : ? target : NULL_RTX),
12877 : VOIDmode,
12878 : modifier == EXPAND_SUM ? EXPAND_NORMAL : modifier,
12879 : NULL, true);
12880 :
12881 0 : if (MEM_P (orig_op0))
12882 : {
12883 0 : op0 = orig_op0;
12884 :
12885 : /* Get a reference to just this component. */
12886 0 : if (modifier == EXPAND_CONST_ADDRESS
12887 : || modifier == EXPAND_SUM
12888 0 : || modifier == EXPAND_INITIALIZER)
12889 0 : op0 = adjust_address_nv (op0, mode, bytepos);
12890 : else
12891 0 : op0 = adjust_address (op0, mode, bytepos);
12892 :
12893 0 : if (op0 == orig_op0)
12894 0 : op0 = copy_rtx (op0);
12895 :
12896 0 : set_mem_attributes (op0, treeop0, 0, may_store_p);
12897 0 : if (REG_P (XEXP (op0, 0)))
12898 0 : mark_reg_pointer (XEXP (op0, 0), MEM_ALIGN (op0));
12899 :
12900 0 : MEM_VOLATILE_P (op0) |= volatilep;
12901 : }
12902 : }
12903 : }
12904 :
12905 0 : if (!op0)
12906 421381 : op0 = expand_expr_real (treeop0, NULL_RTX, VOIDmode, modifier,
12907 : NULL, inner_reference_p || mode == BLKmode);
12908 :
12909 : /* If the input and output modes are both the same, we are done. */
12910 347307 : if (mode == GET_MODE (op0))
12911 : ;
12912 : /* Similarly if the output mode is BLKmode and input is a MEM,
12913 : adjust_address done below is all we need. */
12914 152011 : else if (mode == BLKmode && MEM_P (op0))
12915 : ;
12916 : /* If neither mode is BLKmode, and both modes are the same size
12917 : then we can use gen_lowpart. */
12918 : else if (mode != BLKmode
12919 151977 : && GET_MODE (op0) != BLKmode
12920 150641 : && known_eq (GET_MODE_PRECISION (mode),
12921 : GET_MODE_PRECISION (GET_MODE (op0)))
12922 146742 : && !COMPLEX_MODE_P (GET_MODE (op0)))
12923 : {
12924 145354 : if (GET_CODE (op0) == SUBREG)
12925 111 : op0 = force_reg (GET_MODE (op0), op0);
12926 145354 : temp = gen_lowpart_common (mode, op0);
12927 145354 : if (temp)
12928 : op0 = temp;
12929 : else
12930 : {
12931 28136 : if (!REG_P (op0) && !MEM_P (op0))
12932 18 : op0 = force_reg (GET_MODE (op0), op0);
12933 28136 : op0 = gen_lowpart (mode, op0);
12934 : }
12935 : }
12936 : /* If both types are integral, convert from one mode to the other. */
12937 6655 : else if (INTEGRAL_TYPE_P (type)
12938 3130 : && INTEGRAL_TYPE_P (TREE_TYPE (treeop0))
12939 0 : && mode != BLKmode
12940 6655 : && GET_MODE (op0) != BLKmode)
12941 0 : op0 = convert_modes (mode, GET_MODE (op0), op0,
12942 0 : TYPE_UNSIGNED (TREE_TYPE (treeop0)));
12943 : /* If the output type is a bit-field type, do an extraction. */
12944 6655 : else if (reduce_bit_field
12945 2 : && mode != BLKmode
12946 2 : && (MEM_P (op0) || !COMPLEX_MODE_P (GET_MODE (op0))))
12947 0 : return extract_bit_field (op0, TYPE_PRECISION (type), 0,
12948 0 : TYPE_UNSIGNED (type), NULL_RTX,
12949 : mode, mode, false, NULL);
12950 : /* If source is a complex number and destination is a
12951 : two-component vector with same inner type, try to use
12952 : vector initialization. */
12953 6655 : else if ((temp = try_expand_complex_as_vector (mode, op0, target))
12954 : != NULL_RTX)
12955 : return temp;
12956 : /* As a last resort, spill op0 to memory, and reload it in a
12957 : different mode. */
12958 6642 : else if (!MEM_P (op0))
12959 : {
12960 : /* If the operand is not a MEM, force it into memory. Since we
12961 : are going to be changing the mode of the MEM, don't call
12962 : force_const_mem for constants because we don't allow pool
12963 : constants to change mode. */
12964 5296 : tree inner_type = TREE_TYPE (treeop0);
12965 :
12966 5296 : gcc_assert (!TREE_ADDRESSABLE (exp));
12967 :
12968 5296 : if (target == 0 || GET_MODE (target) != TYPE_MODE (inner_type))
12969 5292 : target
12970 : = assign_stack_temp_for_type
12971 5292 : (TYPE_MODE (inner_type),
12972 10584 : GET_MODE_SIZE (TYPE_MODE (inner_type)), inner_type);
12973 :
12974 5296 : emit_move_insn (target, op0);
12975 5296 : op0 = target;
12976 :
12977 5296 : if (reduce_bit_field && mode != BLKmode)
12978 2 : return extract_bit_field (op0, TYPE_PRECISION (type), 0,
12979 2 : TYPE_UNSIGNED (type), NULL_RTX,
12980 : mode, mode, false, NULL);
12981 : }
12982 :
12983 : /* If OP0 is (now) a MEM, we need to deal with alignment issues. If the
12984 : output type is such that the operand is known to be aligned, indicate
12985 : that it is. Otherwise, we need only be concerned about alignment for
12986 : non-BLKmode results. */
12987 347292 : if (MEM_P (op0))
12988 : {
12989 128693 : enum insn_code icode;
12990 :
12991 128693 : if (modifier != EXPAND_WRITE
12992 128693 : && modifier != EXPAND_MEMORY
12993 128693 : && !inner_reference_p
12994 128692 : && mode != BLKmode
12995 183322 : && MEM_ALIGN (op0) < GET_MODE_ALIGNMENT (mode))
12996 : {
12997 : /* If the target does have special handling for unaligned
12998 : loads of mode then use them. */
12999 15853 : if ((icode = optab_handler (movmisalign_optab, mode))
13000 : != CODE_FOR_nothing)
13001 : {
13002 1904 : rtx reg;
13003 :
13004 1904 : op0 = adjust_address (op0, mode, 0);
13005 : /* We've already validated the memory, and we're creating a
13006 : new pseudo destination. The predicates really can't
13007 : fail. */
13008 1904 : reg = gen_reg_rtx (mode);
13009 :
13010 : /* Nor can the insn generator. */
13011 1904 : rtx_insn *insn = GEN_FCN (icode) (reg, op0);
13012 1904 : emit_insn (insn);
13013 1904 : return reg;
13014 : }
13015 : else if (STRICT_ALIGNMENT)
13016 : {
13017 : poly_uint64 mode_size = GET_MODE_SIZE (mode);
13018 : poly_uint64 temp_size = mode_size;
13019 : if (GET_MODE (op0) != BLKmode)
13020 : temp_size = upper_bound (temp_size,
13021 : GET_MODE_SIZE (GET_MODE (op0)));
13022 : rtx new_rtx
13023 : = assign_stack_temp_for_type (mode, temp_size, type);
13024 : rtx new_with_op0_mode
13025 : = adjust_address (new_rtx, GET_MODE (op0), 0);
13026 :
13027 : gcc_assert (!TREE_ADDRESSABLE (exp));
13028 :
13029 : if (GET_MODE (op0) == BLKmode)
13030 : {
13031 : rtx size_rtx = gen_int_mode (mode_size, Pmode);
13032 : emit_block_move (new_with_op0_mode, op0, size_rtx,
13033 : (modifier == EXPAND_STACK_PARM
13034 : ? BLOCK_OP_CALL_PARM
13035 : : BLOCK_OP_NORMAL));
13036 : }
13037 : else
13038 : emit_move_insn (new_with_op0_mode, op0);
13039 :
13040 : op0 = new_rtx;
13041 : }
13042 : }
13043 :
13044 126789 : op0 = adjust_address (op0, mode, 0);
13045 : }
13046 :
13047 : return op0;
13048 :
13049 63650 : case MODIFY_EXPR:
13050 63650 : {
13051 63650 : tree lhs = treeop0;
13052 63650 : tree rhs = treeop1;
13053 63650 : gcc_assert (ignore);
13054 :
13055 : /* Check for |= or &= of a bitfield of size one into another bitfield
13056 : of size 1. In this case, (unless we need the result of the
13057 : assignment) we can do this more efficiently with a
13058 : test followed by an assignment, if necessary.
13059 :
13060 : ??? At this point, we can't get a BIT_FIELD_REF here. But if
13061 : things change so we do, this code should be enhanced to
13062 : support it. */
13063 63650 : if (TREE_CODE (lhs) == COMPONENT_REF
13064 60508 : && (TREE_CODE (rhs) == BIT_IOR_EXPR
13065 60508 : || TREE_CODE (rhs) == BIT_AND_EXPR)
13066 0 : && TREE_OPERAND (rhs, 0) == lhs
13067 0 : && TREE_CODE (TREE_OPERAND (rhs, 1)) == COMPONENT_REF
13068 0 : && integer_onep (DECL_SIZE (TREE_OPERAND (lhs, 1)))
13069 63650 : && integer_onep (DECL_SIZE (TREE_OPERAND (TREE_OPERAND (rhs, 1), 1))))
13070 : {
13071 0 : rtx_code_label *label = gen_label_rtx ();
13072 0 : int value = TREE_CODE (rhs) == BIT_IOR_EXPR;
13073 0 : profile_probability prob = profile_probability::uninitialized ();
13074 0 : if (value)
13075 0 : jumpifnot (TREE_OPERAND (rhs, 1), label, prob);
13076 : else
13077 0 : jumpif (TREE_OPERAND (rhs, 1), label, prob);
13078 0 : expand_assignment (lhs, build_int_cst (TREE_TYPE (rhs), value),
13079 : false);
13080 0 : do_pending_stack_adjust ();
13081 0 : emit_label (label);
13082 0 : return const0_rtx;
13083 : }
13084 :
13085 63650 : expand_assignment (lhs, rhs, false);
13086 63650 : return const0_rtx;
13087 : }
13088 :
13089 13327996 : case ADDR_EXPR:
13090 13327996 : return expand_expr_addr_expr (exp, target, tmode, modifier);
13091 :
13092 172865 : case REALPART_EXPR:
13093 172865 : op0 = expand_normal (treeop0);
13094 172865 : return read_complex_part (op0, false);
13095 :
13096 196786 : case IMAGPART_EXPR:
13097 196786 : op0 = expand_normal (treeop0);
13098 196786 : return read_complex_part (op0, true);
13099 :
13100 0 : case RETURN_EXPR:
13101 0 : case LABEL_EXPR:
13102 0 : case GOTO_EXPR:
13103 0 : case SWITCH_EXPR:
13104 0 : case ASM_EXPR:
13105 : /* Expanded in cfgexpand.cc. */
13106 0 : gcc_unreachable ();
13107 :
13108 0 : case TRY_CATCH_EXPR:
13109 0 : case CATCH_EXPR:
13110 0 : case EH_FILTER_EXPR:
13111 0 : case TRY_FINALLY_EXPR:
13112 0 : case EH_ELSE_EXPR:
13113 : /* Lowered by tree-eh.cc. */
13114 0 : gcc_unreachable ();
13115 :
13116 0 : case WITH_CLEANUP_EXPR:
13117 0 : case CLEANUP_POINT_EXPR:
13118 0 : case TARGET_EXPR:
13119 0 : case CASE_LABEL_EXPR:
13120 0 : case VA_ARG_EXPR:
13121 0 : case BIND_EXPR:
13122 0 : case INIT_EXPR:
13123 0 : case CONJ_EXPR:
13124 0 : case COMPOUND_EXPR:
13125 0 : case PREINCREMENT_EXPR:
13126 0 : case PREDECREMENT_EXPR:
13127 0 : case POSTINCREMENT_EXPR:
13128 0 : case POSTDECREMENT_EXPR:
13129 0 : case LOOP_EXPR:
13130 0 : case EXIT_EXPR:
13131 0 : case COMPOUND_LITERAL_EXPR:
13132 : /* Lowered by gimplify.cc. */
13133 0 : gcc_unreachable ();
13134 :
13135 0 : case FDESC_EXPR:
13136 : /* Function descriptors are not valid except for as
13137 : initialization constants, and should not be expanded. */
13138 0 : gcc_unreachable ();
13139 :
13140 258 : case WITH_SIZE_EXPR:
13141 : /* WITH_SIZE_EXPR expands to its first argument. The caller should
13142 : have pulled out the size to use in whatever context it needed. */
13143 258 : return expand_expr_real (treeop0, original_target, tmode,
13144 258 : modifier, alt_rtl, inner_reference_p);
13145 :
13146 1864129 : default:
13147 1864129 : return expand_expr_real_2 (&ops, target, tmode, modifier);
13148 : }
13149 : }
13150 : #undef EXTEND_BITINT
13151 :
13152 : /* Subroutine of above: reduce EXP to the precision of TYPE (in the
13153 : signedness of TYPE), possibly returning the result in TARGET.
13154 : TYPE is known to be a partial integer type. */
13155 : static rtx
13156 89926 : reduce_to_bit_field_precision (rtx exp, rtx target, tree type)
13157 : {
13158 89926 : scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
13159 89926 : HOST_WIDE_INT prec = TYPE_PRECISION (type);
13160 89926 : gcc_assert ((GET_MODE (exp) == VOIDmode || GET_MODE (exp) == mode)
13161 : && (!target || GET_MODE (target) == mode));
13162 :
13163 : /* For constant values, reduce using wide_int_to_tree. */
13164 89926 : if (poly_int_rtx_p (exp))
13165 : {
13166 0 : auto value = wi::to_poly_wide (exp, mode);
13167 0 : tree t = wide_int_to_tree (type, value);
13168 0 : return expand_expr (t, target, VOIDmode, EXPAND_NORMAL);
13169 : }
13170 89926 : else if (TYPE_UNSIGNED (type))
13171 : {
13172 70937 : rtx mask = immed_wide_int_const
13173 70937 : (wi::mask (prec, false, GET_MODE_PRECISION (mode)), mode);
13174 70937 : return expand_and (mode, exp, mask, target);
13175 : }
13176 : else
13177 : {
13178 18989 : int count = GET_MODE_PRECISION (mode) - prec;
13179 18989 : exp = expand_shift (LSHIFT_EXPR, mode, exp, count, target, 0);
13180 18989 : return expand_shift (RSHIFT_EXPR, mode, exp, count, target, 0);
13181 : }
13182 : }
13183 :
13184 : /* Subroutine of above: returns true if OFFSET corresponds to an offset that
13185 : when applied to the address of EXP produces an address known to be
13186 : aligned more than BIGGEST_ALIGNMENT. */
13187 :
13188 : static bool
13189 279811 : is_aligning_offset (const_tree offset, const_tree exp)
13190 : {
13191 : /* Strip off any conversions. */
13192 297748 : while (CONVERT_EXPR_P (offset))
13193 17937 : offset = TREE_OPERAND (offset, 0);
13194 :
13195 : /* We must now have a BIT_AND_EXPR with a constant that is one less than
13196 : power of 2 and which is larger than BIGGEST_ALIGNMENT. */
13197 279811 : if (TREE_CODE (offset) != BIT_AND_EXPR
13198 0 : || !tree_fits_uhwi_p (TREE_OPERAND (offset, 1))
13199 0 : || compare_tree_int (TREE_OPERAND (offset, 1),
13200 0 : BIGGEST_ALIGNMENT / BITS_PER_UNIT) <= 0
13201 279811 : || !pow2p_hwi (tree_to_uhwi (TREE_OPERAND (offset, 1)) + 1))
13202 : return false;
13203 :
13204 : /* Look at the first operand of BIT_AND_EXPR and strip any conversion.
13205 : It must be NEGATE_EXPR. Then strip any more conversions. */
13206 0 : offset = TREE_OPERAND (offset, 0);
13207 0 : while (CONVERT_EXPR_P (offset))
13208 0 : offset = TREE_OPERAND (offset, 0);
13209 :
13210 0 : if (TREE_CODE (offset) != NEGATE_EXPR)
13211 : return false;
13212 :
13213 0 : offset = TREE_OPERAND (offset, 0);
13214 0 : while (CONVERT_EXPR_P (offset))
13215 0 : offset = TREE_OPERAND (offset, 0);
13216 :
13217 : /* This must now be the address of EXP. */
13218 0 : return TREE_CODE (offset) == ADDR_EXPR && TREE_OPERAND (offset, 0) == exp;
13219 : }
13220 :
13221 : /* Return a STRING_CST corresponding to ARG's constant initializer either
13222 : if it's a string constant, or, when VALREP is set, any other constant,
13223 : or null otherwise.
13224 : On success, set *PTR_OFFSET to the (possibly non-constant) byte offset
13225 : within the byte string that ARG is references. If nonnull set *MEM_SIZE
13226 : to the size of the byte string. If nonnull, set *DECL to the constant
13227 : declaration ARG refers to. */
13228 :
13229 : static tree
13230 15959011 : constant_byte_string (tree arg, tree *ptr_offset, tree *mem_size, tree *decl,
13231 : bool valrep = false)
13232 : {
13233 15959011 : tree dummy = NULL_TREE;
13234 15959011 : if (!mem_size)
13235 11137 : mem_size = &dummy;
13236 :
13237 : /* Store the type of the original expression before conversions
13238 : via NOP_EXPR or POINTER_PLUS_EXPR to other types have been
13239 : removed. */
13240 15959011 : tree argtype = TREE_TYPE (arg);
13241 :
13242 15959011 : tree array;
13243 15959011 : STRIP_NOPS (arg);
13244 :
13245 : /* Non-constant index into the character array in an ARRAY_REF
13246 : expression or null. */
13247 15959011 : tree varidx = NULL_TREE;
13248 :
13249 15959011 : poly_int64 base_off = 0;
13250 :
13251 15959011 : if (TREE_CODE (arg) == ADDR_EXPR)
13252 : {
13253 6479389 : arg = TREE_OPERAND (arg, 0);
13254 6479389 : tree ref = arg;
13255 6479389 : if (TREE_CODE (arg) == ARRAY_REF)
13256 : {
13257 573605 : tree idx = TREE_OPERAND (arg, 1);
13258 573605 : if (TREE_CODE (idx) != INTEGER_CST)
13259 : {
13260 : /* From a pointer (but not array) argument extract the variable
13261 : index to prevent get_addr_base_and_unit_offset() from failing
13262 : due to it. Use it later to compute the non-constant offset
13263 : into the string and return it to the caller. */
13264 168696 : varidx = idx;
13265 168696 : ref = TREE_OPERAND (arg, 0);
13266 :
13267 168696 : if (TREE_CODE (TREE_TYPE (arg)) == ARRAY_TYPE)
13268 : return NULL_TREE;
13269 :
13270 28869 : if (!integer_zerop (array_ref_low_bound (arg)))
13271 : return NULL_TREE;
13272 :
13273 28119 : if (!integer_onep (array_ref_element_size (arg)))
13274 : return NULL_TREE;
13275 : }
13276 : }
13277 6337709 : array = get_addr_base_and_unit_offset (ref, &base_off);
13278 6337709 : if (!array
13279 6289956 : || (TREE_CODE (array) != VAR_DECL
13280 6289956 : && TREE_CODE (array) != CONST_DECL
13281 3944037 : && TREE_CODE (array) != STRING_CST))
13282 : return NULL_TREE;
13283 : }
13284 9479622 : else if (TREE_CODE (arg) == PLUS_EXPR || TREE_CODE (arg) == POINTER_PLUS_EXPR)
13285 : {
13286 27192 : tree arg0 = TREE_OPERAND (arg, 0);
13287 27192 : tree arg1 = TREE_OPERAND (arg, 1);
13288 :
13289 27192 : tree offset;
13290 27192 : tree str = string_constant (arg0, &offset, mem_size, decl);
13291 27192 : if (!str)
13292 : {
13293 17550 : str = string_constant (arg1, &offset, mem_size, decl);
13294 17550 : arg1 = arg0;
13295 : }
13296 :
13297 17550 : if (str)
13298 : {
13299 : /* Avoid pointers to arrays (see bug 86622). */
13300 9642 : if (POINTER_TYPE_P (TREE_TYPE (arg))
13301 9642 : && TREE_CODE (TREE_TYPE (TREE_TYPE (arg))) == ARRAY_TYPE
13302 1890 : && !(decl && !*decl)
13303 12162 : && !(decl && tree_fits_uhwi_p (DECL_SIZE_UNIT (*decl))
13304 1260 : && tree_fits_uhwi_p (*mem_size)
13305 1260 : && tree_int_cst_equal (*mem_size, DECL_SIZE_UNIT (*decl))))
13306 : return NULL_TREE;
13307 :
13308 7752 : tree type = TREE_TYPE (offset);
13309 7752 : arg1 = fold_convert (type, arg1);
13310 7752 : *ptr_offset = fold_build2 (PLUS_EXPR, type, offset, arg1);
13311 7752 : return str;
13312 : }
13313 : return NULL_TREE;
13314 : }
13315 9452430 : else if (TREE_CODE (arg) == SSA_NAME)
13316 : {
13317 3927182 : gimple *stmt = SSA_NAME_DEF_STMT (arg);
13318 3927182 : if (!is_gimple_assign (stmt))
13319 : return NULL_TREE;
13320 :
13321 1059983 : tree rhs1 = gimple_assign_rhs1 (stmt);
13322 1059983 : tree_code code = gimple_assign_rhs_code (stmt);
13323 1059983 : if (code == ADDR_EXPR)
13324 237417 : return string_constant (rhs1, ptr_offset, mem_size, decl);
13325 822566 : else if (code != POINTER_PLUS_EXPR)
13326 : return NULL_TREE;
13327 :
13328 194356 : tree offset;
13329 194356 : if (tree str = string_constant (rhs1, &offset, mem_size, decl))
13330 : {
13331 : /* Avoid pointers to arrays (see bug 86622). */
13332 23619 : if (POINTER_TYPE_P (TREE_TYPE (rhs1))
13333 23619 : && TREE_CODE (TREE_TYPE (TREE_TYPE (rhs1))) == ARRAY_TYPE
13334 19684 : && !(decl && !*decl)
13335 39867 : && !(decl && tree_fits_uhwi_p (DECL_SIZE_UNIT (*decl))
13336 8124 : && tree_fits_uhwi_p (*mem_size)
13337 8124 : && tree_int_cst_equal (*mem_size, DECL_SIZE_UNIT (*decl))))
13338 : return NULL_TREE;
13339 :
13340 7229 : tree rhs2 = gimple_assign_rhs2 (stmt);
13341 7229 : tree type = TREE_TYPE (offset);
13342 7229 : rhs2 = fold_convert (type, rhs2);
13343 7229 : *ptr_offset = fold_build2 (PLUS_EXPR, type, offset, rhs2);
13344 7229 : return str;
13345 : }
13346 : return NULL_TREE;
13347 : }
13348 5525248 : else if (DECL_P (arg))
13349 : array = arg;
13350 : else
13351 : return NULL_TREE;
13352 :
13353 8888569 : tree offset = wide_int_to_tree (sizetype, base_off);
13354 8888569 : if (varidx)
13355 : {
13356 2852 : if (TREE_CODE (TREE_TYPE (array)) != ARRAY_TYPE)
13357 : return NULL_TREE;
13358 :
13359 2141 : gcc_assert (TREE_CODE (arg) == ARRAY_REF);
13360 2141 : tree chartype = TREE_TYPE (TREE_TYPE (TREE_OPERAND (arg, 0)));
13361 2141 : if (TREE_CODE (chartype) != INTEGER_TYPE)
13362 : return NULL;
13363 :
13364 1985 : offset = fold_convert (sizetype, varidx);
13365 : }
13366 :
13367 8887702 : if (TREE_CODE (array) == STRING_CST)
13368 : {
13369 3546111 : *ptr_offset = fold_convert (sizetype, offset);
13370 3546111 : *mem_size = TYPE_SIZE_UNIT (TREE_TYPE (array));
13371 3546111 : if (decl)
13372 762040 : *decl = NULL_TREE;
13373 3546111 : gcc_checking_assert (tree_to_shwi (TYPE_SIZE_UNIT (TREE_TYPE (array)))
13374 : >= TREE_STRING_LENGTH (array));
13375 : return array;
13376 : }
13377 :
13378 5341591 : tree init = ctor_for_folding (array);
13379 5341591 : if (!init || init == error_mark_node)
13380 : return NULL_TREE;
13381 :
13382 163601 : if (valrep)
13383 : {
13384 58527 : HOST_WIDE_INT cstoff;
13385 58527 : if (!base_off.is_constant (&cstoff))
13386 : return NULL_TREE;
13387 :
13388 : /* Check that the host and target are sane. */
13389 58527 : if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
13390 : return NULL_TREE;
13391 :
13392 58527 : HOST_WIDE_INT typesz = int_size_in_bytes (TREE_TYPE (init));
13393 58527 : if (typesz <= 0 || (int) typesz != typesz)
13394 : return NULL_TREE;
13395 :
13396 58374 : HOST_WIDE_INT size = typesz;
13397 58374 : if (VAR_P (array)
13398 58362 : && DECL_SIZE_UNIT (array)
13399 116736 : && tree_fits_shwi_p (DECL_SIZE_UNIT (array)))
13400 : {
13401 58362 : size = tree_to_shwi (DECL_SIZE_UNIT (array));
13402 58362 : gcc_checking_assert (size >= typesz);
13403 : }
13404 :
13405 : /* If value representation was requested convert the initializer
13406 : for the whole array or object into a string of bytes forming
13407 : its value representation and return it. */
13408 58374 : unsigned char *bytes = XNEWVEC (unsigned char, size);
13409 58374 : int r = native_encode_initializer (init, bytes, size);
13410 58374 : if (r < typesz)
13411 : {
13412 64 : XDELETEVEC (bytes);
13413 64 : return NULL_TREE;
13414 : }
13415 :
13416 58310 : if (r < size)
13417 0 : memset (bytes + r, '\0', size - r);
13418 :
13419 58310 : const char *p = reinterpret_cast<const char *>(bytes);
13420 58310 : init = build_string_literal (size, p, char_type_node);
13421 58310 : init = TREE_OPERAND (init, 0);
13422 58310 : init = TREE_OPERAND (init, 0);
13423 58310 : XDELETE (bytes);
13424 :
13425 58310 : *mem_size = size_int (TREE_STRING_LENGTH (init));
13426 58310 : *ptr_offset = wide_int_to_tree (ssizetype, base_off);
13427 :
13428 58310 : if (decl)
13429 0 : *decl = array;
13430 :
13431 : return init;
13432 : }
13433 :
13434 105074 : if (TREE_CODE (init) == CONSTRUCTOR)
13435 : {
13436 : /* Convert the 64-bit constant offset to a wider type to avoid
13437 : overflow and use it to obtain the initializer for the subobject
13438 : it points into. */
13439 84212 : offset_int wioff;
13440 84212 : if (!base_off.is_constant (&wioff))
13441 155 : return NULL_TREE;
13442 :
13443 84212 : wioff *= BITS_PER_UNIT;
13444 84212 : if (!wi::fits_uhwi_p (wioff))
13445 : return NULL_TREE;
13446 :
13447 84104 : base_off = wioff.to_uhwi ();
13448 84104 : unsigned HOST_WIDE_INT fieldoff = 0;
13449 84104 : init = fold_ctor_reference (TREE_TYPE (arg), init, base_off, 0, array,
13450 : &fieldoff);
13451 84104 : if (!init || init == error_mark_node)
13452 : return NULL_TREE;
13453 :
13454 84057 : HOST_WIDE_INT cstoff;
13455 84057 : if (!base_off.is_constant (&cstoff))
13456 : return NULL_TREE;
13457 :
13458 84057 : cstoff = (cstoff - fieldoff) / BITS_PER_UNIT;
13459 84057 : tree off = build_int_cst (sizetype, cstoff);
13460 84057 : if (varidx)
13461 1122 : offset = fold_build2 (PLUS_EXPR, TREE_TYPE (offset), offset, off);
13462 : else
13463 : offset = off;
13464 : }
13465 :
13466 104919 : *ptr_offset = offset;
13467 :
13468 104919 : tree inittype = TREE_TYPE (init);
13469 :
13470 104919 : if (TREE_CODE (init) == INTEGER_CST
13471 104919 : && (TREE_CODE (TREE_TYPE (array)) == INTEGER_TYPE
13472 1116 : || TYPE_MAIN_VARIANT (inittype) == char_type_node))
13473 : {
13474 : /* Check that the host and target are sane. */
13475 903 : if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
13476 : return NULL_TREE;
13477 :
13478 : /* For a reference to (address of) a single constant character,
13479 : store the native representation of the character in CHARBUF.
13480 : If the reference is to an element of an array or a member
13481 : of a struct, only consider narrow characters until ctors
13482 : for wide character arrays are transformed to STRING_CSTs
13483 : like those for narrow arrays. */
13484 903 : unsigned char charbuf[MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT];
13485 903 : int len = native_encode_expr (init, charbuf, sizeof charbuf, 0);
13486 903 : if (len > 0)
13487 : {
13488 : /* Construct a string literal with elements of INITTYPE and
13489 : the representation above. Then strip
13490 : the ADDR_EXPR (ARRAY_REF (...)) around the STRING_CST. */
13491 903 : init = build_string_literal (len, (char *)charbuf, inittype);
13492 903 : init = TREE_OPERAND (TREE_OPERAND (init, 0), 0);
13493 : }
13494 : }
13495 :
13496 104919 : tree initsize = TYPE_SIZE_UNIT (inittype);
13497 :
13498 104919 : if (TREE_CODE (init) == CONSTRUCTOR && initializer_zerop (init))
13499 : {
13500 : /* Fold an empty/zero constructor for an implicitly initialized
13501 : object or subobject into the empty string. */
13502 :
13503 : /* Determine the character type from that of the original
13504 : expression. */
13505 9339 : tree chartype = argtype;
13506 9339 : if (POINTER_TYPE_P (chartype))
13507 9332 : chartype = TREE_TYPE (chartype);
13508 14962 : while (TREE_CODE (chartype) == ARRAY_TYPE)
13509 5623 : chartype = TREE_TYPE (chartype);
13510 :
13511 9339 : if (INTEGRAL_TYPE_P (chartype)
13512 9339 : && TYPE_PRECISION (chartype) == TYPE_PRECISION (char_type_node))
13513 : {
13514 : /* Convert a char array to an empty STRING_CST having an array
13515 : of the expected type and size. */
13516 9133 : if (!initsize)
13517 3090 : initsize = integer_zero_node;
13518 6043 : else if (!tree_fits_uhwi_p (initsize))
13519 : return NULL_TREE;
13520 :
13521 9127 : unsigned HOST_WIDE_INT size = tree_to_uhwi (initsize);
13522 9127 : if (size > (unsigned HOST_WIDE_INT) INT_MAX)
13523 : return NULL_TREE;
13524 :
13525 9127 : init = build_string_literal (size, NULL, chartype, size);
13526 9127 : init = TREE_OPERAND (init, 0);
13527 9127 : init = TREE_OPERAND (init, 0);
13528 :
13529 9127 : *ptr_offset = integer_zero_node;
13530 : }
13531 : }
13532 :
13533 104913 : if (decl)
13534 54433 : *decl = array;
13535 :
13536 104913 : if (TREE_CODE (init) != STRING_CST)
13537 : return NULL_TREE;
13538 :
13539 72580 : *mem_size = initsize;
13540 :
13541 72580 : gcc_checking_assert (tree_to_shwi (initsize) >= TREE_STRING_LENGTH (init));
13542 :
13543 : return init;
13544 : }
13545 :
13546 : /* Return STRING_CST if an ARG corresponds to a string constant or zero
13547 : if it doesn't. If we return nonzero, set *PTR_OFFSET to the (possibly
13548 : non-constant) offset in bytes within the string that ARG is accessing.
13549 : If MEM_SIZE is non-zero the storage size of the memory is returned.
13550 : If DECL is non-zero the constant declaration is returned if available. */
13551 :
13552 : tree
13553 11267981 : string_constant (tree arg, tree *ptr_offset, tree *mem_size, tree *decl)
13554 : {
13555 11267981 : return constant_byte_string (arg, ptr_offset, mem_size, decl, false);
13556 : }
13557 :
13558 : /* Similar to string_constant, return a STRING_CST corresponding
13559 : to the value representation of the first argument if it's
13560 : a constant. */
13561 :
13562 : tree
13563 4691030 : byte_representation (tree arg, tree *ptr_offset, tree *mem_size, tree *decl)
13564 : {
13565 4691030 : return constant_byte_string (arg, ptr_offset, mem_size, decl, true);
13566 : }
13567 :
13568 : /* Optimize x % C1 == C2 for signed modulo if C1 is a power of two and C2
13569 : is non-zero and C3 ((1<<(prec-1)) | (C1 - 1)):
13570 : for C2 > 0 to x & C3 == C2
13571 : for C2 < 0 to x & C3 == (C2 & C3). */
13572 : enum tree_code
13573 35 : maybe_optimize_pow2p_mod_cmp (enum tree_code code, tree *arg0, tree *arg1)
13574 : {
13575 35 : gimple *stmt = get_def_for_expr (*arg0, TRUNC_MOD_EXPR);
13576 35 : tree treeop0 = gimple_assign_rhs1 (stmt);
13577 35 : tree treeop1 = gimple_assign_rhs2 (stmt);
13578 35 : tree type = TREE_TYPE (*arg0);
13579 35 : scalar_int_mode mode;
13580 35 : if (!is_a <scalar_int_mode> (TYPE_MODE (type), &mode))
13581 : return code;
13582 70 : if (GET_MODE_BITSIZE (mode) != TYPE_PRECISION (type)
13583 35 : || TYPE_PRECISION (type) <= 1
13584 35 : || TYPE_UNSIGNED (type)
13585 : /* Signed x % c == 0 should have been optimized into unsigned modulo
13586 : earlier. */
13587 35 : || integer_zerop (*arg1)
13588 : /* If c is known to be non-negative, modulo will be expanded as unsigned
13589 : modulo. */
13590 70 : || get_range_pos_neg (treeop0, currently_expanding_gimple_stmt) == 1)
13591 : return code;
13592 :
13593 : /* x % c == d where d < 0 && d <= -c should be always false. */
13594 35 : if (tree_int_cst_sgn (*arg1) == -1
13595 51 : && -wi::to_widest (treeop1) >= wi::to_widest (*arg1))
13596 : return code;
13597 :
13598 35 : int prec = TYPE_PRECISION (type);
13599 35 : wide_int w = wi::to_wide (treeop1) - 1;
13600 35 : w |= wi::shifted_mask (0, prec - 1, true, prec);
13601 35 : tree c3 = wide_int_to_tree (type, w);
13602 35 : tree c4 = *arg1;
13603 35 : if (tree_int_cst_sgn (*arg1) == -1)
13604 16 : c4 = wide_int_to_tree (type, w & wi::to_wide (*arg1));
13605 :
13606 35 : rtx op0 = expand_normal (treeop0);
13607 35 : treeop0 = make_tree (TREE_TYPE (treeop0), op0);
13608 :
13609 35 : bool speed_p = optimize_insn_for_speed_p ();
13610 :
13611 35 : do_pending_stack_adjust ();
13612 :
13613 35 : location_t loc = gimple_location (stmt);
13614 35 : struct separate_ops ops;
13615 35 : ops.code = TRUNC_MOD_EXPR;
13616 35 : ops.location = loc;
13617 35 : ops.type = TREE_TYPE (treeop0);
13618 35 : ops.op0 = treeop0;
13619 35 : ops.op1 = treeop1;
13620 35 : ops.op2 = NULL_TREE;
13621 35 : start_sequence ();
13622 35 : rtx mor = expand_expr_real_2 (&ops, NULL_RTX, TYPE_MODE (ops.type),
13623 : EXPAND_NORMAL);
13624 35 : rtx_insn *moinsns = end_sequence ();
13625 :
13626 35 : unsigned mocost = seq_cost (moinsns, speed_p);
13627 35 : mocost += rtx_cost (mor, mode, EQ, 0, speed_p);
13628 35 : mocost += rtx_cost (expand_normal (*arg1), mode, EQ, 1, speed_p);
13629 :
13630 35 : ops.code = BIT_AND_EXPR;
13631 35 : ops.location = loc;
13632 35 : ops.type = TREE_TYPE (treeop0);
13633 35 : ops.op0 = treeop0;
13634 35 : ops.op1 = c3;
13635 35 : ops.op2 = NULL_TREE;
13636 35 : start_sequence ();
13637 35 : rtx mur = expand_expr_real_2 (&ops, NULL_RTX, TYPE_MODE (ops.type),
13638 : EXPAND_NORMAL);
13639 35 : rtx_insn *muinsns = end_sequence ();
13640 :
13641 35 : unsigned mucost = seq_cost (muinsns, speed_p);
13642 35 : mucost += rtx_cost (mur, mode, EQ, 0, speed_p);
13643 35 : mucost += rtx_cost (expand_normal (c4), mode, EQ, 1, speed_p);
13644 :
13645 35 : if (mocost <= mucost)
13646 : {
13647 0 : emit_insn (moinsns);
13648 0 : *arg0 = make_tree (TREE_TYPE (*arg0), mor);
13649 0 : return code;
13650 : }
13651 :
13652 35 : emit_insn (muinsns);
13653 35 : *arg0 = make_tree (TREE_TYPE (*arg0), mur);
13654 35 : *arg1 = c4;
13655 35 : return code;
13656 35 : }
13657 :
13658 : /* Attempt to optimize unsigned (X % C1) == C2 (or (X % C1) != C2).
13659 : If C1 is odd to:
13660 : (X - C2) * C3 <= C4 (or >), where
13661 : C3 is modular multiplicative inverse of C1 and 1<<prec and
13662 : C4 is ((1<<prec) - 1) / C1 or ((1<<prec) - 1) / C1 - 1 (the latter
13663 : if C2 > ((1<<prec) - 1) % C1).
13664 : If C1 is even, S = ctz (C1) and C2 is 0, use
13665 : ((X * C3) r>> S) <= C4, where C3 is modular multiplicative
13666 : inverse of C1>>S and 1<<prec and C4 is (((1<<prec) - 1) / (C1>>S)) >> S.
13667 :
13668 : For signed (X % C1) == 0 if C1 is odd to (all operations in it
13669 : unsigned):
13670 : (X * C3) + C4 <= 2 * C4, where
13671 : C3 is modular multiplicative inverse of (unsigned) C1 and 1<<prec and
13672 : C4 is ((1<<(prec - 1) - 1) / C1).
13673 : If C1 is even, S = ctz(C1), use
13674 : ((X * C3) + C4) r>> S <= (C4 >> (S - 1))
13675 : where C3 is modular multiplicative inverse of (unsigned)(C1>>S) and 1<<prec
13676 : and C4 is ((1<<(prec - 1) - 1) / (C1>>S)) & (-1<<S).
13677 :
13678 : See the Hacker's Delight book, section 10-17. */
13679 : enum tree_code
13680 3391551 : maybe_optimize_mod_cmp (enum tree_code code, tree *arg0, tree *arg1)
13681 : {
13682 3391551 : gcc_checking_assert (code == EQ_EXPR || code == NE_EXPR);
13683 3391551 : gcc_checking_assert (TREE_CODE (*arg1) == INTEGER_CST);
13684 :
13685 3391551 : if (optimize < 2)
13686 : return code;
13687 :
13688 2577396 : gimple *stmt = get_def_for_expr (*arg0, TRUNC_MOD_EXPR);
13689 2577396 : if (stmt == NULL)
13690 : return code;
13691 :
13692 2115 : tree treeop0 = gimple_assign_rhs1 (stmt);
13693 2115 : tree treeop1 = gimple_assign_rhs2 (stmt);
13694 2115 : if (TREE_CODE (treeop0) != SSA_NAME
13695 2013 : || TREE_CODE (treeop1) != INTEGER_CST
13696 : /* Don't optimize the undefined behavior case x % 0;
13697 : x % 1 should have been optimized into zero, punt if
13698 : it makes it here for whatever reason;
13699 : x % -c should have been optimized into x % c. */
13700 1654 : || compare_tree_int (treeop1, 2) <= 0
13701 : /* Likewise x % c == d where d >= c should be always false. */
13702 3693 : || tree_int_cst_le (treeop1, *arg1))
13703 : return code;
13704 :
13705 : /* Unsigned x % pow2 is handled right already, for signed
13706 : modulo handle it in maybe_optimize_pow2p_mod_cmp. */
13707 1578 : if (integer_pow2p (treeop1))
13708 35 : return maybe_optimize_pow2p_mod_cmp (code, arg0, arg1);
13709 :
13710 1543 : tree type = TREE_TYPE (*arg0);
13711 1543 : scalar_int_mode mode;
13712 1543 : if (!is_a <scalar_int_mode> (TYPE_MODE (type), &mode))
13713 : return code;
13714 3086 : if (GET_MODE_BITSIZE (mode) != TYPE_PRECISION (type)
13715 1543 : || TYPE_PRECISION (type) <= 1)
13716 : return code;
13717 :
13718 1543 : signop sgn = UNSIGNED;
13719 : /* If both operands are known to have the sign bit clear, handle
13720 : even the signed modulo case as unsigned. treeop1 is always
13721 : positive >= 2, checked above. */
13722 1543 : if (!TYPE_UNSIGNED (type)
13723 1543 : && get_range_pos_neg (treeop0, currently_expanding_gimple_stmt) != 1)
13724 : sgn = SIGNED;
13725 :
13726 1543 : if (!TYPE_UNSIGNED (type))
13727 : {
13728 1315 : if (tree_int_cst_sgn (*arg1) == -1)
13729 : return code;
13730 1308 : type = unsigned_type_for (type);
13731 1308 : if (!type || TYPE_MODE (type) != TYPE_MODE (TREE_TYPE (*arg0)))
13732 : return code;
13733 : }
13734 :
13735 1536 : int prec = TYPE_PRECISION (type);
13736 1536 : wide_int w = wi::to_wide (treeop1);
13737 1536 : int shift = wi::ctz (w);
13738 : /* Unsigned (X % C1) == C2 is equivalent to (X - C2) % C1 == 0 if
13739 : C2 <= -1U % C1, because for any Z >= 0U - C2 in that case (Z % C1) != 0.
13740 : If C1 is odd, we can handle all cases by subtracting
13741 : C4 below. We could handle even the even C1 and C2 > -1U % C1 cases
13742 : e.g. by testing for overflow on the subtraction, punt on that for now
13743 : though. */
13744 1536 : if ((sgn == SIGNED || shift) && !integer_zerop (*arg1))
13745 : {
13746 186 : if (sgn == SIGNED)
13747 171 : return code;
13748 26 : wide_int x = wi::umod_trunc (wi::mask (prec, false, prec), w);
13749 26 : if (wi::gtu_p (wi::to_wide (*arg1), x))
13750 11 : return code;
13751 26 : }
13752 :
13753 1365 : imm_use_iterator imm_iter;
13754 1365 : use_operand_p use_p;
13755 7028 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, treeop0)
13756 : {
13757 5673 : gimple *use_stmt = USE_STMT (use_p);
13758 : /* Punt if treeop0 is used in the same bb in a division
13759 : or another modulo with the same divisor. We should expect
13760 : the division and modulo combined together. */
13761 11243 : if (use_stmt == stmt
13762 5673 : || gimple_bb (use_stmt) != gimple_bb (stmt))
13763 5570 : continue;
13764 103 : if (!is_gimple_assign (use_stmt)
13765 103 : || (gimple_assign_rhs_code (use_stmt) != TRUNC_DIV_EXPR
13766 87 : && gimple_assign_rhs_code (use_stmt) != TRUNC_MOD_EXPR))
13767 78 : continue;
13768 25 : if (gimple_assign_rhs1 (use_stmt) != treeop0
13769 25 : || !operand_equal_p (gimple_assign_rhs2 (use_stmt), treeop1, 0))
13770 15 : continue;
13771 10 : return code;
13772 10 : }
13773 :
13774 1355 : w = wi::lrshift (w, shift);
13775 1355 : wide_int a = wide_int::from (w, prec + 1, UNSIGNED);
13776 1355 : wide_int b = wi::shifted_mask (prec, 1, false, prec + 1);
13777 1355 : wide_int m = wide_int::from (wi::mod_inv (a, b), prec, UNSIGNED);
13778 1355 : tree c3 = wide_int_to_tree (type, m);
13779 1355 : tree c5 = NULL_TREE;
13780 1355 : wide_int d, e;
13781 1355 : if (sgn == UNSIGNED)
13782 : {
13783 1018 : d = wi::divmod_trunc (wi::mask (prec, false, prec), w, UNSIGNED, &e);
13784 : /* Use <= floor ((1<<prec) - 1) / C1 only if C2 <= ((1<<prec) - 1) % C1,
13785 : otherwise use < or subtract one from C4. E.g. for
13786 : x % 3U == 0 we transform this into x * 0xaaaaaaab <= 0x55555555, but
13787 : x % 3U == 1 already needs to be
13788 : (x - 1) * 0xaaaaaaabU <= 0x55555554. */
13789 1018 : if (!shift && wi::gtu_p (wi::to_wide (*arg1), e))
13790 22 : d -= 1;
13791 1018 : if (shift)
13792 442 : d = wi::lrshift (d, shift);
13793 : }
13794 : else
13795 : {
13796 337 : e = wi::udiv_trunc (wi::mask (prec - 1, false, prec), w);
13797 337 : if (!shift)
13798 142 : d = wi::lshift (e, 1);
13799 : else
13800 : {
13801 195 : e = wi::bit_and (e, wi::mask (shift, true, prec));
13802 195 : d = wi::lrshift (e, shift - 1);
13803 : }
13804 337 : c5 = wide_int_to_tree (type, e);
13805 : }
13806 1355 : tree c4 = wide_int_to_tree (type, d);
13807 :
13808 1355 : rtx op0 = expand_normal (treeop0);
13809 1355 : treeop0 = make_tree (TREE_TYPE (treeop0), op0);
13810 :
13811 1355 : bool speed_p = optimize_insn_for_speed_p ();
13812 :
13813 1355 : do_pending_stack_adjust ();
13814 :
13815 1355 : location_t loc = gimple_location (stmt);
13816 1355 : struct separate_ops ops;
13817 1355 : ops.code = TRUNC_MOD_EXPR;
13818 1355 : ops.location = loc;
13819 1355 : ops.type = TREE_TYPE (treeop0);
13820 1355 : ops.op0 = treeop0;
13821 1355 : ops.op1 = treeop1;
13822 1355 : ops.op2 = NULL_TREE;
13823 1355 : start_sequence ();
13824 1355 : rtx mor = expand_expr_real_2 (&ops, NULL_RTX, TYPE_MODE (ops.type),
13825 : EXPAND_NORMAL);
13826 1355 : rtx_insn *moinsns = end_sequence ();
13827 :
13828 1355 : unsigned mocost = seq_cost (moinsns, speed_p);
13829 1355 : mocost += rtx_cost (mor, mode, EQ, 0, speed_p);
13830 1355 : mocost += rtx_cost (expand_normal (*arg1), mode, EQ, 1, speed_p);
13831 :
13832 1355 : tree t = fold_convert_loc (loc, type, treeop0);
13833 1355 : if (!integer_zerop (*arg1))
13834 45 : t = fold_build2_loc (loc, MINUS_EXPR, type, t, fold_convert (type, *arg1));
13835 1355 : t = fold_build2_loc (loc, MULT_EXPR, type, t, c3);
13836 1355 : if (sgn == SIGNED)
13837 337 : t = fold_build2_loc (loc, PLUS_EXPR, type, t, c5);
13838 1355 : if (shift)
13839 : {
13840 637 : tree s = build_int_cst (integer_type_node, shift);
13841 637 : t = fold_build2_loc (loc, RROTATE_EXPR, type, t, s);
13842 : }
13843 :
13844 1355 : start_sequence ();
13845 1355 : rtx mur = expand_normal (t);
13846 1355 : rtx_insn *muinsns = end_sequence ();
13847 :
13848 1355 : unsigned mucost = seq_cost (muinsns, speed_p);
13849 1355 : mucost += rtx_cost (mur, mode, LE, 0, speed_p);
13850 1355 : mucost += rtx_cost (expand_normal (c4), mode, LE, 1, speed_p);
13851 :
13852 1355 : if (mocost <= mucost)
13853 : {
13854 226 : emit_insn (moinsns);
13855 226 : *arg0 = make_tree (TREE_TYPE (*arg0), mor);
13856 226 : return code;
13857 : }
13858 :
13859 1129 : emit_insn (muinsns);
13860 1129 : *arg0 = make_tree (type, mur);
13861 1129 : *arg1 = c4;
13862 1129 : return code == EQ_EXPR ? LE_EXPR : GT_EXPR;
13863 2891 : }
13864 :
13865 : /* Optimize x - y < 0 into x < 0 if x - y has undefined overflow. */
13866 :
13867 : void
13868 242813 : maybe_optimize_sub_cmp_0 (enum tree_code code, tree *arg0, tree *arg1)
13869 : {
13870 242813 : gcc_checking_assert (code == GT_EXPR || code == GE_EXPR
13871 : || code == LT_EXPR || code == LE_EXPR);
13872 242813 : gcc_checking_assert (integer_zerop (*arg1));
13873 :
13874 242813 : if (!optimize)
13875 : return;
13876 :
13877 208882 : gimple *stmt = get_def_for_expr (*arg0, MINUS_EXPR);
13878 208882 : if (stmt == NULL)
13879 : return;
13880 :
13881 1361 : tree treeop0 = gimple_assign_rhs1 (stmt);
13882 1361 : tree treeop1 = gimple_assign_rhs2 (stmt);
13883 1361 : if (!TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (treeop0)))
13884 : return;
13885 :
13886 1264 : *arg0 = treeop0;
13887 1264 : *arg1 = treeop1;
13888 : }
13889 :
13890 :
13891 : /* Expand CODE with arguments INNER & (1<<BITNUM) and 0 that represents
13892 : a single bit equality/inequality test, returns where the result is located. */
13893 :
13894 : static rtx
13895 10032 : expand_single_bit_test (location_t loc, enum tree_code code,
13896 : tree inner, int bitnum,
13897 : tree result_type, rtx target,
13898 : machine_mode mode)
13899 : {
13900 10032 : gcc_assert (code == NE_EXPR || code == EQ_EXPR);
13901 :
13902 10032 : tree type = TREE_TYPE (inner);
13903 10032 : scalar_int_mode operand_mode = SCALAR_INT_TYPE_MODE (type);
13904 10032 : int ops_unsigned;
13905 10032 : tree signed_type, unsigned_type, intermediate_type;
13906 10032 : gimple *inner_def;
13907 :
13908 : /* First, see if we can fold the single bit test into a sign-bit
13909 : test. */
13910 10032 : if (bitnum == TYPE_PRECISION (type) - 1
13911 10032 : && type_has_mode_precision_p (type))
13912 : {
13913 241 : tree stype = signed_type_for (type);
13914 444 : tree tmp = fold_build2_loc (loc, code == EQ_EXPR ? GE_EXPR : LT_EXPR,
13915 : result_type,
13916 : fold_convert_loc (loc, stype, inner),
13917 : build_int_cst (stype, 0));
13918 241 : return expand_expr (tmp, target, VOIDmode, EXPAND_NORMAL);
13919 : }
13920 :
13921 : /* Otherwise we have (A & C) != 0 where C is a single bit,
13922 : convert that into ((A >> C2) & 1). Where C2 = log2(C).
13923 : Similarly for (A & C) == 0. */
13924 :
13925 : /* If INNER is a right shift of a constant and it plus BITNUM does
13926 : not overflow, adjust BITNUM and INNER. */
13927 9791 : if ((inner_def = get_def_for_expr (inner, RSHIFT_EXPR))
13928 59 : && TREE_CODE (gimple_assign_rhs2 (inner_def)) == INTEGER_CST
13929 16 : && bitnum < TYPE_PRECISION (type)
13930 9807 : && wi::ltu_p (wi::to_wide (gimple_assign_rhs2 (inner_def)),
13931 9791 : TYPE_PRECISION (type) - bitnum))
13932 : {
13933 16 : bitnum += tree_to_uhwi (gimple_assign_rhs2 (inner_def));
13934 16 : inner = gimple_assign_rhs1 (inner_def);
13935 : }
13936 :
13937 : /* If we are going to be able to omit the AND below, we must do our
13938 : operations as unsigned. If we must use the AND, we have a choice.
13939 : Normally unsigned is faster, but for some machines signed is. */
13940 9791 : ops_unsigned = (load_extend_op (operand_mode) == SIGN_EXTEND
13941 : && !flag_syntax_only) ? 0 : 1;
13942 :
13943 9791 : signed_type = lang_hooks.types.type_for_mode (operand_mode, 0);
13944 9791 : unsigned_type = lang_hooks.types.type_for_mode (operand_mode, 1);
13945 9791 : intermediate_type = ops_unsigned ? unsigned_type : signed_type;
13946 9791 : inner = fold_convert_loc (loc, intermediate_type, inner);
13947 :
13948 9791 : rtx inner0 = expand_expr (inner, NULL_RTX, VOIDmode, EXPAND_NORMAL);
13949 :
13950 9791 : if (CONST_SCALAR_INT_P (inner0))
13951 : {
13952 0 : wide_int t = rtx_mode_t (inner0, operand_mode);
13953 0 : bool setp = (wi::lrshift (t, bitnum) & 1) != 0;
13954 0 : return (setp ^ (code == EQ_EXPR)) ? const1_rtx : const0_rtx;
13955 0 : }
13956 9791 : int bitpos = bitnum;
13957 :
13958 9791 : if (BYTES_BIG_ENDIAN)
13959 : bitpos = GET_MODE_BITSIZE (operand_mode) - 1 - bitpos;
13960 :
13961 9791 : inner0 = extract_bit_field (inner0, 1, bitpos, 1, target,
13962 : operand_mode, mode, 0, NULL);
13963 :
13964 9791 : if (code == EQ_EXPR)
13965 2680 : inner0 = expand_binop (GET_MODE (inner0), xor_optab, inner0, const1_rtx,
13966 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
13967 9791 : if (GET_MODE (inner0) != mode)
13968 : {
13969 0 : rtx t = gen_reg_rtx (mode);
13970 0 : convert_move (t, inner0, 0);
13971 0 : return t;
13972 : }
13973 : return inner0;
13974 : }
13975 :
13976 : /* Generate code to calculate OPS, and exploded expression
13977 : using a store-flag instruction and return an rtx for the result.
13978 : OPS reflects a comparison.
13979 :
13980 : If TARGET is nonzero, store the result there if convenient.
13981 :
13982 : Return zero if there is no suitable set-flag instruction
13983 : available on this machine.
13984 :
13985 : Once expand_expr has been called on the arguments of the comparison,
13986 : we are committed to doing the store flag, since it is not safe to
13987 : re-evaluate the expression. We emit the store-flag insn by calling
13988 : emit_store_flag, but only expand the arguments if we have a reason
13989 : to believe that emit_store_flag will be successful. If we think that
13990 : it will, but it isn't, we have to simulate the store-flag with a
13991 : set/jump/set sequence. */
13992 :
13993 : static rtx
13994 632055 : do_store_flag (const_sepops ops, rtx target, machine_mode mode)
13995 : {
13996 632055 : enum rtx_code code;
13997 632055 : tree arg0, arg1, type;
13998 632055 : machine_mode operand_mode;
13999 632055 : int unsignedp;
14000 632055 : rtx op0, op1;
14001 632055 : rtx subtarget = target;
14002 632055 : location_t loc = ops->location;
14003 632055 : unsigned HOST_WIDE_INT nunits;
14004 :
14005 632055 : arg0 = ops->op0;
14006 632055 : arg1 = ops->op1;
14007 :
14008 : /* Don't crash if the comparison was erroneous. */
14009 632055 : if (arg0 == error_mark_node || arg1 == error_mark_node)
14010 0 : return const0_rtx;
14011 :
14012 632055 : type = TREE_TYPE (arg0);
14013 632055 : operand_mode = TYPE_MODE (type);
14014 632055 : unsignedp = TYPE_UNSIGNED (type);
14015 :
14016 : /* We won't bother with BLKmode store-flag operations because it would mean
14017 : passing a lot of information to emit_store_flag. */
14018 632055 : if (operand_mode == BLKmode)
14019 : return 0;
14020 :
14021 : /* We won't bother with store-flag operations involving function pointers
14022 : when function pointers must be canonicalized before comparisons. */
14023 632055 : if (targetm.have_canonicalize_funcptr_for_compare ()
14024 632055 : && ((POINTER_TYPE_P (TREE_TYPE (arg0))
14025 0 : && FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (arg0))))
14026 0 : || (POINTER_TYPE_P (TREE_TYPE (arg1))
14027 0 : && FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (arg1))))))
14028 : return 0;
14029 :
14030 632055 : STRIP_NOPS (arg0);
14031 632055 : STRIP_NOPS (arg1);
14032 :
14033 : /* For vector typed comparisons emit code to generate the desired
14034 : all-ones or all-zeros mask. */
14035 632055 : if (VECTOR_TYPE_P (ops->type))
14036 : {
14037 26282 : tree ifexp = build2 (ops->code, ops->type, arg0, arg1);
14038 26282 : if (VECTOR_BOOLEAN_TYPE_P (ops->type)
14039 52564 : && expand_vec_cmp_expr_p (TREE_TYPE (arg0), ops->type, ops->code))
14040 26282 : return expand_vec_cmp_expr (ops->type, ifexp, target);
14041 : else
14042 0 : gcc_unreachable ();
14043 : }
14044 :
14045 : /* Optimize (x % C1) == C2 or (x % C1) != C2 if it is beneficial
14046 : into (x - C2) * C3 < C4. */
14047 605773 : if ((ops->code == EQ_EXPR || ops->code == NE_EXPR)
14048 415437 : && TREE_CODE (arg0) == SSA_NAME
14049 414862 : && TREE_CODE (arg1) == INTEGER_CST)
14050 : {
14051 248138 : enum tree_code new_code = maybe_optimize_mod_cmp (ops->code,
14052 : &arg0, &arg1);
14053 248138 : if (new_code != ops->code)
14054 : {
14055 110 : struct separate_ops nops = *ops;
14056 110 : nops.code = new_code;
14057 110 : nops.op0 = arg0;
14058 110 : nops.op1 = arg1;
14059 110 : nops.type = TREE_TYPE (arg0);
14060 110 : return do_store_flag (&nops, target, mode);
14061 : }
14062 : }
14063 :
14064 : /* Optimize (x - y) < 0 into x < y if x - y has undefined overflow. */
14065 605663 : if (!unsignedp
14066 432328 : && (ops->code == LT_EXPR || ops->code == LE_EXPR
14067 432328 : || ops->code == GT_EXPR || ops->code == GE_EXPR)
14068 116686 : && integer_zerop (arg1)
14069 644424 : && TREE_CODE (arg0) == SSA_NAME)
14070 38757 : maybe_optimize_sub_cmp_0 (ops->code, &arg0, &arg1);
14071 :
14072 : /* Get the rtx comparison code to use. We know that EXP is a comparison
14073 : operation of some type. Some comparisons against 1 and -1 can be
14074 : converted to comparisons with zero. Do so here so that the tests
14075 : below will be aware that we have a comparison with zero. These
14076 : tests will not catch constants in the first operand, but constants
14077 : are rarely passed as the first operand. */
14078 :
14079 605663 : switch (ops->code)
14080 : {
14081 : case EQ_EXPR:
14082 : code = EQ;
14083 : break;
14084 237084 : case NE_EXPR:
14085 237084 : code = NE;
14086 237084 : break;
14087 33567 : case LT_EXPR:
14088 33567 : if (integer_onep (arg1))
14089 0 : arg1 = integer_zero_node, code = unsignedp ? LEU : LE;
14090 : else
14091 33567 : code = unsignedp ? LTU : LT;
14092 : break;
14093 33548 : case LE_EXPR:
14094 33548 : if (! unsignedp && integer_all_onesp (arg1))
14095 0 : arg1 = integer_zero_node, code = LT;
14096 : else
14097 33548 : code = unsignedp ? LEU : LE;
14098 : break;
14099 50994 : case GT_EXPR:
14100 50994 : if (! unsignedp && integer_all_onesp (arg1))
14101 0 : arg1 = integer_zero_node, code = GE;
14102 : else
14103 50994 : code = unsignedp ? GTU : GT;
14104 : break;
14105 47624 : case GE_EXPR:
14106 47624 : if (integer_onep (arg1))
14107 0 : arg1 = integer_zero_node, code = unsignedp ? GTU : GT;
14108 : else
14109 47624 : code = unsignedp ? GEU : GE;
14110 : break;
14111 :
14112 1032 : case UNORDERED_EXPR:
14113 1032 : code = UNORDERED;
14114 1032 : break;
14115 722 : case ORDERED_EXPR:
14116 722 : code = ORDERED;
14117 722 : break;
14118 536 : case UNLT_EXPR:
14119 536 : code = UNLT;
14120 536 : break;
14121 11137 : case UNLE_EXPR:
14122 11137 : code = UNLE;
14123 11137 : break;
14124 1009 : case UNGT_EXPR:
14125 1009 : code = UNGT;
14126 1009 : break;
14127 9942 : case UNGE_EXPR:
14128 9942 : code = UNGE;
14129 9942 : break;
14130 137 : case UNEQ_EXPR:
14131 137 : code = UNEQ;
14132 137 : break;
14133 88 : case LTGT_EXPR:
14134 88 : code = LTGT;
14135 88 : break;
14136 :
14137 0 : default:
14138 0 : gcc_unreachable ();
14139 : }
14140 :
14141 : /* Put a constant second. */
14142 605663 : if (TREE_CODE (arg0) == REAL_CST || TREE_CODE (arg0) == INTEGER_CST
14143 603698 : || TREE_CODE (arg0) == FIXED_CST)
14144 : {
14145 1965 : std::swap (arg0, arg1);
14146 1965 : code = swap_condition (code);
14147 : }
14148 :
14149 : /* If this is an equality or inequality test of a single bit, we can
14150 : do this by shifting the bit being tested to the low-order bit and
14151 : masking the result with the constant 1. If the condition was EQ,
14152 : we xor it with 1. This does not require an scc insn and is faster
14153 : than an scc insn even if we have it. */
14154 :
14155 605663 : if ((code == NE || code == EQ)
14156 415327 : && (integer_zerop (arg1)
14157 260839 : || integer_pow2p (arg1))
14158 : /* vector types are not handled here. */
14159 187717 : && TREE_CODE (TREE_TYPE (arg1)) != VECTOR_TYPE
14160 793349 : && (TYPE_PRECISION (ops->type) != 1 || TYPE_UNSIGNED (ops->type)))
14161 : {
14162 187686 : tree narg0 = arg0;
14163 187686 : wide_int nz = tree_nonzero_bits (narg0);
14164 187686 : gimple *srcstmt = get_def_for_expr (narg0, BIT_AND_EXPR);
14165 : /* If the defining statement was (x & POW2), then use that instead of
14166 : the non-zero bits. */
14167 187686 : if (srcstmt && integer_pow2p (gimple_assign_rhs2 (srcstmt)))
14168 : {
14169 5678 : nz = wi::to_wide (gimple_assign_rhs2 (srcstmt));
14170 5678 : narg0 = gimple_assign_rhs1 (srcstmt);
14171 : }
14172 :
14173 187686 : if (wi::popcount (nz) == 1
14174 187686 : && (integer_zerop (arg1)
14175 187686 : || wi::to_wide (arg1) == nz))
14176 : {
14177 10032 : int bitnum = wi::exact_log2 (nz);
14178 10032 : enum tree_code tcode = EQ_EXPR;
14179 10032 : if ((code == NE) ^ !integer_zerop (arg1))
14180 7314 : tcode = NE_EXPR;
14181 :
14182 10032 : type = lang_hooks.types.type_for_mode (mode, unsignedp);
14183 10032 : return expand_single_bit_test (loc, tcode,
14184 : narg0,
14185 : bitnum, type, target, mode);
14186 : }
14187 187686 : }
14188 :
14189 :
14190 595631 : if (! get_subtarget (target)
14191 595631 : || GET_MODE (subtarget) != operand_mode)
14192 : subtarget = 0;
14193 :
14194 595631 : expand_operands (arg0, arg1, subtarget, &op0, &op1, EXPAND_NORMAL);
14195 :
14196 : /* For boolean vectors with less than mode precision
14197 : make sure to fill padding with consistent values. */
14198 31 : if (VECTOR_BOOLEAN_TYPE_P (type)
14199 27 : && SCALAR_INT_MODE_P (operand_mode)
14200 595631 : && TYPE_VECTOR_SUBPARTS (type).is_constant (&nunits)
14201 595631 : && maybe_ne (GET_MODE_PRECISION (operand_mode), nunits))
14202 : {
14203 0 : gcc_assert (code == EQ || code == NE);
14204 0 : op0 = expand_binop (mode, and_optab, op0,
14205 0 : GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
14206 : NULL_RTX, true, OPTAB_WIDEN);
14207 0 : op1 = expand_binop (mode, and_optab, op1,
14208 : GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
14209 : NULL_RTX, true, OPTAB_WIDEN);
14210 : }
14211 :
14212 595631 : if (target == 0)
14213 406481 : target = gen_reg_rtx (mode);
14214 :
14215 : /* Try a cstore if possible. */
14216 595631 : return emit_store_flag_force (target, code, op0, op1,
14217 : operand_mode, unsignedp,
14218 595631 : (TYPE_PRECISION (ops->type) == 1
14219 1190587 : && !TYPE_UNSIGNED (ops->type)) ? -1 : 1);
14220 : }
14221 :
14222 : /* Attempt to generate a casesi instruction. Returns true if successful,
14223 : false otherwise (i.e. if there is no casesi instruction).
14224 :
14225 : DEFAULT_PROBABILITY is the probability of jumping to the default
14226 : label. */
14227 : bool
14228 7461 : try_casesi (tree index_type, tree index_expr, tree minval, tree range,
14229 : rtx table_label, rtx default_label, rtx fallback_label,
14230 : profile_probability default_probability)
14231 : {
14232 7461 : class expand_operand ops[5];
14233 7461 : scalar_int_mode index_mode = SImode;
14234 7461 : rtx op1, op2, index;
14235 :
14236 7461 : if (! targetm.have_casesi ())
14237 : return false;
14238 :
14239 : /* The index must be some form of integer. Convert it to SImode. */
14240 0 : scalar_int_mode omode = SCALAR_INT_TYPE_MODE (index_type);
14241 0 : if (GET_MODE_BITSIZE (omode) > GET_MODE_BITSIZE (index_mode))
14242 : {
14243 0 : rtx rangertx = expand_normal (range);
14244 :
14245 : /* We must handle the endpoints in the original mode. */
14246 0 : index_expr = build2 (MINUS_EXPR, index_type,
14247 : index_expr, minval);
14248 0 : minval = integer_zero_node;
14249 0 : index = expand_normal (index_expr);
14250 0 : if (default_label)
14251 0 : emit_cmp_and_jump_insns (rangertx, index, LTU, NULL_RTX,
14252 : omode, 1, default_label,
14253 : default_probability);
14254 : /* Now we can safely truncate. */
14255 0 : index = convert_to_mode (index_mode, index, 0);
14256 : }
14257 : else
14258 : {
14259 0 : if (omode != index_mode)
14260 : {
14261 0 : index_type = lang_hooks.types.type_for_mode (index_mode, 0);
14262 0 : index_expr = fold_convert (index_type, index_expr);
14263 : }
14264 :
14265 0 : index = expand_normal (index_expr);
14266 : }
14267 :
14268 0 : do_pending_stack_adjust ();
14269 :
14270 0 : op1 = expand_normal (minval);
14271 0 : op2 = expand_normal (range);
14272 :
14273 0 : create_input_operand (&ops[0], index, index_mode);
14274 0 : create_convert_operand_from_type (&ops[1], op1, TREE_TYPE (minval));
14275 0 : create_convert_operand_from_type (&ops[2], op2, TREE_TYPE (range));
14276 0 : create_fixed_operand (&ops[3], table_label);
14277 0 : create_fixed_operand (&ops[4], (default_label
14278 : ? default_label
14279 : : fallback_label));
14280 0 : expand_jump_insn (targetm.code_for_casesi, 5, ops);
14281 0 : return true;
14282 : }
14283 :
14284 : /* Attempt to generate a tablejump instruction; same concept. */
14285 : /* Subroutine of the next function.
14286 :
14287 : INDEX is the value being switched on, with the lowest value
14288 : in the table already subtracted.
14289 : MODE is its expected mode (needed if INDEX is constant).
14290 : RANGE is the length of the jump table.
14291 : TABLE_LABEL is a CODE_LABEL rtx for the table itself.
14292 :
14293 : DEFAULT_LABEL is a CODE_LABEL rtx to jump to if the
14294 : index value is out of range.
14295 : DEFAULT_PROBABILITY is the probability of jumping to
14296 : the default label. */
14297 :
14298 : static void
14299 7461 : do_tablejump (rtx index, machine_mode mode, rtx range, rtx table_label,
14300 : rtx default_label, profile_probability default_probability)
14301 : {
14302 7461 : rtx temp, vector;
14303 :
14304 7461 : if (INTVAL (range) > cfun->cfg->max_jumptable_ents)
14305 6194 : cfun->cfg->max_jumptable_ents = INTVAL (range);
14306 :
14307 : /* Do an unsigned comparison (in the proper mode) between the index
14308 : expression and the value which represents the length of the range.
14309 : Since we just finished subtracting the lower bound of the range
14310 : from the index expression, this comparison allows us to simultaneously
14311 : check that the original index expression value is both greater than
14312 : or equal to the minimum value of the range and less than or equal to
14313 : the maximum value of the range. */
14314 :
14315 7461 : if (default_label)
14316 7415 : emit_cmp_and_jump_insns (index, range, GTU, NULL_RTX, mode, 1,
14317 : default_label, default_probability);
14318 :
14319 : /* If index is in range, it must fit in Pmode.
14320 : Convert to Pmode so we can index with it. */
14321 8530 : if (mode != Pmode)
14322 : {
14323 6402 : unsigned int width;
14324 :
14325 : /* We know the value of INDEX is between 0 and RANGE. If we have a
14326 : sign-extended subreg, and RANGE does not have the sign bit set, then
14327 : we have a value that is valid for both sign and zero extension. In
14328 : this case, we get better code if we sign extend. */
14329 6402 : if (GET_CODE (index) == SUBREG
14330 9 : && SUBREG_PROMOTED_VAR_P (index)
14331 0 : && SUBREG_PROMOTED_SIGNED_P (index)
14332 0 : && ((width = GET_MODE_PRECISION (as_a <scalar_int_mode> (mode)))
14333 : <= HOST_BITS_PER_WIDE_INT)
14334 6402 : && ! (UINTVAL (range) & (HOST_WIDE_INT_1U << (width - 1))))
14335 0 : index = convert_to_mode (Pmode, index, 0);
14336 : else
14337 6402 : index = convert_to_mode (Pmode, index, 1);
14338 : }
14339 :
14340 : /* Don't let a MEM slip through, because then INDEX that comes
14341 : out of PIC_CASE_VECTOR_ADDRESS won't be a valid address,
14342 : and break_out_memory_refs will go to work on it and mess it up. */
14343 : #ifdef PIC_CASE_VECTOR_ADDRESS
14344 : if (flag_pic && !REG_P (index))
14345 : index = copy_to_mode_reg (Pmode, index);
14346 : #endif
14347 :
14348 : /* ??? The only correct use of CASE_VECTOR_MODE is the one inside the
14349 : GET_MODE_SIZE, because this indicates how large insns are. The other
14350 : uses should all be Pmode, because they are addresses. This code
14351 : could fail if addresses and insns are not the same size. */
14352 8530 : index = simplify_gen_binary (MULT, Pmode, index,
14353 14922 : gen_int_mode (GET_MODE_SIZE (CASE_VECTOR_MODE),
14354 7461 : Pmode));
14355 8530 : index = simplify_gen_binary (PLUS, Pmode, index,
14356 7461 : gen_rtx_LABEL_REF (Pmode, table_label));
14357 :
14358 : #ifdef PIC_CASE_VECTOR_ADDRESS
14359 : if (flag_pic)
14360 : index = PIC_CASE_VECTOR_ADDRESS (index);
14361 : else
14362 : #endif
14363 9142 : index = memory_address (CASE_VECTOR_MODE, index);
14364 9142 : temp = gen_reg_rtx (CASE_VECTOR_MODE);
14365 9142 : vector = gen_const_mem (CASE_VECTOR_MODE, index);
14366 7461 : convert_move (temp, vector, 0);
14367 :
14368 7461 : emit_jump_insn (targetm.gen_tablejump (temp, table_label));
14369 :
14370 : /* If we are generating PIC code or if the table is PC-relative, the
14371 : table and JUMP_INSN must be adjacent, so don't output a BARRIER. */
14372 7461 : if (! CASE_VECTOR_PC_RELATIVE && ! flag_pic)
14373 6283 : emit_barrier ();
14374 7461 : }
14375 :
14376 : bool
14377 7461 : try_tablejump (tree index_type, tree index_expr, tree minval, tree range,
14378 : rtx table_label, rtx default_label,
14379 : profile_probability default_probability)
14380 : {
14381 7461 : rtx index;
14382 :
14383 7461 : if (! targetm.have_tablejump ())
14384 : return false;
14385 :
14386 7461 : index_expr = fold_build2 (MINUS_EXPR, index_type,
14387 : fold_convert (index_type, index_expr),
14388 : fold_convert (index_type, minval));
14389 7461 : index = expand_normal (index_expr);
14390 7461 : do_pending_stack_adjust ();
14391 :
14392 7461 : do_tablejump (index, TYPE_MODE (index_type),
14393 7461 : convert_modes (TYPE_MODE (index_type),
14394 7461 : TYPE_MODE (TREE_TYPE (range)),
14395 : expand_normal (range),
14396 7461 : TYPE_UNSIGNED (TREE_TYPE (range))),
14397 : table_label, default_label, default_probability);
14398 7461 : return true;
14399 : }
14400 :
14401 : /* Return a CONST_VECTOR rtx representing vector mask for
14402 : a VECTOR_CST of booleans. */
14403 : static rtx
14404 684 : const_vector_mask_from_tree (tree exp)
14405 : {
14406 684 : machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
14407 684 : machine_mode inner = GET_MODE_INNER (mode);
14408 :
14409 684 : rtx_vector_builder builder (mode, VECTOR_CST_NPATTERNS (exp),
14410 684 : VECTOR_CST_NELTS_PER_PATTERN (exp));
14411 684 : unsigned int count = builder.encoded_nelts ();
14412 3680 : for (unsigned int i = 0; i < count; ++i)
14413 : {
14414 2996 : tree elt = VECTOR_CST_ELT (exp, i);
14415 2996 : gcc_assert (TREE_CODE (elt) == INTEGER_CST);
14416 2996 : if (integer_zerop (elt))
14417 1177 : builder.quick_push (CONST0_RTX (inner));
14418 1819 : else if (integer_onep (elt)
14419 1819 : || integer_minus_onep (elt))
14420 1819 : builder.quick_push (CONSTM1_RTX (inner));
14421 : else
14422 0 : gcc_unreachable ();
14423 : }
14424 684 : return builder.build ();
14425 684 : }
14426 :
14427 : /* Return a CONST_VECTOR rtx for a VECTOR_CST tree. */
14428 : static rtx
14429 576167 : const_vector_from_tree (tree exp)
14430 : {
14431 576167 : machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
14432 :
14433 576167 : if (initializer_zerop (exp))
14434 168690 : return CONST0_RTX (mode);
14435 :
14436 407477 : if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (exp)))
14437 684 : return const_vector_mask_from_tree (exp);
14438 :
14439 406793 : machine_mode inner = GET_MODE_INNER (mode);
14440 :
14441 406793 : rtx_vector_builder builder (mode, VECTOR_CST_NPATTERNS (exp),
14442 406793 : VECTOR_CST_NELTS_PER_PATTERN (exp));
14443 406793 : unsigned int count = builder.encoded_nelts ();
14444 1306689 : for (unsigned int i = 0; i < count; ++i)
14445 : {
14446 899896 : tree elt = VECTOR_CST_ELT (exp, i);
14447 899896 : if (TREE_CODE (elt) == REAL_CST)
14448 136610 : builder.quick_push (const_double_from_real_value (TREE_REAL_CST (elt),
14449 : inner));
14450 763286 : else if (TREE_CODE (elt) == FIXED_CST)
14451 0 : builder.quick_push (CONST_FIXED_FROM_FIXED_VALUE (TREE_FIXED_CST (elt),
14452 : inner));
14453 : else
14454 763286 : builder.quick_push (immed_wide_int_const (wi::to_poly_wide (elt),
14455 : inner));
14456 : }
14457 406793 : return builder.build ();
14458 406793 : }
14459 :
14460 : /* Build a decl for a personality function given a language prefix. */
14461 :
14462 : tree
14463 33354 : build_personality_function (const char *lang)
14464 : {
14465 33354 : const char *unwind_and_version;
14466 33354 : tree decl, type;
14467 33354 : char *name;
14468 :
14469 33354 : switch (targetm_common.except_unwind_info (&global_options))
14470 : {
14471 : case UI_NONE:
14472 : return NULL;
14473 : case UI_SJLJ:
14474 : unwind_and_version = "_sj0";
14475 : break;
14476 33354 : case UI_DWARF2:
14477 33354 : case UI_TARGET:
14478 33354 : unwind_and_version = "_v0";
14479 33354 : break;
14480 0 : case UI_SEH:
14481 0 : unwind_and_version = "_seh0";
14482 0 : break;
14483 0 : default:
14484 0 : gcc_unreachable ();
14485 : }
14486 :
14487 33354 : name = ACONCAT (("__", lang, "_personality", unwind_and_version, NULL));
14488 :
14489 33354 : type = build_function_type_list (unsigned_type_node,
14490 : integer_type_node, integer_type_node,
14491 : long_long_unsigned_type_node,
14492 : ptr_type_node, ptr_type_node, NULL_TREE);
14493 33354 : decl = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL,
14494 : get_identifier (name), type);
14495 33354 : DECL_ARTIFICIAL (decl) = 1;
14496 33354 : DECL_EXTERNAL (decl) = 1;
14497 33354 : TREE_PUBLIC (decl) = 1;
14498 :
14499 : /* Zap the nonsensical SYMBOL_REF_DECL for this. What we're left with
14500 : are the flags assigned by targetm.encode_section_info. */
14501 33354 : SET_SYMBOL_REF_DECL (XEXP (DECL_RTL (decl), 0), NULL);
14502 :
14503 33354 : return decl;
14504 : }
14505 :
14506 : /* Extracts the personality function of DECL and returns the corresponding
14507 : libfunc. */
14508 :
14509 : rtx
14510 1669117 : get_personality_function (tree decl)
14511 : {
14512 1669117 : tree personality = DECL_FUNCTION_PERSONALITY (decl);
14513 1669117 : enum eh_personality_kind pk;
14514 :
14515 1669117 : pk = function_needs_eh_personality (DECL_STRUCT_FUNCTION (decl));
14516 1669117 : if (pk == eh_personality_none)
14517 : return NULL;
14518 :
14519 162374 : if (!personality
14520 162374 : && pk == eh_personality_any)
14521 74526 : personality = lang_hooks.eh_personality ();
14522 :
14523 162374 : if (pk == eh_personality_lang)
14524 87848 : gcc_assert (personality != NULL_TREE);
14525 :
14526 162374 : return XEXP (DECL_RTL (personality), 0);
14527 : }
14528 :
14529 : /* Returns a tree for the size of EXP in bytes. */
14530 :
14531 : static tree
14532 15013102 : tree_expr_size (const_tree exp)
14533 : {
14534 15013102 : if (DECL_P (exp)
14535 15013102 : && DECL_SIZE_UNIT (exp) != 0)
14536 1950207 : return DECL_SIZE_UNIT (exp);
14537 : else
14538 13062895 : return size_in_bytes (TREE_TYPE (exp));
14539 : }
14540 :
14541 : /* Return an rtx for the size in bytes of the value of EXP. */
14542 :
14543 : rtx
14544 14815986 : expr_size (tree exp)
14545 : {
14546 14815986 : tree size;
14547 :
14548 14815986 : if (TREE_CODE (exp) == WITH_SIZE_EXPR)
14549 773 : size = TREE_OPERAND (exp, 1);
14550 : else
14551 : {
14552 14815213 : size = tree_expr_size (exp);
14553 14815213 : gcc_assert (size);
14554 14815213 : gcc_assert (size == SUBSTITUTE_PLACEHOLDER_IN_EXPR (size, exp));
14555 : }
14556 :
14557 14815986 : return expand_expr (size, NULL_RTX, TYPE_MODE (sizetype), EXPAND_NORMAL);
14558 : }
14559 :
14560 : /* Return a wide integer for the size in bytes of the value of EXP, or -1
14561 : if the size can vary or is larger than an integer. */
14562 :
14563 : HOST_WIDE_INT
14564 197889 : int_expr_size (const_tree exp)
14565 : {
14566 197889 : tree size;
14567 :
14568 197889 : if (TREE_CODE (exp) == WITH_SIZE_EXPR)
14569 0 : size = TREE_OPERAND (exp, 1);
14570 : else
14571 : {
14572 197889 : size = tree_expr_size (exp);
14573 197889 : gcc_assert (size);
14574 : }
14575 :
14576 197889 : if (size == 0 || !tree_fits_shwi_p (size))
14577 : return -1;
14578 :
14579 197889 : return tree_to_shwi (size);
14580 : }
14581 :
14582 : /* Return the quotient of polynomial long division of x^2N by POLYNOMIAL
14583 : in GF (2^N).
14584 : Author: Richard Sandiford <richard.sandiford@arm.com> */
14585 :
14586 : unsigned HOST_WIDE_INT
14587 0 : gf2n_poly_long_div_quotient (unsigned HOST_WIDE_INT polynomial,
14588 : unsigned short n)
14589 : {
14590 : /* The result has degree N, so needs N + 1 bits. */
14591 0 : gcc_assert (n < 64);
14592 :
14593 : /* Perform a division step for the x^2N coefficient. At this point the
14594 : quotient and remainder have N implicit trailing zeros. */
14595 : unsigned HOST_WIDE_INT quotient = 1;
14596 : unsigned HOST_WIDE_INT remainder = polynomial;
14597 :
14598 : /* Process the coefficients for x^(2N-1) down to x^N, with each step
14599 : reducing the number of implicit trailing zeros by one. */
14600 0 : for (unsigned int i = 0; i < n; ++i)
14601 : {
14602 0 : bool coeff = remainder & (HOST_WIDE_INT_1U << (n - 1));
14603 0 : quotient = (quotient << 1) | coeff;
14604 0 : remainder = (remainder << 1) ^ (coeff ? polynomial : 0);
14605 : }
14606 0 : return quotient;
14607 : }
14608 :
14609 : /* Assemble CRC table with 256 elements for the given POLYNOM and CRC_BITS.
14610 : POLYNOM is the polynomial used to calculate the CRC table's elements.
14611 : CRC_BITS is the size of CRC, may be 8, 16, ... . */
14612 :
14613 : static rtx
14614 144 : assemble_crc_table (unsigned HOST_WIDE_INT polynom, unsigned short crc_bits)
14615 : {
14616 144 : unsigned table_el_n = 0x100;
14617 144 : tree ar = build_array_type (make_unsigned_type (crc_bits),
14618 144 : build_index_type (size_int (table_el_n - 1)));
14619 :
14620 : /* Initialize the table. */
14621 144 : vec<tree, va_gc> *initial_values;
14622 144 : vec_alloc (initial_values, table_el_n);
14623 37152 : for (size_t i = 0; i < table_el_n; ++i)
14624 : {
14625 36864 : unsigned HOST_WIDE_INT crc = calculate_crc (0, i, polynom, crc_bits, 8);
14626 36864 : tree element = build_int_cstu (make_unsigned_type (crc_bits), crc);
14627 36864 : vec_safe_push (initial_values, element);
14628 : }
14629 144 : tree ctor = build_constructor_from_vec (ar, initial_values);
14630 144 : rtx mem = output_constant_def (ctor, 1);
14631 144 : gcc_assert (MEM_P (mem));
14632 144 : if (dump_file && (dump_flags & TDF_DETAILS))
14633 : {
14634 16 : fprintf (dump_file,
14635 : ";; emitting crc table crc_%u_polynomial_"
14636 : HOST_WIDE_INT_PRINT_HEX " ",
14637 : crc_bits, polynom);
14638 16 : print_rtl_single (dump_file, XEXP (mem, 0));
14639 16 : fprintf (dump_file, "\n");
14640 : }
14641 :
14642 144 : return XEXP (mem, 0);
14643 : }
14644 :
14645 : /* Generate CRC lookup table by calculating CRC for all possible
14646 : 8-bit data values. The table is stored with a specific name in the read-only
14647 : static data section.
14648 : POLYNOM is the polynomial used to calculate the CRC table's elements.
14649 : CRC_BITS is the size of CRC, may be 8, 16, ... . */
14650 :
14651 : static rtx
14652 144 : generate_crc_table (unsigned HOST_WIDE_INT polynom, unsigned short crc_bits)
14653 : {
14654 144 : gcc_assert (crc_bits <= 64);
14655 :
14656 144 : return assemble_crc_table (polynom, crc_bits);
14657 : }
14658 :
14659 : /* Assemble CRC table with 256 elements for the given POLYNOM and CRC_BITS.
14660 : POLYNOM is the polynomial used to calculate the CRC table's elements.
14661 : CRC_BITS is the size of CRC, may be 8, 16, ... . */
14662 :
14663 : static rtx
14664 136 : assemble_reversed_crc_table (unsigned HOST_WIDE_INT polynom, unsigned short crc_bits)
14665 : {
14666 136 : unsigned table_el_n = 0x100;
14667 136 : tree ar = build_array_type (make_unsigned_type (crc_bits),
14668 136 : build_index_type (size_int (table_el_n - 1)));
14669 :
14670 : /* Initialize the table. */
14671 136 : vec<tree, va_gc> *initial_values;
14672 136 : vec_alloc (initial_values, table_el_n);
14673 35088 : for (size_t i = 0; i < table_el_n; ++i)
14674 : {
14675 34816 : unsigned HOST_WIDE_INT crc = calculate_reversed_crc (0, i, polynom,
14676 34816 : crc_bits, 8);
14677 34816 : tree element = build_int_cstu (make_unsigned_type (crc_bits), crc);
14678 34816 : vec_safe_push (initial_values, element);
14679 : }
14680 136 : tree ctor = build_constructor_from_vec (ar, initial_values);
14681 136 : rtx mem = output_constant_def (ctor, 1);
14682 136 : gcc_assert (MEM_P (mem));
14683 136 : if (dump_file && (dump_flags & TDF_DETAILS))
14684 : {
14685 16 : fprintf (dump_file,
14686 : ";; emitting reversed crc table crc_%u_polynomial_"
14687 : HOST_WIDE_INT_PRINT_HEX " ",
14688 : crc_bits, polynom);
14689 16 : print_rtl_single (dump_file, XEXP (mem, 0));
14690 16 : fprintf (dump_file, "\n");
14691 : }
14692 :
14693 136 : return XEXP (mem, 0);
14694 : }
14695 :
14696 : /* Generate reversed CRC table for the given POLYNOM and CRC_BITS. */
14697 :
14698 : static rtx
14699 136 : generate_reversed_crc_table (unsigned HOST_WIDE_INT polynom,
14700 : unsigned short crc_bits)
14701 : {
14702 136 : gcc_assert (crc_bits <= 64);
14703 :
14704 136 : return assemble_reversed_crc_table (polynom, crc_bits);
14705 : }
14706 :
14707 : /* Generate table-based CRC code for the given CRC, INPUT_DATA and the
14708 : POLYNOMIAL (without leading 1).
14709 :
14710 : First, using POLYNOMIAL's value generates CRC table of 256 elements,
14711 : then generates the assembly for the following code,
14712 : where crc_bit_size and data_bit_size may be 8, 16, 32, 64, depending on CRC:
14713 :
14714 : for (int i = 0; i < data_bit_size / 8; i++)
14715 : crc = (crc << 8) ^ crc_table[(crc >> (crc_bit_size - 8))
14716 : ^ (data >> (data_bit_size - (i + 1) * 8)
14717 : & 0xFF))];
14718 :
14719 : So to take values from the table, we need 8-bit data.
14720 : If input data size is not 8, then first we extract upper 8 bits,
14721 : then the other 8 bits, and so on. */
14722 :
14723 : static void
14724 144 : calculate_table_based_CRC (rtx *crc, const rtx &input_data,
14725 : const rtx &polynomial,
14726 : machine_mode data_mode)
14727 : {
14728 144 : machine_mode mode = GET_MODE (*crc);
14729 144 : unsigned short crc_bit_size = GET_MODE_BITSIZE (mode).to_constant ();
14730 144 : unsigned short data_size = GET_MODE_SIZE (data_mode).to_constant ();
14731 144 : rtx tab = generate_crc_table (UINTVAL (polynomial), crc_bit_size);
14732 :
14733 586 : for (unsigned short i = 0; i < data_size; i++)
14734 : {
14735 : /* crc >> (crc_bit_size - 8). */
14736 298 : *crc = force_reg (mode, *crc);
14737 298 : rtx op1 = expand_shift (RSHIFT_EXPR, mode, *crc, crc_bit_size - 8,
14738 : NULL_RTX, 1);
14739 :
14740 : /* data >> (8 * (GET_MODE_SIZE (data_mode).to_constant () - i - 1)). */
14741 298 : unsigned range_8 = 8 * (data_size - i - 1);
14742 : /* CRC's mode is always at least as wide as INPUT_DATA. Convert
14743 : INPUT_DATA into CRC's mode. */
14744 298 : rtx data = gen_reg_rtx (mode);
14745 298 : convert_move (data, input_data, 1);
14746 298 : data = expand_shift (RSHIFT_EXPR, mode, data, range_8, NULL_RTX, 1);
14747 :
14748 : /* data >> (8 * (GET_MODE_SIZE (mode)
14749 : .to_constant () - i - 1)) & 0xFF. */
14750 298 : rtx data_final = expand_and (mode, data,
14751 : gen_int_mode (255, mode), NULL_RTX);
14752 :
14753 : /* (crc >> (crc_bit_size - 8)) ^ data_8bit. */
14754 298 : rtx in = expand_binop (mode, xor_optab, op1, data_final,
14755 : NULL_RTX, 1, OPTAB_WIDEN);
14756 :
14757 : /* ((crc >> (crc_bit_size - 8)) ^ data_8bit) & 0xFF. */
14758 298 : rtx index = expand_and (mode, in, gen_int_mode (255, mode),
14759 : NULL_RTX);
14760 596 : int log_crc_size = exact_log2 (GET_MODE_SIZE (mode).to_constant ());
14761 298 : index = expand_shift (LSHIFT_EXPR, mode, index,
14762 298 : log_crc_size, NULL_RTX, 0);
14763 :
14764 303 : rtx addr = gen_reg_rtx (Pmode);
14765 298 : convert_move (addr, index, 1);
14766 303 : addr = expand_binop (Pmode, add_optab, addr, tab, NULL_RTX,
14767 : 0, OPTAB_DIRECT);
14768 :
14769 : /* crc_table[(crc >> (crc_bit_size - 8)) ^ data_8bit] */
14770 298 : rtx tab_el = validize_mem (gen_rtx_MEM (mode, addr));
14771 :
14772 : /* (crc << 8) if CRC is larger than 8, otherwise crc = 0. */
14773 298 : rtx high = NULL_RTX;
14774 298 : if (crc_bit_size != 8)
14775 269 : high = expand_shift (LSHIFT_EXPR, mode, *crc, 8, NULL_RTX, 0);
14776 : else
14777 29 : high = gen_int_mode (0, mode);
14778 :
14779 : /* crc = (crc << 8)
14780 : ^ crc_table[(crc >> (crc_bit_size - 8)) ^ data_8bit]; */
14781 298 : *crc = expand_binop (mode, xor_optab, tab_el, high, NULL_RTX, 1,
14782 : OPTAB_WIDEN);
14783 : }
14784 144 : }
14785 :
14786 : /* Generate table-based reversed CRC code for the given CRC, INPUT_DATA
14787 : and the POLYNOMIAL (without leading 1).
14788 :
14789 : This function generates code for reversed (bit-reflected) CRC calculation
14790 : using a pre-computed lookup table. Unlike the standard CRC calculation,
14791 : this processes data from LSB to MSB, eliminating the need for explicit
14792 : bit reflection before and after the CRC computation. */
14793 :
14794 : static void
14795 136 : calculate_table_based_reversed_CRC (rtx *crc, const rtx &input_data,
14796 : const rtx &polynomial,
14797 : machine_mode data_mode)
14798 : {
14799 136 : machine_mode mode = GET_MODE (*crc);
14800 136 : unsigned short crc_bit_size = GET_MODE_BITSIZE (mode).to_constant ();
14801 136 : unsigned short data_size = GET_MODE_SIZE (data_mode).to_constant ();
14802 136 : rtx tab = generate_reversed_crc_table (UINTVAL (polynomial), crc_bit_size);
14803 :
14804 : /* CRC's mode is always at least as wide as INPUT_DATA. Convert
14805 : INPUT_DATA into CRC's mode once outside the loop since INPUT_DATA
14806 : is loop-invariant. */
14807 136 : rtx data_in_crc_mode = gen_reg_rtx (mode);
14808 136 : convert_move (data_in_crc_mode, input_data, 1);
14809 :
14810 490 : for (unsigned short i = 0; i < data_size; i++)
14811 : {
14812 218 : *crc = force_reg (mode, *crc);
14813 :
14814 : /* data >> (8 * i). */
14815 218 : unsigned range_8 = 8 * i;
14816 218 : rtx data = expand_shift (RSHIFT_EXPR, mode, data_in_crc_mode,
14817 218 : range_8, NULL_RTX, 1);
14818 :
14819 : /* crc ^ (data >> (8 * i)). */
14820 218 : rtx in = expand_binop (mode, xor_optab, *crc, data,
14821 : NULL_RTX, 1, OPTAB_WIDEN);
14822 :
14823 : /* (crc ^ data) & 0xFF. */
14824 218 : rtx index = expand_and (mode, in, gen_int_mode (255, mode),
14825 : NULL_RTX);
14826 436 : int log_crc_size = exact_log2 (GET_MODE_SIZE (mode).to_constant ());
14827 218 : index = expand_shift (LSHIFT_EXPR, mode, index,
14828 218 : log_crc_size, NULL_RTX, 0);
14829 :
14830 231 : rtx addr = gen_reg_rtx (Pmode);
14831 218 : convert_move (addr, index, 1);
14832 231 : addr = expand_binop (Pmode, add_optab, addr, tab, NULL_RTX,
14833 : 0, OPTAB_DIRECT);
14834 :
14835 : /* crc_table[(crc ^ data) & 0xFF]. */
14836 218 : rtx tab_el = validize_mem (gen_rtx_MEM (mode, addr));
14837 :
14838 : /* (crc >> 8) if CRC is larger than 8, otherwise 0. */
14839 218 : rtx high = NULL_RTX;
14840 218 : if (crc_bit_size != 8)
14841 209 : high = expand_shift (RSHIFT_EXPR, mode, *crc, 8, NULL_RTX, 1);
14842 : else
14843 9 : high = gen_int_mode (0, mode);
14844 :
14845 : /* crc = (crc >> 8) ^ crc_table[(crc ^ data) & 0xFF]. */
14846 218 : *crc = expand_binop (mode, xor_optab, tab_el, high, NULL_RTX, 1,
14847 : OPTAB_WIDEN);
14848 : }
14849 136 : }
14850 :
14851 : /* Generate table-based CRC code for the given CRC, INPUT_DATA and the
14852 : POLYNOMIAL (without leading 1).
14853 :
14854 : CRC is OP1, data is OP2 and the polynomial is OP3.
14855 : This must generate a CRC table and an assembly for the following code,
14856 : where crc_bit_size and data_bit_size may be 8, 16, 32, 64:
14857 : uint_crc_bit_size_t
14858 : crc_crc_bit_size (uint_crc_bit_size_t crc_init,
14859 : uint_data_bit_size_t data, size_t size)
14860 : {
14861 : uint_crc_bit_size_t crc = crc_init;
14862 : for (int i = 0; i < data_bit_size / 8; i++)
14863 : crc = (crc << 8) ^ crc_table[(crc >> (crc_bit_size - 8))
14864 : ^ (data >> (data_bit_size - (i + 1) * 8)
14865 : & 0xFF))];
14866 : return crc;
14867 : } */
14868 :
14869 : void
14870 144 : expand_crc_table_based (rtx op0, rtx op1, rtx op2, rtx op3,
14871 : machine_mode data_mode)
14872 : {
14873 144 : gcc_assert (!CONST_INT_P (op0));
14874 144 : gcc_assert (CONST_INT_P (op3));
14875 144 : machine_mode crc_mode = GET_MODE (op0);
14876 144 : rtx crc = gen_reg_rtx (crc_mode);
14877 144 : convert_move (crc, op1, 0);
14878 144 : calculate_table_based_CRC (&crc, op2, op3, data_mode);
14879 144 : convert_move (op0, crc, 0);
14880 144 : }
14881 :
14882 : /* Generate table-based reversed CRC code for the given CRC, INPUT_DATA and
14883 : the POLYNOMIAL (without leading 1).
14884 :
14885 : CRC is OP1, data is OP2 and the polynomial is OP3.
14886 : This generates a reversed CRC table and assembly for the following code,
14887 : where crc_bit_size and data_bit_size may be 8, 16, 32, 64:
14888 : uint_crc_bit_size_t
14889 : crc_crc_bit_size (uint_crc_bit_size_t crc_init,
14890 : uint_data_bit_size_t data)
14891 : {
14892 : uint_crc_bit_size_t crc = crc_init;
14893 : for (int i = 0; i < data_bit_size / 8; i++)
14894 : crc = (crc >> 8) ^ crc_table[(crc ^ (data >> (i * 8))) & 0xFF];
14895 : return crc;
14896 : }
14897 :
14898 : This approach uses a pre-computed reversed polynomial table, eliminating
14899 : the need for explicit bit reflection before and after the CRC computation. */
14900 :
14901 : void
14902 136 : expand_reversed_crc_table_based (rtx op0, rtx op1, rtx op2, rtx op3,
14903 : machine_mode data_mode)
14904 : {
14905 136 : gcc_assert (!CONST_INT_P (op0));
14906 136 : gcc_assert (CONST_INT_P (op3));
14907 136 : machine_mode crc_mode = GET_MODE (op0);
14908 136 : rtx crc = gen_reg_rtx (crc_mode);
14909 136 : convert_move (crc, op1, 0);
14910 136 : calculate_table_based_reversed_CRC (&crc, op2, op3, data_mode);
14911 136 : convert_move (op0, crc, 0);
14912 136 : }
|
