Line data Source code
1 : /* Convert tree expression to rtl instructions, for GNU compiler.
2 : Copyright (C) 1988-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "predict.h"
29 : #include "memmodel.h"
30 : #include "tm_p.h"
31 : #include "ssa.h"
32 : #include "optabs.h"
33 : #include "expmed.h"
34 : #include "regs.h"
35 : #include "emit-rtl.h"
36 : #include "recog.h"
37 : #include "cgraph.h"
38 : #include "diagnostic.h"
39 : #include "alias.h"
40 : #include "fold-const.h"
41 : #include "stor-layout.h"
42 : #include "attribs.h"
43 : #include "varasm.h"
44 : #include "except.h"
45 : #include "insn-attr.h"
46 : #include "dojump.h"
47 : #include "explow.h"
48 : #include "calls.h"
49 : #include "stmt.h"
50 : /* Include expr.h after insn-config.h so we get HAVE_conditional_move. */
51 : #include "expr.h"
52 : #include "optabs-tree.h"
53 : #include "libfuncs.h"
54 : #include "reload.h"
55 : #include "langhooks.h"
56 : #include "common/common-target.h"
57 : #include "tree-dfa.h"
58 : #include "tree-ssa-live.h"
59 : #include "tree-outof-ssa.h"
60 : #include "tree-ssa-address.h"
61 : #include "builtins.h"
62 : #include "ccmp.h"
63 : #include "gimple-iterator.h"
64 : #include "gimple-fold.h"
65 : #include "rtx-vector-builder.h"
66 : #include "tree-pretty-print.h"
67 : #include "flags.h"
68 : #include "internal-fn.h"
69 : #include "gimple-range.h"
70 :
71 :
72 : /* If this is nonzero, we do not bother generating VOLATILE
73 : around volatile memory references, and we are willing to
74 : output indirect addresses. If cse is to follow, we reject
75 : indirect addresses so a useful potential cse is generated;
76 : if it is used only once, instruction combination will produce
77 : the same indirect address eventually. */
78 : int cse_not_expected;
79 :
80 : /* Cache of the "extended" flag in the target's _BitInt description
81 : for use during expand. */
82 : int bitint_extended = -1;
83 :
84 : static bool block_move_libcall_safe_for_call_parm (void);
85 : static bool emit_block_move_via_pattern (rtx, rtx, rtx, unsigned, unsigned,
86 : HOST_WIDE_INT, unsigned HOST_WIDE_INT,
87 : unsigned HOST_WIDE_INT,
88 : unsigned HOST_WIDE_INT, bool);
89 : static void emit_block_move_via_loop (rtx, rtx, rtx, unsigned, int);
90 : static void emit_block_move_via_sized_loop (rtx, rtx, rtx, unsigned, unsigned);
91 : static void emit_block_move_via_oriented_loop (rtx, rtx, rtx, unsigned, unsigned);
92 : static rtx emit_block_cmp_via_loop (rtx, rtx, rtx, tree, rtx, bool,
93 : unsigned, unsigned);
94 : static rtx_insn *compress_float_constant (rtx, rtx);
95 : static rtx get_subtarget (rtx);
96 : static rtx store_field (rtx, poly_int64, poly_int64, poly_uint64, poly_uint64,
97 : machine_mode, tree, alias_set_type, bool, bool);
98 :
99 : static unsigned HOST_WIDE_INT highest_pow2_factor_for_target (const_tree, const_tree);
100 :
101 : static bool is_aligning_offset (const_tree, const_tree);
102 : static rtx reduce_to_bit_field_precision (rtx, rtx, tree);
103 : static rtx do_store_flag (const_sepops, rtx, machine_mode);
104 : #ifdef PUSH_ROUNDING
105 : static void emit_single_push_insn (machine_mode, rtx, tree);
106 : #endif
107 : static void do_tablejump (rtx, machine_mode, rtx, rtx, rtx,
108 : profile_probability);
109 : static rtx const_vector_from_tree (tree);
110 : static tree tree_expr_size (const_tree);
111 : static void convert_mode_scalar (rtx, rtx, int);
112 :
113 :
114 : /* This is run to set up which modes can be used
115 : directly in memory and to initialize the block move optab. It is run
116 : at the beginning of compilation and when the target is reinitialized. */
117 :
118 : void
119 220178 : init_expr_target (void)
120 : {
121 220178 : rtx pat;
122 220178 : int num_clobbers;
123 220178 : rtx mem, mem1;
124 220178 : rtx reg;
125 :
126 : /* Try indexing by frame ptr and try by stack ptr.
127 : It is known that on the Convex the stack ptr isn't a valid index.
128 : With luck, one or the other is valid on any machine. */
129 220178 : mem = gen_rtx_MEM (word_mode, stack_pointer_rtx);
130 220178 : mem1 = gen_rtx_MEM (word_mode, frame_pointer_rtx);
131 :
132 : /* A scratch register we can modify in-place below to avoid
133 : useless RTL allocations. */
134 220178 : reg = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
135 :
136 220178 : rtx_insn *insn = as_a<rtx_insn *> (rtx_alloc (INSN));
137 220178 : pat = gen_rtx_SET (NULL_RTX, NULL_RTX);
138 220178 : PATTERN (insn) = pat;
139 :
140 27522250 : for (machine_mode mode = VOIDmode; (int) mode < NUM_MACHINE_MODES;
141 27302072 : mode = (machine_mode) ((int) mode + 1))
142 : {
143 27302072 : int regno;
144 :
145 27302072 : direct_load[(int) mode] = direct_store[(int) mode] = 0;
146 27302072 : PUT_MODE (mem, mode);
147 27302072 : PUT_MODE (mem1, mode);
148 :
149 : /* See if there is some register that can be used in this mode and
150 : directly loaded or stored from memory. */
151 :
152 27302072 : if (mode != VOIDmode && mode != BLKmode)
153 1840335905 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER
154 1867197621 : && (direct_load[(int) mode] == 0 || direct_store[(int) mode] == 0);
155 : regno++)
156 : {
157 1840335905 : if (!targetm.hard_regno_mode_ok (regno, mode))
158 1719829833 : continue;
159 :
160 120506072 : set_mode_and_regno (reg, mode, regno);
161 :
162 120506072 : SET_SRC (pat) = mem;
163 120506072 : SET_DEST (pat) = reg;
164 120506072 : if (recog (pat, insn, &num_clobbers) >= 0)
165 7402321 : direct_load[(int) mode] = 1;
166 :
167 120506072 : SET_SRC (pat) = mem1;
168 120506072 : SET_DEST (pat) = reg;
169 120506072 : if (recog (pat, insn, &num_clobbers) >= 0)
170 7402321 : direct_load[(int) mode] = 1;
171 :
172 120506072 : SET_SRC (pat) = reg;
173 120506072 : SET_DEST (pat) = mem;
174 120506072 : if (recog (pat, insn, &num_clobbers) >= 0)
175 7402321 : direct_store[(int) mode] = 1;
176 :
177 120506072 : SET_SRC (pat) = reg;
178 120506072 : SET_DEST (pat) = mem1;
179 120506072 : if (recog (pat, insn, &num_clobbers) >= 0)
180 7402321 : direct_store[(int) mode] = 1;
181 : }
182 : }
183 :
184 225887 : mem = gen_rtx_MEM (VOIDmode, gen_raw_REG (Pmode, LAST_VIRTUAL_REGISTER + 1));
185 :
186 220178 : opt_scalar_float_mode mode_iter;
187 1541246 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_FLOAT)
188 : {
189 1321068 : scalar_float_mode mode = mode_iter.require ();
190 1321068 : scalar_float_mode srcmode;
191 4623738 : FOR_EACH_MODE_UNTIL (srcmode, mode)
192 : {
193 3302670 : enum insn_code ic;
194 :
195 3302670 : ic = can_extend_p (mode, srcmode, 0);
196 3302670 : if (ic == CODE_FOR_nothing)
197 2637396 : continue;
198 :
199 665274 : PUT_MODE (mem, srcmode);
200 :
201 665274 : if (insn_operand_matches (ic, 1, mem))
202 663123 : float_extend_from_mem[mode][srcmode] = true;
203 : }
204 : }
205 220178 : }
206 :
207 : /* This is run at the start of compiling a function. */
208 :
209 : void
210 1741754 : init_expr (void)
211 : {
212 1741754 : memset (&crtl->expr, 0, sizeof (crtl->expr));
213 1741754 : }
214 :
215 : /* Copy data from FROM to TO, where the machine modes are not the same.
216 : Both modes may be integer, or both may be floating, or both may be
217 : fixed-point.
218 : UNSIGNEDP should be nonzero if FROM is an unsigned type.
219 : This causes zero-extension instead of sign-extension. */
220 :
221 : void
222 2188466 : convert_move (rtx to, rtx from, int unsignedp)
223 : {
224 2188466 : machine_mode to_mode = GET_MODE (to);
225 2188466 : machine_mode from_mode = GET_MODE (from);
226 :
227 2188466 : gcc_assert (to_mode != BLKmode);
228 2188466 : gcc_assert (from_mode != BLKmode);
229 :
230 : /* If the source and destination are already the same, then there's
231 : nothing to do. */
232 2188466 : if (to == from)
233 2188466 : return;
234 :
235 : /* If FROM is a SUBREG that indicates that we have already done at least
236 : the required extension, strip it. We don't handle such SUBREGs as
237 : TO here. */
238 :
239 2188466 : scalar_int_mode to_int_mode;
240 2188466 : if (GET_CODE (from) == SUBREG
241 188315 : && SUBREG_PROMOTED_VAR_P (from)
242 2188473 : && is_a <scalar_int_mode> (to_mode, &to_int_mode)
243 2188473 : && (GET_MODE_PRECISION (subreg_promoted_mode (from))
244 7 : >= GET_MODE_PRECISION (to_int_mode))
245 2188473 : && SUBREG_CHECK_PROMOTED_SIGN (from, unsignedp))
246 : {
247 0 : scalar_int_mode int_orig_mode;
248 0 : scalar_int_mode int_inner_mode;
249 0 : machine_mode orig_mode = GET_MODE (from);
250 :
251 0 : from = gen_lowpart (to_int_mode, SUBREG_REG (from));
252 0 : from_mode = to_int_mode;
253 :
254 : /* Preserve SUBREG_PROMOTED_VAR_P if the new mode is wider than
255 : the original mode, but narrower than the inner mode. */
256 0 : if (GET_CODE (from) == SUBREG
257 0 : && is_a <scalar_int_mode> (orig_mode, &int_orig_mode)
258 0 : && GET_MODE_PRECISION (to_int_mode)
259 0 : > GET_MODE_PRECISION (int_orig_mode)
260 0 : && is_a <scalar_int_mode> (GET_MODE (SUBREG_REG (from)),
261 : &int_inner_mode)
262 0 : && GET_MODE_PRECISION (int_inner_mode)
263 0 : > GET_MODE_PRECISION (to_int_mode))
264 : {
265 0 : SUBREG_PROMOTED_VAR_P (from) = 1;
266 0 : SUBREG_PROMOTED_SET (from, unsignedp);
267 : }
268 : }
269 :
270 2188466 : gcc_assert (GET_CODE (to) != SUBREG || !SUBREG_PROMOTED_VAR_P (to));
271 :
272 2188466 : if (to_mode == from_mode
273 2131474 : || (from_mode == VOIDmode && CONSTANT_P (from)))
274 : {
275 57263 : emit_move_insn (to, from);
276 57263 : return;
277 : }
278 :
279 2131203 : if (VECTOR_MODE_P (to_mode) || VECTOR_MODE_P (from_mode))
280 : {
281 30478 : if (GET_MODE_UNIT_PRECISION (to_mode)
282 15239 : > GET_MODE_UNIT_PRECISION (from_mode))
283 : {
284 5000 : optab op = unsignedp ? zext_optab : sext_optab;
285 5000 : insn_code icode = convert_optab_handler (op, to_mode, from_mode);
286 5000 : if (icode != CODE_FOR_nothing)
287 : {
288 1170 : emit_unop_insn (icode, to, from,
289 : unsignedp ? ZERO_EXTEND : SIGN_EXTEND);
290 1170 : return;
291 : }
292 : }
293 :
294 28138 : if (GET_MODE_UNIT_PRECISION (to_mode)
295 14069 : < GET_MODE_UNIT_PRECISION (from_mode))
296 : {
297 2154 : insn_code icode = convert_optab_handler (trunc_optab,
298 : to_mode, from_mode);
299 2154 : if (icode != CODE_FOR_nothing)
300 : {
301 908 : emit_unop_insn (icode, to, from, TRUNCATE);
302 908 : return;
303 : }
304 : }
305 :
306 39483 : gcc_assert (known_eq (GET_MODE_BITSIZE (from_mode),
307 : GET_MODE_BITSIZE (to_mode)));
308 :
309 13161 : if (VECTOR_MODE_P (to_mode))
310 13161 : from = force_subreg (to_mode, from, GET_MODE (from), 0);
311 : else
312 0 : to = simplify_gen_subreg (from_mode, to, GET_MODE (to), 0);
313 :
314 13161 : emit_move_insn (to, from);
315 13161 : return;
316 : }
317 :
318 2115964 : if (GET_CODE (to) == CONCAT && GET_CODE (from) == CONCAT)
319 : {
320 0 : convert_move (XEXP (to, 0), XEXP (from, 0), unsignedp);
321 0 : convert_move (XEXP (to, 1), XEXP (from, 1), unsignedp);
322 0 : return;
323 : }
324 :
325 2115964 : convert_mode_scalar (to, from, unsignedp);
326 : }
327 :
328 : /* Like convert_move, but deals only with scalar modes. */
329 :
330 : static void
331 2116026 : convert_mode_scalar (rtx to, rtx from, int unsignedp)
332 : {
333 : /* Both modes should be scalar types. */
334 2116031 : scalar_mode from_mode = as_a <scalar_mode> (GET_MODE (from));
335 2116031 : scalar_mode to_mode = as_a <scalar_mode> (GET_MODE (to));
336 2116031 : bool to_real = SCALAR_FLOAT_MODE_P (to_mode);
337 2116031 : bool from_real = SCALAR_FLOAT_MODE_P (from_mode);
338 2116031 : enum insn_code code;
339 2116031 : rtx libcall;
340 :
341 2116031 : gcc_assert (to_real == from_real);
342 :
343 : /* rtx code for making an equivalent value. */
344 3224342 : enum rtx_code equiv_code = (unsignedp < 0 ? UNKNOWN
345 2116031 : : (unsignedp ? ZERO_EXTEND : SIGN_EXTEND));
346 :
347 2116031 : auto acceptable_same_precision_modes
348 580 : = [] (scalar_mode from_mode, scalar_mode to_mode) -> bool
349 : {
350 580 : if (DECIMAL_FLOAT_MODE_P (from_mode) != DECIMAL_FLOAT_MODE_P (to_mode))
351 : return true;
352 :
353 : /* arm_bfloat_half_format <-> ieee_half_format */
354 2 : if ((REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
355 1 : && REAL_MODE_FORMAT (to_mode) == &ieee_half_format)
356 2 : || (REAL_MODE_FORMAT (to_mode) == &arm_bfloat_half_format
357 1 : && REAL_MODE_FORMAT (from_mode) == &ieee_half_format))
358 : return true;
359 :
360 : /* ibm_extended_format <-> ieee_quad_format */
361 0 : if ((REAL_MODE_FORMAT (from_mode) == &ibm_extended_format
362 0 : && REAL_MODE_FORMAT (to_mode) == &ieee_quad_format)
363 0 : || (REAL_MODE_FORMAT (from_mode) == &ieee_quad_format
364 0 : && REAL_MODE_FORMAT (to_mode) == &ibm_extended_format))
365 0 : return true;
366 :
367 : return false;
368 : };
369 :
370 2116031 : if (to_real)
371 : {
372 189283 : rtx value;
373 189283 : rtx_insn *insns;
374 189283 : convert_optab tab;
375 :
376 189283 : gcc_assert ((GET_MODE_PRECISION (from_mode)
377 : != GET_MODE_PRECISION (to_mode))
378 : || acceptable_same_precision_modes (from_mode, to_mode));
379 :
380 189283 : if (GET_MODE_PRECISION (from_mode) == GET_MODE_PRECISION (to_mode))
381 : {
382 580 : if ((REAL_MODE_FORMAT (to_mode) == &arm_bfloat_half_format
383 1 : && REAL_MODE_FORMAT (from_mode) == &ieee_half_format)
384 580 : || (REAL_MODE_FORMAT (to_mode) == &ieee_quad_format
385 0 : && REAL_MODE_FORMAT (from_mode) == &ibm_extended_format))
386 : /* libgcc implements just __trunchfbf2, not __extendhfbf2;
387 : and __trunctfkf2, not __extendtfkf2. */
388 : tab = trunc_optab;
389 : else
390 : /* Conversion between decimal float and binary float, same
391 : size. */
392 579 : tab = DECIMAL_FLOAT_MODE_P (from_mode) ? trunc_optab : sext_optab;
393 : }
394 188703 : else if (GET_MODE_PRECISION (from_mode) < GET_MODE_PRECISION (to_mode))
395 : tab = sext_optab;
396 : else
397 18761 : tab = trunc_optab;
398 :
399 : /* Try converting directly if the insn is supported. */
400 :
401 189283 : code = convert_optab_handler (tab, to_mode, from_mode);
402 189283 : if (code != CODE_FOR_nothing)
403 : {
404 161425 : emit_unop_insn (code, to, from,
405 : tab == sext_optab ? FLOAT_EXTEND : FLOAT_TRUNCATE);
406 161425 : return;
407 : }
408 :
409 : #ifdef HAVE_SFmode
410 27858 : if (REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
411 27858 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
412 : {
413 3245 : if (GET_MODE_PRECISION (to_mode) > GET_MODE_PRECISION (SFmode))
414 : {
415 : /* To cut down on libgcc size, implement
416 : BFmode -> {DF,XF,TF}mode conversions by
417 : BFmode -> SFmode -> {DF,XF,TF}mode conversions. */
418 4 : rtx temp = gen_reg_rtx (SFmode);
419 4 : convert_mode_scalar (temp, from, unsignedp);
420 4 : convert_mode_scalar (to, temp, unsignedp);
421 4 : return;
422 : }
423 3241 : if (REAL_MODE_FORMAT (to_mode) == &ieee_half_format)
424 : {
425 : /* Similarly, implement BFmode -> HFmode as
426 : BFmode -> SFmode -> HFmode conversion where SFmode
427 : has superset of BFmode values. We don't need
428 : to handle sNaNs by raising exception and turning
429 : it into qNaN though, as that can be done in the
430 : SFmode -> HFmode conversion too. */
431 1 : rtx temp = gen_reg_rtx (SFmode);
432 1 : int save_flag_finite_math_only = flag_finite_math_only;
433 1 : flag_finite_math_only = true;
434 1 : convert_mode_scalar (temp, from, unsignedp);
435 1 : flag_finite_math_only = save_flag_finite_math_only;
436 1 : convert_mode_scalar (to, temp, unsignedp);
437 1 : return;
438 : }
439 3240 : if (to_mode == SFmode
440 3240 : && !HONOR_NANS (from_mode)
441 58 : && !HONOR_NANS (to_mode)
442 3298 : && optimize_insn_for_speed_p ())
443 : {
444 : /* If we don't expect sNaNs, for BFmode -> SFmode we can just
445 : shift the bits up. */
446 57 : machine_mode fromi_mode, toi_mode;
447 114 : if (int_mode_for_size (GET_MODE_BITSIZE (from_mode),
448 57 : 0).exists (&fromi_mode)
449 57 : && int_mode_for_size (GET_MODE_BITSIZE (to_mode),
450 0 : 0).exists (&toi_mode))
451 : {
452 57 : start_sequence ();
453 57 : rtx fromi = force_lowpart_subreg (fromi_mode, from,
454 : from_mode);
455 57 : rtx tof = NULL_RTX;
456 57 : if (fromi)
457 : {
458 57 : rtx toi;
459 57 : if (GET_MODE (fromi) == VOIDmode)
460 0 : toi = simplify_unary_operation (ZERO_EXTEND, toi_mode,
461 : fromi, fromi_mode);
462 : else
463 : {
464 57 : toi = gen_reg_rtx (toi_mode);
465 57 : convert_mode_scalar (toi, fromi, 1);
466 : }
467 57 : toi
468 114 : = maybe_expand_shift (LSHIFT_EXPR, toi_mode, toi,
469 57 : GET_MODE_PRECISION (to_mode)
470 57 : - GET_MODE_PRECISION (from_mode),
471 : NULL_RTX, 1);
472 57 : if (toi)
473 : {
474 57 : tof = force_lowpart_subreg (to_mode, toi, toi_mode);
475 57 : if (tof)
476 57 : emit_move_insn (to, tof);
477 : }
478 : }
479 57 : insns = end_sequence ();
480 57 : if (tof)
481 : {
482 57 : emit_insn (insns);
483 57 : return;
484 : }
485 : }
486 : }
487 : }
488 27796 : if (REAL_MODE_FORMAT (from_mode) == &ieee_single_format
489 4274 : && REAL_MODE_FORMAT (to_mode) == &arm_bfloat_half_format
490 1572 : && !HONOR_NANS (from_mode)
491 0 : && !HONOR_NANS (to_mode)
492 0 : && !flag_rounding_math
493 27796 : && optimize_insn_for_speed_p ())
494 : {
495 : /* If we don't expect qNaNs nor sNaNs and can assume rounding
496 : to nearest, we can expand the conversion inline as
497 : (fromi + 0x7fff + ((fromi >> 16) & 1)) >> 16. */
498 0 : machine_mode fromi_mode, toi_mode;
499 0 : if (int_mode_for_size (GET_MODE_BITSIZE (from_mode),
500 0 : 0).exists (&fromi_mode)
501 0 : && int_mode_for_size (GET_MODE_BITSIZE (to_mode),
502 0 : 0).exists (&toi_mode))
503 : {
504 0 : start_sequence ();
505 0 : rtx fromi = force_lowpart_subreg (fromi_mode, from, from_mode);
506 0 : rtx tof = NULL_RTX;
507 0 : do
508 : {
509 0 : if (!fromi)
510 : break;
511 0 : int shift = (GET_MODE_PRECISION (from_mode)
512 0 : - GET_MODE_PRECISION (to_mode));
513 0 : rtx temp1
514 0 : = maybe_expand_shift (RSHIFT_EXPR, fromi_mode, fromi,
515 : shift, NULL_RTX, 1);
516 0 : if (!temp1)
517 : break;
518 0 : rtx temp2
519 0 : = expand_binop (fromi_mode, and_optab, temp1, const1_rtx,
520 : NULL_RTX, 1, OPTAB_DIRECT);
521 0 : if (!temp2)
522 : break;
523 0 : rtx temp3
524 0 : = expand_binop (fromi_mode, add_optab, fromi,
525 : gen_int_mode ((HOST_WIDE_INT_1U
526 0 : << (shift - 1)) - 1,
527 : fromi_mode), NULL_RTX,
528 : 1, OPTAB_DIRECT);
529 0 : if (!temp3)
530 : break;
531 0 : rtx temp4
532 0 : = expand_binop (fromi_mode, add_optab, temp3, temp2,
533 : NULL_RTX, 1, OPTAB_DIRECT);
534 0 : if (!temp4)
535 : break;
536 0 : rtx temp5 = maybe_expand_shift (RSHIFT_EXPR, fromi_mode,
537 : temp4, shift, NULL_RTX, 1);
538 0 : if (!temp5)
539 : break;
540 0 : rtx temp6 = force_lowpart_subreg (toi_mode, temp5,
541 : fromi_mode);
542 0 : if (!temp6)
543 : break;
544 0 : tof = force_lowpart_subreg (to_mode, temp6, toi_mode);
545 0 : if (tof)
546 0 : emit_move_insn (to, tof);
547 : }
548 : while (0);
549 0 : insns = end_sequence ();
550 0 : if (tof)
551 : {
552 0 : emit_insn (insns);
553 0 : return;
554 : }
555 : }
556 : }
557 : #endif
558 :
559 : /* Otherwise use a libcall. */
560 27796 : libcall = convert_optab_libfunc (tab, to_mode, from_mode);
561 :
562 : /* Is this conversion implemented yet? */
563 27796 : gcc_assert (libcall);
564 :
565 27796 : start_sequence ();
566 27796 : value = emit_library_call_value (libcall, NULL_RTX, LCT_CONST, to_mode,
567 : from, from_mode);
568 27796 : insns = end_sequence ();
569 27796 : emit_libcall_block (insns, to, value,
570 6785 : tab == trunc_optab ? gen_rtx_FLOAT_TRUNCATE (to_mode,
571 : from)
572 21011 : : gen_rtx_FLOAT_EXTEND (to_mode, from));
573 27796 : return;
574 : }
575 :
576 : /* Handle pointer conversion. */ /* SPEE 900220. */
577 : /* If the target has a converter from FROM_MODE to TO_MODE, use it. */
578 1926748 : {
579 1926748 : convert_optab ctab;
580 :
581 1926748 : if (GET_MODE_PRECISION (from_mode) > GET_MODE_PRECISION (to_mode))
582 : ctab = trunc_optab;
583 1723813 : else if (unsignedp)
584 : ctab = zext_optab;
585 : else
586 833450 : ctab = sext_optab;
587 :
588 1926748 : if (convert_optab_handler (ctab, to_mode, from_mode)
589 : != CODE_FOR_nothing)
590 : {
591 1653218 : emit_unop_insn (convert_optab_handler (ctab, to_mode, from_mode),
592 : to, from, UNKNOWN);
593 1653218 : return;
594 : }
595 : }
596 :
597 : /* Targets are expected to provide conversion insns between PxImode and
598 : xImode for all MODE_PARTIAL_INT modes they use, but no others. */
599 273530 : if (GET_MODE_CLASS (to_mode) == MODE_PARTIAL_INT)
600 : {
601 0 : scalar_int_mode full_mode
602 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE (to_mode)).require ();
603 :
604 0 : gcc_assert (convert_optab_handler (trunc_optab, to_mode, full_mode)
605 : != CODE_FOR_nothing);
606 :
607 0 : if (full_mode != from_mode)
608 0 : from = convert_to_mode (full_mode, from, unsignedp);
609 0 : emit_unop_insn (convert_optab_handler (trunc_optab, to_mode, full_mode),
610 : to, from, UNKNOWN);
611 0 : return;
612 : }
613 273530 : if (GET_MODE_CLASS (from_mode) == MODE_PARTIAL_INT)
614 : {
615 0 : rtx new_from;
616 0 : scalar_int_mode full_mode
617 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE (from_mode)).require ();
618 0 : convert_optab ctab = unsignedp ? zext_optab : sext_optab;
619 0 : enum insn_code icode;
620 :
621 0 : icode = convert_optab_handler (ctab, full_mode, from_mode);
622 0 : gcc_assert (icode != CODE_FOR_nothing);
623 :
624 0 : if (to_mode == full_mode)
625 : {
626 0 : emit_unop_insn (icode, to, from, UNKNOWN);
627 0 : return;
628 : }
629 :
630 0 : new_from = gen_reg_rtx (full_mode);
631 0 : emit_unop_insn (icode, new_from, from, UNKNOWN);
632 :
633 : /* else proceed to integer conversions below. */
634 0 : from_mode = full_mode;
635 0 : from = new_from;
636 : }
637 :
638 : /* Make sure both are fixed-point modes or both are not. */
639 273530 : gcc_assert (ALL_SCALAR_FIXED_POINT_MODE_P (from_mode) ==
640 : ALL_SCALAR_FIXED_POINT_MODE_P (to_mode));
641 273530 : if (ALL_SCALAR_FIXED_POINT_MODE_P (from_mode))
642 : {
643 : /* If we widen from_mode to to_mode and they are in the same class,
644 : we won't saturate the result.
645 : Otherwise, always saturate the result to play safe. */
646 0 : if (GET_MODE_CLASS (from_mode) == GET_MODE_CLASS (to_mode)
647 0 : && GET_MODE_SIZE (from_mode) < GET_MODE_SIZE (to_mode))
648 0 : expand_fixed_convert (to, from, 0, 0);
649 : else
650 0 : expand_fixed_convert (to, from, 0, 1);
651 : return;
652 : }
653 :
654 : /* Now both modes are integers. */
655 :
656 : /* Handle expanding beyond a word. */
657 273530 : if (GET_MODE_PRECISION (from_mode) < GET_MODE_PRECISION (to_mode)
658 276073 : && GET_MODE_PRECISION (to_mode) > BITS_PER_WORD)
659 : {
660 70595 : rtx_insn *insns;
661 70595 : rtx lowpart;
662 70595 : rtx fill_value;
663 70595 : rtx lowfrom;
664 70595 : int i;
665 70595 : scalar_mode lowpart_mode;
666 141190 : int nwords = CEIL (GET_MODE_SIZE (to_mode), UNITS_PER_WORD);
667 :
668 : /* Try converting directly if the insn is supported. */
669 70595 : if ((code = can_extend_p (to_mode, from_mode, unsignedp))
670 : != CODE_FOR_nothing)
671 : {
672 : /* If FROM is a SUBREG, put it into a register. Do this
673 : so that we always generate the same set of insns for
674 : better cse'ing; if an intermediate assignment occurred,
675 : we won't be doing the operation directly on the SUBREG. */
676 0 : if (optimize > 0 && GET_CODE (from) == SUBREG)
677 0 : from = force_reg (from_mode, from);
678 0 : emit_unop_insn (code, to, from, equiv_code);
679 0 : return;
680 : }
681 : /* Next, try converting via full word. */
682 70595 : else if (GET_MODE_PRECISION (from_mode) < BITS_PER_WORD
683 70595 : && ((code = can_extend_p (to_mode, word_mode, unsignedp))
684 : != CODE_FOR_nothing))
685 : {
686 70456 : rtx word_to = gen_reg_rtx (word_mode);
687 70456 : if (REG_P (to))
688 : {
689 70383 : if (reg_overlap_mentioned_p (to, from))
690 0 : from = force_reg (from_mode, from);
691 70383 : emit_clobber (to);
692 : }
693 70456 : convert_move (word_to, from, unsignedp);
694 70456 : emit_unop_insn (code, to, word_to, equiv_code);
695 70456 : return;
696 : }
697 :
698 : /* No special multiword conversion insn; do it by hand. */
699 139 : start_sequence ();
700 :
701 : /* Since we will turn this into a no conflict block, we must ensure
702 : the source does not overlap the target so force it into an isolated
703 : register when maybe so. Likewise for any MEM input, since the
704 : conversion sequence might require several references to it and we
705 : must ensure we're getting the same value every time. */
706 :
707 139 : if (MEM_P (from) || reg_overlap_mentioned_p (to, from))
708 0 : from = force_reg (from_mode, from);
709 :
710 : /* Get a copy of FROM widened to a word, if necessary. */
711 278 : if (GET_MODE_PRECISION (from_mode) < BITS_PER_WORD)
712 0 : lowpart_mode = word_mode;
713 : else
714 : lowpart_mode = from_mode;
715 :
716 139 : lowfrom = convert_to_mode (lowpart_mode, from, unsignedp);
717 :
718 139 : lowpart = gen_lowpart (lowpart_mode, to);
719 139 : emit_move_insn (lowpart, lowfrom);
720 :
721 : /* Compute the value to put in each remaining word. */
722 139 : if (unsignedp)
723 117 : fill_value = const0_rtx;
724 : else
725 22 : fill_value = emit_store_flag_force (gen_reg_rtx (word_mode),
726 : LT, lowfrom, const0_rtx,
727 : lowpart_mode, 0, -1);
728 :
729 : /* Fill the remaining words. */
730 695 : for (i = GET_MODE_SIZE (lowpart_mode) / UNITS_PER_WORD; i < nwords; i++)
731 : {
732 278 : int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
733 278 : rtx subword = operand_subword (to, index, 1, to_mode);
734 :
735 278 : gcc_assert (subword);
736 :
737 278 : if (fill_value != subword)
738 278 : emit_move_insn (subword, fill_value);
739 : }
740 :
741 139 : insns = end_sequence ();
742 :
743 139 : emit_insn (insns);
744 139 : return;
745 : }
746 :
747 : /* Truncating multi-word to a word or less. */
748 202935 : if (GET_MODE_PRECISION (from_mode) > BITS_PER_WORD
749 202935 : && GET_MODE_PRECISION (to_mode) <= BITS_PER_WORD)
750 : {
751 78996 : if (!((MEM_P (from)
752 567 : && ! MEM_VOLATILE_P (from)
753 567 : && direct_load[(int) to_mode]
754 567 : && ! mode_dependent_address_p (XEXP (from, 0),
755 567 : MEM_ADDR_SPACE (from)))
756 48460 : || REG_P (from)
757 : || GET_CODE (from) == SUBREG))
758 0 : from = force_reg (from_mode, from);
759 49027 : convert_move (to, gen_lowpart (word_mode, from), 0);
760 49027 : return;
761 : }
762 :
763 : /* Now follow all the conversions between integers
764 : no more than a word long. */
765 :
766 : /* For truncation, usually we can just refer to FROM in a narrower mode. */
767 307816 : if (GET_MODE_BITSIZE (to_mode) < GET_MODE_BITSIZE (from_mode)
768 153908 : && TRULY_NOOP_TRUNCATION_MODES_P (to_mode, from_mode))
769 : {
770 165965 : if (!((MEM_P (from)
771 13965 : && ! MEM_VOLATILE_P (from)
772 13906 : && direct_load[(int) to_mode]
773 13906 : && ! mode_dependent_address_p (XEXP (from, 0),
774 13906 : MEM_ADDR_SPACE (from)))
775 140002 : || REG_P (from)
776 : || GET_CODE (from) == SUBREG))
777 926 : from = force_reg (from_mode, from);
778 128871 : if (REG_P (from) && REGNO (from) < FIRST_PSEUDO_REGISTER
779 153909 : && !targetm.hard_regno_mode_ok (REGNO (from), to_mode))
780 0 : from = copy_to_reg (from);
781 153908 : emit_move_insn (to, gen_lowpart (to_mode, from));
782 153908 : return;
783 : }
784 :
785 : /* Handle extension. */
786 0 : if (GET_MODE_PRECISION (to_mode) > GET_MODE_PRECISION (from_mode))
787 : {
788 : /* Convert directly if that works. */
789 0 : if ((code = can_extend_p (to_mode, from_mode, unsignedp))
790 : != CODE_FOR_nothing)
791 : {
792 0 : emit_unop_insn (code, to, from, equiv_code);
793 0 : return;
794 : }
795 : else
796 : {
797 0 : rtx tmp;
798 0 : int shift_amount;
799 :
800 : /* Search for a mode to convert via. */
801 0 : opt_scalar_mode intermediate_iter;
802 0 : FOR_EACH_MODE_FROM (intermediate_iter, from_mode)
803 : {
804 0 : scalar_mode intermediate = intermediate_iter.require ();
805 0 : if (((can_extend_p (to_mode, intermediate, unsignedp)
806 : != CODE_FOR_nothing)
807 0 : || (GET_MODE_SIZE (to_mode) < GET_MODE_SIZE (intermediate)
808 0 : && TRULY_NOOP_TRUNCATION_MODES_P (to_mode,
809 : intermediate)))
810 0 : && (can_extend_p (intermediate, from_mode, unsignedp)
811 : != CODE_FOR_nothing))
812 : {
813 0 : convert_move (to, convert_to_mode (intermediate, from,
814 : unsignedp), unsignedp);
815 0 : return;
816 : }
817 : }
818 :
819 : /* No suitable intermediate mode.
820 : Generate what we need with shifts. */
821 0 : shift_amount = (GET_MODE_PRECISION (to_mode)
822 0 : - GET_MODE_PRECISION (from_mode));
823 0 : from = gen_lowpart (to_mode, force_reg (from_mode, from));
824 0 : tmp = expand_shift (LSHIFT_EXPR, to_mode, from, shift_amount,
825 : to, unsignedp);
826 0 : tmp = expand_shift (RSHIFT_EXPR, to_mode, tmp, shift_amount,
827 : to, unsignedp);
828 0 : if (tmp != to)
829 0 : emit_move_insn (to, tmp);
830 : return;
831 : }
832 : }
833 :
834 : /* Support special truncate insns for certain modes. */
835 0 : if (convert_optab_handler (trunc_optab, to_mode,
836 : from_mode) != CODE_FOR_nothing)
837 : {
838 0 : emit_unop_insn (convert_optab_handler (trunc_optab, to_mode, from_mode),
839 : to, from, UNKNOWN);
840 0 : return;
841 : }
842 :
843 : /* Handle truncation of volatile memrefs, and so on;
844 : the things that couldn't be truncated directly,
845 : and for which there was no special instruction.
846 :
847 : ??? Code above formerly short-circuited this, for most integer
848 : mode pairs, with a force_reg in from_mode followed by a recursive
849 : call to this routine. Appears always to have been wrong. */
850 0 : if (GET_MODE_PRECISION (to_mode) < GET_MODE_PRECISION (from_mode))
851 : {
852 0 : rtx temp = force_reg (to_mode, gen_lowpart (to_mode, from));
853 0 : emit_move_insn (to, temp);
854 0 : return;
855 : }
856 :
857 : /* Mode combination is not recognized. */
858 0 : gcc_unreachable ();
859 : }
860 :
861 : /* Return an rtx for a value that would result
862 : from converting X to mode MODE.
863 : Both X and MODE may be floating, or both integer.
864 : UNSIGNEDP is nonzero if X is an unsigned value.
865 : This can be done by referring to a part of X in place
866 : or by copying to a new temporary with conversion. */
867 :
868 : rtx
869 2395930 : convert_to_mode (machine_mode mode, rtx x, int unsignedp)
870 : {
871 2395930 : return convert_modes (mode, VOIDmode, x, unsignedp);
872 : }
873 :
874 : /* Return an rtx for a value that would result
875 : from converting X from mode OLDMODE to mode MODE.
876 : Both modes may be floating, or both integer.
877 : UNSIGNEDP is nonzero if X is an unsigned value.
878 :
879 : This can be done by referring to a part of X in place
880 : or by copying to a new temporary with conversion.
881 :
882 : You can give VOIDmode for OLDMODE, if you are sure X has a nonvoid mode. */
883 :
884 : rtx
885 5226904 : convert_modes (machine_mode mode, machine_mode oldmode, rtx x, int unsignedp)
886 : {
887 5226904 : rtx temp;
888 5226904 : scalar_int_mode int_mode;
889 :
890 : /* If FROM is a SUBREG that indicates that we have already done at least
891 : the required extension, strip it. */
892 :
893 5226904 : if (GET_CODE (x) == SUBREG
894 141070 : && SUBREG_PROMOTED_VAR_P (x)
895 5226911 : && is_a <scalar_int_mode> (mode, &int_mode)
896 5226911 : && (GET_MODE_PRECISION (subreg_promoted_mode (x))
897 14 : >= GET_MODE_PRECISION (int_mode))
898 5226918 : && SUBREG_CHECK_PROMOTED_SIGN (x, unsignedp))
899 : {
900 7 : scalar_int_mode int_orig_mode;
901 7 : scalar_int_mode int_inner_mode;
902 7 : machine_mode orig_mode = GET_MODE (x);
903 7 : x = gen_lowpart (int_mode, SUBREG_REG (x));
904 :
905 : /* Preserve SUBREG_PROMOTED_VAR_P if the new mode is wider than
906 : the original mode, but narrower than the inner mode. */
907 7 : if (GET_CODE (x) == SUBREG
908 0 : && is_a <scalar_int_mode> (orig_mode, &int_orig_mode)
909 0 : && GET_MODE_PRECISION (int_mode)
910 0 : > GET_MODE_PRECISION (int_orig_mode)
911 0 : && is_a <scalar_int_mode> (GET_MODE (SUBREG_REG (x)),
912 : &int_inner_mode)
913 7 : && GET_MODE_PRECISION (int_inner_mode)
914 0 : > GET_MODE_PRECISION (int_mode))
915 : {
916 0 : SUBREG_PROMOTED_VAR_P (x) = 1;
917 0 : SUBREG_PROMOTED_SET (x, unsignedp);
918 : }
919 : }
920 :
921 5226904 : if (GET_MODE (x) != VOIDmode)
922 2886415 : oldmode = GET_MODE (x);
923 :
924 5226904 : if (mode == oldmode)
925 : return x;
926 :
927 3781543 : if (CONST_SCALAR_INT_P (x)
928 3781543 : && is_a <scalar_int_mode> (mode, &int_mode))
929 : {
930 : /* If the caller did not tell us the old mode, then there is not
931 : much to do with respect to canonicalization. We have to
932 : assume that all the bits are significant. */
933 2019351 : if (!is_a <scalar_int_mode> (oldmode))
934 1690050 : oldmode = MAX_MODE_INT;
935 2019351 : wide_int w = wide_int::from (rtx_mode_t (x, oldmode),
936 2019351 : GET_MODE_PRECISION (int_mode),
937 2664355 : unsignedp ? UNSIGNED : SIGNED);
938 2019351 : return immed_wide_int_const (w, int_mode);
939 2019351 : }
940 :
941 : /* We can do this with a gen_lowpart if both desired and current modes
942 : are integer, and this is either a constant integer, a register, or a
943 : non-volatile MEM. */
944 1762192 : scalar_int_mode int_oldmode;
945 1762192 : if (is_int_mode (mode, &int_mode)
946 1667490 : && is_int_mode (oldmode, &int_oldmode)
947 1667490 : && GET_MODE_PRECISION (int_mode) <= GET_MODE_PRECISION (int_oldmode)
948 459124 : && ((MEM_P (x) && !MEM_VOLATILE_P (x) && direct_load[(int) int_mode])
949 13000 : || CONST_POLY_INT_P (x)
950 446124 : || (REG_P (x)
951 409932 : && (!HARD_REGISTER_P (x)
952 1069 : || targetm.hard_regno_mode_ok (REGNO (x), int_mode))
953 422932 : && TRULY_NOOP_TRUNCATION_MODES_P (int_mode, GET_MODE (x)))))
954 422932 : return gen_lowpart (int_mode, x);
955 :
956 : /* Converting from integer constant into mode is always equivalent to an
957 : subreg operation. */
958 1339260 : if (VECTOR_MODE_P (mode) && GET_MODE (x) == VOIDmode)
959 : {
960 0 : gcc_assert (known_eq (GET_MODE_BITSIZE (mode),
961 : GET_MODE_BITSIZE (oldmode)));
962 0 : return force_subreg (mode, x, oldmode, 0);
963 : }
964 :
965 1339260 : temp = gen_reg_rtx (mode);
966 1339260 : convert_move (temp, x, unsignedp);
967 1339260 : return temp;
968 : }
969 :
970 : /* Variant of convert_modes for ABI parameter passing/return.
971 : Return an rtx for a value that would result from converting X from
972 : a floating point mode FMODE to wider integer mode MODE. */
973 :
974 : rtx
975 0 : convert_float_to_wider_int (machine_mode mode, machine_mode fmode, rtx x)
976 : {
977 0 : gcc_assert (SCALAR_INT_MODE_P (mode) && SCALAR_FLOAT_MODE_P (fmode));
978 0 : scalar_int_mode tmp_mode = int_mode_for_mode (fmode).require ();
979 0 : rtx tmp = force_reg (tmp_mode, gen_lowpart (tmp_mode, x));
980 0 : return convert_modes (mode, tmp_mode, tmp, 1);
981 : }
982 :
983 : /* Variant of convert_modes for ABI parameter passing/return.
984 : Return an rtx for a value that would result from converting X from
985 : an integer mode IMODE to a narrower floating point mode MODE. */
986 :
987 : rtx
988 0 : convert_wider_int_to_float (machine_mode mode, machine_mode imode, rtx x)
989 : {
990 0 : gcc_assert (SCALAR_FLOAT_MODE_P (mode) && SCALAR_INT_MODE_P (imode));
991 0 : scalar_int_mode tmp_mode = int_mode_for_mode (mode).require ();
992 0 : rtx tmp = force_reg (tmp_mode, gen_lowpart (tmp_mode, x));
993 0 : return gen_lowpart_SUBREG (mode, tmp);
994 : }
995 :
996 : /* Return the largest alignment we can use for doing a move (or store)
997 : of MAX_PIECES. ALIGN is the largest alignment we could use. */
998 :
999 : static unsigned int
1000 2775878 : alignment_for_piecewise_move (unsigned int max_pieces, unsigned int align)
1001 : {
1002 2775878 : scalar_int_mode tmode
1003 2775878 : = int_mode_for_size (max_pieces * BITS_PER_UNIT, 0).require ();
1004 :
1005 2775878 : if (align >= GET_MODE_ALIGNMENT (tmode))
1006 2190818 : align = GET_MODE_ALIGNMENT (tmode);
1007 : else
1008 : {
1009 585060 : scalar_int_mode xmode = NARROWEST_INT_MODE;
1010 585060 : opt_scalar_int_mode mode_iter;
1011 3526799 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
1012 : {
1013 3521071 : tmode = mode_iter.require ();
1014 3521071 : if (GET_MODE_SIZE (tmode) > max_pieces
1015 3521071 : || targetm.slow_unaligned_access (tmode, align))
1016 : break;
1017 2941739 : xmode = tmode;
1018 : }
1019 :
1020 585060 : align = MAX (align, GET_MODE_ALIGNMENT (xmode));
1021 : }
1022 :
1023 2775878 : return align;
1024 : }
1025 :
1026 : /* Return true if we know how to implement OP using vectors of bytes. */
1027 : static bool
1028 5870172 : can_use_qi_vectors (by_pieces_operation op)
1029 : {
1030 5870172 : return (op == COMPARE_BY_PIECES
1031 0 : || op == SET_BY_PIECES
1032 0 : || op == CLEAR_BY_PIECES);
1033 : }
1034 :
1035 : /* Return true if optabs exists for the mode and certain by pieces
1036 : operations. */
1037 : static bool
1038 26003294 : by_pieces_mode_supported_p (fixed_size_mode mode, by_pieces_operation op)
1039 : {
1040 26003294 : if (optab_handler (mov_optab, mode) == CODE_FOR_nothing)
1041 : return false;
1042 :
1043 24641398 : if ((op == SET_BY_PIECES || op == CLEAR_BY_PIECES)
1044 687418 : && VECTOR_MODE_P (mode)
1045 25135299 : && optab_handler (vec_duplicate_optab, mode) == CODE_FOR_nothing)
1046 : return false;
1047 :
1048 24592735 : if (op == COMPARE_BY_PIECES
1049 24592735 : && !can_compare_p (EQ, mode, ccp_jump))
1050 : return false;
1051 :
1052 : return true;
1053 : }
1054 :
1055 : /* Return the widest mode that can be used to perform part of an
1056 : operation OP on SIZE bytes. Try to use QI vector modes where
1057 : possible. */
1058 : static fixed_size_mode
1059 5376384 : widest_fixed_size_mode_for_size (unsigned int size, by_pieces_operation op)
1060 : {
1061 5376384 : fixed_size_mode result = NARROWEST_INT_MODE;
1062 :
1063 5376384 : gcc_checking_assert (size > 1);
1064 :
1065 : /* Use QI vector only if size is wider than a WORD. */
1066 5376384 : if (can_use_qi_vectors (op))
1067 : {
1068 729469 : machine_mode mode;
1069 729469 : fixed_size_mode candidate;
1070 12046501 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_INT)
1071 12046501 : if (is_a<fixed_size_mode> (mode, &candidate)
1072 13153007 : && GET_MODE_SIZE (candidate) > UNITS_PER_WORD
1073 12970690 : && GET_MODE_INNER (candidate) == QImode)
1074 : {
1075 5674516 : if (GET_MODE_SIZE (candidate) >= size)
1076 : break;
1077 2107789 : if (by_pieces_mode_supported_p (candidate, op))
1078 11317032 : result = candidate;
1079 : }
1080 :
1081 729469 : if (result != NARROWEST_INT_MODE)
1082 472430 : return result;
1083 : }
1084 :
1085 4903954 : opt_scalar_int_mode tmode;
1086 4903954 : scalar_int_mode mode;
1087 39231632 : FOR_EACH_MODE_IN_CLASS (tmode, MODE_INT)
1088 : {
1089 34327678 : mode = tmode.require ();
1090 34327678 : if (GET_MODE_SIZE (mode) < size
1091 34327678 : && by_pieces_mode_supported_p (mode, op))
1092 23853760 : result = mode;
1093 : }
1094 :
1095 4903954 : return result;
1096 : }
1097 :
1098 : /* Determine whether an operation OP on LEN bytes with alignment ALIGN can
1099 : and should be performed piecewise. */
1100 :
1101 : static bool
1102 968412 : can_do_by_pieces (unsigned HOST_WIDE_INT len, unsigned int align,
1103 : enum by_pieces_operation op)
1104 : {
1105 968412 : return targetm.use_by_pieces_infrastructure_p (len, align, op,
1106 968412 : optimize_insn_for_speed_p ());
1107 : }
1108 :
1109 : /* Determine whether the LEN bytes can be moved by using several move
1110 : instructions. Return nonzero if a call to move_by_pieces should
1111 : succeed. */
1112 :
1113 : bool
1114 898779 : can_move_by_pieces (unsigned HOST_WIDE_INT len, unsigned int align)
1115 : {
1116 898779 : return can_do_by_pieces (len, align, MOVE_BY_PIECES);
1117 : }
1118 :
1119 : /* Return number of insns required to perform operation OP by pieces
1120 : for L bytes. ALIGN (in bits) is maximum alignment we can assume. */
1121 :
1122 : unsigned HOST_WIDE_INT
1123 1962707 : by_pieces_ninsns (unsigned HOST_WIDE_INT l, unsigned int align,
1124 : unsigned int max_size, by_pieces_operation op)
1125 : {
1126 1962707 : unsigned HOST_WIDE_INT n_insns = 0;
1127 1962707 : fixed_size_mode mode;
1128 :
1129 1962707 : if (targetm.overlap_op_by_pieces_p ())
1130 : {
1131 : /* NB: Round up L and ALIGN to the widest integer mode for
1132 : MAX_SIZE. */
1133 1962707 : mode = widest_fixed_size_mode_for_size (max_size, op);
1134 1962707 : gcc_assert (optab_handler (mov_optab, mode) != CODE_FOR_nothing);
1135 3925414 : unsigned HOST_WIDE_INT up = ROUND_UP (l, GET_MODE_SIZE (mode));
1136 1962707 : if (up > l)
1137 : l = up;
1138 1962707 : align = GET_MODE_ALIGNMENT (mode);
1139 : }
1140 :
1141 1962707 : align = alignment_for_piecewise_move (MOVE_MAX_PIECES, align);
1142 :
1143 5896468 : while (max_size > 1 && l > 0)
1144 : {
1145 1971054 : mode = widest_fixed_size_mode_for_size (max_size, op);
1146 1971054 : gcc_assert (optab_handler (mov_optab, mode) != CODE_FOR_nothing);
1147 :
1148 1971054 : unsigned int modesize = GET_MODE_SIZE (mode);
1149 :
1150 1971054 : if (align >= GET_MODE_ALIGNMENT (mode))
1151 : {
1152 1971054 : unsigned HOST_WIDE_INT n_pieces = l / modesize;
1153 1971054 : l %= modesize;
1154 1971054 : switch (op)
1155 : {
1156 1901421 : default:
1157 1901421 : n_insns += n_pieces;
1158 1901421 : break;
1159 :
1160 69633 : case COMPARE_BY_PIECES:
1161 69633 : int batch = targetm.compare_by_pieces_branch_ratio (mode);
1162 69633 : int batch_ops = 4 * batch - 1;
1163 69633 : unsigned HOST_WIDE_INT full = n_pieces / batch;
1164 69633 : n_insns += full * batch_ops;
1165 69633 : if (n_pieces % batch != 0)
1166 0 : n_insns++;
1167 : break;
1168 :
1169 : }
1170 : }
1171 1971054 : max_size = modesize;
1172 : }
1173 :
1174 1962707 : gcc_assert (!l);
1175 1962707 : return n_insns;
1176 : }
1177 :
1178 : /* Used when performing piecewise block operations, holds information
1179 : about one of the memory objects involved. The member functions
1180 : can be used to generate code for loading from the object and
1181 : updating the address when iterating. */
1182 :
1183 : class pieces_addr
1184 : {
1185 : /* The object being referenced, a MEM. Can be NULL_RTX to indicate
1186 : stack pushes. */
1187 : rtx m_obj;
1188 : /* The address of the object. Can differ from that seen in the
1189 : MEM rtx if we copied the address to a register. */
1190 : rtx m_addr;
1191 : /* Nonzero if the address on the object has an autoincrement already,
1192 : signifies whether that was an increment or decrement. */
1193 : signed char m_addr_inc;
1194 : /* Nonzero if we intend to use autoinc without the address already
1195 : having autoinc form. We will insert add insns around each memory
1196 : reference, expecting later passes to form autoinc addressing modes.
1197 : The only supported options are predecrement and postincrement. */
1198 : signed char m_explicit_inc;
1199 : /* True if we have either of the two possible cases of using
1200 : autoincrement. */
1201 : bool m_auto;
1202 : /* True if this is an address to be used for load operations rather
1203 : than stores. */
1204 : bool m_is_load;
1205 :
1206 : /* Optionally, a function to obtain constants for any given offset into
1207 : the objects, and data associated with it. */
1208 : by_pieces_constfn m_constfn;
1209 : void *m_cfndata;
1210 : public:
1211 : pieces_addr (rtx, bool, by_pieces_constfn, void *);
1212 : rtx adjust (fixed_size_mode, HOST_WIDE_INT, by_pieces_prev * = nullptr);
1213 : void increment_address (HOST_WIDE_INT);
1214 : void maybe_predec (HOST_WIDE_INT);
1215 : void maybe_postinc (HOST_WIDE_INT);
1216 : void decide_autoinc (machine_mode, bool, HOST_WIDE_INT);
1217 759205 : int get_addr_inc ()
1218 : {
1219 759205 : return m_addr_inc;
1220 : }
1221 : };
1222 :
1223 : /* Initialize a pieces_addr structure from an object OBJ. IS_LOAD is
1224 : true if the operation to be performed on this object is a load
1225 : rather than a store. For stores, OBJ can be NULL, in which case we
1226 : assume the operation is a stack push. For loads, the optional
1227 : CONSTFN and its associated CFNDATA can be used in place of the
1228 : memory load. */
1229 :
1230 1518410 : pieces_addr::pieces_addr (rtx obj, bool is_load, by_pieces_constfn constfn,
1231 : void *cfndata)
1232 1518410 : : m_obj (obj), m_is_load (is_load), m_constfn (constfn), m_cfndata (cfndata)
1233 : {
1234 1518410 : m_addr_inc = 0;
1235 1518410 : m_auto = false;
1236 1518410 : if (obj)
1237 : {
1238 1393148 : rtx addr = XEXP (obj, 0);
1239 1393148 : rtx_code code = GET_CODE (addr);
1240 1393148 : m_addr = addr;
1241 1393148 : bool dec = code == PRE_DEC || code == POST_DEC;
1242 1393148 : bool inc = code == PRE_INC || code == POST_INC;
1243 1393148 : m_auto = inc || dec;
1244 1393148 : if (m_auto)
1245 0 : m_addr_inc = dec ? -1 : 1;
1246 :
1247 : /* While we have always looked for these codes here, the code
1248 : implementing the memory operation has never handled them.
1249 : Support could be added later if necessary or beneficial. */
1250 1393148 : gcc_assert (code != PRE_INC && code != POST_DEC);
1251 : }
1252 : else
1253 : {
1254 125262 : m_addr = NULL_RTX;
1255 125262 : if (!is_load)
1256 : {
1257 45 : m_auto = true;
1258 45 : if (STACK_GROWS_DOWNWARD)
1259 45 : m_addr_inc = -1;
1260 : else
1261 : m_addr_inc = 1;
1262 : }
1263 : else
1264 125217 : gcc_assert (constfn != NULL);
1265 : }
1266 1518410 : m_explicit_inc = 0;
1267 1518410 : if (constfn)
1268 151462 : gcc_assert (is_load);
1269 1518410 : }
1270 :
1271 : /* Decide whether to use autoinc for an address involved in a memory op.
1272 : MODE is the mode of the accesses, REVERSE is true if we've decided to
1273 : perform the operation starting from the end, and LEN is the length of
1274 : the operation. Don't override an earlier decision to set m_auto. */
1275 :
1276 : void
1277 377220 : pieces_addr::decide_autoinc (machine_mode ARG_UNUSED (mode), bool reverse,
1278 : HOST_WIDE_INT len)
1279 : {
1280 377220 : if (m_auto || m_obj == NULL_RTX)
1281 : return;
1282 :
1283 345792 : bool use_predec = (m_is_load
1284 : ? USE_LOAD_PRE_DECREMENT (mode)
1285 : : USE_STORE_PRE_DECREMENT (mode));
1286 345792 : bool use_postinc = (m_is_load
1287 : ? USE_LOAD_POST_INCREMENT (mode)
1288 : : USE_STORE_POST_INCREMENT (mode));
1289 345792 : machine_mode addr_mode = get_address_mode (m_obj);
1290 :
1291 345792 : if (use_predec && reverse)
1292 : {
1293 : m_addr = copy_to_mode_reg (addr_mode,
1294 : plus_constant (addr_mode,
1295 : m_addr, len));
1296 : m_auto = true;
1297 : m_explicit_inc = -1;
1298 : }
1299 345792 : else if (use_postinc && !reverse)
1300 : {
1301 : m_addr = copy_to_mode_reg (addr_mode, m_addr);
1302 : m_auto = true;
1303 : m_explicit_inc = 1;
1304 : }
1305 345792 : else if (CONSTANT_P (m_addr))
1306 71982 : m_addr = copy_to_mode_reg (addr_mode, m_addr);
1307 : }
1308 :
1309 : /* Adjust the address to refer to the data at OFFSET in MODE. If we
1310 : are using autoincrement for this address, we don't add the offset,
1311 : but we still modify the MEM's properties. */
1312 :
1313 : rtx
1314 4143605 : pieces_addr::adjust (fixed_size_mode mode, HOST_WIDE_INT offset,
1315 : by_pieces_prev *prev)
1316 : {
1317 4143605 : if (m_constfn)
1318 : /* Pass the previous data to m_constfn. */
1319 371378 : return m_constfn (m_cfndata, prev, offset, mode);
1320 3772227 : if (m_obj == NULL_RTX)
1321 : return NULL_RTX;
1322 3772180 : if (m_auto)
1323 0 : return adjust_automodify_address (m_obj, mode, m_addr, offset);
1324 : else
1325 3772180 : return adjust_address (m_obj, mode, offset);
1326 : }
1327 :
1328 : /* Emit an add instruction to increment the address by SIZE. */
1329 :
1330 : void
1331 0 : pieces_addr::increment_address (HOST_WIDE_INT size)
1332 : {
1333 0 : rtx amount = gen_int_mode (size, GET_MODE (m_addr));
1334 0 : emit_insn (gen_add2_insn (m_addr, amount));
1335 0 : }
1336 :
1337 : /* If we are supposed to decrement the address after each access, emit code
1338 : to do so now. Increment by SIZE (which has should have the correct sign
1339 : already). */
1340 :
1341 : void
1342 4142958 : pieces_addr::maybe_predec (HOST_WIDE_INT size)
1343 : {
1344 4142958 : if (m_explicit_inc >= 0)
1345 4142958 : return;
1346 0 : gcc_assert (HAVE_PRE_DECREMENT);
1347 : increment_address (size);
1348 : }
1349 :
1350 : /* If we are supposed to decrement the address after each access, emit code
1351 : to do so now. Increment by SIZE. */
1352 :
1353 : void
1354 4142981 : pieces_addr::maybe_postinc (HOST_WIDE_INT size)
1355 : {
1356 4142981 : if (m_explicit_inc <= 0)
1357 4142981 : return;
1358 0 : gcc_assert (HAVE_POST_INCREMENT);
1359 : increment_address (size);
1360 : }
1361 :
1362 : /* This structure is used by do_op_by_pieces to describe the operation
1363 : to be performed. */
1364 :
1365 : class op_by_pieces_d
1366 : {
1367 : private:
1368 : fixed_size_mode get_usable_mode (fixed_size_mode, unsigned int);
1369 : fixed_size_mode smallest_fixed_size_mode_for_size (unsigned int);
1370 :
1371 : protected:
1372 : pieces_addr m_to, m_from;
1373 : /* Make m_len read-only so that smallest_fixed_size_mode_for_size can
1374 : use it to check the valid mode size. */
1375 : const unsigned HOST_WIDE_INT m_len;
1376 : HOST_WIDE_INT m_offset;
1377 : unsigned int m_align;
1378 : unsigned int m_max_size;
1379 : bool m_reverse;
1380 : /* True if this is a stack push. */
1381 : bool m_push;
1382 : /* True if targetm.overlap_op_by_pieces_p () returns true. */
1383 : bool m_overlap_op_by_pieces;
1384 : /* The type of operation that we're performing. */
1385 : by_pieces_operation m_op;
1386 :
1387 : /* Virtual functions, overridden by derived classes for the specific
1388 : operation. */
1389 : virtual void generate (rtx, rtx, machine_mode) = 0;
1390 : virtual bool prepare_mode (machine_mode, unsigned int) = 0;
1391 1151843 : virtual void finish_mode (machine_mode)
1392 : {
1393 1151843 : }
1394 :
1395 : public:
1396 : op_by_pieces_d (unsigned int, rtx, bool, rtx, bool, by_pieces_constfn,
1397 : void *, unsigned HOST_WIDE_INT, unsigned int, bool,
1398 : by_pieces_operation);
1399 : void run ();
1400 : };
1401 :
1402 : /* The constructor for an op_by_pieces_d structure. We require two
1403 : objects named TO and FROM, which are identified as loads or stores
1404 : by TO_LOAD and FROM_LOAD. If FROM is a load, the optional FROM_CFN
1405 : and its associated FROM_CFN_DATA can be used to replace loads with
1406 : constant values. MAX_PIECES describes the maximum number of bytes
1407 : at a time which can be moved efficiently. LEN describes the length
1408 : of the operation. */
1409 :
1410 759205 : op_by_pieces_d::op_by_pieces_d (unsigned int max_pieces, rtx to,
1411 : bool to_load, rtx from, bool from_load,
1412 : by_pieces_constfn from_cfn,
1413 : void *from_cfn_data,
1414 : unsigned HOST_WIDE_INT len,
1415 : unsigned int align, bool push,
1416 : by_pieces_operation op)
1417 759205 : : m_to (to, to_load, NULL, NULL),
1418 759205 : m_from (from, from_load, from_cfn, from_cfn_data),
1419 759205 : m_len (len), m_max_size (max_pieces + 1),
1420 759205 : m_push (push), m_op (op)
1421 : {
1422 759205 : int toi = m_to.get_addr_inc ();
1423 759205 : int fromi = m_from.get_addr_inc ();
1424 759205 : if (toi >= 0 && fromi >= 0)
1425 759160 : m_reverse = false;
1426 45 : else if (toi <= 0 && fromi <= 0)
1427 45 : m_reverse = true;
1428 : else
1429 0 : gcc_unreachable ();
1430 :
1431 759205 : m_offset = m_reverse ? len : 0;
1432 2786351 : align = MIN (to ? MEM_ALIGN (to) : align,
1433 : from ? MEM_ALIGN (from) : align);
1434 :
1435 : /* If copying requires more than two move insns,
1436 : copy addresses to registers (to make displacements shorter)
1437 : and use post-increment if available. */
1438 759205 : if (by_pieces_ninsns (len, align, m_max_size, MOVE_BY_PIECES) > 2)
1439 : {
1440 : /* Find the mode of the largest comparison. */
1441 188610 : fixed_size_mode mode
1442 188610 : = widest_fixed_size_mode_for_size (m_max_size, m_op);
1443 :
1444 188610 : m_from.decide_autoinc (mode, m_reverse, len);
1445 188610 : m_to.decide_autoinc (mode, m_reverse, len);
1446 : }
1447 :
1448 759215 : align = alignment_for_piecewise_move (MOVE_MAX_PIECES, align);
1449 759205 : m_align = align;
1450 :
1451 759205 : m_overlap_op_by_pieces = targetm.overlap_op_by_pieces_p ();
1452 759205 : }
1453 :
1454 : /* This function returns the largest usable integer mode for LEN bytes
1455 : whose size is no bigger than size of MODE. */
1456 :
1457 : fixed_size_mode
1458 1246637 : op_by_pieces_d::get_usable_mode (fixed_size_mode mode, unsigned int len)
1459 : {
1460 1546279 : unsigned int size;
1461 1845921 : do
1462 : {
1463 1546279 : size = GET_MODE_SIZE (mode);
1464 1546279 : if (len >= size && prepare_mode (mode, m_align))
1465 : break;
1466 : /* widest_fixed_size_mode_for_size checks SIZE > 1. */
1467 299642 : mode = widest_fixed_size_mode_for_size (size, m_op);
1468 : }
1469 : while (1);
1470 1246637 : return mode;
1471 : }
1472 :
1473 : /* Return the smallest integer or QI vector mode that is not narrower
1474 : than SIZE bytes. */
1475 :
1476 : fixed_size_mode
1477 493788 : op_by_pieces_d::smallest_fixed_size_mode_for_size (unsigned int size)
1478 : {
1479 : /* Use QI vector only for > size of WORD. */
1480 505996 : if (can_use_qi_vectors (m_op) && size > UNITS_PER_WORD)
1481 : {
1482 8234 : machine_mode mode;
1483 8234 : fixed_size_mode candidate;
1484 111425 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_INT)
1485 111425 : if (is_a<fixed_size_mode> (mode, &candidate)
1486 222850 : && GET_MODE_INNER (candidate) == QImode)
1487 : {
1488 : /* Don't return a mode wider than M_LEN. */
1489 103602 : if (GET_MODE_SIZE (candidate) > m_len)
1490 : break;
1491 :
1492 49865 : if (GET_MODE_SIZE (candidate) >= size
1493 49865 : && by_pieces_mode_supported_p (candidate, m_op))
1494 6298 : return candidate;
1495 : }
1496 : }
1497 :
1498 487490 : return smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
1499 : }
1500 :
1501 : /* This function contains the main loop used for expanding a block
1502 : operation. First move what we can in the largest integer mode,
1503 : then go to successively smaller modes. For every access, call
1504 : GENFUN with the two operands and the EXTRA_DATA. */
1505 :
1506 : void
1507 759205 : op_by_pieces_d::run ()
1508 : {
1509 759205 : if (m_len == 0)
1510 : return;
1511 :
1512 752849 : unsigned HOST_WIDE_INT length = m_len;
1513 :
1514 : /* widest_fixed_size_mode_for_size checks M_MAX_SIZE > 1. */
1515 752849 : fixed_size_mode mode
1516 752849 : = widest_fixed_size_mode_for_size (m_max_size, m_op);
1517 752849 : mode = get_usable_mode (mode, length);
1518 :
1519 752849 : by_pieces_prev to_prev = { nullptr, mode };
1520 752849 : by_pieces_prev from_prev = { nullptr, mode };
1521 :
1522 1246637 : do
1523 : {
1524 1246637 : unsigned int size = GET_MODE_SIZE (mode);
1525 1246637 : rtx to1 = NULL_RTX, from1;
1526 :
1527 3318116 : while (length >= size)
1528 : {
1529 2071479 : if (m_reverse)
1530 47 : m_offset -= size;
1531 :
1532 2071479 : to1 = m_to.adjust (mode, m_offset, &to_prev);
1533 2071479 : to_prev.data = to1;
1534 2071479 : to_prev.mode = mode;
1535 2071479 : from1 = m_from.adjust (mode, m_offset, &from_prev);
1536 2071479 : from_prev.data = from1;
1537 2071479 : from_prev.mode = mode;
1538 :
1539 2071479 : m_to.maybe_predec (-(HOST_WIDE_INT)size);
1540 2071479 : m_from.maybe_predec (-(HOST_WIDE_INT)size);
1541 :
1542 2071479 : generate (to1, from1, mode);
1543 :
1544 2071479 : m_to.maybe_postinc (size);
1545 2071479 : m_from.maybe_postinc (size);
1546 :
1547 2071479 : if (!m_reverse)
1548 2071432 : m_offset += size;
1549 :
1550 2071479 : length -= size;
1551 : }
1552 :
1553 1246637 : finish_mode (mode);
1554 :
1555 1246637 : if (length == 0)
1556 : return;
1557 :
1558 493788 : if (!m_push && m_overlap_op_by_pieces)
1559 : {
1560 : /* NB: Generate overlapping operations if it is not a stack
1561 : push since stack push must not overlap. Get the smallest
1562 : fixed size mode for M_LEN bytes. */
1563 493788 : mode = smallest_fixed_size_mode_for_size (length);
1564 987576 : mode = get_usable_mode (mode, GET_MODE_SIZE (mode));
1565 493788 : int gap = GET_MODE_SIZE (mode) - length;
1566 493788 : if (gap > 0)
1567 : {
1568 : /* If size of MODE > M_LEN, generate the last operation
1569 : in MODE for the remaining bytes with overlapping memory
1570 : from the previois operation. */
1571 49039 : if (m_reverse)
1572 0 : m_offset += gap;
1573 : else
1574 49039 : m_offset -= gap;
1575 49039 : length += gap;
1576 : }
1577 : }
1578 : else
1579 : {
1580 : /* widest_fixed_size_mode_for_size checks SIZE > 1. */
1581 0 : mode = widest_fixed_size_mode_for_size (size, m_op);
1582 0 : mode = get_usable_mode (mode, length);
1583 : }
1584 : }
1585 : while (1);
1586 : }
1587 :
1588 : /* Derived class from op_by_pieces_d, providing support for block move
1589 : operations. */
1590 :
1591 : #ifdef PUSH_ROUNDING
1592 : #define PUSHG_P(to) ((to) == nullptr)
1593 : #else
1594 : #define PUSHG_P(to) false
1595 : #endif
1596 :
1597 : class move_by_pieces_d : public op_by_pieces_d
1598 : {
1599 : insn_gen_fn m_gen_fun;
1600 : void generate (rtx, rtx, machine_mode) final override;
1601 : bool prepare_mode (machine_mode, unsigned int) final override;
1602 :
1603 : public:
1604 571138 : move_by_pieces_d (rtx to, rtx from, unsigned HOST_WIDE_INT len,
1605 : unsigned int align)
1606 571138 : : op_by_pieces_d (MOVE_MAX_PIECES, to, false, from, true, NULL,
1607 1142276 : NULL, len, align, PUSHG_P (to), MOVE_BY_PIECES)
1608 : {
1609 571138 : }
1610 : rtx finish_retmode (memop_ret);
1611 : };
1612 :
1613 : /* Return true if MODE can be used for a set of copies, given an
1614 : alignment ALIGN. Prepare whatever data is necessary for later
1615 : calls to generate. */
1616 :
1617 : bool
1618 952186 : move_by_pieces_d::prepare_mode (machine_mode mode, unsigned int align)
1619 : {
1620 952186 : insn_code icode = optab_handler (mov_optab, mode);
1621 952186 : m_gen_fun = GEN_FCN (icode);
1622 952186 : return icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode);
1623 : }
1624 :
1625 : /* A callback used when iterating for a compare_by_pieces_operation.
1626 : OP0 and OP1 are the values that have been loaded and should be
1627 : compared in MODE. If OP0 is NULL, this means we should generate a
1628 : push; otherwise EXTRA_DATA holds a pointer to a pointer to the insn
1629 : gen function that should be used to generate the mode. */
1630 :
1631 : void
1632 1654582 : move_by_pieces_d::generate (rtx op0, rtx op1,
1633 : machine_mode mode ATTRIBUTE_UNUSED)
1634 : {
1635 : #ifdef PUSH_ROUNDING
1636 1654582 : if (op0 == NULL_RTX)
1637 : {
1638 47 : emit_single_push_insn (mode, op1, NULL);
1639 47 : return;
1640 : }
1641 : #endif
1642 1654535 : emit_insn (m_gen_fun (op0, op1));
1643 : }
1644 :
1645 : /* Perform the final adjustment at the end of a string to obtain the
1646 : correct return value for the block operation.
1647 : Return value is based on RETMODE argument. */
1648 :
1649 : rtx
1650 0 : move_by_pieces_d::finish_retmode (memop_ret retmode)
1651 : {
1652 0 : gcc_assert (!m_reverse);
1653 0 : if (retmode == RETURN_END_MINUS_ONE)
1654 : {
1655 0 : m_to.maybe_postinc (-1);
1656 0 : --m_offset;
1657 : }
1658 0 : return m_to.adjust (QImode, m_offset);
1659 : }
1660 :
1661 : /* Generate several move instructions to copy LEN bytes from block FROM to
1662 : block TO. (These are MEM rtx's with BLKmode).
1663 :
1664 : If PUSH_ROUNDING is defined and TO is NULL, emit_single_push_insn is
1665 : used to push FROM to the stack.
1666 :
1667 : ALIGN is maximum stack alignment we can assume.
1668 :
1669 : Return value is based on RETMODE argument. */
1670 :
1671 : rtx
1672 571138 : move_by_pieces (rtx to, rtx from, unsigned HOST_WIDE_INT len,
1673 : unsigned int align, memop_ret retmode)
1674 : {
1675 : #ifndef PUSH_ROUNDING
1676 : if (to == NULL)
1677 : gcc_unreachable ();
1678 : #endif
1679 :
1680 571138 : move_by_pieces_d data (to, from, len, align);
1681 :
1682 571138 : data.run ();
1683 :
1684 571138 : if (retmode != RETURN_BEGIN)
1685 0 : return data.finish_retmode (retmode);
1686 : else
1687 : return to;
1688 : }
1689 :
1690 : /* Derived class from op_by_pieces_d, providing support for block move
1691 : operations. */
1692 :
1693 : class store_by_pieces_d : public op_by_pieces_d
1694 : {
1695 : insn_gen_fn m_gen_fun;
1696 :
1697 : void generate (rtx, rtx, machine_mode) final override;
1698 : bool prepare_mode (machine_mode, unsigned int) final override;
1699 :
1700 : public:
1701 125217 : store_by_pieces_d (rtx to, by_pieces_constfn cfn, void *cfn_data,
1702 : unsigned HOST_WIDE_INT len, unsigned int align,
1703 : by_pieces_operation op)
1704 125217 : : op_by_pieces_d (STORE_MAX_PIECES, to, false, NULL_RTX, true, cfn,
1705 250434 : cfn_data, len, align, false, op)
1706 : {
1707 125217 : }
1708 : rtx finish_retmode (memop_ret);
1709 : };
1710 :
1711 : /* Return true if MODE can be used for a set of stores, given an
1712 : alignment ALIGN. Prepare whatever data is necessary for later
1713 : calls to generate. */
1714 :
1715 : bool
1716 199657 : store_by_pieces_d::prepare_mode (machine_mode mode, unsigned int align)
1717 : {
1718 199657 : insn_code icode = optab_handler (mov_optab, mode);
1719 199657 : m_gen_fun = GEN_FCN (icode);
1720 199657 : return icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode);
1721 : }
1722 :
1723 : /* A callback used when iterating for a store_by_pieces_operation.
1724 : OP0 and OP1 are the values that have been loaded and should be
1725 : compared in MODE. If OP0 is NULL, this means we should generate a
1726 : push; otherwise EXTRA_DATA holds a pointer to a pointer to the insn
1727 : gen function that should be used to generate the mode. */
1728 :
1729 : void
1730 318846 : store_by_pieces_d::generate (rtx op0, rtx op1, machine_mode)
1731 : {
1732 318846 : emit_insn (m_gen_fun (op0, op1));
1733 318846 : }
1734 :
1735 : /* Perform the final adjustment at the end of a string to obtain the
1736 : correct return value for the block operation.
1737 : Return value is based on RETMODE argument. */
1738 :
1739 : rtx
1740 647 : store_by_pieces_d::finish_retmode (memop_ret retmode)
1741 : {
1742 647 : gcc_assert (!m_reverse);
1743 647 : if (retmode == RETURN_END_MINUS_ONE)
1744 : {
1745 23 : m_to.maybe_postinc (-1);
1746 23 : --m_offset;
1747 : }
1748 647 : return m_to.adjust (QImode, m_offset);
1749 : }
1750 :
1751 : /* Determine whether the LEN bytes generated by CONSTFUN can be
1752 : stored to memory using several move instructions. CONSTFUNDATA is
1753 : a pointer which will be passed as argument in every CONSTFUN call.
1754 : ALIGN is maximum alignment we can assume. MEMSETP is true if this is
1755 : a memset operation and false if it's a copy of a constant string.
1756 : Return true if a call to store_by_pieces should succeed. */
1757 :
1758 : bool
1759 91082 : can_store_by_pieces (unsigned HOST_WIDE_INT len,
1760 : by_pieces_constfn constfun,
1761 : void *constfundata, unsigned int align, bool memsetp)
1762 : {
1763 91082 : unsigned HOST_WIDE_INT l;
1764 91082 : unsigned int max_size;
1765 91082 : HOST_WIDE_INT offset = 0;
1766 91082 : enum insn_code icode;
1767 91082 : int reverse;
1768 : /* cst is set but not used if LEGITIMATE_CONSTANT doesn't use it. */
1769 91082 : rtx cst ATTRIBUTE_UNUSED;
1770 :
1771 91082 : if (len == 0)
1772 : return true;
1773 :
1774 137859 : if (!targetm.use_by_pieces_infrastructure_p (len, align,
1775 : memsetp
1776 : ? SET_BY_PIECES
1777 : : STORE_BY_PIECES,
1778 : optimize_insn_for_speed_p ()))
1779 : return false;
1780 :
1781 53966 : align = alignment_for_piecewise_move (STORE_MAX_PIECES, align);
1782 :
1783 : /* We would first store what we can in the largest integer mode, then go to
1784 : successively smaller modes. */
1785 :
1786 53966 : for (reverse = 0;
1787 107932 : reverse <= (HAVE_PRE_DECREMENT || HAVE_POST_DECREMENT);
1788 : reverse++)
1789 : {
1790 53966 : l = len;
1791 53990 : max_size = STORE_MAX_PIECES + 1;
1792 255488 : while (max_size > 1 && l > 0)
1793 : {
1794 201522 : auto op = memsetp ? SET_BY_PIECES : STORE_BY_PIECES;
1795 201522 : auto mode = widest_fixed_size_mode_for_size (max_size, op);
1796 :
1797 201522 : icode = optab_handler (mov_optab, mode);
1798 201522 : if (icode != CODE_FOR_nothing
1799 201522 : && align >= GET_MODE_ALIGNMENT (mode))
1800 : {
1801 201522 : unsigned int size = GET_MODE_SIZE (mode);
1802 :
1803 348710 : while (l >= size)
1804 : {
1805 147188 : if (reverse)
1806 : offset -= size;
1807 :
1808 147188 : cst = (*constfun) (constfundata, nullptr, offset, mode);
1809 : /* All CONST_VECTORs can be loaded for memset since
1810 : vec_duplicate_optab is a precondition to pick a
1811 : vector mode for the memset expander. */
1812 273589 : if (!((memsetp && VECTOR_MODE_P (mode))
1813 126401 : || targetm.legitimate_constant_p (mode, cst)))
1814 91082 : return false;
1815 :
1816 147188 : if (!reverse)
1817 147188 : offset += size;
1818 :
1819 147188 : l -= size;
1820 : }
1821 : }
1822 :
1823 403044 : max_size = GET_MODE_SIZE (mode);
1824 : }
1825 :
1826 : /* The code above should have handled everything. */
1827 53966 : gcc_assert (!l);
1828 : }
1829 :
1830 : return true;
1831 : }
1832 :
1833 : /* Generate several move instructions to store LEN bytes generated by
1834 : CONSTFUN to block TO. (A MEM rtx with BLKmode). CONSTFUNDATA is a
1835 : pointer which will be passed as argument in every CONSTFUN call.
1836 : ALIGN is maximum alignment we can assume. MEMSETP is true if this is
1837 : a memset operation and false if it's a copy of a constant string.
1838 : Return value is based on RETMODE argument. */
1839 :
1840 : rtx
1841 61177 : store_by_pieces (rtx to, unsigned HOST_WIDE_INT len,
1842 : by_pieces_constfn constfun,
1843 : void *constfundata, unsigned int align, bool memsetp,
1844 : memop_ret retmode)
1845 : {
1846 61177 : if (len == 0)
1847 : {
1848 7949 : gcc_assert (retmode != RETURN_END_MINUS_ONE);
1849 : return to;
1850 : }
1851 :
1852 95850 : gcc_assert (targetm.use_by_pieces_infrastructure_p
1853 : (len, align,
1854 : memsetp ? SET_BY_PIECES : STORE_BY_PIECES,
1855 : optimize_insn_for_speed_p ()));
1856 :
1857 53228 : store_by_pieces_d data (to, constfun, constfundata, len, align,
1858 53228 : memsetp ? SET_BY_PIECES : STORE_BY_PIECES);
1859 53228 : data.run ();
1860 :
1861 53228 : if (retmode != RETURN_BEGIN)
1862 647 : return data.finish_retmode (retmode);
1863 : else
1864 : return to;
1865 : }
1866 :
1867 : void
1868 71989 : clear_by_pieces (rtx to, unsigned HOST_WIDE_INT len, unsigned int align)
1869 : {
1870 71989 : if (len == 0)
1871 : return;
1872 :
1873 : /* Use builtin_memset_read_str to support vector mode broadcast. */
1874 71989 : char c = 0;
1875 71989 : store_by_pieces_d data (to, builtin_memset_read_str, &c, len, align,
1876 71989 : CLEAR_BY_PIECES);
1877 71989 : data.run ();
1878 : }
1879 :
1880 : /* Context used by compare_by_pieces_genfn. It stores the fail label
1881 : to jump to in case of miscomparison, and for branch ratios greater than 1,
1882 : it stores an accumulator and the current and maximum counts before
1883 : emitting another branch. */
1884 :
1885 : class compare_by_pieces_d : public op_by_pieces_d
1886 : {
1887 : rtx_code_label *m_fail_label;
1888 : rtx m_accumulator;
1889 : int m_count, m_batch;
1890 :
1891 : void generate (rtx, rtx, machine_mode) final override;
1892 : bool prepare_mode (machine_mode, unsigned int) final override;
1893 : void finish_mode (machine_mode) final override;
1894 :
1895 : public:
1896 62850 : compare_by_pieces_d (rtx op0, rtx op1, by_pieces_constfn op1_cfn,
1897 : void *op1_cfn_data, HOST_WIDE_INT len, int align,
1898 : rtx_code_label *fail_label)
1899 62850 : : op_by_pieces_d (COMPARE_MAX_PIECES, op0, true, op1, true, op1_cfn,
1900 125700 : op1_cfn_data, len, align, false, COMPARE_BY_PIECES)
1901 : {
1902 62850 : m_fail_label = fail_label;
1903 62850 : }
1904 : };
1905 :
1906 : /* A callback used when iterating for a compare_by_pieces_operation.
1907 : OP0 and OP1 are the values that have been loaded and should be
1908 : compared in MODE. DATA holds a pointer to the compare_by_pieces_data
1909 : context structure. */
1910 :
1911 : void
1912 98051 : compare_by_pieces_d::generate (rtx op0, rtx op1, machine_mode mode)
1913 : {
1914 98051 : if (m_batch > 1)
1915 : {
1916 0 : rtx temp = expand_binop (mode, sub_optab, op0, op1, NULL_RTX,
1917 : true, OPTAB_LIB_WIDEN);
1918 0 : if (m_count != 0)
1919 0 : temp = expand_binop (mode, ior_optab, m_accumulator, temp, temp,
1920 : true, OPTAB_LIB_WIDEN);
1921 0 : m_accumulator = temp;
1922 :
1923 0 : if (++m_count < m_batch)
1924 : return;
1925 :
1926 0 : m_count = 0;
1927 0 : op0 = m_accumulator;
1928 0 : op1 = const0_rtx;
1929 0 : m_accumulator = NULL_RTX;
1930 : }
1931 98051 : do_compare_rtx_and_jump (op0, op1, NE, true, mode, NULL_RTX, NULL,
1932 : m_fail_label, profile_probability::uninitialized ());
1933 : }
1934 :
1935 : /* Return true if MODE can be used for a set of moves and comparisons,
1936 : given an alignment ALIGN. Prepare whatever data is necessary for
1937 : later calls to generate. */
1938 :
1939 : bool
1940 94794 : compare_by_pieces_d::prepare_mode (machine_mode mode, unsigned int align)
1941 : {
1942 94794 : insn_code icode = optab_handler (mov_optab, mode);
1943 94794 : if (icode == CODE_FOR_nothing
1944 94794 : || align < GET_MODE_ALIGNMENT (mode)
1945 189588 : || !can_compare_p (EQ, mode, ccp_jump))
1946 : return false;
1947 94794 : m_batch = targetm.compare_by_pieces_branch_ratio (mode);
1948 94794 : if (m_batch < 0)
1949 : return false;
1950 94794 : m_accumulator = NULL_RTX;
1951 94794 : m_count = 0;
1952 94794 : return true;
1953 : }
1954 :
1955 : /* Called after expanding a series of comparisons in MODE. If we have
1956 : accumulated results for which we haven't emitted a branch yet, do
1957 : so now. */
1958 :
1959 : void
1960 94794 : compare_by_pieces_d::finish_mode (machine_mode mode)
1961 : {
1962 94794 : if (m_accumulator != NULL_RTX)
1963 0 : do_compare_rtx_and_jump (m_accumulator, const0_rtx, NE, true, mode,
1964 : NULL_RTX, NULL, m_fail_label,
1965 : profile_probability::uninitialized ());
1966 94794 : }
1967 :
1968 : /* Generate several move instructions to compare LEN bytes from blocks
1969 : ARG0 and ARG1. (These are MEM rtx's with BLKmode).
1970 :
1971 : If PUSH_ROUNDING is defined and TO is NULL, emit_single_push_insn is
1972 : used to push FROM to the stack.
1973 :
1974 : ALIGN is maximum stack alignment we can assume.
1975 :
1976 : Optionally, the caller can pass a constfn and associated data in A1_CFN
1977 : and A1_CFN_DATA. describing that the second operand being compared is a
1978 : known constant and how to obtain its data. */
1979 :
1980 : static rtx
1981 62850 : compare_by_pieces (rtx arg0, rtx arg1, unsigned HOST_WIDE_INT len,
1982 : rtx target, unsigned int align,
1983 : by_pieces_constfn a1_cfn, void *a1_cfn_data)
1984 : {
1985 62850 : rtx_code_label *fail_label = gen_label_rtx ();
1986 62850 : rtx_code_label *end_label = gen_label_rtx ();
1987 :
1988 62850 : if (target == NULL_RTX
1989 62850 : || !REG_P (target) || REGNO (target) < FIRST_PSEUDO_REGISTER)
1990 2 : target = gen_reg_rtx (TYPE_MODE (integer_type_node));
1991 :
1992 62850 : compare_by_pieces_d data (arg0, arg1, a1_cfn, a1_cfn_data, len, align,
1993 62850 : fail_label);
1994 :
1995 62850 : data.run ();
1996 :
1997 62850 : emit_move_insn (target, const0_rtx);
1998 62850 : emit_jump (end_label);
1999 62850 : emit_barrier ();
2000 62850 : emit_label (fail_label);
2001 62850 : emit_move_insn (target, const1_rtx);
2002 62850 : emit_label (end_label);
2003 :
2004 62850 : return target;
2005 : }
2006 :
2007 : /* Emit code to move a block Y to a block X. This may be done with
2008 : string-move instructions, with multiple scalar move instructions,
2009 : or with a library call.
2010 :
2011 : Both X and Y must be MEM rtx's (perhaps inside VOLATILE) with mode BLKmode.
2012 : SIZE is an rtx that says how long they are.
2013 : ALIGN is the maximum alignment we can assume they have.
2014 : METHOD describes what kind of copy this is, and what mechanisms may be used.
2015 : MIN_SIZE is the minimal size of block to move
2016 : MAX_SIZE is the maximal size of block to move, if it cannot be represented
2017 : in unsigned HOST_WIDE_INT, than it is mask of all ones.
2018 : CTZ_SIZE is the trailing-zeros count of SIZE; even a nonconstant SIZE is
2019 : known to be a multiple of 1<<CTZ_SIZE.
2020 :
2021 : Return the address of the new block, if memcpy is called and returns it,
2022 : 0 otherwise. */
2023 :
2024 : rtx
2025 671303 : emit_block_move_hints (rtx x, rtx y, rtx size, enum block_op_methods method,
2026 : unsigned int expected_align, HOST_WIDE_INT expected_size,
2027 : unsigned HOST_WIDE_INT min_size,
2028 : unsigned HOST_WIDE_INT max_size,
2029 : unsigned HOST_WIDE_INT probable_max_size,
2030 : bool bail_out_libcall, bool *is_move_done,
2031 : bool might_overlap, unsigned ctz_size)
2032 : {
2033 671303 : int may_use_call;
2034 671303 : rtx retval = 0;
2035 671303 : unsigned int align;
2036 :
2037 671303 : if (is_move_done)
2038 98966 : *is_move_done = true;
2039 :
2040 671303 : gcc_assert (size);
2041 671303 : if (CONST_INT_P (size) && INTVAL (size) == 0)
2042 : return 0;
2043 :
2044 671026 : switch (method)
2045 : {
2046 : case BLOCK_OP_NORMAL:
2047 : case BLOCK_OP_TAILCALL:
2048 : may_use_call = 1;
2049 : break;
2050 :
2051 269022 : case BLOCK_OP_CALL_PARM:
2052 269022 : may_use_call = block_move_libcall_safe_for_call_parm ();
2053 :
2054 : /* Make inhibit_defer_pop nonzero around the library call
2055 : to force it to pop the arguments right away. */
2056 269022 : NO_DEFER_POP;
2057 269022 : break;
2058 :
2059 354 : case BLOCK_OP_NO_LIBCALL:
2060 354 : may_use_call = 0;
2061 354 : break;
2062 :
2063 1074 : case BLOCK_OP_NO_LIBCALL_RET:
2064 1074 : may_use_call = -1;
2065 1074 : break;
2066 :
2067 0 : default:
2068 0 : gcc_unreachable ();
2069 : }
2070 :
2071 671026 : gcc_assert (MEM_P (x) && MEM_P (y));
2072 671111 : align = MIN (MEM_ALIGN (x), MEM_ALIGN (y));
2073 671026 : gcc_assert (align >= BITS_PER_UNIT);
2074 :
2075 : /* Make sure we've got BLKmode addresses; store_one_arg can decide that
2076 : block copy is more efficient for other large modes, e.g. DCmode. */
2077 671026 : x = adjust_address (x, BLKmode, 0);
2078 671026 : y = adjust_address (y, BLKmode, 0);
2079 :
2080 : /* If source and destination are the same, no need to copy anything. */
2081 671026 : if (rtx_equal_p (x, y)
2082 13 : && !MEM_VOLATILE_P (x)
2083 671039 : && !MEM_VOLATILE_P (y))
2084 : return 0;
2085 :
2086 : /* Set MEM_SIZE as appropriate for this block copy. The main place this
2087 : can be incorrect is coming from __builtin_memcpy. */
2088 671013 : poly_int64 const_size;
2089 671013 : if (poly_int_rtx_p (size, &const_size))
2090 : {
2091 591156 : x = shallow_copy_rtx (x);
2092 591156 : y = shallow_copy_rtx (y);
2093 591156 : set_mem_size (x, const_size);
2094 591156 : set_mem_size (y, const_size);
2095 : }
2096 :
2097 671013 : bool pieces_ok = CONST_INT_P (size)
2098 671013 : && can_move_by_pieces (INTVAL (size), align);
2099 671013 : bool pattern_ok = false;
2100 :
2101 671013 : if (!pieces_ok || might_overlap)
2102 : {
2103 106855 : pattern_ok
2104 106855 : = emit_block_move_via_pattern (x, y, size, align,
2105 : expected_align, expected_size,
2106 : min_size, max_size, probable_max_size,
2107 : might_overlap);
2108 106855 : if (!pattern_ok && might_overlap)
2109 : {
2110 : /* Do not try any of the other methods below as they are not safe
2111 : for overlapping moves. */
2112 12772 : *is_move_done = false;
2113 12772 : return retval;
2114 : }
2115 : }
2116 :
2117 94083 : bool dynamic_direction = false;
2118 94083 : if (!pattern_ok && !pieces_ok && may_use_call
2119 137120 : && (flag_inline_stringops & (might_overlap ? ILSOP_MEMMOVE : ILSOP_MEMCPY)))
2120 : {
2121 658241 : may_use_call = 0;
2122 658241 : dynamic_direction = might_overlap;
2123 : }
2124 :
2125 658241 : if (pattern_ok)
2126 : ;
2127 632718 : else if (pieces_ok)
2128 564158 : move_by_pieces (x, y, INTVAL (size), align, RETURN_BEGIN);
2129 68560 : else if (may_use_call && !might_overlap
2130 68512 : && ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (x))
2131 137072 : && ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (y)))
2132 : {
2133 68512 : if (bail_out_libcall)
2134 : {
2135 259 : if (is_move_done)
2136 259 : *is_move_done = false;
2137 : return retval;
2138 : }
2139 :
2140 68253 : if (may_use_call < 0)
2141 0 : return pc_rtx;
2142 :
2143 68253 : retval = emit_block_copy_via_libcall (x, y, size,
2144 : method == BLOCK_OP_TAILCALL);
2145 : }
2146 48 : else if (dynamic_direction)
2147 0 : emit_block_move_via_oriented_loop (x, y, size, align, ctz_size);
2148 48 : else if (might_overlap)
2149 0 : *is_move_done = false;
2150 : else
2151 48 : emit_block_move_via_sized_loop (x, y, size, align, ctz_size);
2152 :
2153 657982 : if (method == BLOCK_OP_CALL_PARM)
2154 269011 : OK_DEFER_POP;
2155 :
2156 : return retval;
2157 : }
2158 :
2159 : rtx
2160 572337 : emit_block_move (rtx x, rtx y, rtx size, enum block_op_methods method,
2161 : unsigned int ctz_size)
2162 : {
2163 572337 : unsigned HOST_WIDE_INT max, min = 0;
2164 572337 : if (GET_CODE (size) == CONST_INT)
2165 572117 : min = max = UINTVAL (size);
2166 : else
2167 220 : max = GET_MODE_MASK (GET_MODE (size));
2168 572337 : return emit_block_move_hints (x, y, size, method, 0, -1,
2169 : min, max, max,
2170 572337 : false, NULL, false, ctz_size);
2171 : }
2172 :
2173 : /* A subroutine of emit_block_move. Returns true if calling the
2174 : block move libcall will not clobber any parameters which may have
2175 : already been placed on the stack. */
2176 :
2177 : static bool
2178 269022 : block_move_libcall_safe_for_call_parm (void)
2179 : {
2180 269022 : tree fn;
2181 :
2182 : /* If arguments are pushed on the stack, then they're safe. */
2183 269022 : if (targetm.calls.push_argument (0))
2184 : return true;
2185 :
2186 : /* If registers go on the stack anyway, any argument is sure to clobber
2187 : an outgoing argument. */
2188 : #if defined (REG_PARM_STACK_SPACE)
2189 3 : fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
2190 : /* Avoid set but not used warning if *REG_PARM_STACK_SPACE doesn't
2191 : depend on its argument. */
2192 3 : (void) fn;
2193 3 : if (OUTGOING_REG_PARM_STACK_SPACE ((!fn ? NULL_TREE : TREE_TYPE (fn)))
2194 3 : && REG_PARM_STACK_SPACE (fn) != 0)
2195 : return false;
2196 : #endif
2197 :
2198 : /* If any argument goes in memory, then it might clobber an outgoing
2199 : argument. */
2200 3 : {
2201 3 : CUMULATIVE_ARGS args_so_far_v;
2202 3 : cumulative_args_t args_so_far;
2203 3 : tree arg;
2204 :
2205 3 : fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
2206 3 : INIT_CUMULATIVE_ARGS (args_so_far_v, TREE_TYPE (fn), NULL_RTX, 0, 3);
2207 3 : args_so_far = pack_cumulative_args (&args_so_far_v);
2208 :
2209 3 : arg = TYPE_ARG_TYPES (TREE_TYPE (fn));
2210 12 : for ( ; arg != void_list_node ; arg = TREE_CHAIN (arg))
2211 : {
2212 9 : machine_mode mode = TYPE_MODE (TREE_VALUE (arg));
2213 9 : function_arg_info arg_info (mode, /*named=*/true);
2214 9 : rtx tmp = targetm.calls.function_arg (args_so_far, arg_info);
2215 9 : if (!tmp || !REG_P (tmp))
2216 0 : return false;
2217 9 : if (targetm.calls.arg_partial_bytes (args_so_far, arg_info))
2218 : return false;
2219 9 : targetm.calls.function_arg_advance (args_so_far, arg_info);
2220 : }
2221 : }
2222 3 : return true;
2223 : }
2224 :
2225 : /* A subroutine of emit_block_move. Expand a cpymem or movmem pattern;
2226 : return true if successful.
2227 :
2228 : X is the destination of the copy or move.
2229 : Y is the source of the copy or move.
2230 : SIZE is the size of the block to be moved.
2231 :
2232 : MIGHT_OVERLAP indicates this originated with expansion of a
2233 : builtin_memmove() and the source and destination blocks may
2234 : overlap.
2235 : */
2236 :
2237 : static bool
2238 106855 : emit_block_move_via_pattern (rtx x, rtx y, rtx size, unsigned int align,
2239 : unsigned int expected_align,
2240 : HOST_WIDE_INT expected_size,
2241 : unsigned HOST_WIDE_INT min_size,
2242 : unsigned HOST_WIDE_INT max_size,
2243 : unsigned HOST_WIDE_INT probable_max_size,
2244 : bool might_overlap)
2245 : {
2246 106855 : if (expected_align < align)
2247 : expected_align = align;
2248 106855 : if (expected_size != -1)
2249 : {
2250 22 : if ((unsigned HOST_WIDE_INT)expected_size > probable_max_size)
2251 0 : expected_size = probable_max_size;
2252 22 : if ((unsigned HOST_WIDE_INT)expected_size < min_size)
2253 0 : expected_size = min_size;
2254 : }
2255 :
2256 : /* Since this is a move insn, we don't care about volatility. */
2257 106855 : temporary_volatile_ok v (true);
2258 :
2259 : /* Try the most limited insn first, because there's no point
2260 : including more than one in the machine description unless
2261 : the more limited one has some advantage. */
2262 :
2263 106855 : opt_scalar_int_mode mode_iter;
2264 731215 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
2265 : {
2266 649883 : scalar_int_mode mode = mode_iter.require ();
2267 649883 : enum insn_code code;
2268 649883 : if (might_overlap)
2269 103583 : code = direct_optab_handler (movmem_optab, mode);
2270 : else
2271 546300 : code = direct_optab_handler (cpymem_optab, mode);
2272 :
2273 649883 : if (code != CODE_FOR_nothing
2274 : /* We don't need MODE to be narrower than BITS_PER_HOST_WIDE_INT
2275 : here because if SIZE is less than the mode mask, as it is
2276 : returned by the macro, it will definitely be less than the
2277 : actual mode mask. Since SIZE is within the Pmode address
2278 : space, we limit MODE to Pmode. */
2279 649883 : && ((CONST_INT_P (size)
2280 41855 : && ((unsigned HOST_WIDE_INT) INTVAL (size)
2281 41855 : <= (GET_MODE_MASK (mode) >> 1)))
2282 145282 : || max_size <= (GET_MODE_MASK (mode) >> 1)
2283 181796 : || GET_MODE_BITSIZE (mode) >= GET_MODE_BITSIZE (Pmode)))
2284 : {
2285 123622 : class expand_operand ops[9];
2286 123622 : unsigned int nops;
2287 :
2288 : /* ??? When called via emit_block_move_for_call, it'd be
2289 : nice if there were some way to inform the backend, so
2290 : that it doesn't fail the expansion because it thinks
2291 : emitting the libcall would be more efficient. */
2292 123622 : nops = insn_data[(int) code].n_generator_args;
2293 123622 : gcc_assert (nops == 4 || nops == 6 || nops == 8 || nops == 9);
2294 :
2295 123622 : create_fixed_operand (&ops[0], x);
2296 123622 : create_fixed_operand (&ops[1], y);
2297 : /* The check above guarantees that this size conversion is valid. */
2298 123622 : create_convert_operand_to (&ops[2], size, mode, true);
2299 123622 : create_integer_operand (&ops[3], align / BITS_PER_UNIT);
2300 123622 : if (nops >= 6)
2301 : {
2302 123622 : create_integer_operand (&ops[4], expected_align / BITS_PER_UNIT);
2303 123622 : create_integer_operand (&ops[5], expected_size);
2304 : }
2305 123622 : if (nops >= 8)
2306 : {
2307 123622 : create_integer_operand (&ops[6], min_size);
2308 : /* If we cannot represent the maximal size,
2309 : make parameter NULL. */
2310 123622 : if ((HOST_WIDE_INT) max_size != -1)
2311 106001 : create_integer_operand (&ops[7], max_size);
2312 : else
2313 17621 : create_fixed_operand (&ops[7], NULL);
2314 : }
2315 123622 : if (nops == 9)
2316 : {
2317 : /* If we cannot represent the maximal size,
2318 : make parameter NULL. */
2319 123622 : if ((HOST_WIDE_INT) probable_max_size != -1)
2320 108089 : create_integer_operand (&ops[8], probable_max_size);
2321 : else
2322 15533 : create_fixed_operand (&ops[8], NULL);
2323 : }
2324 123622 : gcc_assert (min_size != max_size
2325 : || rtx_equal_p (ops[2].value, GEN_INT (min_size)));
2326 123622 : if (maybe_expand_insn (code, nops, ops))
2327 25523 : return true;
2328 : }
2329 : }
2330 :
2331 : return false;
2332 106855 : }
2333 :
2334 : /* Like emit_block_move_via_loop, but choose a suitable INCR based on
2335 : ALIGN and CTZ_SIZE. */
2336 :
2337 : static void
2338 48 : emit_block_move_via_sized_loop (rtx x, rtx y, rtx size,
2339 : unsigned int align,
2340 : unsigned int ctz_size)
2341 : {
2342 48 : int incr = align / BITS_PER_UNIT;
2343 :
2344 48 : if (CONST_INT_P (size))
2345 0 : ctz_size = MAX (ctz_size, (unsigned) wi::ctz (UINTVAL (size)));
2346 :
2347 48 : if (HOST_WIDE_INT_1U << ctz_size < (unsigned HOST_WIDE_INT) incr)
2348 0 : incr = HOST_WIDE_INT_1U << ctz_size;
2349 :
2350 48 : while (incr > 1 && !can_move_by_pieces (incr, align))
2351 0 : incr >>= 1;
2352 :
2353 48 : gcc_checking_assert (incr);
2354 :
2355 48 : return emit_block_move_via_loop (x, y, size, align, incr);
2356 : }
2357 :
2358 : /* Like emit_block_move_via_sized_loop, but besides choosing INCR so
2359 : as to ensure safe moves even in case of overlap, output dynamic
2360 : tests to choose between two loops, one moving downwards, another
2361 : moving upwards. */
2362 :
2363 : static void
2364 0 : emit_block_move_via_oriented_loop (rtx x, rtx y, rtx size,
2365 : unsigned int align,
2366 : unsigned int ctz_size)
2367 : {
2368 0 : int incr = align / BITS_PER_UNIT;
2369 :
2370 0 : if (CONST_INT_P (size))
2371 0 : ctz_size = MAX (ctz_size, (unsigned) wi::ctz (UINTVAL (size)));
2372 :
2373 0 : if (HOST_WIDE_INT_1U << ctz_size < (unsigned HOST_WIDE_INT) incr)
2374 0 : incr = HOST_WIDE_INT_1U << ctz_size;
2375 :
2376 0 : while (incr > 1 && !int_mode_for_size (incr, 0).exists ())
2377 0 : incr >>= 1;
2378 :
2379 0 : gcc_checking_assert (incr);
2380 :
2381 0 : rtx_code_label *upw_label, *end_label;
2382 0 : upw_label = gen_label_rtx ();
2383 0 : end_label = gen_label_rtx ();
2384 :
2385 0 : rtx x_addr = force_operand (XEXP (x, 0), NULL_RTX);
2386 0 : rtx y_addr = force_operand (XEXP (y, 0), NULL_RTX);
2387 0 : do_pending_stack_adjust ();
2388 :
2389 0 : machine_mode mode = GET_MODE (x_addr);
2390 0 : if (mode != GET_MODE (y_addr))
2391 : {
2392 0 : scalar_int_mode xmode
2393 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE (mode)).require ();
2394 0 : scalar_int_mode ymode
2395 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE
2396 0 : (GET_MODE (y_addr))).require ();
2397 0 : if (GET_MODE_BITSIZE (xmode) < GET_MODE_BITSIZE (ymode))
2398 : mode = ymode;
2399 : else
2400 0 : mode = xmode;
2401 :
2402 : #ifndef POINTERS_EXTEND_UNSIGNED
2403 : const int POINTERS_EXTEND_UNSIGNED = 1;
2404 : #endif
2405 0 : x_addr = convert_modes (mode, GET_MODE (x_addr), x_addr,
2406 : POINTERS_EXTEND_UNSIGNED);
2407 0 : y_addr = convert_modes (mode, GET_MODE (y_addr), y_addr,
2408 : POINTERS_EXTEND_UNSIGNED);
2409 : }
2410 :
2411 : /* Test for overlap: if (x >= y || x + size <= y) goto upw_label. */
2412 0 : emit_cmp_and_jump_insns (x_addr, y_addr, GEU, NULL_RTX, mode,
2413 : true, upw_label,
2414 0 : profile_probability::guessed_always ()
2415 : .apply_scale (5, 10));
2416 0 : rtx tmp = convert_modes (GET_MODE (x_addr), GET_MODE (size), size, true);
2417 0 : tmp = simplify_gen_binary (PLUS, GET_MODE (x_addr), x_addr, tmp);
2418 :
2419 0 : emit_cmp_and_jump_insns (tmp, y_addr, LEU, NULL_RTX, mode,
2420 : true, upw_label,
2421 0 : profile_probability::guessed_always ()
2422 : .apply_scale (8, 10));
2423 :
2424 0 : emit_block_move_via_loop (x, y, size, align, -incr);
2425 :
2426 0 : emit_jump (end_label);
2427 0 : emit_label (upw_label);
2428 :
2429 0 : emit_block_move_via_loop (x, y, size, align, incr);
2430 :
2431 0 : emit_label (end_label);
2432 0 : }
2433 :
2434 : /* A subroutine of emit_block_move. Copy the data via an explicit
2435 : loop. This is used only when libcalls are forbidden, or when
2436 : inlining is required. INCR is the block size to be copied in each
2437 : loop iteration. If it is negative, the absolute value is used, and
2438 : the block is copied backwards. INCR must be a power of two, an
2439 : exact divisor for SIZE and ALIGN, and imply a mode that can be
2440 : safely copied per iteration assuming no overlap. */
2441 :
2442 : static void
2443 48 : emit_block_move_via_loop (rtx x, rtx y, rtx size,
2444 : unsigned int align, int incr)
2445 : {
2446 48 : rtx_code_label *cmp_label, *top_label;
2447 48 : rtx iter, x_addr, y_addr, tmp;
2448 48 : machine_mode x_addr_mode = get_address_mode (x);
2449 48 : machine_mode y_addr_mode = get_address_mode (y);
2450 48 : machine_mode iter_mode;
2451 :
2452 48 : iter_mode = GET_MODE (size);
2453 48 : if (iter_mode == VOIDmode)
2454 0 : iter_mode = word_mode;
2455 :
2456 48 : top_label = gen_label_rtx ();
2457 48 : cmp_label = gen_label_rtx ();
2458 48 : iter = gen_reg_rtx (iter_mode);
2459 :
2460 48 : bool downwards = incr < 0;
2461 48 : rtx iter_init;
2462 48 : rtx_code iter_cond;
2463 48 : rtx iter_limit;
2464 48 : rtx iter_incr;
2465 48 : machine_mode move_mode;
2466 48 : if (downwards)
2467 : {
2468 0 : incr = -incr;
2469 0 : iter_init = size;
2470 0 : iter_cond = GEU;
2471 0 : iter_limit = const0_rtx;
2472 0 : iter_incr = GEN_INT (incr);
2473 : }
2474 : else
2475 : {
2476 48 : iter_init = const0_rtx;
2477 48 : iter_cond = LTU;
2478 48 : iter_limit = size;
2479 48 : iter_incr = GEN_INT (incr);
2480 : }
2481 48 : emit_move_insn (iter, iter_init);
2482 :
2483 48 : opt_scalar_int_mode int_move_mode
2484 48 : = int_mode_for_size (incr * BITS_PER_UNIT, 1);
2485 48 : if (!int_move_mode.exists (&move_mode)
2486 96 : || GET_MODE_BITSIZE (int_move_mode.require ()) != incr * BITS_PER_UNIT)
2487 : {
2488 0 : move_mode = BLKmode;
2489 0 : gcc_checking_assert (can_move_by_pieces (incr, align));
2490 : }
2491 :
2492 48 : x_addr = force_operand (XEXP (x, 0), NULL_RTX);
2493 48 : y_addr = force_operand (XEXP (y, 0), NULL_RTX);
2494 48 : do_pending_stack_adjust ();
2495 :
2496 48 : emit_jump (cmp_label);
2497 48 : emit_label (top_label);
2498 :
2499 48 : tmp = convert_modes (x_addr_mode, iter_mode, iter, true);
2500 48 : x_addr = simplify_gen_binary (PLUS, x_addr_mode, x_addr, tmp);
2501 :
2502 48 : if (x_addr_mode != y_addr_mode)
2503 0 : tmp = convert_modes (y_addr_mode, iter_mode, iter, true);
2504 48 : y_addr = simplify_gen_binary (PLUS, y_addr_mode, y_addr, tmp);
2505 :
2506 48 : x = change_address (x, move_mode, x_addr);
2507 48 : y = change_address (y, move_mode, y_addr);
2508 :
2509 48 : if (move_mode == BLKmode)
2510 : {
2511 0 : bool done;
2512 0 : emit_block_move_hints (x, y, iter_incr, BLOCK_OP_NO_LIBCALL,
2513 : align, incr, incr, incr, incr,
2514 : false, &done, false);
2515 0 : gcc_checking_assert (done);
2516 : }
2517 : else
2518 48 : emit_move_insn (x, y);
2519 :
2520 48 : if (downwards)
2521 0 : emit_label (cmp_label);
2522 :
2523 48 : tmp = expand_simple_binop (iter_mode, PLUS, iter, iter_incr, iter,
2524 : true, OPTAB_LIB_WIDEN);
2525 48 : if (tmp != iter)
2526 0 : emit_move_insn (iter, tmp);
2527 :
2528 48 : if (!downwards)
2529 48 : emit_label (cmp_label);
2530 :
2531 48 : emit_cmp_and_jump_insns (iter, iter_limit, iter_cond, NULL_RTX, iter_mode,
2532 : true, top_label,
2533 48 : profile_probability::guessed_always ()
2534 : .apply_scale (9, 10));
2535 48 : }
2536 :
2537 : /* Expand a call to memcpy or memmove or memcmp, and return the result.
2538 : TAILCALL is true if this is a tail call. */
2539 :
2540 : rtx
2541 68254 : emit_block_op_via_libcall (enum built_in_function fncode, rtx dst, rtx src,
2542 : rtx size, bool tailcall)
2543 : {
2544 68254 : rtx dst_addr, src_addr;
2545 68254 : tree call_expr, dst_tree, src_tree, size_tree;
2546 68254 : machine_mode size_mode;
2547 :
2548 : /* Since dst and src are passed to a libcall, mark the corresponding
2549 : tree EXPR as addressable. */
2550 68254 : tree dst_expr = MEM_EXPR (dst);
2551 68254 : tree src_expr = MEM_EXPR (src);
2552 68254 : if (dst_expr)
2553 61535 : mark_addressable (dst_expr);
2554 68254 : if (src_expr)
2555 66842 : mark_addressable (src_expr);
2556 :
2557 68254 : dst_addr = copy_addr_to_reg (XEXP (dst, 0));
2558 68254 : dst_addr = convert_memory_address (ptr_mode, dst_addr);
2559 68254 : dst_tree = make_tree (ptr_type_node, dst_addr);
2560 :
2561 68254 : src_addr = copy_addr_to_reg (XEXP (src, 0));
2562 68254 : src_addr = convert_memory_address (ptr_mode, src_addr);
2563 68254 : src_tree = make_tree (ptr_type_node, src_addr);
2564 :
2565 68254 : size_mode = TYPE_MODE (sizetype);
2566 68254 : size = convert_to_mode (size_mode, size, 1);
2567 68254 : size = copy_to_mode_reg (size_mode, size);
2568 68254 : size_tree = make_tree (sizetype, size);
2569 :
2570 : /* It is incorrect to use the libcall calling conventions for calls to
2571 : memcpy/memmove/memcmp because they can be provided by the user. */
2572 68254 : tree fn = builtin_decl_implicit (fncode);
2573 68254 : call_expr = build_call_expr (fn, 3, dst_tree, src_tree, size_tree);
2574 68254 : CALL_EXPR_TAILCALL (call_expr) = tailcall;
2575 :
2576 68254 : return expand_call (call_expr, NULL_RTX, false);
2577 : }
2578 :
2579 : /* Try to expand cmpstrn or cmpmem operation ICODE with the given operands.
2580 : ARG3_TYPE is the type of ARG3_RTX. Return the result rtx on success,
2581 : otherwise return null. */
2582 :
2583 : rtx
2584 171564 : expand_cmpstrn_or_cmpmem (insn_code icode, rtx target, rtx arg1_rtx,
2585 : rtx arg2_rtx, tree arg3_type, rtx arg3_rtx,
2586 : HOST_WIDE_INT align)
2587 : {
2588 171564 : machine_mode insn_mode = insn_data[icode].operand[0].mode;
2589 :
2590 171564 : if (target && (!REG_P (target) || HARD_REGISTER_P (target)))
2591 : target = NULL_RTX;
2592 :
2593 171564 : class expand_operand ops[5];
2594 171564 : create_output_operand (&ops[0], target, insn_mode);
2595 171564 : create_fixed_operand (&ops[1], arg1_rtx);
2596 171564 : create_fixed_operand (&ops[2], arg2_rtx);
2597 171564 : create_convert_operand_from (&ops[3], arg3_rtx, TYPE_MODE (arg3_type),
2598 171564 : TYPE_UNSIGNED (arg3_type));
2599 171564 : create_integer_operand (&ops[4], align);
2600 171564 : if (maybe_expand_insn (icode, 5, ops))
2601 5796 : return ops[0].value;
2602 : return NULL_RTX;
2603 : }
2604 :
2605 : /* Expand a block compare between X and Y with length LEN using the
2606 : cmpmem optab, placing the result in TARGET. LEN_TYPE is the type
2607 : of the expression that was used to calculate the length. ALIGN
2608 : gives the known minimum common alignment. */
2609 :
2610 : static rtx
2611 43626 : emit_block_cmp_via_cmpmem (rtx x, rtx y, rtx len, tree len_type, rtx target,
2612 : unsigned align)
2613 : {
2614 : /* Note: The cmpstrnsi pattern, if it exists, is not suitable for
2615 : implementing memcmp because it will stop if it encounters two
2616 : zero bytes. */
2617 43626 : insn_code icode = direct_optab_handler (cmpmem_optab, SImode);
2618 :
2619 43626 : if (icode == CODE_FOR_nothing)
2620 : return NULL_RTX;
2621 :
2622 43626 : return expand_cmpstrn_or_cmpmem (icode, target, x, y, len_type, len, align);
2623 : }
2624 :
2625 : /* Emit code to compare a block Y to a block X. This may be done with
2626 : string-compare instructions, with multiple scalar instructions,
2627 : or with a library call.
2628 :
2629 : Both X and Y must be MEM rtx's. LEN is an rtx that says how long
2630 : they are. LEN_TYPE is the type of the expression that was used to
2631 : calculate it, and CTZ_LEN is the known trailing-zeros count of LEN,
2632 : so LEN must be a multiple of 1<<CTZ_LEN even if it's not constant.
2633 :
2634 : If EQUALITY_ONLY is true, it means we don't have to return the tri-state
2635 : value of a normal memcmp call, instead we can just compare for equality.
2636 : If FORCE_LIBCALL is true, we should emit a call to memcmp rather than
2637 : returning NULL_RTX.
2638 :
2639 : Optionally, the caller can pass a constfn and associated data in Y_CFN
2640 : and Y_CFN_DATA. describing that the second operand being compared is a
2641 : known constant and how to obtain its data.
2642 : Return the result of the comparison, or NULL_RTX if we failed to
2643 : perform the operation. */
2644 :
2645 : rtx
2646 106475 : emit_block_cmp_hints (rtx x, rtx y, rtx len, tree len_type, rtx target,
2647 : bool equality_only, by_pieces_constfn y_cfn,
2648 : void *y_cfndata, unsigned ctz_len)
2649 : {
2650 106475 : rtx result = 0;
2651 :
2652 106475 : if (CONST_INT_P (len) && INTVAL (len) == 0)
2653 0 : return const0_rtx;
2654 :
2655 106475 : gcc_assert (MEM_P (x) && MEM_P (y));
2656 106475 : unsigned int align = MIN (MEM_ALIGN (x), MEM_ALIGN (y));
2657 106475 : gcc_assert (align >= BITS_PER_UNIT);
2658 :
2659 106475 : x = adjust_address (x, BLKmode, 0);
2660 106475 : y = adjust_address (y, BLKmode, 0);
2661 :
2662 106475 : if (equality_only
2663 91612 : && CONST_INT_P (len)
2664 176062 : && can_do_by_pieces (INTVAL (len), align, COMPARE_BY_PIECES))
2665 62849 : result = compare_by_pieces (x, y, INTVAL (len), target, align,
2666 : y_cfn, y_cfndata);
2667 : else
2668 43626 : result = emit_block_cmp_via_cmpmem (x, y, len, len_type, target, align);
2669 :
2670 106475 : if (!result && (flag_inline_stringops & ILSOP_MEMCMP))
2671 361 : result = emit_block_cmp_via_loop (x, y, len, len_type,
2672 : target, equality_only,
2673 : align, ctz_len);
2674 :
2675 : return result;
2676 : }
2677 :
2678 : /* Like emit_block_cmp_hints, but with known alignment and no support
2679 : for constats. Always expand to a loop with iterations that compare
2680 : blocks of the largest compare-by-pieces size that divides both len
2681 : and align, and then, if !EQUALITY_ONLY, identify the word and then
2682 : the unit that first differs to return the result. */
2683 :
2684 : rtx
2685 403 : emit_block_cmp_via_loop (rtx x, rtx y, rtx len, tree len_type, rtx target,
2686 : bool equality_only, unsigned align, unsigned ctz_len)
2687 : {
2688 403 : unsigned incr = align / BITS_PER_UNIT;
2689 :
2690 403 : if (CONST_INT_P (len))
2691 319 : ctz_len = MAX (ctz_len, (unsigned) wi::ctz (UINTVAL (len)));
2692 :
2693 403 : if (HOST_WIDE_INT_1U << ctz_len < (unsigned HOST_WIDE_INT) incr)
2694 148 : incr = HOST_WIDE_INT_1U << ctz_len;
2695 :
2696 : while (incr > 1
2697 407 : && !can_do_by_pieces (incr, align, COMPARE_BY_PIECES))
2698 4 : incr >>= 1;
2699 :
2700 403 : rtx_code_label *cmp_label, *top_label, *ne_label, *res_label;
2701 403 : rtx iter, x_addr, y_addr, tmp;
2702 403 : machine_mode x_addr_mode = get_address_mode (x);
2703 403 : machine_mode y_addr_mode = get_address_mode (y);
2704 403 : machine_mode iter_mode;
2705 :
2706 403 : iter_mode = GET_MODE (len);
2707 403 : if (iter_mode == VOIDmode)
2708 319 : iter_mode = word_mode;
2709 :
2710 403 : rtx iter_init = const0_rtx;
2711 403 : rtx_code iter_cond = LTU;
2712 403 : rtx_code entry_cond = GEU;
2713 403 : rtx iter_limit = len;
2714 403 : rtx iter_incr = GEN_INT (incr);
2715 403 : machine_mode cmp_mode;
2716 :
2717 : /* We can drop the loop back edge if we know there's exactly one
2718 : iteration. */
2719 403 : top_label = (!rtx_equal_p (len, iter_incr)
2720 403 : ? gen_label_rtx ()
2721 : : NULL);
2722 : /* We need not test before entering the loop if len is known
2723 : nonzero. ??? This could be even stricter, testing whether a
2724 : nonconstant LEN could possibly be zero. */
2725 403 : cmp_label = (!CONSTANT_P (len) || rtx_equal_p (len, iter_init)
2726 403 : ? gen_label_rtx ()
2727 : : NULL);
2728 403 : ne_label = gen_label_rtx ();
2729 403 : res_label = gen_label_rtx ();
2730 :
2731 403 : iter = gen_reg_rtx (iter_mode);
2732 403 : emit_move_insn (iter, iter_init);
2733 :
2734 403 : opt_scalar_int_mode int_cmp_mode
2735 403 : = int_mode_for_size (incr * BITS_PER_UNIT, 1);
2736 403 : if (!int_cmp_mode.exists (&cmp_mode)
2737 1207 : || GET_MODE_BITSIZE (int_cmp_mode.require ()) != incr * BITS_PER_UNIT
2738 402 : || !can_compare_p (NE, cmp_mode, ccp_jump))
2739 : {
2740 1 : cmp_mode = BLKmode;
2741 1 : gcc_checking_assert (incr != 1);
2742 : }
2743 :
2744 : /* Save the base addresses. */
2745 403 : x_addr = force_operand (XEXP (x, 0), NULL_RTX);
2746 403 : y_addr = force_operand (XEXP (y, 0), NULL_RTX);
2747 403 : do_pending_stack_adjust ();
2748 :
2749 403 : if (cmp_label)
2750 : {
2751 84 : if (top_label)
2752 84 : emit_jump (cmp_label);
2753 : else
2754 0 : emit_cmp_and_jump_insns (iter, iter_limit, entry_cond,
2755 : NULL_RTX, iter_mode,
2756 : true, cmp_label,
2757 0 : profile_probability::guessed_always ()
2758 : .apply_scale (1, 10));
2759 : }
2760 403 : if (top_label)
2761 386 : emit_label (top_label);
2762 :
2763 : /* Offset the base addresses by ITER. */
2764 403 : tmp = convert_modes (x_addr_mode, iter_mode, iter, true);
2765 403 : x_addr = simplify_gen_binary (PLUS, x_addr_mode, x_addr, tmp);
2766 :
2767 403 : if (x_addr_mode != y_addr_mode)
2768 0 : tmp = convert_modes (y_addr_mode, iter_mode, iter, true);
2769 403 : y_addr = simplify_gen_binary (PLUS, y_addr_mode, y_addr, tmp);
2770 :
2771 403 : x = change_address (x, cmp_mode, x_addr);
2772 403 : y = change_address (y, cmp_mode, y_addr);
2773 :
2774 : /* Compare one block. */
2775 403 : rtx part_res;
2776 403 : if (cmp_mode == BLKmode)
2777 1 : part_res = compare_by_pieces (x, y, incr, target, align, 0, 0);
2778 : else
2779 402 : part_res = expand_binop (cmp_mode, sub_optab, x, y, NULL_RTX,
2780 : true, OPTAB_LIB_WIDEN);
2781 :
2782 : /* Stop if we found a difference. */
2783 806 : emit_cmp_and_jump_insns (part_res, GEN_INT (0), NE, NULL_RTX,
2784 403 : GET_MODE (part_res), true, ne_label,
2785 403 : profile_probability::guessed_always ()
2786 : .apply_scale (1, 10));
2787 :
2788 : /* Increment ITER. */
2789 403 : tmp = expand_simple_binop (iter_mode, PLUS, iter, iter_incr, iter,
2790 : true, OPTAB_LIB_WIDEN);
2791 403 : if (tmp != iter)
2792 0 : emit_move_insn (iter, tmp);
2793 :
2794 403 : if (cmp_label)
2795 84 : emit_label (cmp_label);
2796 : /* Loop until we reach the limit. */
2797 :
2798 403 : if (top_label)
2799 386 : emit_cmp_and_jump_insns (iter, iter_limit, iter_cond, NULL_RTX, iter_mode,
2800 : true, top_label,
2801 772 : profile_probability::guessed_always ()
2802 : .apply_scale (9, 10));
2803 :
2804 : /* We got to the end without differences, so the result is zero. */
2805 403 : if (target == NULL_RTX
2806 403 : || !REG_P (target) || REGNO (target) < FIRST_PSEUDO_REGISTER)
2807 49 : target = gen_reg_rtx (TYPE_MODE (integer_type_node));
2808 :
2809 403 : emit_move_insn (target, const0_rtx);
2810 403 : emit_jump (res_label);
2811 :
2812 403 : emit_label (ne_label);
2813 :
2814 : /* Return nonzero, or pinpoint the difference to return the expected
2815 : result for non-equality tests. */
2816 403 : if (equality_only)
2817 0 : emit_move_insn (target, const1_rtx);
2818 : else
2819 : {
2820 403 : if (incr > UNITS_PER_WORD)
2821 : /* ??? Re-compare the block found to be different one word at a
2822 : time. */
2823 5 : part_res = emit_block_cmp_via_loop (x, y, GEN_INT (incr), len_type,
2824 : target, equality_only,
2825 : BITS_PER_WORD, 0);
2826 398 : else if (incr > 1)
2827 : /* ??? Re-compare the block found to be different one byte at a
2828 : time. We could do better using part_res, and being careful
2829 : about endianness. */
2830 37 : part_res = emit_block_cmp_via_loop (x, y, GEN_INT (incr), len_type,
2831 : target, equality_only,
2832 : BITS_PER_UNIT, 0);
2833 1083 : else if (known_gt (GET_MODE_BITSIZE (GET_MODE (target)),
2834 : GET_MODE_BITSIZE (cmp_mode)))
2835 361 : part_res = expand_binop (GET_MODE (target), sub_optab, x, y, target,
2836 : true, OPTAB_LIB_WIDEN);
2837 : else
2838 : {
2839 : /* In the odd chance target is QImode, we can't count on
2840 : widening subtract to capture the result of the unsigned
2841 : compares. */
2842 0 : rtx_code_label *ltu_label;
2843 0 : ltu_label = gen_label_rtx ();
2844 0 : emit_cmp_and_jump_insns (x, y, LTU, NULL_RTX,
2845 : cmp_mode, true, ltu_label,
2846 0 : profile_probability::guessed_always ()
2847 : .apply_scale (5, 10));
2848 :
2849 0 : emit_move_insn (target, const1_rtx);
2850 0 : emit_jump (res_label);
2851 :
2852 0 : emit_label (ltu_label);
2853 0 : emit_move_insn (target, constm1_rtx);
2854 0 : part_res = target;
2855 : }
2856 :
2857 403 : if (target != part_res)
2858 0 : convert_move (target, part_res, false);
2859 : }
2860 :
2861 403 : emit_label (res_label);
2862 :
2863 403 : return target;
2864 : }
2865 :
2866 :
2867 : /* Copy all or part of a value X into registers starting at REGNO.
2868 : The number of registers to be filled is NREGS. */
2869 :
2870 : void
2871 434 : move_block_to_reg (int regno, rtx x, int nregs, machine_mode mode)
2872 : {
2873 434 : if (nregs == 0)
2874 : return;
2875 :
2876 420 : if (CONSTANT_P (x) && !targetm.legitimate_constant_p (mode, x))
2877 0 : x = validize_mem (force_const_mem (mode, x));
2878 :
2879 : /* See if the machine can do this with a load multiple insn. */
2880 420 : if (targetm.have_load_multiple ())
2881 : {
2882 0 : rtx_insn *last = get_last_insn ();
2883 0 : rtx first = gen_rtx_REG (word_mode, regno);
2884 0 : if (rtx_insn *pat = targetm.gen_load_multiple (first, x,
2885 : GEN_INT (nregs)))
2886 : {
2887 0 : emit_insn (pat);
2888 0 : return;
2889 : }
2890 : else
2891 0 : delete_insns_since (last);
2892 : }
2893 :
2894 848 : for (int i = 0; i < nregs; i++)
2895 428 : emit_move_insn (gen_rtx_REG (word_mode, regno + i),
2896 428 : operand_subword_force (x, i, mode));
2897 : }
2898 :
2899 : /* Copy all or part of a BLKmode value X out of registers starting at REGNO.
2900 : The number of registers to be filled is NREGS. */
2901 :
2902 : void
2903 1216 : move_block_from_reg (int regno, rtx x, int nregs)
2904 : {
2905 1216 : if (nregs == 0)
2906 : return;
2907 :
2908 : /* See if the machine can do this with a store multiple insn. */
2909 1216 : if (targetm.have_store_multiple ())
2910 : {
2911 0 : rtx_insn *last = get_last_insn ();
2912 0 : rtx first = gen_rtx_REG (word_mode, regno);
2913 0 : if (rtx_insn *pat = targetm.gen_store_multiple (x, first,
2914 : GEN_INT (nregs)))
2915 : {
2916 0 : emit_insn (pat);
2917 0 : return;
2918 : }
2919 : else
2920 0 : delete_insns_since (last);
2921 : }
2922 :
2923 2436 : for (int i = 0; i < nregs; i++)
2924 : {
2925 1220 : rtx tem = operand_subword (x, i, 1, BLKmode);
2926 :
2927 1220 : gcc_assert (tem);
2928 :
2929 1220 : emit_move_insn (tem, gen_rtx_REG (word_mode, regno + i));
2930 : }
2931 : }
2932 :
2933 : /* Generate a PARALLEL rtx for a new non-consecutive group of registers from
2934 : ORIG, where ORIG is a non-consecutive group of registers represented by
2935 : a PARALLEL. The clone is identical to the original except in that the
2936 : original set of registers is replaced by a new set of pseudo registers.
2937 : The new set has the same modes as the original set. */
2938 :
2939 : rtx
2940 3418 : gen_group_rtx (rtx orig)
2941 : {
2942 3418 : int i, length;
2943 3418 : rtx *tmps;
2944 :
2945 3418 : gcc_assert (GET_CODE (orig) == PARALLEL);
2946 :
2947 3418 : length = XVECLEN (orig, 0);
2948 3418 : tmps = XALLOCAVEC (rtx, length);
2949 :
2950 : /* Skip a NULL entry in first slot. */
2951 3418 : i = XEXP (XVECEXP (orig, 0, 0), 0) ? 0 : 1;
2952 :
2953 3418 : if (i)
2954 0 : tmps[0] = 0;
2955 :
2956 8334 : for (; i < length; i++)
2957 : {
2958 4916 : machine_mode mode = GET_MODE (XEXP (XVECEXP (orig, 0, i), 0));
2959 4916 : rtx offset = XEXP (XVECEXP (orig, 0, i), 1);
2960 :
2961 4916 : tmps[i] = gen_rtx_EXPR_LIST (VOIDmode, gen_reg_rtx (mode), offset);
2962 : }
2963 :
2964 3418 : return gen_rtx_PARALLEL (GET_MODE (orig), gen_rtvec_v (length, tmps));
2965 : }
2966 :
2967 : /* A subroutine of emit_group_load. Arguments as for emit_group_load,
2968 : except that values are placed in TMPS[i], and must later be moved
2969 : into corresponding XEXP (XVECEXP (DST, 0, i), 0) element. */
2970 :
2971 : static void
2972 299609 : emit_group_load_1 (rtx *tmps, rtx dst, rtx orig_src, tree type,
2973 : poly_int64 ssize)
2974 : {
2975 299612 : rtx src;
2976 299612 : int start, i;
2977 299612 : machine_mode m = GET_MODE (orig_src);
2978 :
2979 299612 : gcc_assert (GET_CODE (dst) == PARALLEL);
2980 :
2981 299612 : if (m != VOIDmode
2982 280727 : && !SCALAR_INT_MODE_P (m)
2983 19908 : && !MEM_P (orig_src)
2984 6078 : && GET_CODE (orig_src) != CONCAT)
2985 : {
2986 3 : scalar_int_mode imode;
2987 3 : if (int_mode_for_mode (GET_MODE (orig_src)).exists (&imode))
2988 : {
2989 3 : src = gen_reg_rtx (imode);
2990 3 : emit_move_insn (gen_lowpart (GET_MODE (orig_src), src), orig_src);
2991 : }
2992 : else
2993 : {
2994 0 : src = assign_stack_temp (GET_MODE (orig_src), ssize);
2995 0 : emit_move_insn (src, orig_src);
2996 : }
2997 3 : emit_group_load_1 (tmps, dst, src, type, ssize);
2998 : return;
2999 : }
3000 :
3001 : /* Check for a NULL entry, used to indicate that the parameter goes
3002 : both on the stack and in registers. */
3003 299609 : if (XEXP (XVECEXP (dst, 0, 0), 0))
3004 : start = 0;
3005 : else
3006 0 : start = 1;
3007 :
3008 : /* Process the pieces. */
3009 886958 : for (i = start; i < XVECLEN (dst, 0); i++)
3010 : {
3011 587349 : machine_mode mode = GET_MODE (XEXP (XVECEXP (dst, 0, i), 0));
3012 587349 : poly_int64 bytepos = rtx_to_poly_int64 (XEXP (XVECEXP (dst, 0, i), 1));
3013 1174698 : poly_int64 bytelen = GET_MODE_SIZE (mode);
3014 587349 : poly_int64 shift = 0;
3015 :
3016 : /* Handle trailing fragments that run over the size of the struct.
3017 : It's the target's responsibility to make sure that the fragment
3018 : cannot be strictly smaller in some cases and strictly larger
3019 : in others. */
3020 587349 : gcc_checking_assert (ordered_p (bytepos + bytelen, ssize));
3021 587349 : if (known_size_p (ssize) && maybe_gt (bytepos + bytelen, ssize))
3022 : {
3023 : /* Arrange to shift the fragment to where it belongs.
3024 : extract_bit_field loads to the lsb of the reg. */
3025 : if (
3026 : #ifdef BLOCK_REG_PADDING
3027 : BLOCK_REG_PADDING (GET_MODE (orig_src), type, i == start)
3028 : == (BYTES_BIG_ENDIAN ? PAD_UPWARD : PAD_DOWNWARD)
3029 : #else
3030 : BYTES_BIG_ENDIAN
3031 : #endif
3032 : )
3033 : shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
3034 1839 : bytelen = ssize - bytepos;
3035 1839 : gcc_assert (maybe_gt (bytelen, 0));
3036 : }
3037 :
3038 : /* If we won't be loading directly from memory, protect the real source
3039 : from strange tricks we might play; but make sure that the source can
3040 : be loaded directly into the destination. */
3041 587349 : src = orig_src;
3042 587349 : if (!MEM_P (orig_src)
3043 485826 : && (!REG_P (orig_src) || HARD_REGISTER_P (orig_src))
3044 46986 : && GET_CODE (orig_src) != CONCAT
3045 625119 : && !CONSTANT_P (orig_src))
3046 : {
3047 0 : gcc_assert (GET_MODE (orig_src) != VOIDmode);
3048 0 : src = force_reg (GET_MODE (orig_src), orig_src);
3049 : }
3050 :
3051 : /* Optimize the access just a bit. */
3052 587349 : if (MEM_P (src)
3053 101523 : && (! targetm.slow_unaligned_access (mode, MEM_ALIGN (src))
3054 0 : || MEM_ALIGN (src) >= GET_MODE_ALIGNMENT (mode))
3055 203046 : && multiple_p (bytepos * BITS_PER_UNIT, GET_MODE_ALIGNMENT (mode))
3056 688872 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3057 : {
3058 99703 : tmps[i] = gen_reg_rtx (mode);
3059 99703 : emit_move_insn (tmps[i], adjust_address (src, mode, bytepos));
3060 : }
3061 487646 : else if (COMPLEX_MODE_P (mode)
3062 0 : && GET_MODE (src) == mode
3063 487646 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3064 : /* Let emit_move_complex do the bulk of the work. */
3065 0 : tmps[i] = src;
3066 487646 : else if (SCALAR_INT_MODE_P (mode)
3067 480737 : && COMPLEX_MODE_P (GET_MODE (src))
3068 9903 : && known_eq (GET_MODE_SIZE (mode),
3069 : GET_MODE_SIZE (GET_MODE (src)))
3070 490561 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3071 : {
3072 : /* When passing a complex value in an integer mode of the same
3073 : size, explicitly construct (highpart<<isize)+lowpart to
3074 : avoid spilling to memory before reload. */
3075 2915 : rtx tmp = read_complex_part (src, !BYTES_BIG_ENDIAN);
3076 2915 : scalar_int_mode imode = int_mode_for_mode (GET_MODE (tmp)).require();
3077 2915 : tmp = gen_lowpart (imode, tmp);
3078 2915 : tmp = simplify_gen_unary (ZERO_EXTEND, mode, tmp, imode);
3079 2915 : rtx result = force_reg (mode, tmp);
3080 5830 : result = expand_shift (LSHIFT_EXPR, mode, result,
3081 2915 : GET_MODE_BITSIZE (imode), NULL_RTX, 1);
3082 2915 : tmp = read_complex_part (src, BYTES_BIG_ENDIAN);
3083 2915 : tmp = gen_lowpart (imode, tmp);
3084 2915 : tmp = simplify_gen_unary (ZERO_EXTEND, mode, tmp, imode);
3085 2915 : result = simplify_gen_binary (PLUS, mode, result, tmp);
3086 2915 : tmps[i] = force_reg (mode, result);
3087 : }
3088 484731 : else if (GET_CODE (src) == CONCAT)
3089 : {
3090 12602 : poly_int64 slen = GET_MODE_SIZE (GET_MODE (src));
3091 12602 : poly_int64 slen0 = GET_MODE_SIZE (GET_MODE (XEXP (src, 0)));
3092 6301 : unsigned int elt;
3093 6301 : poly_int64 subpos;
3094 :
3095 6301 : if (can_div_trunc_p (bytepos, slen0, &elt, &subpos)
3096 6301 : && known_le (subpos + bytelen, slen0))
3097 : {
3098 : /* The following assumes that the concatenated objects all
3099 : have the same size. In this case, a simple calculation
3100 : can be used to determine the object and the bit field
3101 : to be extracted. */
3102 6282 : tmps[i] = XEXP (src, elt);
3103 6282 : if (maybe_ne (subpos, 0)
3104 6282 : || maybe_ne (subpos + bytelen, slen0)
3105 12564 : || (!CONSTANT_P (tmps[i])
3106 6282 : && (!REG_P (tmps[i]) || GET_MODE (tmps[i]) != mode)))
3107 0 : tmps[i] = extract_bit_field (tmps[i], bytelen * BITS_PER_UNIT,
3108 0 : subpos * BITS_PER_UNIT,
3109 : 1, NULL_RTX, mode, mode, false,
3110 : NULL);
3111 : }
3112 : else
3113 : {
3114 19 : rtx mem;
3115 :
3116 19 : gcc_assert (known_eq (bytepos, 0));
3117 19 : mem = assign_stack_temp (GET_MODE (src), slen);
3118 19 : emit_move_insn (mem, src);
3119 19 : tmps[i] = extract_bit_field (mem, bytelen * BITS_PER_UNIT,
3120 : 0, 1, NULL_RTX, mode, mode, false,
3121 : NULL);
3122 : }
3123 : }
3124 478430 : else if (CONSTANT_P (src) && GET_MODE (dst) != BLKmode
3125 37770 : && XVECLEN (dst, 0) > 1)
3126 37770 : tmps[i] = force_subreg (mode, src, GET_MODE (dst), bytepos);
3127 440660 : else if (CONSTANT_P (src))
3128 : {
3129 0 : if (known_eq (bytelen, ssize))
3130 0 : tmps[i] = src;
3131 : else
3132 : {
3133 : rtx first, second;
3134 :
3135 : /* TODO: const_wide_int can have sizes other than this... */
3136 0 : gcc_assert (known_eq (2 * bytelen, ssize));
3137 0 : split_double (src, &first, &second);
3138 0 : if (i)
3139 0 : tmps[i] = second;
3140 : else
3141 0 : tmps[i] = first;
3142 : }
3143 : }
3144 440660 : else if (REG_P (src) && GET_MODE (src) == mode)
3145 0 : tmps[i] = src;
3146 : else
3147 440660 : tmps[i] = extract_bit_field (src, bytelen * BITS_PER_UNIT,
3148 881320 : bytepos * BITS_PER_UNIT, 1, NULL_RTX,
3149 : mode, mode, false, NULL);
3150 :
3151 587349 : if (maybe_ne (shift, 0))
3152 : tmps[i] = expand_shift (LSHIFT_EXPR, mode, tmps[i],
3153 : shift, tmps[i], 0);
3154 : }
3155 : }
3156 :
3157 : /* Emit code to move a block SRC of type TYPE to a block DST,
3158 : where DST is non-consecutive registers represented by a PARALLEL.
3159 : SSIZE represents the total size of block ORIG_SRC in bytes, or -1
3160 : if not known. */
3161 :
3162 : void
3163 11216 : emit_group_load (rtx dst, rtx src, tree type, poly_int64 ssize)
3164 : {
3165 11216 : rtx *tmps;
3166 11216 : int i;
3167 :
3168 11216 : tmps = XALLOCAVEC (rtx, XVECLEN (dst, 0));
3169 11216 : emit_group_load_1 (tmps, dst, src, type, ssize);
3170 :
3171 : /* Copy the extracted pieces into the proper (probable) hard regs. */
3172 42404 : for (i = 0; i < XVECLEN (dst, 0); i++)
3173 : {
3174 19972 : rtx d = XEXP (XVECEXP (dst, 0, i), 0);
3175 19972 : if (d == NULL)
3176 0 : continue;
3177 19972 : emit_move_insn (d, tmps[i]);
3178 : }
3179 11216 : }
3180 :
3181 : /* Similar, but load SRC into new pseudos in a format that looks like
3182 : PARALLEL. This can later be fed to emit_group_move to get things
3183 : in the right place. */
3184 :
3185 : rtx
3186 288393 : emit_group_load_into_temps (rtx parallel, rtx src, tree type, poly_int64 ssize)
3187 : {
3188 288393 : rtvec vec;
3189 288393 : int i;
3190 :
3191 288393 : vec = rtvec_alloc (XVECLEN (parallel, 0));
3192 288393 : emit_group_load_1 (&RTVEC_ELT (vec, 0), parallel, src, type, ssize);
3193 :
3194 : /* Convert the vector to look just like the original PARALLEL, except
3195 : with the computed values. */
3196 1144163 : for (i = 0; i < XVECLEN (parallel, 0); i++)
3197 : {
3198 567377 : rtx e = XVECEXP (parallel, 0, i);
3199 567377 : rtx d = XEXP (e, 0);
3200 :
3201 567377 : if (d)
3202 : {
3203 567377 : d = force_reg (GET_MODE (d), RTVEC_ELT (vec, i));
3204 567377 : e = alloc_EXPR_LIST (REG_NOTE_KIND (e), d, XEXP (e, 1));
3205 : }
3206 567377 : RTVEC_ELT (vec, i) = e;
3207 : }
3208 :
3209 288393 : return gen_rtx_PARALLEL (GET_MODE (parallel), vec);
3210 : }
3211 :
3212 : /* Emit code to move a block SRC to block DST, where SRC and DST are
3213 : non-consecutive groups of registers, each represented by a PARALLEL. */
3214 :
3215 : void
3216 291811 : emit_group_move (rtx dst, rtx src)
3217 : {
3218 291811 : int i;
3219 :
3220 291811 : gcc_assert (GET_CODE (src) == PARALLEL
3221 : && GET_CODE (dst) == PARALLEL
3222 : && XVECLEN (src, 0) == XVECLEN (dst, 0));
3223 :
3224 : /* Skip first entry if NULL. */
3225 864104 : for (i = XEXP (XVECEXP (src, 0, 0), 0) ? 0 : 1; i < XVECLEN (src, 0); i++)
3226 572293 : emit_move_insn (XEXP (XVECEXP (dst, 0, i), 0),
3227 572293 : XEXP (XVECEXP (src, 0, i), 0));
3228 291811 : }
3229 :
3230 : /* Move a group of registers represented by a PARALLEL into pseudos. */
3231 :
3232 : rtx
3233 5803 : emit_group_move_into_temps (rtx src)
3234 : {
3235 5803 : rtvec vec = rtvec_alloc (XVECLEN (src, 0));
3236 5803 : int i;
3237 :
3238 19443 : for (i = 0; i < XVECLEN (src, 0); i++)
3239 : {
3240 7837 : rtx e = XVECEXP (src, 0, i);
3241 7837 : rtx d = XEXP (e, 0);
3242 :
3243 7837 : if (d)
3244 7837 : e = alloc_EXPR_LIST (REG_NOTE_KIND (e), copy_to_reg (d), XEXP (e, 1));
3245 7837 : RTVEC_ELT (vec, i) = e;
3246 : }
3247 :
3248 5803 : return gen_rtx_PARALLEL (GET_MODE (src), vec);
3249 : }
3250 :
3251 : /* Emit code to move a block SRC to a block ORIG_DST of type TYPE,
3252 : where SRC is non-consecutive registers represented by a PARALLEL.
3253 : SSIZE represents the total size of block ORIG_DST, or -1 if not
3254 : known. */
3255 :
3256 : void
3257 66230 : emit_group_store (rtx orig_dst, rtx src, tree type ATTRIBUTE_UNUSED,
3258 : poly_int64 ssize)
3259 : {
3260 66230 : rtx *tmps, dst;
3261 66230 : int start, finish, i;
3262 66230 : machine_mode m = GET_MODE (orig_dst);
3263 :
3264 66230 : gcc_assert (GET_CODE (src) == PARALLEL);
3265 :
3266 66230 : if (!SCALAR_INT_MODE_P (m)
3267 15052 : && !MEM_P (orig_dst) && GET_CODE (orig_dst) != CONCAT)
3268 : {
3269 0 : scalar_int_mode imode;
3270 0 : if (int_mode_for_mode (GET_MODE (orig_dst)).exists (&imode))
3271 : {
3272 0 : dst = gen_reg_rtx (imode);
3273 0 : emit_group_store (dst, src, type, ssize);
3274 0 : dst = gen_lowpart (GET_MODE (orig_dst), dst);
3275 : }
3276 : else
3277 : {
3278 0 : dst = assign_stack_temp (GET_MODE (orig_dst), ssize);
3279 0 : emit_group_store (dst, src, type, ssize);
3280 : }
3281 0 : emit_move_insn (orig_dst, dst);
3282 0 : return;
3283 : }
3284 :
3285 : /* Check for a NULL entry, used to indicate that the parameter goes
3286 : both on the stack and in registers. */
3287 66230 : if (XEXP (XVECEXP (src, 0, 0), 0))
3288 : start = 0;
3289 : else
3290 0 : start = 1;
3291 66230 : finish = XVECLEN (src, 0);
3292 :
3293 66230 : tmps = XALLOCAVEC (rtx, finish);
3294 :
3295 : /* Copy the (probable) hard regs into pseudos. */
3296 190108 : for (i = start; i < finish; i++)
3297 : {
3298 123878 : rtx reg = XEXP (XVECEXP (src, 0, i), 0);
3299 123878 : if (!REG_P (reg) || REGNO (reg) < FIRST_PSEUDO_REGISTER)
3300 : {
3301 116041 : tmps[i] = gen_reg_rtx (GET_MODE (reg));
3302 116041 : emit_move_insn (tmps[i], reg);
3303 : }
3304 : else
3305 7837 : tmps[i] = reg;
3306 : }
3307 :
3308 : /* If we won't be storing directly into memory, protect the real destination
3309 : from strange tricks we might play. */
3310 66230 : dst = orig_dst;
3311 66230 : if (GET_CODE (dst) == PARALLEL)
3312 : {
3313 0 : rtx temp;
3314 :
3315 : /* We can get a PARALLEL dst if there is a conditional expression in
3316 : a return statement. In that case, the dst and src are the same,
3317 : so no action is necessary. */
3318 0 : if (rtx_equal_p (dst, src))
3319 : return;
3320 :
3321 : /* It is unclear if we can ever reach here, but we may as well handle
3322 : it. Allocate a temporary, and split this into a store/load to/from
3323 : the temporary. */
3324 0 : temp = assign_stack_temp (GET_MODE (dst), ssize);
3325 0 : emit_group_store (temp, src, type, ssize);
3326 0 : emit_group_load (dst, temp, type, ssize);
3327 0 : return;
3328 : }
3329 66230 : else if (!MEM_P (dst) && GET_CODE (dst) != CONCAT)
3330 : {
3331 50544 : machine_mode outer = GET_MODE (dst);
3332 50544 : machine_mode inner;
3333 50544 : poly_int64 bytepos;
3334 50544 : bool done = false;
3335 50544 : rtx temp;
3336 :
3337 50544 : if (!REG_P (dst) || REGNO (dst) < FIRST_PSEUDO_REGISTER)
3338 0 : dst = gen_reg_rtx (outer);
3339 :
3340 : /* Make life a bit easier for combine: if the first element of the
3341 : vector is the low part of the destination mode, use a paradoxical
3342 : subreg to initialize the destination. */
3343 50544 : if (start < finish)
3344 : {
3345 50544 : inner = GET_MODE (tmps[start]);
3346 50544 : bytepos = subreg_lowpart_offset (inner, outer);
3347 50544 : if (known_eq (rtx_to_poly_int64 (XEXP (XVECEXP (src, 0, start), 1)),
3348 : bytepos))
3349 : {
3350 50520 : temp = force_subreg (outer, tmps[start], inner, 0);
3351 50520 : if (temp)
3352 : {
3353 49848 : emit_move_insn (dst, temp);
3354 49848 : done = true;
3355 49848 : start++;
3356 : }
3357 : }
3358 : }
3359 :
3360 : /* If the first element wasn't the low part, try the last. */
3361 49848 : if (!done
3362 696 : && start < finish - 1)
3363 : {
3364 629 : inner = GET_MODE (tmps[finish - 1]);
3365 629 : bytepos = subreg_lowpart_offset (inner, outer);
3366 629 : if (known_eq (rtx_to_poly_int64 (XEXP (XVECEXP (src, 0,
3367 : finish - 1), 1)),
3368 : bytepos))
3369 : {
3370 0 : temp = force_subreg (outer, tmps[finish - 1], inner, 0);
3371 0 : if (temp)
3372 : {
3373 0 : emit_move_insn (dst, temp);
3374 0 : done = true;
3375 0 : finish--;
3376 : }
3377 : }
3378 : }
3379 :
3380 : /* Otherwise, simply initialize the result to zero. */
3381 49848 : if (!done)
3382 696 : emit_move_insn (dst, CONST0_RTX (outer));
3383 : }
3384 :
3385 : /* Process the pieces. */
3386 136459 : for (i = start; i < finish; i++)
3387 : {
3388 74030 : poly_int64 bytepos = rtx_to_poly_int64 (XEXP (XVECEXP (src, 0, i), 1));
3389 74030 : machine_mode mode = GET_MODE (tmps[i]);
3390 148060 : poly_int64 bytelen = GET_MODE_SIZE (mode);
3391 74030 : poly_uint64 adj_bytelen;
3392 74030 : rtx dest = dst;
3393 :
3394 : /* Handle trailing fragments that run over the size of the struct.
3395 : It's the target's responsibility to make sure that the fragment
3396 : cannot be strictly smaller in some cases and strictly larger
3397 : in others. */
3398 74030 : gcc_checking_assert (ordered_p (bytepos + bytelen, ssize));
3399 147109 : if (known_size_p (ssize) && maybe_gt (bytepos + bytelen, ssize))
3400 951 : adj_bytelen = ssize - bytepos;
3401 : else
3402 74030 : adj_bytelen = bytelen;
3403 :
3404 : /* Deal with destination CONCATs by either storing into one of the parts
3405 : or doing a copy after storing into a register or stack temporary. */
3406 74030 : if (GET_CODE (dst) == CONCAT)
3407 : {
3408 30490 : if (known_le (bytepos + adj_bytelen,
3409 : GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)))))
3410 : dest = XEXP (dst, 0);
3411 :
3412 19046 : else if (known_ge (bytepos, GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)))))
3413 : {
3414 11444 : bytepos -= GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)));
3415 5722 : dest = XEXP (dst, 1);
3416 : }
3417 :
3418 : else
3419 : {
3420 3801 : machine_mode dest_mode = GET_MODE (dest);
3421 3801 : machine_mode tmp_mode = GET_MODE (tmps[i]);
3422 3801 : scalar_int_mode dest_imode;
3423 :
3424 3801 : gcc_assert (known_eq (bytepos, 0) && XVECLEN (src, 0));
3425 :
3426 : /* If the source is a single scalar integer register, and the
3427 : destination has a complex mode for which a same-sized integer
3428 : mode exists, then we can take the left-justified part of the
3429 : source in the complex mode. */
3430 3801 : if (finish == start + 1
3431 3801 : && REG_P (tmps[i])
3432 3801 : && SCALAR_INT_MODE_P (tmp_mode)
3433 3801 : && COMPLEX_MODE_P (dest_mode)
3434 7602 : && int_mode_for_mode (dest_mode).exists (&dest_imode))
3435 : {
3436 3801 : const scalar_int_mode tmp_imode
3437 3801 : = as_a <scalar_int_mode> (tmp_mode);
3438 :
3439 7602 : if (GET_MODE_BITSIZE (dest_imode)
3440 3801 : < GET_MODE_BITSIZE (tmp_imode))
3441 : {
3442 59 : dest = gen_reg_rtx (dest_imode);
3443 59 : if (BYTES_BIG_ENDIAN)
3444 : tmps[i] = expand_shift (RSHIFT_EXPR, tmp_mode, tmps[i],
3445 : GET_MODE_BITSIZE (tmp_imode)
3446 : - GET_MODE_BITSIZE (dest_imode),
3447 : NULL_RTX, 1);
3448 59 : emit_move_insn (dest, gen_lowpart (dest_imode, tmps[i]));
3449 59 : dst = gen_lowpart (dest_mode, dest);
3450 : }
3451 : else
3452 3742 : dst = gen_lowpart (dest_mode, tmps[i]);
3453 : }
3454 :
3455 : /* Otherwise spill the source onto the stack using the more
3456 : aligned of the two modes. */
3457 0 : else if (GET_MODE_ALIGNMENT (dest_mode)
3458 0 : >= GET_MODE_ALIGNMENT (tmp_mode))
3459 : {
3460 0 : dest = assign_stack_temp (dest_mode,
3461 0 : GET_MODE_SIZE (dest_mode));
3462 0 : emit_move_insn (adjust_address (dest, tmp_mode, bytepos),
3463 : tmps[i]);
3464 0 : dst = dest;
3465 : }
3466 :
3467 : else
3468 : {
3469 0 : dest = assign_stack_temp (tmp_mode,
3470 0 : GET_MODE_SIZE (tmp_mode));
3471 0 : emit_move_insn (dest, tmps[i]);
3472 0 : dst = adjust_address (dest, dest_mode, bytepos);
3473 : }
3474 :
3475 : break;
3476 : }
3477 : }
3478 :
3479 : /* Handle trailing fragments that run over the size of the struct. */
3480 70229 : if (known_size_p (ssize) && maybe_gt (bytepos + bytelen, ssize))
3481 : {
3482 : /* store_bit_field always takes its value from the lsb.
3483 : Move the fragment to the lsb if it's not already there. */
3484 892 : if (
3485 : #ifdef BLOCK_REG_PADDING
3486 : BLOCK_REG_PADDING (GET_MODE (orig_dst), type, i == start)
3487 : == (BYTES_BIG_ENDIAN ? PAD_UPWARD : PAD_DOWNWARD)
3488 : #else
3489 : BYTES_BIG_ENDIAN
3490 : #endif
3491 : )
3492 : {
3493 : poly_int64 shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
3494 : tmps[i] = expand_shift (RSHIFT_EXPR, mode, tmps[i],
3495 : shift, tmps[i], 0);
3496 : }
3497 :
3498 : /* Make sure not to write past the end of the struct. */
3499 892 : store_bit_field (dest,
3500 892 : adj_bytelen * BITS_PER_UNIT, bytepos * BITS_PER_UNIT,
3501 3568 : bytepos * BITS_PER_UNIT, ssize * BITS_PER_UNIT - 1,
3502 : VOIDmode, tmps[i], false, false);
3503 : }
3504 :
3505 : /* Optimize the access just a bit. */
3506 69337 : else if (MEM_P (dest)
3507 7553 : && (!targetm.slow_unaligned_access (mode, MEM_ALIGN (dest))
3508 0 : || MEM_ALIGN (dest) >= GET_MODE_ALIGNMENT (mode))
3509 15106 : && multiple_p (bytepos * BITS_PER_UNIT,
3510 : GET_MODE_ALIGNMENT (mode))
3511 76890 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3512 7553 : emit_move_insn (adjust_address (dest, mode, bytepos), tmps[i]);
3513 :
3514 : else
3515 123568 : store_bit_field (dest, bytelen * BITS_PER_UNIT, bytepos * BITS_PER_UNIT,
3516 : 0, 0, mode, tmps[i], false, false);
3517 : }
3518 :
3519 : /* Copy from the pseudo into the (probable) hard reg. */
3520 66230 : if (orig_dst != dst)
3521 3801 : emit_move_insn (orig_dst, dst);
3522 : }
3523 :
3524 : /* Return a form of X that does not use a PARALLEL. TYPE is the type
3525 : of the value stored in X. */
3526 :
3527 : rtx
3528 337319 : maybe_emit_group_store (rtx x, tree type)
3529 : {
3530 337319 : machine_mode mode = TYPE_MODE (type);
3531 337319 : gcc_checking_assert (GET_MODE (x) == VOIDmode || GET_MODE (x) == mode);
3532 337319 : if (GET_CODE (x) == PARALLEL)
3533 : {
3534 0 : rtx result = gen_reg_rtx (mode);
3535 0 : emit_group_store (result, x, type, int_size_in_bytes (type));
3536 0 : return result;
3537 : }
3538 : return x;
3539 : }
3540 :
3541 : /* Copy a BLKmode object of TYPE out of a register SRCREG into TARGET.
3542 :
3543 : This is used on targets that return BLKmode values in registers. */
3544 :
3545 : static void
3546 256 : copy_blkmode_from_reg (rtx target, rtx srcreg, tree type)
3547 : {
3548 256 : unsigned HOST_WIDE_INT bytes = int_size_in_bytes (type);
3549 256 : rtx src = NULL, dst = NULL;
3550 256 : unsigned HOST_WIDE_INT bitsize = MIN (TYPE_ALIGN (type), BITS_PER_WORD);
3551 256 : unsigned HOST_WIDE_INT bitpos, xbitpos, padding_correction = 0;
3552 : /* No current ABI uses variable-sized modes to pass a BLKmnode type. */
3553 256 : fixed_size_mode mode = as_a <fixed_size_mode> (GET_MODE (srcreg));
3554 256 : fixed_size_mode tmode = as_a <fixed_size_mode> (GET_MODE (target));
3555 256 : fixed_size_mode copy_mode;
3556 :
3557 : /* BLKmode registers created in the back-end shouldn't have survived. */
3558 256 : gcc_assert (mode != BLKmode);
3559 :
3560 : /* If the structure doesn't take up a whole number of words, see whether
3561 : SRCREG is padded on the left or on the right. If it's on the left,
3562 : set PADDING_CORRECTION to the number of bits to skip.
3563 :
3564 : In most ABIs, the structure will be returned at the least end of
3565 : the register, which translates to right padding on little-endian
3566 : targets and left padding on big-endian targets. The opposite
3567 : holds if the structure is returned at the most significant
3568 : end of the register. */
3569 256 : if (bytes % UNITS_PER_WORD != 0
3570 256 : && (targetm.calls.return_in_msb (type)
3571 225 : ? !BYTES_BIG_ENDIAN
3572 : : BYTES_BIG_ENDIAN))
3573 0 : padding_correction
3574 0 : = (BITS_PER_WORD - ((bytes % UNITS_PER_WORD) * BITS_PER_UNIT));
3575 :
3576 : /* We can use a single move if we have an exact mode for the size. */
3577 256 : else if (MEM_P (target)
3578 256 : && (!targetm.slow_unaligned_access (mode, MEM_ALIGN (target))
3579 0 : || MEM_ALIGN (target) >= GET_MODE_ALIGNMENT (mode))
3580 512 : && bytes == GET_MODE_SIZE (mode))
3581 : {
3582 38 : emit_move_insn (adjust_address (target, mode, 0), srcreg);
3583 38 : return;
3584 : }
3585 :
3586 : /* And if we additionally have the same mode for a register. */
3587 218 : else if (REG_P (target)
3588 0 : && GET_MODE (target) == mode
3589 218 : && bytes == GET_MODE_SIZE (mode))
3590 : {
3591 0 : emit_move_insn (target, srcreg);
3592 0 : return;
3593 : }
3594 :
3595 : /* This code assumes srcreg is at least a full word. If it isn't, copy it
3596 : into a new pseudo which is a full word. */
3597 436 : if (GET_MODE_SIZE (mode) < UNITS_PER_WORD)
3598 : {
3599 77 : srcreg = convert_to_mode (word_mode, srcreg, TYPE_UNSIGNED (type));
3600 77 : mode = word_mode;
3601 : }
3602 :
3603 : /* Copy the structure BITSIZE bits at a time. If the target lives in
3604 : memory, take care of not reading/writing past its end by selecting
3605 : a copy mode suited to BITSIZE. This should always be possible given
3606 : how it is computed.
3607 :
3608 : If the target lives in register, make sure not to select a copy mode
3609 : larger than the mode of the register.
3610 :
3611 : We could probably emit more efficient code for machines which do not use
3612 : strict alignment, but it doesn't seem worth the effort at the current
3613 : time. */
3614 :
3615 218 : copy_mode = word_mode;
3616 218 : if (MEM_P (target))
3617 : {
3618 218 : opt_scalar_int_mode mem_mode = int_mode_for_size (bitsize, 1);
3619 218 : if (mem_mode.exists ())
3620 218 : copy_mode = mem_mode.require ();
3621 : }
3622 0 : else if (REG_P (target) && GET_MODE_BITSIZE (tmode) < BITS_PER_WORD)
3623 : copy_mode = tmode;
3624 :
3625 218 : for (bitpos = 0, xbitpos = padding_correction;
3626 1068 : bitpos < bytes * BITS_PER_UNIT;
3627 850 : bitpos += bitsize, xbitpos += bitsize)
3628 : {
3629 : /* We need a new source operand each time xbitpos is on a
3630 : word boundary and when xbitpos == padding_correction
3631 : (the first time through). */
3632 850 : if (xbitpos % BITS_PER_WORD == 0 || xbitpos == padding_correction)
3633 218 : src = operand_subword_force (srcreg, xbitpos / BITS_PER_WORD, mode);
3634 :
3635 : /* We need a new destination operand each time bitpos is on
3636 : a word boundary. */
3637 850 : if (REG_P (target) && GET_MODE_BITSIZE (tmode) < BITS_PER_WORD)
3638 : dst = target;
3639 850 : else if (bitpos % BITS_PER_WORD == 0)
3640 218 : dst = operand_subword (target, bitpos / BITS_PER_WORD, 1, tmode);
3641 :
3642 : /* Use xbitpos for the source extraction (right justified) and
3643 : bitpos for the destination store (left justified). */
3644 850 : store_bit_field (dst, bitsize, bitpos % BITS_PER_WORD, 0, 0, copy_mode,
3645 850 : extract_bit_field (src, bitsize,
3646 850 : xbitpos % BITS_PER_WORD, 1,
3647 : NULL_RTX, copy_mode, copy_mode,
3648 : false, NULL),
3649 : false, false);
3650 : }
3651 : }
3652 :
3653 : /* Copy BLKmode value SRC into a register of mode MODE_IN. Return the
3654 : register if it contains any data, otherwise return null.
3655 :
3656 : This is used on targets that return BLKmode values in registers. */
3657 :
3658 : rtx
3659 3588 : copy_blkmode_to_reg (machine_mode mode_in, tree src)
3660 : {
3661 3588 : int i, n_regs;
3662 3588 : unsigned HOST_WIDE_INT bitpos, xbitpos, padding_correction = 0, bytes;
3663 3588 : unsigned int bitsize;
3664 3588 : rtx *dst_words, dst, x, src_word = NULL_RTX, dst_word = NULL_RTX;
3665 : /* No current ABI uses variable-sized modes to pass a BLKmnode type. */
3666 3588 : fixed_size_mode mode = as_a <fixed_size_mode> (mode_in);
3667 3588 : fixed_size_mode dst_mode;
3668 3588 : scalar_int_mode min_mode;
3669 :
3670 3588 : gcc_assert (TYPE_MODE (TREE_TYPE (src)) == BLKmode);
3671 :
3672 3588 : x = expand_normal (src);
3673 :
3674 3588 : bytes = arg_int_size_in_bytes (TREE_TYPE (src));
3675 3588 : if (bytes == 0)
3676 : return NULL_RTX;
3677 :
3678 : /* If the structure doesn't take up a whole number of words, see
3679 : whether the register value should be padded on the left or on
3680 : the right. Set PADDING_CORRECTION to the number of padding
3681 : bits needed on the left side.
3682 :
3683 : In most ABIs, the structure will be returned at the least end of
3684 : the register, which translates to right padding on little-endian
3685 : targets and left padding on big-endian targets. The opposite
3686 : holds if the structure is returned at the most significant
3687 : end of the register. */
3688 1224 : if (bytes % UNITS_PER_WORD != 0
3689 1224 : && (targetm.calls.return_in_msb (TREE_TYPE (src))
3690 1188 : ? !BYTES_BIG_ENDIAN
3691 : : BYTES_BIG_ENDIAN))
3692 0 : padding_correction = (BITS_PER_WORD - ((bytes % UNITS_PER_WORD)
3693 0 : * BITS_PER_UNIT));
3694 :
3695 1224 : n_regs = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
3696 1224 : dst_words = XALLOCAVEC (rtx, n_regs);
3697 1224 : bitsize = MIN (TYPE_ALIGN (TREE_TYPE (src)), BITS_PER_WORD);
3698 1224 : min_mode = smallest_int_mode_for_size (bitsize).require ();
3699 :
3700 : /* Copy the structure BITSIZE bits at a time. */
3701 1224 : for (bitpos = 0, xbitpos = padding_correction;
3702 3974 : bitpos < bytes * BITS_PER_UNIT;
3703 2750 : bitpos += bitsize, xbitpos += bitsize)
3704 : {
3705 : /* We need a new destination pseudo each time xbitpos is
3706 : on a word boundary and when xbitpos == padding_correction
3707 : (the first time through). */
3708 2750 : if (xbitpos % BITS_PER_WORD == 0
3709 1523 : || xbitpos == padding_correction)
3710 : {
3711 : /* Generate an appropriate register. */
3712 1227 : dst_word = gen_reg_rtx (word_mode);
3713 1227 : dst_words[xbitpos / BITS_PER_WORD] = dst_word;
3714 :
3715 : /* Clear the destination before we move anything into it. */
3716 1227 : emit_move_insn (dst_word, CONST0_RTX (word_mode));
3717 : }
3718 :
3719 : /* Find the largest integer mode that can be used to copy all or as
3720 : many bits as possible of the structure if the target supports larger
3721 : copies. There are too many corner cases here w.r.t to alignments on
3722 : the read/writes. So if there is any padding just use single byte
3723 : operations. */
3724 2750 : opt_scalar_int_mode mode_iter;
3725 2750 : if (padding_correction == 0 && !STRICT_ALIGNMENT)
3726 : {
3727 7488 : FOR_EACH_MODE_FROM (mode_iter, min_mode)
3728 : {
3729 7488 : unsigned int msize = GET_MODE_BITSIZE (mode_iter.require ());
3730 7488 : if (msize <= ((bytes * BITS_PER_UNIT) - bitpos)
3731 4741 : && msize <= BITS_PER_WORD)
3732 4738 : bitsize = msize;
3733 : else
3734 : break;
3735 : }
3736 : }
3737 :
3738 : /* We need a new source operand each time bitpos is on a word
3739 : boundary. */
3740 2750 : if (bitpos % BITS_PER_WORD == 0)
3741 1227 : src_word = operand_subword_force (x, bitpos / BITS_PER_WORD, BLKmode);
3742 :
3743 : /* Use bitpos for the source extraction (left justified) and
3744 : xbitpos for the destination store (right justified). */
3745 5500 : store_bit_field (dst_word, bitsize, xbitpos % BITS_PER_WORD,
3746 : 0, 0, word_mode,
3747 2750 : extract_bit_field (src_word, bitsize,
3748 2750 : bitpos % BITS_PER_WORD, 1,
3749 : NULL_RTX, word_mode, word_mode,
3750 : false, NULL),
3751 : false, false);
3752 : }
3753 :
3754 1224 : if (mode == BLKmode)
3755 : {
3756 : /* Find the smallest integer mode large enough to hold the
3757 : entire structure. */
3758 0 : opt_scalar_int_mode mode_iter;
3759 0 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
3760 0 : if (GET_MODE_SIZE (mode_iter.require ()) >= bytes)
3761 : break;
3762 :
3763 : /* A suitable mode should have been found. */
3764 0 : mode = mode_iter.require ();
3765 : }
3766 :
3767 3672 : if (GET_MODE_SIZE (mode) < GET_MODE_SIZE (word_mode))
3768 : dst_mode = word_mode;
3769 : else
3770 498 : dst_mode = mode;
3771 1224 : dst = gen_reg_rtx (dst_mode);
3772 :
3773 3675 : for (i = 0; i < n_regs; i++)
3774 1227 : emit_move_insn (operand_subword (dst, i, 0, dst_mode), dst_words[i]);
3775 :
3776 1224 : if (mode != dst_mode)
3777 726 : dst = gen_lowpart (mode, dst);
3778 :
3779 : return dst;
3780 : }
3781 :
3782 : /* Add a USE expression for REG to the (possibly empty) list pointed
3783 : to by CALL_FUSAGE. REG must denote a hard register. */
3784 :
3785 : void
3786 11661220 : use_reg_mode (rtx *call_fusage, rtx reg, machine_mode mode)
3787 : {
3788 11661220 : gcc_assert (REG_P (reg));
3789 :
3790 11661220 : if (!HARD_REGISTER_P (reg))
3791 : return;
3792 :
3793 11661220 : *call_fusage
3794 11661220 : = gen_rtx_EXPR_LIST (mode, gen_rtx_USE (VOIDmode, reg), *call_fusage);
3795 : }
3796 :
3797 : /* Add a CLOBBER expression for REG to the (possibly empty) list pointed
3798 : to by CALL_FUSAGE. REG must denote a hard register. */
3799 :
3800 : void
3801 0 : clobber_reg_mode (rtx *call_fusage, rtx reg, machine_mode mode)
3802 : {
3803 0 : gcc_assert (REG_P (reg) && REGNO (reg) < FIRST_PSEUDO_REGISTER);
3804 :
3805 0 : *call_fusage
3806 0 : = gen_rtx_EXPR_LIST (mode, gen_rtx_CLOBBER (VOIDmode, reg), *call_fusage);
3807 0 : }
3808 :
3809 : /* Add USE expressions to *CALL_FUSAGE for each of NREGS consecutive regs,
3810 : starting at REGNO. All of these registers must be hard registers. */
3811 :
3812 : void
3813 1608 : use_regs (rtx *call_fusage, int regno, int nregs)
3814 : {
3815 1608 : int i;
3816 :
3817 1608 : gcc_assert (regno + nregs <= FIRST_PSEUDO_REGISTER);
3818 :
3819 3224 : for (i = 0; i < nregs; i++)
3820 1616 : use_reg (call_fusage, regno_reg_rtx[regno + i]);
3821 1608 : }
3822 :
3823 : /* Add USE expressions to *CALL_FUSAGE for each REG contained in the
3824 : PARALLEL REGS. This is for calls that pass values in multiple
3825 : non-contiguous locations. The Irix 6 ABI has examples of this. */
3826 :
3827 : void
3828 295111 : use_group_regs (rtx *call_fusage, rtx regs)
3829 : {
3830 295111 : int i;
3831 :
3832 875924 : for (i = 0; i < XVECLEN (regs, 0); i++)
3833 : {
3834 580813 : rtx reg = XEXP (XVECEXP (regs, 0, i), 0);
3835 :
3836 : /* A NULL entry means the parameter goes both on the stack and in
3837 : registers. This can also be a MEM for targets that pass values
3838 : partially on the stack and partially in registers. */
3839 580813 : if (reg != 0 && REG_P (reg))
3840 580813 : use_reg (call_fusage, reg);
3841 : }
3842 295111 : }
3843 :
3844 : /* Return the defining gimple statement for SSA_NAME NAME if it is an
3845 : assignment and the code of the expression on the RHS is CODE. Return
3846 : NULL otherwise. */
3847 :
3848 : static gimple *
3849 12112723 : get_def_for_expr (tree name, enum tree_code code)
3850 : {
3851 12112723 : gimple *def_stmt;
3852 :
3853 12112723 : if (TREE_CODE (name) != SSA_NAME)
3854 : return NULL;
3855 :
3856 9435795 : def_stmt = get_gimple_for_ssa_name (name);
3857 9435795 : if (!def_stmt
3858 2022148 : || !is_gimple_assign (def_stmt)
3859 11456468 : || gimple_assign_rhs_code (def_stmt) != code)
3860 9403520 : return NULL;
3861 :
3862 : return def_stmt;
3863 : }
3864 :
3865 : /* Return the defining gimple statement for SSA_NAME NAME if it is an
3866 : assignment and the class of the expression on the RHS is CLASS. Return
3867 : NULL otherwise. */
3868 :
3869 : static gimple *
3870 16660 : get_def_for_expr_class (tree name, enum tree_code_class tclass)
3871 : {
3872 16660 : gimple *def_stmt;
3873 :
3874 16660 : if (TREE_CODE (name) != SSA_NAME)
3875 : return NULL;
3876 :
3877 16660 : def_stmt = get_gimple_for_ssa_name (name);
3878 16660 : if (!def_stmt
3879 14561 : || !is_gimple_assign (def_stmt)
3880 31221 : || TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) != tclass)
3881 2233 : return NULL;
3882 :
3883 : return def_stmt;
3884 : }
3885 :
3886 : /* Write zeros through the storage of OBJECT. If OBJECT has BLKmode, SIZE is
3887 : its length in bytes. */
3888 :
3889 : rtx
3890 158233 : clear_storage_hints (rtx object, rtx size, enum block_op_methods method,
3891 : unsigned int expected_align, HOST_WIDE_INT expected_size,
3892 : unsigned HOST_WIDE_INT min_size,
3893 : unsigned HOST_WIDE_INT max_size,
3894 : unsigned HOST_WIDE_INT probable_max_size,
3895 : unsigned ctz_size)
3896 : {
3897 158233 : machine_mode mode = GET_MODE (object);
3898 158233 : unsigned int align;
3899 :
3900 158233 : gcc_assert (method == BLOCK_OP_NORMAL || method == BLOCK_OP_TAILCALL);
3901 :
3902 : /* If OBJECT is not BLKmode and SIZE is the same size as its mode,
3903 : just move a zero. Otherwise, do this a piece at a time. */
3904 158233 : poly_int64 size_val;
3905 158233 : if (mode != BLKmode
3906 57952 : && poly_int_rtx_p (size, &size_val)
3907 216185 : && known_eq (size_val, GET_MODE_SIZE (mode)))
3908 : {
3909 57952 : rtx zero = CONST0_RTX (mode);
3910 57952 : if (zero != NULL)
3911 : {
3912 57952 : emit_move_insn (object, zero);
3913 57952 : return NULL;
3914 : }
3915 :
3916 0 : if (COMPLEX_MODE_P (mode))
3917 : {
3918 0 : zero = CONST0_RTX (GET_MODE_INNER (mode));
3919 0 : if (zero != NULL)
3920 : {
3921 0 : write_complex_part (object, zero, 0, true);
3922 0 : write_complex_part (object, zero, 1, false);
3923 0 : return NULL;
3924 : }
3925 : }
3926 : }
3927 :
3928 100281 : if (size == const0_rtx)
3929 : return NULL;
3930 :
3931 100280 : align = MEM_ALIGN (object);
3932 :
3933 100280 : if (CONST_INT_P (size)
3934 100280 : && targetm.use_by_pieces_infrastructure_p (INTVAL (size), align,
3935 : CLEAR_BY_PIECES,
3936 : optimize_insn_for_speed_p ()))
3937 71989 : clear_by_pieces (object, INTVAL (size), align);
3938 28291 : else if (set_storage_via_setmem (object, size, const0_rtx, align,
3939 : expected_align, expected_size,
3940 : min_size, max_size, probable_max_size))
3941 : ;
3942 16122 : else if (try_store_by_multiple_pieces (object, size, ctz_size,
3943 : min_size, max_size,
3944 : NULL_RTX, 0, align))
3945 : ;
3946 15858 : else if (ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (object)))
3947 15858 : return set_storage_via_libcall (object, size, const0_rtx,
3948 15858 : method == BLOCK_OP_TAILCALL);
3949 : else
3950 0 : gcc_unreachable ();
3951 :
3952 : return NULL;
3953 : }
3954 :
3955 : rtx
3956 110513 : clear_storage (rtx object, rtx size, enum block_op_methods method)
3957 : {
3958 110513 : unsigned HOST_WIDE_INT max, min = 0;
3959 110513 : if (GET_CODE (size) == CONST_INT)
3960 110513 : min = max = UINTVAL (size);
3961 : else
3962 0 : max = GET_MODE_MASK (GET_MODE (size));
3963 110513 : return clear_storage_hints (object, size, method, 0, -1, min, max, max, 0);
3964 : }
3965 :
3966 :
3967 : /* A subroutine of clear_storage. Expand a call to memset.
3968 : Return the return value of memset, 0 otherwise. */
3969 :
3970 : rtx
3971 15859 : set_storage_via_libcall (rtx object, rtx size, rtx val, bool tailcall)
3972 : {
3973 15859 : tree call_expr, fn, object_tree, size_tree, val_tree;
3974 15859 : machine_mode size_mode;
3975 :
3976 15859 : object = copy_addr_to_reg (XEXP (object, 0));
3977 15859 : object_tree = make_tree (ptr_type_node, object);
3978 :
3979 15859 : if (!CONST_INT_P (val))
3980 0 : val = convert_to_mode (TYPE_MODE (integer_type_node), val, 1);
3981 15859 : val_tree = make_tree (integer_type_node, val);
3982 :
3983 15859 : size_mode = TYPE_MODE (sizetype);
3984 15859 : size = convert_to_mode (size_mode, size, 1);
3985 15859 : size = copy_to_mode_reg (size_mode, size);
3986 15859 : size_tree = make_tree (sizetype, size);
3987 :
3988 : /* It is incorrect to use the libcall calling conventions for calls to
3989 : memset because it can be provided by the user. */
3990 15859 : fn = builtin_decl_implicit (BUILT_IN_MEMSET);
3991 15859 : call_expr = build_call_expr (fn, 3, object_tree, val_tree, size_tree);
3992 15859 : CALL_EXPR_TAILCALL (call_expr) = tailcall;
3993 :
3994 15859 : return expand_call (call_expr, NULL_RTX, false);
3995 : }
3996 :
3997 : /* Expand a setmem pattern; return true if successful. */
3998 :
3999 : bool
4000 36913 : set_storage_via_setmem (rtx object, rtx size, rtx val, unsigned int align,
4001 : unsigned int expected_align, HOST_WIDE_INT expected_size,
4002 : unsigned HOST_WIDE_INT min_size,
4003 : unsigned HOST_WIDE_INT max_size,
4004 : unsigned HOST_WIDE_INT probable_max_size)
4005 : {
4006 : /* Try the most limited insn first, because there's no point
4007 : including more than one in the machine description unless
4008 : the more limited one has some advantage. */
4009 :
4010 36913 : if (expected_align < align)
4011 : expected_align = align;
4012 36913 : if (expected_size != -1)
4013 : {
4014 8 : if ((unsigned HOST_WIDE_INT)expected_size > max_size)
4015 0 : expected_size = max_size;
4016 8 : if ((unsigned HOST_WIDE_INT)expected_size < min_size)
4017 0 : expected_size = min_size;
4018 : }
4019 :
4020 36913 : opt_scalar_int_mode mode_iter;
4021 223890 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
4022 : {
4023 201409 : scalar_int_mode mode = mode_iter.require ();
4024 201409 : enum insn_code code = direct_optab_handler (setmem_optab, mode);
4025 :
4026 201409 : if (code != CODE_FOR_nothing
4027 : /* We don't need MODE to be narrower than BITS_PER_HOST_WIDE_INT
4028 : here because if SIZE is less than the mode mask, as it is
4029 : returned by the macro, it will definitely be less than the
4030 : actual mode mask. Since SIZE is within the Pmode address
4031 : space, we limit MODE to Pmode. */
4032 201409 : && ((CONST_INT_P (size)
4033 31777 : && ((unsigned HOST_WIDE_INT) INTVAL (size)
4034 31777 : <= (GET_MODE_MASK (mode) >> 1)))
4035 26733 : || max_size <= (GET_MODE_MASK (mode) >> 1)
4036 31988 : || GET_MODE_BITSIZE (mode) >= GET_MODE_BITSIZE (Pmode)))
4037 : {
4038 47829 : class expand_operand ops[9];
4039 47829 : unsigned int nops;
4040 :
4041 47829 : nops = insn_data[(int) code].n_generator_args;
4042 47829 : gcc_assert (nops == 4 || nops == 6 || nops == 8 || nops == 9);
4043 :
4044 47829 : create_fixed_operand (&ops[0], object);
4045 : /* The check above guarantees that this size conversion is valid. */
4046 47829 : create_convert_operand_to (&ops[1], size, mode, true);
4047 47829 : create_convert_operand_from (&ops[2], val, byte_mode, true);
4048 47829 : create_integer_operand (&ops[3], align / BITS_PER_UNIT);
4049 47829 : if (nops >= 6)
4050 : {
4051 47829 : create_integer_operand (&ops[4], expected_align / BITS_PER_UNIT);
4052 47829 : create_integer_operand (&ops[5], expected_size);
4053 : }
4054 47829 : if (nops >= 8)
4055 : {
4056 47829 : create_integer_operand (&ops[6], min_size);
4057 : /* If we cannot represent the maximal size,
4058 : make parameter NULL. */
4059 47829 : if ((HOST_WIDE_INT) max_size != -1)
4060 44562 : create_integer_operand (&ops[7], max_size);
4061 : else
4062 3267 : create_fixed_operand (&ops[7], NULL);
4063 : }
4064 47829 : if (nops == 9)
4065 : {
4066 : /* If we cannot represent the maximal size,
4067 : make parameter NULL. */
4068 47829 : if ((HOST_WIDE_INT) probable_max_size != -1)
4069 44722 : create_integer_operand (&ops[8], probable_max_size);
4070 : else
4071 3107 : create_fixed_operand (&ops[8], NULL);
4072 : }
4073 47829 : gcc_assert (min_size != max_size
4074 : || rtx_equal_p (ops[1].value, GEN_INT (min_size)));
4075 47829 : if (maybe_expand_insn (code, nops, ops))
4076 14432 : return true;
4077 : }
4078 : }
4079 :
4080 : return false;
4081 : }
4082 :
4083 :
4084 : /* Write to one of the components of the complex value CPLX. Write VAL to
4085 : the real part if IMAG_P is false, and the imaginary part if its true.
4086 : If UNDEFINED_P then the value in CPLX is currently undefined. */
4087 :
4088 : void
4089 578083 : write_complex_part (rtx cplx, rtx val, bool imag_p, bool undefined_p)
4090 : {
4091 578083 : machine_mode cmode;
4092 578083 : scalar_mode imode;
4093 578083 : unsigned ibitsize;
4094 :
4095 578083 : if (GET_CODE (cplx) == CONCAT)
4096 : {
4097 518733 : emit_move_insn (XEXP (cplx, imag_p), val);
4098 518733 : return;
4099 : }
4100 :
4101 59350 : cmode = GET_MODE (cplx);
4102 59350 : imode = GET_MODE_INNER (cmode);
4103 59350 : ibitsize = GET_MODE_BITSIZE (imode);
4104 :
4105 : /* For MEMs simplify_gen_subreg may generate an invalid new address
4106 : because, e.g., the original address is considered mode-dependent
4107 : by the target, which restricts simplify_subreg from invoking
4108 : adjust_address_nv. Instead of preparing fallback support for an
4109 : invalid address, we call adjust_address_nv directly. */
4110 59350 : if (MEM_P (cplx))
4111 : {
4112 69588 : emit_move_insn (adjust_address_nv (cplx, imode,
4113 : imag_p ? GET_MODE_SIZE (imode) : 0),
4114 : val);
4115 46392 : return;
4116 : }
4117 :
4118 : /* If the sub-object is at least word sized, then we know that subregging
4119 : will work. This special case is important, since store_bit_field
4120 : wants to operate on integer modes, and there's rarely an OImode to
4121 : correspond to TCmode. */
4122 12958 : if (ibitsize >= BITS_PER_WORD
4123 : /* For hard regs we have exact predicates. Assume we can split
4124 : the original object if it spans an even number of hard regs.
4125 : This special case is important for SCmode on 64-bit platforms
4126 : where the natural size of floating-point regs is 32-bit. */
4127 12958 : || (REG_P (cplx)
4128 7324 : && REGNO (cplx) < FIRST_PSEUDO_REGISTER
4129 7122 : && REG_NREGS (cplx) % 2 == 0))
4130 : {
4131 5650 : rtx part = simplify_gen_subreg (imode, cplx, cmode,
4132 8475 : imag_p ? GET_MODE_SIZE (imode) : 0);
4133 5650 : if (part)
4134 : {
4135 5650 : emit_move_insn (part, val);
4136 5650 : return;
4137 : }
4138 : else
4139 : /* simplify_gen_subreg may fail for sub-word MEMs. */
4140 0 : gcc_assert (MEM_P (cplx) && ibitsize < BITS_PER_WORD);
4141 : }
4142 :
4143 10962 : store_bit_field (cplx, ibitsize, imag_p ? ibitsize : 0, 0, 0, imode, val,
4144 : false, undefined_p);
4145 : }
4146 :
4147 : /* Extract one of the components of the complex value CPLX. Extract the
4148 : real part if IMAG_P is false, and the imaginary part if it's true. */
4149 :
4150 : rtx
4151 566634 : read_complex_part (rtx cplx, bool imag_p)
4152 : {
4153 566634 : machine_mode cmode;
4154 566634 : scalar_mode imode;
4155 566634 : unsigned ibitsize;
4156 :
4157 566634 : if (GET_CODE (cplx) == CONCAT)
4158 349938 : return XEXP (cplx, imag_p);
4159 :
4160 216696 : cmode = GET_MODE (cplx);
4161 216696 : imode = GET_MODE_INNER (cmode);
4162 216696 : ibitsize = GET_MODE_BITSIZE (imode);
4163 :
4164 : /* Special case reads from complex constants that got spilled to memory. */
4165 216696 : if (MEM_P (cplx) && GET_CODE (XEXP (cplx, 0)) == SYMBOL_REF)
4166 : {
4167 37943 : tree decl = SYMBOL_REF_DECL (XEXP (cplx, 0));
4168 37943 : if (decl && TREE_CODE (decl) == COMPLEX_CST)
4169 : {
4170 0 : tree part = imag_p ? TREE_IMAGPART (decl) : TREE_REALPART (decl);
4171 0 : if (CONSTANT_CLASS_P (part))
4172 0 : return expand_expr (part, NULL_RTX, imode, EXPAND_NORMAL);
4173 : }
4174 : }
4175 :
4176 : /* For MEMs simplify_gen_subreg may generate an invalid new address
4177 : because, e.g., the original address is considered mode-dependent
4178 : by the target, which restricts simplify_subreg from invoking
4179 : adjust_address_nv. Instead of preparing fallback support for an
4180 : invalid address, we call adjust_address_nv directly. */
4181 216696 : if (MEM_P (cplx))
4182 238688 : return adjust_address_nv (cplx, imode,
4183 : imag_p ? GET_MODE_SIZE (imode) : 0);
4184 :
4185 : /* If the sub-object is at least word sized, then we know that subregging
4186 : will work. This special case is important, since extract_bit_field
4187 : wants to operate on integer modes, and there's rarely an OImode to
4188 : correspond to TCmode. */
4189 57404 : if (ibitsize >= BITS_PER_WORD
4190 : /* For hard regs we have exact predicates. Assume we can split
4191 : the original object if it spans an even number of hard regs.
4192 : This special case is important for SCmode on 64-bit platforms
4193 : where the natural size of floating-point regs is 32-bit. */
4194 57404 : || (REG_P (cplx)
4195 35412 : && REGNO (cplx) < FIRST_PSEUDO_REGISTER
4196 292 : && REG_NREGS (cplx) % 2 == 0))
4197 : {
4198 10869 : rtx ret = simplify_gen_subreg (imode, cplx, cmode,
4199 16303 : imag_p ? GET_MODE_SIZE (imode) : 0);
4200 10869 : if (ret)
4201 : return ret;
4202 : else
4203 : /* simplify_gen_subreg may fail for sub-word MEMs. */
4204 0 : gcc_assert (MEM_P (cplx) && ibitsize < BITS_PER_WORD);
4205 : }
4206 :
4207 69800 : return extract_bit_field (cplx, ibitsize, imag_p ? ibitsize : 0,
4208 : true, NULL_RTX, imode, imode, false, NULL);
4209 : }
4210 :
4211 : /* A subroutine of emit_move_insn_1. Yet another lowpart generator.
4212 : NEW_MODE and OLD_MODE are the same size. Return NULL if X cannot be
4213 : represented in NEW_MODE. If FORCE is true, this will never happen, as
4214 : we'll force-create a SUBREG if needed. */
4215 :
4216 : static rtx
4217 278106 : emit_move_change_mode (machine_mode new_mode,
4218 : machine_mode old_mode, rtx x, bool force)
4219 : {
4220 278106 : rtx ret;
4221 :
4222 278106 : if (push_operand (x, GET_MODE (x)))
4223 : {
4224 1318 : ret = gen_rtx_MEM (new_mode, XEXP (x, 0));
4225 1318 : MEM_COPY_ATTRIBUTES (ret, x);
4226 : }
4227 276788 : else if (MEM_P (x))
4228 : {
4229 : /* We don't have to worry about changing the address since the
4230 : size in bytes is supposed to be the same. */
4231 52774 : if (reload_in_progress)
4232 : {
4233 : /* Copy the MEM to change the mode and move any
4234 : substitutions from the old MEM to the new one. */
4235 0 : ret = adjust_address_nv (x, new_mode, 0);
4236 0 : copy_replacements (x, ret);
4237 : }
4238 : else
4239 52774 : ret = adjust_address (x, new_mode, 0);
4240 : }
4241 : else
4242 : {
4243 : /* Note that we do want simplify_subreg's behavior of validating
4244 : that the new mode is ok for a hard register. If we were to use
4245 : simplify_gen_subreg, we would create the subreg, but would
4246 : probably run into the target not being able to implement it. */
4247 : /* Except, of course, when FORCE is true, when this is exactly what
4248 : we want. Which is needed for CCmodes on some targets. */
4249 224014 : if (force)
4250 224014 : ret = simplify_gen_subreg (new_mode, x, old_mode, 0);
4251 : else
4252 0 : ret = simplify_subreg (new_mode, x, old_mode, 0);
4253 : }
4254 :
4255 278106 : return ret;
4256 : }
4257 :
4258 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X using
4259 : an integer mode of the same size as MODE. Returns the instruction
4260 : emitted, or NULL if such a move could not be generated. */
4261 :
4262 : static rtx_insn *
4263 139053 : emit_move_via_integer (machine_mode mode, rtx x, rtx y, bool force)
4264 : {
4265 139053 : scalar_int_mode imode;
4266 139053 : enum insn_code code;
4267 :
4268 : /* There must exist a mode of the exact size we require. */
4269 139053 : if (!int_mode_for_mode (mode).exists (&imode))
4270 0 : return NULL;
4271 :
4272 : /* The target must support moves in this mode. */
4273 139053 : code = optab_handler (mov_optab, imode);
4274 139053 : if (code == CODE_FOR_nothing)
4275 : return NULL;
4276 :
4277 139053 : x = emit_move_change_mode (imode, mode, x, force);
4278 139053 : if (x == NULL_RTX)
4279 : return NULL;
4280 139053 : y = emit_move_change_mode (imode, mode, y, force);
4281 139053 : if (y == NULL_RTX)
4282 : return NULL;
4283 139053 : return emit_insn (GEN_FCN (code) (x, y));
4284 : }
4285 :
4286 : /* A subroutine of emit_move_insn_1. X is a push_operand in MODE.
4287 : Return an equivalent MEM that does not use an auto-increment. */
4288 :
4289 : rtx
4290 3637 : emit_move_resolve_push (machine_mode mode, rtx x)
4291 : {
4292 3637 : enum rtx_code code = GET_CODE (XEXP (x, 0));
4293 3637 : rtx temp;
4294 :
4295 7274 : poly_int64 adjust = GET_MODE_SIZE (mode);
4296 : #ifdef PUSH_ROUNDING
4297 3637 : adjust = PUSH_ROUNDING (adjust);
4298 : #endif
4299 3637 : if (code == PRE_DEC || code == POST_DEC)
4300 3241 : adjust = -adjust;
4301 396 : else if (code == PRE_MODIFY || code == POST_MODIFY)
4302 : {
4303 396 : rtx expr = XEXP (XEXP (x, 0), 1);
4304 :
4305 396 : gcc_assert (GET_CODE (expr) == PLUS || GET_CODE (expr) == MINUS);
4306 396 : poly_int64 val = rtx_to_poly_int64 (XEXP (expr, 1));
4307 396 : if (GET_CODE (expr) == MINUS)
4308 0 : val = -val;
4309 396 : gcc_assert (known_eq (adjust, val) || known_eq (adjust, -val));
4310 : adjust = val;
4311 : }
4312 :
4313 : /* Do not use anti_adjust_stack, since we don't want to update
4314 : stack_pointer_delta. */
4315 3641 : temp = expand_simple_binop (Pmode, PLUS, stack_pointer_rtx,
4316 3637 : gen_int_mode (adjust, Pmode), stack_pointer_rtx,
4317 : 0, OPTAB_LIB_WIDEN);
4318 3637 : if (temp != stack_pointer_rtx)
4319 0 : emit_move_insn (stack_pointer_rtx, temp);
4320 :
4321 3637 : switch (code)
4322 : {
4323 3637 : case PRE_INC:
4324 3637 : case PRE_DEC:
4325 3637 : case PRE_MODIFY:
4326 3637 : temp = stack_pointer_rtx;
4327 3637 : break;
4328 : case POST_INC:
4329 : case POST_DEC:
4330 : case POST_MODIFY:
4331 0 : temp = plus_constant (Pmode, stack_pointer_rtx, -adjust);
4332 0 : break;
4333 0 : default:
4334 0 : gcc_unreachable ();
4335 : }
4336 :
4337 3637 : return replace_equiv_address (x, temp);
4338 : }
4339 :
4340 : /* A subroutine of emit_move_complex. Generate a move from Y into X.
4341 : X is known to satisfy push_operand, and MODE is known to be complex.
4342 : Returns the last instruction emitted. */
4343 :
4344 : rtx_insn *
4345 5760 : emit_move_complex_push (machine_mode mode, rtx x, rtx y)
4346 : {
4347 5760 : scalar_mode submode = GET_MODE_INNER (mode);
4348 5760 : bool imag_first;
4349 :
4350 : #ifdef PUSH_ROUNDING
4351 11520 : poly_int64 submodesize = GET_MODE_SIZE (submode);
4352 :
4353 : /* In case we output to the stack, but the size is smaller than the
4354 : machine can push exactly, we need to use move instructions. */
4355 5760 : if (maybe_ne (PUSH_ROUNDING (submodesize), submodesize))
4356 : {
4357 718 : x = emit_move_resolve_push (mode, x);
4358 718 : return emit_move_insn (x, y);
4359 : }
4360 : #endif
4361 :
4362 : /* Note that the real part always precedes the imag part in memory
4363 : regardless of machine's endianness. */
4364 5042 : switch (GET_CODE (XEXP (x, 0)))
4365 : {
4366 : case PRE_DEC:
4367 : case POST_DEC:
4368 : imag_first = true;
4369 : break;
4370 0 : case PRE_INC:
4371 0 : case POST_INC:
4372 0 : imag_first = false;
4373 0 : break;
4374 0 : default:
4375 0 : gcc_unreachable ();
4376 : }
4377 :
4378 5042 : emit_move_insn (gen_rtx_MEM (submode, XEXP (x, 0)),
4379 : read_complex_part (y, imag_first));
4380 5042 : return emit_move_insn (gen_rtx_MEM (submode, XEXP (x, 0)),
4381 5042 : read_complex_part (y, !imag_first));
4382 : }
4383 :
4384 : /* A subroutine of emit_move_complex. Perform the move from Y to X
4385 : via two moves of the parts. Returns the last instruction emitted. */
4386 :
4387 : rtx_insn *
4388 90125 : emit_move_complex_parts (rtx x, rtx y)
4389 : {
4390 : /* Show the output dies here. This is necessary for SUBREGs
4391 : of pseudos since we cannot track their lifetimes correctly;
4392 : hard regs shouldn't appear here except as return values. */
4393 90125 : if (!reload_completed && !reload_in_progress
4394 180250 : && REG_P (x) && !reg_overlap_mentioned_p (x, y))
4395 6472 : emit_clobber (x);
4396 :
4397 90125 : write_complex_part (x, read_complex_part (y, false), false, true);
4398 90125 : write_complex_part (x, read_complex_part (y, true), true, false);
4399 :
4400 90125 : return get_last_insn ();
4401 : }
4402 :
4403 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X.
4404 : MODE is known to be complex. Returns the last instruction emitted. */
4405 :
4406 : static rtx_insn *
4407 112770 : emit_move_complex (machine_mode mode, rtx x, rtx y)
4408 : {
4409 112770 : bool try_int;
4410 :
4411 : /* Need to take special care for pushes, to maintain proper ordering
4412 : of the data, and possibly extra padding. */
4413 112770 : if (push_operand (x, mode))
4414 5152 : return emit_move_complex_push (mode, x, y);
4415 :
4416 : /* See if we can coerce the target into moving both values at once, except
4417 : for floating point where we favor moving as parts if this is easy. */
4418 107618 : if (GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT
4419 64305 : && optab_handler (mov_optab, GET_MODE_INNER (mode)) != CODE_FOR_nothing
4420 64305 : && !(REG_P (x)
4421 2549 : && HARD_REGISTER_P (x)
4422 740 : && REG_NREGS (x) == 1)
4423 171923 : && !(REG_P (y)
4424 3622 : && HARD_REGISTER_P (y)
4425 1811 : && REG_NREGS (y) == 1))
4426 : try_int = false;
4427 : /* Not possible if the values are inherently not adjacent. */
4428 43313 : else if (GET_CODE (x) == CONCAT || GET_CODE (y) == CONCAT)
4429 : try_int = false;
4430 : /* Is possible if both are registers (or subregs of registers). */
4431 18302 : else if (register_operand (x, mode) && register_operand (y, mode))
4432 : try_int = true;
4433 : /* If one of the operands is a memory, and alignment constraints
4434 : are friendly enough, we may be able to do combined memory operations.
4435 : We do not attempt this if Y is a constant because that combination is
4436 : usually better with the by-parts thing below. */
4437 635 : else if ((MEM_P (x) ? !CONSTANT_P (y) : MEM_P (y))
4438 : && (!STRICT_ALIGNMENT
4439 : || get_mode_alignment (mode) == BIGGEST_ALIGNMENT))
4440 : try_int = true;
4441 : else
4442 : try_int = false;
4443 :
4444 : if (try_int)
4445 : {
4446 18302 : rtx_insn *ret;
4447 :
4448 : /* For memory to memory moves, optimal behavior can be had with the
4449 : existing block move logic. But use normal expansion if optimizing
4450 : for size. */
4451 18302 : if (MEM_P (x) && MEM_P (y))
4452 : {
4453 726 : emit_block_move (x, y, gen_int_mode (GET_MODE_SIZE (mode), Pmode),
4454 360 : (optimize_insn_for_speed_p()
4455 : ? BLOCK_OP_NO_LIBCALL : BLOCK_OP_NORMAL));
4456 360 : return get_last_insn ();
4457 : }
4458 :
4459 17942 : ret = emit_move_via_integer (mode, x, y, true);
4460 17942 : if (ret)
4461 : return ret;
4462 : }
4463 :
4464 89316 : return emit_move_complex_parts (x, y);
4465 : }
4466 :
4467 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X.
4468 : MODE is known to be MODE_CC. Returns the last instruction emitted. */
4469 :
4470 : static rtx_insn *
4471 0 : emit_move_ccmode (machine_mode mode, rtx x, rtx y)
4472 : {
4473 0 : rtx_insn *ret;
4474 :
4475 : /* Assume all MODE_CC modes are equivalent; if we have movcc, use it. */
4476 0 : if (mode != CCmode)
4477 : {
4478 0 : enum insn_code code = optab_handler (mov_optab, CCmode);
4479 0 : if (code != CODE_FOR_nothing)
4480 : {
4481 0 : x = emit_move_change_mode (CCmode, mode, x, true);
4482 0 : y = emit_move_change_mode (CCmode, mode, y, true);
4483 0 : return emit_insn (GEN_FCN (code) (x, y));
4484 : }
4485 : }
4486 :
4487 : /* Otherwise, find the MODE_INT mode of the same width. */
4488 0 : ret = emit_move_via_integer (mode, x, y, false);
4489 0 : gcc_assert (ret != NULL);
4490 : return ret;
4491 : }
4492 :
4493 : /* Return true if word I of OP lies entirely in the
4494 : undefined bits of a paradoxical subreg. */
4495 :
4496 : static bool
4497 0 : undefined_operand_subword_p (const_rtx op, int i)
4498 : {
4499 0 : if (GET_CODE (op) != SUBREG)
4500 : return false;
4501 0 : machine_mode innermostmode = GET_MODE (SUBREG_REG (op));
4502 0 : poly_int64 offset = i * UNITS_PER_WORD + subreg_memory_offset (op);
4503 0 : return (known_ge (offset, GET_MODE_SIZE (innermostmode))
4504 0 : || known_le (offset, -UNITS_PER_WORD));
4505 : }
4506 :
4507 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X.
4508 : MODE is any multi-word or full-word mode that lacks a move_insn
4509 : pattern. Note that you will get better code if you define such
4510 : patterns, even if they must turn into multiple assembler instructions. */
4511 :
4512 : static rtx_insn *
4513 0 : emit_move_multi_word (machine_mode mode, rtx x, rtx y)
4514 : {
4515 0 : rtx_insn *last_insn = 0;
4516 0 : rtx_insn *seq;
4517 0 : rtx inner;
4518 0 : bool need_clobber;
4519 0 : int i, mode_size;
4520 :
4521 : /* This function can only handle cases where the number of words is
4522 : known at compile time. */
4523 0 : mode_size = GET_MODE_SIZE (mode).to_constant ();
4524 0 : gcc_assert (mode_size >= UNITS_PER_WORD);
4525 :
4526 : /* If X is a push on the stack, do the push now and replace
4527 : X with a reference to the stack pointer. */
4528 0 : if (push_operand (x, mode))
4529 0 : x = emit_move_resolve_push (mode, x);
4530 :
4531 : /* If we are in reload, see if either operand is a MEM whose address
4532 : is scheduled for replacement. */
4533 0 : if (reload_in_progress && MEM_P (x)
4534 0 : && (inner = find_replacement (&XEXP (x, 0))) != XEXP (x, 0))
4535 0 : x = replace_equiv_address_nv (x, inner);
4536 0 : if (reload_in_progress && MEM_P (y)
4537 0 : && (inner = find_replacement (&XEXP (y, 0))) != XEXP (y, 0))
4538 0 : y = replace_equiv_address_nv (y, inner);
4539 :
4540 0 : start_sequence ();
4541 :
4542 0 : need_clobber = false;
4543 0 : for (i = 0; i < CEIL (mode_size, UNITS_PER_WORD); i++)
4544 : {
4545 : /* Do not generate code for a move if it would go entirely
4546 : to the non-existing bits of a paradoxical subreg. */
4547 0 : if (undefined_operand_subword_p (x, i))
4548 0 : continue;
4549 :
4550 0 : rtx xpart = operand_subword (x, i, 1, mode);
4551 0 : rtx ypart;
4552 :
4553 : /* Do not generate code for a move if it would come entirely
4554 : from the undefined bits of a paradoxical subreg. */
4555 0 : if (undefined_operand_subword_p (y, i))
4556 0 : continue;
4557 :
4558 0 : ypart = operand_subword (y, i, 1, mode);
4559 :
4560 : /* If we can't get a part of Y, put Y into memory if it is a
4561 : constant. Otherwise, force it into a register. Then we must
4562 : be able to get a part of Y. */
4563 0 : if (ypart == 0 && CONSTANT_P (y))
4564 : {
4565 0 : y = use_anchored_address (force_const_mem (mode, y));
4566 0 : ypart = operand_subword (y, i, 1, mode);
4567 : }
4568 0 : else if (ypart == 0)
4569 0 : ypart = operand_subword_force (y, i, mode);
4570 :
4571 0 : gcc_assert (xpart && ypart);
4572 :
4573 0 : need_clobber |= (GET_CODE (xpart) == SUBREG);
4574 :
4575 0 : last_insn = emit_move_insn (xpart, ypart);
4576 : }
4577 :
4578 0 : seq = end_sequence ();
4579 :
4580 : /* Show the output dies here. This is necessary for SUBREGs
4581 : of pseudos since we cannot track their lifetimes correctly;
4582 : hard regs shouldn't appear here except as return values.
4583 : We never want to emit such a clobber after reload. */
4584 0 : if (x != y
4585 0 : && ! (reload_in_progress || reload_completed)
4586 0 : && need_clobber != 0)
4587 0 : emit_clobber (x);
4588 :
4589 0 : emit_insn (seq);
4590 :
4591 0 : return last_insn;
4592 : }
4593 :
4594 : /* Low level part of emit_move_insn.
4595 : Called just like emit_move_insn, but assumes X and Y
4596 : are basically valid. */
4597 :
4598 : rtx_insn *
4599 77813259 : emit_move_insn_1 (rtx x, rtx y)
4600 : {
4601 77813259 : machine_mode mode = GET_MODE (x);
4602 77813259 : enum insn_code code;
4603 :
4604 77813259 : gcc_assert ((unsigned int) mode < (unsigned int) MAX_MACHINE_MODE);
4605 :
4606 77813259 : code = optab_handler (mov_optab, mode);
4607 77813259 : if (code != CODE_FOR_nothing)
4608 77579378 : return emit_insn (GEN_FCN (code) (x, y));
4609 :
4610 : /* Expand complex moves by moving real part and imag part. */
4611 233881 : if (COMPLEX_MODE_P (mode))
4612 112770 : return emit_move_complex (mode, x, y);
4613 :
4614 : if (GET_MODE_CLASS (mode) == MODE_DECIMAL_FLOAT
4615 : || ALL_FIXED_POINT_MODE_P (mode))
4616 : {
4617 121111 : rtx_insn *result = emit_move_via_integer (mode, x, y, true);
4618 :
4619 : /* If we can't find an integer mode, use multi words. */
4620 121111 : if (result)
4621 : return result;
4622 : else
4623 0 : return emit_move_multi_word (mode, x, y);
4624 : }
4625 :
4626 : if (GET_MODE_CLASS (mode) == MODE_CC)
4627 0 : return emit_move_ccmode (mode, x, y);
4628 :
4629 : /* Try using a move pattern for the corresponding integer mode. This is
4630 : only safe when simplify_subreg can convert MODE constants into integer
4631 : constants. At present, it can only do this reliably if the value
4632 : fits within a HOST_WIDE_INT. */
4633 0 : if (!CONSTANT_P (y)
4634 0 : || known_le (GET_MODE_BITSIZE (mode), HOST_BITS_PER_WIDE_INT))
4635 : {
4636 0 : rtx_insn *ret = emit_move_via_integer (mode, x, y, lra_in_progress);
4637 :
4638 0 : if (ret)
4639 : {
4640 0 : if (! lra_in_progress || recog (PATTERN (ret), ret, 0) >= 0)
4641 : return ret;
4642 : }
4643 : }
4644 :
4645 0 : return emit_move_multi_word (mode, x, y);
4646 : }
4647 :
4648 : /* Generate code to copy Y into X.
4649 : Both Y and X must have the same mode, except that
4650 : Y can be a constant with VOIDmode.
4651 : This mode cannot be BLKmode; use emit_block_move for that.
4652 :
4653 : Return the last instruction emitted. */
4654 :
4655 : rtx_insn *
4656 71840981 : emit_move_insn (rtx x, rtx y)
4657 : {
4658 71840981 : machine_mode mode = GET_MODE (x);
4659 71840981 : rtx y_cst = NULL_RTX;
4660 71840981 : rtx_insn *last_insn;
4661 71840981 : rtx set;
4662 :
4663 71840981 : gcc_assert (mode != BLKmode
4664 : && (GET_MODE (y) == mode || GET_MODE (y) == VOIDmode));
4665 :
4666 : /* If we have a copy that looks like one of the following patterns:
4667 : (set (subreg:M1 (reg:M2 ...)) (subreg:M1 (reg:M2 ...)))
4668 : (set (subreg:M1 (reg:M2 ...)) (mem:M1 ADDR))
4669 : (set (mem:M1 ADDR) (subreg:M1 (reg:M2 ...)))
4670 : (set (subreg:M1 (reg:M2 ...)) (constant C))
4671 : where mode M1 is equal in size to M2, try to detect whether the
4672 : mode change involves an implicit round trip through memory.
4673 : If so, see if we can avoid that by removing the subregs and
4674 : doing the move in mode M2 instead. */
4675 :
4676 71840981 : rtx x_inner = NULL_RTX;
4677 71840981 : rtx y_inner = NULL_RTX;
4678 :
4679 74654363 : auto candidate_subreg_p = [&](rtx subreg) {
4680 2813382 : return (REG_P (SUBREG_REG (subreg))
4681 8438325 : && known_eq (GET_MODE_SIZE (GET_MODE (SUBREG_REG (subreg))),
4682 : GET_MODE_SIZE (GET_MODE (subreg)))
4683 3200277 : && optab_handler (mov_optab, GET_MODE (SUBREG_REG (subreg)))
4684 2813382 : != CODE_FOR_nothing);
4685 : };
4686 :
4687 71847213 : auto candidate_mem_p = [&](machine_mode innermode, rtx mem) {
4688 6232 : return (!targetm.can_change_mode_class (innermode, GET_MODE (mem), ALL_REGS)
4689 6232 : && !push_operand (mem, GET_MODE (mem))
4690 : /* Not a candidate if innermode requires too much alignment. */
4691 12404 : && (MEM_ALIGN (mem) >= GET_MODE_ALIGNMENT (innermode)
4692 232 : || targetm.slow_unaligned_access (GET_MODE (mem),
4693 116 : MEM_ALIGN (mem))
4694 232 : || !targetm.slow_unaligned_access (innermode,
4695 116 : MEM_ALIGN (mem))));
4696 : };
4697 :
4698 71840981 : if (SUBREG_P (x) && candidate_subreg_p (x))
4699 14587 : x_inner = SUBREG_REG (x);
4700 :
4701 71840981 : if (SUBREG_P (y) && candidate_subreg_p (y))
4702 335844 : y_inner = SUBREG_REG (y);
4703 :
4704 71840981 : if (x_inner != NULL_RTX
4705 71840981 : && y_inner != NULL_RTX
4706 1172 : && GET_MODE (x_inner) == GET_MODE (y_inner)
4707 71841734 : && !targetm.can_change_mode_class (GET_MODE (x_inner), mode, ALL_REGS))
4708 : {
4709 753 : x = x_inner;
4710 753 : y = y_inner;
4711 753 : mode = GET_MODE (x_inner);
4712 : }
4713 71840228 : else if (x_inner != NULL_RTX
4714 13834 : && MEM_P (y)
4715 71840552 : && candidate_mem_p (GET_MODE (x_inner), y))
4716 : {
4717 324 : x = x_inner;
4718 324 : y = adjust_address (y, GET_MODE (x_inner), 0);
4719 324 : mode = GET_MODE (x_inner);
4720 : }
4721 71839904 : else if (y_inner != NULL_RTX
4722 335091 : && MEM_P (x)
4723 71845812 : && candidate_mem_p (GET_MODE (y_inner), x))
4724 : {
4725 5848 : x = adjust_address (x, GET_MODE (y_inner), 0);
4726 5848 : y = y_inner;
4727 5848 : mode = GET_MODE (y_inner);
4728 : }
4729 71834056 : else if (x_inner != NULL_RTX
4730 13510 : && CONSTANT_P (y)
4731 475 : && !targetm.can_change_mode_class (GET_MODE (x_inner),
4732 : mode, ALL_REGS)
4733 71834531 : && (y_inner = simplify_subreg (GET_MODE (x_inner), y, mode, 0)))
4734 : {
4735 475 : x = x_inner;
4736 475 : y = y_inner;
4737 475 : mode = GET_MODE (x_inner);
4738 : }
4739 :
4740 71840981 : if (CONSTANT_P (y))
4741 : {
4742 16774983 : if (optimize
4743 13036296 : && SCALAR_FLOAT_MODE_P (GET_MODE (x))
4744 17550762 : && (last_insn = compress_float_constant (x, y)))
4745 : return last_insn;
4746 :
4747 16705565 : y_cst = y;
4748 :
4749 16705565 : if (!targetm.legitimate_constant_p (mode, y))
4750 : {
4751 513332 : y = force_const_mem (mode, y);
4752 :
4753 : /* If the target's cannot_force_const_mem prevented the spill,
4754 : assume that the target's move expanders will also take care
4755 : of the non-legitimate constant. */
4756 513332 : if (!y)
4757 : y = y_cst;
4758 : else
4759 493215 : y = use_anchored_address (y);
4760 : }
4761 : }
4762 :
4763 : /* If X or Y are memory references, verify that their addresses are valid
4764 : for the machine. */
4765 71771563 : if (MEM_P (x)
4766 88060273 : && (! memory_address_addr_space_p (GET_MODE (x), XEXP (x, 0),
4767 14026531 : MEM_ADDR_SPACE (x))
4768 2262353 : && ! push_operand (x, GET_MODE (x))))
4769 174 : x = validize_mem (x);
4770 :
4771 71771563 : if (MEM_P (y)
4772 89618987 : && ! memory_address_addr_space_p (GET_MODE (y), XEXP (y, 0),
4773 17847424 : MEM_ADDR_SPACE (y)))
4774 8710 : y = validize_mem (y);
4775 :
4776 71771563 : gcc_assert (mode != BLKmode);
4777 :
4778 71771563 : last_insn = emit_move_insn_1 (x, y);
4779 :
4780 16705565 : if (y_cst && REG_P (x)
4781 12267690 : && (set = single_set (last_insn)) != NULL_RTX
4782 12267690 : && SET_DEST (set) == x
4783 84024738 : && ! rtx_equal_p (y_cst, SET_SRC (set)))
4784 1275158 : set_unique_reg_note (last_insn, REG_EQUAL, copy_rtx (y_cst));
4785 :
4786 : return last_insn;
4787 : }
4788 :
4789 : /* Generate the body of an instruction to copy Y into X.
4790 : It may be a list of insns, if one insn isn't enough. */
4791 :
4792 : rtx_insn *
4793 6041616 : gen_move_insn (rtx x, rtx y)
4794 : {
4795 6041616 : start_sequence ();
4796 6041616 : emit_move_insn_1 (x, y);
4797 6041616 : return end_sequence ();
4798 : }
4799 :
4800 : /* If Y is representable exactly in a narrower mode, and the target can
4801 : perform the extension directly from constant or memory, then emit the
4802 : move as an extension. */
4803 :
4804 : static rtx_insn *
4805 775779 : compress_float_constant (rtx x, rtx y)
4806 : {
4807 775779 : machine_mode dstmode = GET_MODE (x);
4808 775779 : machine_mode orig_srcmode = GET_MODE (y);
4809 775779 : machine_mode srcmode;
4810 775779 : const REAL_VALUE_TYPE *r;
4811 775779 : int oldcost, newcost;
4812 775779 : bool speed = optimize_insn_for_speed_p ();
4813 :
4814 775779 : r = CONST_DOUBLE_REAL_VALUE (y);
4815 :
4816 775779 : if (targetm.legitimate_constant_p (dstmode, y))
4817 773634 : oldcost = set_src_cost (y, orig_srcmode, speed);
4818 : else
4819 2145 : oldcost = set_src_cost (force_const_mem (dstmode, y), dstmode, speed);
4820 :
4821 2747240 : FOR_EACH_MODE_UNTIL (srcmode, orig_srcmode)
4822 : {
4823 2040879 : enum insn_code ic;
4824 2040879 : rtx trunc_y;
4825 2040879 : rtx_insn *last_insn;
4826 :
4827 : /* Skip if the target can't extend this way. */
4828 2040879 : ic = can_extend_p (dstmode, srcmode, 0);
4829 2040879 : if (ic == CODE_FOR_nothing)
4830 1615652 : continue;
4831 :
4832 : /* Skip if the narrowed value isn't exact. */
4833 425227 : if (! exact_real_truncate (srcmode, r))
4834 52640 : continue;
4835 :
4836 372587 : trunc_y = const_double_from_real_value (*r, srcmode);
4837 :
4838 372587 : if (targetm.legitimate_constant_p (srcmode, trunc_y))
4839 : {
4840 : /* Skip if the target needs extra instructions to perform
4841 : the extension. */
4842 368618 : if (!insn_operand_matches (ic, 1, trunc_y))
4843 3995 : continue;
4844 : /* This is valid, but may not be cheaper than the original. */
4845 364623 : newcost = set_src_cost (gen_rtx_FLOAT_EXTEND (dstmode, trunc_y),
4846 : dstmode, speed);
4847 364623 : if (oldcost < newcost)
4848 295205 : continue;
4849 : }
4850 3969 : else if (float_extend_from_mem[dstmode][srcmode])
4851 : {
4852 0 : trunc_y = force_const_mem (srcmode, trunc_y);
4853 : /* This is valid, but may not be cheaper than the original. */
4854 0 : newcost = set_src_cost (gen_rtx_FLOAT_EXTEND (dstmode, trunc_y),
4855 : dstmode, speed);
4856 0 : if (oldcost < newcost)
4857 0 : continue;
4858 0 : trunc_y = validize_mem (trunc_y);
4859 : }
4860 : else
4861 3969 : continue;
4862 :
4863 : /* For CSE's benefit, force the compressed constant pool entry
4864 : into a new pseudo. This constant may be used in different modes,
4865 : and if not, combine will put things back together for us. */
4866 69418 : trunc_y = force_reg (srcmode, trunc_y);
4867 :
4868 : /* If x is a hard register, perform the extension into a pseudo,
4869 : so that e.g. stack realignment code is aware of it. */
4870 69418 : rtx target = x;
4871 69418 : if (REG_P (x) && HARD_REGISTER_P (x))
4872 1 : target = gen_reg_rtx (dstmode);
4873 :
4874 69418 : emit_unop_insn (ic, target, trunc_y, UNKNOWN);
4875 69418 : last_insn = get_last_insn ();
4876 :
4877 69418 : if (REG_P (target))
4878 58625 : set_unique_reg_note (last_insn, REG_EQUAL, y);
4879 :
4880 69418 : if (target != x)
4881 1 : return emit_move_insn (x, target);
4882 : return last_insn;
4883 : }
4884 :
4885 : return NULL;
4886 : }
4887 :
4888 : /* Pushing data onto the stack. */
4889 :
4890 : /* Push a block of length SIZE (perhaps variable)
4891 : and return an rtx to address the beginning of the block.
4892 : The value may be virtual_outgoing_args_rtx.
4893 :
4894 : EXTRA is the number of bytes of padding to push in addition to SIZE.
4895 : BELOW nonzero means this padding comes at low addresses;
4896 : otherwise, the padding comes at high addresses. */
4897 :
4898 : rtx
4899 268980 : push_block (rtx size, poly_int64 extra, int below)
4900 : {
4901 268980 : rtx temp;
4902 :
4903 333939 : size = convert_modes (Pmode, ptr_mode, size, 1);
4904 268980 : if (CONSTANT_P (size))
4905 333939 : anti_adjust_stack (plus_constant (Pmode, size, extra));
4906 0 : else if (REG_P (size) && known_eq (extra, 0))
4907 0 : anti_adjust_stack (size);
4908 : else
4909 : {
4910 0 : temp = copy_to_mode_reg (Pmode, size);
4911 0 : if (maybe_ne (extra, 0))
4912 0 : temp = expand_binop (Pmode, add_optab, temp,
4913 0 : gen_int_mode (extra, Pmode),
4914 : temp, 0, OPTAB_LIB_WIDEN);
4915 0 : anti_adjust_stack (temp);
4916 : }
4917 :
4918 268980 : if (STACK_GROWS_DOWNWARD)
4919 : {
4920 268980 : temp = virtual_outgoing_args_rtx;
4921 268980 : if (maybe_ne (extra, 0) && below)
4922 0 : temp = plus_constant (Pmode, temp, extra);
4923 : }
4924 : else
4925 : {
4926 : poly_int64 csize;
4927 : if (poly_int_rtx_p (size, &csize))
4928 : temp = plus_constant (Pmode, virtual_outgoing_args_rtx,
4929 : -csize - (below ? 0 : extra));
4930 : else if (maybe_ne (extra, 0) && !below)
4931 : temp = gen_rtx_PLUS (Pmode, virtual_outgoing_args_rtx,
4932 : negate_rtx (Pmode, plus_constant (Pmode, size,
4933 : extra)));
4934 : else
4935 : temp = gen_rtx_PLUS (Pmode, virtual_outgoing_args_rtx,
4936 : negate_rtx (Pmode, size));
4937 : }
4938 :
4939 268980 : return memory_address (NARROWEST_INT_MODE, temp);
4940 : }
4941 :
4942 : /* A utility routine that returns the base of an auto-inc memory, or NULL. */
4943 :
4944 : static rtx
4945 2576154 : mem_autoinc_base (rtx mem)
4946 : {
4947 0 : if (MEM_P (mem))
4948 : {
4949 1572521 : rtx addr = XEXP (mem, 0);
4950 1572521 : if (GET_RTX_CLASS (GET_CODE (addr)) == RTX_AUTOINC)
4951 1029003 : return XEXP (addr, 0);
4952 : }
4953 : return NULL;
4954 : }
4955 :
4956 : /* A utility routine used here, in reload, and in try_split. The insns
4957 : after PREV up to and including LAST are known to adjust the stack,
4958 : with a final value of END_ARGS_SIZE. Iterate backward from LAST
4959 : placing notes as appropriate. PREV may be NULL, indicating the
4960 : entire insn sequence prior to LAST should be scanned.
4961 :
4962 : The set of allowed stack pointer modifications is small:
4963 : (1) One or more auto-inc style memory references (aka pushes),
4964 : (2) One or more addition/subtraction with the SP as destination,
4965 : (3) A single move insn with the SP as destination,
4966 : (4) A call_pop insn,
4967 : (5) Noreturn call insns if !ACCUMULATE_OUTGOING_ARGS.
4968 :
4969 : Insns in the sequence that do not modify the SP are ignored,
4970 : except for noreturn calls.
4971 :
4972 : The return value is the amount of adjustment that can be trivially
4973 : verified, via immediate operand or auto-inc. If the adjustment
4974 : cannot be trivially extracted, the return value is HOST_WIDE_INT_MIN. */
4975 :
4976 : poly_int64
4977 4642846 : find_args_size_adjust (rtx_insn *insn)
4978 : {
4979 4642846 : rtx dest, set, pat;
4980 4642846 : int i;
4981 :
4982 4642846 : pat = PATTERN (insn);
4983 4642846 : set = NULL;
4984 :
4985 : /* Look for a call_pop pattern. */
4986 4642846 : if (CALL_P (insn))
4987 : {
4988 : /* We have to allow non-call_pop patterns for the case
4989 : of emit_single_push_insn of a TLS address. */
4990 2993178 : if (GET_CODE (pat) != PARALLEL)
4991 2978301 : return 0;
4992 :
4993 : /* All call_pop have a stack pointer adjust in the parallel.
4994 : The call itself is always first, and the stack adjust is
4995 : usually last, so search from the end. */
4996 14877 : for (i = XVECLEN (pat, 0) - 1; i > 0; --i)
4997 : {
4998 14877 : set = XVECEXP (pat, 0, i);
4999 14877 : if (GET_CODE (set) != SET)
5000 0 : continue;
5001 14877 : dest = SET_DEST (set);
5002 14877 : if (dest == stack_pointer_rtx)
5003 : break;
5004 : }
5005 : /* We'd better have found the stack pointer adjust. */
5006 14877 : if (i == 0)
5007 0 : return 0;
5008 : /* Fall through to process the extracted SET and DEST
5009 : as if it was a standalone insn. */
5010 : }
5011 1649668 : else if (GET_CODE (pat) == SET)
5012 : set = pat;
5013 259105 : else if ((set = single_set (insn)) != NULL)
5014 : ;
5015 46950 : else if (GET_CODE (pat) == PARALLEL)
5016 : {
5017 : /* ??? Some older ports use a parallel with a stack adjust
5018 : and a store for a PUSH_ROUNDING pattern, rather than a
5019 : PRE/POST_MODIFY rtx. Don't force them to update yet... */
5020 : /* ??? See h8300 and m68k, pushqi1. */
5021 0 : for (i = XVECLEN (pat, 0) - 1; i >= 0; --i)
5022 : {
5023 0 : set = XVECEXP (pat, 0, i);
5024 0 : if (GET_CODE (set) != SET)
5025 0 : continue;
5026 0 : dest = SET_DEST (set);
5027 0 : if (dest == stack_pointer_rtx)
5028 : break;
5029 :
5030 : /* We do not expect an auto-inc of the sp in the parallel. */
5031 0 : gcc_checking_assert (mem_autoinc_base (dest) != stack_pointer_rtx);
5032 0 : gcc_checking_assert (mem_autoinc_base (SET_SRC (set))
5033 : != stack_pointer_rtx);
5034 : }
5035 0 : if (i < 0)
5036 0 : return 0;
5037 : }
5038 : else
5039 46950 : return 0;
5040 :
5041 1617595 : dest = SET_DEST (set);
5042 :
5043 : /* Look for direct modifications of the stack pointer. */
5044 1617595 : if (REG_P (dest) && REGNO (dest) == STACK_POINTER_REGNUM)
5045 : {
5046 : /* Look for a trivial adjustment, otherwise assume nothing. */
5047 : /* Note that the SPU restore_stack_block pattern refers to
5048 : the stack pointer in V4SImode. Consider that non-trivial. */
5049 329518 : poly_int64 offset;
5050 329518 : if (SCALAR_INT_MODE_P (GET_MODE (dest))
5051 329518 : && strip_offset (SET_SRC (set), &offset) == stack_pointer_rtx)
5052 327077 : return offset;
5053 : /* ??? Reload can generate no-op moves, which will be cleaned
5054 : up later. Recognize it and continue searching. */
5055 2441 : else if (rtx_equal_p (dest, SET_SRC (set)))
5056 0 : return 0;
5057 : else
5058 2441 : return HOST_WIDE_INT_MIN;
5059 : }
5060 : else
5061 : {
5062 1288077 : rtx mem, addr;
5063 :
5064 : /* Otherwise only think about autoinc patterns. */
5065 2329693 : if (mem_autoinc_base (dest) == stack_pointer_rtx)
5066 : {
5067 920217 : mem = dest;
5068 1273705 : gcc_checking_assert (mem_autoinc_base (SET_SRC (set))
5069 : != stack_pointer_rtx);
5070 : }
5071 545277 : else if (mem_autoinc_base (SET_SRC (set)) == stack_pointer_rtx)
5072 : mem = SET_SRC (set);
5073 : else
5074 259074 : return 0;
5075 :
5076 1029003 : addr = XEXP (mem, 0);
5077 1029003 : switch (GET_CODE (addr))
5078 : {
5079 108786 : case PRE_INC:
5080 108786 : case POST_INC:
5081 217572 : return GET_MODE_SIZE (GET_MODE (mem));
5082 902156 : case PRE_DEC:
5083 902156 : case POST_DEC:
5084 1804312 : return -GET_MODE_SIZE (GET_MODE (mem));
5085 18061 : case PRE_MODIFY:
5086 18061 : case POST_MODIFY:
5087 18061 : addr = XEXP (addr, 1);
5088 18061 : gcc_assert (GET_CODE (addr) == PLUS);
5089 18061 : gcc_assert (XEXP (addr, 0) == stack_pointer_rtx);
5090 18061 : return rtx_to_poly_int64 (XEXP (addr, 1));
5091 0 : default:
5092 0 : gcc_unreachable ();
5093 : }
5094 : }
5095 : }
5096 :
5097 : poly_int64
5098 667438 : fixup_args_size_notes (rtx_insn *prev, rtx_insn *last,
5099 : poly_int64 end_args_size)
5100 : {
5101 667438 : poly_int64 args_size = end_args_size;
5102 667438 : bool saw_unknown = false;
5103 667438 : rtx_insn *insn;
5104 :
5105 1903849 : for (insn = last; insn != prev; insn = PREV_INSN (insn))
5106 : {
5107 1236411 : if (!NONDEBUG_INSN_P (insn))
5108 0 : continue;
5109 :
5110 : /* We might have existing REG_ARGS_SIZE notes, e.g. when pushing
5111 : a call argument containing a TLS address that itself requires
5112 : a call to __tls_get_addr. The handling of stack_pointer_delta
5113 : in emit_single_push_insn is supposed to ensure that any such
5114 : notes are already correct. */
5115 1236411 : rtx note = find_reg_note (insn, REG_ARGS_SIZE, NULL_RTX);
5116 1236411 : gcc_assert (!note || known_eq (args_size, get_args_size (note)));
5117 :
5118 1236411 : poly_int64 this_delta = find_args_size_adjust (insn);
5119 1236411 : if (known_eq (this_delta, 0))
5120 : {
5121 607803 : if (!CALL_P (insn)
5122 9 : || ACCUMULATE_OUTGOING_ARGS
5123 303915 : || find_reg_note (insn, REG_NORETURN, NULL_RTX) == NULL_RTX)
5124 303897 : continue;
5125 : }
5126 :
5127 932514 : gcc_assert (!saw_unknown);
5128 932514 : if (known_eq (this_delta, HOST_WIDE_INT_MIN))
5129 2419 : saw_unknown = true;
5130 :
5131 932514 : if (!note)
5132 932514 : add_args_size_note (insn, args_size);
5133 932514 : if (STACK_GROWS_DOWNWARD)
5134 1865028 : this_delta = -poly_uint64 (this_delta);
5135 :
5136 932514 : if (saw_unknown)
5137 : args_size = HOST_WIDE_INT_MIN;
5138 : else
5139 1236411 : args_size -= this_delta;
5140 : }
5141 :
5142 667438 : return args_size;
5143 : }
5144 :
5145 : #ifdef PUSH_ROUNDING
5146 : /* Emit single push insn. */
5147 :
5148 : static void
5149 1847231 : emit_single_push_insn_1 (machine_mode mode, rtx x, tree type)
5150 : {
5151 1847231 : rtx dest_addr;
5152 3694462 : poly_int64 rounded_size = PUSH_ROUNDING (GET_MODE_SIZE (mode));
5153 1847231 : rtx dest;
5154 1847231 : enum insn_code icode;
5155 :
5156 : /* If there is push pattern, use it. Otherwise try old way of throwing
5157 : MEM representing push operation to move expander. */
5158 1847231 : icode = optab_handler (push_optab, mode);
5159 1847231 : if (icode != CODE_FOR_nothing)
5160 : {
5161 0 : class expand_operand ops[1];
5162 :
5163 0 : create_input_operand (&ops[0], x, mode);
5164 0 : if (maybe_expand_insn (icode, 1, ops))
5165 0 : return;
5166 : }
5167 3694462 : if (known_eq (GET_MODE_SIZE (mode), rounded_size))
5168 3189031 : dest_addr = gen_rtx_fmt_e (STACK_PUSH_CODE, Pmode, stack_pointer_rtx);
5169 : /* If we are to pad downward, adjust the stack pointer first and
5170 : then store X into the stack location using an offset. This is
5171 : because emit_move_insn does not know how to pad; it does not have
5172 : access to type. */
5173 104835 : else if (targetm.calls.function_arg_padding (mode, type) == PAD_DOWNWARD)
5174 : {
5175 0 : emit_move_insn (stack_pointer_rtx,
5176 0 : expand_binop (Pmode,
5177 : STACK_GROWS_DOWNWARD ? sub_optab
5178 : : add_optab,
5179 : stack_pointer_rtx,
5180 0 : gen_int_mode (rounded_size, Pmode),
5181 : NULL_RTX, 0, OPTAB_LIB_WIDEN));
5182 :
5183 0 : poly_int64 offset = rounded_size - GET_MODE_SIZE (mode);
5184 0 : if (STACK_GROWS_DOWNWARD && STACK_PUSH_CODE == POST_DEC)
5185 : /* We have already decremented the stack pointer, so get the
5186 : previous value. */
5187 : offset += rounded_size;
5188 :
5189 0 : if (!STACK_GROWS_DOWNWARD && STACK_PUSH_CODE == POST_INC)
5190 : /* We have already incremented the stack pointer, so get the
5191 : previous value. */
5192 : offset -= rounded_size;
5193 :
5194 0 : dest_addr = plus_constant (Pmode, stack_pointer_rtx, offset);
5195 : }
5196 : else
5197 : {
5198 : if (STACK_GROWS_DOWNWARD)
5199 : /* ??? This seems wrong if STACK_PUSH_CODE == POST_DEC. */
5200 105409 : dest_addr = plus_constant (Pmode, stack_pointer_rtx, -rounded_size);
5201 : else
5202 : /* ??? This seems wrong if STACK_PUSH_CODE == POST_INC. */
5203 : dest_addr = plus_constant (Pmode, stack_pointer_rtx, rounded_size);
5204 :
5205 105409 : dest_addr = gen_rtx_PRE_MODIFY (Pmode, stack_pointer_rtx, dest_addr);
5206 : }
5207 :
5208 1847231 : dest = gen_rtx_MEM (mode, dest_addr);
5209 :
5210 1847231 : if (type != 0)
5211 : {
5212 1847184 : set_mem_attributes (dest, type, 1);
5213 :
5214 1847184 : if (cfun->tail_call_marked)
5215 : /* Function incoming arguments may overlap with sibling call
5216 : outgoing arguments and we cannot allow reordering of reads
5217 : from function arguments with stores to outgoing arguments
5218 : of sibling calls. */
5219 142776 : set_mem_alias_set (dest, 0);
5220 : }
5221 1847231 : emit_move_insn (dest, x);
5222 : }
5223 :
5224 : /* Emit and annotate a single push insn. */
5225 :
5226 : static void
5227 1847231 : emit_single_push_insn (machine_mode mode, rtx x, tree type)
5228 : {
5229 1847231 : poly_int64 delta, old_delta = stack_pointer_delta;
5230 1847231 : rtx_insn *prev = get_last_insn ();
5231 1847231 : rtx_insn *last;
5232 :
5233 1847231 : emit_single_push_insn_1 (mode, x, type);
5234 :
5235 : /* Adjust stack_pointer_delta to describe the situation after the push
5236 : we just performed. Note that we must do this after the push rather
5237 : than before the push in case calculating X needs pushes and pops of
5238 : its own (e.g. if calling __tls_get_addr). The REG_ARGS_SIZE notes
5239 : for such pushes and pops must not include the effect of the future
5240 : push of X. */
5241 3694462 : stack_pointer_delta += PUSH_ROUNDING (GET_MODE_SIZE (mode));
5242 :
5243 1847231 : last = get_last_insn ();
5244 :
5245 : /* Notice the common case where we emitted exactly one insn. */
5246 1847231 : if (PREV_INSN (last) == prev)
5247 : {
5248 1736101 : add_args_size_note (last, stack_pointer_delta);
5249 1736101 : return;
5250 : }
5251 :
5252 111130 : delta = fixup_args_size_notes (prev, last, stack_pointer_delta);
5253 111130 : gcc_assert (known_eq (delta, HOST_WIDE_INT_MIN)
5254 : || known_eq (delta, old_delta));
5255 : }
5256 : #endif
5257 :
5258 : /* If reading SIZE bytes from X will end up reading from
5259 : Y return the number of bytes that overlap. Return -1
5260 : if there is no overlap or -2 if we can't determine
5261 : (for example when X and Y have different base registers). */
5262 :
5263 : static int
5264 0 : memory_load_overlap (rtx x, rtx y, HOST_WIDE_INT size)
5265 : {
5266 0 : rtx tmp = plus_constant (Pmode, x, size);
5267 0 : rtx sub = simplify_gen_binary (MINUS, Pmode, tmp, y);
5268 :
5269 0 : if (!CONST_INT_P (sub))
5270 : return -2;
5271 :
5272 0 : HOST_WIDE_INT val = INTVAL (sub);
5273 :
5274 0 : return IN_RANGE (val, 1, size) ? val : -1;
5275 : }
5276 :
5277 : /* Generate code to push X onto the stack, assuming it has mode MODE and
5278 : type TYPE.
5279 : MODE is redundant except when X is a CONST_INT (since they don't
5280 : carry mode info).
5281 : SIZE is an rtx for the size of data to be copied (in bytes),
5282 : needed only if X is BLKmode.
5283 : Return true if successful. May return false if asked to push a
5284 : partial argument during a sibcall optimization (as specified by
5285 : SIBCALL_P) and the incoming and outgoing pointers cannot be shown
5286 : to not overlap.
5287 :
5288 : ALIGN (in bits) is maximum alignment we can assume.
5289 :
5290 : If PARTIAL and REG are both nonzero, then copy that many of the first
5291 : bytes of X into registers starting with REG, and push the rest of X.
5292 : The amount of space pushed is decreased by PARTIAL bytes.
5293 : REG must be a hard register in this case.
5294 : If REG is zero but PARTIAL is not, take any all others actions for an
5295 : argument partially in registers, but do not actually load any
5296 : registers.
5297 :
5298 : EXTRA is the amount in bytes of extra space to leave next to this arg.
5299 : This is ignored if an argument block has already been allocated.
5300 :
5301 : On a machine that lacks real push insns, ARGS_ADDR is the address of
5302 : the bottom of the argument block for this call. We use indexing off there
5303 : to store the arg. On machines with push insns, ARGS_ADDR is 0 when a
5304 : argument block has not been preallocated.
5305 :
5306 : ARGS_SO_FAR is the size of args previously pushed for this call.
5307 :
5308 : REG_PARM_STACK_SPACE is nonzero if functions require stack space
5309 : for arguments passed in registers. If nonzero, it will be the number
5310 : of bytes required. */
5311 :
5312 : bool
5313 2171651 : emit_push_insn (rtx x, machine_mode mode, tree type, rtx size,
5314 : unsigned int align, int partial, rtx reg, poly_int64 extra,
5315 : rtx args_addr, rtx args_so_far, int reg_parm_stack_space,
5316 : rtx alignment_pad, bool sibcall_p)
5317 : {
5318 2171651 : rtx xinner;
5319 2171651 : pad_direction stack_direction
5320 : = STACK_GROWS_DOWNWARD ? PAD_DOWNWARD : PAD_UPWARD;
5321 :
5322 : /* Decide where to pad the argument: PAD_DOWNWARD for below,
5323 : PAD_UPWARD for above, or PAD_NONE for don't pad it.
5324 : Default is below for small data on big-endian machines; else above. */
5325 2171651 : pad_direction where_pad = targetm.calls.function_arg_padding (mode, type);
5326 :
5327 : /* Invert direction if stack is post-decrement.
5328 : FIXME: why? */
5329 2171651 : if (STACK_PUSH_CODE == POST_DEC)
5330 : if (where_pad != PAD_NONE)
5331 : where_pad = (where_pad == PAD_DOWNWARD ? PAD_UPWARD : PAD_DOWNWARD);
5332 :
5333 2171651 : xinner = x;
5334 :
5335 2171651 : int nregs = partial / UNITS_PER_WORD;
5336 2171651 : rtx *tmp_regs = NULL;
5337 2171651 : int overlapping = 0;
5338 :
5339 2171651 : if (mode == BLKmode
5340 : || (STRICT_ALIGNMENT && align < GET_MODE_ALIGNMENT (mode)))
5341 : {
5342 : /* Copy a block into the stack, entirely or partially. */
5343 :
5344 269070 : rtx temp;
5345 269070 : int used;
5346 269070 : int offset;
5347 269070 : int skip;
5348 :
5349 269070 : offset = partial % (PARM_BOUNDARY / BITS_PER_UNIT);
5350 269070 : used = partial - offset;
5351 :
5352 269070 : if (mode != BLKmode)
5353 : {
5354 : /* A value is to be stored in an insufficiently aligned
5355 : stack slot; copy via a suitably aligned slot if
5356 : necessary. */
5357 : size = gen_int_mode (GET_MODE_SIZE (mode), Pmode);
5358 : if (!MEM_P (xinner))
5359 : {
5360 : temp = assign_temp (type, 1, 1);
5361 : emit_move_insn (temp, xinner);
5362 : xinner = temp;
5363 : }
5364 : }
5365 :
5366 269070 : gcc_assert (size);
5367 :
5368 : /* USED is now the # of bytes we need not copy to the stack
5369 : because registers will take care of them. */
5370 :
5371 269070 : if (partial != 0)
5372 0 : xinner = adjust_address (xinner, BLKmode, used);
5373 :
5374 : /* If the partial register-part of the arg counts in its stack size,
5375 : skip the part of stack space corresponding to the registers.
5376 : Otherwise, start copying to the beginning of the stack space,
5377 : by setting SKIP to 0. */
5378 269070 : skip = (reg_parm_stack_space == 0) ? 0 : used;
5379 :
5380 : #ifdef PUSH_ROUNDING
5381 : /* NB: Let the backend known the number of bytes to push and
5382 : decide if push insns should be generated. */
5383 269070 : unsigned int push_size;
5384 269070 : if (CONST_INT_P (size))
5385 269070 : push_size = INTVAL (size);
5386 : else
5387 : push_size = 0;
5388 :
5389 : /* Do it with several push insns if that doesn't take lots of insns
5390 : and if there is no difficulty with push insns that skip bytes
5391 : on the stack for alignment purposes. */
5392 269070 : if (args_addr == 0
5393 269016 : && targetm.calls.push_argument (push_size)
5394 3783 : && CONST_INT_P (size)
5395 3783 : && skip == 0
5396 3783 : && MEM_ALIGN (xinner) >= align
5397 2937 : && can_move_by_pieces ((unsigned) INTVAL (size) - used, align)
5398 : /* Here we avoid the case of a structure whose weak alignment
5399 : forces many pushes of a small amount of data,
5400 : and such small pushes do rounding that causes trouble. */
5401 2937 : && ((!targetm.slow_unaligned_access (word_mode, align))
5402 0 : || align >= BIGGEST_ALIGNMENT
5403 0 : || known_eq (PUSH_ROUNDING (align / BITS_PER_UNIT),
5404 : align / BITS_PER_UNIT))
5405 272007 : && known_eq (PUSH_ROUNDING (INTVAL (size)), INTVAL (size)))
5406 : {
5407 : /* Push padding now if padding above and stack grows down,
5408 : or if padding below and stack grows up.
5409 : But if space already allocated, this has already been done. */
5410 45 : if (maybe_ne (extra, 0)
5411 : && args_addr == 0
5412 0 : && where_pad != PAD_NONE
5413 45 : && where_pad != stack_direction)
5414 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5415 :
5416 45 : move_by_pieces (NULL, xinner, INTVAL (size) - used, align,
5417 : RETURN_BEGIN);
5418 : }
5419 : else
5420 : #endif /* PUSH_ROUNDING */
5421 : {
5422 269025 : rtx target;
5423 :
5424 : /* Otherwise make space on the stack and copy the data
5425 : to the address of that space. */
5426 :
5427 : /* Deduct words put into registers from the size we must copy. */
5428 269025 : if (partial != 0)
5429 : {
5430 0 : if (CONST_INT_P (size))
5431 0 : size = GEN_INT (INTVAL (size) - used);
5432 : else
5433 0 : size = expand_binop (GET_MODE (size), sub_optab, size,
5434 0 : gen_int_mode (used, GET_MODE (size)),
5435 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
5436 : }
5437 :
5438 : /* Get the address of the stack space.
5439 : In this case, we do not deal with EXTRA separately.
5440 : A single stack adjust will do. */
5441 269025 : poly_int64 const_args_so_far;
5442 269025 : if (! args_addr)
5443 : {
5444 268971 : temp = push_block (size, extra, where_pad == PAD_DOWNWARD);
5445 268971 : extra = 0;
5446 : }
5447 54 : else if (poly_int_rtx_p (args_so_far, &const_args_so_far))
5448 72 : temp = memory_address (BLKmode,
5449 : plus_constant (Pmode, args_addr,
5450 : skip + const_args_so_far));
5451 : else
5452 0 : temp = memory_address (BLKmode,
5453 : plus_constant (Pmode,
5454 : gen_rtx_PLUS (Pmode,
5455 : args_addr,
5456 : args_so_far),
5457 : skip));
5458 :
5459 269025 : if (!ACCUMULATE_OUTGOING_ARGS)
5460 : {
5461 : /* If the source is referenced relative to the stack pointer,
5462 : copy it to another register to stabilize it. We do not need
5463 : to do this if we know that we won't be changing sp. */
5464 :
5465 269025 : if (reg_mentioned_p (virtual_stack_dynamic_rtx, temp)
5466 269025 : || reg_mentioned_p (virtual_outgoing_args_rtx, temp))
5467 268971 : temp = copy_to_reg (temp);
5468 : }
5469 :
5470 269025 : target = gen_rtx_MEM (BLKmode, temp);
5471 :
5472 : /* We do *not* set_mem_attributes here, because incoming arguments
5473 : may overlap with sibling call outgoing arguments and we cannot
5474 : allow reordering of reads from function arguments with stores
5475 : to outgoing arguments of sibling calls. We do, however, want
5476 : to record the alignment of the stack slot. */
5477 : /* ALIGN may well be better aligned than TYPE, e.g. due to
5478 : PARM_BOUNDARY. Assume the caller isn't lying. */
5479 269025 : set_mem_align (target, align);
5480 :
5481 : /* If part should go in registers and pushing to that part would
5482 : overwrite some of the values that need to go into regs, load the
5483 : overlapping values into temporary pseudos to be moved into the hard
5484 : regs at the end after the stack pushing has completed.
5485 : We cannot load them directly into the hard regs here because
5486 : they can be clobbered by the block move expansions.
5487 : See PR 65358. */
5488 :
5489 269025 : if (partial > 0 && reg != 0 && mode == BLKmode
5490 0 : && GET_CODE (reg) != PARALLEL)
5491 : {
5492 0 : overlapping = memory_load_overlap (XEXP (x, 0), temp, partial);
5493 0 : if (overlapping > 0)
5494 : {
5495 0 : gcc_assert (overlapping % UNITS_PER_WORD == 0);
5496 0 : overlapping /= UNITS_PER_WORD;
5497 :
5498 0 : tmp_regs = XALLOCAVEC (rtx, overlapping);
5499 :
5500 0 : for (int i = 0; i < overlapping; i++)
5501 0 : tmp_regs[i] = gen_reg_rtx (word_mode);
5502 :
5503 0 : for (int i = 0; i < overlapping; i++)
5504 0 : emit_move_insn (tmp_regs[i],
5505 0 : operand_subword_force (target, i, mode));
5506 : }
5507 0 : else if (overlapping == -1)
5508 : overlapping = 0;
5509 : /* Could not determine whether there is overlap.
5510 : Fail the sibcall. */
5511 : else
5512 : {
5513 0 : overlapping = 0;
5514 0 : if (sibcall_p)
5515 2171651 : return false;
5516 : }
5517 : }
5518 :
5519 : /* If source is a constant VAR_DECL with a simple constructor,
5520 : store the constructor to the stack instead of moving it. */
5521 269025 : const_tree decl;
5522 269025 : HOST_WIDE_INT sz;
5523 269025 : if (partial == 0
5524 269025 : && MEM_P (xinner)
5525 269025 : && SYMBOL_REF_P (XEXP (xinner, 0))
5526 82600 : && (decl = SYMBOL_REF_DECL (XEXP (xinner, 0))) != NULL_TREE
5527 82600 : && VAR_P (decl)
5528 82600 : && TREE_READONLY (decl)
5529 5229 : && !TREE_SIDE_EFFECTS (decl)
5530 5229 : && immediate_const_ctor_p (DECL_INITIAL (decl), 2)
5531 9 : && (sz = int_expr_size (DECL_INITIAL (decl))) > 0
5532 9 : && CONST_INT_P (size)
5533 269034 : && INTVAL (size) == sz)
5534 3 : store_constructor (DECL_INITIAL (decl), target, 0, sz, false);
5535 : else
5536 269022 : emit_block_move (target, xinner, size, BLOCK_OP_CALL_PARM);
5537 : }
5538 : }
5539 1902581 : else if (partial > 0)
5540 : {
5541 : /* Scalar partly in registers. This case is only supported
5542 : for fixed-wdth modes. */
5543 0 : int num_words = GET_MODE_SIZE (mode).to_constant ();
5544 0 : num_words /= UNITS_PER_WORD;
5545 0 : int i;
5546 0 : int not_stack;
5547 : /* # bytes of start of argument
5548 : that we must make space for but need not store. */
5549 0 : int offset = partial % (PARM_BOUNDARY / BITS_PER_UNIT);
5550 0 : int args_offset = INTVAL (args_so_far);
5551 0 : int skip;
5552 :
5553 : /* Push padding now if padding above and stack grows down,
5554 : or if padding below and stack grows up.
5555 : But if space already allocated, this has already been done. */
5556 0 : if (maybe_ne (extra, 0)
5557 0 : && args_addr == 0
5558 0 : && where_pad != PAD_NONE
5559 0 : && where_pad != stack_direction)
5560 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5561 :
5562 : /* If we make space by pushing it, we might as well push
5563 : the real data. Otherwise, we can leave OFFSET nonzero
5564 : and leave the space uninitialized. */
5565 0 : if (args_addr == 0)
5566 0 : offset = 0;
5567 :
5568 : /* Now NOT_STACK gets the number of words that we don't need to
5569 : allocate on the stack. Convert OFFSET to words too. */
5570 0 : not_stack = (partial - offset) / UNITS_PER_WORD;
5571 0 : offset /= UNITS_PER_WORD;
5572 :
5573 : /* If the partial register-part of the arg counts in its stack size,
5574 : skip the part of stack space corresponding to the registers.
5575 : Otherwise, start copying to the beginning of the stack space,
5576 : by setting SKIP to 0. */
5577 0 : skip = (reg_parm_stack_space == 0) ? 0 : not_stack;
5578 :
5579 0 : if (CONSTANT_P (x) && !targetm.legitimate_constant_p (mode, x))
5580 0 : x = validize_mem (force_const_mem (mode, x));
5581 :
5582 : /* If X is a hard register in a non-integer mode, copy it into a pseudo;
5583 : SUBREGs of such registers are not allowed. */
5584 0 : if ((REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER
5585 0 : && GET_MODE_CLASS (GET_MODE (x)) != MODE_INT))
5586 0 : x = copy_to_reg (x);
5587 :
5588 : /* Loop over all the words allocated on the stack for this arg. */
5589 : /* We can do it by words, because any scalar bigger than a word
5590 : has a size a multiple of a word. */
5591 0 : tree word_mode_type = lang_hooks.types.type_for_mode (word_mode, 1);
5592 0 : for (i = num_words - 1; i >= not_stack; i--)
5593 0 : if (i >= not_stack + offset)
5594 0 : if (!emit_push_insn (operand_subword_force (x, i, mode),
5595 : word_mode, word_mode_type, NULL_RTX, align, 0,
5596 : NULL_RTX, 0, args_addr,
5597 0 : GEN_INT (args_offset + ((i - not_stack + skip)
5598 : * UNITS_PER_WORD)),
5599 : reg_parm_stack_space, alignment_pad, sibcall_p))
5600 2171651 : return false;
5601 : }
5602 : else
5603 : {
5604 1902581 : rtx addr;
5605 1902581 : rtx dest;
5606 :
5607 : /* Push padding now if padding above and stack grows down,
5608 : or if padding below and stack grows up.
5609 : But if space already allocated, this has already been done. */
5610 1902581 : if (maybe_ne (extra, 0)
5611 0 : && args_addr == 0
5612 0 : && where_pad != PAD_NONE
5613 1902581 : && where_pad != stack_direction)
5614 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5615 :
5616 : #ifdef PUSH_ROUNDING
5617 1902581 : if (args_addr == 0 && targetm.calls.push_argument (0))
5618 1847184 : emit_single_push_insn (mode, x, type);
5619 : else
5620 : #endif
5621 : {
5622 63644 : addr = simplify_gen_binary (PLUS, Pmode, args_addr, args_so_far);
5623 55397 : dest = gen_rtx_MEM (mode, memory_address (mode, addr));
5624 :
5625 : /* We do *not* set_mem_attributes here, because incoming arguments
5626 : may overlap with sibling call outgoing arguments and we cannot
5627 : allow reordering of reads from function arguments with stores
5628 : to outgoing arguments of sibling calls. We do, however, want
5629 : to record the alignment of the stack slot. */
5630 : /* ALIGN may well be better aligned than TYPE, e.g. due to
5631 : PARM_BOUNDARY. Assume the caller isn't lying. */
5632 55397 : set_mem_align (dest, align);
5633 :
5634 55397 : emit_move_insn (dest, x);
5635 : }
5636 : }
5637 :
5638 : /* Move the partial arguments into the registers and any overlapping
5639 : values that we moved into the pseudos in tmp_regs. */
5640 2171651 : if (partial > 0 && reg != 0)
5641 : {
5642 : /* Handle calls that pass values in multiple non-contiguous locations.
5643 : The Irix 6 ABI has examples of this. */
5644 0 : if (GET_CODE (reg) == PARALLEL)
5645 0 : emit_group_load (reg, x, type, -1);
5646 : else
5647 : {
5648 0 : gcc_assert (partial % UNITS_PER_WORD == 0);
5649 0 : move_block_to_reg (REGNO (reg), x, nregs - overlapping, mode);
5650 :
5651 0 : for (int i = 0; i < overlapping; i++)
5652 0 : emit_move_insn (gen_rtx_REG (word_mode, REGNO (reg)
5653 0 : + nregs - overlapping + i),
5654 0 : tmp_regs[i]);
5655 :
5656 : }
5657 : }
5658 :
5659 2171651 : if (maybe_ne (extra, 0) && args_addr == 0 && where_pad == stack_direction)
5660 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5661 :
5662 2171651 : if (alignment_pad && args_addr == 0)
5663 2116200 : anti_adjust_stack (alignment_pad);
5664 :
5665 : return true;
5666 : }
5667 :
5668 : /* Return X if X can be used as a subtarget in a sequence of arithmetic
5669 : operations. */
5670 :
5671 : static rtx
5672 205435495 : get_subtarget (rtx x)
5673 : {
5674 205435495 : return (optimize
5675 53129735 : || x == 0
5676 : /* Only registers can be subtargets. */
5677 17794248 : || !REG_P (x)
5678 : /* Don't use hard regs to avoid extending their life. */
5679 12102305 : || REGNO (x) < FIRST_PSEUDO_REGISTER
5680 205435495 : ? 0 : x);
5681 : }
5682 :
5683 : /* A subroutine of expand_assignment. Optimize FIELD op= VAL, where
5684 : FIELD is a bitfield. Returns true if the optimization was successful,
5685 : and there's nothing else to do. */
5686 :
5687 : static bool
5688 4799577 : optimize_bitfield_assignment_op (poly_uint64 pbitsize,
5689 : poly_uint64 pbitpos,
5690 : poly_uint64 pbitregion_start,
5691 : poly_uint64 pbitregion_end,
5692 : machine_mode mode1, rtx str_rtx,
5693 : tree to, tree src, bool reverse)
5694 : {
5695 : /* str_mode is not guaranteed to be a scalar type. */
5696 4799577 : machine_mode str_mode = GET_MODE (str_rtx);
5697 4799577 : unsigned int str_bitsize;
5698 4799577 : tree op0, op1;
5699 4799577 : rtx value, result;
5700 4799577 : optab binop;
5701 4799577 : gimple *srcstmt;
5702 4799577 : enum tree_code code;
5703 :
5704 4799577 : unsigned HOST_WIDE_INT bitsize, bitpos, bitregion_start, bitregion_end;
5705 4799577 : if (mode1 != VOIDmode
5706 67225 : || !pbitsize.is_constant (&bitsize)
5707 67225 : || !pbitpos.is_constant (&bitpos)
5708 67225 : || !pbitregion_start.is_constant (&bitregion_start)
5709 67225 : || !pbitregion_end.is_constant (&bitregion_end)
5710 67909 : || bitsize >= BITS_PER_WORD
5711 66954 : || !GET_MODE_BITSIZE (str_mode).is_constant (&str_bitsize)
5712 67638 : || str_bitsize > BITS_PER_WORD
5713 61558 : || TREE_SIDE_EFFECTS (to)
5714 4861006 : || TREE_THIS_VOLATILE (to))
5715 : return false;
5716 :
5717 61429 : STRIP_NOPS (src);
5718 61429 : if (TREE_CODE (src) != SSA_NAME)
5719 : return false;
5720 20191 : if (TREE_CODE (TREE_TYPE (src)) != INTEGER_TYPE)
5721 : return false;
5722 :
5723 18594 : srcstmt = get_gimple_for_ssa_name (src);
5724 18594 : if (!srcstmt
5725 4952 : || !is_gimple_assign (srcstmt)
5726 23546 : || TREE_CODE_CLASS (gimple_assign_rhs_code (srcstmt)) != tcc_binary)
5727 : return false;
5728 :
5729 845 : code = gimple_assign_rhs_code (srcstmt);
5730 :
5731 845 : op0 = gimple_assign_rhs1 (srcstmt);
5732 :
5733 : /* If OP0 is an SSA_NAME, then we want to walk the use-def chain
5734 : to find its initialization. Hopefully the initialization will
5735 : be from a bitfield load. */
5736 845 : if (TREE_CODE (op0) == SSA_NAME)
5737 : {
5738 845 : gimple *op0stmt = get_gimple_for_ssa_name (op0);
5739 :
5740 : /* We want to eventually have OP0 be the same as TO, which
5741 : should be a bitfield. */
5742 845 : if (!op0stmt
5743 775 : || !is_gimple_assign (op0stmt)
5744 1620 : || gimple_assign_rhs_code (op0stmt) != TREE_CODE (to))
5745 : return false;
5746 548 : op0 = gimple_assign_rhs1 (op0stmt);
5747 : }
5748 :
5749 548 : op1 = gimple_assign_rhs2 (srcstmt);
5750 :
5751 548 : if (!operand_equal_p (to, op0, 0))
5752 : return false;
5753 :
5754 463 : if (MEM_P (str_rtx))
5755 : {
5756 422 : unsigned HOST_WIDE_INT offset1;
5757 :
5758 422 : if (str_bitsize == 0 || str_bitsize > BITS_PER_WORD)
5759 82 : str_bitsize = BITS_PER_WORD;
5760 :
5761 422 : scalar_int_mode best_mode;
5762 422 : if (!get_best_mode (bitsize, bitpos, bitregion_start, bitregion_end,
5763 422 : MEM_ALIGN (str_rtx), str_bitsize, false, &best_mode))
5764 0 : return false;
5765 422 : str_mode = best_mode;
5766 422 : str_bitsize = GET_MODE_BITSIZE (best_mode);
5767 :
5768 422 : offset1 = bitpos;
5769 422 : bitpos %= str_bitsize;
5770 422 : offset1 = (offset1 - bitpos) / BITS_PER_UNIT;
5771 422 : str_rtx = adjust_address (str_rtx, str_mode, offset1);
5772 : }
5773 41 : else if (!REG_P (str_rtx) && GET_CODE (str_rtx) != SUBREG)
5774 : return false;
5775 :
5776 : /* If the bit field covers the whole REG/MEM, store_field
5777 : will likely generate better code. */
5778 463 : if (bitsize >= str_bitsize)
5779 : return false;
5780 :
5781 : /* We can't handle fields split across multiple entities. */
5782 463 : if (bitpos + bitsize > str_bitsize)
5783 : return false;
5784 :
5785 463 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
5786 0 : bitpos = str_bitsize - bitpos - bitsize;
5787 :
5788 463 : switch (code)
5789 : {
5790 140 : case PLUS_EXPR:
5791 140 : case MINUS_EXPR:
5792 : /* For now, just optimize the case of the topmost bitfield
5793 : where we don't need to do any masking and also
5794 : 1 bit bitfields where xor can be used.
5795 : We might win by one instruction for the other bitfields
5796 : too if insv/extv instructions aren't used, so that
5797 : can be added later. */
5798 140 : if ((reverse || bitpos + bitsize != str_bitsize)
5799 96 : && (bitsize != 1 || TREE_CODE (op1) != INTEGER_CST))
5800 : break;
5801 :
5802 69 : value = expand_expr (op1, NULL_RTX, str_mode, EXPAND_NORMAL);
5803 69 : value = convert_modes (str_mode,
5804 69 : TYPE_MODE (TREE_TYPE (op1)), value,
5805 69 : TYPE_UNSIGNED (TREE_TYPE (op1)));
5806 :
5807 : /* We may be accessing data outside the field, which means
5808 : we can alias adjacent data. */
5809 69 : if (MEM_P (str_rtx))
5810 : {
5811 69 : str_rtx = shallow_copy_rtx (str_rtx);
5812 69 : set_mem_alias_set (str_rtx, 0);
5813 69 : set_mem_expr (str_rtx, 0);
5814 : }
5815 :
5816 69 : if (bitsize == 1 && (reverse || bitpos + bitsize != str_bitsize))
5817 : {
5818 25 : value = expand_and (str_mode, value, const1_rtx, NULL);
5819 25 : binop = xor_optab;
5820 : }
5821 : else
5822 44 : binop = code == PLUS_EXPR ? add_optab : sub_optab;
5823 :
5824 69 : value = expand_shift (LSHIFT_EXPR, str_mode, value, bitpos, NULL_RTX, 1);
5825 69 : if (reverse)
5826 0 : value = flip_storage_order (str_mode, value);
5827 69 : result = expand_binop (str_mode, binop, str_rtx,
5828 : value, str_rtx, 1, OPTAB_WIDEN);
5829 69 : if (result != str_rtx)
5830 0 : emit_move_insn (str_rtx, result);
5831 : return true;
5832 :
5833 255 : case BIT_IOR_EXPR:
5834 255 : case BIT_XOR_EXPR:
5835 255 : if (TREE_CODE (op1) != INTEGER_CST)
5836 : break;
5837 97 : value = expand_expr (op1, NULL_RTX, str_mode, EXPAND_NORMAL);
5838 97 : value = convert_modes (str_mode,
5839 97 : TYPE_MODE (TREE_TYPE (op1)), value,
5840 97 : TYPE_UNSIGNED (TREE_TYPE (op1)));
5841 :
5842 : /* We may be accessing data outside the field, which means
5843 : we can alias adjacent data. */
5844 97 : if (MEM_P (str_rtx))
5845 : {
5846 61 : str_rtx = shallow_copy_rtx (str_rtx);
5847 61 : set_mem_alias_set (str_rtx, 0);
5848 61 : set_mem_expr (str_rtx, 0);
5849 : }
5850 :
5851 97 : binop = code == BIT_IOR_EXPR ? ior_optab : xor_optab;
5852 97 : if (bitpos + bitsize != str_bitsize)
5853 : {
5854 55 : rtx mask = gen_int_mode ((HOST_WIDE_INT_1U << bitsize) - 1,
5855 : str_mode);
5856 55 : value = expand_and (str_mode, value, mask, NULL_RTX);
5857 : }
5858 97 : value = expand_shift (LSHIFT_EXPR, str_mode, value, bitpos, NULL_RTX, 1);
5859 97 : if (reverse)
5860 0 : value = flip_storage_order (str_mode, value);
5861 97 : result = expand_binop (str_mode, binop, str_rtx,
5862 : value, str_rtx, 1, OPTAB_WIDEN);
5863 97 : if (result != str_rtx)
5864 0 : emit_move_insn (str_rtx, result);
5865 : return true;
5866 :
5867 : default:
5868 : break;
5869 : }
5870 :
5871 : return false;
5872 : }
5873 :
5874 : /* In the C++ memory model, consecutive bit fields in a structure are
5875 : considered one memory location.
5876 :
5877 : Given a COMPONENT_REF EXP at position (BITPOS, OFFSET), this function
5878 : returns the bit range of consecutive bits in which this COMPONENT_REF
5879 : belongs. The values are returned in *BITSTART and *BITEND. *BITPOS
5880 : and *OFFSET may be adjusted in the process.
5881 :
5882 : If the access does not need to be restricted, 0 is returned in both
5883 : *BITSTART and *BITEND. */
5884 :
5885 : void
5886 935590 : get_bit_range (poly_uint64 *bitstart, poly_uint64 *bitend, tree exp,
5887 : poly_int64 *bitpos, tree *offset)
5888 : {
5889 935590 : poly_int64 bitoffset;
5890 935590 : tree field, repr;
5891 :
5892 935590 : gcc_assert (TREE_CODE (exp) == COMPONENT_REF);
5893 :
5894 935590 : field = TREE_OPERAND (exp, 1);
5895 935590 : repr = DECL_BIT_FIELD_REPRESENTATIVE (field);
5896 : /* If we do not have a DECL_BIT_FIELD_REPRESENTATIVE there is no
5897 : need to limit the range we can access. */
5898 935590 : if (!repr)
5899 : {
5900 65 : *bitstart = *bitend = 0;
5901 65 : return;
5902 : }
5903 :
5904 : /* If we have a DECL_BIT_FIELD_REPRESENTATIVE but the enclosing record is
5905 : part of a larger bit field, then the representative does not serve any
5906 : useful purpose. This can occur in Ada. */
5907 935525 : if (handled_component_p (TREE_OPERAND (exp, 0)))
5908 : {
5909 505436 : machine_mode rmode;
5910 505436 : poly_int64 rbitsize, rbitpos;
5911 505436 : tree roffset;
5912 505436 : int unsignedp, reversep, volatilep = 0;
5913 505436 : get_inner_reference (TREE_OPERAND (exp, 0), &rbitsize, &rbitpos,
5914 : &roffset, &rmode, &unsignedp, &reversep,
5915 : &volatilep);
5916 1010872 : if (!multiple_p (rbitpos, BITS_PER_UNIT))
5917 : {
5918 0 : *bitstart = *bitend = 0;
5919 0 : return;
5920 : }
5921 : }
5922 :
5923 : /* Compute the adjustment to bitpos from the offset of the field
5924 : relative to the representative. DECL_FIELD_OFFSET of field and
5925 : repr are the same by construction if they are not constants,
5926 : see finish_bitfield_layout. */
5927 935525 : poly_uint64 field_offset, repr_offset;
5928 935525 : if (poly_int_tree_p (DECL_FIELD_OFFSET (field), &field_offset)
5929 1871045 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
5930 935520 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
5931 : else
5932 : bitoffset = 0;
5933 935525 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (field))
5934 935525 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
5935 :
5936 : /* If the adjustment is larger than bitpos, we would have a negative bit
5937 : position for the lower bound and this may wreak havoc later. Adjust
5938 : offset and bitpos to make the lower bound non-negative in that case. */
5939 935525 : if (maybe_gt (bitoffset, *bitpos))
5940 : {
5941 9 : poly_int64 adjust_bits = upper_bound (bitoffset, *bitpos) - *bitpos;
5942 9 : poly_int64 adjust_bytes = exact_div (adjust_bits, BITS_PER_UNIT);
5943 :
5944 9 : *bitpos += adjust_bits;
5945 9 : if (*offset == NULL_TREE)
5946 5 : *offset = size_int (-adjust_bytes);
5947 : else
5948 4 : *offset = size_binop (MINUS_EXPR, *offset, size_int (adjust_bytes));
5949 9 : *bitstart = 0;
5950 : }
5951 : else
5952 935516 : *bitstart = *bitpos - bitoffset;
5953 :
5954 935525 : *bitend = *bitstart + tree_to_poly_uint64 (DECL_SIZE (repr)) - 1;
5955 : }
5956 :
5957 : /* Returns true if BASE is a DECL that does not reside in memory and
5958 : has non-BLKmode. DECL_RTL must not be a MEM; if
5959 : DECL_RTL was not set yet, return false. */
5960 :
5961 : bool
5962 5550996 : non_mem_decl_p (tree base)
5963 : {
5964 5550996 : if (!DECL_P (base)
5965 5550599 : || TREE_ADDRESSABLE (base)
5966 8013608 : || DECL_MODE (base) == BLKmode)
5967 : return false;
5968 :
5969 1595868 : if (!DECL_RTL_SET_P (base))
5970 : return false;
5971 :
5972 1477127 : return (!MEM_P (DECL_RTL (base)));
5973 : }
5974 :
5975 : /* Returns true if REF refers to an object that does not
5976 : reside in memory and has non-BLKmode. */
5977 :
5978 : bool
5979 12131219 : mem_ref_refers_to_non_mem_p (tree ref)
5980 : {
5981 12131219 : tree base;
5982 :
5983 12131219 : if (TREE_CODE (ref) == MEM_REF
5984 12131219 : || TREE_CODE (ref) == TARGET_MEM_REF)
5985 : {
5986 10597837 : tree addr = TREE_OPERAND (ref, 0);
5987 :
5988 10597837 : if (TREE_CODE (addr) != ADDR_EXPR)
5989 : return false;
5990 :
5991 3991471 : base = TREE_OPERAND (addr, 0);
5992 : }
5993 : else
5994 : base = ref;
5995 :
5996 5524853 : return non_mem_decl_p (base);
5997 : }
5998 :
5999 : /* Helper function of expand_assignment. Check if storing field of
6000 : size BITSIZE at position BITPOS overlaps with the most significant
6001 : bit of TO_RTX, known to be SUBREG_PROMOTED_VAR_P.
6002 : Updating this field requires an explicit extension. */
6003 : static bool
6004 0 : store_field_updates_msb_p (poly_int64 bitpos, poly_int64 bitsize, rtx to_rtx)
6005 : {
6006 0 : return (BYTES_BIG_ENDIAN
6007 : ? maybe_le (bitpos, 0)
6008 0 : : maybe_ge (bitpos + bitsize, GET_MODE_BITSIZE (GET_MODE (to_rtx))));
6009 : }
6010 :
6011 : /* Expand an assignment that stores the value of FROM into TO. If NONTEMPORAL
6012 : is true, try generating a nontemporal store. */
6013 :
6014 : void
6015 18735514 : expand_assignment (tree to, tree from, bool nontemporal)
6016 : {
6017 18735514 : rtx to_rtx = 0;
6018 18735514 : rtx result;
6019 18735514 : machine_mode mode;
6020 18735514 : unsigned int align;
6021 18735514 : enum insn_code icode;
6022 :
6023 : /* Don't crash if the lhs of the assignment was erroneous. */
6024 18735514 : if (TREE_CODE (to) == ERROR_MARK)
6025 : {
6026 0 : expand_normal (from);
6027 0 : return;
6028 : }
6029 :
6030 : /* Optimize away no-op moves without side-effects. */
6031 18735514 : if (operand_equal_p (to, from, 0))
6032 : return;
6033 :
6034 : /* Handle misaligned stores. */
6035 18735359 : mode = TYPE_MODE (TREE_TYPE (to));
6036 18735359 : if ((TREE_CODE (to) == MEM_REF
6037 18735359 : || TREE_CODE (to) == TARGET_MEM_REF
6038 16502718 : || DECL_P (to))
6039 4094859 : && mode != BLKmode
6040 3627954 : && !mem_ref_refers_to_non_mem_p (to)
6041 6505276 : && ((align = get_object_alignment (to))
6042 3252638 : < GET_MODE_ALIGNMENT (mode))
6043 19169469 : && (((icode = optab_handler (movmisalign_optab, mode))
6044 : != CODE_FOR_nothing)
6045 97205 : || targetm.slow_unaligned_access (mode, align)))
6046 : {
6047 336905 : rtx reg, mem;
6048 :
6049 336905 : reg = expand_expr (from, NULL_RTX, VOIDmode, EXPAND_NORMAL);
6050 : /* Handle PARALLEL. */
6051 336905 : reg = maybe_emit_group_store (reg, TREE_TYPE (from));
6052 336905 : reg = force_not_mem (reg);
6053 336905 : mem = expand_expr (to, NULL_RTX, VOIDmode, EXPAND_WRITE);
6054 336905 : if (TREE_CODE (to) == MEM_REF && REF_REVERSE_STORAGE_ORDER (to))
6055 0 : reg = flip_storage_order (mode, reg);
6056 :
6057 336905 : if (icode != CODE_FOR_nothing)
6058 : {
6059 336905 : class expand_operand ops[2];
6060 :
6061 336905 : create_fixed_operand (&ops[0], mem);
6062 336905 : create_input_operand (&ops[1], reg, mode);
6063 : /* The movmisalign<mode> pattern cannot fail, else the assignment
6064 : would silently be omitted. */
6065 336905 : expand_insn (icode, 2, ops);
6066 : }
6067 : else
6068 0 : store_bit_field (mem, GET_MODE_BITSIZE (mode), 0, 0, 0, mode, reg,
6069 : false, false);
6070 : return;
6071 : }
6072 :
6073 : /* Assignment of a structure component needs special treatment
6074 : if the structure component's rtx is not simply a MEM.
6075 : Assignment of an array element at a constant index, and assignment of
6076 : an array element in an unaligned packed structure field, has the same
6077 : problem. Same for (partially) storing into a non-memory object. */
6078 18398454 : if (handled_component_p (to)
6079 13830151 : || (TREE_CODE (to) == MEM_REF
6080 1660068 : && (REF_REVERSE_STORAGE_ORDER (to)
6081 1659988 : || mem_ref_refers_to_non_mem_p (to)))
6082 13720478 : || TREE_CODE (TREE_TYPE (to)) == ARRAY_TYPE)
6083 : {
6084 4799708 : machine_mode mode1;
6085 4799708 : poly_int64 bitsize, bitpos;
6086 4799708 : poly_uint64 bitregion_start = 0;
6087 4799708 : poly_uint64 bitregion_end = 0;
6088 4799708 : tree offset;
6089 4799708 : int unsignedp, reversep, volatilep = 0;
6090 4799708 : tree tem;
6091 :
6092 4799708 : push_temp_slots ();
6093 4799708 : tem = get_inner_reference (to, &bitsize, &bitpos, &offset, &mode1,
6094 : &unsignedp, &reversep, &volatilep);
6095 :
6096 : /* Make sure bitpos is not negative, it can wreak havoc later. */
6097 4799708 : if (maybe_lt (bitpos, 0))
6098 : {
6099 199 : gcc_assert (offset == NULL_TREE);
6100 199 : offset = size_int (bits_to_bytes_round_down (bitpos));
6101 199 : bitpos = num_trailing_bits (bitpos);
6102 : }
6103 :
6104 4799708 : if (TREE_CODE (to) == COMPONENT_REF
6105 4799708 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (to, 1)))
6106 85400 : get_bit_range (&bitregion_start, &bitregion_end, to, &bitpos, &offset);
6107 : /* The C++ memory model naturally applies to byte-aligned fields.
6108 : However, if we do not have a DECL_BIT_FIELD_TYPE but BITPOS or
6109 : BITSIZE are not byte-aligned, there is no need to limit the range
6110 : we can access. This can occur with packed structures in Ada. */
6111 4714308 : else if (maybe_gt (bitsize, 0)
6112 4714293 : && multiple_p (bitsize, BITS_PER_UNIT)
6113 9428334 : && multiple_p (bitpos, BITS_PER_UNIT))
6114 : {
6115 4714026 : bitregion_start = bitpos;
6116 4714026 : bitregion_end = bitpos + bitsize - 1;
6117 : }
6118 :
6119 4799708 : to_rtx = expand_expr (tem, NULL_RTX, VOIDmode, EXPAND_WRITE);
6120 :
6121 : /* If the field has a mode, we want to access it in the
6122 : field's mode, not the computed mode.
6123 : If a MEM has VOIDmode (external with incomplete type),
6124 : use BLKmode for it instead. */
6125 4799708 : if (MEM_P (to_rtx))
6126 : {
6127 4108772 : if (mode1 != VOIDmode)
6128 4042266 : to_rtx = adjust_address (to_rtx, mode1, 0);
6129 66506 : else if (GET_MODE (to_rtx) == VOIDmode)
6130 0 : to_rtx = adjust_address (to_rtx, BLKmode, 0);
6131 : }
6132 :
6133 4799708 : rtx stemp = NULL_RTX, old_to_rtx = NULL_RTX;
6134 4799708 : if (offset != 0)
6135 : {
6136 179408 : machine_mode address_mode;
6137 179408 : rtx offset_rtx;
6138 :
6139 179408 : if (!MEM_P (to_rtx))
6140 : {
6141 : /* We can get constant negative offsets into arrays with broken
6142 : user code. Translate this to a trap instead of ICEing. */
6143 5 : if (TREE_CODE (offset) == INTEGER_CST)
6144 : {
6145 2 : expand_builtin_trap ();
6146 2 : to_rtx = gen_rtx_MEM (BLKmode, const0_rtx);
6147 : }
6148 : /* Else spill for variable offset to the destination. We expect
6149 : to run into this only for hard registers. */
6150 : else
6151 : {
6152 3 : gcc_assert (VAR_P (tem) && DECL_HARD_REGISTER (tem));
6153 3 : stemp = assign_stack_temp (GET_MODE (to_rtx),
6154 6 : GET_MODE_SIZE (GET_MODE (to_rtx)));
6155 3 : emit_move_insn (stemp, to_rtx);
6156 3 : old_to_rtx = to_rtx;
6157 3 : to_rtx = stemp;
6158 : }
6159 : }
6160 :
6161 179408 : offset_rtx = expand_expr (offset, NULL_RTX, VOIDmode, EXPAND_SUM);
6162 179408 : address_mode = get_address_mode (to_rtx);
6163 179408 : if (GET_MODE (offset_rtx) != address_mode)
6164 : {
6165 : /* We cannot be sure that the RTL in offset_rtx is valid outside
6166 : of a memory address context, so force it into a register
6167 : before attempting to convert it to the desired mode. */
6168 224 : offset_rtx = force_operand (offset_rtx, NULL_RTX);
6169 224 : offset_rtx = convert_to_mode (address_mode, offset_rtx, 0);
6170 : }
6171 :
6172 : /* If we have an expression in OFFSET_RTX and a non-zero
6173 : byte offset in BITPOS, adding the byte offset before the
6174 : OFFSET_RTX results in better intermediate code, which makes
6175 : later rtl optimization passes perform better.
6176 :
6177 : We prefer intermediate code like this:
6178 :
6179 : r124:DI=r123:DI+0x18
6180 : [r124:DI]=r121:DI
6181 :
6182 : ... instead of ...
6183 :
6184 : r124:DI=r123:DI+0x10
6185 : [r124:DI+0x8]=r121:DI
6186 :
6187 : This is only done for aligned data values, as these can
6188 : be expected to result in single move instructions. */
6189 179408 : poly_int64 bytepos;
6190 179408 : if (mode1 != VOIDmode
6191 179300 : && maybe_ne (bitpos, 0)
6192 47762 : && maybe_gt (bitsize, 0)
6193 227044 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
6194 227044 : && multiple_p (bitpos, bitsize)
6195 95272 : && multiple_p (bitsize, GET_MODE_ALIGNMENT (mode1))
6196 227044 : && MEM_ALIGN (to_rtx) >= GET_MODE_ALIGNMENT (mode1))
6197 : {
6198 47387 : to_rtx = adjust_address (to_rtx, mode1, bytepos);
6199 47387 : bitregion_start = 0;
6200 47387 : if (known_ge (bitregion_end, poly_uint64 (bitpos)))
6201 47387 : bitregion_end -= bitpos;
6202 47387 : bitpos = 0;
6203 : }
6204 :
6205 179408 : to_rtx = offset_address (to_rtx, offset_rtx,
6206 : highest_pow2_factor_for_target (to,
6207 : offset));
6208 : }
6209 :
6210 : /* No action is needed if the target is not a memory and the field
6211 : lies completely outside that target. This can occur if the source
6212 : code contains an out-of-bounds access to a small array. */
6213 4799708 : if (!MEM_P (to_rtx)
6214 690931 : && GET_MODE (to_rtx) != BLKmode
6215 5490639 : && known_ge (bitpos, GET_MODE_PRECISION (GET_MODE (to_rtx))))
6216 : {
6217 3 : expand_normal (from);
6218 3 : result = NULL;
6219 : }
6220 : /* Handle expand_expr of a complex value returning a CONCAT. */
6221 4799705 : else if (GET_CODE (to_rtx) == CONCAT)
6222 : {
6223 122 : machine_mode to_mode = GET_MODE (to_rtx);
6224 122 : gcc_checking_assert (COMPLEX_MODE_P (to_mode));
6225 244 : poly_int64 mode_bitsize = GET_MODE_BITSIZE (to_mode);
6226 122 : unsigned short inner_bitsize = GET_MODE_UNIT_BITSIZE (to_mode);
6227 122 : if (TYPE_MODE (TREE_TYPE (from)) == to_mode
6228 0 : && known_eq (bitpos, 0)
6229 122 : && known_eq (bitsize, mode_bitsize))
6230 0 : result = store_expr (from, to_rtx, false, nontemporal, reversep);
6231 122 : else if (TYPE_MODE (TREE_TYPE (from)) == GET_MODE_INNER (to_mode)
6232 70 : && known_eq (bitsize, inner_bitsize)
6233 192 : && (known_eq (bitpos, 0)
6234 31 : || known_eq (bitpos, inner_bitsize)))
6235 70 : result = store_expr (from, XEXP (to_rtx, maybe_ne (bitpos, 0)),
6236 : false, nontemporal, reversep);
6237 52 : else if (known_le (bitpos + bitsize, inner_bitsize))
6238 5 : result = store_field (XEXP (to_rtx, 0), bitsize, bitpos,
6239 : bitregion_start, bitregion_end,
6240 : mode1, from, get_alias_set (to),
6241 : nontemporal, reversep);
6242 47 : else if (known_ge (bitpos, inner_bitsize))
6243 3 : result = store_field (XEXP (to_rtx, 1), bitsize,
6244 : bitpos - inner_bitsize,
6245 : bitregion_start, bitregion_end,
6246 : mode1, from, get_alias_set (to),
6247 : nontemporal, reversep);
6248 44 : else if (known_eq (bitpos, 0) && known_eq (bitsize, mode_bitsize))
6249 : {
6250 37 : result = expand_normal (from);
6251 37 : if (GET_CODE (result) == CONCAT)
6252 : {
6253 0 : to_mode = GET_MODE_INNER (to_mode);
6254 0 : machine_mode from_mode = GET_MODE_INNER (GET_MODE (result));
6255 0 : rtx from_real
6256 0 : = force_subreg (to_mode, XEXP (result, 0), from_mode, 0);
6257 0 : rtx from_imag
6258 0 : = force_subreg (to_mode, XEXP (result, 1), from_mode, 0);
6259 0 : if (!from_real || !from_imag)
6260 0 : goto concat_store_slow;
6261 0 : emit_move_insn (XEXP (to_rtx, 0), from_real);
6262 0 : emit_move_insn (XEXP (to_rtx, 1), from_imag);
6263 : }
6264 : else
6265 : {
6266 37 : machine_mode from_mode
6267 37 : = GET_MODE (result) == VOIDmode
6268 37 : ? TYPE_MODE (TREE_TYPE (from))
6269 37 : : GET_MODE (result);
6270 37 : rtx from_rtx;
6271 37 : if (MEM_P (result))
6272 1 : from_rtx = change_address (result, to_mode, NULL_RTX);
6273 : else
6274 36 : from_rtx = force_subreg (to_mode, result, from_mode, 0);
6275 37 : if (from_rtx)
6276 : {
6277 37 : emit_move_insn (XEXP (to_rtx, 0),
6278 : read_complex_part (from_rtx, false));
6279 37 : emit_move_insn (XEXP (to_rtx, 1),
6280 : read_complex_part (from_rtx, true));
6281 : }
6282 : else
6283 : {
6284 0 : to_mode = GET_MODE_INNER (to_mode);
6285 0 : rtx from_real
6286 0 : = force_subreg (to_mode, result, from_mode, 0);
6287 0 : rtx from_imag
6288 0 : = force_subreg (to_mode, result, from_mode,
6289 0 : GET_MODE_SIZE (to_mode));
6290 0 : if (!from_real || !from_imag)
6291 0 : goto concat_store_slow;
6292 0 : emit_move_insn (XEXP (to_rtx, 0), from_real);
6293 0 : emit_move_insn (XEXP (to_rtx, 1), from_imag);
6294 : }
6295 : }
6296 : }
6297 : else
6298 : {
6299 7 : concat_store_slow:;
6300 7 : rtx temp = assign_stack_temp (GET_MODE (to_rtx),
6301 14 : GET_MODE_SIZE (GET_MODE (to_rtx)));
6302 7 : write_complex_part (temp, XEXP (to_rtx, 0), false, true);
6303 7 : write_complex_part (temp, XEXP (to_rtx, 1), true, false);
6304 7 : result = store_field (temp, bitsize, bitpos,
6305 : bitregion_start, bitregion_end,
6306 : mode1, from, get_alias_set (to),
6307 : nontemporal, reversep);
6308 7 : emit_move_insn (XEXP (to_rtx, 0), read_complex_part (temp, false));
6309 7 : emit_move_insn (XEXP (to_rtx, 1), read_complex_part (temp, true));
6310 : }
6311 : }
6312 : /* For calls to functions returning variable length structures, if TO_RTX
6313 : is not a MEM, go through a MEM because we must not create temporaries
6314 : of the VLA type. */
6315 4799583 : else if (!MEM_P (to_rtx)
6316 690806 : && TREE_CODE (from) == CALL_EXPR
6317 235 : && COMPLETE_TYPE_P (TREE_TYPE (from))
6318 4799818 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (from))) != INTEGER_CST)
6319 : {
6320 6 : rtx temp = assign_stack_temp (GET_MODE (to_rtx),
6321 12 : GET_MODE_SIZE (GET_MODE (to_rtx)));
6322 6 : result = store_field (temp, bitsize, bitpos, bitregion_start,
6323 : bitregion_end, mode1, from, get_alias_set (to),
6324 : nontemporal, reversep);
6325 6 : emit_move_insn (to_rtx, temp);
6326 : }
6327 : else
6328 : {
6329 4799577 : if (MEM_P (to_rtx))
6330 : {
6331 : /* If the field is at offset zero, we could have been given the
6332 : DECL_RTX of the parent struct. Don't munge it. */
6333 4108777 : to_rtx = shallow_copy_rtx (to_rtx);
6334 4108777 : set_mem_attributes_minus_bitpos (to_rtx, to, 0, bitpos);
6335 4108777 : if (volatilep)
6336 8722 : MEM_VOLATILE_P (to_rtx) = 1;
6337 : }
6338 :
6339 4799577 : gcc_checking_assert (known_ge (bitpos, 0));
6340 9599154 : if (optimize_bitfield_assignment_op (bitsize, bitpos,
6341 : bitregion_start, bitregion_end,
6342 : mode1, to_rtx, to, from,
6343 : reversep))
6344 : result = NULL;
6345 4799411 : else if (SUBREG_P (to_rtx)
6346 4799411 : && SUBREG_PROMOTED_VAR_P (to_rtx))
6347 : {
6348 : /* If to_rtx is a promoted subreg, we need to zero or sign
6349 : extend the value afterwards. */
6350 0 : if (TREE_CODE (to) == MEM_REF
6351 0 : && TYPE_MODE (TREE_TYPE (from)) != BLKmode
6352 0 : && !REF_REVERSE_STORAGE_ORDER (to)
6353 0 : && known_eq (bitpos, 0)
6354 0 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (to_rtx))))
6355 0 : result = store_expr (from, to_rtx, 0, nontemporal, false);
6356 : /* Check if the field overlaps the MSB, requiring extension. */
6357 0 : else if (store_field_updates_msb_p (bitpos, bitsize, to_rtx))
6358 : {
6359 0 : scalar_int_mode imode = subreg_unpromoted_mode (to_rtx);
6360 0 : scalar_int_mode omode = subreg_promoted_mode (to_rtx);
6361 0 : rtx to_rtx1 = lowpart_subreg (imode, SUBREG_REG (to_rtx),
6362 : omode);
6363 0 : result = store_field (to_rtx1, bitsize, bitpos,
6364 : bitregion_start, bitregion_end,
6365 : mode1, from, get_alias_set (to),
6366 : nontemporal, reversep);
6367 : /* If the target usually keeps IMODE appropriately
6368 : extended in OMODE it's unsafe to refer to it using
6369 : a SUBREG whilst this invariant doesn't hold. */
6370 0 : if (targetm.mode_rep_extended (imode, omode) != UNKNOWN)
6371 0 : to_rtx1 = simplify_gen_unary (TRUNCATE, imode,
6372 : SUBREG_REG (to_rtx), omode);
6373 0 : convert_move (SUBREG_REG (to_rtx), to_rtx1,
6374 0 : SUBREG_PROMOTED_SIGN (to_rtx));
6375 : }
6376 : else
6377 0 : result = store_field (to_rtx, bitsize, bitpos,
6378 : bitregion_start, bitregion_end,
6379 : mode1, from, get_alias_set (to),
6380 : nontemporal, reversep);
6381 : }
6382 : else
6383 4799411 : result = store_field (to_rtx, bitsize, bitpos,
6384 : bitregion_start, bitregion_end,
6385 : mode1, from, get_alias_set (to),
6386 : nontemporal, reversep);
6387 : /* Move the temporary storage back to the non-MEM_P. */
6388 4799577 : if (stemp)
6389 3 : emit_move_insn (old_to_rtx, stemp);
6390 : }
6391 :
6392 4799708 : if (result)
6393 832245 : preserve_temp_slots (result);
6394 4799708 : pop_temp_slots ();
6395 4799708 : return;
6396 : }
6397 :
6398 : /* If the rhs is a function call and its value is not an aggregate,
6399 : call the function before we start to compute the lhs.
6400 : This is needed for correct code for cases such as
6401 : val = setjmp (buf) on machines where reference to val
6402 : requires loading up part of an address in a separate insn.
6403 :
6404 : Don't do this if TO is a VAR_DECL or PARM_DECL whose DECL_RTL is REG
6405 : since it might be a promoted variable where the zero- or sign- extension
6406 : needs to be done. Handling this in the normal way is safe because no
6407 : computation is done before the call. The same is true for SSA names. */
6408 2360529 : if (TREE_CODE (from) == CALL_EXPR && ! aggregate_value_p (from, from)
6409 2161231 : && COMPLETE_TYPE_P (TREE_TYPE (from))
6410 2161231 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (from))) == INTEGER_CST
6411 15759977 : && ! (((VAR_P (to)
6412 : || TREE_CODE (to) == PARM_DECL
6413 : || TREE_CODE (to) == RESULT_DECL)
6414 122301 : && REG_P (DECL_RTL (to)))
6415 2049063 : || TREE_CODE (to) == SSA_NAME))
6416 : {
6417 10465 : rtx value;
6418 :
6419 10465 : push_temp_slots ();
6420 10465 : value = expand_normal (from);
6421 :
6422 10465 : if (to_rtx == 0)
6423 10465 : to_rtx = expand_expr (to, NULL_RTX, VOIDmode, EXPAND_WRITE);
6424 :
6425 : /* Handle calls that return values in multiple non-contiguous locations.
6426 : The Irix 6 ABI has examples of this. */
6427 10465 : if (GET_CODE (to_rtx) == PARALLEL)
6428 : {
6429 0 : if (GET_CODE (value) == PARALLEL)
6430 0 : emit_group_move (to_rtx, value);
6431 : else
6432 0 : emit_group_load (to_rtx, value, TREE_TYPE (from),
6433 0 : int_size_in_bytes (TREE_TYPE (from)));
6434 : }
6435 10465 : else if (GET_CODE (value) == PARALLEL)
6436 1957 : emit_group_store (to_rtx, value, TREE_TYPE (from),
6437 1957 : int_size_in_bytes (TREE_TYPE (from)));
6438 8508 : else if (GET_MODE (to_rtx) == BLKmode)
6439 : {
6440 : /* Handle calls that return BLKmode values in registers. */
6441 254 : if (REG_P (value))
6442 254 : copy_blkmode_from_reg (to_rtx, value, TREE_TYPE (from));
6443 : else
6444 0 : emit_block_move (to_rtx, value, expr_size (from), BLOCK_OP_NORMAL);
6445 : }
6446 : else
6447 : {
6448 8254 : if (POINTER_TYPE_P (TREE_TYPE (to)))
6449 0 : value = convert_memory_address_addr_space
6450 0 : (as_a <scalar_int_mode> (GET_MODE (to_rtx)), value,
6451 0 : TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (to))));
6452 :
6453 8254 : emit_move_insn (to_rtx, value);
6454 : }
6455 :
6456 10465 : preserve_temp_slots (to_rtx);
6457 10465 : pop_temp_slots ();
6458 10465 : return;
6459 : }
6460 :
6461 : /* Ordinary treatment. Expand TO to get a REG or MEM rtx. */
6462 13588281 : to_rtx = expand_expr (to, NULL_RTX, VOIDmode, EXPAND_WRITE);
6463 :
6464 : /* Don't move directly into a return register. */
6465 13588281 : if (TREE_CODE (to) == RESULT_DECL
6466 29433 : && (REG_P (to_rtx) || GET_CODE (to_rtx) == PARALLEL))
6467 : {
6468 0 : rtx temp;
6469 :
6470 0 : push_temp_slots ();
6471 :
6472 : /* If the source is itself a return value, it still is in a pseudo at
6473 : this point so we can move it back to the return register directly. */
6474 0 : if (REG_P (to_rtx)
6475 0 : && TYPE_MODE (TREE_TYPE (from)) == BLKmode
6476 0 : && TREE_CODE (from) != CALL_EXPR)
6477 0 : temp = copy_blkmode_to_reg (GET_MODE (to_rtx), from);
6478 : else
6479 0 : temp = expand_expr (from, NULL_RTX, GET_MODE (to_rtx), EXPAND_NORMAL);
6480 :
6481 : /* Handle calls that return values in multiple non-contiguous locations.
6482 : The Irix 6 ABI has examples of this. */
6483 0 : if (GET_CODE (to_rtx) == PARALLEL)
6484 : {
6485 0 : if (GET_CODE (temp) == PARALLEL)
6486 0 : emit_group_move (to_rtx, temp);
6487 : else
6488 0 : emit_group_load (to_rtx, temp, TREE_TYPE (from),
6489 0 : int_size_in_bytes (TREE_TYPE (from)));
6490 : }
6491 0 : else if (temp)
6492 0 : emit_move_insn (to_rtx, temp);
6493 :
6494 0 : preserve_temp_slots (to_rtx);
6495 0 : pop_temp_slots ();
6496 0 : return;
6497 : }
6498 :
6499 : /* In case we are returning the contents of an object which overlaps
6500 : the place the value is being stored, use a safe function when copying
6501 : a value through a pointer into a structure value return block. */
6502 13588281 : if (TREE_CODE (to) == RESULT_DECL
6503 29433 : && TREE_CODE (from) == INDIRECT_REF
6504 0 : && ADDR_SPACE_GENERIC_P
6505 : (TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (from, 0)))))
6506 0 : && refs_may_alias_p (to, from)
6507 0 : && cfun->returns_struct
6508 13588281 : && !cfun->returns_pcc_struct)
6509 : {
6510 0 : rtx from_rtx, size;
6511 :
6512 0 : push_temp_slots ();
6513 0 : size = expr_size (from);
6514 0 : from_rtx = expand_normal (from);
6515 :
6516 0 : emit_block_move_via_libcall (XEXP (to_rtx, 0), XEXP (from_rtx, 0), size);
6517 :
6518 0 : preserve_temp_slots (to_rtx);
6519 0 : pop_temp_slots ();
6520 0 : return;
6521 : }
6522 :
6523 : /* Compute FROM and store the value in the rtx we got. */
6524 :
6525 13588281 : push_temp_slots ();
6526 13588281 : result = store_expr (from, to_rtx, 0, nontemporal, false);
6527 13588279 : preserve_temp_slots (result);
6528 13588279 : pop_temp_slots ();
6529 13588279 : return;
6530 : }
6531 :
6532 : /* Emits nontemporal store insn that moves FROM to TO. Returns true if this
6533 : succeeded, false otherwise. */
6534 :
6535 : bool
6536 17 : emit_storent_insn (rtx to, rtx from)
6537 : {
6538 17 : class expand_operand ops[2];
6539 17 : machine_mode mode = GET_MODE (to);
6540 17 : enum insn_code code = optab_handler (storent_optab, mode);
6541 :
6542 17 : if (code == CODE_FOR_nothing)
6543 : return false;
6544 :
6545 17 : create_fixed_operand (&ops[0], to);
6546 17 : create_input_operand (&ops[1], from, mode);
6547 17 : return maybe_expand_insn (code, 2, ops);
6548 : }
6549 :
6550 : /* Helper function for store_expr storing of STRING_CST. */
6551 :
6552 : static rtx
6553 80362 : string_cst_read_str (void *data, void *, HOST_WIDE_INT offset,
6554 : fixed_size_mode mode)
6555 : {
6556 80362 : tree str = (tree) data;
6557 :
6558 80362 : gcc_assert (offset >= 0);
6559 80362 : if (offset >= TREE_STRING_LENGTH (str))
6560 3344 : return const0_rtx;
6561 :
6562 77018 : if ((unsigned HOST_WIDE_INT) offset + GET_MODE_SIZE (mode)
6563 77018 : > (unsigned HOST_WIDE_INT) TREE_STRING_LENGTH (str))
6564 : {
6565 2476 : char *p = XALLOCAVEC (char, GET_MODE_SIZE (mode));
6566 2476 : size_t l = TREE_STRING_LENGTH (str) - offset;
6567 2476 : memcpy (p, TREE_STRING_POINTER (str) + offset, l);
6568 2476 : memset (p + l, '\0', GET_MODE_SIZE (mode) - l);
6569 2476 : return c_readstr (p, mode, false);
6570 : }
6571 :
6572 74542 : return c_readstr (TREE_STRING_POINTER (str) + offset, mode, false);
6573 : }
6574 :
6575 : /* Helper function for store_expr storing of RAW_DATA_CST. */
6576 :
6577 : static rtx
6578 36 : raw_data_cst_read_str (void *data, void *, HOST_WIDE_INT offset,
6579 : fixed_size_mode mode)
6580 : {
6581 36 : tree cst = (tree) data;
6582 :
6583 36 : gcc_assert (offset >= 0);
6584 36 : if (offset >= RAW_DATA_LENGTH (cst))
6585 0 : return const0_rtx;
6586 :
6587 36 : if ((unsigned HOST_WIDE_INT) offset + GET_MODE_SIZE (mode)
6588 36 : > (unsigned HOST_WIDE_INT) RAW_DATA_LENGTH (cst))
6589 : {
6590 0 : char *p = XALLOCAVEC (char, GET_MODE_SIZE (mode));
6591 0 : size_t l = RAW_DATA_LENGTH (cst) - offset;
6592 0 : memcpy (p, RAW_DATA_POINTER (cst) + offset, l);
6593 0 : memset (p + l, '\0', GET_MODE_SIZE (mode) - l);
6594 0 : return c_readstr (p, mode, false);
6595 : }
6596 :
6597 36 : return c_readstr (RAW_DATA_POINTER (cst) + offset, mode, false);
6598 : }
6599 :
6600 : /* Generate code for computing expression EXP,
6601 : and storing the value into TARGET.
6602 :
6603 : If the mode is BLKmode then we may return TARGET itself.
6604 : It turns out that in BLKmode it doesn't cause a problem.
6605 : because C has no operators that could combine two different
6606 : assignments into the same BLKmode object with different values
6607 : with no sequence point. Will other languages need this to
6608 : be more thorough?
6609 :
6610 : If CALL_PARAM_P is nonzero, this is a store into a call param on the
6611 : stack, and block moves may need to be treated specially.
6612 :
6613 : If NONTEMPORAL is true, try using a nontemporal store instruction.
6614 :
6615 : If REVERSE is true, the store is to be done in reverse order. */
6616 :
6617 : rtx
6618 17598205 : store_expr (tree exp, rtx target, int call_param_p,
6619 : bool nontemporal, bool reverse)
6620 : {
6621 17598205 : rtx temp;
6622 17598205 : rtx alt_rtl = NULL_RTX;
6623 17598205 : location_t loc = curr_insn_location ();
6624 17598205 : bool shortened_string_cst = false;
6625 :
6626 17598205 : if (VOID_TYPE_P (TREE_TYPE (exp)))
6627 : {
6628 : /* C++ can generate ?: expressions with a throw expression in one
6629 : branch and an rvalue in the other. Here, we resolve attempts to
6630 : store the throw expression's nonexistent result. */
6631 0 : gcc_assert (!call_param_p);
6632 0 : expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
6633 0 : return NULL_RTX;
6634 : }
6635 17598205 : if (TREE_CODE (exp) == COMPOUND_EXPR)
6636 : {
6637 : /* Perform first part of compound expression, then assign from second
6638 : part. */
6639 0 : expand_expr (TREE_OPERAND (exp, 0), const0_rtx, VOIDmode,
6640 : call_param_p ? EXPAND_STACK_PARM : EXPAND_NORMAL);
6641 0 : return store_expr (TREE_OPERAND (exp, 1), target,
6642 0 : call_param_p, nontemporal, reverse);
6643 : }
6644 17598205 : else if (TREE_CODE (exp) == COND_EXPR && GET_MODE (target) == BLKmode)
6645 : {
6646 : /* For conditional expression, get safe form of the target. Then
6647 : test the condition, doing the appropriate assignment on either
6648 : side. This avoids the creation of unnecessary temporaries.
6649 : For non-BLKmode, it is more efficient not to do this. */
6650 :
6651 0 : rtx_code_label *lab1 = gen_label_rtx (), *lab2 = gen_label_rtx ();
6652 :
6653 0 : do_pending_stack_adjust ();
6654 0 : NO_DEFER_POP;
6655 0 : jumpifnot (TREE_OPERAND (exp, 0), lab1,
6656 : profile_probability::uninitialized ());
6657 0 : store_expr (TREE_OPERAND (exp, 1), target, call_param_p,
6658 : nontemporal, reverse);
6659 0 : emit_jump_insn (targetm.gen_jump (lab2));
6660 0 : emit_barrier ();
6661 0 : emit_label (lab1);
6662 0 : store_expr (TREE_OPERAND (exp, 2), target, call_param_p,
6663 : nontemporal, reverse);
6664 0 : emit_label (lab2);
6665 0 : OK_DEFER_POP;
6666 :
6667 0 : return NULL_RTX;
6668 : }
6669 17598205 : else if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
6670 : /* If this is a scalar in a register that is stored in a wider mode
6671 : than the declared mode, compute the result into its declared mode
6672 : and then convert to the wider mode. Our value is the computed
6673 : expression. */
6674 : {
6675 7 : rtx inner_target = 0;
6676 7 : scalar_int_mode outer_mode = subreg_unpromoted_mode (target);
6677 7 : scalar_int_mode inner_mode = subreg_promoted_mode (target);
6678 :
6679 : /* We can do the conversion inside EXP, which will often result
6680 : in some optimizations. Do the conversion in two steps: first
6681 : change the signedness, if needed, then the extend. But don't
6682 : do this if the type of EXP is a subtype of something else
6683 : since then the conversion might involve more than just
6684 : converting modes. */
6685 14 : if (INTEGRAL_TYPE_P (TREE_TYPE (exp))
6686 0 : && TREE_TYPE (TREE_TYPE (exp)) == 0
6687 7 : && GET_MODE_PRECISION (outer_mode)
6688 0 : == TYPE_PRECISION (TREE_TYPE (exp)))
6689 : {
6690 0 : if (!SUBREG_CHECK_PROMOTED_SIGN (target,
6691 : TYPE_UNSIGNED (TREE_TYPE (exp))))
6692 : {
6693 : /* Some types, e.g. Fortran's logical*4, won't have a signed
6694 : version, so use the mode instead. */
6695 0 : tree ntype
6696 : = (signed_or_unsigned_type_for
6697 0 : (SUBREG_PROMOTED_SIGN (target), TREE_TYPE (exp)));
6698 0 : if (ntype == NULL)
6699 0 : ntype = lang_hooks.types.type_for_mode
6700 0 : (TYPE_MODE (TREE_TYPE (exp)),
6701 0 : SUBREG_PROMOTED_SIGN (target));
6702 :
6703 0 : exp = fold_convert_loc (loc, ntype, exp);
6704 : }
6705 :
6706 0 : exp = fold_convert_loc (loc, lang_hooks.types.type_for_mode
6707 0 : (inner_mode, SUBREG_PROMOTED_SIGN (target)),
6708 : exp);
6709 :
6710 0 : inner_target = SUBREG_REG (target);
6711 : }
6712 :
6713 7 : temp = expand_expr (exp, inner_target, VOIDmode,
6714 : call_param_p ? EXPAND_STACK_PARM : EXPAND_NORMAL);
6715 :
6716 :
6717 : /* If TEMP is a VOIDmode constant, use convert_modes to make
6718 : sure that we properly convert it. */
6719 7 : if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode)
6720 : {
6721 0 : temp = convert_modes (outer_mode, TYPE_MODE (TREE_TYPE (exp)),
6722 0 : temp, SUBREG_PROMOTED_SIGN (target));
6723 0 : temp = convert_modes (inner_mode, outer_mode, temp,
6724 0 : SUBREG_PROMOTED_SIGN (target));
6725 0 : }
6726 7 : else if (!SCALAR_INT_MODE_P (GET_MODE (temp)))
6727 0 : temp = convert_modes (outer_mode, TYPE_MODE (TREE_TYPE (exp)),
6728 0 : temp, SUBREG_PROMOTED_SIGN (target));
6729 :
6730 21 : convert_move (SUBREG_REG (target), temp,
6731 7 : SUBREG_PROMOTED_SIGN (target));
6732 :
6733 7 : return NULL_RTX;
6734 : }
6735 17598198 : else if ((TREE_CODE (exp) == STRING_CST
6736 17590902 : || (TREE_CODE (exp) == MEM_REF
6737 1360708 : && TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR
6738 443685 : && TREE_CODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
6739 : == STRING_CST
6740 6545 : && integer_zerop (TREE_OPERAND (exp, 1))))
6741 13841 : && !nontemporal && !call_param_p
6742 17612039 : && MEM_P (target))
6743 : {
6744 : /* Optimize initialization of an array with a STRING_CST. */
6745 13823 : HOST_WIDE_INT exp_len, str_copy_len;
6746 13823 : rtx dest_mem;
6747 13823 : tree str = TREE_CODE (exp) == STRING_CST
6748 13823 : ? exp : TREE_OPERAND (TREE_OPERAND (exp, 0), 0);
6749 :
6750 13823 : exp_len = int_expr_size (exp);
6751 13823 : if (exp_len <= 0)
6752 0 : goto normal_expr;
6753 :
6754 13823 : if (TREE_STRING_LENGTH (str) <= 0)
6755 0 : goto normal_expr;
6756 :
6757 27646 : if (can_store_by_pieces (exp_len, string_cst_read_str, (void *) str,
6758 13823 : MEM_ALIGN (target), false))
6759 : {
6760 13627 : store_by_pieces (target, exp_len, string_cst_read_str, (void *) str,
6761 13627 : MEM_ALIGN (target), false, RETURN_BEGIN);
6762 13627 : return NULL_RTX;
6763 : }
6764 :
6765 196 : str_copy_len = TREE_STRING_LENGTH (str);
6766 :
6767 : /* Trailing NUL bytes in EXP will be handled by the call to
6768 : clear_storage, which is more efficient than copying them from
6769 : the STRING_CST, so trim those from STR_COPY_LEN. */
6770 329 : while (str_copy_len)
6771 : {
6772 276 : if (TREE_STRING_POINTER (str)[str_copy_len - 1])
6773 : break;
6774 133 : str_copy_len--;
6775 : }
6776 :
6777 196 : if ((STORE_MAX_PIECES & (STORE_MAX_PIECES - 1)) == 0)
6778 : {
6779 196 : str_copy_len += STORE_MAX_PIECES - 1;
6780 196 : str_copy_len &= ~(STORE_MAX_PIECES - 1);
6781 : }
6782 196 : if (str_copy_len >= exp_len)
6783 135 : goto normal_expr;
6784 :
6785 122 : if (!can_store_by_pieces (str_copy_len, string_cst_read_str,
6786 61 : (void *) str, MEM_ALIGN (target), false))
6787 2 : goto normal_expr;
6788 :
6789 59 : dest_mem = store_by_pieces (target, str_copy_len, string_cst_read_str,
6790 59 : (void *) str, MEM_ALIGN (target), false,
6791 : RETURN_END);
6792 59 : clear_storage (adjust_address_1 (dest_mem, BLKmode, 0, 1, 1, 0,
6793 59 : exp_len - str_copy_len),
6794 : GEN_INT (exp_len - str_copy_len), BLOCK_OP_NORMAL);
6795 59 : return NULL_RTX;
6796 : }
6797 : else
6798 : {
6799 17584512 : rtx tmp_target;
6800 :
6801 17584512 : normal_expr:
6802 : /* If we want to use a nontemporal or a reverse order store, force the
6803 : value into a register first. */
6804 17584512 : tmp_target = nontemporal || reverse ? NULL_RTX : target;
6805 17584512 : tree rexp = exp;
6806 17584512 : if (TREE_CODE (exp) == STRING_CST
6807 20 : && tmp_target == target
6808 20 : && GET_MODE (target) == BLKmode
6809 17584532 : && TYPE_MODE (TREE_TYPE (exp)) == BLKmode)
6810 : {
6811 20 : rtx size = expr_size (exp);
6812 20 : if (CONST_INT_P (size)
6813 20 : && size != const0_rtx
6814 40 : && (UINTVAL (size)
6815 20 : > ((unsigned HOST_WIDE_INT) TREE_STRING_LENGTH (exp) + 32)))
6816 : {
6817 : /* If the STRING_CST has much larger array type than
6818 : TREE_STRING_LENGTH, only emit the TREE_STRING_LENGTH part of
6819 : it into the rodata section as the code later on will use
6820 : memset zero for the remainder anyway. See PR95052. */
6821 2 : tmp_target = NULL_RTX;
6822 2 : rexp = copy_node (exp);
6823 2 : tree index
6824 2 : = build_index_type (size_int (TREE_STRING_LENGTH (exp) - 1));
6825 2 : TREE_TYPE (rexp) = build_array_type (TREE_TYPE (TREE_TYPE (exp)),
6826 : index);
6827 2 : shortened_string_cst = true;
6828 : }
6829 : }
6830 35169024 : temp = expand_expr_real (rexp, tmp_target, GET_MODE (target),
6831 : (call_param_p
6832 : ? EXPAND_STACK_PARM : EXPAND_NORMAL),
6833 : &alt_rtl, false);
6834 17584510 : if (shortened_string_cst)
6835 : {
6836 2 : gcc_assert (MEM_P (temp));
6837 2 : temp = change_address (temp, BLKmode, NULL_RTX);
6838 : }
6839 : }
6840 :
6841 : /* If TEMP is a VOIDmode constant and the mode of the type of EXP is not
6842 : the same as that of TARGET, adjust the constant. This is needed, for
6843 : example, in case it is a CONST_DOUBLE or CONST_WIDE_INT and we want
6844 : only a word-sized value. */
6845 3633841 : if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode
6846 2623384 : && TREE_CODE (exp) != ERROR_MARK
6847 20207894 : && GET_MODE (target) != TYPE_MODE (TREE_TYPE (exp)))
6848 : {
6849 0 : gcc_assert (!shortened_string_cst);
6850 0 : if (GET_MODE_CLASS (GET_MODE (target))
6851 0 : != GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (exp)))
6852 0 : && known_eq (GET_MODE_BITSIZE (GET_MODE (target)),
6853 : GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (exp)))))
6854 : {
6855 0 : rtx t = simplify_gen_subreg (GET_MODE (target), temp,
6856 0 : TYPE_MODE (TREE_TYPE (exp)), 0);
6857 0 : if (t)
6858 0 : temp = t;
6859 : }
6860 0 : if (GET_MODE (temp) == VOIDmode)
6861 0 : temp = convert_modes (GET_MODE (target), TYPE_MODE (TREE_TYPE (exp)),
6862 0 : temp, TYPE_UNSIGNED (TREE_TYPE (exp)));
6863 : }
6864 :
6865 : /* If value was not generated in the target, store it there.
6866 : Convert the value to TARGET's type first if necessary and emit the
6867 : pending incrementations that have been queued when expanding EXP.
6868 : Note that we cannot emit the whole queue blindly because this will
6869 : effectively disable the POST_INC optimization later.
6870 :
6871 : If TEMP and TARGET compare equal according to rtx_equal_p, but
6872 : one or both of them are volatile memory refs, we have to distinguish
6873 : two cases:
6874 : - expand_expr has used TARGET. In this case, we must not generate
6875 : another copy. This can be detected by TARGET being equal according
6876 : to == .
6877 : - expand_expr has not used TARGET - that means that the source just
6878 : happens to have the same RTX form. Since temp will have been created
6879 : by expand_expr, it will compare unequal according to == .
6880 : We must generate a copy in this case, to reach the correct number
6881 : of volatile memory references. */
6882 :
6883 17584510 : if ((! rtx_equal_p (temp, target)
6884 3133118 : || (temp != target && (side_effects_p (temp)
6885 47135 : || side_effects_p (target)
6886 47135 : || (MEM_P (temp)
6887 46656 : && !mems_same_for_tbaa_p (temp, target)))))
6888 14451396 : && TREE_CODE (exp) != ERROR_MARK
6889 : /* If store_expr stores a DECL whose DECL_RTL(exp) == TARGET,
6890 : but TARGET is not valid memory reference, TEMP will differ
6891 : from TARGET although it is really the same location. */
6892 14451396 : && !(alt_rtl
6893 1985462 : && rtx_equal_p (alt_rtl, target)
6894 1 : && !side_effects_p (alt_rtl)
6895 0 : && !side_effects_p (target))
6896 : /* If there's nothing to copy, don't bother. Don't call
6897 : expr_size unless necessary, because some front-ends (C++)
6898 : expr_size-hook must not be given objects that are not
6899 : supposed to be bit-copied or bit-initialized. */
6900 32035906 : && expr_size (exp) != const0_rtx)
6901 : {
6902 14451396 : if (GET_MODE (temp) != GET_MODE (target) && GET_MODE (temp) != VOIDmode)
6903 : {
6904 1378 : gcc_assert (!shortened_string_cst);
6905 1378 : if (GET_MODE (target) == BLKmode)
6906 : {
6907 : /* Handle calls that return BLKmode values in registers. */
6908 2 : if (REG_P (temp) && TREE_CODE (exp) == CALL_EXPR)
6909 2 : copy_blkmode_from_reg (target, temp, TREE_TYPE (exp));
6910 : else
6911 0 : store_bit_field (target,
6912 0 : rtx_to_poly_int64 (expr_size (exp))
6913 0 : * BITS_PER_UNIT,
6914 : 0, 0, 0, GET_MODE (temp), temp, reverse,
6915 : false);
6916 : }
6917 : else
6918 1376 : convert_move (target, temp, TYPE_UNSIGNED (TREE_TYPE (exp)));
6919 : }
6920 :
6921 14450018 : else if (GET_MODE (temp) == BLKmode && TREE_CODE (exp) == STRING_CST)
6922 : {
6923 : /* Handle copying a string constant into an array. The string
6924 : constant may be shorter than the array. So copy just the string's
6925 : actual length, and clear the rest. First get the size of the data
6926 : type of the string, which is actually the size of the target. */
6927 20 : rtx size = expr_size (exp);
6928 :
6929 20 : if (CONST_INT_P (size)
6930 20 : && INTVAL (size) < TREE_STRING_LENGTH (exp))
6931 0 : emit_block_move (target, temp, size,
6932 : (call_param_p
6933 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
6934 : else
6935 : {
6936 20 : machine_mode pointer_mode
6937 20 : = targetm.addr_space.pointer_mode (MEM_ADDR_SPACE (target));
6938 20 : machine_mode address_mode = get_address_mode (target);
6939 :
6940 : /* Compute the size of the data to copy from the string. */
6941 20 : tree copy_size
6942 20 : = size_binop_loc (loc, MIN_EXPR,
6943 : make_tree (sizetype, size),
6944 20 : size_int (TREE_STRING_LENGTH (exp)));
6945 20 : rtx copy_size_rtx
6946 20 : = expand_expr (copy_size, NULL_RTX, VOIDmode,
6947 : (call_param_p
6948 : ? EXPAND_STACK_PARM : EXPAND_NORMAL));
6949 20 : rtx_code_label *label = 0;
6950 :
6951 : /* Copy that much. */
6952 60 : copy_size_rtx = convert_to_mode (pointer_mode, copy_size_rtx,
6953 20 : TYPE_UNSIGNED (sizetype));
6954 40 : emit_block_move (target, temp, copy_size_rtx,
6955 : (call_param_p
6956 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
6957 :
6958 : /* Figure out how much is left in TARGET that we have to clear.
6959 : Do all calculations in pointer_mode. */
6960 20 : poly_int64 const_copy_size;
6961 20 : if (poly_int_rtx_p (copy_size_rtx, &const_copy_size))
6962 : {
6963 20 : size = plus_constant (address_mode, size, -const_copy_size);
6964 20 : target = adjust_address (target, BLKmode, const_copy_size);
6965 : }
6966 : else
6967 : {
6968 0 : size = expand_binop (TYPE_MODE (sizetype), sub_optab, size,
6969 : copy_size_rtx, NULL_RTX, 0,
6970 : OPTAB_LIB_WIDEN);
6971 :
6972 0 : if (GET_MODE (copy_size_rtx) != address_mode)
6973 0 : copy_size_rtx = convert_to_mode (address_mode,
6974 : copy_size_rtx,
6975 0 : TYPE_UNSIGNED (sizetype));
6976 :
6977 0 : target = offset_address (target, copy_size_rtx,
6978 : highest_pow2_factor (copy_size));
6979 0 : label = gen_label_rtx ();
6980 0 : emit_cmp_and_jump_insns (size, const0_rtx, LT, NULL_RTX,
6981 0 : GET_MODE (size), 0, label);
6982 : }
6983 :
6984 20 : if (size != const0_rtx)
6985 2 : clear_storage (target, size, BLOCK_OP_NORMAL);
6986 :
6987 20 : if (label)
6988 0 : emit_label (label);
6989 : }
6990 : }
6991 14449998 : else if (shortened_string_cst)
6992 0 : gcc_unreachable ();
6993 : /* Handle calls that return values in multiple non-contiguous locations.
6994 : The Irix 6 ABI has examples of this. */
6995 14449998 : else if (GET_CODE (target) == PARALLEL)
6996 : {
6997 0 : if (GET_CODE (temp) == PARALLEL)
6998 0 : emit_group_move (target, temp);
6999 : else
7000 0 : emit_group_load (target, temp, TREE_TYPE (exp),
7001 0 : int_size_in_bytes (TREE_TYPE (exp)));
7002 : }
7003 14449998 : else if (GET_CODE (temp) == PARALLEL)
7004 0 : emit_group_store (target, temp, TREE_TYPE (exp),
7005 0 : int_size_in_bytes (TREE_TYPE (exp)));
7006 14449998 : else if (GET_MODE (temp) == BLKmode)
7007 603616 : emit_block_move (target, temp, expr_size (exp),
7008 : (call_param_p
7009 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
7010 : /* If we emit a nontemporal store, there is nothing else to do. */
7011 14148190 : else if (nontemporal && emit_storent_insn (target, temp))
7012 : ;
7013 : else
7014 : {
7015 14148173 : if (reverse)
7016 727 : temp = flip_storage_order (GET_MODE (target), temp);
7017 14148173 : temp = force_operand (temp, target);
7018 14148173 : if (temp != target)
7019 14146580 : emit_move_insn (target, temp);
7020 : }
7021 : }
7022 : else
7023 3133114 : gcc_assert (!shortened_string_cst);
7024 :
7025 : return NULL_RTX;
7026 : }
7027 :
7028 : /* Return true if field F of structure TYPE is a flexible array. */
7029 :
7030 : static bool
7031 3943396 : flexible_array_member_p (const_tree f, const_tree type)
7032 : {
7033 3943396 : const_tree tf;
7034 :
7035 3943396 : tf = TREE_TYPE (f);
7036 3943396 : return (DECL_CHAIN (f) == NULL
7037 1082244 : && TREE_CODE (tf) == ARRAY_TYPE
7038 2525 : && TYPE_DOMAIN (tf)
7039 2525 : && TYPE_MIN_VALUE (TYPE_DOMAIN (tf))
7040 2525 : && integer_zerop (TYPE_MIN_VALUE (TYPE_DOMAIN (tf)))
7041 2525 : && !TYPE_MAX_VALUE (TYPE_DOMAIN (tf))
7042 3943435 : && int_size_in_bytes (type) >= 0);
7043 : }
7044 :
7045 : /* If FOR_CTOR_P, return the number of top-level elements that a constructor
7046 : must have in order for it to completely initialize a value of type TYPE.
7047 : Return -1 if the number isn't known.
7048 :
7049 : If !FOR_CTOR_P, return an estimate of the number of scalars in TYPE. */
7050 :
7051 : static HOST_WIDE_INT
7052 3727980 : count_type_elements (const_tree type, bool for_ctor_p)
7053 : {
7054 3729320 : switch (TREE_CODE (type))
7055 : {
7056 155455 : case ARRAY_TYPE:
7057 155455 : {
7058 155455 : tree nelts_minus_one;
7059 :
7060 155455 : nelts_minus_one = array_type_nelts_minus_one (type);
7061 155455 : if (nelts_minus_one && tree_fits_uhwi_p (nelts_minus_one))
7062 : {
7063 155444 : unsigned HOST_WIDE_INT n;
7064 :
7065 155444 : n = tree_to_uhwi (nelts_minus_one) + 1;
7066 155444 : if (n == 0 || for_ctor_p)
7067 154104 : return n;
7068 : else
7069 1340 : return n * count_type_elements (TREE_TYPE (type), false);
7070 : }
7071 11 : return for_ctor_p ? -1 : 1;
7072 : }
7073 :
7074 1822868 : case RECORD_TYPE:
7075 1822868 : {
7076 1822868 : unsigned HOST_WIDE_INT n;
7077 1822868 : tree f;
7078 :
7079 1822868 : n = 0;
7080 17319852 : for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
7081 15496984 : if (TREE_CODE (f) == FIELD_DECL)
7082 : {
7083 4407404 : if (!for_ctor_p)
7084 464008 : n += count_type_elements (TREE_TYPE (f), false);
7085 3943396 : else if (!flexible_array_member_p (f, type))
7086 : /* Don't count flexible arrays, which are not supposed
7087 : to be initialized. */
7088 3943357 : n += 1;
7089 : }
7090 :
7091 1822868 : return n;
7092 : }
7093 :
7094 3356 : case UNION_TYPE:
7095 3356 : case QUAL_UNION_TYPE:
7096 3356 : {
7097 3356 : tree f;
7098 3356 : HOST_WIDE_INT n, m;
7099 :
7100 3356 : gcc_assert (!for_ctor_p);
7101 : /* Estimate the number of scalars in each field and pick the
7102 : maximum. Other estimates would do instead; the idea is simply
7103 : to make sure that the estimate is not sensitive to the ordering
7104 : of the fields. */
7105 3356 : n = 1;
7106 76281 : for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
7107 72925 : if (TREE_CODE (f) == FIELD_DECL)
7108 : {
7109 68442 : m = count_type_elements (TREE_TYPE (f), false);
7110 : /* If the field doesn't span the whole union, add an extra
7111 : scalar for the rest. */
7112 68442 : if (simple_cst_equal (TYPE_SIZE (TREE_TYPE (f)),
7113 68442 : TYPE_SIZE (type)) != 1)
7114 48780 : m++;
7115 68442 : if (n < m)
7116 72925 : n = m;
7117 : }
7118 : return n;
7119 : }
7120 :
7121 : case COMPLEX_TYPE:
7122 : return 2;
7123 :
7124 421 : case VECTOR_TYPE:
7125 421 : {
7126 421 : unsigned HOST_WIDE_INT nelts;
7127 421 : if (TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts))
7128 421 : return nelts;
7129 : else
7130 : return -1;
7131 : }
7132 :
7133 : case INTEGER_TYPE:
7134 : case REAL_TYPE:
7135 : case FIXED_POINT_TYPE:
7136 : case ENUMERAL_TYPE:
7137 : case BOOLEAN_TYPE:
7138 : case POINTER_TYPE:
7139 : case OFFSET_TYPE:
7140 : case REFERENCE_TYPE:
7141 : case NULLPTR_TYPE:
7142 : case OPAQUE_TYPE:
7143 : case BITINT_TYPE:
7144 : return 1;
7145 :
7146 0 : case ERROR_MARK:
7147 0 : return 0;
7148 :
7149 0 : case VOID_TYPE:
7150 0 : case METHOD_TYPE:
7151 0 : case FUNCTION_TYPE:
7152 0 : case LANG_TYPE:
7153 0 : default:
7154 0 : gcc_unreachable ();
7155 : }
7156 : }
7157 :
7158 : /* Helper for categorize_ctor_elements. Identical interface. */
7159 :
7160 : static bool
7161 1550016 : categorize_ctor_elements_1 (const_tree ctor, HOST_WIDE_INT *p_nz_elts,
7162 : HOST_WIDE_INT *p_unique_nz_elts,
7163 : HOST_WIDE_INT *p_init_elts,
7164 : ctor_completeness *p_complete)
7165 : {
7166 1550016 : unsigned HOST_WIDE_INT idx;
7167 1550016 : HOST_WIDE_INT nz_elts, unique_nz_elts, init_elts, num_fields;
7168 1550016 : tree value, purpose, elt_type;
7169 :
7170 : /* Whether CTOR is a valid constant initializer, in accordance with what
7171 : initializer_constant_valid_p does. If inferred from the constructor
7172 : elements, true until proven otherwise. */
7173 1550016 : bool const_from_elts_p = constructor_static_from_elts_p (ctor);
7174 1550016 : bool const_p = const_from_elts_p ? true : TREE_STATIC (ctor);
7175 :
7176 1550016 : nz_elts = 0;
7177 1550016 : unique_nz_elts = 0;
7178 1550016 : init_elts = 0;
7179 1550016 : num_fields = 0;
7180 1550016 : elt_type = NULL_TREE;
7181 :
7182 5877670 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (ctor), idx, purpose, value)
7183 : {
7184 4327654 : HOST_WIDE_INT mult = 1;
7185 :
7186 4327654 : if (purpose && TREE_CODE (purpose) == RANGE_EXPR)
7187 : {
7188 821 : tree lo_index = TREE_OPERAND (purpose, 0);
7189 821 : tree hi_index = TREE_OPERAND (purpose, 1);
7190 :
7191 821 : if (tree_fits_uhwi_p (lo_index) && tree_fits_uhwi_p (hi_index))
7192 821 : mult = (tree_to_uhwi (hi_index)
7193 821 : - tree_to_uhwi (lo_index) + 1);
7194 : }
7195 4327654 : num_fields += mult;
7196 4327654 : elt_type = TREE_TYPE (value);
7197 :
7198 4327654 : switch (TREE_CODE (value))
7199 : {
7200 546198 : case CONSTRUCTOR:
7201 546198 : {
7202 546198 : HOST_WIDE_INT nz = 0, unz = 0, ic = 0;
7203 :
7204 546198 : bool const_elt_p = categorize_ctor_elements_1 (value, &nz, &unz,
7205 : &ic, p_complete);
7206 :
7207 546198 : nz_elts += mult * nz;
7208 546198 : unique_nz_elts += unz;
7209 546198 : init_elts += mult * ic;
7210 :
7211 546198 : if (const_from_elts_p && const_p)
7212 333038 : const_p = const_elt_p;
7213 : }
7214 546198 : break;
7215 :
7216 2311554 : case INTEGER_CST:
7217 2311554 : case REAL_CST:
7218 2311554 : case FIXED_CST:
7219 2311554 : if (!initializer_zerop (value))
7220 : {
7221 1691160 : nz_elts += mult;
7222 1691160 : unique_nz_elts++;
7223 : }
7224 2311554 : init_elts += mult;
7225 2311554 : break;
7226 :
7227 6298 : case STRING_CST:
7228 6298 : nz_elts += mult * TREE_STRING_LENGTH (value);
7229 6298 : unique_nz_elts += TREE_STRING_LENGTH (value);
7230 6298 : init_elts += mult * TREE_STRING_LENGTH (value);
7231 6298 : break;
7232 :
7233 166 : case RAW_DATA_CST:
7234 166 : nz_elts += mult * RAW_DATA_LENGTH (value);
7235 166 : unique_nz_elts += RAW_DATA_LENGTH (value);
7236 166 : init_elts += mult * RAW_DATA_LENGTH (value);
7237 166 : num_fields += mult * (RAW_DATA_LENGTH (value) - 1);
7238 166 : break;
7239 :
7240 3234 : case COMPLEX_CST:
7241 3234 : if (!initializer_zerop (TREE_REALPART (value)))
7242 : {
7243 2630 : nz_elts += mult;
7244 2630 : unique_nz_elts++;
7245 : }
7246 3234 : if (!initializer_zerop (TREE_IMAGPART (value)))
7247 : {
7248 2272 : nz_elts += mult;
7249 2272 : unique_nz_elts++;
7250 : }
7251 3234 : init_elts += 2 * mult;
7252 3234 : break;
7253 :
7254 6420 : case VECTOR_CST:
7255 6420 : {
7256 6420 : unsigned int nunits
7257 : = constant_lower_bound
7258 6420 : (TYPE_VECTOR_SUBPARTS (TREE_TYPE (value)));
7259 39931 : for (unsigned int i = 0; i < nunits; ++i)
7260 : {
7261 33511 : tree v = VECTOR_CST_ELT (value, i);
7262 33511 : if (!initializer_zerop (v))
7263 : {
7264 19057 : nz_elts += mult;
7265 19057 : unique_nz_elts++;
7266 : }
7267 33511 : init_elts += mult;
7268 : }
7269 : }
7270 : break;
7271 :
7272 1453784 : default:
7273 1453784 : {
7274 1453784 : HOST_WIDE_INT tc = count_type_elements (elt_type, false);
7275 1453784 : nz_elts += mult * tc;
7276 1453784 : unique_nz_elts += tc;
7277 1453784 : init_elts += mult * tc;
7278 :
7279 1453784 : if (const_from_elts_p && const_p)
7280 599892 : const_p
7281 599892 : = initializer_constant_valid_p (value,
7282 : elt_type,
7283 599892 : TYPE_REVERSE_STORAGE_ORDER
7284 : (TREE_TYPE (ctor)))
7285 : != NULL_TREE;
7286 : }
7287 : break;
7288 : }
7289 : }
7290 :
7291 1550016 : if (!p_complete->sparse && !complete_ctor_at_level_p (TREE_TYPE (ctor),
7292 : num_fields, elt_type))
7293 186651 : p_complete->sparse = true;
7294 1363365 : else if (TREE_CODE (TREE_TYPE (ctor)) == UNION_TYPE
7295 1363365 : || TREE_CODE (TREE_TYPE (ctor)) == QUAL_UNION_TYPE)
7296 : {
7297 12112 : if (!p_complete->sparse
7298 7877 : && CONSTRUCTOR_ZERO_PADDING_BITS (ctor)
7299 22193 : && (num_fields
7300 5042 : ? simple_cst_equal (TYPE_SIZE (TREE_TYPE (ctor)),
7301 5039 : TYPE_SIZE (elt_type)) != 1
7302 3 : : type_has_padding_at_level_p (TREE_TYPE (ctor))))
7303 3367 : p_complete->sparse = true;
7304 4510 : else if (!p_complete->sparse && !p_complete->padded_union
7305 17757 : && (num_fields
7306 4506 : ? simple_cst_equal (TYPE_SIZE (TREE_TYPE (ctor)),
7307 4506 : TYPE_SIZE (elt_type)) != 1
7308 0 : : type_has_padding_at_level_p (TREE_TYPE (ctor))))
7309 380 : p_complete->padded_union = true;
7310 : }
7311 1351253 : else if (!p_complete->sparse
7312 1233435 : && (CONSTRUCTOR_ZERO_PADDING_BITS (ctor)
7313 1214484 : || flag_zero_init_padding_bits == ZERO_INIT_PADDING_BITS_ALL)
7314 1370233 : && type_has_padding_at_level_p (TREE_TYPE (ctor)))
7315 2200 : p_complete->sparse = true;
7316 1231235 : else if (!p_complete->sparse && !p_complete->padded_non_union
7317 2532554 : && type_has_padding_at_level_p (TREE_TYPE (ctor)))
7318 15460 : p_complete->padded_non_union = true;
7319 :
7320 1550016 : *p_nz_elts += nz_elts;
7321 1550016 : *p_unique_nz_elts += unique_nz_elts;
7322 1550016 : *p_init_elts += init_elts;
7323 :
7324 1550016 : return const_p;
7325 : }
7326 :
7327 : /* Examine CTOR to discover:
7328 : * how many scalar fields are set to nonzero values,
7329 : and place it in *P_NZ_ELTS;
7330 : * the same, but counting RANGE_EXPRs as multiplier of 1 instead of
7331 : high - low + 1 (this can be useful for callers to determine ctors
7332 : that could be cheaply initialized with - perhaps nested - loops
7333 : compared to copied from huge read-only data),
7334 : and place it in *P_UNIQUE_NZ_ELTS;
7335 : * how many scalar fields in total are in CTOR,
7336 : and place it in *P_ELT_COUNT.
7337 : * whether the constructor is complete -- in the sense that every
7338 : meaningful byte is explicitly given a value --
7339 : and place it in *P_COMPLETE:
7340 :
7341 : Return whether or not CTOR is a valid static constant initializer, the same
7342 : as "initializer_constant_valid_p (CTOR, TREE_TYPE (CTOR)) != 0". */
7343 :
7344 : bool
7345 1003818 : categorize_ctor_elements (const_tree ctor, HOST_WIDE_INT *p_nz_elts,
7346 : HOST_WIDE_INT *p_unique_nz_elts,
7347 : HOST_WIDE_INT *p_init_elts,
7348 : ctor_completeness *p_complete)
7349 : {
7350 1003818 : *p_nz_elts = 0;
7351 1003818 : *p_unique_nz_elts = 0;
7352 1003818 : *p_init_elts = 0;
7353 1003818 : *p_complete = {};
7354 :
7355 1003818 : return categorize_ctor_elements_1 (ctor, p_nz_elts, p_unique_nz_elts,
7356 1003818 : p_init_elts, p_complete);
7357 : }
7358 :
7359 : /* Return true if constructor CTOR is simple enough to be materialized
7360 : in an integer mode register. Limit the size to WORDS words, which
7361 : is 1 by default. */
7362 :
7363 : bool
7364 23879 : immediate_const_ctor_p (const_tree ctor, unsigned int words)
7365 : {
7366 : /* Allow function to be called with a VAR_DECL's DECL_INITIAL. */
7367 23879 : if (!ctor || TREE_CODE (ctor) != CONSTRUCTOR)
7368 : return false;
7369 :
7370 2820 : return TREE_CONSTANT (ctor)
7371 2820 : && !TREE_ADDRESSABLE (ctor)
7372 2714 : && CONSTRUCTOR_NELTS (ctor)
7373 2688 : && TREE_CODE (TREE_TYPE (ctor)) != ARRAY_TYPE
7374 633 : && int_expr_size (ctor) <= words * UNITS_PER_WORD
7375 3009 : && initializer_constant_valid_for_bitfield_p (ctor);
7376 : }
7377 :
7378 : /* TYPE is initialized by a constructor with NUM_ELTS elements, the last
7379 : of which had type LAST_TYPE. Each element was itself a complete
7380 : initializer, in the sense that every meaningful byte was explicitly
7381 : given a value. Return true if the same is true for the constructor
7382 : as a whole. */
7383 :
7384 : bool
7385 1750843 : complete_ctor_at_level_p (const_tree type, HOST_WIDE_INT num_elts,
7386 : const_tree last_type)
7387 : {
7388 1750843 : if (TREE_CODE (type) == UNION_TYPE || TREE_CODE (type) == QUAL_UNION_TYPE)
7389 : {
7390 9097 : if (num_elts == 0)
7391 : {
7392 24 : if (flag_zero_init_padding_bits >= ZERO_INIT_PADDING_BITS_UNIONS)
7393 : return false;
7394 :
7395 : /* If the CONSTRUCTOR doesn't have any elts, it is
7396 : incomplete if the union has at least one field. */
7397 327 : for (tree f = TYPE_FIELDS (type); f; f = DECL_CHAIN (f))
7398 324 : if (TREE_CODE (f) == FIELD_DECL)
7399 : return false;
7400 :
7401 : return true;
7402 : }
7403 :
7404 9073 : gcc_assert (num_elts == 1 && last_type);
7405 :
7406 9073 : if (flag_zero_init_padding_bits >= ZERO_INIT_PADDING_BITS_UNIONS)
7407 : /* ??? We could look at each element of the union, and find the
7408 : largest element. Which would avoid comparing the size of the
7409 : initialized element against any tail padding in the union.
7410 : Doesn't seem worth the effort... */
7411 102 : return simple_cst_equal (TYPE_SIZE (type), TYPE_SIZE (last_type)) == 1;
7412 :
7413 : return true;
7414 : }
7415 :
7416 1741746 : return count_type_elements (type, true) == num_elts;
7417 : }
7418 :
7419 : /* Return true if EXP contains mostly (3/4) zeros. */
7420 :
7421 : static bool
7422 367234 : mostly_zeros_p (const_tree exp)
7423 : {
7424 367234 : if (TREE_CODE (exp) == CONSTRUCTOR)
7425 : {
7426 197 : HOST_WIDE_INT nz_elts, unz_elts, init_elts;
7427 197 : ctor_completeness complete_p;
7428 :
7429 197 : categorize_ctor_elements (exp, &nz_elts, &unz_elts, &init_elts,
7430 : &complete_p);
7431 197 : return complete_p.sparse || nz_elts < init_elts / 4;
7432 : }
7433 :
7434 367037 : return initializer_zerop (exp);
7435 : }
7436 :
7437 : /* Return true if EXP contains all zeros. */
7438 :
7439 : static bool
7440 1108 : all_zeros_p (const_tree exp)
7441 : {
7442 1108 : if (TREE_CODE (exp) == CONSTRUCTOR)
7443 : {
7444 1108 : HOST_WIDE_INT nz_elts, unz_elts, init_elts;
7445 1108 : ctor_completeness complete_p;
7446 :
7447 1108 : categorize_ctor_elements (exp, &nz_elts, &unz_elts, &init_elts,
7448 : &complete_p);
7449 1108 : return nz_elts == 0;
7450 : }
7451 :
7452 0 : return initializer_zerop (exp);
7453 : }
7454 :
7455 : /* Helper function for store_constructor.
7456 : TARGET, BITSIZE, BITPOS, MODE, EXP are as for store_field.
7457 : CLEARED is as for store_constructor.
7458 : ALIAS_SET is the alias set to use for any stores.
7459 : If REVERSE is true, the store is to be done in reverse order.
7460 :
7461 : This provides a recursive shortcut back to store_constructor when it isn't
7462 : necessary to go through store_field. This is so that we can pass through
7463 : the cleared field to let store_constructor know that we may not have to
7464 : clear a substructure if the outer structure has already been cleared. */
7465 :
7466 : static void
7467 36934 : store_constructor_field (rtx target, poly_uint64 bitsize, poly_int64 bitpos,
7468 : poly_uint64 bitregion_start,
7469 : poly_uint64 bitregion_end,
7470 : machine_mode mode,
7471 : tree exp, int cleared,
7472 : alias_set_type alias_set, bool reverse)
7473 : {
7474 36934 : poly_int64 bytepos;
7475 36934 : poly_uint64 bytesize;
7476 36934 : if (TREE_CODE (exp) == CONSTRUCTOR
7477 : /* We can only call store_constructor recursively if the size and
7478 : bit position are on a byte boundary. */
7479 36934 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
7480 82 : && maybe_ne (bitsize, 0U)
7481 36852 : && multiple_p (bitsize, BITS_PER_UNIT, &bytesize)
7482 : /* If we have a nonzero bitpos for a register target, then we just
7483 : let store_field do the bitfield handling. This is unlikely to
7484 : generate unnecessary clear instructions anyways. */
7485 37016 : && (known_eq (bitpos, 0) || MEM_P (target)))
7486 : {
7487 82 : if (MEM_P (target))
7488 : {
7489 40 : machine_mode target_mode = GET_MODE (target);
7490 40 : if (target_mode != BLKmode
7491 40 : && !multiple_p (bitpos, GET_MODE_ALIGNMENT (target_mode)))
7492 : target_mode = BLKmode;
7493 40 : target = adjust_address (target, target_mode, bytepos);
7494 : }
7495 :
7496 :
7497 : /* Update the alias set, if required. */
7498 40 : if (MEM_P (target) && ! MEM_KEEP_ALIAS_SET_P (target)
7499 122 : && MEM_ALIAS_SET (target) != 0)
7500 : {
7501 36 : target = copy_rtx (target);
7502 36 : set_mem_alias_set (target, alias_set);
7503 : }
7504 :
7505 82 : store_constructor (exp, target, cleared, bytesize, reverse);
7506 : }
7507 : else
7508 36852 : store_field (target, bitsize, bitpos, bitregion_start, bitregion_end, mode,
7509 : exp, alias_set, false, reverse);
7510 36934 : }
7511 :
7512 :
7513 : /* Returns the number of FIELD_DECLs in TYPE. */
7514 :
7515 : static int
7516 68761 : fields_length (const_tree type)
7517 : {
7518 68761 : tree t = TYPE_FIELDS (type);
7519 68761 : int count = 0;
7520 :
7521 902195 : for (; t; t = DECL_CHAIN (t))
7522 833434 : if (TREE_CODE (t) == FIELD_DECL)
7523 299806 : ++count;
7524 :
7525 68761 : return count;
7526 : }
7527 :
7528 : /* Store the value of constructor EXP into the rtx TARGET.
7529 : TARGET is either a REG or a MEM; we know it cannot conflict, since
7530 : safe_from_p has been called.
7531 : CLEARED is true if TARGET is known to have been zero'd.
7532 : If the constructor EXP has a vector type then elements of TARGET for which
7533 : there is no corresponding element in EXP are zero'd. For a variable-length
7534 : vector type, only elements up to the minimum number of subparts of the type
7535 : are explicitly zero'd; any elements beyond that are implicitly zero.
7536 : SIZE is the number of bytes of TARGET we are allowed to modify: this
7537 : may not be the same as the size of EXP if we are assigning to a field
7538 : which has been packed to exclude padding bits.
7539 : If REVERSE is true, the store is to be done in reverse order. */
7540 :
7541 : void
7542 248692 : store_constructor (tree exp, rtx target, int cleared, poly_int64 size,
7543 : bool reverse)
7544 : {
7545 248692 : tree type = TREE_TYPE (exp);
7546 248692 : HOST_WIDE_INT exp_size = int_size_in_bytes (type);
7547 248692 : poly_int64 bitregion_end = known_gt (size, 0) ? size * BITS_PER_UNIT - 1 : 0;
7548 :
7549 248692 : switch (TREE_CODE (type))
7550 : {
7551 69250 : case RECORD_TYPE:
7552 69250 : case UNION_TYPE:
7553 69250 : case QUAL_UNION_TYPE:
7554 69250 : {
7555 69250 : unsigned HOST_WIDE_INT idx;
7556 69250 : tree field, value;
7557 :
7558 : /* The storage order is specified for every aggregate type. */
7559 69250 : reverse = TYPE_REVERSE_STORAGE_ORDER (type);
7560 :
7561 : /* If size is zero or the target is already cleared, do nothing. */
7562 69250 : if (known_eq (size, 0) || cleared)
7563 : cleared = 1;
7564 : /* We either clear the aggregate or indicate the value is dead. */
7565 69250 : else if ((TREE_CODE (type) == UNION_TYPE
7566 69250 : || TREE_CODE (type) == QUAL_UNION_TYPE)
7567 69554 : && ! CONSTRUCTOR_ELTS (exp))
7568 : /* If the constructor is empty, clear the union. */
7569 : {
7570 255 : clear_storage (target, expr_size (exp), BLOCK_OP_NORMAL);
7571 255 : cleared = 1;
7572 : }
7573 :
7574 : /* If we are building a static constructor into a register,
7575 : set the initial value as zero so we can fold the value into
7576 : a constant. But if more than one register is involved,
7577 : this probably loses. */
7578 4248 : else if (REG_P (target) && TREE_STATIC (exp)
7579 69585 : && known_le (GET_MODE_SIZE (GET_MODE (target)),
7580 : REGMODE_NATURAL_SIZE (GET_MODE (target))))
7581 : {
7582 234 : emit_move_insn (target, CONST0_RTX (GET_MODE (target)));
7583 234 : cleared = 1;
7584 : }
7585 :
7586 : /* If the constructor has fewer fields than the structure or
7587 : if we are initializing the structure to mostly zeros, clear
7588 : the whole structure first. Don't do this if TARGET is a
7589 : register whose mode size isn't equal to SIZE since
7590 : clear_storage can't handle this case. */
7591 68761 : else if (known_size_p (size)
7592 68930 : && (((int) CONSTRUCTOR_NELTS (exp) != fields_length (type))
7593 156 : || mostly_zeros_p (exp))
7594 137370 : && (!REG_P (target)
7595 8018 : || known_eq (GET_MODE_SIZE (GET_MODE (target)), size)))
7596 : {
7597 79322 : clear_storage (target, gen_int_mode (size, Pmode),
7598 : BLOCK_OP_NORMAL);
7599 68609 : cleared = 1;
7600 : }
7601 :
7602 69250 : if (REG_P (target) && !cleared)
7603 5 : emit_clobber (target);
7604 :
7605 : /* Store each element of the constructor into the
7606 : corresponding field of TARGET. */
7607 69596 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (exp), idx, field, value)
7608 : {
7609 346 : machine_mode mode;
7610 346 : HOST_WIDE_INT bitsize;
7611 346 : HOST_WIDE_INT bitpos = 0;
7612 346 : tree offset;
7613 346 : rtx to_rtx = target;
7614 :
7615 : /* Just ignore missing fields. We cleared the whole
7616 : structure, above, if any fields are missing. */
7617 346 : if (field == 0)
7618 346 : continue;
7619 :
7620 346 : if (cleared && initializer_zerop (value))
7621 17 : continue;
7622 :
7623 : /* Variable sized arrays are ignored. */
7624 329 : tree decl_size = DECL_SIZE (field);
7625 329 : if (!decl_size)
7626 4 : continue;
7627 :
7628 325 : if (tree_fits_uhwi_p (decl_size))
7629 325 : bitsize = tree_to_uhwi (decl_size);
7630 : else
7631 0 : gcc_unreachable ();
7632 :
7633 325 : mode = DECL_MODE (field);
7634 325 : if (DECL_BIT_FIELD (field))
7635 24 : mode = VOIDmode;
7636 :
7637 325 : offset = DECL_FIELD_OFFSET (field);
7638 325 : if (tree_fits_shwi_p (offset)
7639 325 : && tree_fits_shwi_p (bit_position (field)))
7640 : {
7641 325 : bitpos = int_bit_position (field);
7642 325 : offset = NULL_TREE;
7643 : }
7644 : else
7645 0 : gcc_unreachable ();
7646 :
7647 : /* If this initializes a field that is smaller than a
7648 : word, at the start of a word, try to widen it to a full
7649 : word. This special case allows us to output C++ member
7650 : function initializations in a form that the optimizers
7651 : can understand. */
7652 325 : if (WORD_REGISTER_OPERATIONS
7653 : && REG_P (target)
7654 : && bitsize < BITS_PER_WORD
7655 : && bitpos % BITS_PER_WORD == 0
7656 : && GET_MODE_CLASS (mode) == MODE_INT
7657 : && TREE_CODE (value) == INTEGER_CST
7658 : && exp_size >= 0
7659 : && bitpos + BITS_PER_WORD <= exp_size * BITS_PER_UNIT)
7660 : {
7661 : type = TREE_TYPE (value);
7662 :
7663 : if (TYPE_PRECISION (type) < BITS_PER_WORD)
7664 : {
7665 : type = lang_hooks.types.type_for_mode
7666 : (word_mode, TYPE_UNSIGNED (type));
7667 : value = fold_convert (type, value);
7668 : /* Make sure the bits beyond the original bitsize are zero
7669 : so that we can correctly avoid extra zeroing stores in
7670 : later constructor elements. */
7671 : tree bitsize_mask
7672 : = wide_int_to_tree (type, wi::mask (bitsize, false,
7673 : BITS_PER_WORD));
7674 : value = fold_build2 (BIT_AND_EXPR, type, value, bitsize_mask);
7675 : }
7676 :
7677 : if (BYTES_BIG_ENDIAN)
7678 : value
7679 : = fold_build2 (LSHIFT_EXPR, type, value,
7680 : build_int_cst (type,
7681 : BITS_PER_WORD - bitsize));
7682 : bitsize = BITS_PER_WORD;
7683 : mode = word_mode;
7684 : }
7685 :
7686 207 : if (MEM_P (to_rtx) && !MEM_KEEP_ALIAS_SET_P (to_rtx)
7687 532 : && DECL_NONADDRESSABLE_P (field))
7688 : {
7689 0 : to_rtx = copy_rtx (to_rtx);
7690 0 : MEM_KEEP_ALIAS_SET_P (to_rtx) = 1;
7691 : }
7692 :
7693 325 : store_constructor_field (to_rtx, bitsize, bitpos,
7694 325 : 0, bitregion_end, mode,
7695 : value, cleared,
7696 325 : get_alias_set (TREE_TYPE (field)),
7697 : reverse);
7698 : }
7699 : break;
7700 : }
7701 39043 : case ARRAY_TYPE:
7702 39043 : {
7703 39043 : tree value, index;
7704 39043 : unsigned HOST_WIDE_INT i, j = 0;
7705 39043 : bool need_to_clear;
7706 39043 : tree domain;
7707 39043 : tree elttype = TREE_TYPE (type);
7708 39043 : bool const_bounds_p;
7709 39043 : unsigned HOST_WIDE_INT minelt = 0;
7710 39043 : unsigned HOST_WIDE_INT maxelt = 0;
7711 :
7712 : /* The storage order is specified for every aggregate type. */
7713 39043 : reverse = TYPE_REVERSE_STORAGE_ORDER (type);
7714 :
7715 39043 : domain = TYPE_DOMAIN (type);
7716 39043 : const_bounds_p = (TYPE_MIN_VALUE (domain)
7717 39043 : && TYPE_MAX_VALUE (domain)
7718 39043 : && tree_fits_uhwi_p (TYPE_MIN_VALUE (domain))
7719 78086 : && tree_fits_uhwi_p (TYPE_MAX_VALUE (domain)));
7720 :
7721 : /* If we have constant bounds for the range of the type, get them. */
7722 39043 : if (const_bounds_p)
7723 : {
7724 39043 : minelt = tree_to_uhwi (TYPE_MIN_VALUE (domain));
7725 39043 : maxelt = tree_to_uhwi (TYPE_MAX_VALUE (domain));
7726 : }
7727 :
7728 : /* If the constructor has fewer elements than the array, clear
7729 : the whole array first. Similarly if this is static
7730 : constructor of a non-BLKmode object. */
7731 39043 : if (cleared)
7732 : need_to_clear = false;
7733 38997 : else if (REG_P (target) && TREE_STATIC (exp))
7734 : need_to_clear = true;
7735 : else
7736 : {
7737 38890 : unsigned HOST_WIDE_INT idx;
7738 38890 : unsigned HOST_WIDE_INT count = 0, zero_count = 0;
7739 38890 : need_to_clear = ! const_bounds_p;
7740 :
7741 : /* This loop is a more accurate version of the loop in
7742 : mostly_zeros_p (it handles RANGE_EXPR in an index). It
7743 : is also needed to check for missing elements. */
7744 38898 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (exp), idx, index, value)
7745 : {
7746 8 : unsigned HOST_WIDE_INT this_node_count;
7747 :
7748 8 : if (need_to_clear)
7749 : break;
7750 :
7751 8 : if (index != NULL_TREE && TREE_CODE (index) == RANGE_EXPR)
7752 : {
7753 0 : tree lo_index = TREE_OPERAND (index, 0);
7754 0 : tree hi_index = TREE_OPERAND (index, 1);
7755 :
7756 0 : if (! tree_fits_uhwi_p (lo_index)
7757 0 : || ! tree_fits_uhwi_p (hi_index))
7758 : {
7759 : need_to_clear = true;
7760 : break;
7761 : }
7762 :
7763 0 : this_node_count = (tree_to_uhwi (hi_index)
7764 0 : - tree_to_uhwi (lo_index) + 1);
7765 0 : }
7766 8 : else if (TREE_CODE (value) == RAW_DATA_CST)
7767 0 : this_node_count = RAW_DATA_LENGTH (value);
7768 : else
7769 : this_node_count = 1;
7770 :
7771 8 : count += this_node_count;
7772 8 : if (mostly_zeros_p (value))
7773 0 : zero_count += this_node_count;
7774 : }
7775 :
7776 : /* Clear the entire array first if there are any missing
7777 : elements, or if the incidence of zero elements is >=
7778 : 75%. */
7779 38890 : if (! need_to_clear
7780 38890 : && (count < maxelt - minelt + 1
7781 1 : || 4 * zero_count >= 3 * count))
7782 : need_to_clear = true;
7783 : }
7784 :
7785 38996 : if (need_to_clear && maybe_gt (size, 0))
7786 : {
7787 38996 : if (REG_P (target))
7788 1639 : emit_move_insn (target, CONST0_RTX (GET_MODE (target)));
7789 : else
7790 42121 : clear_storage (target, gen_int_mode (size, Pmode),
7791 : BLOCK_OP_NORMAL);
7792 : cleared = 1;
7793 : }
7794 :
7795 39043 : if (!cleared && REG_P (target))
7796 : /* Inform later passes that the old value is dead. */
7797 0 : emit_clobber (target);
7798 :
7799 : /* Store each element of the constructor into the
7800 : corresponding element of TARGET, determined by counting the
7801 : elements. */
7802 40159 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (exp), i, index, value)
7803 : {
7804 1116 : machine_mode mode;
7805 1116 : poly_int64 bitsize;
7806 1116 : HOST_WIDE_INT bitpos;
7807 1116 : rtx xtarget = target;
7808 :
7809 1116 : if (cleared && initializer_zerop (value))
7810 : {
7811 672 : if (TREE_CODE (value) == RAW_DATA_CST)
7812 0 : j += RAW_DATA_LENGTH (value) - 1;
7813 674 : continue;
7814 : }
7815 :
7816 444 : mode = TYPE_MODE (elttype);
7817 444 : if (mode != BLKmode)
7818 888 : bitsize = GET_MODE_BITSIZE (mode);
7819 0 : else if (!poly_int_tree_p (TYPE_SIZE (elttype), &bitsize))
7820 0 : bitsize = -1;
7821 :
7822 444 : if (index != NULL_TREE && TREE_CODE (index) == RANGE_EXPR)
7823 : {
7824 0 : tree lo_index = TREE_OPERAND (index, 0);
7825 0 : tree hi_index = TREE_OPERAND (index, 1);
7826 0 : rtx index_r;
7827 0 : unsigned HOST_WIDE_INT lo, hi, count;
7828 0 : tree offset;
7829 :
7830 0 : gcc_assert (TREE_CODE (value) != RAW_DATA_CST);
7831 :
7832 : /* If the range is constant and "small", unroll the loop. */
7833 0 : if (const_bounds_p
7834 0 : && tree_fits_uhwi_p (lo_index)
7835 0 : && tree_fits_uhwi_p (hi_index)
7836 0 : && (lo = tree_to_uhwi (lo_index),
7837 0 : hi = tree_to_uhwi (hi_index),
7838 0 : count = hi - lo + 1,
7839 0 : (!MEM_P (target)
7840 0 : || count <= 2
7841 0 : || (tree_fits_uhwi_p (TYPE_SIZE (elttype))
7842 0 : && (tree_to_uhwi (TYPE_SIZE (elttype)) * count
7843 : <= 40 * 8)))))
7844 : {
7845 0 : lo -= minelt; hi -= minelt;
7846 0 : for (; lo <= hi; lo++)
7847 : {
7848 0 : bitpos = lo * tree_to_uhwi (TYPE_SIZE (elttype));
7849 :
7850 0 : if (MEM_P (target)
7851 0 : && !MEM_KEEP_ALIAS_SET_P (target)
7852 0 : && TREE_CODE (type) == ARRAY_TYPE
7853 0 : && TYPE_NONALIASED_COMPONENT (type))
7854 : {
7855 0 : target = copy_rtx (target);
7856 0 : MEM_KEEP_ALIAS_SET_P (target) = 1;
7857 : }
7858 :
7859 0 : store_constructor_field
7860 0 : (target, bitsize, bitpos, 0, bitregion_end,
7861 : mode, value, cleared,
7862 : get_alias_set (elttype), reverse);
7863 : }
7864 : }
7865 : else
7866 : {
7867 0 : rtx_code_label *loop_start = gen_label_rtx ();
7868 0 : rtx_code_label *loop_end = gen_label_rtx ();
7869 0 : tree exit_cond;
7870 :
7871 0 : expand_normal (hi_index);
7872 :
7873 0 : index = build_decl (EXPR_LOCATION (exp),
7874 : VAR_DECL, NULL_TREE, domain);
7875 0 : index_r = gen_reg_rtx (promote_decl_mode (index, NULL));
7876 0 : SET_DECL_RTL (index, index_r);
7877 0 : store_expr (lo_index, index_r, 0, false, reverse);
7878 :
7879 : /* Build the head of the loop. */
7880 0 : do_pending_stack_adjust ();
7881 0 : emit_label (loop_start);
7882 :
7883 : /* Assign value to element index. */
7884 0 : offset = fold_build2 (MINUS_EXPR,
7885 : TREE_TYPE (index),
7886 : index,
7887 : TYPE_MIN_VALUE (domain));
7888 :
7889 0 : offset = size_binop (MULT_EXPR,
7890 : fold_convert (sizetype, offset),
7891 : TYPE_SIZE_UNIT (elttype));
7892 :
7893 0 : xtarget = offset_address (target,
7894 : expand_normal (offset),
7895 : highest_pow2_factor (offset));
7896 0 : xtarget = adjust_address (xtarget, mode, 0);
7897 0 : if (TREE_CODE (value) == CONSTRUCTOR)
7898 0 : store_constructor (value, xtarget, cleared,
7899 : exact_div (bitsize, BITS_PER_UNIT),
7900 : reverse);
7901 : else
7902 0 : store_expr (value, xtarget, 0, false, reverse);
7903 :
7904 : /* Generate a conditional jump to exit the loop. */
7905 0 : exit_cond = build2 (LT_EXPR, integer_type_node,
7906 : index, hi_index);
7907 0 : jumpif (exit_cond, loop_end,
7908 : profile_probability::uninitialized ());
7909 :
7910 : /* Update the loop counter, and jump to the head of
7911 : the loop. */
7912 0 : expand_assignment (index,
7913 0 : build2 (PLUS_EXPR, TREE_TYPE (index),
7914 : index, integer_one_node),
7915 : false);
7916 :
7917 0 : emit_jump (loop_start);
7918 :
7919 : /* Build the end of the loop. */
7920 0 : emit_label (loop_end);
7921 : }
7922 : }
7923 444 : else if ((index && !tree_fits_uhwi_p (index))
7924 444 : || !tree_fits_uhwi_p (TYPE_SIZE (elttype)))
7925 : {
7926 0 : tree offset;
7927 :
7928 0 : gcc_assert (TREE_CODE (value) != RAW_DATA_CST);
7929 0 : if (index)
7930 0 : offset = fold_build2 (MINUS_EXPR,
7931 : TREE_TYPE (index),
7932 : index,
7933 : TYPE_MIN_VALUE (domain));
7934 : else
7935 0 : offset = size_int (i + j);
7936 :
7937 0 : offset = size_binop (MULT_EXPR,
7938 : fold_convert (sizetype, offset),
7939 : TYPE_SIZE_UNIT (elttype));
7940 0 : xtarget = offset_address (target,
7941 : expand_normal (offset),
7942 : highest_pow2_factor (offset));
7943 0 : xtarget = adjust_address (xtarget, mode, 0);
7944 0 : store_expr (value, xtarget, 0, false, reverse);
7945 : }
7946 : else
7947 : {
7948 444 : if (index)
7949 888 : bitpos = ((tree_to_uhwi (index) - minelt)
7950 444 : * tree_to_uhwi (TYPE_SIZE (elttype)));
7951 : else
7952 0 : bitpos = ((i + j) * tree_to_uhwi (TYPE_SIZE (elttype)));
7953 :
7954 12 : if (MEM_P (target) && !MEM_KEEP_ALIAS_SET_P (target)
7955 12 : && TREE_CODE (type) == ARRAY_TYPE
7956 456 : && TYPE_NONALIASED_COMPONENT (type))
7957 : {
7958 0 : target = copy_rtx (target);
7959 0 : MEM_KEEP_ALIAS_SET_P (target) = 1;
7960 : }
7961 444 : if (TREE_CODE (value) != RAW_DATA_CST)
7962 440 : store_constructor_field (target, bitsize, bitpos, 0,
7963 440 : bitregion_end, mode, value,
7964 : cleared, get_alias_set (elttype),
7965 : reverse);
7966 : else
7967 : {
7968 4 : j += RAW_DATA_LENGTH (value) - 1;
7969 4 : gcc_assert (known_eq (bitsize, BITS_PER_UNIT));
7970 4 : rtx to_rtx = adjust_address (target, mode,
7971 : bitpos / BITS_PER_UNIT);
7972 :
7973 4 : if (to_rtx == target)
7974 0 : to_rtx = copy_rtx (to_rtx);
7975 :
7976 4 : if (!MEM_KEEP_ALIAS_SET_P (to_rtx)
7977 8 : && MEM_ALIAS_SET (to_rtx) != 0)
7978 0 : set_mem_alias_set (to_rtx, get_alias_set (elttype));
7979 :
7980 8 : if (can_store_by_pieces (RAW_DATA_LENGTH (value),
7981 : raw_data_cst_read_str,
7982 : (void *) value,
7983 4 : MEM_ALIGN (target), false))
7984 : {
7985 4 : store_by_pieces (target, RAW_DATA_LENGTH (value),
7986 : raw_data_cst_read_str, (void *) value,
7987 2 : MEM_ALIGN (target), false,
7988 : RETURN_BEGIN);
7989 2 : continue;
7990 : }
7991 :
7992 2 : elttype
7993 2 : = build_array_type_nelts (TREE_TYPE (value),
7994 2 : RAW_DATA_LENGTH (value));
7995 2 : tree ctor = build_constructor_single (elttype, NULL_TREE,
7996 : value);
7997 2 : ctor = tree_output_constant_def (ctor);
7998 2 : mode = TYPE_MODE (type);
7999 2 : store_constructor_field (target,
8000 2 : bitsize * RAW_DATA_LENGTH (value),
8001 2 : bitpos, 0, bitregion_end, mode,
8002 : ctor, cleared,
8003 : get_alias_set (elttype), reverse);
8004 : }
8005 : }
8006 : }
8007 : break;
8008 : }
8009 :
8010 140399 : case VECTOR_TYPE:
8011 140399 : {
8012 140399 : unsigned HOST_WIDE_INT idx;
8013 140399 : constructor_elt *ce;
8014 140399 : bool need_to_clear;
8015 140399 : insn_code icode = CODE_FOR_nothing;
8016 140399 : tree elt;
8017 140399 : tree elttype = TREE_TYPE (type);
8018 140399 : int elt_size = vector_element_bits (type);
8019 140399 : machine_mode eltmode = TYPE_MODE (elttype);
8020 140399 : poly_int64 bitsize;
8021 140399 : poly_int64 bitpos;
8022 140399 : rtvec vector = NULL;
8023 140399 : poly_uint64 n_elts;
8024 140399 : unsigned HOST_WIDE_INT const_n_elts;
8025 140399 : alias_set_type alias;
8026 140399 : bool vec_vec_init_p = false;
8027 140399 : machine_mode mode = GET_MODE (target);
8028 :
8029 140399 : gcc_assert (eltmode != BLKmode);
8030 :
8031 : /* Try using vec_duplicate_optab for uniform vectors. */
8032 140399 : icode = optab_handler (vec_duplicate_optab, mode);
8033 140399 : if (!TREE_SIDE_EFFECTS (exp)
8034 140399 : && VECTOR_MODE_P (mode)
8035 136843 : && icode != CODE_FOR_nothing
8036 : /* If the vec_duplicate target pattern does not specify an element
8037 : mode check that eltmode is the normal inner mode of the
8038 : requested vector mode. But if the target allows eltmode
8039 : explicitly go ahead and use it. */
8040 93343 : && (eltmode == GET_MODE_INNER (mode)
8041 0 : || insn_data[icode].operand[1].mode == eltmode)
8042 93343 : && (elt = uniform_vector_p (exp))
8043 158114 : && !VECTOR_TYPE_P (TREE_TYPE (elt)))
8044 : {
8045 17715 : class expand_operand ops[2];
8046 17715 : create_output_operand (&ops[0], target, mode);
8047 17715 : create_input_operand (&ops[1], expand_normal (elt), eltmode);
8048 17715 : expand_insn (icode, 2, ops);
8049 17715 : if (!rtx_equal_p (target, ops[0].value))
8050 0 : emit_move_insn (target, ops[0].value);
8051 17715 : break;
8052 : }
8053 : /* Use sign-extension for uniform boolean vectors with
8054 : integer modes and single-bit mask entries.
8055 : Effectively "vec_duplicate" for bitmasks. */
8056 122684 : if (elt_size == 1
8057 146 : && !TREE_SIDE_EFFECTS (exp)
8058 146 : && VECTOR_BOOLEAN_TYPE_P (type)
8059 146 : && SCALAR_INT_MODE_P (TYPE_MODE (type))
8060 146 : && (elt = uniform_vector_p (exp))
8061 143 : && !VECTOR_TYPE_P (TREE_TYPE (elt))
8062 122684 : && !BYTES_BIG_ENDIAN)
8063 : {
8064 143 : rtx op0 = force_reg (TYPE_MODE (TREE_TYPE (elt)),
8065 : expand_normal (elt));
8066 143 : rtx tmp = gen_reg_rtx (mode);
8067 143 : convert_move (tmp, op0, 0);
8068 :
8069 : /* Ensure no excess bits are set.
8070 : GCN needs this for nunits < 64.
8071 : x86 needs this for nunits < 8. */
8072 143 : unsigned int nunits = TYPE_VECTOR_SUBPARTS (type).to_constant ();
8073 143 : if (maybe_ne (GET_MODE_PRECISION (mode), nunits))
8074 2 : tmp = expand_binop (mode, and_optab, tmp,
8075 2 : GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
8076 : target, true, OPTAB_WIDEN);
8077 143 : if (tmp != target)
8078 141 : emit_move_insn (target, tmp);
8079 : break;
8080 : }
8081 :
8082 122541 : n_elts = TYPE_VECTOR_SUBPARTS (type);
8083 122541 : if (REG_P (target)
8084 119322 : && VECTOR_MODE_P (mode))
8085 : {
8086 119128 : const_n_elts = 0;
8087 119128 : if (CONSTRUCTOR_NELTS (exp)
8088 119128 : && (TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (exp, 0)->value))
8089 : == VECTOR_TYPE))
8090 : {
8091 1438 : tree etype = TREE_TYPE (CONSTRUCTOR_ELT (exp, 0)->value);
8092 1438 : gcc_assert (known_eq (CONSTRUCTOR_NELTS (exp)
8093 : * TYPE_VECTOR_SUBPARTS (etype),
8094 : n_elts));
8095 :
8096 4314 : icode = convert_optab_handler (vec_init_optab, mode,
8097 1438 : TYPE_MODE (etype));
8098 1438 : const_n_elts = CONSTRUCTOR_NELTS (exp);
8099 1438 : vec_vec_init_p = icode != CODE_FOR_nothing;
8100 : }
8101 235380 : else if (exact_div (n_elts, GET_MODE_NUNITS (eltmode))
8102 117690 : .is_constant (&const_n_elts))
8103 : {
8104 : /* For a non-const type vector, we check it is made up of
8105 : similarly non-const type vectors. */
8106 117690 : icode = convert_optab_handler (vec_init_optab, mode, eltmode);
8107 : }
8108 : else
8109 : {
8110 : /* Handle variable-length vector types. */
8111 : icode = convert_optab_handler (vec_init_optab, mode, eltmode);
8112 : const_n_elts = constant_lower_bound (n_elts);
8113 : }
8114 :
8115 119128 : if (const_n_elts && icode != CODE_FOR_nothing)
8116 : {
8117 118393 : vector = rtvec_alloc (const_n_elts);
8118 559914 : for (unsigned int k = 0; k < const_n_elts; k++)
8119 323128 : RTVEC_ELT (vector, k) = CONST0_RTX (eltmode);
8120 : }
8121 : }
8122 : else
8123 : gcc_assert (n_elts.is_constant ());
8124 :
8125 : /* Compute the size of the elements in the CTOR. It differs
8126 : from the size of the vector type elements only when the
8127 : CTOR elements are vectors themselves. */
8128 122541 : tree val_type = (CONSTRUCTOR_NELTS (exp) != 0
8129 122541 : ? TREE_TYPE (CONSTRUCTOR_ELT (exp, 0)->value)
8130 122541 : : elttype);
8131 122541 : if (VECTOR_TYPE_P (val_type))
8132 1788 : bitsize = tree_to_poly_uint64 (TYPE_SIZE (val_type));
8133 : else
8134 120753 : bitsize = elt_size;
8135 :
8136 : /* If the constructor has fewer elements than the vector,
8137 : clear the whole array first. Similarly if this is static
8138 : constructor of a non-BLKmode object. */
8139 122541 : if (cleared)
8140 : need_to_clear = false;
8141 122541 : else if (REG_P (target) && TREE_STATIC (exp))
8142 : need_to_clear = true;
8143 : else
8144 : {
8145 : poly_uint64 count = 0, zero_count = 0;
8146 : tree value;
8147 :
8148 489570 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), idx, value)
8149 : {
8150 367029 : poly_int64 n_elts_here = exact_div (bitsize, elt_size);
8151 367029 : count += n_elts_here;
8152 367029 : if (mostly_zeros_p (value))
8153 367029 : zero_count += n_elts_here;
8154 : }
8155 :
8156 : /* Clear the entire vector first if there are any missing elements,
8157 : or if the incidence of zero elements is >= 75%. */
8158 122541 : need_to_clear = (maybe_lt (count, n_elts)
8159 122541 : || maybe_gt (4 * zero_count, 3 * count));
8160 : }
8161 :
8162 1307 : if (need_to_clear && maybe_gt (size, 0) && !vector)
8163 : {
8164 761 : if (REG_P (target))
8165 25 : emit_move_insn (target, CONST0_RTX (mode));
8166 : else
8167 736 : clear_storage (target, gen_int_mode (size, Pmode),
8168 : BLOCK_OP_NORMAL);
8169 : cleared = 1;
8170 : }
8171 :
8172 : /* Inform later passes that the old value is dead. */
8173 122541 : if (!cleared && !vector && REG_P (target) && maybe_gt (n_elts, 1u))
8174 : {
8175 587 : emit_move_insn (target, CONST0_RTX (mode));
8176 587 : cleared = 1;
8177 : }
8178 :
8179 122541 : if (MEM_P (target))
8180 3219 : alias = MEM_ALIAS_SET (target);
8181 : else
8182 119322 : alias = get_alias_set (elttype);
8183 :
8184 : /* Store each element of the constructor into the corresponding
8185 : element of TARGET, determined by counting the elements. */
8186 122541 : HOST_WIDE_INT chunk_size = 0;
8187 122541 : bool chunk_multiple_p = constant_multiple_p (bitsize, elt_size,
8188 : &chunk_size);
8189 122541 : gcc_assert (chunk_multiple_p || vec_vec_init_p);
8190 :
8191 489570 : for (idx = 0; vec_safe_iterate (CONSTRUCTOR_ELTS (exp), idx, &ce);
8192 : idx++)
8193 : {
8194 367029 : HOST_WIDE_INT eltpos;
8195 367029 : tree value = ce->value;
8196 :
8197 367029 : if (cleared && initializer_zerop (value))
8198 8817 : continue;
8199 :
8200 358212 : if (ce->index)
8201 49538 : eltpos = tree_to_uhwi (ce->index);
8202 : else
8203 308674 : eltpos = idx * chunk_size;
8204 :
8205 358212 : if (vector)
8206 : {
8207 322045 : if (vec_vec_init_p)
8208 : {
8209 2772 : gcc_assert (ce->index == NULL_TREE);
8210 2772 : gcc_assert (TREE_CODE (TREE_TYPE (value)) == VECTOR_TYPE);
8211 2772 : eltpos = idx;
8212 : }
8213 : else
8214 319273 : gcc_assert (TREE_CODE (TREE_TYPE (value)) != VECTOR_TYPE);
8215 322045 : RTVEC_ELT (vector, eltpos) = expand_normal (value);
8216 : }
8217 : else
8218 : {
8219 36167 : machine_mode value_mode
8220 36167 : = (TREE_CODE (TREE_TYPE (value)) == VECTOR_TYPE
8221 36167 : ? TYPE_MODE (TREE_TYPE (value)) : eltmode);
8222 36167 : bitpos = eltpos * elt_size;
8223 36167 : store_constructor_field (target, bitsize, bitpos, 0,
8224 36167 : bitregion_end, value_mode,
8225 : value, cleared, alias, reverse);
8226 : }
8227 : }
8228 :
8229 122541 : if (vector)
8230 118393 : emit_insn (GEN_FCN (icode) (target,
8231 : gen_rtx_PARALLEL (mode, vector)));
8232 : break;
8233 : }
8234 :
8235 0 : default:
8236 0 : gcc_unreachable ();
8237 : }
8238 248692 : }
8239 :
8240 : /* Store the value of EXP (an expression tree)
8241 : into a subfield of TARGET which has mode MODE and occupies
8242 : BITSIZE bits, starting BITPOS bits from the start of TARGET.
8243 : If MODE is VOIDmode, it means that we are storing into a bit-field.
8244 :
8245 : BITREGION_START is bitpos of the first bitfield in this region.
8246 : BITREGION_END is the bitpos of the ending bitfield in this region.
8247 : These two fields are 0, if the C++ memory model does not apply,
8248 : or we are not interested in keeping track of bitfield regions.
8249 :
8250 : Always return const0_rtx unless we have something particular to
8251 : return.
8252 :
8253 : ALIAS_SET is the alias set for the destination. This value will
8254 : (in general) be different from that for TARGET, since TARGET is a
8255 : reference to the containing structure.
8256 :
8257 : If NONTEMPORAL is true, try generating a nontemporal store.
8258 :
8259 : If REVERSE is true, the store is to be done in reverse order. */
8260 :
8261 : static rtx
8262 4836284 : store_field (rtx target, poly_int64 bitsize, poly_int64 bitpos,
8263 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
8264 : machine_mode mode, tree exp,
8265 : alias_set_type alias_set, bool nontemporal, bool reverse)
8266 : {
8267 4836284 : if (TREE_CODE (exp) == ERROR_MARK)
8268 0 : return const0_rtx;
8269 :
8270 : /* If we have nothing to store, do nothing unless the expression has
8271 : side-effects. Don't do that for zero sized addressable lhs of
8272 : calls. */
8273 4836284 : if (known_eq (bitsize, 0)
8274 4836284 : && (!TREE_ADDRESSABLE (TREE_TYPE (exp))
8275 3 : || TREE_CODE (exp) != CALL_EXPR))
8276 0 : return expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
8277 :
8278 4836284 : if (GET_CODE (target) == CONCAT)
8279 : {
8280 : /* We're storing into a struct containing a single __complex. */
8281 :
8282 0 : gcc_assert (known_eq (bitpos, 0));
8283 0 : return store_expr (exp, target, 0, nontemporal, reverse);
8284 : }
8285 :
8286 : /* If the structure is in a register or if the component
8287 : is a bit field, we cannot use addressing to access it.
8288 : Use bit-field techniques or SUBREG to store in it. */
8289 :
8290 4836284 : poly_int64 decl_bitsize;
8291 4836284 : if (mode == VOIDmode
8292 4769201 : || (mode != BLKmode && ! direct_store[(int) mode]
8293 5338 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_INT
8294 5336 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_FLOAT)
8295 4763894 : || REG_P (target)
8296 4071639 : || GET_CODE (target) == SUBREG
8297 : /* If the field isn't aligned enough to store as an ordinary memref,
8298 : store it as a bit field. */
8299 4071639 : || (mode != BLKmode
8300 3956217 : && ((((MEM_ALIGN (target) < GET_MODE_ALIGNMENT (mode))
8301 7823016 : || !multiple_p (bitpos, GET_MODE_ALIGNMENT (mode)))
8302 47421 : && targetm.slow_unaligned_access (mode, MEM_ALIGN (target)))
8303 3956217 : || !multiple_p (bitpos, BITS_PER_UNIT)))
8304 4071639 : || (known_size_p (bitsize)
8305 4071627 : && mode != BLKmode
8306 3956217 : && maybe_gt (GET_MODE_BITSIZE (mode), bitsize))
8307 : /* If the RHS and field are a constant size and the size of the
8308 : RHS isn't the same size as the bitfield, we must use bitfield
8309 : operations. */
8310 4071630 : || (known_size_p (bitsize)
8311 4071618 : && poly_int_tree_p (TYPE_SIZE (TREE_TYPE (exp)))
8312 4836558 : && maybe_ne (wi::to_poly_offset (TYPE_SIZE (TREE_TYPE (exp))),
8313 : bitsize)
8314 : /* Except for initialization of full bytes from a CONSTRUCTOR, which
8315 : we will handle specially below. */
8316 34 : && !(TREE_CODE (exp) == CONSTRUCTOR
8317 11 : && multiple_p (bitsize, BITS_PER_UNIT))
8318 : /* And except for bitwise copying of TREE_ADDRESSABLE types,
8319 : where the FIELD_DECL has the right bitsize, but TREE_TYPE (exp)
8320 : includes some extra padding. store_expr / expand_expr will in
8321 : that case call get_inner_reference that will have the bitsize
8322 : we check here and thus the block move will not clobber the
8323 : padding that shouldn't be clobbered. In the future we could
8324 : replace the TREE_ADDRESSABLE check with a check that
8325 : get_base_address needs to live in memory. */
8326 23 : && (!TREE_ADDRESSABLE (TREE_TYPE (exp))
8327 9 : || TREE_CODE (exp) != COMPONENT_REF
8328 6 : || !multiple_p (bitsize, BITS_PER_UNIT)
8329 6 : || !multiple_p (bitpos, BITS_PER_UNIT)
8330 6 : || !poly_int_tree_p (DECL_SIZE (TREE_OPERAND (exp, 1)),
8331 : &decl_bitsize)
8332 6 : || maybe_ne (decl_bitsize, bitsize))
8333 : /* A call with an addressable return type and return-slot
8334 : optimization must not need bitfield operations but we must
8335 : pass down the original target. */
8336 17 : && (TREE_CODE (exp) != CALL_EXPR
8337 6 : || !TREE_ADDRESSABLE (TREE_TYPE (exp))
8338 0 : || !CALL_EXPR_RETURN_SLOT_OPT (exp)))
8339 : /* If we are expanding a MEM_REF of a non-BLKmode non-addressable
8340 : decl we must use bitfield operations. */
8341 8907897 : || (known_size_p (bitsize)
8342 4071601 : && TREE_CODE (exp) == MEM_REF
8343 32829 : && TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR
8344 26873 : && DECL_P (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
8345 20360 : && !TREE_ADDRESSABLE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
8346 5687 : && DECL_MODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0)) != BLKmode))
8347 : {
8348 764940 : rtx temp;
8349 764940 : gimple *nop_def;
8350 :
8351 : /* If EXP is a NOP_EXPR of precision less than its mode, then that
8352 : implies a mask operation. If the precision is the same size as
8353 : the field we're storing into, that mask is redundant. This is
8354 : particularly common with bit field assignments generated by the
8355 : C front end. */
8356 764940 : nop_def = get_def_for_expr (exp, NOP_EXPR);
8357 764940 : if (nop_def)
8358 : {
8359 5527 : tree type = TREE_TYPE (exp);
8360 5527 : if (INTEGRAL_TYPE_P (type)
8361 5317 : && maybe_ne (TYPE_PRECISION (type),
8362 10634 : GET_MODE_BITSIZE (TYPE_MODE (type)))
8363 8810 : && known_eq (bitsize, TYPE_PRECISION (type)))
8364 : {
8365 3283 : tree op = gimple_assign_rhs1 (nop_def);
8366 3283 : type = TREE_TYPE (op);
8367 3283 : if (INTEGRAL_TYPE_P (type)
8368 3283 : && known_ge (TYPE_PRECISION (type), bitsize))
8369 : exp = op;
8370 : }
8371 : }
8372 :
8373 764940 : temp = expand_normal (exp);
8374 :
8375 : /* We don't support variable-sized BLKmode bitfields, since our
8376 : handling of BLKmode is bound up with the ability to break
8377 : things into words. */
8378 764940 : gcc_assert (mode != BLKmode || bitsize.is_constant ());
8379 :
8380 : /* Handle calls that return values in multiple non-contiguous locations.
8381 : The Irix 6 ABI has examples of this. */
8382 764940 : if (GET_CODE (temp) == PARALLEL)
8383 : {
8384 6 : HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));
8385 6 : machine_mode temp_mode = GET_MODE (temp);
8386 6 : if (temp_mode == BLKmode || temp_mode == VOIDmode)
8387 6 : temp_mode
8388 6 : = smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
8389 6 : rtx temp_target = gen_reg_rtx (temp_mode);
8390 6 : emit_group_store (temp_target, temp, TREE_TYPE (exp), size);
8391 6 : temp = temp_target;
8392 : }
8393 :
8394 : /* Handle calls that return BLKmode values in registers. */
8395 764934 : else if (mode == BLKmode && REG_P (temp) && TREE_CODE (exp) == CALL_EXPR)
8396 : {
8397 0 : rtx temp_target = gen_reg_rtx (GET_MODE (temp));
8398 0 : copy_blkmode_from_reg (temp_target, temp, TREE_TYPE (exp));
8399 0 : temp = temp_target;
8400 : }
8401 :
8402 : /* If the value has aggregate type and an integral mode then, if BITSIZE
8403 : is narrower than this mode and this is for big-endian data, we first
8404 : need to put the value into the low-order bits for store_bit_field,
8405 : except when MODE is BLKmode and BITSIZE larger than the word size
8406 : (see the handling of fields larger than a word in store_bit_field).
8407 : Moreover, the field may be not aligned on a byte boundary; in this
8408 : case, if it has reverse storage order, it needs to be accessed as a
8409 : scalar field with reverse storage order and we must first put the
8410 : value into target order. */
8411 764940 : scalar_int_mode temp_mode;
8412 1527810 : if (AGGREGATE_TYPE_P (TREE_TYPE (exp))
8413 765691 : && is_int_mode (GET_MODE (temp), &temp_mode))
8414 : {
8415 2494 : HOST_WIDE_INT size = GET_MODE_BITSIZE (temp_mode);
8416 :
8417 2494 : reverse = TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (exp));
8418 :
8419 2494 : if (reverse)
8420 0 : temp = flip_storage_order (temp_mode, temp);
8421 :
8422 2494 : gcc_checking_assert (known_le (bitsize, size));
8423 2494 : if (maybe_lt (bitsize, size)
8424 2494 : && reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN
8425 : /* Use of to_constant for BLKmode was checked above. */
8426 : && !(mode == BLKmode && bitsize.to_constant () > BITS_PER_WORD))
8427 0 : temp = expand_shift (RSHIFT_EXPR, temp_mode, temp,
8428 : size - bitsize, NULL_RTX, 1);
8429 : }
8430 :
8431 : /* Unless MODE is VOIDmode or BLKmode, convert TEMP to MODE. */
8432 697857 : if (mode != VOIDmode && mode != BLKmode
8433 1462520 : && mode != TYPE_MODE (TREE_TYPE (exp)))
8434 4 : temp = convert_modes (mode, TYPE_MODE (TREE_TYPE (exp)), temp, 1);
8435 :
8436 : /* If the mode of TEMP and TARGET is BLKmode, both must be in memory
8437 : and BITPOS must be aligned on a byte boundary. If so, we simply do
8438 : a block copy. Likewise for a BLKmode-like TARGET. */
8439 764940 : if (GET_MODE (temp) == BLKmode
8440 764940 : && (GET_MODE (target) == BLKmode
8441 182 : || (MEM_P (target)
8442 0 : && GET_MODE_CLASS (GET_MODE (target)) == MODE_INT
8443 0 : && multiple_p (bitpos, BITS_PER_UNIT)
8444 0 : && multiple_p (bitsize, BITS_PER_UNIT))))
8445 : {
8446 89 : gcc_assert (MEM_P (target) && MEM_P (temp));
8447 89 : poly_int64 bytepos = exact_div (bitpos, BITS_PER_UNIT);
8448 89 : poly_int64 bytesize = bits_to_bytes_round_up (bitsize);
8449 :
8450 89 : target = adjust_address (target, VOIDmode, bytepos);
8451 89 : emit_block_move (target, temp,
8452 89 : gen_int_mode (bytesize, Pmode),
8453 : BLOCK_OP_NORMAL);
8454 :
8455 89 : return const0_rtx;
8456 : }
8457 :
8458 : /* If the mode of TEMP is still BLKmode and BITSIZE not larger than the
8459 : word size, we need to load the value (see again store_bit_field). */
8460 764869 : if (GET_MODE (temp) == BLKmode && known_le (bitsize, BITS_PER_WORD))
8461 : {
8462 61 : temp_mode = smallest_int_mode_for_size (bitsize).require ();
8463 61 : temp = extract_bit_field (temp, bitsize, 0, 1, NULL_RTX, temp_mode,
8464 : temp_mode, false, NULL);
8465 : }
8466 :
8467 : /* Store the value in the bitfield. */
8468 764851 : gcc_checking_assert (known_ge (bitpos, 0));
8469 764851 : store_bit_field (target, bitsize, bitpos,
8470 : bitregion_start, bitregion_end,
8471 : mode, temp, reverse, false);
8472 :
8473 764851 : return const0_rtx;
8474 : }
8475 : else
8476 : {
8477 : /* Now build a reference to just the desired component. */
8478 4071344 : rtx to_rtx = adjust_address (target, mode,
8479 : exact_div (bitpos, BITS_PER_UNIT));
8480 :
8481 4071344 : if (to_rtx == target)
8482 1299134 : to_rtx = copy_rtx (to_rtx);
8483 :
8484 7866499 : if (!MEM_KEEP_ALIAS_SET_P (to_rtx) && MEM_ALIAS_SET (to_rtx) != 0)
8485 3535718 : set_mem_alias_set (to_rtx, alias_set);
8486 :
8487 : /* Above we avoided using bitfield operations for storing a CONSTRUCTOR
8488 : into a target smaller than its type; handle that case now. */
8489 4071344 : if (TREE_CODE (exp) == CONSTRUCTOR && known_size_p (bitsize))
8490 : {
8491 69777 : poly_int64 bytesize = exact_div (bitsize, BITS_PER_UNIT);
8492 69777 : store_constructor (exp, to_rtx, 0, bytesize, reverse);
8493 69777 : return to_rtx;
8494 : }
8495 :
8496 4001567 : return store_expr (exp, to_rtx, 0, nontemporal, reverse);
8497 : }
8498 : }
8499 :
8500 : /* Given an expression EXP that may be a COMPONENT_REF, a BIT_FIELD_REF,
8501 : an ARRAY_REF, or an ARRAY_RANGE_REF, look for nested operations of these
8502 : codes and find the ultimate containing object, which we return.
8503 :
8504 : We set *PBITSIZE to the size in bits that we want, *PBITPOS to the
8505 : bit position, *PUNSIGNEDP to the signedness and *PREVERSEP to the
8506 : storage order of the field.
8507 : If the position of the field is variable, we store a tree
8508 : giving the variable offset (in units) in *POFFSET.
8509 : This offset is in addition to the bit position.
8510 : If the position is not variable, we store 0 in *POFFSET.
8511 :
8512 : If any of the extraction expressions is volatile,
8513 : we store 1 in *PVOLATILEP. Otherwise we don't change that.
8514 :
8515 : If the field is a non-BLKmode bit-field, *PMODE is set to VOIDmode.
8516 : Otherwise, it is a mode that can be used to access the field.
8517 :
8518 : If the field describes a variable-sized object, *PMODE is set to
8519 : BLKmode and *PBITSIZE is set to -1. An access cannot be made in
8520 : this case, but the address of the object can be found. */
8521 :
8522 : tree
8523 296681317 : get_inner_reference (tree exp, poly_int64 *pbitsize,
8524 : poly_int64 *pbitpos, tree *poffset,
8525 : machine_mode *pmode, int *punsignedp,
8526 : int *preversep, int *pvolatilep)
8527 : {
8528 296681317 : tree size_tree = 0;
8529 296681317 : machine_mode mode = VOIDmode;
8530 296681317 : bool blkmode_bitfield = false;
8531 296681317 : tree offset = size_zero_node;
8532 296681317 : poly_offset_int bit_offset = 0;
8533 :
8534 : /* First get the mode, signedness, storage order and size. We do this from
8535 : just the outermost expression. */
8536 296681317 : *pbitsize = -1;
8537 296681317 : if (TREE_CODE (exp) == COMPONENT_REF)
8538 : {
8539 127371209 : tree field = TREE_OPERAND (exp, 1);
8540 127371209 : size_tree = DECL_SIZE (field);
8541 127371209 : if (flag_strict_volatile_bitfields > 0
8542 58 : && TREE_THIS_VOLATILE (exp)
8543 40 : && DECL_BIT_FIELD_TYPE (field)
8544 127371231 : && DECL_MODE (field) != BLKmode)
8545 : /* Volatile bitfields should be accessed in the mode of the
8546 : field's type, not the mode computed based on the bit
8547 : size. */
8548 22 : mode = TYPE_MODE (DECL_BIT_FIELD_TYPE (field));
8549 127371187 : else if (!DECL_BIT_FIELD (field))
8550 : {
8551 126030280 : mode = DECL_MODE (field);
8552 : /* For vector fields re-check the target flags, as DECL_MODE
8553 : could have been set with different target flags than
8554 : the current function has. */
8555 126030280 : if (VECTOR_TYPE_P (TREE_TYPE (field))
8556 126030280 : && VECTOR_MODE_P (TYPE_MODE_RAW (TREE_TYPE (field))))
8557 380589 : mode = TYPE_MODE (TREE_TYPE (field));
8558 : }
8559 1340907 : else if (DECL_MODE (field) == BLKmode)
8560 624 : blkmode_bitfield = true;
8561 :
8562 127371209 : *punsignedp = DECL_UNSIGNED (field);
8563 : }
8564 169310108 : else if (TREE_CODE (exp) == BIT_FIELD_REF)
8565 : {
8566 613860 : size_tree = TREE_OPERAND (exp, 1);
8567 1227301 : *punsignedp = (! INTEGRAL_TYPE_P (TREE_TYPE (exp))
8568 1077353 : || TYPE_UNSIGNED (TREE_TYPE (exp)));
8569 :
8570 : /* For vector element types with the correct size of access or for
8571 : vector typed accesses use the mode of the access type. */
8572 613860 : if ((TREE_CODE (TREE_TYPE (TREE_OPERAND (exp, 0))) == VECTOR_TYPE
8573 461155 : && TREE_TYPE (exp) == TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0)))
8574 414492 : && tree_int_cst_equal (size_tree, TYPE_SIZE (TREE_TYPE (exp))))
8575 660558 : || VECTOR_TYPE_P (TREE_TYPE (exp)))
8576 438709 : mode = TYPE_MODE (TREE_TYPE (exp));
8577 : }
8578 : else
8579 : {
8580 168696248 : mode = TYPE_MODE (TREE_TYPE (exp));
8581 168696248 : *punsignedp = TYPE_UNSIGNED (TREE_TYPE (exp));
8582 :
8583 168696248 : if (mode == BLKmode)
8584 74687481 : size_tree = TYPE_SIZE (TREE_TYPE (exp));
8585 : else
8586 188017534 : *pbitsize = GET_MODE_BITSIZE (mode);
8587 : }
8588 :
8589 296681317 : if (size_tree != 0)
8590 : {
8591 201842557 : if (!poly_int_tree_p (size_tree, pbitsize))
8592 487998 : mode = BLKmode, *pbitsize = -1;
8593 : }
8594 :
8595 296681317 : *preversep = reverse_storage_order_for_component_p (exp);
8596 :
8597 : /* Compute cumulative bit-offset for nested component-refs and array-refs,
8598 : and find the ultimate containing object. */
8599 758366183 : while (1)
8600 : {
8601 527523750 : switch (TREE_CODE (exp))
8602 : {
8603 613860 : case BIT_FIELD_REF:
8604 613860 : bit_offset += wi::to_poly_offset (TREE_OPERAND (exp, 2));
8605 613860 : break;
8606 :
8607 194198123 : case COMPONENT_REF:
8608 194198123 : {
8609 194198123 : tree field = TREE_OPERAND (exp, 1);
8610 194198123 : tree this_offset = component_ref_field_offset (exp);
8611 :
8612 : /* If this field hasn't been filled in yet, don't go past it.
8613 : This should only happen when folding expressions made during
8614 : type construction. */
8615 194198123 : if (this_offset == 0)
8616 : break;
8617 :
8618 194198123 : offset = size_binop (PLUS_EXPR, offset, this_offset);
8619 194198123 : bit_offset += wi::to_poly_offset (DECL_FIELD_BIT_OFFSET (field));
8620 :
8621 : /* ??? Right now we don't do anything with DECL_OFFSET_ALIGN. */
8622 : }
8623 194198123 : break;
8624 :
8625 32723949 : case ARRAY_REF:
8626 32723949 : case ARRAY_RANGE_REF:
8627 32723949 : {
8628 32723949 : tree index = TREE_OPERAND (exp, 1);
8629 32723949 : tree low_bound = array_ref_low_bound (exp);
8630 32723949 : tree unit_size = array_ref_element_size (exp);
8631 :
8632 : /* We assume all arrays have sizes that are a multiple of a byte.
8633 : First subtract the lower bound, if any, in the type of the
8634 : index, then convert to sizetype and multiply by the size of
8635 : the array element. */
8636 32723949 : if (! integer_zerop (low_bound))
8637 969143 : index = fold_build2 (MINUS_EXPR, TREE_TYPE (index),
8638 : index, low_bound);
8639 :
8640 32723949 : offset = size_binop (PLUS_EXPR, offset,
8641 : size_binop (MULT_EXPR,
8642 : fold_convert (sizetype, index),
8643 : unit_size));
8644 : }
8645 32723949 : break;
8646 :
8647 : case REALPART_EXPR:
8648 : break;
8649 :
8650 : case IMAGPART_EXPR:
8651 230842433 : bit_offset += *pbitsize;
8652 : break;
8653 :
8654 : case VIEW_CONVERT_EXPR:
8655 : break;
8656 :
8657 93189413 : case MEM_REF:
8658 : /* Hand back the decl for MEM[&decl, off]. */
8659 93189413 : if (TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR)
8660 : {
8661 21928393 : tree off = TREE_OPERAND (exp, 1);
8662 21928393 : if (!integer_zerop (off))
8663 : {
8664 12435824 : poly_offset_int boff = mem_ref_offset (exp);
8665 12435824 : boff <<= LOG2_BITS_PER_UNIT;
8666 12435824 : bit_offset += boff;
8667 : }
8668 21928393 : exp = TREE_OPERAND (TREE_OPERAND (exp, 0), 0);
8669 : }
8670 93189413 : goto done;
8671 :
8672 203491904 : default:
8673 203491904 : goto done;
8674 : }
8675 :
8676 : /* If any reference in the chain is volatile, the effect is volatile. */
8677 230842433 : if (TREE_THIS_VOLATILE (exp))
8678 566716 : *pvolatilep = 1;
8679 :
8680 230842433 : exp = TREE_OPERAND (exp, 0);
8681 230842433 : }
8682 296681317 : done:
8683 :
8684 : /* If OFFSET is constant, see if we can return the whole thing as a
8685 : constant bit position. Make sure to handle overflow during
8686 : this conversion. */
8687 296681317 : if (poly_int_tree_p (offset))
8688 : {
8689 282935394 : poly_offset_int tem = wi::sext (wi::to_poly_offset (offset),
8690 282935394 : TYPE_PRECISION (sizetype));
8691 282935394 : tem <<= LOG2_BITS_PER_UNIT;
8692 282935394 : tem += bit_offset;
8693 282935394 : if (tem.to_shwi (pbitpos))
8694 282933966 : *poffset = offset = NULL_TREE;
8695 : }
8696 :
8697 : /* Otherwise, split it up. */
8698 282935394 : if (offset)
8699 : {
8700 : /* Avoid returning a negative bitpos as this may wreak havoc later. */
8701 13747351 : if (!bit_offset.to_shwi (pbitpos) || maybe_lt (*pbitpos, 0))
8702 : {
8703 303 : *pbitpos = num_trailing_bits (bit_offset.force_shwi ());
8704 303 : poly_offset_int bytes = bits_to_bytes_round_down (bit_offset);
8705 303 : offset = size_binop (PLUS_EXPR, offset,
8706 : build_int_cst (sizetype, bytes.force_shwi ()));
8707 : }
8708 :
8709 13747351 : *poffset = offset;
8710 : }
8711 :
8712 : /* We can use BLKmode for a byte-aligned BLKmode bitfield. */
8713 296681317 : if (mode == VOIDmode
8714 5672675 : && blkmode_bitfield
8715 624 : && multiple_p (*pbitpos, BITS_PER_UNIT)
8716 296681805 : && multiple_p (*pbitsize, BITS_PER_UNIT))
8717 5 : *pmode = BLKmode;
8718 : else
8719 296681312 : *pmode = mode;
8720 :
8721 296681317 : return exp;
8722 : }
8723 :
8724 : /* Alignment in bits the TARGET of an assignment may be assumed to have. */
8725 :
8726 : static unsigned HOST_WIDE_INT
8727 514884 : target_align (const_tree target)
8728 : {
8729 : /* We might have a chain of nested references with intermediate misaligning
8730 : bitfields components, so need to recurse to find out. */
8731 :
8732 514884 : unsigned HOST_WIDE_INT this_align, outer_align;
8733 :
8734 514884 : switch (TREE_CODE (target))
8735 : {
8736 : case BIT_FIELD_REF:
8737 : return 1;
8738 :
8739 140117 : case COMPONENT_REF:
8740 140117 : this_align = DECL_ALIGN (TREE_OPERAND (target, 1));
8741 140117 : outer_align = target_align (TREE_OPERAND (target, 0));
8742 140117 : return MIN (this_align, outer_align);
8743 :
8744 190659 : case ARRAY_REF:
8745 190659 : case ARRAY_RANGE_REF:
8746 190659 : this_align = TYPE_ALIGN (TREE_TYPE (target));
8747 190659 : outer_align = target_align (TREE_OPERAND (target, 0));
8748 190659 : return MIN (this_align, outer_align);
8749 :
8750 4700 : CASE_CONVERT:
8751 4700 : case NON_LVALUE_EXPR:
8752 4700 : case VIEW_CONVERT_EXPR:
8753 4700 : this_align = TYPE_ALIGN (TREE_TYPE (target));
8754 4700 : outer_align = target_align (TREE_OPERAND (target, 0));
8755 4700 : return MAX (this_align, outer_align);
8756 :
8757 179403 : default:
8758 179403 : return TYPE_ALIGN (TREE_TYPE (target));
8759 : }
8760 : }
8761 :
8762 :
8763 : /* Given an rtx VALUE that may contain additions and multiplications, return
8764 : an equivalent value that just refers to a register, memory, or constant.
8765 : This is done by generating instructions to perform the arithmetic and
8766 : returning a pseudo-register containing the value.
8767 :
8768 : The returned value may be a REG, SUBREG, MEM or constant. */
8769 :
8770 : rtx
8771 31624161 : force_operand (rtx value, rtx target)
8772 : {
8773 31624161 : rtx op1, op2;
8774 : /* Use subtarget as the target for operand 0 of a binary operation. */
8775 31624161 : rtx subtarget = get_subtarget (target);
8776 31624161 : enum rtx_code code = GET_CODE (value);
8777 :
8778 : /* Check for subreg applied to an expression produced by loop optimizer. */
8779 31624161 : if (code == SUBREG
8780 306255 : && !REG_P (SUBREG_REG (value))
8781 182 : && !MEM_P (SUBREG_REG (value)))
8782 : {
8783 182 : value
8784 182 : = simplify_gen_subreg (GET_MODE (value),
8785 182 : force_reg (GET_MODE (SUBREG_REG (value)),
8786 : force_operand (SUBREG_REG (value),
8787 : NULL_RTX)),
8788 182 : GET_MODE (SUBREG_REG (value)),
8789 182 : SUBREG_BYTE (value));
8790 182 : code = GET_CODE (value);
8791 : }
8792 :
8793 : /* Check for a PIC address load. */
8794 31624161 : if ((code == PLUS || code == MINUS)
8795 3736559 : && XEXP (value, 0) == pic_offset_table_rtx
8796 1926 : && (GET_CODE (XEXP (value, 1)) == SYMBOL_REF
8797 1926 : || GET_CODE (XEXP (value, 1)) == LABEL_REF
8798 1926 : || GET_CODE (XEXP (value, 1)) == CONST))
8799 : {
8800 184 : if (!subtarget)
8801 184 : subtarget = gen_reg_rtx (GET_MODE (value));
8802 184 : emit_move_insn (subtarget, value);
8803 184 : return subtarget;
8804 : }
8805 :
8806 31623977 : if (ARITHMETIC_P (value))
8807 : {
8808 3848945 : op2 = XEXP (value, 1);
8809 3848945 : if (!CONSTANT_P (op2) && !(REG_P (op2) && op2 != subtarget))
8810 3848945 : subtarget = 0;
8811 3848945 : if (code == MINUS && CONST_INT_P (op2))
8812 : {
8813 0 : code = PLUS;
8814 0 : op2 = negate_rtx (GET_MODE (value), op2);
8815 : }
8816 :
8817 : /* Check for an addition with OP2 a constant integer and our first
8818 : operand a PLUS of a virtual register and something else. In that
8819 : case, we want to emit the sum of the virtual register and the
8820 : constant first and then add the other value. This allows virtual
8821 : register instantiation to simply modify the constant rather than
8822 : creating another one around this addition. */
8823 3677240 : if (code == PLUS && CONST_INT_P (op2)
8824 3306940 : && GET_CODE (XEXP (value, 0)) == PLUS
8825 64552 : && REG_P (XEXP (XEXP (value, 0), 0))
8826 3856292 : && VIRTUAL_REGISTER_P (XEXP (XEXP (value, 0), 0)))
8827 : {
8828 1150 : rtx temp = expand_simple_binop (GET_MODE (value), code,
8829 : XEXP (XEXP (value, 0), 0), op2,
8830 : subtarget, 0, OPTAB_LIB_WIDEN);
8831 1150 : return expand_simple_binop (GET_MODE (value), code, temp,
8832 1150 : force_operand (XEXP (XEXP (value,
8833 : 0), 1), 0),
8834 1150 : target, 0, OPTAB_LIB_WIDEN);
8835 : }
8836 :
8837 3847795 : op1 = force_operand (XEXP (value, 0), subtarget);
8838 3847795 : op2 = force_operand (op2, NULL_RTX);
8839 3847795 : switch (code)
8840 : {
8841 96974 : case MULT:
8842 96974 : return expand_mult (GET_MODE (value), op1, op2, target, 1);
8843 222 : case DIV:
8844 222 : if (!INTEGRAL_MODE_P (GET_MODE (value)))
8845 222 : return expand_simple_binop (GET_MODE (value), code, op1, op2,
8846 222 : target, 1, OPTAB_LIB_WIDEN);
8847 : else
8848 0 : return expand_divmod (0,
8849 : FLOAT_MODE_P (GET_MODE (value))
8850 : ? RDIV_EXPR : TRUNC_DIV_EXPR,
8851 0 : GET_MODE (value), op1, op2, target, 0);
8852 0 : case MOD:
8853 0 : return expand_divmod (1, TRUNC_MOD_EXPR, GET_MODE (value), op1, op2,
8854 0 : target, 0);
8855 348 : case UDIV:
8856 348 : return expand_divmod (0, TRUNC_DIV_EXPR, GET_MODE (value), op1, op2,
8857 348 : target, 1);
8858 0 : case UMOD:
8859 0 : return expand_divmod (1, TRUNC_MOD_EXPR, GET_MODE (value), op1, op2,
8860 0 : target, 1);
8861 42 : case ASHIFTRT:
8862 42 : return expand_simple_binop (GET_MODE (value), code, op1, op2,
8863 42 : target, 0, OPTAB_LIB_WIDEN);
8864 3750209 : default:
8865 3750209 : return expand_simple_binop (GET_MODE (value), code, op1, op2,
8866 3750209 : target, 1, OPTAB_LIB_WIDEN);
8867 : }
8868 : }
8869 27775032 : if (UNARY_P (value))
8870 : {
8871 21755 : if (!target)
8872 11805 : target = gen_reg_rtx (GET_MODE (value));
8873 : /* FIX or UNSIGNED_FIX with integral mode has unspecified rounding,
8874 : while FIX with floating point mode rounds toward zero. So, some
8875 : targets use expressions like (fix:SI (fix:DF (reg:DF ...)))
8876 : to express rounding toward zero during the conversion to int.
8877 : expand_fix isn't able to handle that, it can only handle
8878 : FIX/UNSIGNED_FIX from floating point mode to integral one. */
8879 21755 : if ((code == FIX || code == UNSIGNED_FIX)
8880 8 : && GET_CODE (XEXP (value, 0)) == FIX
8881 0 : && (GET_MODE (XEXP (value, 0))
8882 0 : == GET_MODE (XEXP (XEXP (value, 0), 0))))
8883 0 : op1 = force_operand (XEXP (XEXP (value, 0), 0), NULL_RTX);
8884 : else
8885 21755 : op1 = force_operand (XEXP (value, 0), NULL_RTX);
8886 21755 : switch (code)
8887 : {
8888 10491 : case ZERO_EXTEND:
8889 10491 : case SIGN_EXTEND:
8890 10491 : case TRUNCATE:
8891 10491 : case FLOAT_EXTEND:
8892 10491 : case FLOAT_TRUNCATE:
8893 10491 : convert_move (target, op1, code == ZERO_EXTEND);
8894 10491 : return target;
8895 :
8896 8 : case FIX:
8897 8 : case UNSIGNED_FIX:
8898 8 : expand_fix (target, op1, code == UNSIGNED_FIX);
8899 8 : return target;
8900 :
8901 48 : case FLOAT:
8902 48 : case UNSIGNED_FLOAT:
8903 48 : expand_float (target, op1, code == UNSIGNED_FLOAT);
8904 48 : return target;
8905 :
8906 11208 : default:
8907 11208 : return expand_simple_unop (GET_MODE (value), code, op1, target, 0);
8908 : }
8909 : }
8910 :
8911 : #ifdef INSN_SCHEDULING
8912 : /* On machines that have insn scheduling, we want all memory reference to be
8913 : explicit, so we need to deal with such paradoxical SUBREGs. */
8914 27753277 : if (paradoxical_subreg_p (value) && MEM_P (SUBREG_REG (value)))
8915 0 : value
8916 0 : = simplify_gen_subreg (GET_MODE (value),
8917 0 : force_reg (GET_MODE (SUBREG_REG (value)),
8918 : force_operand (SUBREG_REG (value),
8919 : NULL_RTX)),
8920 : GET_MODE (SUBREG_REG (value)),
8921 0 : SUBREG_BYTE (value));
8922 : #endif
8923 :
8924 : return value;
8925 : }
8926 :
8927 : /* Subroutine of expand_expr: return true iff there is no way that
8928 : EXP can reference X, which is being modified. TOP_P is nonzero if this
8929 : call is going to be used to determine whether we need a temporary
8930 : for EXP, as opposed to a recursive call to this function.
8931 :
8932 : It is always safe for this routine to return false since it merely
8933 : searches for optimization opportunities. */
8934 :
8935 : bool
8936 8938871 : safe_from_p (const_rtx x, tree exp, int top_p)
8937 : {
8938 8938887 : rtx exp_rtl = 0;
8939 8938887 : int i, nops;
8940 :
8941 8938887 : if (x == 0
8942 : /* If EXP has varying size, we MUST use a target since we currently
8943 : have no way of allocating temporaries of variable size
8944 : (except for arrays that have TYPE_ARRAY_MAX_SIZE set).
8945 : So we assume here that something at a higher level has prevented a
8946 : clash. This is somewhat bogus, but the best we can do. Only
8947 : do this when X is BLKmode and when we are at the top level. */
8948 2025958 : || (top_p && TREE_TYPE (exp) != 0 && COMPLETE_TYPE_P (TREE_TYPE (exp))
8949 1888560 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (exp))) != INTEGER_CST
8950 0 : && (TREE_CODE (TREE_TYPE (exp)) != ARRAY_TYPE
8951 0 : || TYPE_ARRAY_MAX_SIZE (TREE_TYPE (exp)) == NULL_TREE
8952 0 : || TREE_CODE (TYPE_ARRAY_MAX_SIZE (TREE_TYPE (exp)))
8953 : != INTEGER_CST)
8954 0 : && GET_MODE (x) == BLKmode)
8955 : /* If X is in the outgoing argument area, it is always safe. */
8956 10964845 : || (MEM_P (x)
8957 180978 : && (XEXP (x, 0) == virtual_outgoing_args_rtx
8958 180978 : || (GET_CODE (XEXP (x, 0)) == PLUS
8959 132926 : && XEXP (XEXP (x, 0), 0) == virtual_outgoing_args_rtx))))
8960 : return true;
8961 :
8962 : /* If this is a subreg of a hard register, declare it unsafe, otherwise,
8963 : find the underlying pseudo. */
8964 2025958 : if (GET_CODE (x) == SUBREG)
8965 : {
8966 0 : x = SUBREG_REG (x);
8967 0 : if (REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER)
8968 : return false;
8969 : }
8970 :
8971 : /* Now look at our tree code and possibly recurse. */
8972 2025958 : switch (TREE_CODE_CLASS (TREE_CODE (exp)))
8973 : {
8974 600 : case tcc_declaration:
8975 600 : exp_rtl = DECL_RTL_IF_SET (exp);
8976 : break;
8977 :
8978 : case tcc_constant:
8979 : return true;
8980 :
8981 878427 : case tcc_exceptional:
8982 878427 : if (TREE_CODE (exp) == TREE_LIST)
8983 : {
8984 0 : while (1)
8985 : {
8986 0 : if (TREE_VALUE (exp) && !safe_from_p (x, TREE_VALUE (exp), 0))
8987 : return false;
8988 0 : exp = TREE_CHAIN (exp);
8989 0 : if (!exp)
8990 : return true;
8991 0 : if (TREE_CODE (exp) != TREE_LIST)
8992 : return safe_from_p (x, exp, 0);
8993 : }
8994 : }
8995 878427 : else if (TREE_CODE (exp) == CONSTRUCTOR)
8996 : {
8997 : constructor_elt *ce;
8998 : unsigned HOST_WIDE_INT idx;
8999 :
9000 8230667 : FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (exp), idx, ce)
9001 5825 : if ((ce->index != NULL_TREE && !safe_from_p (x, ce->index, 0))
9002 136973 : || !safe_from_p (x, ce->value, 0))
9003 : return false;
9004 : return true;
9005 : }
9006 733735 : else if (TREE_CODE (exp) == ERROR_MARK)
9007 : return true; /* An already-visited SAVE_EXPR? */
9008 : else
9009 : return false;
9010 :
9011 0 : case tcc_statement:
9012 : /* The only case we look at here is the DECL_INITIAL inside a
9013 : DECL_EXPR. */
9014 0 : return (TREE_CODE (exp) != DECL_EXPR
9015 0 : || TREE_CODE (DECL_EXPR_DECL (exp)) != VAR_DECL
9016 0 : || !DECL_INITIAL (DECL_EXPR_DECL (exp))
9017 0 : || safe_from_p (x, DECL_INITIAL (DECL_EXPR_DECL (exp)), 0));
9018 :
9019 0 : case tcc_binary:
9020 0 : case tcc_comparison:
9021 0 : if (!safe_from_p (x, TREE_OPERAND (exp, 1), 0))
9022 : return false;
9023 : /* Fall through. */
9024 :
9025 16 : case tcc_unary:
9026 16 : return safe_from_p (x, TREE_OPERAND (exp, 0), 0);
9027 :
9028 471 : case tcc_expression:
9029 471 : case tcc_reference:
9030 471 : case tcc_vl_exp:
9031 : /* Now do code-specific tests. EXP_RTL is set to any rtx we find in
9032 : the expression. If it is set, we conflict iff we are that rtx or
9033 : both are in memory. Otherwise, we check all operands of the
9034 : expression recursively. */
9035 :
9036 471 : switch (TREE_CODE (exp))
9037 : {
9038 383 : case ADDR_EXPR:
9039 : /* If the operand is static or we are static, we can't conflict.
9040 : Likewise if we don't conflict with the operand at all. */
9041 383 : if (staticp (TREE_OPERAND (exp, 0))
9042 233 : || TREE_STATIC (exp)
9043 616 : || safe_from_p (x, TREE_OPERAND (exp, 0), 0))
9044 : return true;
9045 :
9046 : /* Otherwise, the only way this can conflict is if we are taking
9047 : the address of a DECL a that address if part of X, which is
9048 : very rare. */
9049 0 : exp = TREE_OPERAND (exp, 0);
9050 0 : if (DECL_P (exp))
9051 : {
9052 0 : if (!DECL_RTL_SET_P (exp)
9053 0 : || !MEM_P (DECL_RTL (exp)))
9054 : return false;
9055 : else
9056 0 : exp_rtl = XEXP (DECL_RTL (exp), 0);
9057 : }
9058 : break;
9059 :
9060 73 : case MEM_REF:
9061 73 : if (MEM_P (x)
9062 73 : && alias_sets_conflict_p (MEM_ALIAS_SET (x),
9063 : get_alias_set (exp)))
9064 : return false;
9065 : break;
9066 :
9067 0 : case CALL_EXPR:
9068 : /* Assume that the call will clobber all hard registers and
9069 : all of memory. */
9070 0 : if ((REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER)
9071 0 : || MEM_P (x))
9072 : return false;
9073 : break;
9074 :
9075 0 : case WITH_CLEANUP_EXPR:
9076 0 : case CLEANUP_POINT_EXPR:
9077 : /* Lowered by gimplify.cc. */
9078 0 : gcc_unreachable ();
9079 :
9080 0 : case SAVE_EXPR:
9081 0 : return safe_from_p (x, TREE_OPERAND (exp, 0), 0);
9082 :
9083 : default:
9084 : break;
9085 : }
9086 :
9087 : /* If we have an rtx, we do not need to scan our operands. */
9088 0 : if (exp_rtl)
9089 : break;
9090 :
9091 88 : nops = TREE_OPERAND_LENGTH (exp);
9092 371 : for (i = 0; i < nops; i++)
9093 195 : if (TREE_OPERAND (exp, i) != 0
9094 195 : && ! safe_from_p (x, TREE_OPERAND (exp, i), 0))
9095 : return false;
9096 :
9097 : break;
9098 :
9099 0 : case tcc_type:
9100 : /* Should never get a type here. */
9101 0 : gcc_unreachable ();
9102 : }
9103 :
9104 : /* If we have an rtl, find any enclosed object. Then see if we conflict
9105 : with it. */
9106 392 : if (exp_rtl)
9107 : {
9108 304 : if (GET_CODE (exp_rtl) == SUBREG)
9109 : {
9110 0 : exp_rtl = SUBREG_REG (exp_rtl);
9111 0 : if (REG_P (exp_rtl)
9112 0 : && REGNO (exp_rtl) < FIRST_PSEUDO_REGISTER)
9113 : return false;
9114 : }
9115 :
9116 : /* If the rtl is X, then it is not safe. Otherwise, it is unless both
9117 : are memory and they conflict. */
9118 304 : return ! (rtx_equal_p (x, exp_rtl)
9119 304 : || (MEM_P (x) && MEM_P (exp_rtl)
9120 4 : && true_dependence (exp_rtl, VOIDmode, x)));
9121 : }
9122 :
9123 : /* If we reach here, it is safe. */
9124 : return true;
9125 : }
9126 :
9127 :
9128 : /* Return the highest power of two that EXP is known to be a multiple of.
9129 : This is used in updating alignment of MEMs in array references. */
9130 :
9131 : unsigned HOST_WIDE_INT
9132 33425973 : highest_pow2_factor (const_tree exp)
9133 : {
9134 33425973 : unsigned HOST_WIDE_INT ret;
9135 33425973 : int trailing_zeros = tree_ctz (exp);
9136 33425973 : if (trailing_zeros >= HOST_BITS_PER_WIDE_INT)
9137 23464010 : return BIGGEST_ALIGNMENT;
9138 9961963 : ret = HOST_WIDE_INT_1U << trailing_zeros;
9139 19458262 : if (ret > BIGGEST_ALIGNMENT)
9140 6627173 : return BIGGEST_ALIGNMENT;
9141 : return ret;
9142 : }
9143 :
9144 : /* Similar, except that the alignment requirements of TARGET are
9145 : taken into account. Assume it is at least as aligned as its
9146 : type, unless it is a COMPONENT_REF in which case the layout of
9147 : the structure gives the alignment. */
9148 :
9149 : static unsigned HOST_WIDE_INT
9150 179408 : highest_pow2_factor_for_target (const_tree target, const_tree exp)
9151 : {
9152 179408 : unsigned HOST_WIDE_INT talign = target_align (target) / BITS_PER_UNIT;
9153 179408 : unsigned HOST_WIDE_INT factor = highest_pow2_factor (exp);
9154 :
9155 179408 : return MAX (factor, talign);
9156 : }
9157 :
9158 : /* Convert the tree comparison code TCODE to the rtl one where the
9159 : signedness is UNSIGNEDP. */
9160 :
9161 : enum rtx_code
9162 21181 : convert_tree_comp_to_rtx (enum tree_code tcode, int unsignedp)
9163 : {
9164 21181 : enum rtx_code code;
9165 21181 : switch (tcode)
9166 : {
9167 : case EQ_EXPR:
9168 : code = EQ;
9169 : break;
9170 2408 : case NE_EXPR:
9171 2408 : code = NE;
9172 2408 : break;
9173 4055 : case LT_EXPR:
9174 4055 : code = unsignedp ? LTU : LT;
9175 : break;
9176 2735 : case LE_EXPR:
9177 2735 : code = unsignedp ? LEU : LE;
9178 : break;
9179 3207 : case GT_EXPR:
9180 3207 : code = unsignedp ? GTU : GT;
9181 : break;
9182 4386 : case GE_EXPR:
9183 4386 : code = unsignedp ? GEU : GE;
9184 : break;
9185 31 : case UNORDERED_EXPR:
9186 31 : code = UNORDERED;
9187 31 : break;
9188 31 : case ORDERED_EXPR:
9189 31 : code = ORDERED;
9190 31 : break;
9191 10 : case UNLT_EXPR:
9192 10 : code = UNLT;
9193 10 : break;
9194 19 : case UNLE_EXPR:
9195 19 : code = UNLE;
9196 19 : break;
9197 10 : case UNGT_EXPR:
9198 10 : code = UNGT;
9199 10 : break;
9200 11 : case UNGE_EXPR:
9201 11 : code = UNGE;
9202 11 : break;
9203 4 : case UNEQ_EXPR:
9204 4 : code = UNEQ;
9205 4 : break;
9206 4 : case LTGT_EXPR:
9207 4 : code = LTGT;
9208 4 : break;
9209 :
9210 0 : default:
9211 0 : gcc_unreachable ();
9212 : }
9213 21181 : return code;
9214 : }
9215 :
9216 : /* Subroutine of expand_expr. Expand the two operands of a binary
9217 : expression EXP0 and EXP1 placing the results in OP0 and OP1.
9218 : The value may be stored in TARGET if TARGET is nonzero. The
9219 : MODIFIER argument is as documented by expand_expr. */
9220 :
9221 : void
9222 8321113 : expand_operands (tree exp0, tree exp1, rtx target, rtx *op0, rtx *op1,
9223 : enum expand_modifier modifier)
9224 : {
9225 8321113 : if (! safe_from_p (target, exp1, 1))
9226 609583 : target = 0;
9227 8321113 : if (operand_equal_p (exp0, exp1, 0))
9228 : {
9229 47886 : *op0 = expand_expr (exp0, target, VOIDmode, modifier);
9230 47886 : *op1 = copy_rtx (*op0);
9231 : }
9232 : else
9233 : {
9234 8273227 : *op0 = expand_expr (exp0, target, VOIDmode, modifier);
9235 8273227 : *op1 = expand_expr (exp1, NULL_RTX, VOIDmode, modifier);
9236 : }
9237 8321113 : }
9238 :
9239 :
9240 : /* Return a MEM that contains constant EXP. DEFER is as for
9241 : output_constant_def and MODIFIER is as for expand_expr. */
9242 :
9243 : static rtx
9244 2985765 : expand_expr_constant (tree exp, int defer, enum expand_modifier modifier)
9245 : {
9246 2985765 : rtx mem;
9247 :
9248 2985765 : mem = output_constant_def (exp, defer);
9249 2985765 : if (modifier != EXPAND_INITIALIZER)
9250 1908257 : mem = use_anchored_address (mem);
9251 2985765 : return mem;
9252 : }
9253 :
9254 : /* A subroutine of expand_expr_addr_expr. Evaluate the address of EXP.
9255 : The TARGET, TMODE and MODIFIER arguments are as for expand_expr. */
9256 :
9257 : static rtx
9258 14586912 : expand_expr_addr_expr_1 (tree exp, rtx target, scalar_int_mode tmode,
9259 : enum expand_modifier modifier, addr_space_t as)
9260 : {
9261 14586912 : rtx result, subtarget;
9262 14586912 : tree inner, offset;
9263 14586912 : poly_int64 bitsize, bitpos;
9264 14586912 : int unsignedp, reversep, volatilep = 0;
9265 14586912 : machine_mode mode1;
9266 :
9267 : /* If we are taking the address of a constant and are at the top level,
9268 : we have to use output_constant_def since we can't call force_const_mem
9269 : at top level. */
9270 : /* ??? This should be considered a front-end bug. We should not be
9271 : generating ADDR_EXPR of something that isn't an LVALUE. The only
9272 : exception here is STRING_CST. */
9273 14586912 : if (CONSTANT_CLASS_P (exp))
9274 : {
9275 2722717 : result = XEXP (expand_expr_constant (exp, 0, modifier), 0);
9276 2722717 : if (modifier < EXPAND_SUM)
9277 1816598 : result = force_operand (result, target);
9278 : return result;
9279 : }
9280 :
9281 : /* Everything must be something allowed by is_gimple_addressable. */
9282 11864195 : switch (TREE_CODE (exp))
9283 : {
9284 46 : case INDIRECT_REF:
9285 : /* This case will happen via recursion for &a->b. */
9286 46 : return expand_expr (TREE_OPERAND (exp, 0), target, tmode, modifier);
9287 :
9288 579251 : case MEM_REF:
9289 579251 : {
9290 579251 : tree tem = TREE_OPERAND (exp, 0);
9291 579251 : if (!integer_zerop (TREE_OPERAND (exp, 1)))
9292 304790 : tem = fold_build_pointer_plus (tem, TREE_OPERAND (exp, 1));
9293 579251 : return expand_expr (tem, target, tmode, modifier);
9294 : }
9295 :
9296 1270 : case TARGET_MEM_REF:
9297 1270 : return addr_for_mem_ref (exp, as, true);
9298 :
9299 60127 : case CONST_DECL:
9300 : /* Expand the initializer like constants above. */
9301 60127 : result = XEXP (expand_expr_constant (DECL_INITIAL (exp),
9302 : 0, modifier), 0);
9303 60127 : if (modifier < EXPAND_SUM)
9304 60121 : result = force_operand (result, target);
9305 : return result;
9306 :
9307 209 : case REALPART_EXPR:
9308 : /* The real part of the complex number is always first, therefore
9309 : the address is the same as the address of the parent object. */
9310 209 : offset = 0;
9311 209 : bitpos = 0;
9312 209 : inner = TREE_OPERAND (exp, 0);
9313 209 : break;
9314 :
9315 126 : case IMAGPART_EXPR:
9316 : /* The imaginary part of the complex number is always second.
9317 : The expression is therefore always offset by the size of the
9318 : scalar type. */
9319 126 : offset = 0;
9320 252 : bitpos = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (exp)));
9321 126 : inner = TREE_OPERAND (exp, 0);
9322 126 : break;
9323 :
9324 13 : case COMPOUND_LITERAL_EXPR:
9325 : /* Allow COMPOUND_LITERAL_EXPR in initializers or coming from
9326 : initializers, if e.g. rtl_for_decl_init is called on DECL_INITIAL
9327 : with COMPOUND_LITERAL_EXPRs in it, or ARRAY_REF on a const static
9328 : array with address of COMPOUND_LITERAL_EXPR in DECL_INITIAL;
9329 : the initializers aren't gimplified. */
9330 13 : if (COMPOUND_LITERAL_EXPR_DECL (exp)
9331 13 : && is_global_var (COMPOUND_LITERAL_EXPR_DECL (exp)))
9332 13 : return expand_expr_addr_expr_1 (COMPOUND_LITERAL_EXPR_DECL (exp),
9333 13 : target, tmode, modifier, as);
9334 : /* FALLTHRU */
9335 11223153 : default:
9336 : /* If the object is a DECL, then expand it for its rtl. Don't bypass
9337 : expand_expr, as that can have various side effects; LABEL_DECLs for
9338 : example, may not have their DECL_RTL set yet. Expand the rtl of
9339 : CONSTRUCTORs too, which should yield a memory reference for the
9340 : constructor's contents. Assume language specific tree nodes can
9341 : be expanded in some interesting way. */
9342 11223153 : gcc_assert (TREE_CODE (exp) < LAST_AND_UNUSED_TREE_CODE);
9343 11223153 : if (DECL_P (exp)
9344 1338380 : || TREE_CODE (exp) == CONSTRUCTOR
9345 1338380 : || TREE_CODE (exp) == COMPOUND_LITERAL_EXPR)
9346 : {
9347 16253696 : result = expand_expr (exp, target, tmode,
9348 : modifier == EXPAND_INITIALIZER
9349 : ? EXPAND_INITIALIZER : EXPAND_CONST_ADDRESS);
9350 :
9351 : /* If the DECL isn't in memory, then the DECL wasn't properly
9352 : marked TREE_ADDRESSABLE, which will be either a front-end
9353 : or a tree optimizer bug. */
9354 :
9355 9884773 : gcc_assert (MEM_P (result));
9356 9884771 : result = XEXP (result, 0);
9357 :
9358 : /* ??? Is this needed anymore? */
9359 9884771 : if (DECL_P (exp))
9360 9884771 : TREE_USED (exp) = 1;
9361 :
9362 9884771 : if (modifier != EXPAND_INITIALIZER
9363 : && modifier != EXPAND_CONST_ADDRESS
9364 9884771 : && modifier != EXPAND_SUM)
9365 4317183 : result = force_operand (result, target);
9366 : return result;
9367 : }
9368 :
9369 : /* Pass FALSE as the last argument to get_inner_reference although
9370 : we are expanding to RTL. The rationale is that we know how to
9371 : handle "aligning nodes" here: we can just bypass them because
9372 : they won't change the final object whose address will be returned
9373 : (they actually exist only for that purpose). */
9374 1338380 : inner = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
9375 : &unsignedp, &reversep, &volatilep);
9376 1338380 : break;
9377 : }
9378 :
9379 : /* We must have made progress. */
9380 1338715 : gcc_assert (inner != exp);
9381 :
9382 1338715 : subtarget = offset || maybe_ne (bitpos, 0) ? NULL_RTX : target;
9383 : /* For VIEW_CONVERT_EXPR, where the outer alignment is bigger than
9384 : inner alignment, force the inner to be sufficiently aligned. */
9385 1338715 : if (CONSTANT_CLASS_P (inner)
9386 1338715 : && TYPE_ALIGN (TREE_TYPE (inner)) < TYPE_ALIGN (TREE_TYPE (exp)))
9387 : {
9388 0 : inner = copy_node (inner);
9389 0 : TREE_TYPE (inner) = copy_node (TREE_TYPE (inner));
9390 0 : SET_TYPE_ALIGN (TREE_TYPE (inner), TYPE_ALIGN (TREE_TYPE (exp)));
9391 0 : TYPE_USER_ALIGN (TREE_TYPE (inner)) = 1;
9392 : }
9393 1338715 : result = expand_expr_addr_expr_1 (inner, subtarget, tmode, modifier, as);
9394 :
9395 1338715 : if (offset)
9396 : {
9397 55417 : rtx tmp;
9398 :
9399 55417 : if (modifier != EXPAND_NORMAL)
9400 1662 : result = force_operand (result, NULL);
9401 57079 : tmp = expand_expr (offset, NULL_RTX, tmode,
9402 : modifier == EXPAND_INITIALIZER
9403 : ? EXPAND_INITIALIZER : EXPAND_NORMAL);
9404 :
9405 : /* expand_expr is allowed to return an object in a mode other
9406 : than TMODE. If it did, we need to convert. */
9407 55417 : if (GET_MODE (tmp) != VOIDmode && tmode != GET_MODE (tmp))
9408 0 : tmp = convert_modes (tmode, GET_MODE (tmp),
9409 0 : tmp, TYPE_UNSIGNED (TREE_TYPE (offset)));
9410 55417 : result = convert_memory_address_addr_space (tmode, result, as);
9411 55417 : tmp = convert_memory_address_addr_space (tmode, tmp, as);
9412 :
9413 55417 : if (modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER)
9414 1662 : result = simplify_gen_binary (PLUS, tmode, result, tmp);
9415 : else
9416 : {
9417 53755 : subtarget = maybe_ne (bitpos, 0) ? NULL_RTX : target;
9418 53755 : result = expand_simple_binop (tmode, PLUS, result, tmp, subtarget,
9419 : 1, OPTAB_LIB_WIDEN);
9420 : }
9421 : }
9422 :
9423 1338715 : if (maybe_ne (bitpos, 0))
9424 : {
9425 : /* Someone beforehand should have rejected taking the address
9426 : of an object that isn't byte-aligned. */
9427 634388 : poly_int64 bytepos = exact_div (bitpos, BITS_PER_UNIT);
9428 634388 : result = convert_memory_address_addr_space (tmode, result, as);
9429 634388 : result = plus_constant (tmode, result, bytepos);
9430 634388 : if (modifier < EXPAND_SUM)
9431 596883 : result = force_operand (result, target);
9432 : }
9433 :
9434 : return result;
9435 : }
9436 :
9437 : /* A subroutine of expand_expr. Evaluate EXP, which is an ADDR_EXPR.
9438 : The TARGET, TMODE and MODIFIER arguments are as for expand_expr. */
9439 :
9440 : static rtx
9441 13248182 : expand_expr_addr_expr (tree exp, rtx target, machine_mode tmode,
9442 : enum expand_modifier modifier)
9443 : {
9444 13248182 : addr_space_t as = ADDR_SPACE_GENERIC;
9445 13248182 : scalar_int_mode address_mode = Pmode;
9446 13248182 : scalar_int_mode pointer_mode = ptr_mode;
9447 13248182 : machine_mode rmode;
9448 13248182 : rtx result;
9449 :
9450 : /* Target mode of VOIDmode says "whatever's natural". */
9451 13248182 : if (tmode == VOIDmode)
9452 11274860 : tmode = TYPE_MODE (TREE_TYPE (exp));
9453 :
9454 13248182 : if (POINTER_TYPE_P (TREE_TYPE (exp)))
9455 : {
9456 13248182 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)));
9457 13248182 : address_mode = targetm.addr_space.address_mode (as);
9458 13248182 : pointer_mode = targetm.addr_space.pointer_mode (as);
9459 : }
9460 :
9461 : /* We can get called with some Weird Things if the user does silliness
9462 : like "(short) &a". In that case, convert_memory_address won't do
9463 : the right thing, so ignore the given target mode. */
9464 13248182 : scalar_int_mode new_tmode = (tmode == pointer_mode
9465 13248182 : ? pointer_mode
9466 13248182 : : address_mode);
9467 :
9468 13248182 : result = expand_expr_addr_expr_1 (TREE_OPERAND (exp, 0), target,
9469 : new_tmode, modifier, as);
9470 :
9471 : /* Despite expand_expr claims concerning ignoring TMODE when not
9472 : strictly convenient, stuff breaks if we don't honor it. Note
9473 : that combined with the above, we only do this for pointer modes. */
9474 13248180 : rmode = GET_MODE (result);
9475 13248180 : if (rmode == VOIDmode)
9476 6 : rmode = new_tmode;
9477 13248180 : if (rmode != new_tmode)
9478 74 : result = convert_memory_address_addr_space (new_tmode, result, as);
9479 :
9480 13248180 : return result;
9481 : }
9482 :
9483 : /* Generate code for computing CONSTRUCTOR EXP.
9484 : An rtx for the computed value is returned. If AVOID_TEMP_MEM
9485 : is TRUE, instead of creating a temporary variable in memory
9486 : NULL is returned and the caller needs to handle it differently. */
9487 :
9488 : static rtx
9489 179905 : expand_constructor (tree exp, rtx target, enum expand_modifier modifier,
9490 : bool avoid_temp_mem)
9491 : {
9492 179905 : tree type = TREE_TYPE (exp);
9493 179905 : machine_mode mode = TYPE_MODE (type);
9494 :
9495 : /* Try to avoid creating a temporary at all. This is possible
9496 : if all of the initializer is zero.
9497 : FIXME: try to handle all [0..255] initializers we can handle
9498 : with memset. */
9499 179905 : if (TREE_STATIC (exp)
9500 2237 : && !TREE_ADDRESSABLE (exp)
9501 2237 : && target != 0 && mode == BLKmode
9502 181013 : && all_zeros_p (exp))
9503 : {
9504 1099 : clear_storage (target, expr_size (exp), BLOCK_OP_NORMAL);
9505 1099 : return target;
9506 : }
9507 :
9508 : /* All elts simple constants => refer to a constant in memory. But
9509 : if this is a non-BLKmode mode, let it store a field at a time
9510 : since that should make a CONST_INT, CONST_WIDE_INT or
9511 : CONST_DOUBLE when we fold. Likewise, if we have a target we can
9512 : use, it is best to store directly into the target unless the type
9513 : is large enough that memcpy will be used. If we are making an
9514 : initializer and all operands are constant, put it in memory as
9515 : well.
9516 :
9517 : FIXME: Avoid trying to fill vector constructors piece-meal.
9518 : Output them with output_constant_def below unless we're sure
9519 : they're zeros. This should go away when vector initializers
9520 : are treated like VECTOR_CST instead of arrays. */
9521 178806 : if ((TREE_STATIC (exp)
9522 1138 : && ((mode == BLKmode
9523 46 : && ! (target != 0 && safe_from_p (target, exp, 1)))
9524 1101 : || TREE_ADDRESSABLE (exp)
9525 1101 : || (tree_fits_uhwi_p (TYPE_SIZE_UNIT (type))
9526 1101 : && (! can_move_by_pieces
9527 1101 : (tree_to_uhwi (TYPE_SIZE_UNIT (type)),
9528 1101 : TYPE_ALIGN (type)))
9529 4 : && ! mostly_zeros_p (exp))))
9530 179905 : || ((modifier == EXPAND_INITIALIZER || modifier == EXPAND_CONST_ADDRESS)
9531 0 : && TREE_CONSTANT (exp)))
9532 : {
9533 39 : rtx constructor;
9534 :
9535 39 : if (avoid_temp_mem)
9536 : return NULL_RTX;
9537 :
9538 37 : constructor = expand_expr_constant (exp, 1, modifier);
9539 :
9540 37 : if (modifier != EXPAND_CONST_ADDRESS
9541 : && modifier != EXPAND_INITIALIZER
9542 37 : && modifier != EXPAND_SUM)
9543 37 : constructor = validize_mem (constructor);
9544 :
9545 : return constructor;
9546 : }
9547 :
9548 : /* If the CTOR is available in static storage and not mostly
9549 : zeros and we can move it by pieces prefer to do so since
9550 : that's usually more efficient than performing a series of
9551 : stores from immediates. */
9552 178767 : if (avoid_temp_mem
9553 97 : && TREE_STATIC (exp)
9554 38 : && TREE_CONSTANT (exp)
9555 38 : && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type))
9556 38 : && can_move_by_pieces (tree_to_uhwi (TYPE_SIZE_UNIT (type)),
9557 38 : TYPE_ALIGN (type))
9558 178804 : && ! mostly_zeros_p (exp))
9559 : return NULL_RTX;
9560 :
9561 : /* Handle calls that pass values in multiple non-contiguous
9562 : locations. The Irix 6 ABI has examples of this. */
9563 144639 : if (target == 0 || ! safe_from_p (target, exp, 1)
9564 37723 : || GET_CODE (target) == PARALLEL || modifier == EXPAND_STACK_PARM
9565 : /* Also make a temporary if the store is to volatile memory, to
9566 : avoid individual accesses to aggregate members. */
9567 216448 : || (GET_CODE (target) == MEM
9568 33583 : && MEM_VOLATILE_P (target)
9569 134 : && !TREE_ADDRESSABLE (TREE_TYPE (exp))))
9570 : {
9571 141154 : if (avoid_temp_mem)
9572 : return NULL_RTX;
9573 :
9574 141143 : target = assign_temp (type, TREE_ADDRESSABLE (exp), 1);
9575 : }
9576 :
9577 178726 : store_constructor (exp, target, 0, int_expr_size (exp), false);
9578 178726 : return target;
9579 : }
9580 :
9581 :
9582 : /* expand_expr: generate code for computing expression EXP.
9583 : An rtx for the computed value is returned. The value is never null.
9584 : In the case of a void EXP, const0_rtx is returned.
9585 :
9586 : The value may be stored in TARGET if TARGET is nonzero.
9587 : TARGET is just a suggestion; callers must assume that
9588 : the rtx returned may not be the same as TARGET.
9589 :
9590 : If TARGET is CONST0_RTX, it means that the value will be ignored.
9591 :
9592 : If TMODE is not VOIDmode, it suggests generating the
9593 : result in mode TMODE. But this is done only when convenient.
9594 : Otherwise, TMODE is ignored and the value generated in its natural mode.
9595 : TMODE is just a suggestion; callers must assume that
9596 : the rtx returned may not have mode TMODE.
9597 :
9598 : Note that TARGET may have neither TMODE nor MODE. In that case, it
9599 : probably will not be used.
9600 :
9601 : If MODIFIER is EXPAND_SUM then when EXP is an addition
9602 : we can return an rtx of the form (MULT (REG ...) (CONST_INT ...))
9603 : or a nest of (PLUS ...) and (MINUS ...) where the terms are
9604 : products as above, or REG or MEM, or constant.
9605 : Ordinarily in such cases we would output mul or add instructions
9606 : and then return a pseudo reg containing the sum.
9607 :
9608 : EXPAND_INITIALIZER is much like EXPAND_SUM except that
9609 : it also marks a label as absolutely required (it can't be dead).
9610 : It also makes a ZERO_EXTEND or SIGN_EXTEND instead of emitting extend insns.
9611 : This is used for outputting expressions used in initializers.
9612 :
9613 : EXPAND_CONST_ADDRESS says that it is okay to return a MEM
9614 : with a constant address even if that address is not normally legitimate.
9615 : EXPAND_INITIALIZER and EXPAND_SUM also have this effect.
9616 :
9617 : EXPAND_STACK_PARM is used when expanding to a TARGET on the stack for
9618 : a call parameter. Such targets require special care as we haven't yet
9619 : marked TARGET so that it's safe from being trashed by libcalls. We
9620 : don't want to use TARGET for anything but the final result;
9621 : Intermediate values must go elsewhere. Additionally, calls to
9622 : emit_block_move will be flagged with BLOCK_OP_CALL_PARM.
9623 :
9624 : If EXP is a VAR_DECL whose DECL_RTL was a MEM with an invalid
9625 : address, and ALT_RTL is non-NULL, then *ALT_RTL is set to the
9626 : DECL_RTL of the VAR_DECL. *ALT_RTL is also set if EXP is a
9627 : COMPOUND_EXPR whose second argument is such a VAR_DECL, and so on
9628 : recursively.
9629 : If the result can be stored at TARGET, and ALT_RTL is non-NULL,
9630 : then *ALT_RTL is set to TARGET (before legitimziation).
9631 :
9632 : If INNER_REFERENCE_P is true, we are expanding an inner reference.
9633 : In this case, we don't adjust a returned MEM rtx that wouldn't be
9634 : sufficiently aligned for its mode; instead, it's up to the caller
9635 : to deal with it afterwards. This is used to make sure that unaligned
9636 : base objects for which out-of-bounds accesses are supported, for
9637 : example record types with trailing arrays, aren't realigned behind
9638 : the back of the caller.
9639 : The normal operating mode is to pass FALSE for this parameter. */
9640 :
9641 : rtx
9642 159541205 : expand_expr_real (tree exp, rtx target, machine_mode tmode,
9643 : enum expand_modifier modifier, rtx *alt_rtl,
9644 : bool inner_reference_p)
9645 : {
9646 159541205 : rtx ret;
9647 :
9648 : /* Handle ERROR_MARK before anybody tries to access its type. */
9649 159541205 : if (TREE_CODE (exp) == ERROR_MARK
9650 159541205 : || (TREE_CODE (TREE_TYPE (exp)) == ERROR_MARK))
9651 : {
9652 0 : ret = CONST0_RTX (tmode);
9653 0 : return ret ? ret : const0_rtx;
9654 : }
9655 :
9656 159541205 : ret = expand_expr_real_1 (exp, target, tmode, modifier, alt_rtl,
9657 : inner_reference_p);
9658 159541205 : return ret;
9659 : }
9660 :
9661 : /* Try to expand the conditional expression which is represented by
9662 : TREEOP0 ? TREEOP1 : TREEOP2 using conditional moves. If it succeeds
9663 : return the rtl reg which represents the result. Otherwise return
9664 : NULL_RTX. */
9665 :
9666 : static rtx
9667 18096 : expand_cond_expr_using_cmove (tree treeop0 ATTRIBUTE_UNUSED,
9668 : tree treeop1 ATTRIBUTE_UNUSED,
9669 : tree treeop2 ATTRIBUTE_UNUSED)
9670 : {
9671 18096 : rtx insn;
9672 18096 : rtx op00, op01, op1, op2;
9673 18096 : enum rtx_code comparison_code;
9674 18096 : machine_mode comparison_mode;
9675 18096 : gimple *srcstmt;
9676 18096 : rtx temp;
9677 18096 : tree type = TREE_TYPE (treeop1);
9678 18096 : int unsignedp = TYPE_UNSIGNED (type);
9679 18096 : machine_mode mode = TYPE_MODE (type);
9680 18096 : machine_mode orig_mode = mode;
9681 18096 : static bool expanding_cond_expr_using_cmove = false;
9682 :
9683 : /* Conditional move expansion can end up TERing two operands which,
9684 : when recursively hitting conditional expressions can result in
9685 : exponential behavior if the cmove expansion ultimatively fails.
9686 : It's hardly profitable to TER a cmove into a cmove so avoid doing
9687 : that by failing early if we end up recursing. */
9688 18096 : if (expanding_cond_expr_using_cmove)
9689 : return NULL_RTX;
9690 :
9691 : /* If we cannot do a conditional move on the mode, try doing it
9692 : with the promoted mode. */
9693 17110 : if (!can_conditionally_move_p (mode))
9694 : {
9695 287 : mode = promote_mode (type, mode, &unsignedp);
9696 287 : if (!can_conditionally_move_p (mode))
9697 : return NULL_RTX;
9698 0 : temp = assign_temp (type, 0, 0); /* Use promoted mode for temp. */
9699 : }
9700 : else
9701 16823 : temp = assign_temp (type, 0, 1);
9702 :
9703 16823 : expanding_cond_expr_using_cmove = true;
9704 16823 : start_sequence ();
9705 16823 : expand_operands (treeop1, treeop2,
9706 : mode == orig_mode ? temp : NULL_RTX, &op1, &op2,
9707 : EXPAND_NORMAL);
9708 :
9709 16823 : if (TREE_CODE (treeop0) == SSA_NAME
9710 16660 : && (srcstmt = get_def_for_expr_class (treeop0, tcc_comparison))
9711 31250 : && !VECTOR_TYPE_P (TREE_TYPE (gimple_assign_rhs1 (srcstmt))))
9712 : {
9713 14424 : type = TREE_TYPE (gimple_assign_rhs1 (srcstmt));
9714 14424 : enum tree_code cmpcode = gimple_assign_rhs_code (srcstmt);
9715 14424 : op00 = expand_normal (gimple_assign_rhs1 (srcstmt));
9716 14424 : op01 = expand_normal (gimple_assign_rhs2 (srcstmt));
9717 14424 : comparison_mode = TYPE_MODE (type);
9718 14424 : unsignedp = TYPE_UNSIGNED (type);
9719 14424 : comparison_code = convert_tree_comp_to_rtx (cmpcode, unsignedp);
9720 : }
9721 2399 : else if (COMPARISON_CLASS_P (treeop0)
9722 2399 : && !VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (treeop0, 0))))
9723 : {
9724 163 : type = TREE_TYPE (TREE_OPERAND (treeop0, 0));
9725 163 : enum tree_code cmpcode = TREE_CODE (treeop0);
9726 163 : op00 = expand_normal (TREE_OPERAND (treeop0, 0));
9727 163 : op01 = expand_normal (TREE_OPERAND (treeop0, 1));
9728 163 : unsignedp = TYPE_UNSIGNED (type);
9729 163 : comparison_mode = TYPE_MODE (type);
9730 163 : comparison_code = convert_tree_comp_to_rtx (cmpcode, unsignedp);
9731 : }
9732 : else
9733 : {
9734 2236 : op00 = expand_normal (treeop0);
9735 2236 : op01 = const0_rtx;
9736 2236 : comparison_code = NE;
9737 2236 : comparison_mode = GET_MODE (op00);
9738 2236 : if (comparison_mode == VOIDmode)
9739 0 : comparison_mode = TYPE_MODE (TREE_TYPE (treeop0));
9740 : }
9741 16823 : expanding_cond_expr_using_cmove = false;
9742 :
9743 16823 : if (GET_MODE (op1) != mode)
9744 2369 : op1 = gen_lowpart (mode, op1);
9745 :
9746 16823 : if (GET_MODE (op2) != mode)
9747 9067 : op2 = gen_lowpart (mode, op2);
9748 :
9749 : /* Try to emit the conditional move. */
9750 16823 : insn = emit_conditional_move (temp,
9751 : { comparison_code, op00, op01,
9752 : comparison_mode },
9753 : op1, op2, mode,
9754 : unsignedp);
9755 :
9756 : /* If we could do the conditional move, emit the sequence,
9757 : and return. */
9758 16823 : if (insn)
9759 : {
9760 14257 : rtx_insn *seq = end_sequence ();
9761 14257 : emit_insn (seq);
9762 14257 : return convert_modes (orig_mode, mode, temp, 0);
9763 : }
9764 :
9765 : /* Otherwise discard the sequence and fall back to code with
9766 : branches. */
9767 2566 : end_sequence ();
9768 2566 : return NULL_RTX;
9769 : }
9770 :
9771 : /* A helper function for expand_expr_real_2 to be used with a
9772 : misaligned mem_ref TEMP. Assume an unsigned type if UNSIGNEDP
9773 : is nonzero, with alignment ALIGN in bits.
9774 : Store the value at TARGET if possible (if TARGET is nonzero).
9775 : Regardless of TARGET, we return the rtx for where the value is placed.
9776 : If the result can be stored at TARGET, and ALT_RTL is non-NULL,
9777 : then *ALT_RTL is set to TARGET (before legitimziation). */
9778 :
9779 : static rtx
9780 211467 : expand_misaligned_mem_ref (rtx temp, machine_mode mode, int unsignedp,
9781 : unsigned int align, rtx target, rtx *alt_rtl)
9782 : {
9783 211467 : enum insn_code icode;
9784 :
9785 211467 : if ((icode = optab_handler (movmisalign_optab, mode))
9786 : != CODE_FOR_nothing)
9787 : {
9788 135551 : class expand_operand ops[2];
9789 :
9790 : /* We've already validated the memory, and we're creating a
9791 : new pseudo destination. The predicates really can't fail,
9792 : nor can the generator. */
9793 135551 : create_output_operand (&ops[0], NULL_RTX, mode);
9794 135551 : create_fixed_operand (&ops[1], temp);
9795 135551 : expand_insn (icode, 2, ops);
9796 135551 : temp = ops[0].value;
9797 : }
9798 75916 : else if (targetm.slow_unaligned_access (mode, align))
9799 0 : temp = extract_bit_field (temp, GET_MODE_BITSIZE (mode),
9800 : 0, unsignedp, target,
9801 : mode, mode, false, alt_rtl);
9802 211467 : return temp;
9803 : }
9804 :
9805 : /* Return true if OP is known to be either LOWER or LOWER + 1, with one
9806 : value a positive power of two. */
9807 :
9808 : static bool
9809 60241 : near_pow2_divisor_range_p (tree op, wide_int &lower)
9810 : {
9811 60241 : if (TREE_CODE (op) != SSA_NAME)
9812 : return false;
9813 :
9814 32034 : int_range_max range;
9815 32034 : range_query *query = get_range_query (cfun);
9816 32034 : if (!query->range_of_expr (range, op, currently_expanding_gimple_stmt)
9817 32034 : || range.num_pairs () != 1)
9818 : return false;
9819 :
9820 30036 : lower = range.lower_bound ();
9821 30036 : wide_int upper = lower + 1;
9822 60072 : return (range.upper_bound () == upper
9823 87 : && wi::gt_p (lower, 0, TYPE_SIGN (TREE_TYPE (op)))
9824 30227 : && (wi::popcount (lower) == 1 || wi::popcount (upper) == 1));
9825 32034 : }
9826 :
9827 : /* Helper function of expand_expr_2, expand a division or modulo.
9828 : op0 and op1 should be already expanded treeop0 and treeop1, using
9829 : expand_operands. */
9830 :
9831 : static rtx
9832 169319 : expand_expr_divmod (tree_code code, machine_mode mode, tree treeop0,
9833 : tree treeop1, rtx op0, rtx op1, rtx target, int unsignedp)
9834 : {
9835 338638 : bool mod_p = (code == TRUNC_MOD_EXPR || code == FLOOR_MOD_EXPR
9836 169319 : || code == CEIL_MOD_EXPR || code == ROUND_MOD_EXPR);
9837 169319 : bool speed_p = optimize_insn_for_speed_p ();
9838 :
9839 169319 : scalar_int_mode int_mode;
9840 169319 : wide_int lower;
9841 : /* Split x / y when y is one of two neighboring constants and the target can
9842 : select between the constant divisions cheaply. */
9843 169319 : if (code == TRUNC_DIV_EXPR
9844 234666 : && is_a <scalar_int_mode> (mode, &int_mode)
9845 65420 : && speed_p
9846 62850 : && can_conditionally_move_p (int_mode)
9847 229560 : && near_pow2_divisor_range_p (treeop1, lower))
9848 : {
9849 73 : signop sgn = TYPE_SIGN (TREE_TYPE (treeop1));
9850 73 : unsigned int prec = GET_MODE_PRECISION (int_mode);
9851 73 : wide_int upper = lower + 1;
9852 73 : rtx op_lower
9853 73 : = immed_wide_int_const (wide_int::from (lower, prec, sgn), int_mode);
9854 73 : rtx op_upper
9855 73 : = immed_wide_int_const (wide_int::from (upper, prec, sgn), int_mode);
9856 :
9857 73 : do_pending_stack_adjust ();
9858 73 : start_sequence ();
9859 73 : rtx q_lower = expand_divmod (0, TRUNC_DIV_EXPR, mode, op0, op_lower,
9860 : NULL_RTX, unsignedp);
9861 73 : rtx q_upper = expand_divmod (0, TRUNC_DIV_EXPR, mode, op0, op_upper,
9862 : NULL_RTX, unsignedp);
9863 73 : rtx split_ret
9864 73 : = emit_conditional_move (target, { EQ, op1, op_lower, int_mode },
9865 : q_lower, q_upper, int_mode, unsignedp);
9866 73 : rtx_insn *split_insns = end_sequence ();
9867 :
9868 : /* Cost the unsplit form as a single DIV/UDIV. */
9869 73 : rtx div_rtx = gen_rtx_fmt_ee (unsignedp ? UDIV : DIV, int_mode, op0, op1);
9870 73 : unsigned div_cost = set_src_cost (div_rtx, int_mode, speed_p);
9871 73 : if (split_ret && seq_cost (split_insns, speed_p) < div_cost)
9872 : {
9873 73 : emit_insn (split_insns);
9874 73 : return split_ret;
9875 : }
9876 73 : }
9877 :
9878 169246 : if (SCALAR_INT_MODE_P (mode)
9879 169246 : && optimize >= 2
9880 141970 : && get_range_pos_neg (treeop0, currently_expanding_gimple_stmt) == 1
9881 212730 : && get_range_pos_neg (treeop1, currently_expanding_gimple_stmt) == 1)
9882 : {
9883 : /* If both arguments are known to be positive when interpreted
9884 : as signed, we can expand it as both signed and unsigned
9885 : division or modulo. Choose the cheaper sequence in that case. */
9886 14005 : do_pending_stack_adjust ();
9887 14005 : start_sequence ();
9888 14005 : rtx uns_ret = expand_divmod (mod_p, code, mode, op0, op1, target, 1);
9889 14005 : rtx_insn *uns_insns = end_sequence ();
9890 14005 : start_sequence ();
9891 14005 : rtx sgn_ret = expand_divmod (mod_p, code, mode, op0, op1, target, 0);
9892 14005 : rtx_insn *sgn_insns = end_sequence ();
9893 14005 : unsigned uns_cost = seq_cost (uns_insns, speed_p);
9894 14005 : unsigned sgn_cost = seq_cost (sgn_insns, speed_p);
9895 14005 : bool was_tie = false;
9896 :
9897 : /* If costs are the same then use as tie breaker the other other
9898 : factor. */
9899 14005 : if (uns_cost == sgn_cost)
9900 : {
9901 3591 : uns_cost = seq_cost (uns_insns, !speed_p);
9902 3591 : sgn_cost = seq_cost (sgn_insns, !speed_p);
9903 3591 : was_tie = true;
9904 : }
9905 :
9906 14005 : if (dump_file && (dump_flags & TDF_DETAILS))
9907 0 : fprintf (dump_file, ";; positive division:%s unsigned cost: %u; "
9908 : "signed cost: %u\n",
9909 : was_tie ? " (needed tie breaker)" : "", uns_cost, sgn_cost);
9910 :
9911 14005 : if (uns_cost < sgn_cost || (uns_cost == sgn_cost && unsignedp))
9912 : {
9913 11288 : emit_insn (uns_insns);
9914 11288 : return uns_ret;
9915 : }
9916 2717 : emit_insn (sgn_insns);
9917 2717 : return sgn_ret;
9918 : }
9919 155241 : return expand_divmod (mod_p, code, mode, op0, op1, target, unsignedp);
9920 169319 : }
9921 :
9922 : /* Return true if EXP has a range of values [0..1], false
9923 : otherwise. This works for constants and ssa names, calling back into the ranger. */
9924 : static bool
9925 1527538 : expr_has_boolean_range (tree exp, gimple *stmt)
9926 : {
9927 : /* An integral type with a single bit of precision. */
9928 3055063 : if (INTEGRAL_TYPE_P (TREE_TYPE (exp))
9929 1527538 : && TYPE_UNSIGNED (TREE_TYPE (exp))
9930 2539342 : && TYPE_PRECISION (TREE_TYPE (exp)) == 1)
9931 : return true;
9932 :
9933 : /* Signed 1 bit integers are not boolean ranges. */
9934 3055063 : if (!INTEGRAL_TYPE_P (TREE_TYPE (exp))
9935 3055063 : || TYPE_PRECISION (TREE_TYPE (exp)) <= 1)
9936 : return false;
9937 :
9938 1527494 : if (TREE_CODE (exp) == SSA_NAME)
9939 906818 : return ssa_name_has_boolean_range (exp, stmt);
9940 620676 : if (TREE_CODE (exp) == INTEGER_CST)
9941 546772 : return wi::leu_p (wi::to_wide (exp), 1);
9942 : return false;
9943 : }
9944 :
9945 : rtx
9946 13661179 : expand_expr_real_2 (const_sepops ops, rtx target, machine_mode tmode,
9947 : enum expand_modifier modifier)
9948 : {
9949 13661179 : rtx op0, op1, op2, temp;
9950 13661179 : rtx_code_label *lab;
9951 13661179 : tree type;
9952 13661179 : int unsignedp;
9953 13661179 : machine_mode mode;
9954 13661179 : scalar_int_mode int_mode;
9955 13661179 : enum tree_code code = ops->code;
9956 13661179 : optab this_optab;
9957 13661179 : rtx subtarget, original_target;
9958 13661179 : int ignore;
9959 13661179 : bool reduce_bit_field;
9960 13661179 : location_t loc = ops->location;
9961 13661179 : tree treeop0, treeop1, treeop2;
9962 : #define REDUCE_BIT_FIELD(expr) (reduce_bit_field \
9963 : ? reduce_to_bit_field_precision ((expr), \
9964 : target, \
9965 : type) \
9966 : : (expr))
9967 :
9968 13661179 : type = ops->type;
9969 13661179 : mode = TYPE_MODE (type);
9970 13661179 : unsignedp = TYPE_UNSIGNED (type);
9971 :
9972 13661179 : treeop0 = ops->op0;
9973 13661179 : treeop1 = ops->op1;
9974 13661179 : treeop2 = ops->op2;
9975 :
9976 : /* We should be called only on simple (binary or unary) expressions,
9977 : exactly those that are valid in gimple expressions that aren't
9978 : GIMPLE_SINGLE_RHS (or invalid). */
9979 13661179 : gcc_assert (get_gimple_rhs_class (code) == GIMPLE_UNARY_RHS
9980 : || get_gimple_rhs_class (code) == GIMPLE_BINARY_RHS
9981 : || get_gimple_rhs_class (code) == GIMPLE_TERNARY_RHS);
9982 :
9983 27322358 : ignore = (target == const0_rtx
9984 13661179 : || ((CONVERT_EXPR_CODE_P (code)
9985 9822213 : || code == COND_EXPR || code == VIEW_CONVERT_EXPR)
9986 3857062 : && TREE_CODE (type) == VOID_TYPE));
9987 :
9988 : /* We should be called only if we need the result. */
9989 0 : gcc_assert (!ignore);
9990 :
9991 : /* An operation in what may be a bit-field type needs the
9992 : result to be reduced to the precision of the bit-field type,
9993 : which is narrower than that of the type's mode. */
9994 28154810 : reduce_bit_field = (INTEGRAL_TYPE_P (type)
9995 13661179 : && !type_has_mode_precision_p (type));
9996 :
9997 832452 : if (reduce_bit_field
9998 832452 : && (modifier == EXPAND_STACK_PARM
9999 831928 : || (target && GET_MODE (target) != mode)))
10000 513699 : target = 0;
10001 :
10002 : /* Use subtarget as the target for operand 0 of a binary operation. */
10003 13661179 : subtarget = get_subtarget (target);
10004 13661179 : original_target = target;
10005 :
10006 13661179 : switch (code)
10007 : {
10008 3842142 : case NON_LVALUE_EXPR:
10009 3842142 : case PAREN_EXPR:
10010 3842142 : CASE_CONVERT:
10011 3842142 : if (treeop0 == error_mark_node)
10012 0 : return const0_rtx;
10013 :
10014 3842142 : if (TREE_CODE (type) == UNION_TYPE)
10015 : {
10016 0 : tree valtype = TREE_TYPE (treeop0);
10017 :
10018 : /* If both input and output are BLKmode, this conversion isn't doing
10019 : anything except possibly changing memory attribute. */
10020 0 : if (mode == BLKmode && TYPE_MODE (valtype) == BLKmode)
10021 : {
10022 0 : rtx result = expand_expr (treeop0, target, tmode,
10023 : modifier);
10024 :
10025 0 : result = copy_rtx (result);
10026 0 : set_mem_attributes (result, type, 0);
10027 0 : return result;
10028 : }
10029 :
10030 0 : if (target == 0)
10031 : {
10032 0 : if (TYPE_MODE (type) != BLKmode)
10033 0 : target = gen_reg_rtx (TYPE_MODE (type));
10034 : else
10035 0 : target = assign_temp (type, 1, 1);
10036 : }
10037 :
10038 0 : if (MEM_P (target))
10039 : /* Store data into beginning of memory target. */
10040 0 : store_expr (treeop0,
10041 0 : adjust_address (target, TYPE_MODE (valtype), 0),
10042 : modifier == EXPAND_STACK_PARM,
10043 0 : false, TYPE_REVERSE_STORAGE_ORDER (type));
10044 :
10045 : else
10046 : {
10047 0 : gcc_assert (REG_P (target)
10048 : && !TYPE_REVERSE_STORAGE_ORDER (type));
10049 :
10050 : /* Store this field into a union of the proper type. */
10051 0 : poly_uint64 op0_size
10052 0 : = tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (treeop0)));
10053 0 : poly_uint64 union_size = GET_MODE_BITSIZE (mode);
10054 0 : store_field (target,
10055 : /* The conversion must be constructed so that
10056 : we know at compile time how many bits
10057 : to preserve. */
10058 0 : ordered_min (op0_size, union_size),
10059 0 : 0, 0, 0, TYPE_MODE (valtype), treeop0, 0,
10060 : false, false);
10061 : }
10062 :
10063 : /* Return the entire union. */
10064 : return target;
10065 : }
10066 :
10067 3842142 : if (mode == TYPE_MODE (TREE_TYPE (treeop0)))
10068 : {
10069 2278586 : op0 = expand_expr (treeop0, target, VOIDmode,
10070 : modifier);
10071 :
10072 2278586 : return REDUCE_BIT_FIELD (op0);
10073 : }
10074 :
10075 1927975 : op0 = expand_expr (treeop0, NULL_RTX, mode,
10076 : modifier == EXPAND_SUM ? EXPAND_NORMAL : modifier);
10077 1563556 : if (GET_MODE (op0) == mode)
10078 : ;
10079 :
10080 : /* If OP0 is a constant, just convert it into the proper mode. */
10081 1520564 : else if (CONSTANT_P (op0))
10082 : {
10083 905 : tree inner_type = TREE_TYPE (treeop0);
10084 905 : machine_mode inner_mode = GET_MODE (op0);
10085 :
10086 905 : if (inner_mode == VOIDmode)
10087 15 : inner_mode = TYPE_MODE (inner_type);
10088 :
10089 905 : if (modifier == EXPAND_INITIALIZER)
10090 0 : op0 = force_lowpart_subreg (mode, op0, inner_mode);
10091 : else
10092 905 : op0 = convert_modes (mode, inner_mode, op0,
10093 905 : TYPE_UNSIGNED (inner_type));
10094 : }
10095 :
10096 1519659 : else if (modifier == EXPAND_INITIALIZER)
10097 24 : op0 = gen_rtx_fmt_e (TYPE_UNSIGNED (TREE_TYPE (treeop0))
10098 : ? ZERO_EXTEND : SIGN_EXTEND, mode, op0);
10099 :
10100 1519647 : else if (SCALAR_INT_MODE_P (GET_MODE (op0))
10101 1343952 : && optimize >= 2
10102 787542 : && SCALAR_INT_MODE_P (mode)
10103 787542 : && INTEGRAL_TYPE_P (TREE_TYPE (treeop0))
10104 787341 : && (GET_MODE_SIZE (as_a <scalar_int_mode> (mode))
10105 1574682 : > GET_MODE_SIZE (as_a <scalar_int_mode> (GET_MODE (op0))))
10106 2069509 : && get_range_pos_neg (treeop0,
10107 : currently_expanding_gimple_stmt) == 1)
10108 : {
10109 : /* If argument is known to be positive when interpreted
10110 : as signed, we can expand it as both sign and zero
10111 : extension. Choose the cheaper sequence in that case. */
10112 139732 : bool speed_p = optimize_insn_for_speed_p ();
10113 139732 : rtx uns_ret = NULL_RTX, sgn_ret = NULL_RTX;
10114 139732 : do_pending_stack_adjust ();
10115 139732 : start_sequence ();
10116 139732 : if (target == NULL_RTX)
10117 115383 : uns_ret = convert_to_mode (mode, op0, 1);
10118 : else
10119 24349 : convert_move (target, op0, 1);
10120 139732 : rtx_insn *uns_insns = end_sequence ();
10121 139732 : start_sequence ();
10122 139732 : if (target == NULL_RTX)
10123 115383 : sgn_ret = convert_to_mode (mode, op0, 0);
10124 : else
10125 24349 : convert_move (target, op0, 0);
10126 139732 : rtx_insn *sgn_insns = end_sequence ();
10127 139732 : unsigned uns_cost = seq_cost (uns_insns, speed_p);
10128 139732 : unsigned sgn_cost = seq_cost (sgn_insns, speed_p);
10129 139732 : bool was_tie = false;
10130 :
10131 : /* If costs are the same then use as tie breaker the other other
10132 : factor. */
10133 139732 : if (uns_cost == sgn_cost)
10134 : {
10135 38738 : uns_cost = seq_cost (uns_insns, !speed_p);
10136 38738 : sgn_cost = seq_cost (sgn_insns, !speed_p);
10137 38738 : was_tie = true;
10138 : }
10139 :
10140 139732 : if (dump_file && (dump_flags & TDF_DETAILS))
10141 0 : fprintf (dump_file, ";; positive extension:%s unsigned cost: %u; "
10142 : "signed cost: %u\n",
10143 : was_tie ? " (needed tie breaker)" : "",
10144 : uns_cost, sgn_cost);
10145 139732 : if (uns_cost < sgn_cost
10146 139732 : || (uns_cost == sgn_cost && TYPE_UNSIGNED (TREE_TYPE (treeop0))))
10147 : {
10148 124612 : emit_insn (uns_insns);
10149 124612 : sgn_ret = uns_ret;
10150 : }
10151 : else
10152 15120 : emit_insn (sgn_insns);
10153 139732 : if (target == NULL_RTX)
10154 115383 : op0 = sgn_ret;
10155 : else
10156 24349 : op0 = target;
10157 : }
10158 1379915 : else if (target == 0)
10159 866449 : op0 = convert_to_mode (mode, op0, TYPE_UNSIGNED (TREE_TYPE (treeop0)));
10160 : else
10161 : {
10162 513466 : convert_move (target, op0, TYPE_UNSIGNED (TREE_TYPE (treeop0)));
10163 513466 : op0 = target;
10164 : }
10165 :
10166 1563556 : return REDUCE_BIT_FIELD (op0);
10167 :
10168 0 : case ADDR_SPACE_CONVERT_EXPR:
10169 0 : {
10170 0 : tree treeop0_type = TREE_TYPE (treeop0);
10171 :
10172 0 : gcc_assert (POINTER_TYPE_P (type));
10173 0 : gcc_assert (POINTER_TYPE_P (treeop0_type));
10174 :
10175 0 : addr_space_t as_to = TYPE_ADDR_SPACE (TREE_TYPE (type));
10176 0 : addr_space_t as_from = TYPE_ADDR_SPACE (TREE_TYPE (treeop0_type));
10177 :
10178 : /* Conversions between pointers to the same address space should
10179 : have been implemented via CONVERT_EXPR / NOP_EXPR. */
10180 0 : gcc_assert (as_to != as_from);
10181 :
10182 0 : op0 = expand_expr (treeop0, NULL_RTX, VOIDmode, modifier);
10183 :
10184 : /* Ask target code to handle conversion between pointers
10185 : to overlapping address spaces. */
10186 0 : if (targetm.addr_space.subset_p (as_to, as_from)
10187 0 : || targetm.addr_space.subset_p (as_from, as_to))
10188 : {
10189 0 : op0 = targetm.addr_space.convert (op0, treeop0_type, type);
10190 : }
10191 : else
10192 : {
10193 : /* For disjoint address spaces, converting anything but a null
10194 : pointer invokes undefined behavior. We truncate or extend the
10195 : value as if we'd converted via integers, which handles 0 as
10196 : required, and all others as the programmer likely expects. */
10197 : #ifndef POINTERS_EXTEND_UNSIGNED
10198 : const int POINTERS_EXTEND_UNSIGNED = 1;
10199 : #endif
10200 0 : op0 = convert_modes (mode, TYPE_MODE (treeop0_type),
10201 : op0, POINTERS_EXTEND_UNSIGNED);
10202 : }
10203 0 : gcc_assert (op0);
10204 : return op0;
10205 : }
10206 :
10207 1374078 : case POINTER_PLUS_EXPR:
10208 : /* Even though the sizetype mode and the pointer's mode can be different
10209 : expand is able to handle this correctly and get the correct result out
10210 : of the PLUS_EXPR code. */
10211 : /* Make sure to sign-extend the sizetype offset in a POINTER_PLUS_EXPR
10212 : if sizetype precision is smaller than pointer precision. */
10213 1374078 : if (TYPE_PRECISION (sizetype) < TYPE_PRECISION (type))
10214 0 : treeop1 = fold_convert_loc (loc, type,
10215 : fold_convert_loc (loc, ssizetype,
10216 : treeop1));
10217 : /* If sizetype precision is larger than pointer precision, truncate the
10218 : offset to have matching modes. */
10219 1374078 : else if (TYPE_PRECISION (sizetype) > TYPE_PRECISION (type))
10220 0 : treeop1 = fold_convert_loc (loc, type, treeop1);
10221 : /* FALLTHRU */
10222 :
10223 5392710 : case PLUS_EXPR:
10224 : /* If we are adding a constant, a VAR_DECL that is sp, fp, or ap, and
10225 : something else, make sure we add the register to the constant and
10226 : then to the other thing. This case can occur during strength
10227 : reduction and doing it this way will produce better code if the
10228 : frame pointer or argument pointer is eliminated.
10229 :
10230 : fold-const.cc will ensure that the constant is always in the inner
10231 : PLUS_EXPR, so the only case we need to do anything about is if
10232 : sp, ap, or fp is our second argument, in which case we must swap
10233 : the innermost first argument and our second argument. */
10234 :
10235 5392710 : if (TREE_CODE (treeop0) == PLUS_EXPR
10236 6948 : && TREE_CODE (TREE_OPERAND (treeop0, 1)) == INTEGER_CST
10237 0 : && VAR_P (treeop1)
10238 5392710 : && (DECL_RTL (treeop1) == frame_pointer_rtx
10239 0 : || DECL_RTL (treeop1) == stack_pointer_rtx
10240 0 : || DECL_RTL (treeop1) == arg_pointer_rtx))
10241 : {
10242 0 : gcc_unreachable ();
10243 : }
10244 :
10245 : /* If the result is to be ptr_mode and we are adding an integer to
10246 : something, we might be forming a constant. So try to use
10247 : plus_constant. If it produces a sum and we can't accept it,
10248 : use force_operand. This allows P = &ARR[const] to generate
10249 : efficient code on machines where a SYMBOL_REF is not a valid
10250 : address.
10251 :
10252 : If this is an EXPAND_SUM call, always return the sum. */
10253 5392710 : if (modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER
10254 5392710 : || (mode == ptr_mode && (unsignedp || ! flag_trapv)))
10255 : {
10256 3520619 : if (modifier == EXPAND_STACK_PARM)
10257 18888 : target = 0;
10258 3520619 : if (TREE_CODE (treeop0) == INTEGER_CST
10259 3521626 : && HWI_COMPUTABLE_MODE_P (mode)
10260 3521630 : && TREE_CONSTANT (treeop1))
10261 : {
10262 4 : rtx constant_part;
10263 4 : HOST_WIDE_INT wc;
10264 4 : machine_mode wmode = TYPE_MODE (TREE_TYPE (treeop1));
10265 :
10266 4 : op1 = expand_expr (treeop1, subtarget, VOIDmode,
10267 : EXPAND_SUM);
10268 : /* Use wi::shwi to ensure that the constant is
10269 : truncated according to the mode of OP1, then sign extended
10270 : to a HOST_WIDE_INT. Using the constant directly can result
10271 : in non-canonical RTL in a 64x32 cross compile. */
10272 4 : wc = TREE_INT_CST_LOW (treeop0);
10273 4 : constant_part =
10274 4 : immed_wide_int_const (wi::shwi (wc, wmode), wmode);
10275 4 : op1 = plus_constant (mode, op1, INTVAL (constant_part));
10276 4 : if (modifier != EXPAND_SUM && modifier != EXPAND_INITIALIZER)
10277 1 : op1 = force_operand (op1, target);
10278 4 : return REDUCE_BIT_FIELD (op1);
10279 : }
10280 :
10281 3520615 : else if (TREE_CODE (treeop1) == INTEGER_CST
10282 7489975 : && HWI_COMPUTABLE_MODE_P (mode)
10283 5899130 : && TREE_CONSTANT (treeop0))
10284 : {
10285 281246 : rtx constant_part;
10286 281246 : HOST_WIDE_INT wc;
10287 281246 : machine_mode wmode = TYPE_MODE (TREE_TYPE (treeop0));
10288 :
10289 490693 : op0 = expand_expr (treeop0, subtarget, VOIDmode,
10290 : (modifier == EXPAND_INITIALIZER
10291 : ? EXPAND_INITIALIZER : EXPAND_SUM));
10292 281246 : if (! CONSTANT_P (op0))
10293 : {
10294 6 : op1 = expand_expr (treeop1, NULL_RTX,
10295 : VOIDmode, modifier);
10296 : /* Return a PLUS if modifier says it's OK. */
10297 6 : if (modifier == EXPAND_SUM
10298 : || modifier == EXPAND_INITIALIZER)
10299 6 : return simplify_gen_binary (PLUS, mode, op0, op1);
10300 0 : goto binop2;
10301 : }
10302 : /* Use wi::shwi to ensure that the constant is
10303 : truncated according to the mode of OP1, then sign extended
10304 : to a HOST_WIDE_INT. Using the constant directly can result
10305 : in non-canonical RTL in a 64x32 cross compile. */
10306 281240 : wc = TREE_INT_CST_LOW (treeop1);
10307 281240 : constant_part
10308 281240 : = immed_wide_int_const (wi::shwi (wc, wmode), wmode);
10309 281240 : op0 = plus_constant (mode, op0, INTVAL (constant_part));
10310 281240 : if (modifier != EXPAND_SUM && modifier != EXPAND_INITIALIZER)
10311 208616 : op0 = force_operand (op0, target);
10312 281240 : return REDUCE_BIT_FIELD (op0);
10313 : }
10314 : }
10315 :
10316 : /* Use TER to expand pointer addition of a negated value
10317 : as pointer subtraction. */
10318 9149454 : if ((POINTER_TYPE_P (TREE_TYPE (treeop0))
10319 4018619 : || (TREE_CODE (TREE_TYPE (treeop0)) == VECTOR_TYPE
10320 81699 : && POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (treeop0)))))
10321 1092841 : && TREE_CODE (treeop1) == SSA_NAME
10322 6249600 : && TYPE_MODE (TREE_TYPE (treeop0))
10323 569070 : == TYPE_MODE (TREE_TYPE (treeop1)))
10324 : {
10325 569070 : gimple *def = get_def_for_expr (treeop1, NEGATE_EXPR);
10326 569070 : if (def)
10327 : {
10328 2173 : treeop1 = gimple_assign_rhs1 (def);
10329 2173 : code = MINUS_EXPR;
10330 2173 : goto do_minus;
10331 : }
10332 : }
10333 :
10334 : /* No sense saving up arithmetic to be done
10335 : if it's all in the wrong mode to form part of an address.
10336 : And force_operand won't know whether to sign-extend or
10337 : zero-extend. */
10338 5109287 : if (modifier != EXPAND_INITIALIZER
10339 5109287 : && (modifier != EXPAND_SUM || mode != ptr_mode))
10340 : {
10341 4531598 : expand_operands (treeop0, treeop1,
10342 : subtarget, &op0, &op1, modifier);
10343 4531598 : if (op0 == const0_rtx)
10344 8994 : return op1;
10345 4522604 : if (op1 == const0_rtx)
10346 : return op0;
10347 4522353 : goto binop2;
10348 : }
10349 :
10350 577689 : expand_operands (treeop0, treeop1,
10351 : subtarget, &op0, &op1, modifier);
10352 577689 : return REDUCE_BIT_FIELD (simplify_gen_binary (PLUS, mode, op0, op1));
10353 :
10354 552492 : case MINUS_EXPR:
10355 552492 : case POINTER_DIFF_EXPR:
10356 552492 : do_minus:
10357 : /* For initializers, we are allowed to return a MINUS of two
10358 : symbolic constants. Here we handle all cases when both operands
10359 : are constant. */
10360 : /* Handle difference of two symbolic constants,
10361 : for the sake of an initializer. */
10362 552492 : if ((modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER)
10363 2679 : && really_constant_p (treeop0)
10364 553004 : && really_constant_p (treeop1))
10365 : {
10366 68 : expand_operands (treeop0, treeop1,
10367 : NULL_RTX, &op0, &op1, modifier);
10368 68 : return simplify_gen_binary (MINUS, mode, op0, op1);
10369 : }
10370 :
10371 : /* No sense saving up arithmetic to be done
10372 : if it's all in the wrong mode to form part of an address.
10373 : And force_operand won't know whether to sign-extend or
10374 : zero-extend. */
10375 552424 : if (modifier != EXPAND_INITIALIZER
10376 552424 : && (modifier != EXPAND_SUM || mode != ptr_mode))
10377 549813 : goto binop;
10378 :
10379 2611 : expand_operands (treeop0, treeop1,
10380 : subtarget, &op0, &op1, modifier);
10381 :
10382 : /* Convert A - const to A + (-const). */
10383 2611 : if (CONST_INT_P (op1))
10384 : {
10385 0 : op1 = negate_rtx (mode, op1);
10386 0 : return REDUCE_BIT_FIELD (simplify_gen_binary (PLUS, mode, op0, op1));
10387 : }
10388 :
10389 2611 : goto binop2;
10390 :
10391 0 : case WIDEN_MULT_PLUS_EXPR:
10392 0 : case WIDEN_MULT_MINUS_EXPR:
10393 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
10394 0 : op2 = expand_normal (treeop2);
10395 0 : target = expand_widen_pattern_expr (ops, op0, op1, op2,
10396 : target, unsignedp);
10397 0 : return target;
10398 :
10399 24470 : case WIDEN_MULT_EXPR:
10400 : /* If first operand is constant, swap them.
10401 : Thus the following special case checks need only
10402 : check the second operand. */
10403 24470 : if (TREE_CODE (treeop0) == INTEGER_CST)
10404 368 : std::swap (treeop0, treeop1);
10405 :
10406 : /* First, check if we have a multiplication of one signed and one
10407 : unsigned operand. */
10408 24470 : if (TREE_CODE (treeop1) != INTEGER_CST
10409 24470 : && (TYPE_UNSIGNED (TREE_TYPE (treeop0))
10410 18639 : != TYPE_UNSIGNED (TREE_TYPE (treeop1))))
10411 : {
10412 0 : machine_mode innermode = TYPE_MODE (TREE_TYPE (treeop0));
10413 0 : this_optab = usmul_widen_optab;
10414 0 : if (find_widening_optab_handler (this_optab, mode, innermode)
10415 : != CODE_FOR_nothing)
10416 : {
10417 0 : if (TYPE_UNSIGNED (TREE_TYPE (treeop0)))
10418 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1,
10419 : EXPAND_NORMAL);
10420 : else
10421 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op1, &op0,
10422 : EXPAND_NORMAL);
10423 : /* op0 and op1 might still be constant, despite the above
10424 : != INTEGER_CST check. Handle it. */
10425 0 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
10426 : {
10427 0 : op0 = convert_modes (mode, innermode, op0, true);
10428 0 : op1 = convert_modes (mode, innermode, op1, false);
10429 0 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1,
10430 : target, unsignedp));
10431 : }
10432 0 : goto binop3;
10433 : }
10434 : }
10435 : /* Check for a multiplication with matching signedness. */
10436 24470 : else if ((TREE_CODE (treeop1) == INTEGER_CST
10437 5831 : && int_fits_type_p (treeop1, TREE_TYPE (treeop0)))
10438 24474 : || (TYPE_UNSIGNED (TREE_TYPE (treeop1))
10439 18643 : == TYPE_UNSIGNED (TREE_TYPE (treeop0))))
10440 : {
10441 24470 : tree op0type = TREE_TYPE (treeop0);
10442 24470 : machine_mode innermode = TYPE_MODE (op0type);
10443 24470 : bool zextend_p = TYPE_UNSIGNED (op0type);
10444 24470 : optab other_optab = zextend_p ? smul_widen_optab : umul_widen_optab;
10445 2590 : this_optab = zextend_p ? umul_widen_optab : smul_widen_optab;
10446 :
10447 24470 : if (TREE_CODE (treeop0) != INTEGER_CST)
10448 : {
10449 24470 : if (find_widening_optab_handler (this_optab, mode, innermode)
10450 : != CODE_FOR_nothing)
10451 : {
10452 24470 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1,
10453 : EXPAND_NORMAL);
10454 : /* op0 and op1 might still be constant, despite the above
10455 : != INTEGER_CST check. Handle it. */
10456 24470 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
10457 : {
10458 0 : widen_mult_const:
10459 0 : op0 = convert_modes (mode, innermode, op0, zextend_p);
10460 0 : op1
10461 0 : = convert_modes (mode, innermode, op1,
10462 0 : TYPE_UNSIGNED (TREE_TYPE (treeop1)));
10463 0 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1,
10464 : target,
10465 : unsignedp));
10466 : }
10467 24470 : temp = expand_widening_mult (mode, op0, op1, target,
10468 : unsignedp, this_optab);
10469 24470 : return REDUCE_BIT_FIELD (temp);
10470 : }
10471 0 : if (find_widening_optab_handler (other_optab, mode, innermode)
10472 : != CODE_FOR_nothing
10473 0 : && innermode == word_mode)
10474 : {
10475 0 : rtx htem, hipart;
10476 0 : op0 = expand_normal (treeop0);
10477 0 : op1 = expand_normal (treeop1);
10478 : /* op0 and op1 might be constants, despite the above
10479 : != INTEGER_CST check. Handle it. */
10480 0 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
10481 0 : goto widen_mult_const;
10482 0 : temp = expand_binop (mode, other_optab, op0, op1, target,
10483 : unsignedp, OPTAB_LIB_WIDEN);
10484 0 : hipart = gen_highpart (word_mode, temp);
10485 0 : htem = expand_mult_highpart_adjust (word_mode, hipart,
10486 : op0, op1, hipart,
10487 : zextend_p);
10488 0 : if (htem != hipart)
10489 0 : emit_move_insn (hipart, htem);
10490 0 : return REDUCE_BIT_FIELD (temp);
10491 : }
10492 : }
10493 : }
10494 0 : treeop0 = fold_build1 (CONVERT_EXPR, type, treeop0);
10495 0 : treeop1 = fold_build1 (CONVERT_EXPR, type, treeop1);
10496 0 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10497 0 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1, target, unsignedp));
10498 :
10499 1428593 : case MULT_EXPR:
10500 : /* If this is a fixed-point operation, then we cannot use the code
10501 : below because "expand_mult" doesn't support sat/no-sat fixed-point
10502 : multiplications. */
10503 1428593 : if (ALL_FIXED_POINT_MODE_P (mode))
10504 0 : goto binop;
10505 :
10506 : /* If first operand is constant, swap them.
10507 : Thus the following special case checks need only
10508 : check the second operand. */
10509 1428593 : if (TREE_CODE (treeop0) == INTEGER_CST)
10510 686 : std::swap (treeop0, treeop1);
10511 :
10512 : /* Attempt to return something suitable for generating an
10513 : indexed address, for machines that support that. */
10514 :
10515 539051 : if (modifier == EXPAND_SUM && mode == ptr_mode
10516 1967644 : && tree_fits_shwi_p (treeop1))
10517 : {
10518 531305 : tree exp1 = treeop1;
10519 :
10520 531305 : op0 = expand_expr (treeop0, subtarget, VOIDmode,
10521 : EXPAND_SUM);
10522 :
10523 531305 : if (!REG_P (op0))
10524 245019 : op0 = force_operand (op0, NULL_RTX);
10525 531305 : if (!REG_P (op0))
10526 2358 : op0 = copy_to_mode_reg (mode, op0);
10527 :
10528 531305 : op1 = gen_int_mode (tree_to_shwi (exp1),
10529 531305 : TYPE_MODE (TREE_TYPE (exp1)));
10530 531305 : return REDUCE_BIT_FIELD (gen_rtx_MULT (mode, op0, op1));
10531 : }
10532 :
10533 897288 : if (modifier == EXPAND_STACK_PARM)
10534 2941 : target = 0;
10535 :
10536 897288 : if (SCALAR_INT_MODE_P (mode) && optimize >= 2)
10537 : {
10538 541692 : gimple *def_stmt0 = get_def_for_expr (treeop0, TRUNC_DIV_EXPR);
10539 541692 : gimple *def_stmt1 = get_def_for_expr (treeop1, TRUNC_DIV_EXPR);
10540 541692 : if (def_stmt0
10541 541692 : && !operand_equal_p (treeop1, gimple_assign_rhs2 (def_stmt0), 0))
10542 : def_stmt0 = NULL;
10543 541692 : if (def_stmt1
10544 541692 : && !operand_equal_p (treeop0, gimple_assign_rhs2 (def_stmt1), 0))
10545 : def_stmt1 = NULL;
10546 :
10547 541692 : if (def_stmt0 || def_stmt1)
10548 : {
10549 : /* X / Y * Y can be expanded as X - X % Y too.
10550 : Choose the cheaper sequence of those two. */
10551 194 : if (def_stmt0)
10552 194 : treeop0 = gimple_assign_rhs1 (def_stmt0);
10553 : else
10554 : {
10555 0 : treeop1 = treeop0;
10556 0 : treeop0 = gimple_assign_rhs1 (def_stmt1);
10557 : }
10558 194 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1,
10559 : EXPAND_NORMAL);
10560 194 : bool speed_p = optimize_insn_for_speed_p ();
10561 194 : do_pending_stack_adjust ();
10562 194 : start_sequence ();
10563 194 : rtx divmul_ret
10564 194 : = expand_expr_divmod (TRUNC_DIV_EXPR, mode, treeop0, treeop1,
10565 : op0, op1, NULL_RTX, unsignedp);
10566 194 : divmul_ret = expand_mult (mode, divmul_ret, op1, target,
10567 : unsignedp);
10568 194 : rtx_insn *divmul_insns = end_sequence ();
10569 194 : start_sequence ();
10570 194 : rtx modsub_ret
10571 194 : = expand_expr_divmod (TRUNC_MOD_EXPR, mode, treeop0, treeop1,
10572 : op0, op1, NULL_RTX, unsignedp);
10573 194 : this_optab = optab_for_tree_code (MINUS_EXPR, type,
10574 : optab_default);
10575 194 : modsub_ret = expand_binop (mode, this_optab, op0, modsub_ret,
10576 : target, unsignedp, OPTAB_LIB_WIDEN);
10577 194 : rtx_insn *modsub_insns = end_sequence ();
10578 194 : unsigned divmul_cost = seq_cost (divmul_insns, speed_p);
10579 194 : unsigned modsub_cost = seq_cost (modsub_insns, speed_p);
10580 : /* If costs are the same then use as tie breaker the other other
10581 : factor. */
10582 194 : if (divmul_cost == modsub_cost)
10583 : {
10584 5 : divmul_cost = seq_cost (divmul_insns, !speed_p);
10585 5 : modsub_cost = seq_cost (modsub_insns, !speed_p);
10586 : }
10587 :
10588 194 : if (divmul_cost <= modsub_cost)
10589 : {
10590 121 : emit_insn (divmul_insns);
10591 121 : return REDUCE_BIT_FIELD (divmul_ret);
10592 : }
10593 73 : emit_insn (modsub_insns);
10594 73 : return REDUCE_BIT_FIELD (modsub_ret);
10595 : }
10596 : }
10597 :
10598 897094 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10599 :
10600 : /* Expand X*Y as X&-Y when Y must be zero or one. */
10601 897094 : if (SCALAR_INT_MODE_P (mode))
10602 : {
10603 763769 : bool bit0_p = expr_has_boolean_range (treeop0, currently_expanding_gimple_stmt);
10604 763769 : bool bit1_p = expr_has_boolean_range (treeop1, currently_expanding_gimple_stmt);
10605 :
10606 : /* Expand X*Y as X&Y when both X and Y must be zero or one. */
10607 763769 : if (bit0_p && bit1_p)
10608 0 : return REDUCE_BIT_FIELD (expand_and (mode, op0, op1, target));
10609 :
10610 763769 : if (bit0_p || bit1_p)
10611 : {
10612 3987 : bool speed = optimize_insn_for_speed_p ();
10613 3987 : int cost = add_cost (speed, mode) + neg_cost (speed, mode);
10614 3987 : struct algorithm algorithm;
10615 3987 : enum mult_variant variant;
10616 3987 : if (CONST_INT_P (op1)
10617 3987 : ? !choose_mult_variant (mode, INTVAL (op1),
10618 : &algorithm, &variant, cost)
10619 627 : : cost < mul_cost (speed, mode))
10620 : {
10621 1954 : temp = bit0_p ? expand_and (mode, negate_rtx (mode, op0),
10622 : op1, target)
10623 238 : : expand_and (mode, op0,
10624 : negate_rtx (mode, op1),
10625 : target);
10626 1954 : return REDUCE_BIT_FIELD (temp);
10627 : }
10628 : }
10629 : }
10630 :
10631 895140 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1, target, unsignedp));
10632 :
10633 168931 : case TRUNC_MOD_EXPR:
10634 168931 : case FLOOR_MOD_EXPR:
10635 168931 : case CEIL_MOD_EXPR:
10636 168931 : case ROUND_MOD_EXPR:
10637 :
10638 168931 : case TRUNC_DIV_EXPR:
10639 168931 : case FLOOR_DIV_EXPR:
10640 168931 : case CEIL_DIV_EXPR:
10641 168931 : case ROUND_DIV_EXPR:
10642 168931 : case EXACT_DIV_EXPR:
10643 : /* If this is a fixed-point operation, then we cannot use the code
10644 : below because "expand_divmod" doesn't support sat/no-sat fixed-point
10645 : divisions. */
10646 168931 : if (ALL_FIXED_POINT_MODE_P (mode))
10647 0 : goto binop;
10648 :
10649 168931 : if (modifier == EXPAND_STACK_PARM)
10650 312 : target = 0;
10651 : /* Possible optimization: compute the dividend with EXPAND_SUM
10652 : then if the divisor is constant can optimize the case
10653 : where some terms of the dividend have coeffs divisible by it. */
10654 168931 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10655 168931 : return expand_expr_divmod (code, mode, treeop0, treeop1, op0, op1,
10656 168931 : target, unsignedp);
10657 :
10658 30946 : case RDIV_EXPR:
10659 30946 : goto binop;
10660 :
10661 3507 : case MULT_HIGHPART_EXPR:
10662 3507 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10663 3507 : temp = expand_mult_highpart (mode, op0, op1, target, unsignedp);
10664 3507 : gcc_assert (temp);
10665 : return temp;
10666 :
10667 0 : case FIXED_CONVERT_EXPR:
10668 0 : op0 = expand_normal (treeop0);
10669 0 : if (target == 0 || modifier == EXPAND_STACK_PARM)
10670 0 : target = gen_reg_rtx (mode);
10671 :
10672 0 : if ((TREE_CODE (TREE_TYPE (treeop0)) == INTEGER_TYPE
10673 0 : && TYPE_UNSIGNED (TREE_TYPE (treeop0)))
10674 0 : || (TREE_CODE (type) == INTEGER_TYPE && TYPE_UNSIGNED (type)))
10675 0 : expand_fixed_convert (target, op0, 1, TYPE_SATURATING (type));
10676 : else
10677 0 : expand_fixed_convert (target, op0, 0, TYPE_SATURATING (type));
10678 : return target;
10679 :
10680 48303 : case FIX_TRUNC_EXPR:
10681 48303 : op0 = expand_normal (treeop0);
10682 48303 : if (target == 0 || modifier == EXPAND_STACK_PARM)
10683 3427 : target = gen_reg_rtx (mode);
10684 48303 : expand_fix (target, op0, unsignedp);
10685 48303 : return target;
10686 :
10687 133287 : case FLOAT_EXPR:
10688 133287 : op0 = expand_normal (treeop0);
10689 133287 : if (target == 0 || modifier == EXPAND_STACK_PARM)
10690 62946 : target = gen_reg_rtx (mode);
10691 : /* expand_float can't figure out what to do if FROM has VOIDmode.
10692 : So give it the correct mode. With -O, cse will optimize this. */
10693 133287 : if (GET_MODE (op0) == VOIDmode)
10694 109 : op0 = copy_to_mode_reg (TYPE_MODE (TREE_TYPE (treeop0)),
10695 : op0);
10696 266574 : expand_float (target, op0,
10697 133287 : TYPE_UNSIGNED (TREE_TYPE (treeop0)));
10698 133287 : return target;
10699 :
10700 59231 : case NEGATE_EXPR:
10701 59231 : op0 = expand_expr (treeop0, subtarget,
10702 : VOIDmode, EXPAND_NORMAL);
10703 59231 : if (modifier == EXPAND_STACK_PARM)
10704 241 : target = 0;
10705 59231 : temp = expand_unop (mode,
10706 : optab_for_tree_code (NEGATE_EXPR, type,
10707 : optab_default),
10708 : op0, target, 0);
10709 59231 : gcc_assert (temp);
10710 59231 : return REDUCE_BIT_FIELD (temp);
10711 :
10712 28313 : case ABS_EXPR:
10713 28313 : case ABSU_EXPR:
10714 28313 : op0 = expand_expr (treeop0, subtarget,
10715 : VOIDmode, EXPAND_NORMAL);
10716 28313 : if (modifier == EXPAND_STACK_PARM)
10717 65 : target = 0;
10718 :
10719 : /* ABS_EXPR is not valid for complex arguments. */
10720 28313 : gcc_assert (GET_MODE_CLASS (mode) != MODE_COMPLEX_INT
10721 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_FLOAT);
10722 :
10723 : /* Unsigned abs is simply the operand. Testing here means we don't
10724 : risk generating incorrect code below. */
10725 28313 : if (TYPE_UNSIGNED (TREE_TYPE (treeop0)))
10726 : return op0;
10727 :
10728 28313 : return expand_abs (mode, op0, target, unsignedp,
10729 56626 : safe_from_p (target, treeop0, 1));
10730 :
10731 136366 : case MAX_EXPR:
10732 136366 : case MIN_EXPR:
10733 136366 : target = original_target;
10734 136366 : if (target == 0
10735 136366 : || modifier == EXPAND_STACK_PARM
10736 70606 : || (MEM_P (target) && MEM_VOLATILE_P (target))
10737 70606 : || GET_MODE (target) != mode
10738 206972 : || (REG_P (target)
10739 63991 : && REGNO (target) < FIRST_PSEUDO_REGISTER))
10740 65760 : target = gen_reg_rtx (mode);
10741 136366 : expand_operands (treeop0, treeop1,
10742 : target, &op0, &op1, EXPAND_NORMAL);
10743 :
10744 : /* First try to do it with a special MIN or MAX instruction.
10745 : If that does not win, use a conditional jump to select the proper
10746 : value. */
10747 136366 : this_optab = optab_for_tree_code (code, type, optab_default);
10748 136366 : temp = expand_binop (mode, this_optab, op0, op1, target, unsignedp,
10749 : OPTAB_WIDEN);
10750 136366 : if (temp != 0)
10751 : return temp;
10752 :
10753 122 : if (VECTOR_TYPE_P (type))
10754 0 : gcc_unreachable ();
10755 :
10756 : /* At this point, a MEM target is no longer useful; we will get better
10757 : code without it. */
10758 :
10759 122 : if (! REG_P (target))
10760 1 : target = gen_reg_rtx (mode);
10761 :
10762 : /* If op1 was placed in target, swap op0 and op1. */
10763 122 : if (target != op0 && target == op1)
10764 0 : std::swap (op0, op1);
10765 :
10766 : /* We generate better code and avoid problems with op1 mentioning
10767 : target by forcing op1 into a pseudo if it isn't a constant. */
10768 122 : if (! CONSTANT_P (op1))
10769 42 : op1 = force_reg (mode, op1);
10770 :
10771 122 : {
10772 122 : enum rtx_code comparison_code;
10773 122 : rtx cmpop1 = op1;
10774 :
10775 122 : if (code == MAX_EXPR)
10776 62 : comparison_code = unsignedp ? GEU : GE;
10777 : else
10778 60 : comparison_code = unsignedp ? LEU : LE;
10779 :
10780 : /* Canonicalize to comparisons against 0. */
10781 122 : if (op1 == const1_rtx)
10782 : {
10783 : /* Converting (a >= 1 ? a : 1) into (a > 0 ? a : 1)
10784 : or (a != 0 ? a : 1) for unsigned.
10785 : For MIN we are safe converting (a <= 1 ? a : 1)
10786 : into (a <= 0 ? a : 1) */
10787 0 : cmpop1 = const0_rtx;
10788 0 : if (code == MAX_EXPR)
10789 0 : comparison_code = unsignedp ? NE : GT;
10790 : }
10791 122 : if (op1 == constm1_rtx && !unsignedp)
10792 : {
10793 : /* Converting (a >= -1 ? a : -1) into (a >= 0 ? a : -1)
10794 : and (a <= -1 ? a : -1) into (a < 0 ? a : -1) */
10795 0 : cmpop1 = const0_rtx;
10796 0 : if (code == MIN_EXPR)
10797 0 : comparison_code = LT;
10798 : }
10799 :
10800 : /* Use a conditional move if possible. */
10801 122 : if (can_conditionally_move_p (mode))
10802 : {
10803 75 : rtx insn;
10804 :
10805 75 : start_sequence ();
10806 :
10807 : /* Try to emit the conditional move. */
10808 75 : insn = emit_conditional_move (target,
10809 : { comparison_code,
10810 : op0, cmpop1, mode },
10811 : op0, op1, mode,
10812 : unsignedp);
10813 :
10814 : /* If we could do the conditional move, emit the sequence,
10815 : and return. */
10816 75 : if (insn)
10817 : {
10818 41 : rtx_insn *seq = end_sequence ();
10819 41 : emit_insn (seq);
10820 41 : return target;
10821 : }
10822 :
10823 : /* Otherwise discard the sequence and fall back to code with
10824 : branches. */
10825 34 : end_sequence ();
10826 : }
10827 :
10828 81 : if (target != op0)
10829 52 : emit_move_insn (target, op0);
10830 :
10831 81 : lab = gen_label_rtx ();
10832 81 : do_compare_rtx_and_jump (target, cmpop1, comparison_code,
10833 : unsignedp, mode, NULL_RTX, NULL, lab,
10834 : profile_probability::uninitialized ());
10835 : }
10836 81 : emit_move_insn (target, op1);
10837 81 : emit_label (lab);
10838 81 : return target;
10839 :
10840 62174 : case BIT_NOT_EXPR:
10841 62174 : op0 = expand_expr (treeop0, subtarget,
10842 : VOIDmode, EXPAND_NORMAL);
10843 62174 : if (modifier == EXPAND_STACK_PARM)
10844 88 : target = 0;
10845 : /* In case we have to reduce the result to bitfield precision
10846 : for unsigned bitfield expand this as XOR with a proper constant
10847 : instead. */
10848 62174 : if (reduce_bit_field && TYPE_UNSIGNED (type))
10849 : {
10850 23258 : int_mode = SCALAR_INT_TYPE_MODE (type);
10851 23258 : wide_int mask = wi::mask (TYPE_PRECISION (type),
10852 46516 : false, GET_MODE_PRECISION (int_mode));
10853 :
10854 46516 : temp = expand_binop (int_mode, xor_optab, op0,
10855 46516 : immed_wide_int_const (mask, int_mode),
10856 : target, 1, OPTAB_LIB_WIDEN);
10857 23258 : }
10858 : else
10859 38916 : temp = expand_unop (mode, one_cmpl_optab, op0, target, 1);
10860 62174 : gcc_assert (temp);
10861 : return temp;
10862 :
10863 : /* ??? Can optimize bitwise operations with one arg constant.
10864 : Can optimize (a bitwise1 n) bitwise2 (a bitwise3 b)
10865 : and (a bitwise1 b) bitwise2 b (etc)
10866 : but that is probably not worth while. */
10867 :
10868 618838 : case BIT_AND_EXPR:
10869 618838 : case BIT_IOR_EXPR:
10870 618838 : case BIT_XOR_EXPR:
10871 618838 : goto binop;
10872 :
10873 8560 : case LROTATE_EXPR:
10874 8560 : case RROTATE_EXPR:
10875 8560 : gcc_assert (VECTOR_MODE_P (TYPE_MODE (type))
10876 : || type_has_mode_precision_p (type));
10877 : /* fall through */
10878 :
10879 305153 : case LSHIFT_EXPR:
10880 305153 : case RSHIFT_EXPR:
10881 305153 : {
10882 : /* If this is a fixed-point operation, then we cannot use the code
10883 : below because "expand_shift" doesn't support sat/no-sat fixed-point
10884 : shifts. */
10885 305153 : if (ALL_FIXED_POINT_MODE_P (mode))
10886 0 : goto binop;
10887 :
10888 305153 : if (! safe_from_p (subtarget, treeop1, 1))
10889 4976 : subtarget = 0;
10890 305153 : if (modifier == EXPAND_STACK_PARM)
10891 2586 : target = 0;
10892 305153 : op0 = expand_expr (treeop0, subtarget,
10893 : VOIDmode, EXPAND_NORMAL);
10894 :
10895 : /* Left shift optimization when shifting across word_size boundary.
10896 :
10897 : If mode == GET_MODE_WIDER_MODE (word_mode), then normally
10898 : there isn't native instruction to support this wide mode
10899 : left shift. Given below scenario:
10900 :
10901 : Type A = (Type) B << C
10902 :
10903 : |< T >|
10904 : | dest_high | dest_low |
10905 :
10906 : | word_size |
10907 :
10908 : If the shift amount C caused we shift B to across the word
10909 : size boundary, i.e part of B shifted into high half of
10910 : destination register, and part of B remains in the low
10911 : half, then GCC will use the following left shift expand
10912 : logic:
10913 :
10914 : 1. Initialize dest_low to B.
10915 : 2. Initialize every bit of dest_high to the sign bit of B.
10916 : 3. Logic left shift dest_low by C bit to finalize dest_low.
10917 : The value of dest_low before this shift is kept in a temp D.
10918 : 4. Logic left shift dest_high by C.
10919 : 5. Logic right shift D by (word_size - C).
10920 : 6. Or the result of 4 and 5 to finalize dest_high.
10921 :
10922 : While, by checking gimple statements, if operand B is
10923 : coming from signed extension, then we can simplify above
10924 : expand logic into:
10925 :
10926 : 1. dest_high = src_low >> (word_size - C).
10927 : 2. dest_low = src_low << C.
10928 :
10929 : We can use one arithmetic right shift to finish all the
10930 : purpose of steps 2, 4, 5, 6, thus we reduce the steps
10931 : needed from 6 into 2.
10932 :
10933 : The case is similar for zero extension, except that we
10934 : initialize dest_high to zero rather than copies of the sign
10935 : bit from B. Furthermore, we need to use a logical right shift
10936 : in this case.
10937 :
10938 : The choice of sign-extension versus zero-extension is
10939 : determined entirely by whether or not B is signed and is
10940 : independent of the current setting of unsignedp. */
10941 :
10942 305153 : temp = NULL_RTX;
10943 305153 : if (code == LSHIFT_EXPR
10944 305153 : && target
10945 33124 : && REG_P (target)
10946 30163 : && GET_MODE_2XWIDER_MODE (word_mode).exists (&int_mode)
10947 30163 : && mode == int_mode
10948 1505 : && TREE_CONSTANT (treeop1)
10949 305998 : && TREE_CODE (treeop0) == SSA_NAME)
10950 : {
10951 845 : gimple *def = SSA_NAME_DEF_STMT (treeop0);
10952 845 : if (is_gimple_assign (def)
10953 845 : && gimple_assign_rhs_code (def) == NOP_EXPR)
10954 : {
10955 334 : scalar_int_mode rmode = SCALAR_INT_TYPE_MODE
10956 : (TREE_TYPE (gimple_assign_rhs1 (def)));
10957 :
10958 668 : if (GET_MODE_SIZE (rmode) < GET_MODE_SIZE (int_mode)
10959 622 : && TREE_INT_CST_LOW (treeop1) < GET_MODE_BITSIZE (word_mode)
10960 476 : && ((TREE_INT_CST_LOW (treeop1) + GET_MODE_BITSIZE (rmode))
10961 71 : >= GET_MODE_BITSIZE (word_mode)))
10962 : {
10963 67 : rtx_insn *seq, *seq_old;
10964 67 : poly_uint64 high_off = subreg_highpart_offset (word_mode,
10965 : int_mode);
10966 67 : bool extend_unsigned
10967 67 : = TYPE_UNSIGNED (TREE_TYPE (gimple_assign_rhs1 (def)));
10968 67 : rtx low = lowpart_subreg (word_mode, op0, int_mode);
10969 67 : rtx dest_low = lowpart_subreg (word_mode, target, int_mode);
10970 67 : rtx dest_high = simplify_gen_subreg (word_mode, target,
10971 : int_mode, high_off);
10972 67 : HOST_WIDE_INT ramount = (BITS_PER_WORD
10973 67 : - TREE_INT_CST_LOW (treeop1));
10974 67 : tree rshift = build_int_cst (TREE_TYPE (treeop1), ramount);
10975 :
10976 67 : start_sequence ();
10977 : /* dest_high = src_low >> (word_size - C). */
10978 67 : temp = expand_variable_shift (RSHIFT_EXPR, word_mode, low,
10979 : rshift, dest_high,
10980 : extend_unsigned);
10981 67 : if (temp != dest_high)
10982 0 : emit_move_insn (dest_high, temp);
10983 :
10984 : /* dest_low = src_low << C. */
10985 67 : temp = expand_variable_shift (LSHIFT_EXPR, word_mode, low,
10986 : treeop1, dest_low, unsignedp);
10987 67 : if (temp != dest_low)
10988 2 : emit_move_insn (dest_low, temp);
10989 :
10990 67 : seq = end_sequence ();
10991 67 : temp = target ;
10992 :
10993 67 : if (have_insn_for (ASHIFT, int_mode))
10994 : {
10995 67 : bool speed_p = optimize_insn_for_speed_p ();
10996 67 : start_sequence ();
10997 67 : rtx ret_old = expand_variable_shift (code, int_mode,
10998 : op0, treeop1,
10999 : target,
11000 : unsignedp);
11001 :
11002 67 : seq_old = end_sequence ();
11003 134 : if (seq_cost (seq, speed_p)
11004 67 : >= seq_cost (seq_old, speed_p))
11005 : {
11006 67 : seq = seq_old;
11007 67 : temp = ret_old;
11008 : }
11009 : }
11010 67 : emit_insn (seq);
11011 : }
11012 : }
11013 : }
11014 :
11015 67 : if (temp == NULL_RTX)
11016 305086 : temp = expand_variable_shift (code, mode, op0, treeop1, target,
11017 : unsignedp);
11018 305153 : if (code == LSHIFT_EXPR)
11019 80437 : temp = REDUCE_BIT_FIELD (temp);
11020 : return temp;
11021 : }
11022 :
11023 : /* Could determine the answer when only additive constants differ. Also,
11024 : the addition of one can be handled by changing the condition. */
11025 643567 : case LT_EXPR:
11026 643567 : case LE_EXPR:
11027 643567 : case GT_EXPR:
11028 643567 : case GE_EXPR:
11029 643567 : case EQ_EXPR:
11030 643567 : case NE_EXPR:
11031 643567 : case UNORDERED_EXPR:
11032 643567 : case ORDERED_EXPR:
11033 643567 : case UNLT_EXPR:
11034 643567 : case UNLE_EXPR:
11035 643567 : case UNGT_EXPR:
11036 643567 : case UNGE_EXPR:
11037 643567 : case UNEQ_EXPR:
11038 643567 : case LTGT_EXPR:
11039 643567 : {
11040 1056578 : temp = do_store_flag (ops,
11041 : modifier != EXPAND_STACK_PARM ? target : NULL_RTX,
11042 : tmode != VOIDmode ? tmode : mode);
11043 643567 : if (temp)
11044 : return temp;
11045 :
11046 : /* Use a compare and a jump for BLKmode comparisons, or for function
11047 : type comparisons is have_canonicalize_funcptr_for_compare. */
11048 :
11049 0 : if ((target == 0
11050 0 : || modifier == EXPAND_STACK_PARM
11051 0 : || ! safe_from_p (target, treeop0, 1)
11052 0 : || ! safe_from_p (target, treeop1, 1)
11053 : /* Make sure we don't have a hard reg (such as function's return
11054 : value) live across basic blocks, if not optimizing. */
11055 0 : || (!optimize && REG_P (target)
11056 0 : && REGNO (target) < FIRST_PSEUDO_REGISTER)))
11057 0 : target = gen_reg_rtx (tmode != VOIDmode ? tmode : mode);
11058 :
11059 0 : emit_move_insn (target, const0_rtx);
11060 :
11061 0 : rtx_code_label *lab1 = gen_label_rtx ();
11062 0 : jumpifnot_1 (code, treeop0, treeop1, lab1,
11063 : profile_probability::uninitialized ());
11064 :
11065 0 : if (TYPE_PRECISION (type) == 1 && !TYPE_UNSIGNED (type))
11066 0 : emit_move_insn (target, constm1_rtx);
11067 : else
11068 0 : emit_move_insn (target, const1_rtx);
11069 :
11070 0 : emit_label (lab1);
11071 0 : return target;
11072 : }
11073 55972 : case COMPLEX_EXPR:
11074 : /* Get the rtx code of the operands. */
11075 55972 : op0 = expand_normal (treeop0);
11076 55972 : op1 = expand_normal (treeop1);
11077 :
11078 55972 : if (!target)
11079 3097 : target = gen_reg_rtx (TYPE_MODE (type));
11080 : else
11081 : /* If target overlaps with op1, then either we need to force
11082 : op1 into a pseudo (if target also overlaps with op0),
11083 : or write the complex parts in reverse order. */
11084 52875 : switch (GET_CODE (target))
11085 : {
11086 50121 : case CONCAT:
11087 50121 : if (reg_overlap_mentioned_p (XEXP (target, 0), op1))
11088 : {
11089 0 : if (reg_overlap_mentioned_p (XEXP (target, 1), op0))
11090 : {
11091 0 : complex_expr_force_op1:
11092 1724 : temp = gen_reg_rtx (GET_MODE_INNER (GET_MODE (target)));
11093 862 : emit_move_insn (temp, op1);
11094 862 : op1 = temp;
11095 862 : break;
11096 : }
11097 0 : complex_expr_swap_order:
11098 : /* Move the imaginary (op1) and real (op0) parts to their
11099 : location. */
11100 1 : write_complex_part (target, op1, true, true);
11101 1 : write_complex_part (target, op0, false, false);
11102 :
11103 1 : return target;
11104 : }
11105 : break;
11106 2754 : case MEM:
11107 5508 : temp = adjust_address_nv (target,
11108 : GET_MODE_INNER (GET_MODE (target)), 0);
11109 2754 : if (reg_overlap_mentioned_p (temp, op1))
11110 : {
11111 863 : scalar_mode imode = GET_MODE_INNER (GET_MODE (target));
11112 1726 : temp = adjust_address_nv (target, imode,
11113 : GET_MODE_SIZE (imode));
11114 863 : if (reg_overlap_mentioned_p (temp, op0))
11115 862 : goto complex_expr_force_op1;
11116 1 : goto complex_expr_swap_order;
11117 : }
11118 : break;
11119 0 : default:
11120 0 : if (reg_overlap_mentioned_p (target, op1))
11121 : {
11122 0 : if (reg_overlap_mentioned_p (target, op0))
11123 0 : goto complex_expr_force_op1;
11124 0 : goto complex_expr_swap_order;
11125 : }
11126 : break;
11127 : }
11128 :
11129 : /* Move the real (op0) and imaginary (op1) parts to their location. */
11130 55971 : write_complex_part (target, op0, false, true);
11131 55971 : write_complex_part (target, op1, true, false);
11132 :
11133 55971 : return target;
11134 :
11135 0 : case WIDEN_SUM_EXPR:
11136 0 : {
11137 0 : tree oprnd0 = treeop0;
11138 0 : tree oprnd1 = treeop1;
11139 :
11140 0 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11141 0 : target = expand_widen_pattern_expr (ops, op0, NULL_RTX, op1,
11142 : target, unsignedp);
11143 0 : return target;
11144 : }
11145 :
11146 18278 : case VEC_UNPACK_HI_EXPR:
11147 18278 : case VEC_UNPACK_LO_EXPR:
11148 18278 : case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
11149 18278 : case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
11150 18278 : {
11151 18278 : op0 = expand_normal (treeop0);
11152 18278 : temp = expand_widen_pattern_expr (ops, op0, NULL_RTX, NULL_RTX,
11153 : target, unsignedp);
11154 18278 : gcc_assert (temp);
11155 : return temp;
11156 : }
11157 :
11158 1754 : case VEC_UNPACK_FLOAT_HI_EXPR:
11159 1754 : case VEC_UNPACK_FLOAT_LO_EXPR:
11160 1754 : {
11161 1754 : op0 = expand_normal (treeop0);
11162 : /* The signedness is determined from input operand. */
11163 1754 : temp = expand_widen_pattern_expr
11164 3508 : (ops, op0, NULL_RTX, NULL_RTX,
11165 1754 : target, TYPE_UNSIGNED (TREE_TYPE (treeop0)));
11166 :
11167 1754 : gcc_assert (temp);
11168 : return temp;
11169 : }
11170 :
11171 995 : case VEC_WIDEN_MULT_HI_EXPR:
11172 995 : case VEC_WIDEN_MULT_LO_EXPR:
11173 995 : case VEC_WIDEN_MULT_EVEN_EXPR:
11174 995 : case VEC_WIDEN_MULT_ODD_EXPR:
11175 995 : case VEC_WIDEN_LSHIFT_HI_EXPR:
11176 995 : case VEC_WIDEN_LSHIFT_LO_EXPR:
11177 995 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11178 995 : target = expand_widen_pattern_expr (ops, op0, op1, NULL_RTX,
11179 : target, unsignedp);
11180 995 : gcc_assert (target);
11181 : return target;
11182 :
11183 370 : case VEC_PACK_SAT_EXPR:
11184 370 : case VEC_PACK_FIX_TRUNC_EXPR:
11185 370 : mode = TYPE_MODE (TREE_TYPE (treeop0));
11186 370 : subtarget = NULL_RTX;
11187 370 : goto binop;
11188 :
11189 11106 : case VEC_PACK_TRUNC_EXPR:
11190 11106 : if (VECTOR_BOOLEAN_TYPE_P (type)
11191 2478 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (treeop0))
11192 2478 : && mode == TYPE_MODE (TREE_TYPE (treeop0))
11193 11810 : && SCALAR_INT_MODE_P (mode))
11194 : {
11195 704 : class expand_operand eops[4];
11196 704 : machine_mode imode = TYPE_MODE (TREE_TYPE (treeop0));
11197 704 : expand_operands (treeop0, treeop1,
11198 : subtarget, &op0, &op1, EXPAND_NORMAL);
11199 704 : this_optab = vec_pack_sbool_trunc_optab;
11200 704 : enum insn_code icode = optab_handler (this_optab, imode);
11201 704 : create_output_operand (&eops[0], target, mode);
11202 704 : create_convert_operand_from (&eops[1], op0, imode, false);
11203 704 : create_convert_operand_from (&eops[2], op1, imode, false);
11204 704 : temp = GEN_INT (TYPE_VECTOR_SUBPARTS (type).to_constant ());
11205 704 : create_input_operand (&eops[3], temp, imode);
11206 704 : expand_insn (icode, 4, eops);
11207 704 : return eops[0].value;
11208 : }
11209 10402 : mode = TYPE_MODE (TREE_TYPE (treeop0));
11210 10402 : subtarget = NULL_RTX;
11211 10402 : goto binop;
11212 :
11213 27 : case VEC_PACK_FLOAT_EXPR:
11214 27 : mode = TYPE_MODE (TREE_TYPE (treeop0));
11215 27 : expand_operands (treeop0, treeop1,
11216 : subtarget, &op0, &op1, EXPAND_NORMAL);
11217 27 : this_optab = optab_for_tree_code (code, TREE_TYPE (treeop0),
11218 : optab_default);
11219 81 : target = expand_binop (mode, this_optab, op0, op1, target,
11220 27 : TYPE_UNSIGNED (TREE_TYPE (treeop0)),
11221 : OPTAB_LIB_WIDEN);
11222 27 : gcc_assert (target);
11223 : return target;
11224 :
11225 75963 : case VEC_PERM_EXPR:
11226 75963 : {
11227 75963 : expand_operands (treeop0, treeop1, target, &op0, &op1, EXPAND_NORMAL);
11228 75963 : vec_perm_builder sel;
11229 75963 : if (TREE_CODE (treeop2) == VECTOR_CST
11230 75963 : && tree_to_vec_perm_builder (&sel, treeop2))
11231 : {
11232 75945 : machine_mode sel_mode = TYPE_MODE (TREE_TYPE (treeop2));
11233 75945 : temp = expand_vec_perm_const (mode, op0, op1, sel,
11234 : sel_mode, target);
11235 : }
11236 : else
11237 : {
11238 18 : op2 = expand_normal (treeop2);
11239 18 : temp = expand_vec_perm_var (mode, op0, op1, op2, target);
11240 : }
11241 75963 : gcc_assert (temp);
11242 75963 : return temp;
11243 75963 : }
11244 :
11245 429 : case DOT_PROD_EXPR:
11246 429 : {
11247 429 : tree oprnd0 = treeop0;
11248 429 : tree oprnd1 = treeop1;
11249 429 : tree oprnd2 = treeop2;
11250 :
11251 429 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11252 429 : op2 = expand_normal (oprnd2);
11253 429 : target = expand_widen_pattern_expr (ops, op0, op1, op2,
11254 : target, unsignedp);
11255 429 : return target;
11256 : }
11257 :
11258 113 : case SAD_EXPR:
11259 113 : {
11260 113 : tree oprnd0 = treeop0;
11261 113 : tree oprnd1 = treeop1;
11262 113 : tree oprnd2 = treeop2;
11263 :
11264 113 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11265 113 : op2 = expand_normal (oprnd2);
11266 113 : target = expand_widen_pattern_expr (ops, op0, op1, op2,
11267 : target, unsignedp);
11268 113 : return target;
11269 : }
11270 :
11271 0 : case REALIGN_LOAD_EXPR:
11272 0 : {
11273 0 : tree oprnd0 = treeop0;
11274 0 : tree oprnd1 = treeop1;
11275 0 : tree oprnd2 = treeop2;
11276 :
11277 0 : this_optab = optab_for_tree_code (code, type, optab_default);
11278 0 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11279 0 : op2 = expand_normal (oprnd2);
11280 0 : temp = expand_ternary_op (mode, this_optab, op0, op1, op2,
11281 : target, unsignedp);
11282 0 : gcc_assert (temp);
11283 : return temp;
11284 : }
11285 :
11286 18096 : case COND_EXPR:
11287 18096 : {
11288 : /* A COND_EXPR with its type being VOID_TYPE represents a
11289 : conditional jump and is handled in
11290 : expand_gimple_cond_expr. */
11291 18096 : gcc_assert (!VOID_TYPE_P (type));
11292 :
11293 : /* Note that COND_EXPRs whose type is a structure or union
11294 : are required to be constructed to contain assignments of
11295 : a temporary variable, so that we can evaluate them here
11296 : for side effect only. If type is void, we must do likewise. */
11297 :
11298 18096 : gcc_assert (!TREE_ADDRESSABLE (type)
11299 : && !ignore
11300 : && TREE_TYPE (treeop1) != void_type_node
11301 : && TREE_TYPE (treeop2) != void_type_node);
11302 :
11303 18096 : temp = expand_cond_expr_using_cmove (treeop0, treeop1, treeop2);
11304 18096 : if (temp)
11305 : return temp;
11306 :
11307 : /* If we are not to produce a result, we have no target. Otherwise,
11308 : if a target was specified use it; it will not be used as an
11309 : intermediate target unless it is safe. If no target, use a
11310 : temporary. */
11311 :
11312 3839 : if (modifier != EXPAND_STACK_PARM
11313 3839 : && original_target
11314 2262 : && safe_from_p (original_target, treeop0, 1)
11315 0 : && GET_MODE (original_target) == mode
11316 3839 : && !MEM_P (original_target))
11317 : temp = original_target;
11318 : else
11319 3839 : temp = assign_temp (type, 0, 1);
11320 :
11321 3839 : do_pending_stack_adjust ();
11322 3839 : NO_DEFER_POP;
11323 3839 : rtx_code_label *lab0 = gen_label_rtx ();
11324 3839 : rtx_code_label *lab1 = gen_label_rtx ();
11325 3839 : jumpifnot (treeop0, lab0,
11326 : profile_probability::uninitialized ());
11327 3839 : store_expr (treeop1, temp,
11328 : modifier == EXPAND_STACK_PARM,
11329 : false, false);
11330 :
11331 3839 : emit_jump_insn (targetm.gen_jump (lab1));
11332 3839 : emit_barrier ();
11333 3839 : emit_label (lab0);
11334 3839 : store_expr (treeop2, temp,
11335 : modifier == EXPAND_STACK_PARM,
11336 : false, false);
11337 :
11338 3839 : emit_label (lab1);
11339 3839 : OK_DEFER_POP;
11340 3839 : return temp;
11341 : }
11342 :
11343 0 : case VEC_DUPLICATE_EXPR:
11344 0 : op0 = expand_expr (treeop0, NULL_RTX, VOIDmode, modifier);
11345 0 : target = expand_vector_broadcast (mode, op0);
11346 0 : gcc_assert (target);
11347 : return target;
11348 :
11349 0 : case VEC_SERIES_EXPR:
11350 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, modifier);
11351 0 : return expand_vec_series_expr (mode, op0, op1, target);
11352 :
11353 1226 : case BIT_INSERT_EXPR:
11354 1226 : {
11355 1226 : unsigned bitpos = tree_to_uhwi (treeop2);
11356 1226 : unsigned bitsize;
11357 1226 : if (INTEGRAL_TYPE_P (TREE_TYPE (treeop1)))
11358 881 : bitsize = TYPE_PRECISION (TREE_TYPE (treeop1));
11359 : else
11360 345 : bitsize = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (treeop1)));
11361 1226 : op0 = expand_normal (treeop0);
11362 1226 : op1 = expand_normal (treeop1);
11363 1226 : rtx dst = gen_reg_rtx (mode);
11364 1226 : emit_move_insn (dst, op0);
11365 1226 : store_bit_field (dst, bitsize, bitpos, 0, 0,
11366 1226 : TYPE_MODE (TREE_TYPE (treeop1)), op1, false, false);
11367 1226 : return dst;
11368 : }
11369 :
11370 0 : default:
11371 0 : gcc_unreachable ();
11372 : }
11373 :
11374 : /* Here to do an ordinary binary operator. */
11375 1210369 : binop:
11376 1210369 : expand_operands (treeop0, treeop1,
11377 : subtarget, &op0, &op1, EXPAND_NORMAL);
11378 5735333 : binop2:
11379 5735333 : this_optab = optab_for_tree_code (code, type, optab_default);
11380 5735333 : binop3:
11381 5735333 : if (modifier == EXPAND_STACK_PARM)
11382 27181 : target = 0;
11383 5735333 : temp = expand_binop (mode, this_optab, op0, op1, target,
11384 : unsignedp, OPTAB_LIB_WIDEN);
11385 5735333 : gcc_assert (temp);
11386 : /* Bitwise operations do not need bitfield reduction as we expect their
11387 : operands being properly truncated. */
11388 5735333 : if (code == BIT_XOR_EXPR
11389 : || code == BIT_AND_EXPR
11390 5735333 : || code == BIT_IOR_EXPR)
11391 : return temp;
11392 5116495 : return REDUCE_BIT_FIELD (temp);
11393 : }
11394 : #undef REDUCE_BIT_FIELD
11395 :
11396 :
11397 : /* Return TRUE if expression STMT is suitable for replacement.
11398 : Never consider memory loads as replaceable, because those don't ever lead
11399 : into constant expressions. */
11400 :
11401 : static bool
11402 8 : stmt_is_replaceable_p (gimple *stmt)
11403 : {
11404 8 : if (ssa_is_replaceable_p (stmt))
11405 : {
11406 : /* Don't move around loads. */
11407 7 : if (!gimple_assign_single_p (stmt)
11408 7 : || is_gimple_val (gimple_assign_rhs1 (stmt)))
11409 6 : return true;
11410 : }
11411 : return false;
11412 : }
11413 :
11414 : /* A subroutine of expand_expr_real_1. Expand gimple assignment G,
11415 : which is known to set an SSA_NAME result. The other arguments are
11416 : as for expand_expr_real_1. */
11417 :
11418 : rtx
11419 15253371 : expand_expr_real_gassign (gassign *g, rtx target, machine_mode tmode,
11420 : enum expand_modifier modifier, rtx *alt_rtl,
11421 : bool inner_reference_p)
11422 : {
11423 15253371 : separate_ops ops;
11424 15253371 : rtx r;
11425 15253371 : location_t saved_loc = curr_insn_location ();
11426 15253371 : auto loc = gimple_location (g);
11427 15253371 : if (loc != UNKNOWN_LOCATION)
11428 12213334 : set_curr_insn_location (loc);
11429 15253371 : tree lhs = gimple_assign_lhs (g);
11430 15253371 : ops.code = gimple_assign_rhs_code (g);
11431 15253371 : ops.type = TREE_TYPE (lhs);
11432 15253371 : switch (get_gimple_rhs_class (ops.code))
11433 : {
11434 95664 : case GIMPLE_TERNARY_RHS:
11435 191328 : ops.op2 = gimple_assign_rhs3 (g);
11436 : /* Fallthru */
11437 8183807 : case GIMPLE_BINARY_RHS:
11438 8183807 : ops.op1 = gimple_assign_rhs2 (g);
11439 :
11440 : /* Try to expand conditional compare. */
11441 8183807 : if (targetm.have_ccmp ())
11442 : {
11443 164994 : gcc_checking_assert (targetm.gen_ccmp_next != NULL);
11444 164994 : r = expand_ccmp_expr (g, TYPE_MODE (ops.type));
11445 164994 : if (r)
11446 : break;
11447 : }
11448 : /* Fallthru */
11449 11765406 : case GIMPLE_UNARY_RHS:
11450 11765406 : ops.op0 = gimple_assign_rhs1 (g);
11451 11765406 : ops.location = loc;
11452 11765406 : r = expand_expr_real_2 (&ops, target, tmode, modifier);
11453 11765406 : break;
11454 3487945 : case GIMPLE_SINGLE_RHS:
11455 3487945 : {
11456 3487945 : r = expand_expr_real (gimple_assign_rhs1 (g), target,
11457 : tmode, modifier, alt_rtl,
11458 : inner_reference_p);
11459 3487945 : break;
11460 : }
11461 0 : default:
11462 0 : gcc_unreachable ();
11463 : }
11464 15253371 : set_curr_insn_location (saved_loc);
11465 15253371 : if (REG_P (r) && !REG_EXPR (r))
11466 3994104 : set_reg_attrs_for_decl_rtl (lhs, r);
11467 15253371 : return r;
11468 : }
11469 :
11470 : /* A subroutine of expand_expr_real_1. Attempt to VIEW_CONVERT_EXPR
11471 : the complex expression OP0 to the vector mode MODE. Store the
11472 : result at TARGET if possible (if TARGET is nonzero). Returns
11473 : NULL_RTX on failure. */
11474 : static rtx
11475 6662 : try_expand_complex_as_vector (machine_mode mode, rtx op0, rtx target)
11476 : {
11477 6662 : if (COMPLEX_MODE_P (GET_MODE (op0))
11478 1404 : && VECTOR_MODE_P (mode)
11479 58 : && known_eq (GET_MODE_NUNITS (mode), 2)
11480 6720 : && GET_MODE_INNER (mode) == GET_MODE_INNER (GET_MODE (op0)))
11481 : {
11482 29 : enum insn_code icode = convert_optab_handler (vec_init_optab, mode,
11483 29 : GET_MODE_INNER (mode));
11484 29 : if (icode != CODE_FOR_nothing)
11485 : {
11486 29 : if (!target || !REG_P (target))
11487 16 : target = gen_reg_rtx (mode);
11488 29 : rtx rpart = read_complex_part (op0, false);
11489 29 : rtx ipart = read_complex_part (op0, true);
11490 29 : if (!REG_P (rpart) && !CONSTANT_P (rpart))
11491 0 : rpart = force_reg (GET_MODE_INNER (mode), rpart);
11492 29 : if (!REG_P (ipart) && !CONSTANT_P (ipart))
11493 0 : ipart = force_reg (GET_MODE_INNER (mode), ipart);
11494 29 : rtvec vec = rtvec_alloc (2);
11495 29 : RTVEC_ELT (vec, 0) = rpart;
11496 29 : RTVEC_ELT (vec, 1) = ipart;
11497 29 : rtx par = gen_rtx_PARALLEL (mode, vec);
11498 29 : rtx_insn *insn = GEN_FCN (icode) (target, par);
11499 29 : if (insn)
11500 : {
11501 29 : emit_insn (insn);
11502 29 : return target;
11503 : }
11504 : }
11505 : }
11506 : return NULL_RTX;
11507 : }
11508 :
11509 : rtx
11510 159542845 : expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
11511 : enum expand_modifier modifier, rtx *alt_rtl,
11512 : bool inner_reference_p)
11513 : {
11514 159542845 : rtx op0, op1, temp, decl_rtl;
11515 159542845 : tree type;
11516 159542845 : int unsignedp;
11517 159542845 : machine_mode mode, dmode;
11518 159542845 : enum tree_code code = TREE_CODE (exp);
11519 159542845 : rtx subtarget, original_target;
11520 159542845 : int ignore;
11521 159542845 : bool reduce_bit_field;
11522 159542845 : location_t loc = EXPR_LOCATION (exp);
11523 159542845 : struct separate_ops ops;
11524 159542845 : tree treeop0, treeop1, treeop2;
11525 159542845 : tree ssa_name = NULL_TREE;
11526 159542845 : gimple *g;
11527 :
11528 : /* Some ABIs define padding bits in _BitInt uninitialized. Normally, RTL
11529 : expansion sign/zero extends integral types with less than mode precision
11530 : when reading from bit-fields and after arithmetic operations (see
11531 : REDUCE_BIT_FIELD in expand_expr_real_2) and on subsequent loads relies
11532 : on those extensions to have been already performed, but because of the
11533 : above for _BitInt they need to be sign/zero extended when reading from
11534 : locations that could be exposed to ABI boundaries (when loading from
11535 : objects in memory, or function arguments, return value). Because we
11536 : internally extend after arithmetic operations, we can avoid doing that
11537 : when reading from SSA_NAMEs of vars. */
11538 : #define EXTEND_BITINT(expr) \
11539 : ((BITINT_TYPE_P (type) \
11540 : && !bitint_extended \
11541 : && reduce_bit_field \
11542 : && mode != BLKmode \
11543 : && modifier != EXPAND_MEMORY \
11544 : && modifier != EXPAND_WRITE \
11545 : && modifier != EXPAND_INITIALIZER \
11546 : && modifier != EXPAND_CONST_ADDRESS) \
11547 : ? reduce_to_bit_field_precision ((expr), NULL_RTX, type) : (expr))
11548 :
11549 159542845 : type = TREE_TYPE (exp);
11550 159542845 : mode = TYPE_MODE (type);
11551 159542845 : unsignedp = TYPE_UNSIGNED (type);
11552 159542845 : if (BITINT_TYPE_P (type) && bitint_extended == -1)
11553 : {
11554 9099 : struct bitint_info info;
11555 9099 : bool ok = targetm.c.bitint_type_info (TYPE_PRECISION (type), &info);
11556 9099 : gcc_assert (ok);
11557 9099 : bitint_extended = info.extended;
11558 : }
11559 :
11560 159542845 : treeop0 = treeop1 = treeop2 = NULL_TREE;
11561 159542845 : if (!VL_EXP_CLASS_P (exp))
11562 152731085 : switch (TREE_CODE_LENGTH (code))
11563 : {
11564 5602148 : default:
11565 5602148 : case 3: treeop2 = TREE_OPERAND (exp, 2); /* FALLTHRU */
11566 13727147 : case 2: treeop1 = TREE_OPERAND (exp, 1); /* FALLTHRU */
11567 28311036 : case 1: treeop0 = TREE_OPERAND (exp, 0); /* FALLTHRU */
11568 : case 0: break;
11569 : }
11570 159542845 : ops.code = code;
11571 159542845 : ops.type = type;
11572 159542845 : ops.op0 = treeop0;
11573 159542845 : ops.op1 = treeop1;
11574 159542845 : ops.op2 = treeop2;
11575 159542845 : ops.location = loc;
11576 :
11577 319085690 : ignore = (target == const0_rtx
11578 159542845 : || ((CONVERT_EXPR_CODE_P (code)
11579 154432384 : || code == COND_EXPR || code == VIEW_CONVERT_EXPR)
11580 963111 : && TREE_CODE (type) == VOID_TYPE));
11581 :
11582 : /* An operation in what may be a bit-field type needs the
11583 : result to be reduced to the precision of the bit-field type,
11584 : which is narrower than that of the type's mode. */
11585 319085454 : reduce_bit_field = (!ignore
11586 155043364 : && INTEGRAL_TYPE_P (type)
11587 81753483 : && !type_has_mode_precision_p (type));
11588 :
11589 : /* If we are going to ignore this result, we need only do something
11590 : if there is a side-effect somewhere in the expression. If there
11591 : is, short-circuit the most common cases here. Note that we must
11592 : not call expand_expr with anything but const0_rtx in case this
11593 : is an initial expansion of a size that contains a PLACEHOLDER_EXPR. */
11594 :
11595 : if (ignore)
11596 : {
11597 4499481 : if (! TREE_SIDE_EFFECTS (exp))
11598 : return const0_rtx;
11599 :
11600 : /* Ensure we reference a volatile object even if value is ignored, but
11601 : don't do this if all we are doing is taking its address. */
11602 4499245 : if (TREE_THIS_VOLATILE (exp)
11603 0 : && TREE_CODE (exp) != FUNCTION_DECL
11604 0 : && mode != VOIDmode && mode != BLKmode
11605 0 : && modifier != EXPAND_CONST_ADDRESS)
11606 : {
11607 0 : temp = expand_expr (exp, NULL_RTX, VOIDmode, modifier);
11608 0 : if (MEM_P (temp))
11609 0 : copy_to_reg (temp);
11610 0 : return const0_rtx;
11611 : }
11612 :
11613 4499245 : if (TREE_CODE_CLASS (code) == tcc_unary
11614 : || code == BIT_FIELD_REF
11615 4499245 : || code == COMPONENT_REF
11616 4499245 : || code == INDIRECT_REF)
11617 0 : return expand_expr (treeop0, const0_rtx, VOIDmode,
11618 0 : modifier);
11619 :
11620 4499245 : else if (TREE_CODE_CLASS (code) == tcc_binary
11621 4499245 : || TREE_CODE_CLASS (code) == tcc_comparison
11622 4499245 : || code == ARRAY_REF || code == ARRAY_RANGE_REF)
11623 : {
11624 0 : expand_expr (treeop0, const0_rtx, VOIDmode, modifier);
11625 0 : expand_expr (treeop1, const0_rtx, VOIDmode, modifier);
11626 0 : return const0_rtx;
11627 : }
11628 :
11629 : target = 0;
11630 : }
11631 :
11632 159542609 : if (reduce_bit_field && modifier == EXPAND_STACK_PARM)
11633 39592 : target = 0;
11634 :
11635 : /* Use subtarget as the target for operand 0 of a binary operation. */
11636 159542609 : subtarget = get_subtarget (target);
11637 159542609 : original_target = target;
11638 :
11639 159542609 : switch (code)
11640 : {
11641 10821 : case LABEL_DECL:
11642 10821 : {
11643 10821 : tree function = decl_function_context (exp);
11644 :
11645 10821 : temp = label_rtx (exp);
11646 14765 : temp = gen_rtx_LABEL_REF (Pmode, temp);
11647 :
11648 10821 : if (function != current_function_decl
11649 1270 : && function != 0)
11650 1270 : LABEL_REF_NONLOCAL_P (temp) = 1;
11651 :
11652 10821 : temp = gen_rtx_MEM (FUNCTION_MODE, temp);
11653 10821 : return temp;
11654 : }
11655 :
11656 57665394 : case SSA_NAME:
11657 : /* ??? ivopts calls expander, without any preparation from
11658 : out-of-ssa. So fake instructions as if this was an access to the
11659 : base variable. This unnecessarily allocates a pseudo, see how we can
11660 : reuse it, if partition base vars have it set already. */
11661 57665394 : if (!currently_expanding_to_rtl)
11662 : {
11663 0 : tree var = SSA_NAME_VAR (exp);
11664 0 : if (var && DECL_RTL_SET_P (var))
11665 0 : return DECL_RTL (var);
11666 0 : return gen_raw_REG (TYPE_MODE (TREE_TYPE (exp)),
11667 0 : LAST_VIRTUAL_REGISTER + 1);
11668 : }
11669 :
11670 57665394 : g = get_gimple_for_ssa_name (exp);
11671 : /* For EXPAND_INITIALIZER try harder to get something simpler. */
11672 57665394 : if (g == NULL
11673 57665394 : && modifier == EXPAND_INITIALIZER
11674 26 : && !SSA_NAME_IS_DEFAULT_DEF (exp)
11675 11 : && (optimize || !SSA_NAME_VAR (exp)
11676 1 : || DECL_IGNORED_P (SSA_NAME_VAR (exp)))
11677 10 : && is_gimple_assign (SSA_NAME_DEF_STMT (exp))
11678 57665402 : && stmt_is_replaceable_p (SSA_NAME_DEF_STMT (exp)))
11679 6 : g = SSA_NAME_DEF_STMT (exp);
11680 57665394 : if (safe_is_a <gassign *> (g))
11681 9206442 : return expand_expr_real_gassign (as_a<gassign *> (g), target, tmode,
11682 9206442 : modifier, alt_rtl, inner_reference_p);
11683 48458952 : else if (safe_is_a <gcall *> (g))
11684 : {
11685 : /* ??? internal call expansion doesn't follow the usual API
11686 : of returning the destination RTX and being passed a desired
11687 : target. */
11688 75022 : if (modifier == EXPAND_WRITE)
11689 37515 : return DECL_RTL (SSA_NAME_VAR (exp));
11690 37507 : rtx dest = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
11691 37507 : tree tmplhs = make_tree (TREE_TYPE (exp), dest);
11692 37507 : tree var_or_id = SSA_NAME_VAR (exp);
11693 : if (!var_or_id)
11694 27458 : var_or_id = SSA_NAME_IDENTIFIER (exp);
11695 37507 : SET_SSA_NAME_VAR_OR_IDENTIFIER (exp, tmplhs);
11696 37507 : expand_internal_call (as_a <gcall *> (g));
11697 37507 : SET_SSA_NAME_VAR_OR_IDENTIFIER (exp, var_or_id);
11698 37507 : return dest;
11699 : }
11700 :
11701 48383930 : ssa_name = exp;
11702 48383930 : decl_rtl = get_rtx_for_ssa_name (ssa_name);
11703 48383930 : exp = SSA_NAME_VAR (ssa_name);
11704 : /* Optimize and avoid to EXTEND_BITINIT doing anything if it is an
11705 : SSA_NAME computed within the current function. In such case the
11706 : value have been already extended before. While if it is a function
11707 : parameter, result or some memory location, we need to be prepared
11708 : for some other compiler leaving the bits uninitialized. */
11709 18903403 : if (!exp || VAR_P (exp))
11710 : reduce_bit_field = false;
11711 48383930 : goto expand_decl_rtl;
11712 :
11713 19871232 : case VAR_DECL:
11714 : /* Allow accel compiler to handle variables that require special
11715 : treatment, e.g. if they have been modified in some way earlier in
11716 : compilation by the adjust_private_decl OpenACC hook. */
11717 19871232 : if (flag_openacc && targetm.goacc.expand_var_decl)
11718 : {
11719 0 : temp = targetm.goacc.expand_var_decl (exp);
11720 0 : if (temp)
11721 : return temp;
11722 : }
11723 : /* Expand const VAR_DECLs with CONSTRUCTOR initializers that
11724 : have scalar integer modes to a reg via store_constructor. */
11725 19871232 : if (TREE_READONLY (exp)
11726 3379287 : && !TREE_SIDE_EFFECTS (exp)
11727 3356879 : && (modifier == EXPAND_NORMAL || modifier == EXPAND_STACK_PARM)
11728 18634 : && immediate_const_ctor_p (DECL_INITIAL (exp))
11729 163 : && SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (exp)))
11730 127 : && crtl->emit.regno_pointer_align_length
11731 19871359 : && !target)
11732 : {
11733 88 : target = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
11734 88 : store_constructor (DECL_INITIAL (exp), target, 0,
11735 88 : int_expr_size (DECL_INITIAL (exp)), false);
11736 88 : return target;
11737 : }
11738 : /* ... fall through ... */
11739 :
11740 20416234 : case PARM_DECL:
11741 : /* If a static var's type was incomplete when the decl was written,
11742 : but the type is complete now, lay out the decl now. */
11743 20416234 : if (DECL_SIZE (exp) == 0
11744 45232 : && COMPLETE_OR_UNBOUND_ARRAY_TYPE_P (TREE_TYPE (exp))
11745 20461354 : && (TREE_STATIC (exp) || DECL_EXTERNAL (exp)))
11746 45120 : layout_decl (exp, 0);
11747 :
11748 : /* fall through */
11749 :
11750 21885202 : case FUNCTION_DECL:
11751 21885202 : case RESULT_DECL:
11752 21885202 : decl_rtl = DECL_RTL (exp);
11753 70269132 : expand_decl_rtl:
11754 70269132 : gcc_assert (decl_rtl);
11755 :
11756 : /* DECL_MODE might change when TYPE_MODE depends on attribute target
11757 : settings for VECTOR_TYPE_P that might switch for the function. */
11758 70269132 : if (currently_expanding_to_rtl
11759 66586500 : && code == VAR_DECL && MEM_P (decl_rtl)
11760 85109615 : && VECTOR_TYPE_P (type) && exp && DECL_MODE (exp) != mode)
11761 61 : decl_rtl = change_address (decl_rtl, TYPE_MODE (type), 0);
11762 : else
11763 70269071 : decl_rtl = copy_rtx (decl_rtl);
11764 :
11765 : /* Record writes to register variables. */
11766 70269132 : if (modifier == EXPAND_WRITE
11767 22072007 : && REG_P (decl_rtl)
11768 86416188 : && HARD_REGISTER_P (decl_rtl))
11769 2027 : add_to_hard_reg_set (&crtl->asm_clobbers,
11770 2027 : GET_MODE (decl_rtl), REGNO (decl_rtl));
11771 :
11772 : /* Ensure variable marked as used even if it doesn't go through
11773 : a parser. If it hasn't be used yet, write out an external
11774 : definition. */
11775 70269132 : if (exp)
11776 40788605 : TREE_USED (exp) = 1;
11777 :
11778 : /* Show we haven't gotten RTL for this yet. */
11779 111057737 : temp = 0;
11780 :
11781 : /* Variables inherited from containing functions should have
11782 : been lowered by this point. */
11783 40788605 : if (exp)
11784 : {
11785 40788605 : tree context = decl_function_context (exp);
11786 40788605 : gcc_assert (SCOPE_FILE_SCOPE_P (context)
11787 : || context == current_function_decl
11788 : || TREE_STATIC (exp)
11789 : || DECL_EXTERNAL (exp)
11790 : /* ??? C++ creates functions that are not
11791 : TREE_STATIC. */
11792 : || TREE_CODE (exp) == FUNCTION_DECL);
11793 : }
11794 :
11795 : /* This is the case of an array whose size is to be determined
11796 : from its initializer, while the initializer is still being parsed.
11797 : ??? We aren't parsing while expanding anymore. */
11798 :
11799 70269132 : if (MEM_P (decl_rtl) && REG_P (XEXP (decl_rtl, 0)))
11800 394922 : temp = validize_mem (decl_rtl);
11801 :
11802 : /* If DECL_RTL is memory, we are in the normal case and the
11803 : address is not valid, get the address into a register. */
11804 :
11805 69874210 : else if (MEM_P (decl_rtl) && modifier != EXPAND_INITIALIZER)
11806 : {
11807 18934518 : if (alt_rtl)
11808 2032047 : *alt_rtl = decl_rtl;
11809 18934518 : decl_rtl = use_anchored_address (decl_rtl);
11810 18934518 : if (modifier != EXPAND_CONST_ADDRESS
11811 18934518 : && modifier != EXPAND_SUM
11812 31342833 : && !memory_address_addr_space_p (exp ? DECL_MODE (exp)
11813 17568 : : GET_MODE (decl_rtl),
11814 : XEXP (decl_rtl, 0),
11815 12408315 : MEM_ADDR_SPACE (decl_rtl)))
11816 137277 : temp = replace_equiv_address (decl_rtl,
11817 : copy_rtx (XEXP (decl_rtl, 0)));
11818 : }
11819 :
11820 : /* If we got something, return it. But first, set the alignment
11821 : if the address is a register. */
11822 19329440 : if (temp != 0)
11823 : {
11824 532199 : if (exp && MEM_P (temp) && REG_P (XEXP (temp, 0)))
11825 500176 : mark_reg_pointer (XEXP (temp, 0), DECL_ALIGN (exp));
11826 : }
11827 69736933 : else if (MEM_P (decl_rtl))
11828 : temp = decl_rtl;
11829 :
11830 47924767 : if (temp != 0)
11831 : {
11832 22844551 : if (MEM_P (temp)
11833 : && modifier != EXPAND_WRITE
11834 22844551 : && modifier != EXPAND_MEMORY
11835 : && modifier != EXPAND_INITIALIZER
11836 16858998 : && modifier != EXPAND_CONST_ADDRESS
11837 6985041 : && modifier != EXPAND_SUM
11838 6985041 : && !inner_reference_p
11839 4584586 : && mode != BLKmode
11840 27061412 : && MEM_ALIGN (temp) < GET_MODE_ALIGNMENT (mode))
11841 20100 : temp = expand_misaligned_mem_ref (temp, mode, unsignedp,
11842 20100 : MEM_ALIGN (temp), NULL_RTX, NULL);
11843 :
11844 22844551 : return EXTEND_BITINT (temp);
11845 : }
11846 :
11847 47424581 : if (exp)
11848 17961632 : dmode = DECL_MODE (exp);
11849 : else
11850 29462949 : dmode = TYPE_MODE (TREE_TYPE (ssa_name));
11851 :
11852 : /* If the mode of DECL_RTL does not match that of the decl,
11853 : there are two cases: we are dealing with a BLKmode value
11854 : that is returned in a register, or we are dealing with
11855 : a promoted value. In the latter case, return a SUBREG
11856 : of the wanted mode, but mark it so that we know that it
11857 : was already extended. */
11858 47424581 : if (REG_P (decl_rtl)
11859 46887045 : && dmode != BLKmode
11860 46887045 : && GET_MODE (decl_rtl) != dmode)
11861 : {
11862 82 : machine_mode pmode;
11863 :
11864 : /* Get the signedness to be used for this variable. Ensure we get
11865 : the same mode we got when the variable was declared. */
11866 82 : if (code != SSA_NAME)
11867 0 : pmode = promote_decl_mode (exp, &unsignedp);
11868 82 : else if ((g = SSA_NAME_DEF_STMT (ssa_name))
11869 82 : && gimple_code (g) == GIMPLE_CALL
11870 84 : && !gimple_call_internal_p (g))
11871 2 : pmode = promote_function_mode (type, mode, &unsignedp,
11872 2 : gimple_call_fntype (g),
11873 : 2);
11874 : else
11875 80 : pmode = promote_ssa_mode (ssa_name, &unsignedp);
11876 82 : gcc_assert (GET_MODE (decl_rtl) == pmode);
11877 :
11878 : /* Some ABIs require scalar floating point modes to be passed
11879 : in a wider scalar integer mode. We need to explicitly
11880 : truncate to an integer mode of the correct precision before
11881 : using a SUBREG to reinterpret as a floating point value. */
11882 82 : if (SCALAR_FLOAT_MODE_P (mode)
11883 0 : && SCALAR_INT_MODE_P (pmode)
11884 82 : && known_lt (GET_MODE_SIZE (mode), GET_MODE_SIZE (pmode)))
11885 0 : return convert_wider_int_to_float (mode, pmode, decl_rtl);
11886 :
11887 82 : temp = gen_lowpart_SUBREG (mode, decl_rtl);
11888 82 : SUBREG_PROMOTED_VAR_P (temp) = 1;
11889 82 : SUBREG_PROMOTED_SET (temp, unsignedp);
11890 82 : return EXTEND_BITINT (temp);
11891 : }
11892 :
11893 47424499 : return EXTEND_BITINT (decl_rtl);
11894 :
11895 43035787 : case INTEGER_CST:
11896 43035787 : {
11897 43035787 : if (BITINT_TYPE_P (type))
11898 : {
11899 11849 : unsigned int prec = TYPE_PRECISION (type);
11900 11849 : struct bitint_info info;
11901 11849 : bool ok = targetm.c.bitint_type_info (prec, &info);
11902 11849 : gcc_assert (ok);
11903 11849 : scalar_int_mode limb_mode
11904 11849 : = as_a <scalar_int_mode> (info.limb_mode);
11905 11849 : unsigned int limb_prec = GET_MODE_PRECISION (limb_mode);
11906 19467 : if (prec > limb_prec && prec > MAX_FIXED_MODE_SIZE)
11907 : {
11908 : /* Emit large/huge _BitInt INTEGER_CSTs into memory. */
11909 5159 : exp = tree_output_constant_def (exp);
11910 5159 : return expand_expr (exp, target, VOIDmode, modifier);
11911 : }
11912 : }
11913 :
11914 : /* Given that TYPE_PRECISION (type) is not always equal to
11915 : GET_MODE_PRECISION (TYPE_MODE (type)), we need to extend from
11916 : the former to the latter according to the signedness of the
11917 : type. */
11918 43030628 : scalar_int_mode int_mode = SCALAR_INT_TYPE_MODE (type);
11919 43030628 : temp = immed_wide_int_const
11920 43030628 : (wi::to_wide (exp, GET_MODE_PRECISION (int_mode)), int_mode);
11921 43030628 : return temp;
11922 : }
11923 :
11924 573338 : case VECTOR_CST:
11925 573338 : {
11926 573338 : tree tmp = NULL_TREE;
11927 573338 : if (VECTOR_MODE_P (mode))
11928 570469 : return const_vector_from_tree (exp);
11929 2869 : scalar_int_mode int_mode;
11930 2869 : if (is_int_mode (mode, &int_mode))
11931 : {
11932 926 : tree type_for_mode = lang_hooks.types.type_for_mode (int_mode, 1);
11933 926 : if (type_for_mode)
11934 926 : tmp = fold_unary_loc (loc, VIEW_CONVERT_EXPR,
11935 : type_for_mode, exp);
11936 : }
11937 926 : if (!tmp)
11938 : {
11939 1943 : vec<constructor_elt, va_gc> *v;
11940 : /* Constructors need to be fixed-length. FIXME. */
11941 1943 : unsigned int nunits = VECTOR_CST_NELTS (exp).to_constant ();
11942 1943 : vec_alloc (v, nunits);
11943 31617 : for (unsigned int i = 0; i < nunits; ++i)
11944 27731 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, VECTOR_CST_ELT (exp, i));
11945 1943 : tmp = build_constructor (type, v);
11946 : }
11947 2869 : return expand_expr (tmp, ignore ? const0_rtx : target,
11948 2869 : tmode, modifier);
11949 : }
11950 :
11951 136 : case CONST_DECL:
11952 136 : if (modifier == EXPAND_WRITE)
11953 : {
11954 : /* Writing into CONST_DECL is always invalid, but handle it
11955 : gracefully. */
11956 2 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (exp));
11957 2 : scalar_int_mode address_mode = targetm.addr_space.address_mode (as);
11958 2 : op0 = expand_expr_addr_expr_1 (exp, NULL_RTX, address_mode,
11959 : EXPAND_NORMAL, as);
11960 2 : op0 = memory_address_addr_space (mode, op0, as);
11961 2 : temp = gen_rtx_MEM (mode, op0);
11962 2 : set_mem_addr_space (temp, as);
11963 2 : return temp;
11964 : }
11965 134 : return expand_expr (DECL_INITIAL (exp), target, VOIDmode, modifier);
11966 :
11967 866367 : case REAL_CST:
11968 : /* If optimized, generate immediate CONST_DOUBLE
11969 : which will be turned into memory by reload if necessary.
11970 :
11971 : We used to force a register so that loop.c could see it. But
11972 : this does not allow gen_* patterns to perform optimizations with
11973 : the constants. It also produces two insns in cases like "x = 1.0;".
11974 : On most machines, floating-point constants are not permitted in
11975 : many insns, so we'd end up copying it to a register in any case.
11976 :
11977 : Now, we do the copying in expand_binop, if appropriate. */
11978 866367 : return const_double_from_real_value (TREE_REAL_CST (exp),
11979 1732734 : TYPE_MODE (TREE_TYPE (exp)));
11980 :
11981 0 : case FIXED_CST:
11982 0 : return CONST_FIXED_FROM_FIXED_VALUE (TREE_FIXED_CST (exp),
11983 : TYPE_MODE (TREE_TYPE (exp)));
11984 :
11985 18396 : case COMPLEX_CST:
11986 : /* Handle evaluating a complex constant in a CONCAT target. */
11987 18396 : if (original_target && GET_CODE (original_target) == CONCAT)
11988 : {
11989 219 : rtx rtarg, itarg;
11990 :
11991 219 : mode = TYPE_MODE (TREE_TYPE (TREE_TYPE (exp)));
11992 219 : rtarg = XEXP (original_target, 0);
11993 219 : itarg = XEXP (original_target, 1);
11994 :
11995 : /* Move the real and imaginary parts separately. */
11996 219 : op0 = expand_expr (TREE_REALPART (exp), rtarg, mode, EXPAND_NORMAL);
11997 219 : op1 = expand_expr (TREE_IMAGPART (exp), itarg, mode, EXPAND_NORMAL);
11998 :
11999 219 : if (op0 != rtarg)
12000 219 : emit_move_insn (rtarg, op0);
12001 219 : if (op1 != itarg)
12002 219 : emit_move_insn (itarg, op1);
12003 :
12004 : return original_target;
12005 : }
12006 :
12007 : /* fall through */
12008 :
12009 202884 : case STRING_CST:
12010 202884 : temp = expand_expr_constant (exp, 1, modifier);
12011 :
12012 : /* temp contains a constant address.
12013 : On RISC machines where a constant address isn't valid,
12014 : make some insns to get that address into a register. */
12015 202884 : if (modifier != EXPAND_CONST_ADDRESS
12016 : && modifier != EXPAND_INITIALIZER
12017 202884 : && modifier != EXPAND_SUM
12018 221441 : && ! memory_address_addr_space_p (mode, XEXP (temp, 0),
12019 18557 : MEM_ADDR_SPACE (temp)))
12020 12 : return replace_equiv_address (temp,
12021 12 : copy_rtx (XEXP (temp, 0)));
12022 : return temp;
12023 :
12024 0 : case POLY_INT_CST:
12025 0 : return immed_wide_int_const (poly_int_cst_value (exp), mode);
12026 :
12027 1640 : case SAVE_EXPR:
12028 1640 : {
12029 1640 : tree val = treeop0;
12030 1640 : rtx ret = expand_expr_real_1 (val, target, tmode, modifier, alt_rtl,
12031 : inner_reference_p);
12032 :
12033 1640 : if (!SAVE_EXPR_RESOLVED_P (exp))
12034 : {
12035 : /* We can indeed still hit this case, typically via builtin
12036 : expanders calling save_expr immediately before expanding
12037 : something. Assume this means that we only have to deal
12038 : with non-BLKmode values. */
12039 1593 : gcc_assert (GET_MODE (ret) != BLKmode);
12040 :
12041 1593 : val = build_decl (curr_insn_location (),
12042 1593 : VAR_DECL, NULL, TREE_TYPE (exp));
12043 1593 : DECL_ARTIFICIAL (val) = 1;
12044 1593 : DECL_IGNORED_P (val) = 1;
12045 1593 : treeop0 = val;
12046 1593 : TREE_OPERAND (exp, 0) = treeop0;
12047 1593 : SAVE_EXPR_RESOLVED_P (exp) = 1;
12048 :
12049 1593 : if (!CONSTANT_P (ret))
12050 1593 : ret = copy_to_reg (ret);
12051 1593 : SET_DECL_RTL (val, ret);
12052 : }
12053 :
12054 : return ret;
12055 : }
12056 :
12057 :
12058 179805 : case CONSTRUCTOR:
12059 : /* If we don't need the result, just ensure we evaluate any
12060 : subexpressions. */
12061 179805 : if (ignore)
12062 : {
12063 : unsigned HOST_WIDE_INT idx;
12064 : tree value;
12065 :
12066 0 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), idx, value)
12067 0 : expand_expr (value, const0_rtx, VOIDmode, EXPAND_NORMAL);
12068 :
12069 0 : return const0_rtx;
12070 : }
12071 :
12072 179805 : return expand_constructor (exp, target, modifier, false);
12073 :
12074 858516 : case TARGET_MEM_REF:
12075 858516 : {
12076 858516 : addr_space_t as
12077 858516 : = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
12078 858516 : unsigned int align;
12079 :
12080 858516 : op0 = addr_for_mem_ref (exp, as, true);
12081 858516 : op0 = memory_address_addr_space (mode, op0, as);
12082 858516 : temp = gen_rtx_MEM (mode, op0);
12083 858516 : set_mem_attributes (temp, exp, 0);
12084 858516 : set_mem_addr_space (temp, as);
12085 858516 : align = get_object_alignment (exp);
12086 858516 : if (modifier != EXPAND_WRITE
12087 858516 : && modifier != EXPAND_MEMORY
12088 591739 : && mode != BLKmode
12089 1444331 : && align < GET_MODE_ALIGNMENT (mode))
12090 53356 : temp = expand_misaligned_mem_ref (temp, mode, unsignedp,
12091 : align, NULL_RTX, NULL);
12092 858516 : return EXTEND_BITINT (temp);
12093 : }
12094 :
12095 6815306 : case MEM_REF:
12096 6815306 : {
12097 6815306 : const bool reverse = REF_REVERSE_STORAGE_ORDER (exp);
12098 6815306 : addr_space_t as
12099 6815306 : = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
12100 6815306 : machine_mode address_mode;
12101 6815306 : tree base = TREE_OPERAND (exp, 0);
12102 6815306 : gimple *def_stmt;
12103 6815306 : unsigned align;
12104 : /* Handle expansion of non-aliased memory with non-BLKmode. That
12105 : might end up in a register. */
12106 6815306 : if (mem_ref_refers_to_non_mem_p (exp))
12107 : {
12108 80912 : poly_int64 offset = mem_ref_offset (exp).force_shwi ();
12109 80912 : base = TREE_OPERAND (base, 0);
12110 80912 : poly_uint64 type_size;
12111 80912 : if (known_eq (offset, 0)
12112 46438 : && !reverse
12113 46438 : && poly_int_tree_p (TYPE_SIZE (type), &type_size)
12114 173788 : && known_eq (GET_MODE_BITSIZE (DECL_MODE (base)), type_size))
12115 19273 : return expand_expr (build1 (VIEW_CONVERT_EXPR, type, base),
12116 19273 : target, tmode, modifier);
12117 61639 : unsigned align;
12118 61639 : if (TYPE_MODE (type) == BLKmode || maybe_lt (offset, 0))
12119 : {
12120 218 : temp = assign_stack_temp (DECL_MODE (base),
12121 436 : GET_MODE_SIZE (DECL_MODE (base)));
12122 218 : store_expr (base, temp, 0, false, false);
12123 218 : temp = adjust_address (temp, TYPE_MODE (type), offset);
12124 218 : if (TYPE_MODE (type) == BLKmode)
12125 197 : set_mem_size (temp, int_size_in_bytes (type));
12126 : /* When the original ref was misaligned so will be the
12127 : access to the stack temporary. Not all targets handle
12128 : this correctly, some will ICE in sanity checking.
12129 : Handle this by doing bitfield extraction when necessary. */
12130 42 : else if ((align = get_object_alignment (exp))
12131 21 : < GET_MODE_ALIGNMENT (TYPE_MODE (type)))
12132 3 : temp
12133 3 : = expand_misaligned_mem_ref (temp, TYPE_MODE (type),
12134 : unsignedp, align,
12135 : modifier == EXPAND_STACK_PARM
12136 : ? NULL_RTX : target, NULL);
12137 : return temp;
12138 : }
12139 : /* When the access is fully outside of the underlying object
12140 : expand the offset as zero. This avoids out-of-bound
12141 : BIT_FIELD_REFs and generates smaller code for these cases
12142 : with UB. */
12143 61421 : type_size = tree_to_poly_uint64 (TYPE_SIZE_UNIT (type));
12144 122842 : if (!ranges_maybe_overlap_p (offset, type_size, 0,
12145 122842 : GET_MODE_SIZE (DECL_MODE (base))))
12146 37 : offset = 0;
12147 61421 : exp = build3 (BIT_FIELD_REF, type, base, TYPE_SIZE (type),
12148 61421 : bitsize_int (offset * BITS_PER_UNIT));
12149 61421 : REF_REVERSE_STORAGE_ORDER (exp) = reverse;
12150 61421 : return expand_expr (exp, target, tmode, modifier);
12151 : }
12152 6734394 : address_mode = targetm.addr_space.address_mode (as);
12153 6734394 : if ((def_stmt = get_def_for_expr (base, BIT_AND_EXPR)))
12154 : {
12155 44 : tree mask = gimple_assign_rhs2 (def_stmt);
12156 44 : base = build2 (BIT_AND_EXPR, TREE_TYPE (base),
12157 : gimple_assign_rhs1 (def_stmt), mask);
12158 44 : TREE_OPERAND (exp, 0) = base;
12159 : }
12160 6734394 : align = get_object_alignment (exp);
12161 6734394 : op0 = expand_expr (base, NULL_RTX, VOIDmode, EXPAND_SUM);
12162 6734394 : op0 = memory_address_addr_space (mode, op0, as);
12163 6734394 : if (!integer_zerop (TREE_OPERAND (exp, 1)))
12164 : {
12165 2047204 : rtx off = immed_wide_int_const (mem_ref_offset (exp), address_mode);
12166 2047204 : op0 = simplify_gen_binary (PLUS, address_mode, op0, off);
12167 2047204 : op0 = memory_address_addr_space (mode, op0, as);
12168 : }
12169 6734394 : temp = gen_rtx_MEM (mode, op0);
12170 6734394 : set_mem_attributes (temp, exp, 0);
12171 6734394 : set_mem_addr_space (temp, as);
12172 6734394 : if (TREE_THIS_VOLATILE (exp))
12173 12395 : MEM_VOLATILE_P (temp) = 1;
12174 6734394 : if (modifier == EXPAND_WRITE || modifier == EXPAND_MEMORY)
12175 : return temp;
12176 3994508 : if (!inner_reference_p
12177 2061617 : && mode != BLKmode
12178 5994652 : && align < GET_MODE_ALIGNMENT (mode))
12179 143348 : temp = expand_misaligned_mem_ref (temp, mode, unsignedp, align,
12180 : modifier == EXPAND_STACK_PARM
12181 : ? NULL_RTX : target, alt_rtl);
12182 3994508 : if (reverse)
12183 16 : temp = flip_storage_order (mode, temp);
12184 3994508 : return EXTEND_BITINT (temp);
12185 : }
12186 :
12187 629540 : case ARRAY_REF:
12188 :
12189 629540 : {
12190 629540 : tree array = treeop0;
12191 629540 : tree index = treeop1;
12192 629540 : tree init;
12193 :
12194 : /* Fold an expression like: "foo"[2].
12195 : This is not done in fold so it won't happen inside &.
12196 : Don't fold if this is for wide characters since it's too
12197 : difficult to do correctly and this is a very rare case. */
12198 :
12199 629540 : if (modifier != EXPAND_CONST_ADDRESS
12200 629540 : && modifier != EXPAND_INITIALIZER
12201 629540 : && modifier != EXPAND_MEMORY)
12202 : {
12203 629435 : tree t = fold_read_from_constant_string (exp);
12204 :
12205 629435 : if (t)
12206 0 : return expand_expr (t, target, tmode, modifier);
12207 : }
12208 :
12209 : /* If this is a constant index into a constant array,
12210 : just get the value from the array. Handle both the cases when
12211 : we have an explicit constructor and when our operand is a variable
12212 : that was declared const. */
12213 :
12214 629540 : if (modifier != EXPAND_CONST_ADDRESS
12215 : && modifier != EXPAND_INITIALIZER
12216 : && modifier != EXPAND_MEMORY
12217 629435 : && TREE_CODE (array) == CONSTRUCTOR
12218 0 : && ! TREE_SIDE_EFFECTS (array)
12219 629540 : && TREE_CODE (index) == INTEGER_CST)
12220 : {
12221 : unsigned HOST_WIDE_INT ix;
12222 : tree field, value;
12223 :
12224 0 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (array), ix,
12225 : field, value)
12226 0 : if (tree_int_cst_equal (field, index))
12227 : {
12228 0 : if (!TREE_SIDE_EFFECTS (value)
12229 0 : && TREE_CODE (value) != RAW_DATA_CST)
12230 0 : return expand_expr (fold (value), target, tmode, modifier);
12231 : break;
12232 : }
12233 : }
12234 :
12235 629540 : else if (optimize >= 1
12236 : && modifier != EXPAND_CONST_ADDRESS
12237 359260 : && modifier != EXPAND_INITIALIZER
12238 359260 : && modifier != EXPAND_MEMORY
12239 359159 : && TREE_READONLY (array) && ! TREE_SIDE_EFFECTS (array)
12240 9062 : && TREE_CODE (index) == INTEGER_CST
12241 2027 : && (VAR_P (array) || TREE_CODE (array) == CONST_DECL)
12242 629689 : && (init = ctor_for_folding (array)) != error_mark_node)
12243 : {
12244 108 : if (init == NULL_TREE)
12245 : {
12246 5 : tree value = build_zero_cst (type);
12247 5 : if (TREE_CODE (value) == CONSTRUCTOR)
12248 : {
12249 : /* If VALUE is a CONSTRUCTOR, this optimization is only
12250 : useful if this doesn't store the CONSTRUCTOR into
12251 : memory. If it does, it is more efficient to just
12252 : load the data from the array directly. */
12253 5 : rtx ret = expand_constructor (value, target,
12254 : modifier, true);
12255 5 : if (ret == NULL_RTX)
12256 : value = NULL_TREE;
12257 : }
12258 :
12259 : if (value)
12260 0 : return expand_expr (value, target, tmode, modifier);
12261 : }
12262 103 : else if (TREE_CODE (init) == CONSTRUCTOR)
12263 : {
12264 : unsigned HOST_WIDE_INT ix;
12265 : tree field, value;
12266 :
12267 168 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (init), ix,
12268 : field, value)
12269 166 : if (tree_int_cst_equal (field, index))
12270 : {
12271 100 : if (TREE_SIDE_EFFECTS (value)
12272 100 : || TREE_CODE (value) == RAW_DATA_CST)
12273 : break;
12274 :
12275 100 : if (TREE_CODE (value) == CONSTRUCTOR)
12276 : {
12277 : /* If VALUE is a CONSTRUCTOR, this
12278 : optimization is only useful if
12279 : this doesn't store the CONSTRUCTOR
12280 : into memory. If it does, it is more
12281 : efficient to just load the data from
12282 : the array directly. */
12283 95 : rtx ret = expand_constructor (value, target,
12284 : modifier, true);
12285 95 : if (ret == NULL_RTX)
12286 : break;
12287 : }
12288 :
12289 62 : return expand_expr (fold (value), target, tmode,
12290 62 : modifier);
12291 : }
12292 : }
12293 1 : else if (TREE_CODE (init) == STRING_CST)
12294 : {
12295 1 : tree low_bound = array_ref_low_bound (exp);
12296 1 : tree index1 = fold_convert_loc (loc, sizetype, treeop1);
12297 :
12298 : /* Optimize the special case of a zero lower bound.
12299 :
12300 : We convert the lower bound to sizetype to avoid problems
12301 : with constant folding. E.g. suppose the lower bound is
12302 : 1 and its mode is QI. Without the conversion
12303 : (ARRAY + (INDEX - (unsigned char)1))
12304 : becomes
12305 : (ARRAY + (-(unsigned char)1) + INDEX)
12306 : which becomes
12307 : (ARRAY + 255 + INDEX). Oops! */
12308 1 : if (!integer_zerop (low_bound))
12309 0 : index1 = size_diffop_loc (loc, index1,
12310 : fold_convert_loc (loc, sizetype,
12311 : low_bound));
12312 :
12313 1 : if (tree_fits_uhwi_p (index1)
12314 2 : && compare_tree_int (index1, TREE_STRING_LENGTH (init)) < 0)
12315 : {
12316 0 : tree char_type = TREE_TYPE (TREE_TYPE (init));
12317 0 : scalar_int_mode char_mode;
12318 :
12319 629478 : if (is_int_mode (TYPE_MODE (char_type), &char_mode)
12320 0 : && GET_MODE_SIZE (char_mode) == 1)
12321 0 : return gen_int_mode (TREE_STRING_POINTER (init)
12322 0 : [TREE_INT_CST_LOW (index1)],
12323 : char_mode);
12324 : }
12325 : }
12326 : }
12327 : }
12328 629478 : goto normal_inner_ref;
12329 :
12330 3760178 : case COMPONENT_REF:
12331 3760178 : gcc_assert (TREE_CODE (treeop0) != CONSTRUCTOR);
12332 : /* Fall through. */
12333 4658106 : case BIT_FIELD_REF:
12334 4658106 : case ARRAY_RANGE_REF:
12335 3760178 : normal_inner_ref:
12336 4658106 : {
12337 4658106 : machine_mode mode1, mode2;
12338 4658106 : poly_int64 bitsize, bitpos, bytepos;
12339 4658106 : tree offset;
12340 4658106 : int reversep, volatilep = 0;
12341 4658106 : tree tem
12342 4658106 : = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
12343 : &unsignedp, &reversep, &volatilep);
12344 4658106 : rtx orig_op0, memloc;
12345 4658106 : bool clear_mem_expr = false;
12346 4658106 : bool must_force_mem;
12347 :
12348 : /* If we got back the original object, something is wrong. Perhaps
12349 : we are evaluating an expression too early. In any event, don't
12350 : infinitely recurse. */
12351 4658106 : gcc_assert (tem != exp);
12352 :
12353 : /* Make sure bitpos is not negative, this can wreak havoc later. */
12354 4658106 : if (maybe_lt (bitpos, 0))
12355 : {
12356 249 : gcc_checking_assert (offset == NULL_TREE);
12357 249 : offset = size_int (bits_to_bytes_round_down (bitpos));
12358 249 : bitpos = num_trailing_bits (bitpos);
12359 : }
12360 :
12361 : /* If we have either an offset, a BLKmode result, or a reference
12362 : outside the underlying object, we must force it to memory.
12363 : Such a case can occur in Ada if we have unchecked conversion
12364 : of an expression from a scalar type to an aggregate type or
12365 : for an ARRAY_RANGE_REF whose type is BLKmode, or if we were
12366 : passed a partially uninitialized object or a view-conversion
12367 : to a larger size. */
12368 9316212 : must_force_mem = offset != NULL_TREE
12369 4377998 : || mode1 == BLKmode
12370 8963905 : || (mode == BLKmode
12371 0 : && !int_mode_for_size (bitsize, 1).exists ());
12372 :
12373 527249 : const enum expand_modifier tem_modifier
12374 : = must_force_mem
12375 : ? EXPAND_MEMORY
12376 4305799 : : modifier == EXPAND_SUM ? EXPAND_NORMAL : modifier;
12377 :
12378 : /* If TEM's type is a union of variable size, pass TARGET to the inner
12379 : computation, since it will need a temporary and TARGET is known
12380 : to have to do. This occurs in unchecked conversion in Ada. */
12381 4658106 : const rtx tem_target
12382 4658106 : = TREE_CODE (TREE_TYPE (tem)) == UNION_TYPE
12383 36707 : && COMPLETE_TYPE_P (TREE_TYPE (tem))
12384 36702 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (tem))) != INTEGER_CST
12385 0 : && modifier != EXPAND_STACK_PARM
12386 4658106 : ? target
12387 : : NULL_RTX;
12388 :
12389 9316212 : orig_op0 = op0
12390 4658106 : = expand_expr_real (tem, tem_target, VOIDmode, tem_modifier, NULL,
12391 : true);
12392 :
12393 : /* If the field has a mode, we want to access it in the
12394 : field's mode, not the computed mode.
12395 : If a MEM has VOIDmode (external with incomplete type),
12396 : use BLKmode for it instead. */
12397 4658106 : if (MEM_P (op0))
12398 : {
12399 4309177 : if (mode1 != VOIDmode)
12400 4151621 : op0 = adjust_address (op0, mode1, 0);
12401 157556 : else if (GET_MODE (op0) == VOIDmode)
12402 0 : op0 = adjust_address (op0, BLKmode, 0);
12403 : }
12404 :
12405 4658106 : mode2
12406 4658106 : = CONSTANT_P (op0) ? TYPE_MODE (TREE_TYPE (tem)) : GET_MODE (op0);
12407 :
12408 : /* See above for the rationale. */
12409 9316212 : if (maybe_gt (bitpos + bitsize, GET_MODE_BITSIZE (mode2)))
12410 2938033 : must_force_mem = true;
12411 :
12412 : /* Handle CONCAT first. */
12413 4658106 : if (GET_CODE (op0) == CONCAT && !must_force_mem)
12414 : {
12415 114 : if (known_eq (bitpos, 0)
12416 206 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0)))
12417 98 : && COMPLEX_MODE_P (mode1)
12418 93 : && COMPLEX_MODE_P (GET_MODE (op0))
12419 393 : && (GET_MODE_PRECISION (GET_MODE_INNER (mode1))
12420 186 : == GET_MODE_PRECISION (GET_MODE_INNER (GET_MODE (op0)))))
12421 : {
12422 93 : if (reversep)
12423 0 : op0 = flip_storage_order (GET_MODE (op0), op0);
12424 93 : if (mode1 != GET_MODE (op0))
12425 : {
12426 : rtx parts[2];
12427 0 : for (int i = 0; i < 2; i++)
12428 : {
12429 0 : rtx op = read_complex_part (op0, i != 0);
12430 0 : if (GET_CODE (op) == SUBREG)
12431 0 : op = force_reg (GET_MODE (op), op);
12432 0 : temp = gen_lowpart_common (GET_MODE_INNER (mode1), op);
12433 0 : if (temp)
12434 : op = temp;
12435 : else
12436 : {
12437 0 : if (!REG_P (op) && !MEM_P (op))
12438 0 : op = force_reg (GET_MODE (op), op);
12439 0 : op = gen_lowpart (GET_MODE_INNER (mode1), op);
12440 : }
12441 0 : parts[i] = op;
12442 : }
12443 0 : op0 = gen_rtx_CONCAT (mode1, parts[0], parts[1]);
12444 : }
12445 : return op0;
12446 : }
12447 21 : if (known_eq (bitpos, 0)
12448 20 : && known_eq (bitsize,
12449 : GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 0))))
12450 23 : && maybe_ne (bitsize, 0))
12451 : {
12452 : op0 = XEXP (op0, 0);
12453 : mode2 = GET_MODE (op0);
12454 : }
12455 38 : else if (known_eq (bitpos,
12456 : GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 0))))
12457 4 : && known_eq (bitsize,
12458 : GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 1))))
12459 0 : && maybe_ne (bitpos, 0)
12460 19 : && maybe_ne (bitsize, 0))
12461 : {
12462 0 : op0 = XEXP (op0, 1);
12463 0 : bitpos = 0;
12464 0 : mode2 = GET_MODE (op0);
12465 : }
12466 : else
12467 : /* Otherwise force into memory. */
12468 : must_force_mem = true;
12469 : }
12470 :
12471 : /* If this is a constant, put it in a register if it is a legitimate
12472 : constant and we don't need a memory reference. */
12473 4658013 : if (CONSTANT_P (op0)
12474 33 : && mode2 != BLKmode
12475 33 : && targetm.legitimate_constant_p (mode2, op0)
12476 4658034 : && !must_force_mem)
12477 21 : op0 = force_reg (mode2, op0);
12478 :
12479 : /* Otherwise, if this is a constant, try to force it to the constant
12480 : pool. Note that back-ends, e.g. MIPS, may refuse to do so if it
12481 : is a legitimate constant. */
12482 4657992 : else if (CONSTANT_P (op0) && (memloc = force_const_mem (mode2, op0)))
12483 12 : op0 = validize_mem (memloc);
12484 :
12485 : /* Otherwise, if this is a constant or the object is not in memory
12486 : and need be, put it there. */
12487 4657980 : else if (CONSTANT_P (op0) || (!MEM_P (op0) && must_force_mem))
12488 : {
12489 1095 : machine_mode tem_mode = TYPE_MODE (TREE_TYPE (tem));
12490 1095 : poly_int64 size;
12491 1095 : if (!poly_int_tree_p (TYPE_SIZE_UNIT (TREE_TYPE (tem)), &size))
12492 0 : size = max_int_size_in_bytes (TREE_TYPE (tem));
12493 1095 : unsigned int align = TREE_CODE (tem) == SSA_NAME
12494 1095 : ? TYPE_ALIGN (TREE_TYPE (tem))
12495 1095 : : get_object_alignment (tem);
12496 1095 : if (STRICT_ALIGNMENT)
12497 : {
12498 : /* For STRICT_ALIGNMENT targets, when we force the operand to
12499 : memory, we may need to increase the alignment to meet the
12500 : expectation in later RTL lowering passes. The increased
12501 : alignment is capped by MAX_SUPPORTED_STACK_ALIGNMENT. */
12502 : if (tem_mode != BLKmode)
12503 : align = MAX (align, GET_MODE_ALIGNMENT (tem_mode));
12504 : else
12505 : align = MAX (align, TYPE_ALIGN (TREE_TYPE (tem)));
12506 : align = MIN (align, (unsigned) MAX_SUPPORTED_STACK_ALIGNMENT);
12507 : }
12508 1095 : memloc = assign_stack_local (tem_mode, size, align);
12509 1095 : emit_move_insn (memloc, op0);
12510 1095 : op0 = memloc;
12511 1095 : clear_mem_expr = true;
12512 : }
12513 :
12514 4658013 : if (offset)
12515 : {
12516 280108 : machine_mode address_mode;
12517 280108 : rtx offset_rtx = expand_expr (offset, NULL_RTX, VOIDmode,
12518 : EXPAND_SUM);
12519 :
12520 280108 : gcc_assert (MEM_P (op0));
12521 :
12522 280108 : address_mode = get_address_mode (op0);
12523 280108 : if (GET_MODE (offset_rtx) != address_mode)
12524 : {
12525 : /* We cannot be sure that the RTL in offset_rtx is valid outside
12526 : of a memory address context, so force it into a register
12527 : before attempting to convert it to the desired mode. */
12528 422 : offset_rtx = force_operand (offset_rtx, NULL_RTX);
12529 422 : offset_rtx = convert_to_mode (address_mode, offset_rtx, 0);
12530 : }
12531 :
12532 : /* See the comment in expand_assignment for the rationale. */
12533 280108 : if (mode1 != VOIDmode
12534 279892 : && maybe_ne (bitpos, 0)
12535 74596 : && maybe_gt (bitsize, 0)
12536 354165 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
12537 354165 : && multiple_p (bitpos, bitsize)
12538 148114 : && multiple_p (bitsize, GET_MODE_ALIGNMENT (mode1))
12539 354165 : && MEM_ALIGN (op0) >= GET_MODE_ALIGNMENT (mode1))
12540 : {
12541 73699 : op0 = adjust_address (op0, mode1, bytepos);
12542 73699 : bitpos = 0;
12543 : }
12544 :
12545 280108 : op0 = offset_address (op0, offset_rtx,
12546 : highest_pow2_factor (offset));
12547 : }
12548 :
12549 : /* If OFFSET is making OP0 more aligned than BIGGEST_ALIGNMENT,
12550 : record its alignment as BIGGEST_ALIGNMENT. */
12551 4658013 : if (MEM_P (op0)
12552 4310284 : && known_eq (bitpos, 0)
12553 1457328 : && offset != 0
12554 4937074 : && is_aligning_offset (offset, tem))
12555 0 : set_mem_align (op0, BIGGEST_ALIGNMENT);
12556 :
12557 : /* Don't forget about volatility even if this is a bitfield. */
12558 4658013 : if (MEM_P (op0) && volatilep && ! MEM_VOLATILE_P (op0))
12559 : {
12560 528 : if (op0 == orig_op0)
12561 158 : op0 = copy_rtx (op0);
12562 :
12563 528 : MEM_VOLATILE_P (op0) = 1;
12564 : }
12565 :
12566 4658013 : if (MEM_P (op0) && TREE_CODE (tem) == FUNCTION_DECL)
12567 : {
12568 6 : if (op0 == orig_op0)
12569 0 : op0 = copy_rtx (op0);
12570 :
12571 6 : set_mem_align (op0, BITS_PER_UNIT);
12572 : }
12573 :
12574 : /* In cases where an aligned union has an unaligned object
12575 : as a field, we might be extracting a BLKmode value from
12576 : an integer-mode (e.g., SImode) object. Handle this case
12577 : by doing the extract into an object as wide as the field
12578 : (which we know to be the width of a basic mode), then
12579 : storing into memory, and changing the mode to BLKmode. */
12580 4658013 : if (mode1 == VOIDmode
12581 4426727 : || REG_P (op0) || GET_CODE (op0) == SUBREG
12582 4152694 : || (mode1 != BLKmode && ! direct_load[(int) mode1]
12583 27185 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_INT
12584 23074 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_FLOAT
12585 : && modifier != EXPAND_CONST_ADDRESS
12586 15389 : && modifier != EXPAND_INITIALIZER
12587 15389 : && modifier != EXPAND_MEMORY)
12588 : /* If the bitfield is volatile and the bitsize
12589 : is narrower than the access size of the bitfield,
12590 : we need to extract bitfields from the access. */
12591 4137305 : || (volatilep && TREE_CODE (exp) == COMPONENT_REF
12592 1503 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (exp, 1))
12593 10 : && mode1 != BLKmode
12594 20 : && maybe_lt (bitsize, GET_MODE_SIZE (mode1) * BITS_PER_UNIT))
12595 : /* If the field isn't aligned enough to fetch as a memref,
12596 : fetch it as a bit field. */
12597 4137296 : || (mode1 != BLKmode
12598 4064233 : && (((MEM_P (op0)
12599 4064233 : ? MEM_ALIGN (op0) < GET_MODE_ALIGNMENT (mode1)
12600 7983983 : || !multiple_p (bitpos, GET_MODE_ALIGNMENT (mode1))
12601 0 : : TYPE_ALIGN (TREE_TYPE (tem)) < GET_MODE_ALIGNMENT (mode)
12602 0 : || !multiple_p (bitpos, GET_MODE_ALIGNMENT (mode)))
12603 75519 : && modifier != EXPAND_MEMORY
12604 75519 : && ((modifier == EXPAND_CONST_ADDRESS
12605 75519 : || modifier == EXPAND_INITIALIZER)
12606 75519 : ? STRICT_ALIGNMENT
12607 75519 : : targetm.slow_unaligned_access (mode1,
12608 75519 : MEM_ALIGN (op0))))
12609 4064233 : || !multiple_p (bitpos, BITS_PER_UNIT)))
12610 : /* If the type and the field are a constant size and the
12611 : size of the type isn't the same size as the bitfield,
12612 : we must use bitfield operations. */
12613 12859542 : || (known_size_p (bitsize)
12614 4137296 : && TYPE_SIZE (TREE_TYPE (exp))
12615 4137296 : && poly_int_tree_p (TYPE_SIZE (TREE_TYPE (exp)))
12616 4658013 : && maybe_ne (wi::to_poly_offset (TYPE_SIZE (TREE_TYPE (exp))),
12617 : bitsize)))
12618 : {
12619 520747 : machine_mode ext_mode = mode;
12620 :
12621 520747 : if (ext_mode == BLKmode
12622 520747 : && ! (target != 0 && MEM_P (op0)
12623 22 : && MEM_P (target)
12624 22 : && multiple_p (bitpos, BITS_PER_UNIT)))
12625 4 : ext_mode = int_mode_for_size (bitsize, 1).else_blk ();
12626 :
12627 520747 : if (ext_mode == BLKmode)
12628 : {
12629 26 : if (target == 0)
12630 4 : target = assign_temp (type, 1, 1);
12631 :
12632 : /* ??? Unlike the similar test a few lines below, this one is
12633 : very likely obsolete. */
12634 26 : if (known_eq (bitsize, 0))
12635 : return target;
12636 :
12637 : /* In this case, BITPOS must start at a byte boundary and
12638 : TARGET, if specified, must be a MEM. */
12639 26 : gcc_assert (MEM_P (op0)
12640 : && (!target || MEM_P (target)));
12641 :
12642 26 : bytepos = exact_div (bitpos, BITS_PER_UNIT);
12643 26 : poly_int64 bytesize = bits_to_bytes_round_up (bitsize);
12644 52 : emit_block_move (target,
12645 : adjust_address (op0, VOIDmode, bytepos),
12646 26 : gen_int_mode (bytesize, Pmode),
12647 : (modifier == EXPAND_STACK_PARM
12648 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
12649 :
12650 26 : return target;
12651 : }
12652 :
12653 : /* If we have nothing to extract, the result will be 0 for targets
12654 : with SHIFT_COUNT_TRUNCATED == 0 and garbage otherwise. Always
12655 : return 0 for the sake of consistency, as reading a zero-sized
12656 : bitfield is valid in Ada and the value is fully specified. */
12657 520721 : if (known_eq (bitsize, 0))
12658 0 : return const0_rtx;
12659 :
12660 520721 : op0 = validize_mem (op0);
12661 :
12662 520721 : if (MEM_P (op0) && REG_P (XEXP (op0, 0)))
12663 49656 : mark_reg_pointer (XEXP (op0, 0), MEM_ALIGN (op0));
12664 :
12665 : /* If the result has aggregate type and the extraction is done in
12666 : an integral mode, then the field may be not aligned on a byte
12667 : boundary; in this case, if it has reverse storage order, it
12668 : needs to be extracted as a scalar field with reverse storage
12669 : order and put back into memory order afterwards. */
12670 520721 : if (AGGREGATE_TYPE_P (type)
12671 5170 : && GET_MODE_CLASS (ext_mode) == MODE_INT)
12672 5088 : reversep = TYPE_REVERSE_STORAGE_ORDER (type);
12673 :
12674 520721 : gcc_checking_assert (known_ge (bitpos, 0));
12675 1054107 : op0 = extract_bit_field (op0, bitsize, bitpos, unsignedp,
12676 : (modifier == EXPAND_STACK_PARM
12677 : ? NULL_RTX : target),
12678 : ext_mode, ext_mode, reversep, alt_rtl);
12679 :
12680 : /* If the result has aggregate type and the mode of OP0 is an
12681 : integral mode then, if BITSIZE is narrower than this mode
12682 : and this is for big-endian data, we must put the field
12683 : into the high-order bits. And we must also put it back
12684 : into memory order if it has been previously reversed. */
12685 520721 : scalar_int_mode op0_mode;
12686 520721 : if (AGGREGATE_TYPE_P (type)
12687 520721 : && is_int_mode (GET_MODE (op0), &op0_mode))
12688 : {
12689 5088 : HOST_WIDE_INT size = GET_MODE_BITSIZE (op0_mode);
12690 :
12691 5088 : gcc_checking_assert (known_le (bitsize, size));
12692 5088 : if (maybe_lt (bitsize, size)
12693 5088 : && reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
12694 0 : op0 = expand_shift (LSHIFT_EXPR, op0_mode, op0,
12695 : size - bitsize, op0, 1);
12696 :
12697 5088 : if (reversep)
12698 0 : op0 = flip_storage_order (op0_mode, op0);
12699 : }
12700 :
12701 : /* If the result type is BLKmode, store the data into a temporary
12702 : of the appropriate type, but with the mode corresponding to the
12703 : mode for the data we have (op0's mode). */
12704 520721 : if (mode == BLKmode)
12705 : {
12706 0 : rtx new_rtx
12707 0 : = assign_stack_temp_for_type (ext_mode,
12708 0 : GET_MODE_BITSIZE (ext_mode),
12709 : type);
12710 0 : emit_move_insn (new_rtx, op0);
12711 0 : op0 = copy_rtx (new_rtx);
12712 0 : PUT_MODE (op0, BLKmode);
12713 : }
12714 :
12715 : return op0;
12716 : }
12717 :
12718 : /* If the result is BLKmode, use that to access the object
12719 : now as well. */
12720 4137266 : if (mode == BLKmode)
12721 73037 : mode1 = BLKmode;
12722 :
12723 : /* Get a reference to just this component. */
12724 4137266 : bytepos = bits_to_bytes_round_down (bitpos);
12725 4137266 : if (modifier == EXPAND_CONST_ADDRESS
12726 4137266 : || modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER)
12727 165478 : op0 = adjust_address_nv (op0, mode1, bytepos);
12728 : else
12729 3971788 : op0 = adjust_address (op0, mode1, bytepos);
12730 :
12731 4137266 : if (op0 == orig_op0)
12732 127348 : op0 = copy_rtx (op0);
12733 :
12734 : /* Don't set memory attributes if the base expression is
12735 : SSA_NAME that got expanded as a MEM or a CONSTANT. In that case,
12736 : we should just honor its original memory attributes. */
12737 4137266 : if (!(TREE_CODE (tem) == SSA_NAME
12738 9501 : && (MEM_P (orig_op0) || CONSTANT_P (orig_op0))))
12739 4127765 : set_mem_attributes (op0, exp, 0);
12740 :
12741 4137266 : if (REG_P (XEXP (op0, 0)))
12742 667811 : mark_reg_pointer (XEXP (op0, 0), MEM_ALIGN (op0));
12743 :
12744 : /* If op0 is a temporary because the original expressions was forced
12745 : to memory, clear MEM_EXPR so that the original expression cannot
12746 : be marked as addressable through MEM_EXPR of the temporary. */
12747 4137266 : if (clear_mem_expr)
12748 1061 : set_mem_expr (op0, NULL_TREE);
12749 :
12750 4137266 : MEM_VOLATILE_P (op0) |= volatilep;
12751 :
12752 4137266 : if (reversep
12753 : && modifier != EXPAND_MEMORY
12754 497 : && modifier != EXPAND_WRITE)
12755 497 : op0 = flip_storage_order (mode1, op0);
12756 :
12757 4137266 : op0 = EXTEND_BITINT (op0);
12758 :
12759 4137266 : if (mode == mode1 || mode1 == BLKmode || mode1 == tmode
12760 : || modifier == EXPAND_CONST_ADDRESS
12761 0 : || modifier == EXPAND_INITIALIZER)
12762 : return op0;
12763 :
12764 0 : if (target == 0)
12765 0 : target = gen_reg_rtx (tmode != VOIDmode ? tmode : mode);
12766 :
12767 0 : convert_move (target, op0, unsignedp);
12768 0 : return target;
12769 : }
12770 :
12771 85301 : case OBJ_TYPE_REF:
12772 85301 : return expand_expr (OBJ_TYPE_REF_EXPR (exp), target, tmode, modifier);
12773 :
12774 6811532 : case CALL_EXPR:
12775 : /* All valid uses of __builtin_va_arg_pack () are removed during
12776 : inlining. */
12777 6811532 : if (CALL_EXPR_VA_ARG_PACK (exp))
12778 6 : error ("invalid use of %<__builtin_va_arg_pack ()%>");
12779 6811532 : {
12780 6811532 : tree fndecl = get_callee_fndecl (exp), attr;
12781 :
12782 6811532 : if (fndecl
12783 : /* Don't diagnose the error attribute in thunks, those are
12784 : artificially created. */
12785 6624628 : && !CALL_FROM_THUNK_P (exp)
12786 13432574 : && (attr = lookup_attribute ("error",
12787 6621042 : DECL_ATTRIBUTES (fndecl))) != NULL)
12788 : {
12789 5 : const char *ident = lang_hooks.decl_printable_name (fndecl, 1);
12790 10 : error ("call to %qs declared with attribute error: %s",
12791 : identifier_to_locale (ident),
12792 5 : TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr))));
12793 : }
12794 6811532 : if (fndecl
12795 : /* Don't diagnose the warning attribute in thunks, those are
12796 : artificially created. */
12797 6624628 : && !CALL_FROM_THUNK_P (exp)
12798 13432574 : && (attr = lookup_attribute ("warning",
12799 6621042 : DECL_ATTRIBUTES (fndecl))) != NULL)
12800 : {
12801 21 : const char *ident = lang_hooks.decl_printable_name (fndecl, 1);
12802 42 : warning_at (EXPR_LOCATION (exp),
12803 21 : OPT_Wattribute_warning,
12804 : "call to %qs declared with attribute warning: %s",
12805 : identifier_to_locale (ident),
12806 21 : TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr))));
12807 : }
12808 :
12809 : /* Check for a built-in function. */
12810 6811532 : if (fndecl && fndecl_built_in_p (fndecl))
12811 : {
12812 2016791 : gcc_assert (DECL_BUILT_IN_CLASS (fndecl) != BUILT_IN_FRONTEND);
12813 2016791 : return expand_builtin (exp, target, subtarget, tmode, ignore);
12814 : }
12815 : }
12816 4794741 : temp = expand_call (exp, target, ignore);
12817 4794739 : return EXTEND_BITINT (temp);
12818 :
12819 351968 : case VIEW_CONVERT_EXPR:
12820 351968 : op0 = NULL_RTX;
12821 :
12822 : /* If we are converting to BLKmode, try to avoid an intermediate
12823 : temporary by fetching an inner memory reference. */
12824 351968 : if (mode == BLKmode
12825 74118 : && poly_int_tree_p (TYPE_SIZE (type))
12826 74118 : && TYPE_MODE (TREE_TYPE (treeop0)) != BLKmode
12827 351968 : && handled_component_p (treeop0))
12828 : {
12829 0 : machine_mode mode1;
12830 0 : poly_int64 bitsize, bitpos, bytepos;
12831 0 : tree offset;
12832 0 : int reversep, volatilep = 0;
12833 0 : tree tem
12834 0 : = get_inner_reference (treeop0, &bitsize, &bitpos, &offset, &mode1,
12835 : &unsignedp, &reversep, &volatilep);
12836 :
12837 : /* ??? We should work harder and deal with non-zero offsets. */
12838 0 : if (!offset
12839 0 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
12840 0 : && !reversep
12841 0 : && known_size_p (bitsize)
12842 0 : && known_eq (wi::to_poly_offset (TYPE_SIZE (type)), bitsize))
12843 : {
12844 : /* See the normal_inner_ref case for the rationale. */
12845 0 : rtx orig_op0
12846 0 : = expand_expr_real (tem,
12847 0 : (TREE_CODE (TREE_TYPE (tem)) == UNION_TYPE
12848 0 : && (TREE_CODE (TYPE_SIZE (TREE_TYPE (tem)))
12849 : != INTEGER_CST)
12850 0 : && modifier != EXPAND_STACK_PARM
12851 : ? target : NULL_RTX),
12852 : VOIDmode,
12853 : modifier == EXPAND_SUM ? EXPAND_NORMAL : modifier,
12854 : NULL, true);
12855 :
12856 0 : if (MEM_P (orig_op0))
12857 : {
12858 0 : op0 = orig_op0;
12859 :
12860 : /* Get a reference to just this component. */
12861 0 : if (modifier == EXPAND_CONST_ADDRESS
12862 : || modifier == EXPAND_SUM
12863 0 : || modifier == EXPAND_INITIALIZER)
12864 0 : op0 = adjust_address_nv (op0, mode, bytepos);
12865 : else
12866 0 : op0 = adjust_address (op0, mode, bytepos);
12867 :
12868 0 : if (op0 == orig_op0)
12869 0 : op0 = copy_rtx (op0);
12870 :
12871 0 : set_mem_attributes (op0, treeop0, 0);
12872 0 : if (REG_P (XEXP (op0, 0)))
12873 0 : mark_reg_pointer (XEXP (op0, 0), MEM_ALIGN (op0));
12874 :
12875 0 : MEM_VOLATILE_P (op0) |= volatilep;
12876 : }
12877 : }
12878 : }
12879 :
12880 0 : if (!op0)
12881 426097 : op0 = expand_expr_real (treeop0, NULL_RTX, VOIDmode, modifier,
12882 : NULL, inner_reference_p || mode == BLKmode);
12883 :
12884 : /* If the input and output modes are both the same, we are done. */
12885 351968 : if (mode == GET_MODE (op0))
12886 : ;
12887 : /* Similarly if the output mode is BLKmode and input is a MEM,
12888 : adjust_address done below is all we need. */
12889 152430 : else if (mode == BLKmode && MEM_P (op0))
12890 : ;
12891 : /* If neither mode is BLKmode, and both modes are the same size
12892 : then we can use gen_lowpart. */
12893 : else if (mode != BLKmode
12894 152396 : && GET_MODE (op0) != BLKmode
12895 151069 : && known_eq (GET_MODE_PRECISION (mode),
12896 : GET_MODE_PRECISION (GET_MODE (op0)))
12897 147170 : && !COMPLEX_MODE_P (GET_MODE (op0)))
12898 : {
12899 145766 : if (GET_CODE (op0) == SUBREG)
12900 108 : op0 = force_reg (GET_MODE (op0), op0);
12901 145766 : temp = gen_lowpart_common (mode, op0);
12902 145766 : if (temp)
12903 : op0 = temp;
12904 : else
12905 : {
12906 27999 : if (!REG_P (op0) && !MEM_P (op0))
12907 18 : op0 = force_reg (GET_MODE (op0), op0);
12908 27999 : op0 = gen_lowpart (mode, op0);
12909 : }
12910 : }
12911 : /* If both types are integral, convert from one mode to the other. */
12912 6662 : else if (INTEGRAL_TYPE_P (type)
12913 3130 : && INTEGRAL_TYPE_P (TREE_TYPE (treeop0))
12914 0 : && mode != BLKmode
12915 6662 : && GET_MODE (op0) != BLKmode)
12916 0 : op0 = convert_modes (mode, GET_MODE (op0), op0,
12917 0 : TYPE_UNSIGNED (TREE_TYPE (treeop0)));
12918 : /* If the output type is a bit-field type, do an extraction. */
12919 6662 : else if (reduce_bit_field
12920 2 : && mode != BLKmode
12921 2 : && (MEM_P (op0) || !COMPLEX_MODE_P (GET_MODE (op0))))
12922 0 : return extract_bit_field (op0, TYPE_PRECISION (type), 0,
12923 0 : TYPE_UNSIGNED (type), NULL_RTX,
12924 : mode, mode, false, NULL);
12925 : /* If source is a complex number and destination is a
12926 : two-component vector with same inner type, try to use
12927 : vector initialization. */
12928 6662 : else if ((temp = try_expand_complex_as_vector (mode, op0, target))
12929 : != NULL_RTX)
12930 : return temp;
12931 : /* As a last resort, spill op0 to memory, and reload it in a
12932 : different mode. */
12933 6633 : else if (!MEM_P (op0))
12934 : {
12935 : /* If the operand is not a MEM, force it into memory. Since we
12936 : are going to be changing the mode of the MEM, don't call
12937 : force_const_mem for constants because we don't allow pool
12938 : constants to change mode. */
12939 5296 : tree inner_type = TREE_TYPE (treeop0);
12940 :
12941 5296 : gcc_assert (!TREE_ADDRESSABLE (exp));
12942 :
12943 5296 : if (target == 0 || GET_MODE (target) != TYPE_MODE (inner_type))
12944 5292 : target
12945 : = assign_stack_temp_for_type
12946 5292 : (TYPE_MODE (inner_type),
12947 10584 : GET_MODE_SIZE (TYPE_MODE (inner_type)), inner_type);
12948 :
12949 5296 : emit_move_insn (target, op0);
12950 5296 : op0 = target;
12951 :
12952 5296 : if (reduce_bit_field && mode != BLKmode)
12953 2 : return extract_bit_field (op0, TYPE_PRECISION (type), 0,
12954 2 : TYPE_UNSIGNED (type), NULL_RTX,
12955 : mode, mode, false, NULL);
12956 : }
12957 :
12958 : /* If OP0 is (now) a MEM, we need to deal with alignment issues. If the
12959 : output type is such that the operand is known to be aligned, indicate
12960 : that it is. Otherwise, we need only be concerned about alignment for
12961 : non-BLKmode results. */
12962 351937 : if (MEM_P (op0))
12963 : {
12964 128590 : enum insn_code icode;
12965 :
12966 128590 : if (modifier != EXPAND_WRITE
12967 128590 : && modifier != EXPAND_MEMORY
12968 128590 : && !inner_reference_p
12969 128589 : && mode != BLKmode
12970 183061 : && MEM_ALIGN (op0) < GET_MODE_ALIGNMENT (mode))
12971 : {
12972 : /* If the target does have special handling for unaligned
12973 : loads of mode then use them. */
12974 15841 : if ((icode = optab_handler (movmisalign_optab, mode))
12975 : != CODE_FOR_nothing)
12976 : {
12977 1906 : rtx reg;
12978 :
12979 1906 : op0 = adjust_address (op0, mode, 0);
12980 : /* We've already validated the memory, and we're creating a
12981 : new pseudo destination. The predicates really can't
12982 : fail. */
12983 1906 : reg = gen_reg_rtx (mode);
12984 :
12985 : /* Nor can the insn generator. */
12986 1906 : rtx_insn *insn = GEN_FCN (icode) (reg, op0);
12987 1906 : emit_insn (insn);
12988 1906 : return reg;
12989 : }
12990 : else if (STRICT_ALIGNMENT)
12991 : {
12992 : poly_uint64 mode_size = GET_MODE_SIZE (mode);
12993 : poly_uint64 temp_size = mode_size;
12994 : if (GET_MODE (op0) != BLKmode)
12995 : temp_size = upper_bound (temp_size,
12996 : GET_MODE_SIZE (GET_MODE (op0)));
12997 : rtx new_rtx
12998 : = assign_stack_temp_for_type (mode, temp_size, type);
12999 : rtx new_with_op0_mode
13000 : = adjust_address (new_rtx, GET_MODE (op0), 0);
13001 :
13002 : gcc_assert (!TREE_ADDRESSABLE (exp));
13003 :
13004 : if (GET_MODE (op0) == BLKmode)
13005 : {
13006 : rtx size_rtx = gen_int_mode (mode_size, Pmode);
13007 : emit_block_move (new_with_op0_mode, op0, size_rtx,
13008 : (modifier == EXPAND_STACK_PARM
13009 : ? BLOCK_OP_CALL_PARM
13010 : : BLOCK_OP_NORMAL));
13011 : }
13012 : else
13013 : emit_move_insn (new_with_op0_mode, op0);
13014 :
13015 : op0 = new_rtx;
13016 : }
13017 : }
13018 :
13019 126684 : op0 = adjust_address (op0, mode, 0);
13020 : }
13021 :
13022 : return op0;
13023 :
13024 63798 : case MODIFY_EXPR:
13025 63798 : {
13026 63798 : tree lhs = treeop0;
13027 63798 : tree rhs = treeop1;
13028 63798 : gcc_assert (ignore);
13029 :
13030 : /* Check for |= or &= of a bitfield of size one into another bitfield
13031 : of size 1. In this case, (unless we need the result of the
13032 : assignment) we can do this more efficiently with a
13033 : test followed by an assignment, if necessary.
13034 :
13035 : ??? At this point, we can't get a BIT_FIELD_REF here. But if
13036 : things change so we do, this code should be enhanced to
13037 : support it. */
13038 63798 : if (TREE_CODE (lhs) == COMPONENT_REF
13039 60546 : && (TREE_CODE (rhs) == BIT_IOR_EXPR
13040 60546 : || TREE_CODE (rhs) == BIT_AND_EXPR)
13041 0 : && TREE_OPERAND (rhs, 0) == lhs
13042 0 : && TREE_CODE (TREE_OPERAND (rhs, 1)) == COMPONENT_REF
13043 0 : && integer_onep (DECL_SIZE (TREE_OPERAND (lhs, 1)))
13044 63798 : && integer_onep (DECL_SIZE (TREE_OPERAND (TREE_OPERAND (rhs, 1), 1))))
13045 : {
13046 0 : rtx_code_label *label = gen_label_rtx ();
13047 0 : int value = TREE_CODE (rhs) == BIT_IOR_EXPR;
13048 0 : profile_probability prob = profile_probability::uninitialized ();
13049 0 : if (value)
13050 0 : jumpifnot (TREE_OPERAND (rhs, 1), label, prob);
13051 : else
13052 0 : jumpif (TREE_OPERAND (rhs, 1), label, prob);
13053 0 : expand_assignment (lhs, build_int_cst (TREE_TYPE (rhs), value),
13054 : false);
13055 0 : do_pending_stack_adjust ();
13056 0 : emit_label (label);
13057 0 : return const0_rtx;
13058 : }
13059 :
13060 63798 : expand_assignment (lhs, rhs, false);
13061 63798 : return const0_rtx;
13062 : }
13063 :
13064 13248182 : case ADDR_EXPR:
13065 13248182 : return expand_expr_addr_expr (exp, target, tmode, modifier);
13066 :
13067 173202 : case REALPART_EXPR:
13068 173202 : op0 = expand_normal (treeop0);
13069 173202 : return read_complex_part (op0, false);
13070 :
13071 197034 : case IMAGPART_EXPR:
13072 197034 : op0 = expand_normal (treeop0);
13073 197034 : return read_complex_part (op0, true);
13074 :
13075 0 : case RETURN_EXPR:
13076 0 : case LABEL_EXPR:
13077 0 : case GOTO_EXPR:
13078 0 : case SWITCH_EXPR:
13079 0 : case ASM_EXPR:
13080 : /* Expanded in cfgexpand.cc. */
13081 0 : gcc_unreachable ();
13082 :
13083 0 : case TRY_CATCH_EXPR:
13084 0 : case CATCH_EXPR:
13085 0 : case EH_FILTER_EXPR:
13086 0 : case TRY_FINALLY_EXPR:
13087 0 : case EH_ELSE_EXPR:
13088 : /* Lowered by tree-eh.cc. */
13089 0 : gcc_unreachable ();
13090 :
13091 0 : case WITH_CLEANUP_EXPR:
13092 0 : case CLEANUP_POINT_EXPR:
13093 0 : case TARGET_EXPR:
13094 0 : case CASE_LABEL_EXPR:
13095 0 : case VA_ARG_EXPR:
13096 0 : case BIND_EXPR:
13097 0 : case INIT_EXPR:
13098 0 : case CONJ_EXPR:
13099 0 : case COMPOUND_EXPR:
13100 0 : case PREINCREMENT_EXPR:
13101 0 : case PREDECREMENT_EXPR:
13102 0 : case POSTINCREMENT_EXPR:
13103 0 : case POSTDECREMENT_EXPR:
13104 0 : case LOOP_EXPR:
13105 0 : case EXIT_EXPR:
13106 0 : case COMPOUND_LITERAL_EXPR:
13107 : /* Lowered by gimplify.cc. */
13108 0 : gcc_unreachable ();
13109 :
13110 0 : case FDESC_EXPR:
13111 : /* Function descriptors are not valid except for as
13112 : initialization constants, and should not be expanded. */
13113 0 : gcc_unreachable ();
13114 :
13115 258 : case WITH_SIZE_EXPR:
13116 : /* WITH_SIZE_EXPR expands to its first argument. The caller should
13117 : have pulled out the size to use in whatever context it needed. */
13118 258 : return expand_expr_real (treeop0, original_target, tmode,
13119 258 : modifier, alt_rtl, inner_reference_p);
13120 :
13121 1857663 : default:
13122 1857663 : return expand_expr_real_2 (&ops, target, tmode, modifier);
13123 : }
13124 : }
13125 : #undef EXTEND_BITINT
13126 :
13127 : /* Subroutine of above: reduce EXP to the precision of TYPE (in the
13128 : signedness of TYPE), possibly returning the result in TARGET.
13129 : TYPE is known to be a partial integer type. */
13130 : static rtx
13131 89668 : reduce_to_bit_field_precision (rtx exp, rtx target, tree type)
13132 : {
13133 89668 : scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
13134 89668 : HOST_WIDE_INT prec = TYPE_PRECISION (type);
13135 89668 : gcc_assert ((GET_MODE (exp) == VOIDmode || GET_MODE (exp) == mode)
13136 : && (!target || GET_MODE (target) == mode));
13137 :
13138 : /* For constant values, reduce using wide_int_to_tree. */
13139 89668 : if (poly_int_rtx_p (exp))
13140 : {
13141 0 : auto value = wi::to_poly_wide (exp, mode);
13142 0 : tree t = wide_int_to_tree (type, value);
13143 0 : return expand_expr (t, target, VOIDmode, EXPAND_NORMAL);
13144 : }
13145 89668 : else if (TYPE_UNSIGNED (type))
13146 : {
13147 70557 : rtx mask = immed_wide_int_const
13148 70557 : (wi::mask (prec, false, GET_MODE_PRECISION (mode)), mode);
13149 70557 : return expand_and (mode, exp, mask, target);
13150 : }
13151 : else
13152 : {
13153 19111 : int count = GET_MODE_PRECISION (mode) - prec;
13154 19111 : exp = expand_shift (LSHIFT_EXPR, mode, exp, count, target, 0);
13155 19111 : return expand_shift (RSHIFT_EXPR, mode, exp, count, target, 0);
13156 : }
13157 : }
13158 :
13159 : /* Subroutine of above: returns true if OFFSET corresponds to an offset that
13160 : when applied to the address of EXP produces an address known to be
13161 : aligned more than BIGGEST_ALIGNMENT. */
13162 :
13163 : static bool
13164 279061 : is_aligning_offset (const_tree offset, const_tree exp)
13165 : {
13166 : /* Strip off any conversions. */
13167 296862 : while (CONVERT_EXPR_P (offset))
13168 17801 : offset = TREE_OPERAND (offset, 0);
13169 :
13170 : /* We must now have a BIT_AND_EXPR with a constant that is one less than
13171 : power of 2 and which is larger than BIGGEST_ALIGNMENT. */
13172 279061 : if (TREE_CODE (offset) != BIT_AND_EXPR
13173 0 : || !tree_fits_uhwi_p (TREE_OPERAND (offset, 1))
13174 0 : || compare_tree_int (TREE_OPERAND (offset, 1),
13175 0 : BIGGEST_ALIGNMENT / BITS_PER_UNIT) <= 0
13176 279061 : || !pow2p_hwi (tree_to_uhwi (TREE_OPERAND (offset, 1)) + 1))
13177 : return false;
13178 :
13179 : /* Look at the first operand of BIT_AND_EXPR and strip any conversion.
13180 : It must be NEGATE_EXPR. Then strip any more conversions. */
13181 0 : offset = TREE_OPERAND (offset, 0);
13182 0 : while (CONVERT_EXPR_P (offset))
13183 0 : offset = TREE_OPERAND (offset, 0);
13184 :
13185 0 : if (TREE_CODE (offset) != NEGATE_EXPR)
13186 : return false;
13187 :
13188 0 : offset = TREE_OPERAND (offset, 0);
13189 0 : while (CONVERT_EXPR_P (offset))
13190 0 : offset = TREE_OPERAND (offset, 0);
13191 :
13192 : /* This must now be the address of EXP. */
13193 0 : return TREE_CODE (offset) == ADDR_EXPR && TREE_OPERAND (offset, 0) == exp;
13194 : }
13195 :
13196 : /* Return a STRING_CST corresponding to ARG's constant initializer either
13197 : if it's a string constant, or, when VALREP is set, any other constant,
13198 : or null otherwise.
13199 : On success, set *PTR_OFFSET to the (possibly non-constant) byte offset
13200 : within the byte string that ARG is references. If nonnull set *MEM_SIZE
13201 : to the size of the byte string. If nonnull, set *DECL to the constant
13202 : declaration ARG refers to. */
13203 :
13204 : static tree
13205 15891601 : constant_byte_string (tree arg, tree *ptr_offset, tree *mem_size, tree *decl,
13206 : bool valrep = false)
13207 : {
13208 15891601 : tree dummy = NULL_TREE;
13209 15891601 : if (!mem_size)
13210 11128 : mem_size = &dummy;
13211 :
13212 : /* Store the type of the original expression before conversions
13213 : via NOP_EXPR or POINTER_PLUS_EXPR to other types have been
13214 : removed. */
13215 15891601 : tree argtype = TREE_TYPE (arg);
13216 :
13217 15891601 : tree array;
13218 15891601 : STRIP_NOPS (arg);
13219 :
13220 : /* Non-constant index into the character array in an ARRAY_REF
13221 : expression or null. */
13222 15891601 : tree varidx = NULL_TREE;
13223 :
13224 15891601 : poly_int64 base_off = 0;
13225 :
13226 15891601 : if (TREE_CODE (arg) == ADDR_EXPR)
13227 : {
13228 6457190 : arg = TREE_OPERAND (arg, 0);
13229 6457190 : tree ref = arg;
13230 6457190 : if (TREE_CODE (arg) == ARRAY_REF)
13231 : {
13232 569809 : tree idx = TREE_OPERAND (arg, 1);
13233 569809 : if (TREE_CODE (idx) != INTEGER_CST)
13234 : {
13235 : /* From a pointer (but not array) argument extract the variable
13236 : index to prevent get_addr_base_and_unit_offset() from failing
13237 : due to it. Use it later to compute the non-constant offset
13238 : into the string and return it to the caller. */
13239 167954 : varidx = idx;
13240 167954 : ref = TREE_OPERAND (arg, 0);
13241 :
13242 167954 : if (TREE_CODE (TREE_TYPE (arg)) == ARRAY_TYPE)
13243 : return NULL_TREE;
13244 :
13245 28905 : if (!integer_zerop (array_ref_low_bound (arg)))
13246 : return NULL_TREE;
13247 :
13248 28157 : if (!integer_onep (array_ref_element_size (arg)))
13249 : return NULL_TREE;
13250 : }
13251 : }
13252 6316250 : array = get_addr_base_and_unit_offset (ref, &base_off);
13253 6316250 : if (!array
13254 6268543 : || (TREE_CODE (array) != VAR_DECL
13255 6268543 : && TREE_CODE (array) != CONST_DECL
13256 3931516 : && TREE_CODE (array) != STRING_CST))
13257 : return NULL_TREE;
13258 : }
13259 9434411 : else if (TREE_CODE (arg) == PLUS_EXPR || TREE_CODE (arg) == POINTER_PLUS_EXPR)
13260 : {
13261 27056 : tree arg0 = TREE_OPERAND (arg, 0);
13262 27056 : tree arg1 = TREE_OPERAND (arg, 1);
13263 :
13264 27056 : tree offset;
13265 27056 : tree str = string_constant (arg0, &offset, mem_size, decl);
13266 27056 : if (!str)
13267 : {
13268 17550 : str = string_constant (arg1, &offset, mem_size, decl);
13269 17550 : arg1 = arg0;
13270 : }
13271 :
13272 17550 : if (str)
13273 : {
13274 : /* Avoid pointers to arrays (see bug 86622). */
13275 9506 : if (POINTER_TYPE_P (TREE_TYPE (arg))
13276 9506 : && TREE_CODE (TREE_TYPE (TREE_TYPE (arg))) == ARRAY_TYPE
13277 1890 : && !(decl && !*decl)
13278 12026 : && !(decl && tree_fits_uhwi_p (DECL_SIZE_UNIT (*decl))
13279 1260 : && tree_fits_uhwi_p (*mem_size)
13280 1260 : && tree_int_cst_equal (*mem_size, DECL_SIZE_UNIT (*decl))))
13281 : return NULL_TREE;
13282 :
13283 7616 : tree type = TREE_TYPE (offset);
13284 7616 : arg1 = fold_convert (type, arg1);
13285 7616 : *ptr_offset = fold_build2 (PLUS_EXPR, type, offset, arg1);
13286 7616 : return str;
13287 : }
13288 : return NULL_TREE;
13289 : }
13290 9407355 : else if (TREE_CODE (arg) == SSA_NAME)
13291 : {
13292 3919717 : gimple *stmt = SSA_NAME_DEF_STMT (arg);
13293 3919717 : if (!is_gimple_assign (stmt))
13294 : return NULL_TREE;
13295 :
13296 1054324 : tree rhs1 = gimple_assign_rhs1 (stmt);
13297 1054324 : tree_code code = gimple_assign_rhs_code (stmt);
13298 1054324 : if (code == ADDR_EXPR)
13299 236249 : return string_constant (rhs1, ptr_offset, mem_size, decl);
13300 818075 : else if (code != POINTER_PLUS_EXPR)
13301 : return NULL_TREE;
13302 :
13303 194198 : tree offset;
13304 194198 : if (tree str = string_constant (rhs1, &offset, mem_size, decl))
13305 : {
13306 : /* Avoid pointers to arrays (see bug 86622). */
13307 23635 : if (POINTER_TYPE_P (TREE_TYPE (rhs1))
13308 23635 : && TREE_CODE (TREE_TYPE (TREE_TYPE (rhs1))) == ARRAY_TYPE
13309 19700 : && !(decl && !*decl)
13310 39883 : && !(decl && tree_fits_uhwi_p (DECL_SIZE_UNIT (*decl))
13311 8124 : && tree_fits_uhwi_p (*mem_size)
13312 8124 : && tree_int_cst_equal (*mem_size, DECL_SIZE_UNIT (*decl))))
13313 : return NULL_TREE;
13314 :
13315 7237 : tree rhs2 = gimple_assign_rhs2 (stmt);
13316 7237 : tree type = TREE_TYPE (offset);
13317 7237 : rhs2 = fold_convert (type, rhs2);
13318 7237 : *ptr_offset = fold_build2 (PLUS_EXPR, type, offset, rhs2);
13319 7237 : return str;
13320 : }
13321 : return NULL_TREE;
13322 : }
13323 5487638 : else if (DECL_P (arg))
13324 : array = arg;
13325 : else
13326 : return NULL_TREE;
13327 :
13328 8866510 : tree offset = wide_int_to_tree (sizetype, base_off);
13329 8866510 : if (varidx)
13330 : {
13331 2850 : if (TREE_CODE (TREE_TYPE (array)) != ARRAY_TYPE)
13332 : return NULL_TREE;
13333 :
13334 2139 : gcc_assert (TREE_CODE (arg) == ARRAY_REF);
13335 2139 : tree chartype = TREE_TYPE (TREE_TYPE (TREE_OPERAND (arg, 0)));
13336 2139 : if (TREE_CODE (chartype) != INTEGER_TYPE)
13337 : return NULL;
13338 :
13339 1983 : offset = fold_convert (sizetype, varidx);
13340 : }
13341 :
13342 8865643 : if (TREE_CODE (array) == STRING_CST)
13343 : {
13344 3537136 : *ptr_offset = fold_convert (sizetype, offset);
13345 3537136 : *mem_size = TYPE_SIZE_UNIT (TREE_TYPE (array));
13346 3537136 : if (decl)
13347 760349 : *decl = NULL_TREE;
13348 3537136 : gcc_checking_assert (tree_to_shwi (TYPE_SIZE_UNIT (TREE_TYPE (array)))
13349 : >= TREE_STRING_LENGTH (array));
13350 : return array;
13351 : }
13352 :
13353 5328507 : tree init = ctor_for_folding (array);
13354 5328507 : if (!init || init == error_mark_node)
13355 : return NULL_TREE;
13356 :
13357 163244 : if (valrep)
13358 : {
13359 58506 : HOST_WIDE_INT cstoff;
13360 58506 : if (!base_off.is_constant (&cstoff))
13361 : return NULL_TREE;
13362 :
13363 : /* Check that the host and target are sane. */
13364 58506 : if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
13365 : return NULL_TREE;
13366 :
13367 58506 : HOST_WIDE_INT typesz = int_size_in_bytes (TREE_TYPE (init));
13368 58506 : if (typesz <= 0 || (int) typesz != typesz)
13369 : return NULL_TREE;
13370 :
13371 58353 : HOST_WIDE_INT size = typesz;
13372 58353 : if (VAR_P (array)
13373 58341 : && DECL_SIZE_UNIT (array)
13374 116694 : && tree_fits_shwi_p (DECL_SIZE_UNIT (array)))
13375 : {
13376 58341 : size = tree_to_shwi (DECL_SIZE_UNIT (array));
13377 58341 : gcc_checking_assert (size >= typesz);
13378 : }
13379 :
13380 : /* If value representation was requested convert the initializer
13381 : for the whole array or object into a string of bytes forming
13382 : its value representation and return it. */
13383 58353 : unsigned char *bytes = XNEWVEC (unsigned char, size);
13384 58353 : int r = native_encode_initializer (init, bytes, size);
13385 58353 : if (r < typesz)
13386 : {
13387 60 : XDELETEVEC (bytes);
13388 60 : return NULL_TREE;
13389 : }
13390 :
13391 58293 : if (r < size)
13392 0 : memset (bytes + r, '\0', size - r);
13393 :
13394 58293 : const char *p = reinterpret_cast<const char *>(bytes);
13395 58293 : init = build_string_literal (size, p, char_type_node);
13396 58293 : init = TREE_OPERAND (init, 0);
13397 58293 : init = TREE_OPERAND (init, 0);
13398 58293 : XDELETE (bytes);
13399 :
13400 58293 : *mem_size = size_int (TREE_STRING_LENGTH (init));
13401 58293 : *ptr_offset = wide_int_to_tree (ssizetype, base_off);
13402 :
13403 58293 : if (decl)
13404 0 : *decl = array;
13405 :
13406 : return init;
13407 : }
13408 :
13409 104738 : if (TREE_CODE (init) == CONSTRUCTOR)
13410 : {
13411 : /* Convert the 64-bit constant offset to a wider type to avoid
13412 : overflow and use it to obtain the initializer for the subobject
13413 : it points into. */
13414 83997 : offset_int wioff;
13415 83997 : if (!base_off.is_constant (&wioff))
13416 155 : return NULL_TREE;
13417 :
13418 83997 : wioff *= BITS_PER_UNIT;
13419 83997 : if (!wi::fits_uhwi_p (wioff))
13420 : return NULL_TREE;
13421 :
13422 83889 : base_off = wioff.to_uhwi ();
13423 83889 : unsigned HOST_WIDE_INT fieldoff = 0;
13424 83889 : init = fold_ctor_reference (TREE_TYPE (arg), init, base_off, 0, array,
13425 : &fieldoff);
13426 83889 : if (!init || init == error_mark_node)
13427 : return NULL_TREE;
13428 :
13429 83842 : HOST_WIDE_INT cstoff;
13430 83842 : if (!base_off.is_constant (&cstoff))
13431 : return NULL_TREE;
13432 :
13433 83842 : cstoff = (cstoff - fieldoff) / BITS_PER_UNIT;
13434 83842 : tree off = build_int_cst (sizetype, cstoff);
13435 83842 : if (varidx)
13436 1122 : offset = fold_build2 (PLUS_EXPR, TREE_TYPE (offset), offset, off);
13437 : else
13438 : offset = off;
13439 : }
13440 :
13441 104583 : *ptr_offset = offset;
13442 :
13443 104583 : tree inittype = TREE_TYPE (init);
13444 :
13445 104583 : if (TREE_CODE (init) == INTEGER_CST
13446 104583 : && (TREE_CODE (TREE_TYPE (array)) == INTEGER_TYPE
13447 1116 : || TYPE_MAIN_VARIANT (inittype) == char_type_node))
13448 : {
13449 : /* Check that the host and target are sane. */
13450 903 : if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
13451 : return NULL_TREE;
13452 :
13453 : /* For a reference to (address of) a single constant character,
13454 : store the native representation of the character in CHARBUF.
13455 : If the reference is to an element of an array or a member
13456 : of a struct, only consider narrow characters until ctors
13457 : for wide character arrays are transformed to STRING_CSTs
13458 : like those for narrow arrays. */
13459 903 : unsigned char charbuf[MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT];
13460 903 : int len = native_encode_expr (init, charbuf, sizeof charbuf, 0);
13461 903 : if (len > 0)
13462 : {
13463 : /* Construct a string literal with elements of INITTYPE and
13464 : the representation above. Then strip
13465 : the ADDR_EXPR (ARRAY_REF (...)) around the STRING_CST. */
13466 903 : init = build_string_literal (len, (char *)charbuf, inittype);
13467 903 : init = TREE_OPERAND (TREE_OPERAND (init, 0), 0);
13468 : }
13469 : }
13470 :
13471 104583 : tree initsize = TYPE_SIZE_UNIT (inittype);
13472 :
13473 104583 : if (TREE_CODE (init) == CONSTRUCTOR && initializer_zerop (init))
13474 : {
13475 : /* Fold an empty/zero constructor for an implicitly initialized
13476 : object or subobject into the empty string. */
13477 :
13478 : /* Determine the character type from that of the original
13479 : expression. */
13480 9337 : tree chartype = argtype;
13481 9337 : if (POINTER_TYPE_P (chartype))
13482 9330 : chartype = TREE_TYPE (chartype);
13483 14958 : while (TREE_CODE (chartype) == ARRAY_TYPE)
13484 5621 : chartype = TREE_TYPE (chartype);
13485 :
13486 9337 : if (INTEGRAL_TYPE_P (chartype)
13487 9337 : && TYPE_PRECISION (chartype) == TYPE_PRECISION (char_type_node))
13488 : {
13489 : /* Convert a char array to an empty STRING_CST having an array
13490 : of the expected type and size. */
13491 9133 : if (!initsize)
13492 3090 : initsize = integer_zero_node;
13493 6043 : else if (!tree_fits_uhwi_p (initsize))
13494 : return NULL_TREE;
13495 :
13496 9127 : unsigned HOST_WIDE_INT size = tree_to_uhwi (initsize);
13497 9127 : if (size > (unsigned HOST_WIDE_INT) INT_MAX)
13498 : return NULL_TREE;
13499 :
13500 9127 : init = build_string_literal (size, NULL, chartype, size);
13501 9127 : init = TREE_OPERAND (init, 0);
13502 9127 : init = TREE_OPERAND (init, 0);
13503 :
13504 9127 : *ptr_offset = integer_zero_node;
13505 : }
13506 : }
13507 :
13508 104577 : if (decl)
13509 54389 : *decl = array;
13510 :
13511 104577 : if (TREE_CODE (init) != STRING_CST)
13512 : return NULL_TREE;
13513 :
13514 72333 : *mem_size = initsize;
13515 :
13516 72333 : gcc_checking_assert (tree_to_shwi (initsize) >= TREE_STRING_LENGTH (init));
13517 :
13518 : return init;
13519 : }
13520 :
13521 : /* Return STRING_CST if an ARG corresponds to a string constant or zero
13522 : if it doesn't. If we return nonzero, set *PTR_OFFSET to the (possibly
13523 : non-constant) offset in bytes within the string that ARG is accessing.
13524 : If MEM_SIZE is non-zero the storage size of the memory is returned.
13525 : If DECL is non-zero the constant declaration is returned if available. */
13526 :
13527 : tree
13528 11209923 : string_constant (tree arg, tree *ptr_offset, tree *mem_size, tree *decl)
13529 : {
13530 11209923 : return constant_byte_string (arg, ptr_offset, mem_size, decl, false);
13531 : }
13532 :
13533 : /* Similar to string_constant, return a STRING_CST corresponding
13534 : to the value representation of the first argument if it's
13535 : a constant. */
13536 :
13537 : tree
13538 4681678 : byte_representation (tree arg, tree *ptr_offset, tree *mem_size, tree *decl)
13539 : {
13540 4681678 : return constant_byte_string (arg, ptr_offset, mem_size, decl, true);
13541 : }
13542 :
13543 : /* Optimize x % C1 == C2 for signed modulo if C1 is a power of two and C2
13544 : is non-zero and C3 ((1<<(prec-1)) | (C1 - 1)):
13545 : for C2 > 0 to x & C3 == C2
13546 : for C2 < 0 to x & C3 == (C2 & C3). */
13547 : enum tree_code
13548 35 : maybe_optimize_pow2p_mod_cmp (enum tree_code code, tree *arg0, tree *arg1)
13549 : {
13550 35 : gimple *stmt = get_def_for_expr (*arg0, TRUNC_MOD_EXPR);
13551 35 : tree treeop0 = gimple_assign_rhs1 (stmt);
13552 35 : tree treeop1 = gimple_assign_rhs2 (stmt);
13553 35 : tree type = TREE_TYPE (*arg0);
13554 35 : scalar_int_mode mode;
13555 35 : if (!is_a <scalar_int_mode> (TYPE_MODE (type), &mode))
13556 : return code;
13557 70 : if (GET_MODE_BITSIZE (mode) != TYPE_PRECISION (type)
13558 35 : || TYPE_PRECISION (type) <= 1
13559 35 : || TYPE_UNSIGNED (type)
13560 : /* Signed x % c == 0 should have been optimized into unsigned modulo
13561 : earlier. */
13562 35 : || integer_zerop (*arg1)
13563 : /* If c is known to be non-negative, modulo will be expanded as unsigned
13564 : modulo. */
13565 70 : || get_range_pos_neg (treeop0, currently_expanding_gimple_stmt) == 1)
13566 : return code;
13567 :
13568 : /* x % c == d where d < 0 && d <= -c should be always false. */
13569 35 : if (tree_int_cst_sgn (*arg1) == -1
13570 51 : && -wi::to_widest (treeop1) >= wi::to_widest (*arg1))
13571 : return code;
13572 :
13573 35 : int prec = TYPE_PRECISION (type);
13574 35 : wide_int w = wi::to_wide (treeop1) - 1;
13575 35 : w |= wi::shifted_mask (0, prec - 1, true, prec);
13576 35 : tree c3 = wide_int_to_tree (type, w);
13577 35 : tree c4 = *arg1;
13578 35 : if (tree_int_cst_sgn (*arg1) == -1)
13579 16 : c4 = wide_int_to_tree (type, w & wi::to_wide (*arg1));
13580 :
13581 35 : rtx op0 = expand_normal (treeop0);
13582 35 : treeop0 = make_tree (TREE_TYPE (treeop0), op0);
13583 :
13584 35 : bool speed_p = optimize_insn_for_speed_p ();
13585 :
13586 35 : do_pending_stack_adjust ();
13587 :
13588 35 : location_t loc = gimple_location (stmt);
13589 35 : struct separate_ops ops;
13590 35 : ops.code = TRUNC_MOD_EXPR;
13591 35 : ops.location = loc;
13592 35 : ops.type = TREE_TYPE (treeop0);
13593 35 : ops.op0 = treeop0;
13594 35 : ops.op1 = treeop1;
13595 35 : ops.op2 = NULL_TREE;
13596 35 : start_sequence ();
13597 35 : rtx mor = expand_expr_real_2 (&ops, NULL_RTX, TYPE_MODE (ops.type),
13598 : EXPAND_NORMAL);
13599 35 : rtx_insn *moinsns = end_sequence ();
13600 :
13601 35 : unsigned mocost = seq_cost (moinsns, speed_p);
13602 35 : mocost += rtx_cost (mor, mode, EQ, 0, speed_p);
13603 35 : mocost += rtx_cost (expand_normal (*arg1), mode, EQ, 1, speed_p);
13604 :
13605 35 : ops.code = BIT_AND_EXPR;
13606 35 : ops.location = loc;
13607 35 : ops.type = TREE_TYPE (treeop0);
13608 35 : ops.op0 = treeop0;
13609 35 : ops.op1 = c3;
13610 35 : ops.op2 = NULL_TREE;
13611 35 : start_sequence ();
13612 35 : rtx mur = expand_expr_real_2 (&ops, NULL_RTX, TYPE_MODE (ops.type),
13613 : EXPAND_NORMAL);
13614 35 : rtx_insn *muinsns = end_sequence ();
13615 :
13616 35 : unsigned mucost = seq_cost (muinsns, speed_p);
13617 35 : mucost += rtx_cost (mur, mode, EQ, 0, speed_p);
13618 35 : mucost += rtx_cost (expand_normal (c4), mode, EQ, 1, speed_p);
13619 :
13620 35 : if (mocost <= mucost)
13621 : {
13622 0 : emit_insn (moinsns);
13623 0 : *arg0 = make_tree (TREE_TYPE (*arg0), mor);
13624 0 : return code;
13625 : }
13626 :
13627 35 : emit_insn (muinsns);
13628 35 : *arg0 = make_tree (TREE_TYPE (*arg0), mur);
13629 35 : *arg1 = c4;
13630 35 : return code;
13631 35 : }
13632 :
13633 : /* Attempt to optimize unsigned (X % C1) == C2 (or (X % C1) != C2).
13634 : If C1 is odd to:
13635 : (X - C2) * C3 <= C4 (or >), where
13636 : C3 is modular multiplicative inverse of C1 and 1<<prec and
13637 : C4 is ((1<<prec) - 1) / C1 or ((1<<prec) - 1) / C1 - 1 (the latter
13638 : if C2 > ((1<<prec) - 1) % C1).
13639 : If C1 is even, S = ctz (C1) and C2 is 0, use
13640 : ((X * C3) r>> S) <= C4, where C3 is modular multiplicative
13641 : inverse of C1>>S and 1<<prec and C4 is (((1<<prec) - 1) / (C1>>S)) >> S.
13642 :
13643 : For signed (X % C1) == 0 if C1 is odd to (all operations in it
13644 : unsigned):
13645 : (X * C3) + C4 <= 2 * C4, where
13646 : C3 is modular multiplicative inverse of (unsigned) C1 and 1<<prec and
13647 : C4 is ((1<<(prec - 1) - 1) / C1).
13648 : If C1 is even, S = ctz(C1), use
13649 : ((X * C3) + C4) r>> S <= (C4 >> (S - 1))
13650 : where C3 is modular multiplicative inverse of (unsigned)(C1>>S) and 1<<prec
13651 : and C4 is ((1<<(prec - 1) - 1) / (C1>>S)) & (-1<<S).
13652 :
13653 : See the Hacker's Delight book, section 10-17. */
13654 : enum tree_code
13655 3373210 : maybe_optimize_mod_cmp (enum tree_code code, tree *arg0, tree *arg1)
13656 : {
13657 3373210 : gcc_checking_assert (code == EQ_EXPR || code == NE_EXPR);
13658 3373210 : gcc_checking_assert (TREE_CODE (*arg1) == INTEGER_CST);
13659 :
13660 3373210 : if (optimize < 2)
13661 : return code;
13662 :
13663 2555114 : gimple *stmt = get_def_for_expr (*arg0, TRUNC_MOD_EXPR);
13664 2555114 : if (stmt == NULL)
13665 : return code;
13666 :
13667 2111 : tree treeop0 = gimple_assign_rhs1 (stmt);
13668 2111 : tree treeop1 = gimple_assign_rhs2 (stmt);
13669 2111 : if (TREE_CODE (treeop0) != SSA_NAME
13670 2009 : || TREE_CODE (treeop1) != INTEGER_CST
13671 : /* Don't optimize the undefined behavior case x % 0;
13672 : x % 1 should have been optimized into zero, punt if
13673 : it makes it here for whatever reason;
13674 : x % -c should have been optimized into x % c. */
13675 1651 : || compare_tree_int (treeop1, 2) <= 0
13676 : /* Likewise x % c == d where d >= c should be always false. */
13677 3686 : || tree_int_cst_le (treeop1, *arg1))
13678 : return code;
13679 :
13680 : /* Unsigned x % pow2 is handled right already, for signed
13681 : modulo handle it in maybe_optimize_pow2p_mod_cmp. */
13682 1575 : if (integer_pow2p (treeop1))
13683 35 : return maybe_optimize_pow2p_mod_cmp (code, arg0, arg1);
13684 :
13685 1540 : tree type = TREE_TYPE (*arg0);
13686 1540 : scalar_int_mode mode;
13687 1540 : if (!is_a <scalar_int_mode> (TYPE_MODE (type), &mode))
13688 : return code;
13689 3080 : if (GET_MODE_BITSIZE (mode) != TYPE_PRECISION (type)
13690 1540 : || TYPE_PRECISION (type) <= 1)
13691 : return code;
13692 :
13693 1540 : signop sgn = UNSIGNED;
13694 : /* If both operands are known to have the sign bit clear, handle
13695 : even the signed modulo case as unsigned. treeop1 is always
13696 : positive >= 2, checked above. */
13697 1540 : if (!TYPE_UNSIGNED (type)
13698 1540 : && get_range_pos_neg (treeop0, currently_expanding_gimple_stmt) != 1)
13699 : sgn = SIGNED;
13700 :
13701 1540 : if (!TYPE_UNSIGNED (type))
13702 : {
13703 1312 : if (tree_int_cst_sgn (*arg1) == -1)
13704 : return code;
13705 1305 : type = unsigned_type_for (type);
13706 1305 : if (!type || TYPE_MODE (type) != TYPE_MODE (TREE_TYPE (*arg0)))
13707 : return code;
13708 : }
13709 :
13710 1533 : int prec = TYPE_PRECISION (type);
13711 1533 : wide_int w = wi::to_wide (treeop1);
13712 1533 : int shift = wi::ctz (w);
13713 : /* Unsigned (X % C1) == C2 is equivalent to (X - C2) % C1 == 0 if
13714 : C2 <= -1U % C1, because for any Z >= 0U - C2 in that case (Z % C1) != 0.
13715 : If C1 is odd, we can handle all cases by subtracting
13716 : C4 below. We could handle even the even C1 and C2 > -1U % C1 cases
13717 : e.g. by testing for overflow on the subtraction, punt on that for now
13718 : though. */
13719 1533 : if ((sgn == SIGNED || shift) && !integer_zerop (*arg1))
13720 : {
13721 189 : if (sgn == SIGNED)
13722 174 : return code;
13723 26 : wide_int x = wi::umod_trunc (wi::mask (prec, false, prec), w);
13724 26 : if (wi::gtu_p (wi::to_wide (*arg1), x))
13725 11 : return code;
13726 26 : }
13727 :
13728 1359 : imm_use_iterator imm_iter;
13729 1359 : use_operand_p use_p;
13730 6878 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, treeop0)
13731 : {
13732 5529 : gimple *use_stmt = USE_STMT (use_p);
13733 : /* Punt if treeop0 is used in the same bb in a division
13734 : or another modulo with the same divisor. We should expect
13735 : the division and modulo combined together. */
13736 10961 : if (use_stmt == stmt
13737 5529 : || gimple_bb (use_stmt) != gimple_bb (stmt))
13738 5432 : continue;
13739 97 : if (!is_gimple_assign (use_stmt)
13740 97 : || (gimple_assign_rhs_code (use_stmt) != TRUNC_DIV_EXPR
13741 81 : && gimple_assign_rhs_code (use_stmt) != TRUNC_MOD_EXPR))
13742 72 : continue;
13743 25 : if (gimple_assign_rhs1 (use_stmt) != treeop0
13744 25 : || !operand_equal_p (gimple_assign_rhs2 (use_stmt), treeop1, 0))
13745 15 : continue;
13746 10 : return code;
13747 10 : }
13748 :
13749 1349 : w = wi::lrshift (w, shift);
13750 1349 : wide_int a = wide_int::from (w, prec + 1, UNSIGNED);
13751 1349 : wide_int b = wi::shifted_mask (prec, 1, false, prec + 1);
13752 1349 : wide_int m = wide_int::from (wi::mod_inv (a, b), prec, UNSIGNED);
13753 1349 : tree c3 = wide_int_to_tree (type, m);
13754 1349 : tree c5 = NULL_TREE;
13755 1349 : wide_int d, e;
13756 1349 : if (sgn == UNSIGNED)
13757 : {
13758 1008 : d = wi::divmod_trunc (wi::mask (prec, false, prec), w, UNSIGNED, &e);
13759 : /* Use <= floor ((1<<prec) - 1) / C1 only if C2 <= ((1<<prec) - 1) % C1,
13760 : otherwise use < or subtract one from C4. E.g. for
13761 : x % 3U == 0 we transform this into x * 0xaaaaaaab <= 0x55555555, but
13762 : x % 3U == 1 already needs to be
13763 : (x - 1) * 0xaaaaaaabU <= 0x55555554. */
13764 1008 : if (!shift && wi::gtu_p (wi::to_wide (*arg1), e))
13765 15 : d -= 1;
13766 1008 : if (shift)
13767 442 : d = wi::lrshift (d, shift);
13768 : }
13769 : else
13770 : {
13771 341 : e = wi::udiv_trunc (wi::mask (prec - 1, false, prec), w);
13772 341 : if (!shift)
13773 142 : d = wi::lshift (e, 1);
13774 : else
13775 : {
13776 199 : e = wi::bit_and (e, wi::mask (shift, true, prec));
13777 199 : d = wi::lrshift (e, shift - 1);
13778 : }
13779 341 : c5 = wide_int_to_tree (type, e);
13780 : }
13781 1349 : tree c4 = wide_int_to_tree (type, d);
13782 :
13783 1349 : rtx op0 = expand_normal (treeop0);
13784 1349 : treeop0 = make_tree (TREE_TYPE (treeop0), op0);
13785 :
13786 1349 : bool speed_p = optimize_insn_for_speed_p ();
13787 :
13788 1349 : do_pending_stack_adjust ();
13789 :
13790 1349 : location_t loc = gimple_location (stmt);
13791 1349 : struct separate_ops ops;
13792 1349 : ops.code = TRUNC_MOD_EXPR;
13793 1349 : ops.location = loc;
13794 1349 : ops.type = TREE_TYPE (treeop0);
13795 1349 : ops.op0 = treeop0;
13796 1349 : ops.op1 = treeop1;
13797 1349 : ops.op2 = NULL_TREE;
13798 1349 : start_sequence ();
13799 1349 : rtx mor = expand_expr_real_2 (&ops, NULL_RTX, TYPE_MODE (ops.type),
13800 : EXPAND_NORMAL);
13801 1349 : rtx_insn *moinsns = end_sequence ();
13802 :
13803 1349 : unsigned mocost = seq_cost (moinsns, speed_p);
13804 1349 : mocost += rtx_cost (mor, mode, EQ, 0, speed_p);
13805 1349 : mocost += rtx_cost (expand_normal (*arg1), mode, EQ, 1, speed_p);
13806 :
13807 1349 : tree t = fold_convert_loc (loc, type, treeop0);
13808 1349 : if (!integer_zerop (*arg1))
13809 36 : t = fold_build2_loc (loc, MINUS_EXPR, type, t, fold_convert (type, *arg1));
13810 1349 : t = fold_build2_loc (loc, MULT_EXPR, type, t, c3);
13811 1349 : if (sgn == SIGNED)
13812 341 : t = fold_build2_loc (loc, PLUS_EXPR, type, t, c5);
13813 1349 : if (shift)
13814 : {
13815 641 : tree s = build_int_cst (NULL_TREE, shift);
13816 641 : t = fold_build2_loc (loc, RROTATE_EXPR, type, t, s);
13817 : }
13818 :
13819 1349 : start_sequence ();
13820 1349 : rtx mur = expand_normal (t);
13821 1349 : rtx_insn *muinsns = end_sequence ();
13822 :
13823 1349 : unsigned mucost = seq_cost (muinsns, speed_p);
13824 1349 : mucost += rtx_cost (mur, mode, LE, 0, speed_p);
13825 1349 : mucost += rtx_cost (expand_normal (c4), mode, LE, 1, speed_p);
13826 :
13827 1349 : if (mocost <= mucost)
13828 : {
13829 217 : emit_insn (moinsns);
13830 217 : *arg0 = make_tree (TREE_TYPE (*arg0), mor);
13831 217 : return code;
13832 : }
13833 :
13834 1132 : emit_insn (muinsns);
13835 1132 : *arg0 = make_tree (type, mur);
13836 1132 : *arg1 = c4;
13837 1132 : return code == EQ_EXPR ? LE_EXPR : GT_EXPR;
13838 2882 : }
13839 :
13840 : /* Optimize x - y < 0 into x < 0 if x - y has undefined overflow. */
13841 :
13842 : void
13843 241576 : maybe_optimize_sub_cmp_0 (enum tree_code code, tree *arg0, tree *arg1)
13844 : {
13845 241576 : gcc_checking_assert (code == GT_EXPR || code == GE_EXPR
13846 : || code == LT_EXPR || code == LE_EXPR);
13847 241576 : gcc_checking_assert (integer_zerop (*arg1));
13848 :
13849 241576 : if (!optimize)
13850 : return;
13851 :
13852 207797 : gimple *stmt = get_def_for_expr (*arg0, MINUS_EXPR);
13853 207797 : if (stmt == NULL)
13854 : return;
13855 :
13856 1371 : tree treeop0 = gimple_assign_rhs1 (stmt);
13857 1371 : tree treeop1 = gimple_assign_rhs2 (stmt);
13858 1371 : if (!TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (treeop0)))
13859 : return;
13860 :
13861 1274 : *arg0 = treeop0;
13862 1274 : *arg1 = treeop1;
13863 : }
13864 :
13865 :
13866 : /* Expand CODE with arguments INNER & (1<<BITNUM) and 0 that represents
13867 : a single bit equality/inequality test, returns where the result is located. */
13868 :
13869 : static rtx
13870 9846 : expand_single_bit_test (location_t loc, enum tree_code code,
13871 : tree inner, int bitnum,
13872 : tree result_type, rtx target,
13873 : machine_mode mode)
13874 : {
13875 9846 : gcc_assert (code == NE_EXPR || code == EQ_EXPR);
13876 :
13877 9846 : tree type = TREE_TYPE (inner);
13878 9846 : scalar_int_mode operand_mode = SCALAR_INT_TYPE_MODE (type);
13879 9846 : int ops_unsigned;
13880 9846 : tree signed_type, unsigned_type, intermediate_type;
13881 9846 : gimple *inner_def;
13882 :
13883 : /* First, see if we can fold the single bit test into a sign-bit
13884 : test. */
13885 9846 : if (bitnum == TYPE_PRECISION (type) - 1
13886 9846 : && type_has_mode_precision_p (type))
13887 : {
13888 241 : tree stype = signed_type_for (type);
13889 444 : tree tmp = fold_build2_loc (loc, code == EQ_EXPR ? GE_EXPR : LT_EXPR,
13890 : result_type,
13891 : fold_convert_loc (loc, stype, inner),
13892 : build_int_cst (stype, 0));
13893 241 : return expand_expr (tmp, target, VOIDmode, EXPAND_NORMAL);
13894 : }
13895 :
13896 : /* Otherwise we have (A & C) != 0 where C is a single bit,
13897 : convert that into ((A >> C2) & 1). Where C2 = log2(C).
13898 : Similarly for (A & C) == 0. */
13899 :
13900 : /* If INNER is a right shift of a constant and it plus BITNUM does
13901 : not overflow, adjust BITNUM and INNER. */
13902 9605 : if ((inner_def = get_def_for_expr (inner, RSHIFT_EXPR))
13903 61 : && TREE_CODE (gimple_assign_rhs2 (inner_def)) == INTEGER_CST
13904 16 : && bitnum < TYPE_PRECISION (type)
13905 9621 : && wi::ltu_p (wi::to_wide (gimple_assign_rhs2 (inner_def)),
13906 9605 : TYPE_PRECISION (type) - bitnum))
13907 : {
13908 16 : bitnum += tree_to_uhwi (gimple_assign_rhs2 (inner_def));
13909 16 : inner = gimple_assign_rhs1 (inner_def);
13910 : }
13911 :
13912 : /* If we are going to be able to omit the AND below, we must do our
13913 : operations as unsigned. If we must use the AND, we have a choice.
13914 : Normally unsigned is faster, but for some machines signed is. */
13915 9605 : ops_unsigned = (load_extend_op (operand_mode) == SIGN_EXTEND
13916 : && !flag_syntax_only) ? 0 : 1;
13917 :
13918 9605 : signed_type = lang_hooks.types.type_for_mode (operand_mode, 0);
13919 9605 : unsigned_type = lang_hooks.types.type_for_mode (operand_mode, 1);
13920 9605 : intermediate_type = ops_unsigned ? unsigned_type : signed_type;
13921 9605 : inner = fold_convert_loc (loc, intermediate_type, inner);
13922 :
13923 9605 : rtx inner0 = expand_expr (inner, NULL_RTX, VOIDmode, EXPAND_NORMAL);
13924 :
13925 9605 : if (CONST_SCALAR_INT_P (inner0))
13926 : {
13927 0 : wide_int t = rtx_mode_t (inner0, operand_mode);
13928 0 : bool setp = (wi::lrshift (t, bitnum) & 1) != 0;
13929 0 : return (setp ^ (code == EQ_EXPR)) ? const1_rtx : const0_rtx;
13930 0 : }
13931 9605 : int bitpos = bitnum;
13932 :
13933 9605 : if (BYTES_BIG_ENDIAN)
13934 : bitpos = GET_MODE_BITSIZE (operand_mode) - 1 - bitpos;
13935 :
13936 9605 : inner0 = extract_bit_field (inner0, 1, bitpos, 1, target,
13937 : operand_mode, mode, 0, NULL);
13938 :
13939 9605 : if (code == EQ_EXPR)
13940 2694 : inner0 = expand_binop (GET_MODE (inner0), xor_optab, inner0, const1_rtx,
13941 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
13942 9605 : if (GET_MODE (inner0) != mode)
13943 : {
13944 0 : rtx t = gen_reg_rtx (mode);
13945 0 : convert_move (t, inner0, 0);
13946 0 : return t;
13947 : }
13948 : return inner0;
13949 : }
13950 :
13951 : /* Generate code to calculate OPS, and exploded expression
13952 : using a store-flag instruction and return an rtx for the result.
13953 : OPS reflects a comparison.
13954 :
13955 : If TARGET is nonzero, store the result there if convenient.
13956 :
13957 : Return zero if there is no suitable set-flag instruction
13958 : available on this machine.
13959 :
13960 : Once expand_expr has been called on the arguments of the comparison,
13961 : we are committed to doing the store flag, since it is not safe to
13962 : re-evaluate the expression. We emit the store-flag insn by calling
13963 : emit_store_flag, but only expand the arguments if we have a reason
13964 : to believe that emit_store_flag will be successful. If we think that
13965 : it will, but it isn't, we have to simulate the store-flag with a
13966 : set/jump/set sequence. */
13967 :
13968 : static rtx
13969 643674 : do_store_flag (const_sepops ops, rtx target, machine_mode mode)
13970 : {
13971 643674 : enum rtx_code code;
13972 643674 : tree arg0, arg1, type;
13973 643674 : machine_mode operand_mode;
13974 643674 : int unsignedp;
13975 643674 : rtx op0, op1;
13976 643674 : rtx subtarget = target;
13977 643674 : location_t loc = ops->location;
13978 643674 : unsigned HOST_WIDE_INT nunits;
13979 :
13980 643674 : arg0 = ops->op0;
13981 643674 : arg1 = ops->op1;
13982 :
13983 : /* Don't crash if the comparison was erroneous. */
13984 643674 : if (arg0 == error_mark_node || arg1 == error_mark_node)
13985 0 : return const0_rtx;
13986 :
13987 643674 : type = TREE_TYPE (arg0);
13988 643674 : operand_mode = TYPE_MODE (type);
13989 643674 : unsignedp = TYPE_UNSIGNED (type);
13990 :
13991 : /* We won't bother with BLKmode store-flag operations because it would mean
13992 : passing a lot of information to emit_store_flag. */
13993 643674 : if (operand_mode == BLKmode)
13994 : return 0;
13995 :
13996 : /* We won't bother with store-flag operations involving function pointers
13997 : when function pointers must be canonicalized before comparisons. */
13998 643674 : if (targetm.have_canonicalize_funcptr_for_compare ()
13999 643674 : && ((POINTER_TYPE_P (TREE_TYPE (arg0))
14000 0 : && FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (arg0))))
14001 0 : || (POINTER_TYPE_P (TREE_TYPE (arg1))
14002 0 : && FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (arg1))))))
14003 : return 0;
14004 :
14005 643674 : STRIP_NOPS (arg0);
14006 643674 : STRIP_NOPS (arg1);
14007 :
14008 : /* For vector typed comparisons emit code to generate the desired
14009 : all-ones or all-zeros mask. */
14010 643674 : if (VECTOR_TYPE_P (ops->type))
14011 : {
14012 26175 : tree ifexp = build2 (ops->code, ops->type, arg0, arg1);
14013 26175 : if (VECTOR_BOOLEAN_TYPE_P (ops->type)
14014 52350 : && expand_vec_cmp_expr_p (TREE_TYPE (arg0), ops->type, ops->code))
14015 26175 : return expand_vec_cmp_expr (ops->type, ifexp, target);
14016 : else
14017 0 : gcc_unreachable ();
14018 : }
14019 :
14020 : /* Optimize (x % C1) == C2 or (x % C1) != C2 if it is beneficial
14021 : into (x - C2) * C3 < C4. */
14022 617499 : if ((ops->code == EQ_EXPR || ops->code == NE_EXPR)
14023 415109 : && TREE_CODE (arg0) == SSA_NAME
14024 414534 : && TREE_CODE (arg1) == INTEGER_CST)
14025 : {
14026 248635 : enum tree_code new_code = maybe_optimize_mod_cmp (ops->code,
14027 : &arg0, &arg1);
14028 248635 : if (new_code != ops->code)
14029 : {
14030 107 : struct separate_ops nops = *ops;
14031 107 : nops.code = new_code;
14032 107 : nops.op0 = arg0;
14033 107 : nops.op1 = arg1;
14034 107 : nops.type = TREE_TYPE (arg0);
14035 107 : return do_store_flag (&nops, target, mode);
14036 : }
14037 : }
14038 :
14039 : /* Optimize (x - y) < 0 into x < y if x - y has undefined overflow. */
14040 617392 : if (!unsignedp
14041 442905 : && (ops->code == LT_EXPR || ops->code == LE_EXPR
14042 442905 : || ops->code == GT_EXPR || ops->code == GE_EXPR)
14043 127846 : && integer_zerop (arg1)
14044 655994 : && TREE_CODE (arg0) == SSA_NAME)
14045 38598 : maybe_optimize_sub_cmp_0 (ops->code, &arg0, &arg1);
14046 :
14047 : /* Get the rtx comparison code to use. We know that EXP is a comparison
14048 : operation of some type. Some comparisons against 1 and -1 can be
14049 : converted to comparisons with zero. Do so here so that the tests
14050 : below will be aware that we have a comparison with zero. These
14051 : tests will not catch constants in the first operand, but constants
14052 : are rarely passed as the first operand. */
14053 :
14054 617392 : switch (ops->code)
14055 : {
14056 : case EQ_EXPR:
14057 : code = EQ;
14058 : break;
14059 235582 : case NE_EXPR:
14060 235582 : code = NE;
14061 235582 : break;
14062 42307 : case LT_EXPR:
14063 42307 : if (integer_onep (arg1))
14064 0 : arg1 = integer_zero_node, code = unsignedp ? LEU : LE;
14065 : else
14066 42307 : code = unsignedp ? LTU : LT;
14067 : break;
14068 33570 : case LE_EXPR:
14069 33570 : if (! unsignedp && integer_all_onesp (arg1))
14070 0 : arg1 = integer_zero_node, code = LT;
14071 : else
14072 33570 : code = unsignedp ? LEU : LE;
14073 : break;
14074 54257 : case GT_EXPR:
14075 54257 : if (! unsignedp && integer_all_onesp (arg1))
14076 0 : arg1 = integer_zero_node, code = GE;
14077 : else
14078 54257 : code = unsignedp ? GTU : GT;
14079 : break;
14080 47684 : case GE_EXPR:
14081 47684 : if (integer_onep (arg1))
14082 0 : arg1 = integer_zero_node, code = unsignedp ? GTU : GT;
14083 : else
14084 47684 : code = unsignedp ? GEU : GE;
14085 : break;
14086 :
14087 1030 : case UNORDERED_EXPR:
14088 1030 : code = UNORDERED;
14089 1030 : break;
14090 720 : case ORDERED_EXPR:
14091 720 : code = ORDERED;
14092 720 : break;
14093 535 : case UNLT_EXPR:
14094 535 : code = UNLT;
14095 535 : break;
14096 11128 : case UNLE_EXPR:
14097 11128 : code = UNLE;
14098 11128 : break;
14099 1002 : case UNGT_EXPR:
14100 1002 : code = UNGT;
14101 1002 : break;
14102 9936 : case UNGE_EXPR:
14103 9936 : code = UNGE;
14104 9936 : break;
14105 135 : case UNEQ_EXPR:
14106 135 : code = UNEQ;
14107 135 : break;
14108 86 : case LTGT_EXPR:
14109 86 : code = LTGT;
14110 86 : break;
14111 :
14112 0 : default:
14113 0 : gcc_unreachable ();
14114 : }
14115 :
14116 : /* Put a constant second. */
14117 617392 : if (TREE_CODE (arg0) == REAL_CST || TREE_CODE (arg0) == INTEGER_CST
14118 615422 : || TREE_CODE (arg0) == FIXED_CST)
14119 : {
14120 1970 : std::swap (arg0, arg1);
14121 1970 : code = swap_condition (code);
14122 : }
14123 :
14124 : /* If this is an equality or inequality test of a single bit, we can
14125 : do this by shifting the bit being tested to the low-order bit and
14126 : masking the result with the constant 1. If the condition was EQ,
14127 : we xor it with 1. This does not require an scc insn and is faster
14128 : than an scc insn even if we have it. */
14129 :
14130 617392 : if ((code == NE || code == EQ)
14131 415002 : && (integer_zerop (arg1)
14132 259923 : || integer_pow2p (arg1))
14133 : /* vector types are not handled here. */
14134 188424 : && TREE_CODE (TREE_TYPE (arg1)) != VECTOR_TYPE
14135 805776 : && (TYPE_PRECISION (ops->type) != 1 || TYPE_UNSIGNED (ops->type)))
14136 : {
14137 188384 : tree narg0 = arg0;
14138 188384 : wide_int nz = tree_nonzero_bits (narg0);
14139 188384 : gimple *srcstmt = get_def_for_expr (narg0, BIT_AND_EXPR);
14140 : /* If the defining statement was (x & POW2), then use that instead of
14141 : the non-zero bits. */
14142 188384 : if (srcstmt && integer_pow2p (gimple_assign_rhs2 (srcstmt)))
14143 : {
14144 5479 : nz = wi::to_wide (gimple_assign_rhs2 (srcstmt));
14145 5479 : narg0 = gimple_assign_rhs1 (srcstmt);
14146 : }
14147 :
14148 188384 : if (wi::popcount (nz) == 1
14149 188384 : && (integer_zerop (arg1)
14150 188384 : || wi::to_wide (arg1) == nz))
14151 : {
14152 9846 : int bitnum = wi::exact_log2 (nz);
14153 9846 : enum tree_code tcode = EQ_EXPR;
14154 9846 : if ((code == NE) ^ !integer_zerop (arg1))
14155 7114 : tcode = NE_EXPR;
14156 :
14157 9846 : type = lang_hooks.types.type_for_mode (mode, unsignedp);
14158 9846 : return expand_single_bit_test (loc, tcode,
14159 : narg0,
14160 : bitnum, type, target, mode);
14161 : }
14162 188384 : }
14163 :
14164 :
14165 607546 : if (! get_subtarget (target)
14166 607546 : || GET_MODE (subtarget) != operand_mode)
14167 : subtarget = 0;
14168 :
14169 607546 : expand_operands (arg0, arg1, subtarget, &op0, &op1, EXPAND_NORMAL);
14170 :
14171 : /* For boolean vectors with less than mode precision
14172 : make sure to fill padding with consistent values. */
14173 40 : if (VECTOR_BOOLEAN_TYPE_P (type)
14174 27 : && SCALAR_INT_MODE_P (operand_mode)
14175 607546 : && TYPE_VECTOR_SUBPARTS (type).is_constant (&nunits)
14176 607546 : && maybe_ne (GET_MODE_PRECISION (operand_mode), nunits))
14177 : {
14178 0 : gcc_assert (code == EQ || code == NE);
14179 0 : op0 = expand_binop (mode, and_optab, op0,
14180 0 : GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
14181 : NULL_RTX, true, OPTAB_WIDEN);
14182 0 : op1 = expand_binop (mode, and_optab, op1,
14183 : GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
14184 : NULL_RTX, true, OPTAB_WIDEN);
14185 : }
14186 :
14187 607546 : if (target == 0)
14188 415319 : target = gen_reg_rtx (mode);
14189 :
14190 : /* Try a cstore if possible. */
14191 607546 : return emit_store_flag_force (target, code, op0, op1,
14192 : operand_mode, unsignedp,
14193 607546 : (TYPE_PRECISION (ops->type) == 1
14194 1214420 : && !TYPE_UNSIGNED (ops->type)) ? -1 : 1);
14195 : }
14196 :
14197 : /* Attempt to generate a casesi instruction. Returns true if successful,
14198 : false otherwise (i.e. if there is no casesi instruction).
14199 :
14200 : DEFAULT_PROBABILITY is the probability of jumping to the default
14201 : label. */
14202 : bool
14203 7077 : try_casesi (tree index_type, tree index_expr, tree minval, tree range,
14204 : rtx table_label, rtx default_label, rtx fallback_label,
14205 : profile_probability default_probability)
14206 : {
14207 7077 : class expand_operand ops[5];
14208 7077 : scalar_int_mode index_mode = SImode;
14209 7077 : rtx op1, op2, index;
14210 :
14211 7077 : if (! targetm.have_casesi ())
14212 : return false;
14213 :
14214 : /* The index must be some form of integer. Convert it to SImode. */
14215 0 : scalar_int_mode omode = SCALAR_INT_TYPE_MODE (index_type);
14216 0 : if (GET_MODE_BITSIZE (omode) > GET_MODE_BITSIZE (index_mode))
14217 : {
14218 0 : rtx rangertx = expand_normal (range);
14219 :
14220 : /* We must handle the endpoints in the original mode. */
14221 0 : index_expr = build2 (MINUS_EXPR, index_type,
14222 : index_expr, minval);
14223 0 : minval = integer_zero_node;
14224 0 : index = expand_normal (index_expr);
14225 0 : if (default_label)
14226 0 : emit_cmp_and_jump_insns (rangertx, index, LTU, NULL_RTX,
14227 : omode, 1, default_label,
14228 : default_probability);
14229 : /* Now we can safely truncate. */
14230 0 : index = convert_to_mode (index_mode, index, 0);
14231 : }
14232 : else
14233 : {
14234 0 : if (omode != index_mode)
14235 : {
14236 0 : index_type = lang_hooks.types.type_for_mode (index_mode, 0);
14237 0 : index_expr = fold_convert (index_type, index_expr);
14238 : }
14239 :
14240 0 : index = expand_normal (index_expr);
14241 : }
14242 :
14243 0 : do_pending_stack_adjust ();
14244 :
14245 0 : op1 = expand_normal (minval);
14246 0 : op2 = expand_normal (range);
14247 :
14248 0 : create_input_operand (&ops[0], index, index_mode);
14249 0 : create_convert_operand_from_type (&ops[1], op1, TREE_TYPE (minval));
14250 0 : create_convert_operand_from_type (&ops[2], op2, TREE_TYPE (range));
14251 0 : create_fixed_operand (&ops[3], table_label);
14252 0 : create_fixed_operand (&ops[4], (default_label
14253 : ? default_label
14254 : : fallback_label));
14255 0 : expand_jump_insn (targetm.code_for_casesi, 5, ops);
14256 0 : return true;
14257 : }
14258 :
14259 : /* Attempt to generate a tablejump instruction; same concept. */
14260 : /* Subroutine of the next function.
14261 :
14262 : INDEX is the value being switched on, with the lowest value
14263 : in the table already subtracted.
14264 : MODE is its expected mode (needed if INDEX is constant).
14265 : RANGE is the length of the jump table.
14266 : TABLE_LABEL is a CODE_LABEL rtx for the table itself.
14267 :
14268 : DEFAULT_LABEL is a CODE_LABEL rtx to jump to if the
14269 : index value is out of range.
14270 : DEFAULT_PROBABILITY is the probability of jumping to
14271 : the default label. */
14272 :
14273 : static void
14274 7077 : do_tablejump (rtx index, machine_mode mode, rtx range, rtx table_label,
14275 : rtx default_label, profile_probability default_probability)
14276 : {
14277 7077 : rtx temp, vector;
14278 :
14279 7077 : if (INTVAL (range) > cfun->cfg->max_jumptable_ents)
14280 5990 : cfun->cfg->max_jumptable_ents = INTVAL (range);
14281 :
14282 : /* Do an unsigned comparison (in the proper mode) between the index
14283 : expression and the value which represents the length of the range.
14284 : Since we just finished subtracting the lower bound of the range
14285 : from the index expression, this comparison allows us to simultaneously
14286 : check that the original index expression value is both greater than
14287 : or equal to the minimum value of the range and less than or equal to
14288 : the maximum value of the range. */
14289 :
14290 7077 : if (default_label)
14291 7031 : emit_cmp_and_jump_insns (index, range, GTU, NULL_RTX, mode, 1,
14292 : default_label, default_probability);
14293 :
14294 : /* If index is in range, it must fit in Pmode.
14295 : Convert to Pmode so we can index with it. */
14296 8144 : if (mode != Pmode)
14297 : {
14298 6074 : unsigned int width;
14299 :
14300 : /* We know the value of INDEX is between 0 and RANGE. If we have a
14301 : sign-extended subreg, and RANGE does not have the sign bit set, then
14302 : we have a value that is valid for both sign and zero extension. In
14303 : this case, we get better code if we sign extend. */
14304 6074 : if (GET_CODE (index) == SUBREG
14305 9 : && SUBREG_PROMOTED_VAR_P (index)
14306 0 : && SUBREG_PROMOTED_SIGNED_P (index)
14307 0 : && ((width = GET_MODE_PRECISION (as_a <scalar_int_mode> (mode)))
14308 : <= HOST_BITS_PER_WIDE_INT)
14309 6074 : && ! (UINTVAL (range) & (HOST_WIDE_INT_1U << (width - 1))))
14310 0 : index = convert_to_mode (Pmode, index, 0);
14311 : else
14312 6074 : index = convert_to_mode (Pmode, index, 1);
14313 : }
14314 :
14315 : /* Don't let a MEM slip through, because then INDEX that comes
14316 : out of PIC_CASE_VECTOR_ADDRESS won't be a valid address,
14317 : and break_out_memory_refs will go to work on it and mess it up. */
14318 : #ifdef PIC_CASE_VECTOR_ADDRESS
14319 : if (flag_pic && !REG_P (index))
14320 : index = copy_to_mode_reg (Pmode, index);
14321 : #endif
14322 :
14323 : /* ??? The only correct use of CASE_VECTOR_MODE is the one inside the
14324 : GET_MODE_SIZE, because this indicates how large insns are. The other
14325 : uses should all be Pmode, because they are addresses. This code
14326 : could fail if addresses and insns are not the same size. */
14327 8144 : index = simplify_gen_binary (MULT, Pmode, index,
14328 14154 : gen_int_mode (GET_MODE_SIZE (CASE_VECTOR_MODE),
14329 7077 : Pmode));
14330 8144 : index = simplify_gen_binary (PLUS, Pmode, index,
14331 7077 : gen_rtx_LABEL_REF (Pmode, table_label));
14332 :
14333 : #ifdef PIC_CASE_VECTOR_ADDRESS
14334 : if (flag_pic)
14335 : index = PIC_CASE_VECTOR_ADDRESS (index);
14336 : else
14337 : #endif
14338 8756 : index = memory_address (CASE_VECTOR_MODE, index);
14339 8756 : temp = gen_reg_rtx (CASE_VECTOR_MODE);
14340 8756 : vector = gen_const_mem (CASE_VECTOR_MODE, index);
14341 7077 : convert_move (temp, vector, 0);
14342 :
14343 7077 : emit_jump_insn (targetm.gen_tablejump (temp, table_label));
14344 :
14345 : /* If we are generating PIC code or if the table is PC-relative, the
14346 : table and JUMP_INSN must be adjacent, so don't output a BARRIER. */
14347 7077 : if (! CASE_VECTOR_PC_RELATIVE && ! flag_pic)
14348 5900 : emit_barrier ();
14349 7077 : }
14350 :
14351 : bool
14352 7077 : try_tablejump (tree index_type, tree index_expr, tree minval, tree range,
14353 : rtx table_label, rtx default_label,
14354 : profile_probability default_probability)
14355 : {
14356 7077 : rtx index;
14357 :
14358 7077 : if (! targetm.have_tablejump ())
14359 : return false;
14360 :
14361 7077 : index_expr = fold_build2 (MINUS_EXPR, index_type,
14362 : fold_convert (index_type, index_expr),
14363 : fold_convert (index_type, minval));
14364 7077 : index = expand_normal (index_expr);
14365 7077 : do_pending_stack_adjust ();
14366 :
14367 7077 : do_tablejump (index, TYPE_MODE (index_type),
14368 7077 : convert_modes (TYPE_MODE (index_type),
14369 7077 : TYPE_MODE (TREE_TYPE (range)),
14370 : expand_normal (range),
14371 7077 : TYPE_UNSIGNED (TREE_TYPE (range))),
14372 : table_label, default_label, default_probability);
14373 7077 : return true;
14374 : }
14375 :
14376 : /* Return a CONST_VECTOR rtx representing vector mask for
14377 : a VECTOR_CST of booleans. */
14378 : static rtx
14379 684 : const_vector_mask_from_tree (tree exp)
14380 : {
14381 684 : machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
14382 684 : machine_mode inner = GET_MODE_INNER (mode);
14383 :
14384 684 : rtx_vector_builder builder (mode, VECTOR_CST_NPATTERNS (exp),
14385 684 : VECTOR_CST_NELTS_PER_PATTERN (exp));
14386 684 : unsigned int count = builder.encoded_nelts ();
14387 3680 : for (unsigned int i = 0; i < count; ++i)
14388 : {
14389 2996 : tree elt = VECTOR_CST_ELT (exp, i);
14390 2996 : gcc_assert (TREE_CODE (elt) == INTEGER_CST);
14391 2996 : if (integer_zerop (elt))
14392 1177 : builder.quick_push (CONST0_RTX (inner));
14393 1819 : else if (integer_onep (elt)
14394 1819 : || integer_minus_onep (elt))
14395 1819 : builder.quick_push (CONSTM1_RTX (inner));
14396 : else
14397 0 : gcc_unreachable ();
14398 : }
14399 684 : return builder.build ();
14400 684 : }
14401 :
14402 : /* Return a CONST_VECTOR rtx for a VECTOR_CST tree. */
14403 : static rtx
14404 570469 : const_vector_from_tree (tree exp)
14405 : {
14406 570469 : machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
14407 :
14408 570469 : if (initializer_zerop (exp))
14409 170103 : return CONST0_RTX (mode);
14410 :
14411 400366 : if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (exp)))
14412 684 : return const_vector_mask_from_tree (exp);
14413 :
14414 399682 : machine_mode inner = GET_MODE_INNER (mode);
14415 :
14416 399682 : rtx_vector_builder builder (mode, VECTOR_CST_NPATTERNS (exp),
14417 399682 : VECTOR_CST_NELTS_PER_PATTERN (exp));
14418 399682 : unsigned int count = builder.encoded_nelts ();
14419 1288674 : for (unsigned int i = 0; i < count; ++i)
14420 : {
14421 888992 : tree elt = VECTOR_CST_ELT (exp, i);
14422 888992 : if (TREE_CODE (elt) == REAL_CST)
14423 135726 : builder.quick_push (const_double_from_real_value (TREE_REAL_CST (elt),
14424 : inner));
14425 753266 : else if (TREE_CODE (elt) == FIXED_CST)
14426 0 : builder.quick_push (CONST_FIXED_FROM_FIXED_VALUE (TREE_FIXED_CST (elt),
14427 : inner));
14428 : else
14429 753266 : builder.quick_push (immed_wide_int_const (wi::to_poly_wide (elt),
14430 : inner));
14431 : }
14432 399682 : return builder.build ();
14433 399682 : }
14434 :
14435 : /* Build a decl for a personality function given a language prefix. */
14436 :
14437 : tree
14438 33085 : build_personality_function (const char *lang)
14439 : {
14440 33085 : const char *unwind_and_version;
14441 33085 : tree decl, type;
14442 33085 : char *name;
14443 :
14444 33085 : switch (targetm_common.except_unwind_info (&global_options))
14445 : {
14446 : case UI_NONE:
14447 : return NULL;
14448 : case UI_SJLJ:
14449 : unwind_and_version = "_sj0";
14450 : break;
14451 33085 : case UI_DWARF2:
14452 33085 : case UI_TARGET:
14453 33085 : unwind_and_version = "_v0";
14454 33085 : break;
14455 0 : case UI_SEH:
14456 0 : unwind_and_version = "_seh0";
14457 0 : break;
14458 0 : default:
14459 0 : gcc_unreachable ();
14460 : }
14461 :
14462 33085 : name = ACONCAT (("__", lang, "_personality", unwind_and_version, NULL));
14463 :
14464 33085 : type = build_function_type_list (unsigned_type_node,
14465 : integer_type_node, integer_type_node,
14466 : long_long_unsigned_type_node,
14467 : ptr_type_node, ptr_type_node, NULL_TREE);
14468 33085 : decl = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL,
14469 : get_identifier (name), type);
14470 33085 : DECL_ARTIFICIAL (decl) = 1;
14471 33085 : DECL_EXTERNAL (decl) = 1;
14472 33085 : TREE_PUBLIC (decl) = 1;
14473 :
14474 : /* Zap the nonsensical SYMBOL_REF_DECL for this. What we're left with
14475 : are the flags assigned by targetm.encode_section_info. */
14476 33085 : SET_SYMBOL_REF_DECL (XEXP (DECL_RTL (decl), 0), NULL);
14477 :
14478 33085 : return decl;
14479 : }
14480 :
14481 : /* Extracts the personality function of DECL and returns the corresponding
14482 : libfunc. */
14483 :
14484 : rtx
14485 1665407 : get_personality_function (tree decl)
14486 : {
14487 1665407 : tree personality = DECL_FUNCTION_PERSONALITY (decl);
14488 1665407 : enum eh_personality_kind pk;
14489 :
14490 1665407 : pk = function_needs_eh_personality (DECL_STRUCT_FUNCTION (decl));
14491 1665407 : if (pk == eh_personality_none)
14492 : return NULL;
14493 :
14494 158181 : if (!personality
14495 158181 : && pk == eh_personality_any)
14496 72334 : personality = lang_hooks.eh_personality ();
14497 :
14498 158181 : if (pk == eh_personality_lang)
14499 85847 : gcc_assert (personality != NULL_TREE);
14500 :
14501 158181 : return XEXP (DECL_RTL (personality), 0);
14502 : }
14503 :
14504 : /* Returns a tree for the size of EXP in bytes. */
14505 :
14506 : static tree
14507 14947391 : tree_expr_size (const_tree exp)
14508 : {
14509 14947391 : if (DECL_P (exp)
14510 14947391 : && DECL_SIZE_UNIT (exp) != 0)
14511 1953581 : return DECL_SIZE_UNIT (exp);
14512 : else
14513 12993810 : return size_in_bytes (TREE_TYPE (exp));
14514 : }
14515 :
14516 : /* Return an rtx for the size in bytes of the value of EXP. */
14517 :
14518 : rtx
14519 14754876 : expr_size (tree exp)
14520 : {
14521 14754876 : tree size;
14522 :
14523 14754876 : if (TREE_CODE (exp) == WITH_SIZE_EXPR)
14524 773 : size = TREE_OPERAND (exp, 1);
14525 : else
14526 : {
14527 14754103 : size = tree_expr_size (exp);
14528 14754103 : gcc_assert (size);
14529 14754103 : gcc_assert (size == SUBSTITUTE_PLACEHOLDER_IN_EXPR (size, exp));
14530 : }
14531 :
14532 14754876 : return expand_expr (size, NULL_RTX, TYPE_MODE (sizetype), EXPAND_NORMAL);
14533 : }
14534 :
14535 : /* Return a wide integer for the size in bytes of the value of EXP, or -1
14536 : if the size can vary or is larger than an integer. */
14537 :
14538 : HOST_WIDE_INT
14539 193288 : int_expr_size (const_tree exp)
14540 : {
14541 193288 : tree size;
14542 :
14543 193288 : if (TREE_CODE (exp) == WITH_SIZE_EXPR)
14544 0 : size = TREE_OPERAND (exp, 1);
14545 : else
14546 : {
14547 193288 : size = tree_expr_size (exp);
14548 193288 : gcc_assert (size);
14549 : }
14550 :
14551 193288 : if (size == 0 || !tree_fits_shwi_p (size))
14552 : return -1;
14553 :
14554 193288 : return tree_to_shwi (size);
14555 : }
14556 :
14557 : /* Return the quotient of polynomial long division of x^2N by POLYNOMIAL
14558 : in GF (2^N).
14559 : Author: Richard Sandiford <richard.sandiford@arm.com> */
14560 :
14561 : unsigned HOST_WIDE_INT
14562 0 : gf2n_poly_long_div_quotient (unsigned HOST_WIDE_INT polynomial,
14563 : unsigned short n)
14564 : {
14565 : /* The result has degree N, so needs N + 1 bits. */
14566 0 : gcc_assert (n < 64);
14567 :
14568 : /* Perform a division step for the x^2N coefficient. At this point the
14569 : quotient and remainder have N implicit trailing zeros. */
14570 : unsigned HOST_WIDE_INT quotient = 1;
14571 : unsigned HOST_WIDE_INT remainder = polynomial;
14572 :
14573 : /* Process the coefficients for x^(2N-1) down to x^N, with each step
14574 : reducing the number of implicit trailing zeros by one. */
14575 0 : for (unsigned int i = 0; i < n; ++i)
14576 : {
14577 0 : bool coeff = remainder & (HOST_WIDE_INT_1U << (n - 1));
14578 0 : quotient = (quotient << 1) | coeff;
14579 0 : remainder = (remainder << 1) ^ (coeff ? polynomial : 0);
14580 : }
14581 0 : return quotient;
14582 : }
14583 :
14584 : /* Calculate CRC for the initial CRC and given POLYNOMIAL.
14585 : CRC_BITS is CRC size. */
14586 :
14587 : static unsigned HOST_WIDE_INT
14588 34560 : calculate_crc (unsigned HOST_WIDE_INT crc,
14589 : unsigned HOST_WIDE_INT polynomial,
14590 : unsigned short crc_bits)
14591 : {
14592 34560 : unsigned HOST_WIDE_INT msb = HOST_WIDE_INT_1U << (crc_bits - 1);
14593 34560 : crc = crc << (crc_bits - 8);
14594 311040 : for (short i = 8; i > 0; --i)
14595 : {
14596 276480 : if (crc & msb)
14597 138240 : crc = (crc << 1) ^ polynomial;
14598 : else
14599 138240 : crc <<= 1;
14600 : }
14601 : /* Zero out bits in crc beyond the specified number of crc_bits. */
14602 34560 : if (crc_bits < sizeof (crc) * CHAR_BIT)
14603 31488 : crc &= (HOST_WIDE_INT_1U << crc_bits) - 1;
14604 34560 : return crc;
14605 : }
14606 :
14607 : /* Assemble CRC table with 256 elements for the given POLYNOM and CRC_BITS.
14608 : POLYNOM is the polynomial used to calculate the CRC table's elements.
14609 : CRC_BITS is the size of CRC, may be 8, 16, ... . */
14610 :
14611 : static rtx
14612 135 : assemble_crc_table (unsigned HOST_WIDE_INT polynom, unsigned short crc_bits)
14613 : {
14614 135 : unsigned table_el_n = 0x100;
14615 135 : tree ar = build_array_type (make_unsigned_type (crc_bits),
14616 135 : build_index_type (size_int (table_el_n - 1)));
14617 :
14618 : /* Initialize the table. */
14619 135 : vec<tree, va_gc> *initial_values;
14620 135 : vec_alloc (initial_values, table_el_n);
14621 34830 : for (size_t i = 0; i < table_el_n; ++i)
14622 : {
14623 34560 : unsigned HOST_WIDE_INT crc = calculate_crc (i, polynom, crc_bits);
14624 34560 : tree element = build_int_cstu (make_unsigned_type (crc_bits), crc);
14625 34560 : vec_safe_push (initial_values, element);
14626 : }
14627 135 : tree ctor = build_constructor_from_vec (ar, initial_values);
14628 135 : rtx mem = output_constant_def (ctor, 1);
14629 135 : gcc_assert (MEM_P (mem));
14630 135 : if (dump_file && (dump_flags & TDF_DETAILS))
14631 : {
14632 16 : fprintf (dump_file,
14633 : ";; emitting crc table crc_%u_polynomial_"
14634 : HOST_WIDE_INT_PRINT_HEX " ",
14635 : crc_bits, polynom);
14636 16 : print_rtl_single (dump_file, XEXP (mem, 0));
14637 16 : fprintf (dump_file, "\n");
14638 : }
14639 :
14640 135 : return XEXP (mem, 0);
14641 : }
14642 :
14643 : /* Generate CRC lookup table by calculating CRC for all possible
14644 : 8-bit data values. The table is stored with a specific name in the read-only
14645 : static data section.
14646 : POLYNOM is the polynomial used to calculate the CRC table's elements.
14647 : CRC_BITS is the size of CRC, may be 8, 16, ... . */
14648 :
14649 : static rtx
14650 135 : generate_crc_table (unsigned HOST_WIDE_INT polynom, unsigned short crc_bits)
14651 : {
14652 135 : gcc_assert (crc_bits <= 64);
14653 :
14654 135 : return assemble_crc_table (polynom, crc_bits);
14655 : }
14656 :
14657 : /* Calculate CRC for the initial CRC and given POLYNOMIAL.
14658 : CRC_BITS is CRC size. */
14659 :
14660 : static unsigned HOST_WIDE_INT
14661 35328 : calculate_reversed_crc (unsigned HOST_WIDE_INT crc,
14662 : unsigned HOST_WIDE_INT polynomial,
14663 : unsigned short crc_bits)
14664 : {
14665 35328 : unsigned HOST_WIDE_INT rev_polynom = reflect_hwi (polynomial, crc_bits);
14666 353280 : for (int j = 0; j < 8; j++)
14667 : {
14668 282624 : if (crc & 1)
14669 141312 : crc = (crc >> 1) ^ rev_polynom;
14670 : else
14671 141312 : crc >>= 1;
14672 : }
14673 : /* Zero out bits in crc beyond the specified number of crc_bits. */
14674 35328 : if (crc_bits < sizeof (crc) * CHAR_BIT)
14675 29952 : crc &= (HOST_WIDE_INT_1U << crc_bits) - 1;
14676 35328 : return crc;
14677 : }
14678 :
14679 : /* Assemble CRC table with 256 elements for the given POLYNOM and CRC_BITS.
14680 : POLYNOM is the polynomial used to calculate the CRC table's elements.
14681 : CRC_BITS is the size of CRC, may be 8, 16, ... . */
14682 :
14683 : static rtx
14684 138 : assemble_reversed_crc_table (unsigned HOST_WIDE_INT polynom, unsigned short crc_bits)
14685 : {
14686 138 : unsigned table_el_n = 0x100;
14687 138 : tree ar = build_array_type (make_unsigned_type (crc_bits),
14688 138 : build_index_type (size_int (table_el_n - 1)));
14689 :
14690 : /* Initialize the table. */
14691 138 : vec<tree, va_gc> *initial_values;
14692 138 : vec_alloc (initial_values, table_el_n);
14693 35604 : for (size_t i = 0; i < table_el_n; ++i)
14694 : {
14695 35328 : unsigned HOST_WIDE_INT crc = calculate_reversed_crc (i, polynom, crc_bits);
14696 35328 : tree element = build_int_cstu (make_unsigned_type (crc_bits), crc);
14697 35328 : vec_safe_push (initial_values, element);
14698 : }
14699 138 : tree ctor = build_constructor_from_vec (ar, initial_values);
14700 138 : rtx mem = output_constant_def (ctor, 1);
14701 138 : gcc_assert (MEM_P (mem));
14702 138 : if (dump_file && (dump_flags & TDF_DETAILS))
14703 : {
14704 16 : fprintf (dump_file,
14705 : ";; emitting reversed crc table crc_%u_polynomial_"
14706 : HOST_WIDE_INT_PRINT_HEX " ",
14707 : crc_bits, polynom);
14708 16 : print_rtl_single (dump_file, XEXP (mem, 0));
14709 16 : fprintf (dump_file, "\n");
14710 : }
14711 :
14712 138 : return XEXP (mem, 0);
14713 : }
14714 :
14715 : /* Generate reversed CRC table for the given POLYNOM and CRC_BITS. */
14716 :
14717 : static rtx
14718 138 : generate_reversed_crc_table (unsigned HOST_WIDE_INT polynom,
14719 : unsigned short crc_bits)
14720 : {
14721 138 : gcc_assert (crc_bits <= 64);
14722 :
14723 138 : return assemble_reversed_crc_table (polynom, crc_bits);
14724 : }
14725 :
14726 : /* Generate table-based CRC code for the given CRC, INPUT_DATA and the
14727 : POLYNOMIAL (without leading 1).
14728 :
14729 : First, using POLYNOMIAL's value generates CRC table of 256 elements,
14730 : then generates the assembly for the following code,
14731 : where crc_bit_size and data_bit_size may be 8, 16, 32, 64, depending on CRC:
14732 :
14733 : for (int i = 0; i < data_bit_size / 8; i++)
14734 : crc = (crc << 8) ^ crc_table[(crc >> (crc_bit_size - 8))
14735 : ^ (data >> (data_bit_size - (i + 1) * 8)
14736 : & 0xFF))];
14737 :
14738 : So to take values from the table, we need 8-bit data.
14739 : If input data size is not 8, then first we extract upper 8 bits,
14740 : then the other 8 bits, and so on. */
14741 :
14742 : static void
14743 135 : calculate_table_based_CRC (rtx *crc, const rtx &input_data,
14744 : const rtx &polynomial,
14745 : machine_mode data_mode)
14746 : {
14747 135 : machine_mode mode = GET_MODE (*crc);
14748 135 : unsigned short crc_bit_size = GET_MODE_BITSIZE (mode).to_constant ();
14749 135 : unsigned short data_size = GET_MODE_SIZE (data_mode).to_constant ();
14750 135 : rtx tab = generate_crc_table (UINTVAL (polynomial), crc_bit_size);
14751 :
14752 529 : for (unsigned short i = 0; i < data_size; i++)
14753 : {
14754 : /* crc >> (crc_bit_size - 8). */
14755 259 : *crc = force_reg (mode, *crc);
14756 259 : rtx op1 = expand_shift (RSHIFT_EXPR, mode, *crc, crc_bit_size - 8,
14757 : NULL_RTX, 1);
14758 :
14759 : /* data >> (8 * (GET_MODE_SIZE (data_mode).to_constant () - i - 1)). */
14760 259 : unsigned range_8 = 8 * (data_size - i - 1);
14761 : /* CRC's mode is always at least as wide as INPUT_DATA. Convert
14762 : INPUT_DATA into CRC's mode. */
14763 259 : rtx data = gen_reg_rtx (mode);
14764 259 : convert_move (data, input_data, 1);
14765 259 : data = expand_shift (RSHIFT_EXPR, mode, data, range_8, NULL_RTX, 1);
14766 :
14767 : /* data >> (8 * (GET_MODE_SIZE (mode)
14768 : .to_constant () - i - 1)) & 0xFF. */
14769 259 : rtx data_final = expand_and (mode, data,
14770 : gen_int_mode (255, mode), NULL_RTX);
14771 :
14772 : /* (crc >> (crc_bit_size - 8)) ^ data_8bit. */
14773 259 : rtx in = expand_binop (mode, xor_optab, op1, data_final,
14774 : NULL_RTX, 1, OPTAB_WIDEN);
14775 :
14776 : /* ((crc >> (crc_bit_size - 8)) ^ data_8bit) & 0xFF. */
14777 259 : rtx index = expand_and (mode, in, gen_int_mode (255, mode),
14778 : NULL_RTX);
14779 518 : int log_crc_size = exact_log2 (GET_MODE_SIZE (mode).to_constant ());
14780 259 : index = expand_shift (LSHIFT_EXPR, mode, index,
14781 259 : log_crc_size, NULL_RTX, 0);
14782 :
14783 264 : rtx addr = gen_reg_rtx (Pmode);
14784 259 : convert_move (addr, index, 1);
14785 264 : addr = expand_binop (Pmode, add_optab, addr, tab, NULL_RTX,
14786 : 0, OPTAB_DIRECT);
14787 :
14788 : /* crc_table[(crc >> (crc_bit_size - 8)) ^ data_8bit] */
14789 259 : rtx tab_el = validize_mem (gen_rtx_MEM (mode, addr));
14790 :
14791 : /* (crc << 8) if CRC is larger than 8, otherwise crc = 0. */
14792 259 : rtx high = NULL_RTX;
14793 259 : if (crc_bit_size != 8)
14794 229 : high = expand_shift (LSHIFT_EXPR, mode, *crc, 8, NULL_RTX, 0);
14795 : else
14796 30 : high = gen_int_mode (0, mode);
14797 :
14798 : /* crc = (crc << 8)
14799 : ^ crc_table[(crc >> (crc_bit_size - 8)) ^ data_8bit]; */
14800 259 : *crc = expand_binop (mode, xor_optab, tab_el, high, NULL_RTX, 1,
14801 : OPTAB_WIDEN);
14802 : }
14803 135 : }
14804 :
14805 : /* Generate table-based reversed CRC code for the given CRC, INPUT_DATA
14806 : and the POLYNOMIAL (without leading 1).
14807 :
14808 : This function generates code for reversed (bit-reflected) CRC calculation
14809 : using a pre-computed lookup table. Unlike the standard CRC calculation,
14810 : this processes data from LSB to MSB, eliminating the need for explicit
14811 : bit reflection before and after the CRC computation. */
14812 :
14813 : static void
14814 138 : calculate_table_based_reversed_CRC (rtx *crc, const rtx &input_data,
14815 : const rtx &polynomial,
14816 : machine_mode data_mode)
14817 : {
14818 138 : machine_mode mode = GET_MODE (*crc);
14819 138 : unsigned short crc_bit_size = GET_MODE_BITSIZE (mode).to_constant ();
14820 138 : unsigned short data_size = GET_MODE_SIZE (data_mode).to_constant ();
14821 138 : rtx tab = generate_reversed_crc_table (UINTVAL (polynomial), crc_bit_size);
14822 :
14823 : /* CRC's mode is always at least as wide as INPUT_DATA. Convert
14824 : INPUT_DATA into CRC's mode once outside the loop since INPUT_DATA
14825 : is loop-invariant. */
14826 138 : rtx data_in_crc_mode = gen_reg_rtx (mode);
14827 138 : convert_move (data_in_crc_mode, input_data, 1);
14828 :
14829 502 : for (unsigned short i = 0; i < data_size; i++)
14830 : {
14831 226 : *crc = force_reg (mode, *crc);
14832 :
14833 : /* data >> (8 * i). */
14834 226 : unsigned range_8 = 8 * i;
14835 226 : rtx data = expand_shift (RSHIFT_EXPR, mode, data_in_crc_mode,
14836 226 : range_8, NULL_RTX, 1);
14837 :
14838 : /* crc ^ (data >> (8 * i)). */
14839 226 : rtx in = expand_binop (mode, xor_optab, *crc, data,
14840 : NULL_RTX, 1, OPTAB_WIDEN);
14841 :
14842 : /* (crc ^ data) & 0xFF. */
14843 226 : rtx index = expand_and (mode, in, gen_int_mode (255, mode),
14844 : NULL_RTX);
14845 452 : int log_crc_size = exact_log2 (GET_MODE_SIZE (mode).to_constant ());
14846 226 : index = expand_shift (LSHIFT_EXPR, mode, index,
14847 226 : log_crc_size, NULL_RTX, 0);
14848 :
14849 239 : rtx addr = gen_reg_rtx (Pmode);
14850 226 : convert_move (addr, index, 1);
14851 239 : addr = expand_binop (Pmode, add_optab, addr, tab, NULL_RTX,
14852 : 0, OPTAB_DIRECT);
14853 :
14854 : /* crc_table[(crc ^ data) & 0xFF]. */
14855 226 : rtx tab_el = validize_mem (gen_rtx_MEM (mode, addr));
14856 :
14857 : /* (crc >> 8) if CRC is larger than 8, otherwise 0. */
14858 226 : rtx high = NULL_RTX;
14859 226 : if (crc_bit_size != 8)
14860 217 : high = expand_shift (RSHIFT_EXPR, mode, *crc, 8, NULL_RTX, 1);
14861 : else
14862 9 : high = gen_int_mode (0, mode);
14863 :
14864 : /* crc = (crc >> 8) ^ crc_table[(crc ^ data) & 0xFF]. */
14865 226 : *crc = expand_binop (mode, xor_optab, tab_el, high, NULL_RTX, 1,
14866 : OPTAB_WIDEN);
14867 : }
14868 138 : }
14869 :
14870 : /* Generate table-based CRC code for the given CRC, INPUT_DATA and the
14871 : POLYNOMIAL (without leading 1).
14872 :
14873 : CRC is OP1, data is OP2 and the polynomial is OP3.
14874 : This must generate a CRC table and an assembly for the following code,
14875 : where crc_bit_size and data_bit_size may be 8, 16, 32, 64:
14876 : uint_crc_bit_size_t
14877 : crc_crc_bit_size (uint_crc_bit_size_t crc_init,
14878 : uint_data_bit_size_t data, size_t size)
14879 : {
14880 : uint_crc_bit_size_t crc = crc_init;
14881 : for (int i = 0; i < data_bit_size / 8; i++)
14882 : crc = (crc << 8) ^ crc_table[(crc >> (crc_bit_size - 8))
14883 : ^ (data >> (data_bit_size - (i + 1) * 8)
14884 : & 0xFF))];
14885 : return crc;
14886 : } */
14887 :
14888 : void
14889 135 : expand_crc_table_based (rtx op0, rtx op1, rtx op2, rtx op3,
14890 : machine_mode data_mode)
14891 : {
14892 135 : gcc_assert (!CONST_INT_P (op0));
14893 135 : gcc_assert (CONST_INT_P (op3));
14894 135 : machine_mode crc_mode = GET_MODE (op0);
14895 135 : rtx crc = gen_reg_rtx (crc_mode);
14896 135 : convert_move (crc, op1, 0);
14897 135 : calculate_table_based_CRC (&crc, op2, op3, data_mode);
14898 135 : convert_move (op0, crc, 0);
14899 135 : }
14900 :
14901 : /* Generate the common operation for reflecting values:
14902 : *OP = (*OP & AND1_VALUE) << SHIFT_VAL | (*OP & AND2_VALUE) >> SHIFT_VAL; */
14903 :
14904 : void
14905 0 : gen_common_operation_to_reflect (rtx *op,
14906 : unsigned HOST_WIDE_INT and1_value,
14907 : unsigned HOST_WIDE_INT and2_value,
14908 : unsigned shift_val)
14909 : {
14910 0 : rtx op1 = expand_and (GET_MODE (*op), *op,
14911 0 : gen_int_mode (and1_value, GET_MODE (*op)), NULL_RTX);
14912 0 : op1 = expand_shift (LSHIFT_EXPR, GET_MODE (*op), op1, shift_val, op1, 0);
14913 0 : rtx op2 = expand_and (GET_MODE (*op), *op,
14914 0 : gen_int_mode (and2_value, GET_MODE (*op)), NULL_RTX);
14915 0 : op2 = expand_shift (RSHIFT_EXPR, GET_MODE (*op), op2, shift_val, op2, 1);
14916 0 : *op = expand_binop (GET_MODE (*op), ior_optab, op1,
14917 : op2, *op, 0, OPTAB_LIB_WIDEN);
14918 0 : }
14919 :
14920 : /* Reflect 64-bit value for the 64-bit target. */
14921 :
14922 : void
14923 0 : reflect_64_bit_value (rtx *op)
14924 : {
14925 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x00000000FFFFFFFF),
14926 : HOST_WIDE_INT_C (0xFFFFFFFF00000000), 32);
14927 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0000FFFF0000FFFF),
14928 : HOST_WIDE_INT_C (0xFFFF0000FFFF0000), 16);
14929 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x00FF00FF00FF00FF),
14930 : HOST_WIDE_INT_C (0xFF00FF00FF00FF00), 8);
14931 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0F0F0F0F0F0F0F0F),
14932 : HOST_WIDE_INT_C (0xF0F0F0F0F0F0F0F0), 4);
14933 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x3333333333333333),
14934 : HOST_WIDE_INT_C (0xCCCCCCCCCCCCCCCC), 2);
14935 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x5555555555555555),
14936 : HOST_WIDE_INT_C (0xAAAAAAAAAAAAAAAA), 1);
14937 0 : }
14938 :
14939 : /* Reflect 32-bit value for the 32-bit target. */
14940 :
14941 : void
14942 0 : reflect_32_bit_value (rtx *op)
14943 : {
14944 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0000FFFF),
14945 : HOST_WIDE_INT_C (0xFFFF0000), 16);
14946 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x00FF00FF),
14947 : HOST_WIDE_INT_C (0xFF00FF00), 8);
14948 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0F0F0F0F),
14949 : HOST_WIDE_INT_C (0xF0F0F0F0), 4);
14950 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x33333333),
14951 : HOST_WIDE_INT_C (0xCCCCCCCC), 2);
14952 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x55555555),
14953 : HOST_WIDE_INT_C (0xAAAAAAAA), 1);
14954 0 : }
14955 :
14956 : /* Reflect 16-bit value for the 16-bit target. */
14957 :
14958 : void
14959 0 : reflect_16_bit_value (rtx *op)
14960 : {
14961 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x00FF),
14962 : HOST_WIDE_INT_C (0xFF00), 8);
14963 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0F0F),
14964 : HOST_WIDE_INT_C (0xF0F0), 4);
14965 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x3333),
14966 : HOST_WIDE_INT_C (0xCCCC), 2);
14967 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x5555),
14968 : HOST_WIDE_INT_C (0xAAAA), 1);
14969 0 : }
14970 :
14971 : /* Reflect 8-bit value for the 8-bit target. */
14972 :
14973 : void
14974 0 : reflect_8_bit_value (rtx *op)
14975 : {
14976 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0F),
14977 : HOST_WIDE_INT_C (0xF0), 4);
14978 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x33),
14979 : HOST_WIDE_INT_C (0xCC), 2);
14980 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x55),
14981 : HOST_WIDE_INT_C (0xAA), 1);
14982 0 : }
14983 :
14984 : /* Generate instruction sequence which reflects the value of the OP
14985 : using shift, and, or operations. OP's mode may be less than word_mode. */
14986 :
14987 : void
14988 0 : generate_reflecting_code_standard (rtx *op)
14989 : {
14990 0 : gcc_assert (GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () >= 8
14991 : && GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () <= 64);
14992 :
14993 0 : if (GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () == 64)
14994 0 : reflect_64_bit_value (op);
14995 0 : else if (GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () == 32)
14996 0 : reflect_32_bit_value (op);
14997 0 : else if (GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () == 16)
14998 0 : reflect_16_bit_value (op);
14999 : else
15000 0 : reflect_8_bit_value (op);
15001 0 : }
15002 :
15003 : /* Generate table-based reversed CRC code for the given CRC, INPUT_DATA and
15004 : the POLYNOMIAL (without leading 1).
15005 :
15006 : CRC is OP1, data is OP2 and the polynomial is OP3.
15007 : This generates a reversed CRC table and assembly for the following code,
15008 : where crc_bit_size and data_bit_size may be 8, 16, 32, 64:
15009 : uint_crc_bit_size_t
15010 : crc_crc_bit_size (uint_crc_bit_size_t crc_init,
15011 : uint_data_bit_size_t data)
15012 : {
15013 : uint_crc_bit_size_t crc = crc_init;
15014 : for (int i = 0; i < data_bit_size / 8; i++)
15015 : crc = (crc >> 8) ^ crc_table[(crc ^ (data >> (i * 8))) & 0xFF];
15016 : return crc;
15017 : }
15018 :
15019 : This approach uses a pre-computed reversed polynomial table, eliminating
15020 : the need for explicit bit reflection before and after the CRC computation. */
15021 :
15022 : void
15023 138 : expand_reversed_crc_table_based (rtx op0, rtx op1, rtx op2, rtx op3,
15024 : machine_mode data_mode)
15025 : {
15026 138 : gcc_assert (!CONST_INT_P (op0));
15027 138 : gcc_assert (CONST_INT_P (op3));
15028 138 : machine_mode crc_mode = GET_MODE (op0);
15029 138 : rtx crc = gen_reg_rtx (crc_mode);
15030 138 : convert_move (crc, op1, 0);
15031 138 : calculate_table_based_reversed_CRC (&crc, op2, op3, data_mode);
15032 138 : convert_move (op0, crc, 0);
15033 138 : }
|
