Line data Source code
1 : /* Convert tree expression to rtl instructions, for GNU compiler.
2 : Copyright (C) 1988-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "predict.h"
29 : #include "memmodel.h"
30 : #include "tm_p.h"
31 : #include "ssa.h"
32 : #include "optabs.h"
33 : #include "expmed.h"
34 : #include "regs.h"
35 : #include "emit-rtl.h"
36 : #include "recog.h"
37 : #include "cgraph.h"
38 : #include "diagnostic.h"
39 : #include "alias.h"
40 : #include "fold-const.h"
41 : #include "stor-layout.h"
42 : #include "attribs.h"
43 : #include "varasm.h"
44 : #include "except.h"
45 : #include "insn-attr.h"
46 : #include "dojump.h"
47 : #include "explow.h"
48 : #include "calls.h"
49 : #include "stmt.h"
50 : /* Include expr.h after insn-config.h so we get HAVE_conditional_move. */
51 : #include "expr.h"
52 : #include "optabs-tree.h"
53 : #include "libfuncs.h"
54 : #include "reload.h"
55 : #include "langhooks.h"
56 : #include "common/common-target.h"
57 : #include "tree-dfa.h"
58 : #include "tree-ssa-live.h"
59 : #include "tree-outof-ssa.h"
60 : #include "tree-ssa-address.h"
61 : #include "builtins.h"
62 : #include "ccmp.h"
63 : #include "gimple-iterator.h"
64 : #include "gimple-fold.h"
65 : #include "rtx-vector-builder.h"
66 : #include "tree-pretty-print.h"
67 : #include "flags.h"
68 : #include "internal-fn.h"
69 :
70 :
71 : /* If this is nonzero, we do not bother generating VOLATILE
72 : around volatile memory references, and we are willing to
73 : output indirect addresses. If cse is to follow, we reject
74 : indirect addresses so a useful potential cse is generated;
75 : if it is used only once, instruction combination will produce
76 : the same indirect address eventually. */
77 : int cse_not_expected;
78 :
79 : /* Cache of the "extended" flag in the target's _BitInt description
80 : for use during expand. */
81 : int bitint_extended = -1;
82 :
83 : static bool block_move_libcall_safe_for_call_parm (void);
84 : static bool emit_block_move_via_pattern (rtx, rtx, rtx, unsigned, unsigned,
85 : HOST_WIDE_INT, unsigned HOST_WIDE_INT,
86 : unsigned HOST_WIDE_INT,
87 : unsigned HOST_WIDE_INT, bool);
88 : static void emit_block_move_via_loop (rtx, rtx, rtx, unsigned, int);
89 : static void emit_block_move_via_sized_loop (rtx, rtx, rtx, unsigned, unsigned);
90 : static void emit_block_move_via_oriented_loop (rtx, rtx, rtx, unsigned, unsigned);
91 : static rtx emit_block_cmp_via_loop (rtx, rtx, rtx, tree, rtx, bool,
92 : unsigned, unsigned);
93 : static rtx_insn *compress_float_constant (rtx, rtx);
94 : static rtx get_subtarget (rtx);
95 : static rtx store_field (rtx, poly_int64, poly_int64, poly_uint64, poly_uint64,
96 : machine_mode, tree, alias_set_type, bool, bool);
97 :
98 : static unsigned HOST_WIDE_INT highest_pow2_factor_for_target (const_tree, const_tree);
99 :
100 : static bool is_aligning_offset (const_tree, const_tree);
101 : static rtx reduce_to_bit_field_precision (rtx, rtx, tree);
102 : static rtx do_store_flag (const_sepops, rtx, machine_mode);
103 : #ifdef PUSH_ROUNDING
104 : static void emit_single_push_insn (machine_mode, rtx, tree);
105 : #endif
106 : static void do_tablejump (rtx, machine_mode, rtx, rtx, rtx,
107 : profile_probability);
108 : static rtx const_vector_from_tree (tree);
109 : static tree tree_expr_size (const_tree);
110 : static void convert_mode_scalar (rtx, rtx, int);
111 :
112 :
113 : /* This is run to set up which modes can be used
114 : directly in memory and to initialize the block move optab. It is run
115 : at the beginning of compilation and when the target is reinitialized. */
116 :
117 : void
118 218789 : init_expr_target (void)
119 : {
120 218789 : rtx pat;
121 218789 : int num_clobbers;
122 218789 : rtx mem, mem1;
123 218789 : rtx reg;
124 :
125 : /* Try indexing by frame ptr and try by stack ptr.
126 : It is known that on the Convex the stack ptr isn't a valid index.
127 : With luck, one or the other is valid on any machine. */
128 218789 : mem = gen_rtx_MEM (word_mode, stack_pointer_rtx);
129 218789 : mem1 = gen_rtx_MEM (word_mode, frame_pointer_rtx);
130 :
131 : /* A scratch register we can modify in-place below to avoid
132 : useless RTL allocations. */
133 218789 : reg = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
134 :
135 218789 : rtx_insn *insn = as_a<rtx_insn *> (rtx_alloc (INSN));
136 218789 : pat = gen_rtx_SET (NULL_RTX, NULL_RTX);
137 218789 : PATTERN (insn) = pat;
138 :
139 27348625 : for (machine_mode mode = VOIDmode; (int) mode < NUM_MACHINE_MODES;
140 27129836 : mode = (machine_mode) ((int) mode + 1))
141 : {
142 27129836 : int regno;
143 :
144 27129836 : direct_load[(int) mode] = direct_store[(int) mode] = 0;
145 27129836 : PUT_MODE (mem, mode);
146 27129836 : PUT_MODE (mem1, mode);
147 :
148 : /* See if there is some register that can be used in this mode and
149 : directly loaded or stored from memory. */
150 :
151 27129836 : if (mode != VOIDmode && mode != BLKmode)
152 1828750140 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER
153 1855442398 : && (direct_load[(int) mode] == 0 || direct_store[(int) mode] == 0);
154 : regno++)
155 : {
156 1828750140 : if (!targetm.hard_regno_mode_ok (regno, mode))
157 1709006478 : continue;
158 :
159 119743662 : set_mode_and_regno (reg, mode, regno);
160 :
161 119743662 : SET_SRC (pat) = mem;
162 119743662 : SET_DEST (pat) = reg;
163 119743662 : if (recog (pat, insn, &num_clobbers) >= 0)
164 7355308 : direct_load[(int) mode] = 1;
165 :
166 119743662 : SET_SRC (pat) = mem1;
167 119743662 : SET_DEST (pat) = reg;
168 119743662 : if (recog (pat, insn, &num_clobbers) >= 0)
169 7355308 : direct_load[(int) mode] = 1;
170 :
171 119743662 : SET_SRC (pat) = reg;
172 119743662 : SET_DEST (pat) = mem;
173 119743662 : if (recog (pat, insn, &num_clobbers) >= 0)
174 7355308 : direct_store[(int) mode] = 1;
175 :
176 119743662 : SET_SRC (pat) = reg;
177 119743662 : SET_DEST (pat) = mem1;
178 119743662 : if (recog (pat, insn, &num_clobbers) >= 0)
179 7355308 : direct_store[(int) mode] = 1;
180 : }
181 : }
182 :
183 224497 : mem = gen_rtx_MEM (VOIDmode, gen_raw_REG (Pmode, LAST_VIRTUAL_REGISTER + 1));
184 :
185 218789 : opt_scalar_float_mode mode_iter;
186 1531523 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_FLOAT)
187 : {
188 1312734 : scalar_float_mode mode = mode_iter.require ();
189 1312734 : scalar_float_mode srcmode;
190 4594569 : FOR_EACH_MODE_UNTIL (srcmode, mode)
191 : {
192 3281835 : enum insn_code ic;
193 :
194 3281835 : ic = can_extend_p (mode, srcmode, 0);
195 3281835 : if (ic == CODE_FOR_nothing)
196 2620820 : continue;
197 :
198 661015 : PUT_MODE (mem, srcmode);
199 :
200 661015 : if (insn_operand_matches (ic, 1, mem))
201 658876 : float_extend_from_mem[mode][srcmode] = true;
202 : }
203 : }
204 218789 : }
205 :
206 : /* This is run at the start of compiling a function. */
207 :
208 : void
209 1730071 : init_expr (void)
210 : {
211 1730071 : memset (&crtl->expr, 0, sizeof (crtl->expr));
212 1730071 : }
213 :
214 : /* Copy data from FROM to TO, where the machine modes are not the same.
215 : Both modes may be integer, or both may be floating, or both may be
216 : fixed-point.
217 : UNSIGNEDP should be nonzero if FROM is an unsigned type.
218 : This causes zero-extension instead of sign-extension. */
219 :
220 : void
221 2152874 : convert_move (rtx to, rtx from, int unsignedp)
222 : {
223 2152874 : machine_mode to_mode = GET_MODE (to);
224 2152874 : machine_mode from_mode = GET_MODE (from);
225 :
226 2152874 : gcc_assert (to_mode != BLKmode);
227 2152874 : gcc_assert (from_mode != BLKmode);
228 :
229 : /* If the source and destination are already the same, then there's
230 : nothing to do. */
231 2152874 : if (to == from)
232 2152874 : return;
233 :
234 : /* If FROM is a SUBREG that indicates that we have already done at least
235 : the required extension, strip it. We don't handle such SUBREGs as
236 : TO here. */
237 :
238 2152874 : scalar_int_mode to_int_mode;
239 2152874 : if (GET_CODE (from) == SUBREG
240 177988 : && SUBREG_PROMOTED_VAR_P (from)
241 2152874 : && is_a <scalar_int_mode> (to_mode, &to_int_mode)
242 2152874 : && (GET_MODE_PRECISION (subreg_promoted_mode (from))
243 0 : >= GET_MODE_PRECISION (to_int_mode))
244 2152874 : && SUBREG_CHECK_PROMOTED_SIGN (from, unsignedp))
245 : {
246 0 : scalar_int_mode int_orig_mode;
247 0 : scalar_int_mode int_inner_mode;
248 0 : machine_mode orig_mode = GET_MODE (from);
249 :
250 0 : from = gen_lowpart (to_int_mode, SUBREG_REG (from));
251 0 : from_mode = to_int_mode;
252 :
253 : /* Preserve SUBREG_PROMOTED_VAR_P if the new mode is wider than
254 : the original mode, but narrower than the inner mode. */
255 0 : if (GET_CODE (from) == SUBREG
256 0 : && is_a <scalar_int_mode> (orig_mode, &int_orig_mode)
257 0 : && GET_MODE_PRECISION (to_int_mode)
258 0 : > GET_MODE_PRECISION (int_orig_mode)
259 0 : && is_a <scalar_int_mode> (GET_MODE (SUBREG_REG (from)),
260 : &int_inner_mode)
261 0 : && GET_MODE_PRECISION (int_inner_mode)
262 0 : > GET_MODE_PRECISION (to_int_mode))
263 : {
264 0 : SUBREG_PROMOTED_VAR_P (from) = 1;
265 0 : SUBREG_PROMOTED_SET (from, unsignedp);
266 : }
267 : }
268 :
269 2152874 : gcc_assert (GET_CODE (to) != SUBREG || !SUBREG_PROMOTED_VAR_P (to));
270 :
271 2152874 : if (to_mode == from_mode
272 2105893 : || (from_mode == VOIDmode && CONSTANT_P (from)))
273 : {
274 47252 : emit_move_insn (to, from);
275 47252 : return;
276 : }
277 :
278 2105622 : if (VECTOR_MODE_P (to_mode) || VECTOR_MODE_P (from_mode))
279 : {
280 30046 : if (GET_MODE_UNIT_PRECISION (to_mode)
281 15023 : > GET_MODE_UNIT_PRECISION (from_mode))
282 : {
283 4607 : optab op = unsignedp ? zext_optab : sext_optab;
284 4607 : insn_code icode = convert_optab_handler (op, to_mode, from_mode);
285 4607 : if (icode != CODE_FOR_nothing)
286 : {
287 777 : emit_unop_insn (icode, to, from,
288 : unsignedp ? ZERO_EXTEND : SIGN_EXTEND);
289 777 : return;
290 : }
291 : }
292 :
293 28492 : if (GET_MODE_UNIT_PRECISION (to_mode)
294 14246 : < GET_MODE_UNIT_PRECISION (from_mode))
295 : {
296 2381 : insn_code icode = convert_optab_handler (trunc_optab,
297 : to_mode, from_mode);
298 2381 : if (icode != CODE_FOR_nothing)
299 : {
300 880 : emit_unop_insn (icode, to, from, TRUNCATE);
301 880 : return;
302 : }
303 : }
304 :
305 40098 : gcc_assert (known_eq (GET_MODE_BITSIZE (from_mode),
306 : GET_MODE_BITSIZE (to_mode)));
307 :
308 13366 : if (VECTOR_MODE_P (to_mode))
309 13366 : from = force_subreg (to_mode, from, GET_MODE (from), 0);
310 : else
311 0 : to = simplify_gen_subreg (from_mode, to, GET_MODE (to), 0);
312 :
313 13366 : emit_move_insn (to, from);
314 13366 : return;
315 : }
316 :
317 2090599 : if (GET_CODE (to) == CONCAT && GET_CODE (from) == CONCAT)
318 : {
319 0 : convert_move (XEXP (to, 0), XEXP (from, 0), unsignedp);
320 0 : convert_move (XEXP (to, 1), XEXP (from, 1), unsignedp);
321 0 : return;
322 : }
323 :
324 2090599 : convert_mode_scalar (to, from, unsignedp);
325 : }
326 :
327 : /* Like convert_move, but deals only with scalar modes. */
328 :
329 : static void
330 2090661 : convert_mode_scalar (rtx to, rtx from, int unsignedp)
331 : {
332 : /* Both modes should be scalar types. */
333 2090666 : scalar_mode from_mode = as_a <scalar_mode> (GET_MODE (from));
334 2090666 : scalar_mode to_mode = as_a <scalar_mode> (GET_MODE (to));
335 2090666 : bool to_real = SCALAR_FLOAT_MODE_P (to_mode);
336 2090666 : bool from_real = SCALAR_FLOAT_MODE_P (from_mode);
337 2090666 : enum insn_code code;
338 2090666 : rtx libcall;
339 :
340 2090666 : gcc_assert (to_real == from_real);
341 :
342 : /* rtx code for making an equivalent value. */
343 3193193 : enum rtx_code equiv_code = (unsignedp < 0 ? UNKNOWN
344 2090666 : : (unsignedp ? ZERO_EXTEND : SIGN_EXTEND));
345 :
346 2090666 : auto acceptable_same_precision_modes
347 580 : = [] (scalar_mode from_mode, scalar_mode to_mode) -> bool
348 : {
349 580 : if (DECIMAL_FLOAT_MODE_P (from_mode) != DECIMAL_FLOAT_MODE_P (to_mode))
350 : return true;
351 :
352 : /* arm_bfloat_half_format <-> ieee_half_format */
353 2 : if ((REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
354 1 : && REAL_MODE_FORMAT (to_mode) == &ieee_half_format)
355 2 : || (REAL_MODE_FORMAT (to_mode) == &arm_bfloat_half_format
356 1 : && REAL_MODE_FORMAT (from_mode) == &ieee_half_format))
357 : return true;
358 :
359 : /* ibm_extended_format <-> ieee_quad_format */
360 0 : if ((REAL_MODE_FORMAT (from_mode) == &ibm_extended_format
361 0 : && REAL_MODE_FORMAT (to_mode) == &ieee_quad_format)
362 0 : || (REAL_MODE_FORMAT (from_mode) == &ieee_quad_format
363 0 : && REAL_MODE_FORMAT (to_mode) == &ibm_extended_format))
364 0 : return true;
365 :
366 : return false;
367 : };
368 :
369 2090666 : if (to_real)
370 : {
371 186182 : rtx value;
372 186182 : rtx_insn *insns;
373 186182 : convert_optab tab;
374 :
375 186182 : gcc_assert ((GET_MODE_PRECISION (from_mode)
376 : != GET_MODE_PRECISION (to_mode))
377 : || acceptable_same_precision_modes (from_mode, to_mode));
378 :
379 186182 : if (GET_MODE_PRECISION (from_mode) == GET_MODE_PRECISION (to_mode))
380 : {
381 580 : if ((REAL_MODE_FORMAT (to_mode) == &arm_bfloat_half_format
382 1 : && REAL_MODE_FORMAT (from_mode) == &ieee_half_format)
383 580 : || (REAL_MODE_FORMAT (to_mode) == &ieee_quad_format
384 0 : && REAL_MODE_FORMAT (from_mode) == &ibm_extended_format))
385 : /* libgcc implements just __trunchfbf2, not __extendhfbf2;
386 : and __trunctfkf2, not __extendtfkf2. */
387 : tab = trunc_optab;
388 : else
389 : /* Conversion between decimal float and binary float, same
390 : size. */
391 579 : tab = DECIMAL_FLOAT_MODE_P (from_mode) ? trunc_optab : sext_optab;
392 : }
393 185602 : else if (GET_MODE_PRECISION (from_mode) < GET_MODE_PRECISION (to_mode))
394 : tab = sext_optab;
395 : else
396 18584 : tab = trunc_optab;
397 :
398 : /* Try converting directly if the insn is supported. */
399 :
400 186182 : code = convert_optab_handler (tab, to_mode, from_mode);
401 186182 : if (code != CODE_FOR_nothing)
402 : {
403 160976 : emit_unop_insn (code, to, from,
404 : tab == sext_optab ? FLOAT_EXTEND : FLOAT_TRUNCATE);
405 160976 : return;
406 : }
407 :
408 : #ifdef HAVE_SFmode
409 25206 : if (REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
410 25206 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
411 : {
412 2462 : if (GET_MODE_PRECISION (to_mode) > GET_MODE_PRECISION (SFmode))
413 : {
414 : /* To cut down on libgcc size, implement
415 : BFmode -> {DF,XF,TF}mode conversions by
416 : BFmode -> SFmode -> {DF,XF,TF}mode conversions. */
417 4 : rtx temp = gen_reg_rtx (SFmode);
418 4 : convert_mode_scalar (temp, from, unsignedp);
419 4 : convert_mode_scalar (to, temp, unsignedp);
420 4 : return;
421 : }
422 2458 : if (REAL_MODE_FORMAT (to_mode) == &ieee_half_format)
423 : {
424 : /* Similarly, implement BFmode -> HFmode as
425 : BFmode -> SFmode -> HFmode conversion where SFmode
426 : has superset of BFmode values. We don't need
427 : to handle sNaNs by raising exception and turning
428 : it into qNaN though, as that can be done in the
429 : SFmode -> HFmode conversion too. */
430 1 : rtx temp = gen_reg_rtx (SFmode);
431 1 : int save_flag_finite_math_only = flag_finite_math_only;
432 1 : flag_finite_math_only = true;
433 1 : convert_mode_scalar (temp, from, unsignedp);
434 1 : flag_finite_math_only = save_flag_finite_math_only;
435 1 : convert_mode_scalar (to, temp, unsignedp);
436 1 : return;
437 : }
438 2457 : if (to_mode == SFmode
439 2457 : && !HONOR_NANS (from_mode)
440 58 : && !HONOR_NANS (to_mode)
441 2515 : && optimize_insn_for_speed_p ())
442 : {
443 : /* If we don't expect sNaNs, for BFmode -> SFmode we can just
444 : shift the bits up. */
445 57 : machine_mode fromi_mode, toi_mode;
446 114 : if (int_mode_for_size (GET_MODE_BITSIZE (from_mode),
447 57 : 0).exists (&fromi_mode)
448 57 : && int_mode_for_size (GET_MODE_BITSIZE (to_mode),
449 0 : 0).exists (&toi_mode))
450 : {
451 57 : start_sequence ();
452 57 : rtx fromi = force_lowpart_subreg (fromi_mode, from,
453 : from_mode);
454 57 : rtx tof = NULL_RTX;
455 57 : if (fromi)
456 : {
457 57 : rtx toi;
458 57 : if (GET_MODE (fromi) == VOIDmode)
459 0 : toi = simplify_unary_operation (ZERO_EXTEND, toi_mode,
460 : fromi, fromi_mode);
461 : else
462 : {
463 57 : toi = gen_reg_rtx (toi_mode);
464 57 : convert_mode_scalar (toi, fromi, 1);
465 : }
466 57 : toi
467 114 : = maybe_expand_shift (LSHIFT_EXPR, toi_mode, toi,
468 57 : GET_MODE_PRECISION (to_mode)
469 57 : - GET_MODE_PRECISION (from_mode),
470 : NULL_RTX, 1);
471 57 : if (toi)
472 : {
473 57 : tof = force_lowpart_subreg (to_mode, toi, toi_mode);
474 57 : if (tof)
475 57 : emit_move_insn (to, tof);
476 : }
477 : }
478 57 : insns = end_sequence ();
479 57 : if (tof)
480 : {
481 57 : emit_insn (insns);
482 57 : return;
483 : }
484 : }
485 : }
486 : }
487 25144 : if (REAL_MODE_FORMAT (from_mode) == &ieee_single_format
488 4274 : && REAL_MODE_FORMAT (to_mode) == &arm_bfloat_half_format
489 1572 : && !HONOR_NANS (from_mode)
490 0 : && !HONOR_NANS (to_mode)
491 0 : && !flag_rounding_math
492 25144 : && optimize_insn_for_speed_p ())
493 : {
494 : /* If we don't expect qNaNs nor sNaNs and can assume rounding
495 : to nearest, we can expand the conversion inline as
496 : (fromi + 0x7fff + ((fromi >> 16) & 1)) >> 16. */
497 0 : machine_mode fromi_mode, toi_mode;
498 0 : if (int_mode_for_size (GET_MODE_BITSIZE (from_mode),
499 0 : 0).exists (&fromi_mode)
500 0 : && int_mode_for_size (GET_MODE_BITSIZE (to_mode),
501 0 : 0).exists (&toi_mode))
502 : {
503 0 : start_sequence ();
504 0 : rtx fromi = force_lowpart_subreg (fromi_mode, from, from_mode);
505 0 : rtx tof = NULL_RTX;
506 0 : do
507 : {
508 0 : if (!fromi)
509 : break;
510 0 : int shift = (GET_MODE_PRECISION (from_mode)
511 0 : - GET_MODE_PRECISION (to_mode));
512 0 : rtx temp1
513 0 : = maybe_expand_shift (RSHIFT_EXPR, fromi_mode, fromi,
514 : shift, NULL_RTX, 1);
515 0 : if (!temp1)
516 : break;
517 0 : rtx temp2
518 0 : = expand_binop (fromi_mode, and_optab, temp1, const1_rtx,
519 : NULL_RTX, 1, OPTAB_DIRECT);
520 0 : if (!temp2)
521 : break;
522 0 : rtx temp3
523 0 : = expand_binop (fromi_mode, add_optab, fromi,
524 : gen_int_mode ((HOST_WIDE_INT_1U
525 0 : << (shift - 1)) - 1,
526 : fromi_mode), NULL_RTX,
527 : 1, OPTAB_DIRECT);
528 0 : if (!temp3)
529 : break;
530 0 : rtx temp4
531 0 : = expand_binop (fromi_mode, add_optab, temp3, temp2,
532 : NULL_RTX, 1, OPTAB_DIRECT);
533 0 : if (!temp4)
534 : break;
535 0 : rtx temp5 = maybe_expand_shift (RSHIFT_EXPR, fromi_mode,
536 : temp4, shift, NULL_RTX, 1);
537 0 : if (!temp5)
538 : break;
539 0 : rtx temp6 = force_lowpart_subreg (toi_mode, temp5,
540 : fromi_mode);
541 0 : if (!temp6)
542 : break;
543 0 : tof = force_lowpart_subreg (to_mode, temp6, toi_mode);
544 0 : if (tof)
545 0 : emit_move_insn (to, tof);
546 : }
547 : while (0);
548 0 : insns = end_sequence ();
549 0 : if (tof)
550 : {
551 0 : emit_insn (insns);
552 0 : return;
553 : }
554 : }
555 : }
556 : #endif
557 :
558 : /* Otherwise use a libcall. */
559 25144 : libcall = convert_optab_libfunc (tab, to_mode, from_mode);
560 :
561 : /* Is this conversion implemented yet? */
562 25144 : gcc_assert (libcall);
563 :
564 25144 : start_sequence ();
565 25144 : value = emit_library_call_value (libcall, NULL_RTX, LCT_CONST, to_mode,
566 : from, from_mode);
567 25144 : insns = end_sequence ();
568 25144 : emit_libcall_block (insns, to, value,
569 6795 : tab == trunc_optab ? gen_rtx_FLOAT_TRUNCATE (to_mode,
570 : from)
571 18349 : : gen_rtx_FLOAT_EXTEND (to_mode, from));
572 25144 : return;
573 : }
574 :
575 : /* Handle pointer conversion. */ /* SPEE 900220. */
576 : /* If the target has a converter from FROM_MODE to TO_MODE, use it. */
577 1904484 : {
578 1904484 : convert_optab ctab;
579 :
580 1904484 : if (GET_MODE_PRECISION (from_mode) > GET_MODE_PRECISION (to_mode))
581 : ctab = trunc_optab;
582 1711972 : else if (unsignedp)
583 : ctab = zext_optab;
584 : else
585 831090 : ctab = sext_optab;
586 :
587 1904484 : if (convert_optab_handler (ctab, to_mode, from_mode)
588 : != CODE_FOR_nothing)
589 : {
590 1641493 : emit_unop_insn (convert_optab_handler (ctab, to_mode, from_mode),
591 : to, from, UNKNOWN);
592 1641493 : return;
593 : }
594 : }
595 :
596 : /* Targets are expected to provide conversion insns between PxImode and
597 : xImode for all MODE_PARTIAL_INT modes they use, but no others. */
598 262991 : if (GET_MODE_CLASS (to_mode) == MODE_PARTIAL_INT)
599 : {
600 0 : scalar_int_mode full_mode
601 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE (to_mode)).require ();
602 :
603 0 : gcc_assert (convert_optab_handler (trunc_optab, to_mode, full_mode)
604 : != CODE_FOR_nothing);
605 :
606 0 : if (full_mode != from_mode)
607 0 : from = convert_to_mode (full_mode, from, unsignedp);
608 0 : emit_unop_insn (convert_optab_handler (trunc_optab, to_mode, full_mode),
609 : to, from, UNKNOWN);
610 0 : return;
611 : }
612 262991 : if (GET_MODE_CLASS (from_mode) == MODE_PARTIAL_INT)
613 : {
614 0 : rtx new_from;
615 0 : scalar_int_mode full_mode
616 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE (from_mode)).require ();
617 0 : convert_optab ctab = unsignedp ? zext_optab : sext_optab;
618 0 : enum insn_code icode;
619 :
620 0 : icode = convert_optab_handler (ctab, full_mode, from_mode);
621 0 : gcc_assert (icode != CODE_FOR_nothing);
622 :
623 0 : if (to_mode == full_mode)
624 : {
625 0 : emit_unop_insn (icode, to, from, UNKNOWN);
626 0 : return;
627 : }
628 :
629 0 : new_from = gen_reg_rtx (full_mode);
630 0 : emit_unop_insn (icode, new_from, from, UNKNOWN);
631 :
632 : /* else proceed to integer conversions below. */
633 0 : from_mode = full_mode;
634 0 : from = new_from;
635 : }
636 :
637 : /* Make sure both are fixed-point modes or both are not. */
638 262991 : gcc_assert (ALL_SCALAR_FIXED_POINT_MODE_P (from_mode) ==
639 : ALL_SCALAR_FIXED_POINT_MODE_P (to_mode));
640 262991 : if (ALL_SCALAR_FIXED_POINT_MODE_P (from_mode))
641 : {
642 : /* If we widen from_mode to to_mode and they are in the same class,
643 : we won't saturate the result.
644 : Otherwise, always saturate the result to play safe. */
645 0 : if (GET_MODE_CLASS (from_mode) == GET_MODE_CLASS (to_mode)
646 0 : && GET_MODE_SIZE (from_mode) < GET_MODE_SIZE (to_mode))
647 0 : expand_fixed_convert (to, from, 0, 0);
648 : else
649 0 : expand_fixed_convert (to, from, 0, 1);
650 0 : return;
651 : }
652 :
653 : /* Now both modes are integers. */
654 :
655 : /* Handle expanding beyond a word. */
656 262991 : if (GET_MODE_PRECISION (from_mode) < GET_MODE_PRECISION (to_mode)
657 265425 : && GET_MODE_PRECISION (to_mode) > BITS_PER_WORD)
658 : {
659 70479 : rtx_insn *insns;
660 70479 : rtx lowpart;
661 70479 : rtx fill_value;
662 70479 : rtx lowfrom;
663 70479 : int i;
664 70479 : scalar_mode lowpart_mode;
665 140958 : int nwords = CEIL (GET_MODE_SIZE (to_mode), UNITS_PER_WORD);
666 :
667 : /* Try converting directly if the insn is supported. */
668 70479 : if ((code = can_extend_p (to_mode, from_mode, unsignedp))
669 : != CODE_FOR_nothing)
670 : {
671 : /* If FROM is a SUBREG, put it into a register. Do this
672 : so that we always generate the same set of insns for
673 : better cse'ing; if an intermediate assignment occurred,
674 : we won't be doing the operation directly on the SUBREG. */
675 0 : if (optimize > 0 && GET_CODE (from) == SUBREG)
676 0 : from = force_reg (from_mode, from);
677 0 : emit_unop_insn (code, to, from, equiv_code);
678 0 : return;
679 : }
680 : /* Next, try converting via full word. */
681 70479 : else if (GET_MODE_PRECISION (from_mode) < BITS_PER_WORD
682 70479 : && ((code = can_extend_p (to_mode, word_mode, unsignedp))
683 : != CODE_FOR_nothing))
684 : {
685 70479 : rtx word_to = gen_reg_rtx (word_mode);
686 70479 : if (REG_P (to))
687 : {
688 70406 : if (reg_overlap_mentioned_p (to, from))
689 0 : from = force_reg (from_mode, from);
690 70406 : emit_clobber (to);
691 : }
692 70479 : convert_move (word_to, from, unsignedp);
693 70479 : emit_unop_insn (code, to, word_to, equiv_code);
694 70479 : return;
695 : }
696 :
697 : /* No special multiword conversion insn; do it by hand. */
698 0 : start_sequence ();
699 :
700 : /* Since we will turn this into a no conflict block, we must ensure
701 : the source does not overlap the target so force it into an isolated
702 : register when maybe so. Likewise for any MEM input, since the
703 : conversion sequence might require several references to it and we
704 : must ensure we're getting the same value every time. */
705 :
706 0 : if (MEM_P (from) || reg_overlap_mentioned_p (to, from))
707 0 : from = force_reg (from_mode, from);
708 :
709 : /* Get a copy of FROM widened to a word, if necessary. */
710 0 : if (GET_MODE_PRECISION (from_mode) < BITS_PER_WORD)
711 0 : lowpart_mode = word_mode;
712 : else
713 : lowpart_mode = from_mode;
714 :
715 0 : lowfrom = convert_to_mode (lowpart_mode, from, unsignedp);
716 :
717 0 : lowpart = gen_lowpart (lowpart_mode, to);
718 0 : emit_move_insn (lowpart, lowfrom);
719 :
720 : /* Compute the value to put in each remaining word. */
721 0 : if (unsignedp)
722 0 : fill_value = const0_rtx;
723 : else
724 0 : fill_value = emit_store_flag_force (gen_reg_rtx (word_mode),
725 : LT, lowfrom, const0_rtx,
726 : lowpart_mode, 0, -1);
727 :
728 : /* Fill the remaining words. */
729 0 : for (i = GET_MODE_SIZE (lowpart_mode) / UNITS_PER_WORD; i < nwords; i++)
730 : {
731 0 : int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
732 0 : rtx subword = operand_subword (to, index, 1, to_mode);
733 :
734 0 : gcc_assert (subword);
735 :
736 0 : if (fill_value != subword)
737 0 : emit_move_insn (subword, fill_value);
738 : }
739 :
740 0 : insns = end_sequence ();
741 :
742 0 : emit_insn (insns);
743 0 : return;
744 : }
745 :
746 : /* Truncating multi-word to a word or less. */
747 192512 : if (GET_MODE_PRECISION (from_mode) > BITS_PER_WORD
748 192512 : && GET_MODE_PRECISION (to_mode) <= BITS_PER_WORD)
749 : {
750 69210 : if (!((MEM_P (from)
751 568 : && ! MEM_VOLATILE_P (from)
752 568 : && direct_load[(int) to_mode]
753 568 : && ! mode_dependent_address_p (XEXP (from, 0),
754 568 : MEM_ADDR_SPACE (from)))
755 38951 : || REG_P (from)
756 : || GET_CODE (from) == SUBREG))
757 0 : from = force_reg (from_mode, from);
758 39519 : convert_move (to, gen_lowpart (word_mode, from), 0);
759 39519 : return;
760 : }
761 :
762 : /* Now follow all the conversions between integers
763 : no more than a word long. */
764 :
765 : /* For truncation, usually we can just refer to FROM in a narrower mode. */
766 305986 : if (GET_MODE_BITSIZE (to_mode) < GET_MODE_BITSIZE (from_mode)
767 152993 : && TRULY_NOOP_TRUNCATION_MODES_P (to_mode, from_mode))
768 : {
769 164802 : if (!((MEM_P (from)
770 13867 : && ! MEM_VOLATILE_P (from)
771 13808 : && direct_load[(int) to_mode]
772 13808 : && ! mode_dependent_address_p (XEXP (from, 0),
773 13808 : MEM_ADDR_SPACE (from)))
774 139185 : || REG_P (from)
775 : || GET_CODE (from) == SUBREG))
776 925 : from = force_reg (from_mode, from);
777 128301 : if (REG_P (from) && REGNO (from) < FIRST_PSEUDO_REGISTER
778 152994 : && !targetm.hard_regno_mode_ok (REGNO (from), to_mode))
779 0 : from = copy_to_reg (from);
780 152993 : emit_move_insn (to, gen_lowpart (to_mode, from));
781 152993 : return;
782 : }
783 :
784 : /* Handle extension. */
785 0 : if (GET_MODE_PRECISION (to_mode) > GET_MODE_PRECISION (from_mode))
786 : {
787 : /* Convert directly if that works. */
788 0 : if ((code = can_extend_p (to_mode, from_mode, unsignedp))
789 : != CODE_FOR_nothing)
790 : {
791 0 : emit_unop_insn (code, to, from, equiv_code);
792 0 : return;
793 : }
794 : else
795 : {
796 0 : rtx tmp;
797 0 : int shift_amount;
798 :
799 : /* Search for a mode to convert via. */
800 0 : opt_scalar_mode intermediate_iter;
801 0 : FOR_EACH_MODE_FROM (intermediate_iter, from_mode)
802 : {
803 0 : scalar_mode intermediate = intermediate_iter.require ();
804 0 : if (((can_extend_p (to_mode, intermediate, unsignedp)
805 : != CODE_FOR_nothing)
806 0 : || (GET_MODE_SIZE (to_mode) < GET_MODE_SIZE (intermediate)
807 0 : && TRULY_NOOP_TRUNCATION_MODES_P (to_mode,
808 : intermediate)))
809 0 : && (can_extend_p (intermediate, from_mode, unsignedp)
810 : != CODE_FOR_nothing))
811 : {
812 0 : convert_move (to, convert_to_mode (intermediate, from,
813 : unsignedp), unsignedp);
814 0 : return;
815 : }
816 : }
817 :
818 : /* No suitable intermediate mode.
819 : Generate what we need with shifts. */
820 0 : shift_amount = (GET_MODE_PRECISION (to_mode)
821 0 : - GET_MODE_PRECISION (from_mode));
822 0 : from = gen_lowpart (to_mode, force_reg (from_mode, from));
823 0 : tmp = expand_shift (LSHIFT_EXPR, to_mode, from, shift_amount,
824 : to, unsignedp);
825 0 : tmp = expand_shift (RSHIFT_EXPR, to_mode, tmp, shift_amount,
826 : to, unsignedp);
827 0 : if (tmp != to)
828 0 : emit_move_insn (to, tmp);
829 0 : return;
830 : }
831 : }
832 :
833 : /* Support special truncate insns for certain modes. */
834 0 : if (convert_optab_handler (trunc_optab, to_mode,
835 : from_mode) != CODE_FOR_nothing)
836 : {
837 0 : emit_unop_insn (convert_optab_handler (trunc_optab, to_mode, from_mode),
838 : to, from, UNKNOWN);
839 0 : return;
840 : }
841 :
842 : /* Handle truncation of volatile memrefs, and so on;
843 : the things that couldn't be truncated directly,
844 : and for which there was no special instruction.
845 :
846 : ??? Code above formerly short-circuited this, for most integer
847 : mode pairs, with a force_reg in from_mode followed by a recursive
848 : call to this routine. Appears always to have been wrong. */
849 0 : if (GET_MODE_PRECISION (to_mode) < GET_MODE_PRECISION (from_mode))
850 : {
851 0 : rtx temp = force_reg (to_mode, gen_lowpart (to_mode, from));
852 0 : emit_move_insn (to, temp);
853 0 : return;
854 : }
855 :
856 : /* Mode combination is not recognized. */
857 0 : gcc_unreachable ();
858 : }
859 :
860 : /* Return an rtx for a value that would result
861 : from converting X to mode MODE.
862 : Both X and MODE may be floating, or both integer.
863 : UNSIGNEDP is nonzero if X is an unsigned value.
864 : This can be done by referring to a part of X in place
865 : or by copying to a new temporary with conversion. */
866 :
867 : rtx
868 2011989 : convert_to_mode (machine_mode mode, rtx x, int unsignedp)
869 : {
870 2011989 : return convert_modes (mode, VOIDmode, x, unsignedp);
871 : }
872 :
873 : /* Return an rtx for a value that would result
874 : from converting X from mode OLDMODE to mode MODE.
875 : Both modes may be floating, or both integer.
876 : UNSIGNEDP is nonzero if X is an unsigned value.
877 :
878 : This can be done by referring to a part of X in place
879 : or by copying to a new temporary with conversion.
880 :
881 : You can give VOIDmode for OLDMODE, if you are sure X has a nonvoid mode. */
882 :
883 : rtx
884 4835227 : convert_modes (machine_mode mode, machine_mode oldmode, rtx x, int unsignedp)
885 : {
886 4835227 : rtx temp;
887 4835227 : scalar_int_mode int_mode;
888 :
889 : /* If FROM is a SUBREG that indicates that we have already done at least
890 : the required extension, strip it. */
891 :
892 4835227 : if (GET_CODE (x) == SUBREG
893 140253 : && SUBREG_PROMOTED_VAR_P (x)
894 4835227 : && is_a <scalar_int_mode> (mode, &int_mode)
895 4835227 : && (GET_MODE_PRECISION (subreg_promoted_mode (x))
896 7 : >= GET_MODE_PRECISION (int_mode))
897 4835234 : && SUBREG_CHECK_PROMOTED_SIGN (x, unsignedp))
898 : {
899 7 : scalar_int_mode int_orig_mode;
900 7 : scalar_int_mode int_inner_mode;
901 7 : machine_mode orig_mode = GET_MODE (x);
902 7 : x = gen_lowpart (int_mode, SUBREG_REG (x));
903 :
904 : /* Preserve SUBREG_PROMOTED_VAR_P if the new mode is wider than
905 : the original mode, but narrower than the inner mode. */
906 7 : if (GET_CODE (x) == SUBREG
907 0 : && is_a <scalar_int_mode> (orig_mode, &int_orig_mode)
908 0 : && GET_MODE_PRECISION (int_mode)
909 0 : > GET_MODE_PRECISION (int_orig_mode)
910 0 : && is_a <scalar_int_mode> (GET_MODE (SUBREG_REG (x)),
911 : &int_inner_mode)
912 7 : && GET_MODE_PRECISION (int_inner_mode)
913 0 : > GET_MODE_PRECISION (int_mode))
914 : {
915 0 : SUBREG_PROMOTED_VAR_P (x) = 1;
916 0 : SUBREG_PROMOTED_SET (x, unsignedp);
917 : }
918 : }
919 :
920 4835227 : if (GET_MODE (x) != VOIDmode)
921 2498767 : oldmode = GET_MODE (x);
922 :
923 4835227 : if (mode == oldmode)
924 : return x;
925 :
926 3771640 : if (CONST_SCALAR_INT_P (x)
927 3771640 : && is_a <scalar_int_mode> (mode, &int_mode))
928 : {
929 : /* If the caller did not tell us the old mode, then there is not
930 : much to do with respect to canonicalization. We have to
931 : assume that all the bits are significant. */
932 2015919 : if (!is_a <scalar_int_mode> (oldmode))
933 1690117 : oldmode = MAX_MODE_INT;
934 2015919 : wide_int w = wide_int::from (rtx_mode_t (x, oldmode),
935 2015919 : GET_MODE_PRECISION (int_mode),
936 2662612 : unsignedp ? UNSIGNED : SIGNED);
937 2015919 : return immed_wide_int_const (w, int_mode);
938 2015919 : }
939 :
940 : /* We can do this with a gen_lowpart if both desired and current modes
941 : are integer, and this is either a constant integer, a register, or a
942 : non-volatile MEM. */
943 1755721 : scalar_int_mode int_oldmode;
944 1755721 : if (is_int_mode (mode, &int_mode)
945 1662635 : && is_int_mode (oldmode, &int_oldmode)
946 1662635 : && GET_MODE_PRECISION (int_mode) <= GET_MODE_PRECISION (int_oldmode)
947 460064 : && ((MEM_P (x) && !MEM_VOLATILE_P (x) && direct_load[(int) int_mode])
948 12210 : || CONST_POLY_INT_P (x)
949 447854 : || (REG_P (x)
950 412163 : && (!HARD_REGISTER_P (x)
951 1068 : || targetm.hard_regno_mode_ok (REGNO (x), int_mode))
952 412163 : && TRULY_NOOP_TRUNCATION_MODES_P (int_mode, GET_MODE (x)))))
953 424373 : return gen_lowpart (int_mode, x);
954 :
955 : /* Converting from integer constant into mode is always equivalent to an
956 : subreg operation. */
957 1331348 : if (VECTOR_MODE_P (mode) && GET_MODE (x) == VOIDmode)
958 : {
959 0 : gcc_assert (known_eq (GET_MODE_BITSIZE (mode),
960 : GET_MODE_BITSIZE (oldmode)));
961 0 : return force_subreg (mode, x, oldmode, 0);
962 : }
963 :
964 1331348 : temp = gen_reg_rtx (mode);
965 1331348 : convert_move (temp, x, unsignedp);
966 1331348 : return temp;
967 : }
968 :
969 : /* Variant of convert_modes for ABI parameter passing/return.
970 : Return an rtx for a value that would result from converting X from
971 : a floating point mode FMODE to wider integer mode MODE. */
972 :
973 : rtx
974 0 : convert_float_to_wider_int (machine_mode mode, machine_mode fmode, rtx x)
975 : {
976 0 : gcc_assert (SCALAR_INT_MODE_P (mode) && SCALAR_FLOAT_MODE_P (fmode));
977 0 : scalar_int_mode tmp_mode = int_mode_for_mode (fmode).require ();
978 0 : rtx tmp = force_reg (tmp_mode, gen_lowpart (tmp_mode, x));
979 0 : return convert_modes (mode, tmp_mode, tmp, 1);
980 : }
981 :
982 : /* Variant of convert_modes for ABI parameter passing/return.
983 : Return an rtx for a value that would result from converting X from
984 : an integer mode IMODE to a narrower floating point mode MODE. */
985 :
986 : rtx
987 0 : convert_wider_int_to_float (machine_mode mode, machine_mode imode, rtx x)
988 : {
989 0 : gcc_assert (SCALAR_FLOAT_MODE_P (mode) && SCALAR_INT_MODE_P (imode));
990 0 : scalar_int_mode tmp_mode = int_mode_for_mode (mode).require ();
991 0 : rtx tmp = force_reg (tmp_mode, gen_lowpart (tmp_mode, x));
992 0 : return gen_lowpart_SUBREG (mode, tmp);
993 : }
994 :
995 : /* Return the largest alignment we can use for doing a move (or store)
996 : of MAX_PIECES. ALIGN is the largest alignment we could use. */
997 :
998 : static unsigned int
999 2769019 : alignment_for_piecewise_move (unsigned int max_pieces, unsigned int align)
1000 : {
1001 2769019 : scalar_int_mode tmode
1002 2769019 : = int_mode_for_size (max_pieces * BITS_PER_UNIT, 0).require ();
1003 :
1004 2769019 : if (align >= GET_MODE_ALIGNMENT (tmode))
1005 2186052 : align = GET_MODE_ALIGNMENT (tmode);
1006 : else
1007 : {
1008 582967 : scalar_int_mode xmode = NARROWEST_INT_MODE;
1009 582967 : opt_scalar_int_mode mode_iter;
1010 3513863 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
1011 : {
1012 3508203 : tmode = mode_iter.require ();
1013 3508203 : if (GET_MODE_SIZE (tmode) > max_pieces
1014 3508203 : || targetm.slow_unaligned_access (tmode, align))
1015 : break;
1016 2930896 : xmode = tmode;
1017 : }
1018 :
1019 582967 : align = MAX (align, GET_MODE_ALIGNMENT (xmode));
1020 : }
1021 :
1022 2769019 : return align;
1023 : }
1024 :
1025 : /* Return true if we know how to implement OP using vectors of bytes. */
1026 : static bool
1027 5855472 : can_use_qi_vectors (by_pieces_operation op)
1028 : {
1029 5855472 : return (op == COMPARE_BY_PIECES
1030 0 : || op == SET_BY_PIECES
1031 0 : || op == CLEAR_BY_PIECES);
1032 : }
1033 :
1034 : /* Return true if optabs exists for the mode and certain by pieces
1035 : operations. */
1036 : static bool
1037 25937857 : by_pieces_mode_supported_p (fixed_size_mode mode, by_pieces_operation op)
1038 : {
1039 25937857 : if (optab_handler (mov_optab, mode) == CODE_FOR_nothing)
1040 : return false;
1041 :
1042 24579534 : if ((op == SET_BY_PIECES || op == CLEAR_BY_PIECES)
1043 688307 : && VECTOR_MODE_P (mode)
1044 25071823 : && optab_handler (vec_duplicate_optab, mode) == CODE_FOR_nothing)
1045 : return false;
1046 :
1047 24530926 : if (op == COMPARE_BY_PIECES
1048 24530926 : && !can_compare_p (EQ, mode, ccp_jump))
1049 : return false;
1050 :
1051 : return true;
1052 : }
1053 :
1054 : /* Return the widest mode that can be used to perform part of an
1055 : operation OP on SIZE bytes. Try to use QI vector modes where
1056 : possible. */
1057 : static fixed_size_mode
1058 5362533 : widest_fixed_size_mode_for_size (unsigned int size, by_pieces_operation op)
1059 : {
1060 5362533 : fixed_size_mode result = NARROWEST_INT_MODE;
1061 :
1062 5362533 : gcc_checking_assert (size > 1);
1063 :
1064 : /* Use QI vector only if size is wider than a WORD. */
1065 5362533 : if (can_use_qi_vectors (op))
1066 : {
1067 728600 : machine_mode mode;
1068 728600 : fixed_size_mode candidate;
1069 12023471 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_INT)
1070 12023471 : if (is_a<fixed_size_mode> (mode, &candidate)
1071 13129304 : && GET_MODE_SIZE (candidate) > UNITS_PER_WORD
1072 12938214 : && GET_MODE_INNER (candidate) == QImode)
1073 : {
1074 5661740 : if (GET_MODE_SIZE (candidate) >= size)
1075 : break;
1076 2102270 : if (by_pieces_mode_supported_p (candidate, op))
1077 11294871 : result = candidate;
1078 : }
1079 :
1080 728600 : if (result != NARROWEST_INT_MODE)
1081 471065 : return result;
1082 : }
1083 :
1084 4891468 : opt_scalar_int_mode tmode;
1085 4891468 : scalar_int_mode mode;
1086 39131744 : FOR_EACH_MODE_IN_CLASS (tmode, MODE_INT)
1087 : {
1088 34240276 : mode = tmode.require ();
1089 34240276 : if (GET_MODE_SIZE (mode) < size
1090 34240276 : && by_pieces_mode_supported_p (mode, op))
1091 23794006 : result = mode;
1092 : }
1093 :
1094 4891468 : return result;
1095 : }
1096 :
1097 : /* Determine whether an operation OP on LEN bytes with alignment ALIGN can
1098 : and should be performed piecewise. */
1099 :
1100 : static bool
1101 967087 : can_do_by_pieces (unsigned HOST_WIDE_INT len, unsigned int align,
1102 : enum by_pieces_operation op)
1103 : {
1104 967087 : return targetm.use_by_pieces_infrastructure_p (len, align, op,
1105 967087 : optimize_insn_for_speed_p ());
1106 : }
1107 :
1108 : /* Determine whether the LEN bytes can be moved by using several move
1109 : instructions. Return nonzero if a call to move_by_pieces should
1110 : succeed. */
1111 :
1112 : bool
1113 897616 : can_move_by_pieces (unsigned HOST_WIDE_INT len, unsigned int align)
1114 : {
1115 897616 : return can_do_by_pieces (len, align, MOVE_BY_PIECES);
1116 : }
1117 :
1118 : /* Return number of insns required to perform operation OP by pieces
1119 : for L bytes. ALIGN (in bits) is maximum alignment we can assume. */
1120 :
1121 : unsigned HOST_WIDE_INT
1122 1958051 : by_pieces_ninsns (unsigned HOST_WIDE_INT l, unsigned int align,
1123 : unsigned int max_size, by_pieces_operation op)
1124 : {
1125 1958051 : unsigned HOST_WIDE_INT n_insns = 0;
1126 1958051 : fixed_size_mode mode;
1127 :
1128 1958051 : if (targetm.overlap_op_by_pieces_p ())
1129 : {
1130 : /* NB: Round up L and ALIGN to the widest integer mode for
1131 : MAX_SIZE. */
1132 1958051 : mode = widest_fixed_size_mode_for_size (max_size, op);
1133 1958051 : gcc_assert (optab_handler (mov_optab, mode) != CODE_FOR_nothing);
1134 3916102 : unsigned HOST_WIDE_INT up = ROUND_UP (l, GET_MODE_SIZE (mode));
1135 1958051 : if (up > l)
1136 : l = up;
1137 1958051 : align = GET_MODE_ALIGNMENT (mode);
1138 : }
1139 :
1140 1958051 : align = alignment_for_piecewise_move (MOVE_MAX_PIECES, align);
1141 :
1142 5882141 : while (max_size > 1 && l > 0)
1143 : {
1144 1966039 : mode = widest_fixed_size_mode_for_size (max_size, op);
1145 1966039 : gcc_assert (optab_handler (mov_optab, mode) != CODE_FOR_nothing);
1146 :
1147 1966039 : unsigned int modesize = GET_MODE_SIZE (mode);
1148 :
1149 1966039 : if (align >= GET_MODE_ALIGNMENT (mode))
1150 : {
1151 1966039 : unsigned HOST_WIDE_INT n_pieces = l / modesize;
1152 1966039 : l %= modesize;
1153 1966039 : switch (op)
1154 : {
1155 1896568 : default:
1156 1896568 : n_insns += n_pieces;
1157 1896568 : break;
1158 :
1159 69471 : case COMPARE_BY_PIECES:
1160 69471 : int batch = targetm.compare_by_pieces_branch_ratio (mode);
1161 69471 : int batch_ops = 4 * batch - 1;
1162 69471 : unsigned HOST_WIDE_INT full = n_pieces / batch;
1163 69471 : n_insns += full * batch_ops;
1164 69471 : if (n_pieces % batch != 0)
1165 0 : n_insns++;
1166 : break;
1167 :
1168 : }
1169 : }
1170 : max_size = modesize;
1171 : }
1172 :
1173 1958051 : gcc_assert (!l);
1174 1958051 : return n_insns;
1175 : }
1176 :
1177 : /* Used when performing piecewise block operations, holds information
1178 : about one of the memory objects involved. The member functions
1179 : can be used to generate code for loading from the object and
1180 : updating the address when iterating. */
1181 :
1182 : class pieces_addr
1183 : {
1184 : /* The object being referenced, a MEM. Can be NULL_RTX to indicate
1185 : stack pushes. */
1186 : rtx m_obj;
1187 : /* The address of the object. Can differ from that seen in the
1188 : MEM rtx if we copied the address to a register. */
1189 : rtx m_addr;
1190 : /* Nonzero if the address on the object has an autoincrement already,
1191 : signifies whether that was an increment or decrement. */
1192 : signed char m_addr_inc;
1193 : /* Nonzero if we intend to use autoinc without the address already
1194 : having autoinc form. We will insert add insns around each memory
1195 : reference, expecting later passes to form autoinc addressing modes.
1196 : The only supported options are predecrement and postincrement. */
1197 : signed char m_explicit_inc;
1198 : /* True if we have either of the two possible cases of using
1199 : autoincrement. */
1200 : bool m_auto;
1201 : /* True if this is an address to be used for load operations rather
1202 : than stores. */
1203 : bool m_is_load;
1204 :
1205 : /* Optionally, a function to obtain constants for any given offset into
1206 : the objects, and data associated with it. */
1207 : by_pieces_constfn m_constfn;
1208 : void *m_cfndata;
1209 : public:
1210 : pieces_addr (rtx, bool, by_pieces_constfn, void *);
1211 : rtx adjust (fixed_size_mode, HOST_WIDE_INT, by_pieces_prev * = nullptr);
1212 : void increment_address (HOST_WIDE_INT);
1213 : void maybe_predec (HOST_WIDE_INT);
1214 : void maybe_postinc (HOST_WIDE_INT);
1215 : void decide_autoinc (machine_mode, bool, HOST_WIDE_INT);
1216 757492 : int get_addr_inc ()
1217 : {
1218 757492 : return m_addr_inc;
1219 : }
1220 : };
1221 :
1222 : /* Initialize a pieces_addr structure from an object OBJ. IS_LOAD is
1223 : true if the operation to be performed on this object is a load
1224 : rather than a store. For stores, OBJ can be NULL, in which case we
1225 : assume the operation is a stack push. For loads, the optional
1226 : CONSTFN and its associated CFNDATA can be used in place of the
1227 : memory load. */
1228 :
1229 1514984 : pieces_addr::pieces_addr (rtx obj, bool is_load, by_pieces_constfn constfn,
1230 1514984 : void *cfndata)
1231 1514984 : : m_obj (obj), m_is_load (is_load), m_constfn (constfn), m_cfndata (cfndata)
1232 : {
1233 1514984 : m_addr_inc = 0;
1234 1514984 : m_auto = false;
1235 1514984 : if (obj)
1236 : {
1237 1390470 : rtx addr = XEXP (obj, 0);
1238 1390470 : rtx_code code = GET_CODE (addr);
1239 1390470 : m_addr = addr;
1240 1390470 : bool dec = code == PRE_DEC || code == POST_DEC;
1241 1390470 : bool inc = code == PRE_INC || code == POST_INC;
1242 1390470 : m_auto = inc || dec;
1243 1390470 : if (m_auto)
1244 0 : m_addr_inc = dec ? -1 : 1;
1245 :
1246 : /* While we have always looked for these codes here, the code
1247 : implementing the memory operation has never handled them.
1248 : Support could be added later if necessary or beneficial. */
1249 1390470 : gcc_assert (code != PRE_INC && code != POST_DEC);
1250 : }
1251 : else
1252 : {
1253 124514 : m_addr = NULL_RTX;
1254 124514 : if (!is_load)
1255 : {
1256 45 : m_auto = true;
1257 45 : if (STACK_GROWS_DOWNWARD)
1258 45 : m_addr_inc = -1;
1259 : else
1260 : m_addr_inc = 1;
1261 : }
1262 : else
1263 124469 : gcc_assert (constfn != NULL);
1264 : }
1265 1514984 : m_explicit_inc = 0;
1266 1514984 : if (constfn)
1267 150568 : gcc_assert (is_load);
1268 1514984 : }
1269 :
1270 : /* Decide whether to use autoinc for an address involved in a memory op.
1271 : MODE is the mode of the accesses, REVERSE is true if we've decided to
1272 : perform the operation starting from the end, and LEN is the length of
1273 : the operation. Don't override an earlier decision to set m_auto. */
1274 :
1275 : void
1276 375228 : pieces_addr::decide_autoinc (machine_mode ARG_UNUSED (mode), bool reverse,
1277 : HOST_WIDE_INT len)
1278 : {
1279 375228 : if (m_auto || m_obj == NULL_RTX)
1280 : return;
1281 :
1282 344303 : bool use_predec = (m_is_load
1283 : ? USE_LOAD_PRE_DECREMENT (mode)
1284 : : USE_STORE_PRE_DECREMENT (mode));
1285 344303 : bool use_postinc = (m_is_load
1286 : ? USE_LOAD_POST_INCREMENT (mode)
1287 : : USE_STORE_POST_INCREMENT (mode));
1288 344303 : machine_mode addr_mode = get_address_mode (m_obj);
1289 :
1290 344303 : if (use_predec && reverse)
1291 : {
1292 : m_addr = copy_to_mode_reg (addr_mode,
1293 : plus_constant (addr_mode,
1294 : m_addr, len));
1295 : m_auto = true;
1296 : m_explicit_inc = -1;
1297 : }
1298 344303 : else if (use_postinc && !reverse)
1299 : {
1300 : m_addr = copy_to_mode_reg (addr_mode, m_addr);
1301 : m_auto = true;
1302 : m_explicit_inc = 1;
1303 : }
1304 344303 : else if (CONSTANT_P (m_addr))
1305 71838 : m_addr = copy_to_mode_reg (addr_mode, m_addr);
1306 : }
1307 :
1308 : /* Adjust the address to refer to the data at OFFSET in MODE. If we
1309 : are using autoincrement for this address, we don't add the offset,
1310 : but we still modify the MEM's properties. */
1311 :
1312 : rtx
1313 4130343 : pieces_addr::adjust (fixed_size_mode mode, HOST_WIDE_INT offset,
1314 : by_pieces_prev *prev)
1315 : {
1316 4130343 : if (m_constfn)
1317 : /* Pass the previous data to m_constfn. */
1318 368078 : return m_constfn (m_cfndata, prev, offset, mode);
1319 3762265 : if (m_obj == NULL_RTX)
1320 : return NULL_RTX;
1321 3762218 : if (m_auto)
1322 0 : return adjust_automodify_address (m_obj, mode, m_addr, offset);
1323 : else
1324 3762218 : return adjust_address (m_obj, mode, offset);
1325 : }
1326 :
1327 : /* Emit an add instruction to increment the address by SIZE. */
1328 :
1329 : void
1330 0 : pieces_addr::increment_address (HOST_WIDE_INT size)
1331 : {
1332 0 : rtx amount = gen_int_mode (size, GET_MODE (m_addr));
1333 0 : emit_insn (gen_add2_insn (m_addr, amount));
1334 0 : }
1335 :
1336 : /* If we are supposed to decrement the address after each access, emit code
1337 : to do so now. Increment by SIZE (which has should have the correct sign
1338 : already). */
1339 :
1340 : void
1341 4129696 : pieces_addr::maybe_predec (HOST_WIDE_INT size)
1342 : {
1343 4129696 : if (m_explicit_inc >= 0)
1344 4129696 : return;
1345 0 : gcc_assert (HAVE_PRE_DECREMENT);
1346 : increment_address (size);
1347 : }
1348 :
1349 : /* If we are supposed to decrement the address after each access, emit code
1350 : to do so now. Increment by SIZE. */
1351 :
1352 : void
1353 4129719 : pieces_addr::maybe_postinc (HOST_WIDE_INT size)
1354 : {
1355 4129719 : if (m_explicit_inc <= 0)
1356 4129719 : return;
1357 0 : gcc_assert (HAVE_POST_INCREMENT);
1358 : increment_address (size);
1359 : }
1360 :
1361 : /* This structure is used by do_op_by_pieces to describe the operation
1362 : to be performed. */
1363 :
1364 : class op_by_pieces_d
1365 : {
1366 : private:
1367 : fixed_size_mode get_usable_mode (fixed_size_mode, unsigned int);
1368 : fixed_size_mode smallest_fixed_size_mode_for_size (unsigned int);
1369 :
1370 : protected:
1371 : pieces_addr m_to, m_from;
1372 : /* Make m_len read-only so that smallest_fixed_size_mode_for_size can
1373 : use it to check the valid mode size. */
1374 : const unsigned HOST_WIDE_INT m_len;
1375 : HOST_WIDE_INT m_offset;
1376 : unsigned int m_align;
1377 : unsigned int m_max_size;
1378 : bool m_reverse;
1379 : /* True if this is a stack push. */
1380 : bool m_push;
1381 : /* True if targetm.overlap_op_by_pieces_p () returns true. */
1382 : bool m_overlap_op_by_pieces;
1383 : /* The type of operation that we're performing. */
1384 : by_pieces_operation m_op;
1385 :
1386 : /* Virtual functions, overridden by derived classes for the specific
1387 : operation. */
1388 : virtual void generate (rtx, rtx, machine_mode) = 0;
1389 : virtual bool prepare_mode (machine_mode, unsigned int) = 0;
1390 1149741 : virtual void finish_mode (machine_mode)
1391 : {
1392 1149741 : }
1393 :
1394 : public:
1395 : op_by_pieces_d (unsigned int, rtx, bool, rtx, bool, by_pieces_constfn,
1396 : void *, unsigned HOST_WIDE_INT, unsigned int, bool,
1397 : by_pieces_operation);
1398 : void run ();
1399 : };
1400 :
1401 : /* The constructor for an op_by_pieces_d structure. We require two
1402 : objects named TO and FROM, which are identified as loads or stores
1403 : by TO_LOAD and FROM_LOAD. If FROM is a load, the optional FROM_CFN
1404 : and its associated FROM_CFN_DATA can be used to replace loads with
1405 : constant values. MAX_PIECES describes the maximum number of bytes
1406 : at a time which can be moved efficiently. LEN describes the length
1407 : of the operation. */
1408 :
1409 757492 : op_by_pieces_d::op_by_pieces_d (unsigned int max_pieces, rtx to,
1410 : bool to_load, rtx from, bool from_load,
1411 : by_pieces_constfn from_cfn,
1412 : void *from_cfn_data,
1413 : unsigned HOST_WIDE_INT len,
1414 : unsigned int align, bool push,
1415 757492 : by_pieces_operation op)
1416 757492 : : m_to (to, to_load, NULL, NULL),
1417 757492 : m_from (from, from_load, from_cfn, from_cfn_data),
1418 757492 : m_len (len), m_max_size (max_pieces + 1),
1419 757492 : m_push (push), m_op (op)
1420 : {
1421 757492 : int toi = m_to.get_addr_inc ();
1422 757492 : int fromi = m_from.get_addr_inc ();
1423 757492 : if (toi >= 0 && fromi >= 0)
1424 757447 : m_reverse = false;
1425 45 : else if (toi <= 0 && fromi <= 0)
1426 45 : m_reverse = true;
1427 : else
1428 0 : gcc_unreachable ();
1429 :
1430 757492 : m_offset = m_reverse ? len : 0;
1431 2780995 : align = MIN (to ? MEM_ALIGN (to) : align,
1432 : from ? MEM_ALIGN (from) : align);
1433 :
1434 : /* If copying requires more than two move insns,
1435 : copy addresses to registers (to make displacements shorter)
1436 : and use post-increment if available. */
1437 757492 : if (by_pieces_ninsns (len, align, m_max_size, MOVE_BY_PIECES) > 2)
1438 : {
1439 : /* Find the mode of the largest comparison. */
1440 187614 : fixed_size_mode mode
1441 187614 : = widest_fixed_size_mode_for_size (m_max_size, m_op);
1442 :
1443 187614 : m_from.decide_autoinc (mode, m_reverse, len);
1444 187614 : m_to.decide_autoinc (mode, m_reverse, len);
1445 : }
1446 :
1447 757502 : align = alignment_for_piecewise_move (MOVE_MAX_PIECES, align);
1448 757492 : m_align = align;
1449 :
1450 757492 : m_overlap_op_by_pieces = targetm.overlap_op_by_pieces_p ();
1451 757492 : }
1452 :
1453 : /* This function returns the largest usable integer mode for LEN bytes
1454 : whose size is no bigger than size of MODE. */
1455 :
1456 : fixed_size_mode
1457 1244242 : op_by_pieces_d::get_usable_mode (fixed_size_mode mode, unsigned int len)
1458 : {
1459 1543486 : unsigned int size;
1460 1842730 : do
1461 : {
1462 1543486 : size = GET_MODE_SIZE (mode);
1463 1543486 : if (len >= size && prepare_mode (mode, m_align))
1464 : break;
1465 : /* widest_fixed_size_mode_for_size checks SIZE > 1. */
1466 299244 : mode = widest_fixed_size_mode_for_size (size, m_op);
1467 : }
1468 : while (1);
1469 1244242 : return mode;
1470 : }
1471 :
1472 : /* Return the smallest integer or QI vector mode that is not narrower
1473 : than SIZE bytes. */
1474 :
1475 : fixed_size_mode
1476 492939 : op_by_pieces_d::smallest_fixed_size_mode_for_size (unsigned int size)
1477 : {
1478 : /* Use QI vector only for > size of WORD. */
1479 505166 : if (can_use_qi_vectors (m_op) && size > UNITS_PER_WORD)
1480 : {
1481 8177 : machine_mode mode;
1482 8177 : fixed_size_mode candidate;
1483 110621 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_INT)
1484 110621 : if (is_a<fixed_size_mode> (mode, &candidate)
1485 221242 : && GET_MODE_INNER (candidate) == QImode)
1486 : {
1487 : /* Don't return a mode wider than M_LEN. */
1488 102870 : if (GET_MODE_SIZE (candidate) > m_len)
1489 : break;
1490 :
1491 49522 : if (GET_MODE_SIZE (candidate) >= size
1492 49522 : && by_pieces_mode_supported_p (candidate, m_op))
1493 6264 : return candidate;
1494 : }
1495 : }
1496 :
1497 486675 : return smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
1498 : }
1499 :
1500 : /* This function contains the main loop used for expanding a block
1501 : operation. First move what we can in the largest integer mode,
1502 : then go to successively smaller modes. For every access, call
1503 : GENFUN with the two operands and the EXTRA_DATA. */
1504 :
1505 : void
1506 757492 : op_by_pieces_d::run ()
1507 : {
1508 757492 : if (m_len == 0)
1509 : return;
1510 :
1511 751303 : unsigned HOST_WIDE_INT length = m_len;
1512 :
1513 : /* widest_fixed_size_mode_for_size checks M_MAX_SIZE > 1. */
1514 751303 : fixed_size_mode mode
1515 751303 : = widest_fixed_size_mode_for_size (m_max_size, m_op);
1516 751303 : mode = get_usable_mode (mode, length);
1517 :
1518 751303 : by_pieces_prev to_prev = { nullptr, mode };
1519 751303 : by_pieces_prev from_prev = { nullptr, mode };
1520 :
1521 1244242 : do
1522 : {
1523 1244242 : unsigned int size = GET_MODE_SIZE (mode);
1524 1244242 : rtx to1 = NULL_RTX, from1;
1525 :
1526 3309090 : while (length >= size)
1527 : {
1528 2064848 : if (m_reverse)
1529 47 : m_offset -= size;
1530 :
1531 2064848 : to1 = m_to.adjust (mode, m_offset, &to_prev);
1532 2064848 : to_prev.data = to1;
1533 2064848 : to_prev.mode = mode;
1534 2064848 : from1 = m_from.adjust (mode, m_offset, &from_prev);
1535 2064848 : from_prev.data = from1;
1536 2064848 : from_prev.mode = mode;
1537 :
1538 2064848 : m_to.maybe_predec (-(HOST_WIDE_INT)size);
1539 2064848 : m_from.maybe_predec (-(HOST_WIDE_INT)size);
1540 :
1541 2064848 : generate (to1, from1, mode);
1542 :
1543 2064848 : m_to.maybe_postinc (size);
1544 2064848 : m_from.maybe_postinc (size);
1545 :
1546 2064848 : if (!m_reverse)
1547 2064801 : m_offset += size;
1548 :
1549 2064848 : length -= size;
1550 : }
1551 :
1552 1244242 : finish_mode (mode);
1553 :
1554 1244242 : if (length == 0)
1555 : return;
1556 :
1557 492939 : if (!m_push && m_overlap_op_by_pieces)
1558 : {
1559 : /* NB: Generate overlapping operations if it is not a stack
1560 : push since stack push must not overlap. Get the smallest
1561 : fixed size mode for M_LEN bytes. */
1562 492939 : mode = smallest_fixed_size_mode_for_size (length);
1563 985878 : mode = get_usable_mode (mode, GET_MODE_SIZE (mode));
1564 492939 : int gap = GET_MODE_SIZE (mode) - length;
1565 492939 : if (gap > 0)
1566 : {
1567 : /* If size of MODE > M_LEN, generate the last operation
1568 : in MODE for the remaining bytes with overlapping memory
1569 : from the previois operation. */
1570 48985 : if (m_reverse)
1571 0 : m_offset += gap;
1572 : else
1573 48985 : m_offset -= gap;
1574 48985 : length += gap;
1575 : }
1576 : }
1577 : else
1578 : {
1579 : /* widest_fixed_size_mode_for_size checks SIZE > 1. */
1580 0 : mode = widest_fixed_size_mode_for_size (size, m_op);
1581 0 : mode = get_usable_mode (mode, length);
1582 : }
1583 : }
1584 : while (1);
1585 : }
1586 :
1587 : /* Derived class from op_by_pieces_d, providing support for block move
1588 : operations. */
1589 :
1590 : #ifdef PUSH_ROUNDING
1591 : #define PUSHG_P(to) ((to) == nullptr)
1592 : #else
1593 : #define PUSHG_P(to) false
1594 : #endif
1595 :
1596 : class move_by_pieces_d : public op_by_pieces_d
1597 : {
1598 : insn_gen_fn m_gen_fun;
1599 : void generate (rtx, rtx, machine_mode) final override;
1600 : bool prepare_mode (machine_mode, unsigned int) final override;
1601 :
1602 : public:
1603 570323 : move_by_pieces_d (rtx to, rtx from, unsigned HOST_WIDE_INT len,
1604 : unsigned int align)
1605 570323 : : op_by_pieces_d (MOVE_MAX_PIECES, to, false, from, true, NULL,
1606 1140646 : NULL, len, align, PUSHG_P (to), MOVE_BY_PIECES)
1607 : {
1608 570323 : }
1609 : rtx finish_retmode (memop_ret);
1610 : };
1611 :
1612 : /* Return true if MODE can be used for a set of copies, given an
1613 : alignment ALIGN. Prepare whatever data is necessary for later
1614 : calls to generate. */
1615 :
1616 : bool
1617 951292 : move_by_pieces_d::prepare_mode (machine_mode mode, unsigned int align)
1618 : {
1619 951292 : insn_code icode = optab_handler (mov_optab, mode);
1620 951292 : m_gen_fun = GEN_FCN (icode);
1621 951292 : return icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode);
1622 : }
1623 :
1624 : /* A callback used when iterating for a compare_by_pieces_operation.
1625 : OP0 and OP1 are the values that have been loaded and should be
1626 : compared in MODE. If OP0 is NULL, this means we should generate a
1627 : push; otherwise EXTRA_DATA holds a pointer to a pointer to the insn
1628 : gen function that should be used to generate the mode. */
1629 :
1630 : void
1631 1651255 : move_by_pieces_d::generate (rtx op0, rtx op1,
1632 : machine_mode mode ATTRIBUTE_UNUSED)
1633 : {
1634 : #ifdef PUSH_ROUNDING
1635 1651255 : if (op0 == NULL_RTX)
1636 : {
1637 47 : emit_single_push_insn (mode, op1, NULL);
1638 47 : return;
1639 : }
1640 : #endif
1641 1651208 : emit_insn (m_gen_fun (op0, op1));
1642 : }
1643 :
1644 : /* Perform the final adjustment at the end of a string to obtain the
1645 : correct return value for the block operation.
1646 : Return value is based on RETMODE argument. */
1647 :
1648 : rtx
1649 0 : move_by_pieces_d::finish_retmode (memop_ret retmode)
1650 : {
1651 0 : gcc_assert (!m_reverse);
1652 0 : if (retmode == RETURN_END_MINUS_ONE)
1653 : {
1654 0 : m_to.maybe_postinc (-1);
1655 0 : --m_offset;
1656 : }
1657 0 : return m_to.adjust (QImode, m_offset);
1658 : }
1659 :
1660 : /* Generate several move instructions to copy LEN bytes from block FROM to
1661 : block TO. (These are MEM rtx's with BLKmode).
1662 :
1663 : If PUSH_ROUNDING is defined and TO is NULL, emit_single_push_insn is
1664 : used to push FROM to the stack.
1665 :
1666 : ALIGN is maximum stack alignment we can assume.
1667 :
1668 : Return value is based on RETMODE argument. */
1669 :
1670 : rtx
1671 570323 : move_by_pieces (rtx to, rtx from, unsigned HOST_WIDE_INT len,
1672 : unsigned int align, memop_ret retmode)
1673 : {
1674 : #ifndef PUSH_ROUNDING
1675 : if (to == NULL)
1676 : gcc_unreachable ();
1677 : #endif
1678 :
1679 570323 : move_by_pieces_d data (to, from, len, align);
1680 :
1681 570323 : data.run ();
1682 :
1683 570323 : if (retmode != RETURN_BEGIN)
1684 0 : return data.finish_retmode (retmode);
1685 : else
1686 : return to;
1687 : }
1688 :
1689 : /* Derived class from op_by_pieces_d, providing support for block move
1690 : operations. */
1691 :
1692 : class store_by_pieces_d : public op_by_pieces_d
1693 : {
1694 : insn_gen_fn m_gen_fun;
1695 :
1696 : void generate (rtx, rtx, machine_mode) final override;
1697 : bool prepare_mode (machine_mode, unsigned int) final override;
1698 :
1699 : public:
1700 124469 : store_by_pieces_d (rtx to, by_pieces_constfn cfn, void *cfn_data,
1701 : unsigned HOST_WIDE_INT len, unsigned int align,
1702 : by_pieces_operation op)
1703 124469 : : op_by_pieces_d (STORE_MAX_PIECES, to, false, NULL_RTX, true, cfn,
1704 248938 : cfn_data, len, align, false, op)
1705 : {
1706 124469 : }
1707 : rtx finish_retmode (memop_ret);
1708 : };
1709 :
1710 : /* Return true if MODE can be used for a set of stores, given an
1711 : alignment ALIGN. Prepare whatever data is necessary for later
1712 : calls to generate. */
1713 :
1714 : bool
1715 198449 : store_by_pieces_d::prepare_mode (machine_mode mode, unsigned int align)
1716 : {
1717 198449 : insn_code icode = optab_handler (mov_optab, mode);
1718 198449 : m_gen_fun = GEN_FCN (icode);
1719 198449 : return icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode);
1720 : }
1721 :
1722 : /* A callback used when iterating for a store_by_pieces_operation.
1723 : OP0 and OP1 are the values that have been loaded and should be
1724 : compared in MODE. If OP0 is NULL, this means we should generate a
1725 : push; otherwise EXTRA_DATA holds a pointer to a pointer to the insn
1726 : gen function that should be used to generate the mode. */
1727 :
1728 : void
1729 315836 : store_by_pieces_d::generate (rtx op0, rtx op1, machine_mode)
1730 : {
1731 315836 : emit_insn (m_gen_fun (op0, op1));
1732 315836 : }
1733 :
1734 : /* Perform the final adjustment at the end of a string to obtain the
1735 : correct return value for the block operation.
1736 : Return value is based on RETMODE argument. */
1737 :
1738 : rtx
1739 647 : store_by_pieces_d::finish_retmode (memop_ret retmode)
1740 : {
1741 647 : gcc_assert (!m_reverse);
1742 647 : if (retmode == RETURN_END_MINUS_ONE)
1743 : {
1744 23 : m_to.maybe_postinc (-1);
1745 23 : --m_offset;
1746 : }
1747 647 : return m_to.adjust (QImode, m_offset);
1748 : }
1749 :
1750 : /* Determine whether the LEN bytes generated by CONSTFUN can be
1751 : stored to memory using several move instructions. CONSTFUNDATA is
1752 : a pointer which will be passed as argument in every CONSTFUN call.
1753 : ALIGN is maximum alignment we can assume. MEMSETP is true if this is
1754 : a memset operation and false if it's a copy of a constant string.
1755 : Return true if a call to store_by_pieces should succeed. */
1756 :
1757 : bool
1758 90286 : can_store_by_pieces (unsigned HOST_WIDE_INT len,
1759 : by_pieces_constfn constfun,
1760 : void *constfundata, unsigned int align, bool memsetp)
1761 : {
1762 90286 : unsigned HOST_WIDE_INT l;
1763 90286 : unsigned int max_size;
1764 90286 : HOST_WIDE_INT offset = 0;
1765 90286 : enum insn_code icode;
1766 90286 : int reverse;
1767 : /* cst is set but not used if LEGITIMATE_CONSTANT doesn't use it. */
1768 90286 : rtx cst ATTRIBUTE_UNUSED;
1769 :
1770 90286 : if (len == 0)
1771 : return true;
1772 :
1773 136386 : if (!targetm.use_by_pieces_infrastructure_p (len, align,
1774 : memsetp
1775 : ? SET_BY_PIECES
1776 : : STORE_BY_PIECES,
1777 90232 : optimize_insn_for_speed_p ()))
1778 : return false;
1779 :
1780 53476 : align = alignment_for_piecewise_move (STORE_MAX_PIECES, align);
1781 :
1782 : /* We would first store what we can in the largest integer mode, then go to
1783 : successively smaller modes. */
1784 :
1785 53476 : for (reverse = 0;
1786 106952 : reverse <= (HAVE_PRE_DECREMENT || HAVE_POST_DECREMENT);
1787 : reverse++)
1788 : {
1789 53476 : l = len;
1790 53476 : max_size = STORE_MAX_PIECES + 1;
1791 253758 : while (max_size > 1 && l > 0)
1792 : {
1793 200282 : auto op = memsetp ? SET_BY_PIECES : STORE_BY_PIECES;
1794 200282 : auto mode = widest_fixed_size_mode_for_size (max_size, op);
1795 :
1796 200282 : icode = optab_handler (mov_optab, mode);
1797 200282 : if (icode != CODE_FOR_nothing
1798 200282 : && align >= GET_MODE_ALIGNMENT (mode))
1799 : {
1800 200282 : unsigned int size = GET_MODE_SIZE (mode);
1801 :
1802 346238 : while (l >= size)
1803 : {
1804 145956 : if (reverse)
1805 : offset -= size;
1806 :
1807 145956 : cst = (*constfun) (constfundata, nullptr, offset, mode);
1808 : /* All CONST_VECTORs can be loaded for memset since
1809 : vec_duplicate_optab is a precondition to pick a
1810 : vector mode for the memset expander. */
1811 271280 : if (!((memsetp && VECTOR_MODE_P (mode))
1812 125324 : || targetm.legitimate_constant_p (mode, cst)))
1813 36756 : return false;
1814 :
1815 145956 : if (!reverse)
1816 145956 : offset += size;
1817 :
1818 145956 : l -= size;
1819 : }
1820 : }
1821 :
1822 400564 : max_size = GET_MODE_SIZE (mode);
1823 : }
1824 :
1825 : /* The code above should have handled everything. */
1826 53476 : gcc_assert (!l);
1827 : }
1828 :
1829 : return true;
1830 : }
1831 :
1832 : /* Generate several move instructions to store LEN bytes generated by
1833 : CONSTFUN to block TO. (A MEM rtx with BLKmode). CONSTFUNDATA is a
1834 : pointer which will be passed as argument in every CONSTFUN call.
1835 : ALIGN is maximum alignment we can assume. MEMSETP is true if this is
1836 : a memset operation and false if it's a copy of a constant string.
1837 : Return value is based on RETMODE argument. */
1838 :
1839 : rtx
1840 60772 : store_by_pieces (rtx to, unsigned HOST_WIDE_INT len,
1841 : by_pieces_constfn constfun,
1842 : void *constfundata, unsigned int align, bool memsetp,
1843 : memop_ret retmode)
1844 : {
1845 60772 : if (len == 0)
1846 : {
1847 7887 : gcc_assert (retmode != RETURN_END_MINUS_ONE);
1848 : return to;
1849 : }
1850 :
1851 94986 : gcc_assert (targetm.use_by_pieces_infrastructure_p
1852 : (len, align,
1853 : memsetp ? SET_BY_PIECES : STORE_BY_PIECES,
1854 : optimize_insn_for_speed_p ()));
1855 :
1856 52885 : store_by_pieces_d data (to, constfun, constfundata, len, align,
1857 52885 : memsetp ? SET_BY_PIECES : STORE_BY_PIECES);
1858 52885 : data.run ();
1859 :
1860 52885 : if (retmode != RETURN_BEGIN)
1861 647 : return data.finish_retmode (retmode);
1862 : else
1863 : return to;
1864 : }
1865 :
1866 : void
1867 71584 : clear_by_pieces (rtx to, unsigned HOST_WIDE_INT len, unsigned int align)
1868 : {
1869 71584 : if (len == 0)
1870 0 : return;
1871 :
1872 : /* Use builtin_memset_read_str to support vector mode broadcast. */
1873 71584 : char c = 0;
1874 71584 : store_by_pieces_d data (to, builtin_memset_read_str, &c, len, align,
1875 71584 : CLEAR_BY_PIECES);
1876 71584 : data.run ();
1877 : }
1878 :
1879 : /* Context used by compare_by_pieces_genfn. It stores the fail label
1880 : to jump to in case of miscomparison, and for branch ratios greater than 1,
1881 : it stores an accumulator and the current and maximum counts before
1882 : emitting another branch. */
1883 :
1884 : class compare_by_pieces_d : public op_by_pieces_d
1885 : {
1886 : rtx_code_label *m_fail_label;
1887 : rtx m_accumulator;
1888 : int m_count, m_batch;
1889 :
1890 : void generate (rtx, rtx, machine_mode) final override;
1891 : bool prepare_mode (machine_mode, unsigned int) final override;
1892 : void finish_mode (machine_mode) final override;
1893 :
1894 : public:
1895 62700 : compare_by_pieces_d (rtx op0, rtx op1, by_pieces_constfn op1_cfn,
1896 : void *op1_cfn_data, HOST_WIDE_INT len, int align,
1897 : rtx_code_label *fail_label)
1898 62700 : : op_by_pieces_d (COMPARE_MAX_PIECES, op0, true, op1, true, op1_cfn,
1899 125400 : op1_cfn_data, len, align, false, COMPARE_BY_PIECES)
1900 : {
1901 62700 : m_fail_label = fail_label;
1902 62700 : }
1903 : };
1904 :
1905 : /* A callback used when iterating for a compare_by_pieces_operation.
1906 : OP0 and OP1 are the values that have been loaded and should be
1907 : compared in MODE. DATA holds a pointer to the compare_by_pieces_data
1908 : context structure. */
1909 :
1910 : void
1911 97757 : compare_by_pieces_d::generate (rtx op0, rtx op1, machine_mode mode)
1912 : {
1913 97757 : if (m_batch > 1)
1914 : {
1915 0 : rtx temp = expand_binop (mode, sub_optab, op0, op1, NULL_RTX,
1916 : true, OPTAB_LIB_WIDEN);
1917 0 : if (m_count != 0)
1918 0 : temp = expand_binop (mode, ior_optab, m_accumulator, temp, temp,
1919 : true, OPTAB_LIB_WIDEN);
1920 0 : m_accumulator = temp;
1921 :
1922 0 : if (++m_count < m_batch)
1923 : return;
1924 :
1925 0 : m_count = 0;
1926 0 : op0 = m_accumulator;
1927 0 : op1 = const0_rtx;
1928 0 : m_accumulator = NULL_RTX;
1929 : }
1930 97757 : do_compare_rtx_and_jump (op0, op1, NE, true, mode, NULL_RTX, NULL,
1931 : m_fail_label, profile_probability::uninitialized ());
1932 : }
1933 :
1934 : /* Return true if MODE can be used for a set of moves and comparisons,
1935 : given an alignment ALIGN. Prepare whatever data is necessary for
1936 : later calls to generate. */
1937 :
1938 : bool
1939 94501 : compare_by_pieces_d::prepare_mode (machine_mode mode, unsigned int align)
1940 : {
1941 94501 : insn_code icode = optab_handler (mov_optab, mode);
1942 94501 : if (icode == CODE_FOR_nothing
1943 94501 : || align < GET_MODE_ALIGNMENT (mode)
1944 189002 : || !can_compare_p (EQ, mode, ccp_jump))
1945 0 : return false;
1946 94501 : m_batch = targetm.compare_by_pieces_branch_ratio (mode);
1947 94501 : if (m_batch < 0)
1948 : return false;
1949 94501 : m_accumulator = NULL_RTX;
1950 94501 : m_count = 0;
1951 94501 : return true;
1952 : }
1953 :
1954 : /* Called after expanding a series of comparisons in MODE. If we have
1955 : accumulated results for which we haven't emitted a branch yet, do
1956 : so now. */
1957 :
1958 : void
1959 94501 : compare_by_pieces_d::finish_mode (machine_mode mode)
1960 : {
1961 94501 : if (m_accumulator != NULL_RTX)
1962 0 : do_compare_rtx_and_jump (m_accumulator, const0_rtx, NE, true, mode,
1963 : NULL_RTX, NULL, m_fail_label,
1964 : profile_probability::uninitialized ());
1965 94501 : }
1966 :
1967 : /* Generate several move instructions to compare LEN bytes from blocks
1968 : ARG0 and ARG1. (These are MEM rtx's with BLKmode).
1969 :
1970 : If PUSH_ROUNDING is defined and TO is NULL, emit_single_push_insn is
1971 : used to push FROM to the stack.
1972 :
1973 : ALIGN is maximum stack alignment we can assume.
1974 :
1975 : Optionally, the caller can pass a constfn and associated data in A1_CFN
1976 : and A1_CFN_DATA. describing that the second operand being compared is a
1977 : known constant and how to obtain its data. */
1978 :
1979 : static rtx
1980 62700 : compare_by_pieces (rtx arg0, rtx arg1, unsigned HOST_WIDE_INT len,
1981 : rtx target, unsigned int align,
1982 : by_pieces_constfn a1_cfn, void *a1_cfn_data)
1983 : {
1984 62700 : rtx_code_label *fail_label = gen_label_rtx ();
1985 62700 : rtx_code_label *end_label = gen_label_rtx ();
1986 :
1987 62700 : if (target == NULL_RTX
1988 62700 : || !REG_P (target) || REGNO (target) < FIRST_PSEUDO_REGISTER)
1989 2 : target = gen_reg_rtx (TYPE_MODE (integer_type_node));
1990 :
1991 62700 : compare_by_pieces_d data (arg0, arg1, a1_cfn, a1_cfn_data, len, align,
1992 62700 : fail_label);
1993 :
1994 62700 : data.run ();
1995 :
1996 62700 : emit_move_insn (target, const0_rtx);
1997 62700 : emit_jump (end_label);
1998 62700 : emit_barrier ();
1999 62700 : emit_label (fail_label);
2000 62700 : emit_move_insn (target, const1_rtx);
2001 62700 : emit_label (end_label);
2002 :
2003 62700 : return target;
2004 : }
2005 :
2006 : /* Emit code to move a block Y to a block X. This may be done with
2007 : string-move instructions, with multiple scalar move instructions,
2008 : or with a library call.
2009 :
2010 : Both X and Y must be MEM rtx's (perhaps inside VOLATILE) with mode BLKmode.
2011 : SIZE is an rtx that says how long they are.
2012 : ALIGN is the maximum alignment we can assume they have.
2013 : METHOD describes what kind of copy this is, and what mechanisms may be used.
2014 : MIN_SIZE is the minimal size of block to move
2015 : MAX_SIZE is the maximal size of block to move, if it cannot be represented
2016 : in unsigned HOST_WIDE_INT, than it is mask of all ones.
2017 : CTZ_SIZE is the trailing-zeros count of SIZE; even a nonconstant SIZE is
2018 : known to be a multiple of 1<<CTZ_SIZE.
2019 :
2020 : Return the address of the new block, if memcpy is called and returns it,
2021 : 0 otherwise. */
2022 :
2023 : rtx
2024 669378 : emit_block_move_hints (rtx x, rtx y, rtx size, enum block_op_methods method,
2025 : unsigned int expected_align, HOST_WIDE_INT expected_size,
2026 : unsigned HOST_WIDE_INT min_size,
2027 : unsigned HOST_WIDE_INT max_size,
2028 : unsigned HOST_WIDE_INT probable_max_size,
2029 : bool bail_out_libcall, bool *is_move_done,
2030 : bool might_overlap, unsigned ctz_size)
2031 : {
2032 669378 : int may_use_call;
2033 669378 : rtx retval = 0;
2034 669378 : unsigned int align;
2035 :
2036 669378 : if (is_move_done)
2037 97879 : *is_move_done = true;
2038 :
2039 669378 : gcc_assert (size);
2040 669378 : if (CONST_INT_P (size) && INTVAL (size) == 0)
2041 : return 0;
2042 :
2043 669101 : switch (method)
2044 : {
2045 : case BLOCK_OP_NORMAL:
2046 : case BLOCK_OP_TAILCALL:
2047 : may_use_call = 1;
2048 : break;
2049 :
2050 269085 : case BLOCK_OP_CALL_PARM:
2051 269085 : may_use_call = block_move_libcall_safe_for_call_parm ();
2052 :
2053 : /* Make inhibit_defer_pop nonzero around the library call
2054 : to force it to pop the arguments right away. */
2055 269085 : NO_DEFER_POP;
2056 269085 : break;
2057 :
2058 354 : case BLOCK_OP_NO_LIBCALL:
2059 354 : may_use_call = 0;
2060 354 : break;
2061 :
2062 1074 : case BLOCK_OP_NO_LIBCALL_RET:
2063 1074 : may_use_call = -1;
2064 1074 : break;
2065 :
2066 0 : default:
2067 0 : gcc_unreachable ();
2068 : }
2069 :
2070 669101 : gcc_assert (MEM_P (x) && MEM_P (y));
2071 669186 : align = MIN (MEM_ALIGN (x), MEM_ALIGN (y));
2072 669101 : gcc_assert (align >= BITS_PER_UNIT);
2073 :
2074 : /* Make sure we've got BLKmode addresses; store_one_arg can decide that
2075 : block copy is more efficient for other large modes, e.g. DCmode. */
2076 669101 : x = adjust_address (x, BLKmode, 0);
2077 669101 : y = adjust_address (y, BLKmode, 0);
2078 :
2079 : /* If source and destination are the same, no need to copy anything. */
2080 669101 : if (rtx_equal_p (x, y)
2081 13 : && !MEM_VOLATILE_P (x)
2082 669114 : && !MEM_VOLATILE_P (y))
2083 : return 0;
2084 :
2085 : /* Set MEM_SIZE as appropriate for this block copy. The main place this
2086 : can be incorrect is coming from __builtin_memcpy. */
2087 669088 : poly_int64 const_size;
2088 669088 : if (poly_int_rtx_p (size, &const_size))
2089 : {
2090 590295 : x = shallow_copy_rtx (x);
2091 590295 : y = shallow_copy_rtx (y);
2092 590295 : set_mem_size (x, const_size);
2093 590295 : set_mem_size (y, const_size);
2094 : }
2095 :
2096 669088 : bool pieces_ok = CONST_INT_P (size)
2097 669088 : && can_move_by_pieces (INTVAL (size), align);
2098 669088 : bool pattern_ok = false;
2099 :
2100 669088 : if (!pieces_ok || might_overlap)
2101 : {
2102 105584 : pattern_ok
2103 105584 : = emit_block_move_via_pattern (x, y, size, align,
2104 : expected_align, expected_size,
2105 : min_size, max_size, probable_max_size,
2106 : might_overlap);
2107 105584 : if (!pattern_ok && might_overlap)
2108 : {
2109 : /* Do not try any of the other methods below as they are not safe
2110 : for overlapping moves. */
2111 12467 : *is_move_done = false;
2112 12467 : return retval;
2113 : }
2114 : }
2115 :
2116 93117 : bool dynamic_direction = false;
2117 93117 : if (!pattern_ok && !pieces_ok && may_use_call
2118 135626 : && (flag_inline_stringops & (might_overlap ? ILSOP_MEMMOVE : ILSOP_MEMCPY)))
2119 : {
2120 656621 : may_use_call = 0;
2121 656621 : dynamic_direction = might_overlap;
2122 : }
2123 :
2124 656621 : if (pattern_ok)
2125 : ;
2126 631317 : else if (pieces_ok)
2127 563504 : move_by_pieces (x, y, INTVAL (size), align, RETURN_BEGIN);
2128 67813 : else if (may_use_call && !might_overlap
2129 67765 : && ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (x))
2130 135578 : && ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (y)))
2131 : {
2132 67765 : if (bail_out_libcall)
2133 : {
2134 259 : if (is_move_done)
2135 259 : *is_move_done = false;
2136 259 : return retval;
2137 : }
2138 :
2139 67506 : if (may_use_call < 0)
2140 0 : return pc_rtx;
2141 :
2142 67506 : retval = emit_block_copy_via_libcall (x, y, size,
2143 : method == BLOCK_OP_TAILCALL);
2144 : }
2145 48 : else if (dynamic_direction)
2146 0 : emit_block_move_via_oriented_loop (x, y, size, align, ctz_size);
2147 48 : else if (might_overlap)
2148 0 : *is_move_done = false;
2149 : else
2150 48 : emit_block_move_via_sized_loop (x, y, size, align, ctz_size);
2151 :
2152 656362 : if (method == BLOCK_OP_CALL_PARM)
2153 269074 : OK_DEFER_POP;
2154 :
2155 : return retval;
2156 : }
2157 :
2158 : rtx
2159 571499 : emit_block_move (rtx x, rtx y, rtx size, enum block_op_methods method,
2160 : unsigned int ctz_size)
2161 : {
2162 571499 : unsigned HOST_WIDE_INT max, min = 0;
2163 571499 : if (GET_CODE (size) == CONST_INT)
2164 571279 : min = max = UINTVAL (size);
2165 : else
2166 220 : max = GET_MODE_MASK (GET_MODE (size));
2167 571499 : return emit_block_move_hints (x, y, size, method, 0, -1,
2168 : min, max, max,
2169 571499 : false, NULL, false, ctz_size);
2170 : }
2171 :
2172 : /* A subroutine of emit_block_move. Returns true if calling the
2173 : block move libcall will not clobber any parameters which may have
2174 : already been placed on the stack. */
2175 :
2176 : static bool
2177 269085 : block_move_libcall_safe_for_call_parm (void)
2178 : {
2179 269085 : tree fn;
2180 :
2181 : /* If arguments are pushed on the stack, then they're safe. */
2182 269085 : if (targetm.calls.push_argument (0))
2183 : return true;
2184 :
2185 : /* If registers go on the stack anyway, any argument is sure to clobber
2186 : an outgoing argument. */
2187 : #if defined (REG_PARM_STACK_SPACE)
2188 3 : fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
2189 : /* Avoid set but not used warning if *REG_PARM_STACK_SPACE doesn't
2190 : depend on its argument. */
2191 3 : (void) fn;
2192 3 : if (OUTGOING_REG_PARM_STACK_SPACE ((!fn ? NULL_TREE : TREE_TYPE (fn)))
2193 3 : && REG_PARM_STACK_SPACE (fn) != 0)
2194 : return false;
2195 : #endif
2196 :
2197 : /* If any argument goes in memory, then it might clobber an outgoing
2198 : argument. */
2199 3 : {
2200 3 : CUMULATIVE_ARGS args_so_far_v;
2201 3 : cumulative_args_t args_so_far;
2202 3 : tree arg;
2203 :
2204 3 : fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
2205 3 : INIT_CUMULATIVE_ARGS (args_so_far_v, TREE_TYPE (fn), NULL_RTX, 0, 3);
2206 3 : args_so_far = pack_cumulative_args (&args_so_far_v);
2207 :
2208 3 : arg = TYPE_ARG_TYPES (TREE_TYPE (fn));
2209 12 : for ( ; arg != void_list_node ; arg = TREE_CHAIN (arg))
2210 : {
2211 9 : machine_mode mode = TYPE_MODE (TREE_VALUE (arg));
2212 9 : function_arg_info arg_info (mode, /*named=*/true);
2213 9 : rtx tmp = targetm.calls.function_arg (args_so_far, arg_info);
2214 9 : if (!tmp || !REG_P (tmp))
2215 0 : return false;
2216 9 : if (targetm.calls.arg_partial_bytes (args_so_far, arg_info))
2217 : return false;
2218 9 : targetm.calls.function_arg_advance (args_so_far, arg_info);
2219 : }
2220 : }
2221 3 : return true;
2222 : }
2223 :
2224 : /* A subroutine of emit_block_move. Expand a cpymem or movmem pattern;
2225 : return true if successful.
2226 :
2227 : X is the destination of the copy or move.
2228 : Y is the source of the copy or move.
2229 : SIZE is the size of the block to be moved.
2230 :
2231 : MIGHT_OVERLAP indicates this originated with expansion of a
2232 : builtin_memmove() and the source and destination blocks may
2233 : overlap.
2234 : */
2235 :
2236 : static bool
2237 105584 : emit_block_move_via_pattern (rtx x, rtx y, rtx size, unsigned int align,
2238 : unsigned int expected_align,
2239 : HOST_WIDE_INT expected_size,
2240 : unsigned HOST_WIDE_INT min_size,
2241 : unsigned HOST_WIDE_INT max_size,
2242 : unsigned HOST_WIDE_INT probable_max_size,
2243 : bool might_overlap)
2244 : {
2245 105584 : if (expected_align < align)
2246 : expected_align = align;
2247 105584 : if (expected_size != -1)
2248 : {
2249 22 : if ((unsigned HOST_WIDE_INT)expected_size > probable_max_size)
2250 0 : expected_size = probable_max_size;
2251 22 : if ((unsigned HOST_WIDE_INT)expected_size < min_size)
2252 0 : expected_size = min_size;
2253 : }
2254 :
2255 : /* Since this is a move insn, we don't care about volatility. */
2256 105584 : temporary_volatile_ok v (true);
2257 :
2258 : /* Try the most limited insn first, because there's no point
2259 : including more than one in the machine description unless
2260 : the more limited one has some advantage. */
2261 :
2262 105584 : opt_scalar_int_mode mode_iter;
2263 722115 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
2264 : {
2265 641835 : scalar_int_mode mode = mode_iter.require ();
2266 641835 : enum insn_code code;
2267 641835 : if (might_overlap)
2268 101463 : code = direct_optab_handler (movmem_optab, mode);
2269 : else
2270 540372 : code = direct_optab_handler (cpymem_optab, mode);
2271 :
2272 641835 : if (code != CODE_FOR_nothing
2273 : /* We don't need MODE to be narrower than BITS_PER_HOST_WIDE_INT
2274 : here because if SIZE is less than the mode mask, as it is
2275 : returned by the macro, it will definitely be less than the
2276 : actual mode mask. Since SIZE is within the Pmode address
2277 : space, we limit MODE to Pmode. */
2278 641835 : && ((CONST_INT_P (size)
2279 41613 : && ((unsigned HOST_WIDE_INT) INTVAL (size)
2280 41613 : <= (GET_MODE_MASK (mode) >> 1)))
2281 143143 : || max_size <= (GET_MODE_MASK (mode) >> 1)
2282 178406 : || GET_MODE_BITSIZE (mode) >= GET_MODE_BITSIZE (Pmode)))
2283 : {
2284 122306 : class expand_operand ops[9];
2285 122306 : unsigned int nops;
2286 :
2287 : /* ??? When called via emit_block_move_for_call, it'd be
2288 : nice if there were some way to inform the backend, so
2289 : that it doesn't fail the expansion because it thinks
2290 : emitting the libcall would be more efficient. */
2291 122306 : nops = insn_data[(int) code].n_generator_args;
2292 122306 : gcc_assert (nops == 4 || nops == 6 || nops == 8 || nops == 9);
2293 :
2294 122306 : create_fixed_operand (&ops[0], x);
2295 122306 : create_fixed_operand (&ops[1], y);
2296 : /* The check above guarantees that this size conversion is valid. */
2297 122306 : create_convert_operand_to (&ops[2], size, mode, true);
2298 122306 : create_integer_operand (&ops[3], align / BITS_PER_UNIT);
2299 122306 : if (nops >= 6)
2300 : {
2301 122306 : create_integer_operand (&ops[4], expected_align / BITS_PER_UNIT);
2302 122306 : create_integer_operand (&ops[5], expected_size);
2303 : }
2304 122306 : if (nops >= 8)
2305 : {
2306 122306 : create_integer_operand (&ops[6], min_size);
2307 : /* If we cannot represent the maximal size,
2308 : make parameter NULL. */
2309 122306 : if ((HOST_WIDE_INT) max_size != -1)
2310 105146 : create_integer_operand (&ops[7], max_size);
2311 : else
2312 17160 : create_fixed_operand (&ops[7], NULL);
2313 : }
2314 122306 : if (nops == 9)
2315 : {
2316 : /* If we cannot represent the maximal size,
2317 : make parameter NULL. */
2318 122306 : if ((HOST_WIDE_INT) probable_max_size != -1)
2319 107276 : create_integer_operand (&ops[8], probable_max_size);
2320 : else
2321 15030 : create_fixed_operand (&ops[8], NULL);
2322 : }
2323 122306 : gcc_assert (min_size != max_size
2324 : || rtx_equal_p (ops[2].value, GEN_INT (min_size)));
2325 122306 : if (maybe_expand_insn (code, nops, ops))
2326 25304 : return true;
2327 : }
2328 : }
2329 :
2330 : return false;
2331 105584 : }
2332 :
2333 : /* Like emit_block_move_via_loop, but choose a suitable INCR based on
2334 : ALIGN and CTZ_SIZE. */
2335 :
2336 : static void
2337 48 : emit_block_move_via_sized_loop (rtx x, rtx y, rtx size,
2338 : unsigned int align,
2339 : unsigned int ctz_size)
2340 : {
2341 48 : int incr = align / BITS_PER_UNIT;
2342 :
2343 48 : if (CONST_INT_P (size))
2344 0 : ctz_size = MAX (ctz_size, (unsigned) wi::ctz (UINTVAL (size)));
2345 :
2346 48 : if (HOST_WIDE_INT_1U << ctz_size < (unsigned HOST_WIDE_INT) incr)
2347 0 : incr = HOST_WIDE_INT_1U << ctz_size;
2348 :
2349 48 : while (incr > 1 && !can_move_by_pieces (incr, align))
2350 0 : incr >>= 1;
2351 :
2352 48 : gcc_checking_assert (incr);
2353 :
2354 48 : return emit_block_move_via_loop (x, y, size, align, incr);
2355 : }
2356 :
2357 : /* Like emit_block_move_via_sized_loop, but besides choosing INCR so
2358 : as to ensure safe moves even in case of overlap, output dynamic
2359 : tests to choose between two loops, one moving downwards, another
2360 : moving upwards. */
2361 :
2362 : static void
2363 0 : emit_block_move_via_oriented_loop (rtx x, rtx y, rtx size,
2364 : unsigned int align,
2365 : unsigned int ctz_size)
2366 : {
2367 0 : int incr = align / BITS_PER_UNIT;
2368 :
2369 0 : if (CONST_INT_P (size))
2370 0 : ctz_size = MAX (ctz_size, (unsigned) wi::ctz (UINTVAL (size)));
2371 :
2372 0 : if (HOST_WIDE_INT_1U << ctz_size < (unsigned HOST_WIDE_INT) incr)
2373 0 : incr = HOST_WIDE_INT_1U << ctz_size;
2374 :
2375 0 : while (incr > 1 && !int_mode_for_size (incr, 0).exists ())
2376 0 : incr >>= 1;
2377 :
2378 0 : gcc_checking_assert (incr);
2379 :
2380 0 : rtx_code_label *upw_label, *end_label;
2381 0 : upw_label = gen_label_rtx ();
2382 0 : end_label = gen_label_rtx ();
2383 :
2384 0 : rtx x_addr = force_operand (XEXP (x, 0), NULL_RTX);
2385 0 : rtx y_addr = force_operand (XEXP (y, 0), NULL_RTX);
2386 0 : do_pending_stack_adjust ();
2387 :
2388 0 : machine_mode mode = GET_MODE (x_addr);
2389 0 : if (mode != GET_MODE (y_addr))
2390 : {
2391 0 : scalar_int_mode xmode
2392 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE (mode)).require ();
2393 0 : scalar_int_mode ymode
2394 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE
2395 0 : (GET_MODE (y_addr))).require ();
2396 0 : if (GET_MODE_BITSIZE (xmode) < GET_MODE_BITSIZE (ymode))
2397 : mode = ymode;
2398 : else
2399 0 : mode = xmode;
2400 :
2401 : #ifndef POINTERS_EXTEND_UNSIGNED
2402 : const int POINTERS_EXTEND_UNSIGNED = 1;
2403 : #endif
2404 0 : x_addr = convert_modes (mode, GET_MODE (x_addr), x_addr,
2405 : POINTERS_EXTEND_UNSIGNED);
2406 0 : y_addr = convert_modes (mode, GET_MODE (y_addr), y_addr,
2407 : POINTERS_EXTEND_UNSIGNED);
2408 : }
2409 :
2410 : /* Test for overlap: if (x >= y || x + size <= y) goto upw_label. */
2411 0 : emit_cmp_and_jump_insns (x_addr, y_addr, GEU, NULL_RTX, mode,
2412 : true, upw_label,
2413 0 : profile_probability::guessed_always ()
2414 : .apply_scale (5, 10));
2415 0 : rtx tmp = convert_modes (GET_MODE (x_addr), GET_MODE (size), size, true);
2416 0 : tmp = simplify_gen_binary (PLUS, GET_MODE (x_addr), x_addr, tmp);
2417 :
2418 0 : emit_cmp_and_jump_insns (tmp, y_addr, LEU, NULL_RTX, mode,
2419 : true, upw_label,
2420 0 : profile_probability::guessed_always ()
2421 : .apply_scale (8, 10));
2422 :
2423 0 : emit_block_move_via_loop (x, y, size, align, -incr);
2424 :
2425 0 : emit_jump (end_label);
2426 0 : emit_label (upw_label);
2427 :
2428 0 : emit_block_move_via_loop (x, y, size, align, incr);
2429 :
2430 0 : emit_label (end_label);
2431 0 : }
2432 :
2433 : /* A subroutine of emit_block_move. Copy the data via an explicit
2434 : loop. This is used only when libcalls are forbidden, or when
2435 : inlining is required. INCR is the block size to be copied in each
2436 : loop iteration. If it is negative, the absolute value is used, and
2437 : the block is copied backwards. INCR must be a power of two, an
2438 : exact divisor for SIZE and ALIGN, and imply a mode that can be
2439 : safely copied per iteration assuming no overlap. */
2440 :
2441 : static void
2442 48 : emit_block_move_via_loop (rtx x, rtx y, rtx size,
2443 : unsigned int align, int incr)
2444 : {
2445 48 : rtx_code_label *cmp_label, *top_label;
2446 48 : rtx iter, x_addr, y_addr, tmp;
2447 48 : machine_mode x_addr_mode = get_address_mode (x);
2448 48 : machine_mode y_addr_mode = get_address_mode (y);
2449 48 : machine_mode iter_mode;
2450 :
2451 48 : iter_mode = GET_MODE (size);
2452 48 : if (iter_mode == VOIDmode)
2453 0 : iter_mode = word_mode;
2454 :
2455 48 : top_label = gen_label_rtx ();
2456 48 : cmp_label = gen_label_rtx ();
2457 48 : iter = gen_reg_rtx (iter_mode);
2458 :
2459 48 : bool downwards = incr < 0;
2460 48 : rtx iter_init;
2461 48 : rtx_code iter_cond;
2462 48 : rtx iter_limit;
2463 48 : rtx iter_incr;
2464 48 : machine_mode move_mode;
2465 48 : if (downwards)
2466 : {
2467 0 : incr = -incr;
2468 0 : iter_init = size;
2469 0 : iter_cond = GEU;
2470 0 : iter_limit = const0_rtx;
2471 0 : iter_incr = GEN_INT (incr);
2472 : }
2473 : else
2474 : {
2475 48 : iter_init = const0_rtx;
2476 48 : iter_cond = LTU;
2477 48 : iter_limit = size;
2478 48 : iter_incr = GEN_INT (incr);
2479 : }
2480 48 : emit_move_insn (iter, iter_init);
2481 :
2482 48 : opt_scalar_int_mode int_move_mode
2483 48 : = int_mode_for_size (incr * BITS_PER_UNIT, 1);
2484 48 : if (!int_move_mode.exists (&move_mode)
2485 96 : || GET_MODE_BITSIZE (int_move_mode.require ()) != incr * BITS_PER_UNIT)
2486 : {
2487 0 : move_mode = BLKmode;
2488 0 : gcc_checking_assert (can_move_by_pieces (incr, align));
2489 : }
2490 :
2491 48 : x_addr = force_operand (XEXP (x, 0), NULL_RTX);
2492 48 : y_addr = force_operand (XEXP (y, 0), NULL_RTX);
2493 48 : do_pending_stack_adjust ();
2494 :
2495 48 : emit_jump (cmp_label);
2496 48 : emit_label (top_label);
2497 :
2498 48 : tmp = convert_modes (x_addr_mode, iter_mode, iter, true);
2499 48 : x_addr = simplify_gen_binary (PLUS, x_addr_mode, x_addr, tmp);
2500 :
2501 48 : if (x_addr_mode != y_addr_mode)
2502 0 : tmp = convert_modes (y_addr_mode, iter_mode, iter, true);
2503 48 : y_addr = simplify_gen_binary (PLUS, y_addr_mode, y_addr, tmp);
2504 :
2505 48 : x = change_address (x, move_mode, x_addr);
2506 48 : y = change_address (y, move_mode, y_addr);
2507 :
2508 48 : if (move_mode == BLKmode)
2509 : {
2510 0 : bool done;
2511 0 : emit_block_move_hints (x, y, iter_incr, BLOCK_OP_NO_LIBCALL,
2512 : align, incr, incr, incr, incr,
2513 : false, &done, false);
2514 0 : gcc_checking_assert (done);
2515 : }
2516 : else
2517 48 : emit_move_insn (x, y);
2518 :
2519 48 : if (downwards)
2520 0 : emit_label (cmp_label);
2521 :
2522 48 : tmp = expand_simple_binop (iter_mode, PLUS, iter, iter_incr, iter,
2523 : true, OPTAB_LIB_WIDEN);
2524 48 : if (tmp != iter)
2525 0 : emit_move_insn (iter, tmp);
2526 :
2527 48 : if (!downwards)
2528 48 : emit_label (cmp_label);
2529 :
2530 48 : emit_cmp_and_jump_insns (iter, iter_limit, iter_cond, NULL_RTX, iter_mode,
2531 : true, top_label,
2532 48 : profile_probability::guessed_always ()
2533 : .apply_scale (9, 10));
2534 48 : }
2535 :
2536 : /* Expand a call to memcpy or memmove or memcmp, and return the result.
2537 : TAILCALL is true if this is a tail call. */
2538 :
2539 : rtx
2540 67507 : emit_block_op_via_libcall (enum built_in_function fncode, rtx dst, rtx src,
2541 : rtx size, bool tailcall)
2542 : {
2543 67507 : rtx dst_addr, src_addr;
2544 67507 : tree call_expr, dst_tree, src_tree, size_tree;
2545 67507 : machine_mode size_mode;
2546 :
2547 : /* Since dst and src are passed to a libcall, mark the corresponding
2548 : tree EXPR as addressable. */
2549 67507 : tree dst_expr = MEM_EXPR (dst);
2550 67507 : tree src_expr = MEM_EXPR (src);
2551 67507 : if (dst_expr)
2552 60788 : mark_addressable (dst_expr);
2553 67507 : if (src_expr)
2554 66095 : mark_addressable (src_expr);
2555 :
2556 67507 : dst_addr = copy_addr_to_reg (XEXP (dst, 0));
2557 67507 : dst_addr = convert_memory_address (ptr_mode, dst_addr);
2558 67507 : dst_tree = make_tree (ptr_type_node, dst_addr);
2559 :
2560 67507 : src_addr = copy_addr_to_reg (XEXP (src, 0));
2561 67507 : src_addr = convert_memory_address (ptr_mode, src_addr);
2562 67507 : src_tree = make_tree (ptr_type_node, src_addr);
2563 :
2564 67507 : size_mode = TYPE_MODE (sizetype);
2565 67507 : size = convert_to_mode (size_mode, size, 1);
2566 67507 : size = copy_to_mode_reg (size_mode, size);
2567 67507 : size_tree = make_tree (sizetype, size);
2568 :
2569 : /* It is incorrect to use the libcall calling conventions for calls to
2570 : memcpy/memmove/memcmp because they can be provided by the user. */
2571 67507 : tree fn = builtin_decl_implicit (fncode);
2572 67507 : call_expr = build_call_expr (fn, 3, dst_tree, src_tree, size_tree);
2573 67507 : CALL_EXPR_TAILCALL (call_expr) = tailcall;
2574 :
2575 67507 : return expand_call (call_expr, NULL_RTX, false);
2576 : }
2577 :
2578 : /* Try to expand cmpstrn or cmpmem operation ICODE with the given operands.
2579 : ARG3_TYPE is the type of ARG3_RTX. Return the result rtx on success,
2580 : otherwise return null. */
2581 :
2582 : rtx
2583 171425 : expand_cmpstrn_or_cmpmem (insn_code icode, rtx target, rtx arg1_rtx,
2584 : rtx arg2_rtx, tree arg3_type, rtx arg3_rtx,
2585 : HOST_WIDE_INT align)
2586 : {
2587 171425 : machine_mode insn_mode = insn_data[icode].operand[0].mode;
2588 :
2589 171425 : if (target && (!REG_P (target) || HARD_REGISTER_P (target)))
2590 : target = NULL_RTX;
2591 :
2592 171425 : class expand_operand ops[5];
2593 171425 : create_output_operand (&ops[0], target, insn_mode);
2594 171425 : create_fixed_operand (&ops[1], arg1_rtx);
2595 171425 : create_fixed_operand (&ops[2], arg2_rtx);
2596 171425 : create_convert_operand_from (&ops[3], arg3_rtx, TYPE_MODE (arg3_type),
2597 171425 : TYPE_UNSIGNED (arg3_type));
2598 171425 : create_integer_operand (&ops[4], align);
2599 171425 : if (maybe_expand_insn (icode, 5, ops))
2600 5796 : return ops[0].value;
2601 : return NULL_RTX;
2602 : }
2603 :
2604 : /* Expand a block compare between X and Y with length LEN using the
2605 : cmpmem optab, placing the result in TARGET. LEN_TYPE is the type
2606 : of the expression that was used to calculate the length. ALIGN
2607 : gives the known minimum common alignment. */
2608 :
2609 : static rtx
2610 43484 : emit_block_cmp_via_cmpmem (rtx x, rtx y, rtx len, tree len_type, rtx target,
2611 : unsigned align)
2612 : {
2613 : /* Note: The cmpstrnsi pattern, if it exists, is not suitable for
2614 : implementing memcmp because it will stop if it encounters two
2615 : zero bytes. */
2616 43484 : insn_code icode = direct_optab_handler (cmpmem_optab, SImode);
2617 :
2618 43484 : if (icode == CODE_FOR_nothing)
2619 : return NULL_RTX;
2620 :
2621 43484 : return expand_cmpstrn_or_cmpmem (icode, target, x, y, len_type, len, align);
2622 : }
2623 :
2624 : /* Emit code to compare a block Y to a block X. This may be done with
2625 : string-compare instructions, with multiple scalar instructions,
2626 : or with a library call.
2627 :
2628 : Both X and Y must be MEM rtx's. LEN is an rtx that says how long
2629 : they are. LEN_TYPE is the type of the expression that was used to
2630 : calculate it, and CTZ_LEN is the known trailing-zeros count of LEN,
2631 : so LEN must be a multiple of 1<<CTZ_LEN even if it's not constant.
2632 :
2633 : If EQUALITY_ONLY is true, it means we don't have to return the tri-state
2634 : value of a normal memcmp call, instead we can just compare for equality.
2635 : If FORCE_LIBCALL is true, we should emit a call to memcmp rather than
2636 : returning NULL_RTX.
2637 :
2638 : Optionally, the caller can pass a constfn and associated data in Y_CFN
2639 : and Y_CFN_DATA. describing that the second operand being compared is a
2640 : known constant and how to obtain its data.
2641 : Return the result of the comparison, or NULL_RTX if we failed to
2642 : perform the operation. */
2643 :
2644 : rtx
2645 106183 : emit_block_cmp_hints (rtx x, rtx y, rtx len, tree len_type, rtx target,
2646 : bool equality_only, by_pieces_constfn y_cfn,
2647 : void *y_cfndata, unsigned ctz_len)
2648 : {
2649 106183 : rtx result = 0;
2650 :
2651 106183 : if (CONST_INT_P (len) && INTVAL (len) == 0)
2652 0 : return const0_rtx;
2653 :
2654 106183 : gcc_assert (MEM_P (x) && MEM_P (y));
2655 106183 : unsigned int align = MIN (MEM_ALIGN (x), MEM_ALIGN (y));
2656 106183 : gcc_assert (align >= BITS_PER_UNIT);
2657 :
2658 106183 : x = adjust_address (x, BLKmode, 0);
2659 106183 : y = adjust_address (y, BLKmode, 0);
2660 :
2661 106183 : if (equality_only
2662 91326 : && CONST_INT_P (len)
2663 175608 : && can_do_by_pieces (INTVAL (len), align, COMPARE_BY_PIECES))
2664 62699 : result = compare_by_pieces (x, y, INTVAL (len), target, align,
2665 : y_cfn, y_cfndata);
2666 : else
2667 43484 : result = emit_block_cmp_via_cmpmem (x, y, len, len_type, target, align);
2668 :
2669 106183 : if (!result && (flag_inline_stringops & ILSOP_MEMCMP))
2670 361 : result = emit_block_cmp_via_loop (x, y, len, len_type,
2671 : target, equality_only,
2672 : align, ctz_len);
2673 :
2674 : return result;
2675 : }
2676 :
2677 : /* Like emit_block_cmp_hints, but with known alignment and no support
2678 : for constats. Always expand to a loop with iterations that compare
2679 : blocks of the largest compare-by-pieces size that divides both len
2680 : and align, and then, if !EQUALITY_ONLY, identify the word and then
2681 : the unit that first differs to return the result. */
2682 :
2683 : rtx
2684 403 : emit_block_cmp_via_loop (rtx x, rtx y, rtx len, tree len_type, rtx target,
2685 : bool equality_only, unsigned align, unsigned ctz_len)
2686 : {
2687 403 : unsigned incr = align / BITS_PER_UNIT;
2688 :
2689 403 : if (CONST_INT_P (len))
2690 319 : ctz_len = MAX (ctz_len, (unsigned) wi::ctz (UINTVAL (len)));
2691 :
2692 403 : if (HOST_WIDE_INT_1U << ctz_len < (unsigned HOST_WIDE_INT) incr)
2693 148 : incr = HOST_WIDE_INT_1U << ctz_len;
2694 :
2695 : while (incr > 1
2696 407 : && !can_do_by_pieces (incr, align, COMPARE_BY_PIECES))
2697 4 : incr >>= 1;
2698 :
2699 403 : rtx_code_label *cmp_label, *top_label, *ne_label, *res_label;
2700 403 : rtx iter, x_addr, y_addr, tmp;
2701 403 : machine_mode x_addr_mode = get_address_mode (x);
2702 403 : machine_mode y_addr_mode = get_address_mode (y);
2703 403 : machine_mode iter_mode;
2704 :
2705 403 : iter_mode = GET_MODE (len);
2706 403 : if (iter_mode == VOIDmode)
2707 319 : iter_mode = word_mode;
2708 :
2709 403 : rtx iter_init = const0_rtx;
2710 403 : rtx_code iter_cond = LTU;
2711 403 : rtx_code entry_cond = GEU;
2712 403 : rtx iter_limit = len;
2713 403 : rtx iter_incr = GEN_INT (incr);
2714 403 : machine_mode cmp_mode;
2715 :
2716 : /* We can drop the loop back edge if we know there's exactly one
2717 : iteration. */
2718 403 : top_label = (!rtx_equal_p (len, iter_incr)
2719 403 : ? gen_label_rtx ()
2720 : : NULL);
2721 : /* We need not test before entering the loop if len is known
2722 : nonzero. ??? This could be even stricter, testing whether a
2723 : nonconstant LEN could possibly be zero. */
2724 403 : cmp_label = (!CONSTANT_P (len) || rtx_equal_p (len, iter_init)
2725 403 : ? gen_label_rtx ()
2726 : : NULL);
2727 403 : ne_label = gen_label_rtx ();
2728 403 : res_label = gen_label_rtx ();
2729 :
2730 403 : iter = gen_reg_rtx (iter_mode);
2731 403 : emit_move_insn (iter, iter_init);
2732 :
2733 403 : opt_scalar_int_mode int_cmp_mode
2734 403 : = int_mode_for_size (incr * BITS_PER_UNIT, 1);
2735 403 : if (!int_cmp_mode.exists (&cmp_mode)
2736 1206 : || GET_MODE_BITSIZE (int_cmp_mode.require ()) != incr * BITS_PER_UNIT
2737 402 : || !can_compare_p (NE, cmp_mode, ccp_jump))
2738 : {
2739 1 : cmp_mode = BLKmode;
2740 1 : gcc_checking_assert (incr != 1);
2741 : }
2742 :
2743 : /* Save the base addresses. */
2744 403 : x_addr = force_operand (XEXP (x, 0), NULL_RTX);
2745 403 : y_addr = force_operand (XEXP (y, 0), NULL_RTX);
2746 403 : do_pending_stack_adjust ();
2747 :
2748 403 : if (cmp_label)
2749 : {
2750 84 : if (top_label)
2751 84 : emit_jump (cmp_label);
2752 : else
2753 0 : emit_cmp_and_jump_insns (iter, iter_limit, entry_cond,
2754 : NULL_RTX, iter_mode,
2755 : true, cmp_label,
2756 0 : profile_probability::guessed_always ()
2757 : .apply_scale (1, 10));
2758 : }
2759 403 : if (top_label)
2760 386 : emit_label (top_label);
2761 :
2762 : /* Offset the base addresses by ITER. */
2763 403 : tmp = convert_modes (x_addr_mode, iter_mode, iter, true);
2764 403 : x_addr = simplify_gen_binary (PLUS, x_addr_mode, x_addr, tmp);
2765 :
2766 403 : if (x_addr_mode != y_addr_mode)
2767 0 : tmp = convert_modes (y_addr_mode, iter_mode, iter, true);
2768 403 : y_addr = simplify_gen_binary (PLUS, y_addr_mode, y_addr, tmp);
2769 :
2770 403 : x = change_address (x, cmp_mode, x_addr);
2771 403 : y = change_address (y, cmp_mode, y_addr);
2772 :
2773 : /* Compare one block. */
2774 403 : rtx part_res;
2775 403 : if (cmp_mode == BLKmode)
2776 1 : part_res = compare_by_pieces (x, y, incr, target, align, 0, 0);
2777 : else
2778 402 : part_res = expand_binop (cmp_mode, sub_optab, x, y, NULL_RTX,
2779 : true, OPTAB_LIB_WIDEN);
2780 :
2781 : /* Stop if we found a difference. */
2782 806 : emit_cmp_and_jump_insns (part_res, GEN_INT (0), NE, NULL_RTX,
2783 403 : GET_MODE (part_res), true, ne_label,
2784 403 : profile_probability::guessed_always ()
2785 : .apply_scale (1, 10));
2786 :
2787 : /* Increment ITER. */
2788 403 : tmp = expand_simple_binop (iter_mode, PLUS, iter, iter_incr, iter,
2789 : true, OPTAB_LIB_WIDEN);
2790 403 : if (tmp != iter)
2791 0 : emit_move_insn (iter, tmp);
2792 :
2793 403 : if (cmp_label)
2794 84 : emit_label (cmp_label);
2795 : /* Loop until we reach the limit. */
2796 :
2797 403 : if (top_label)
2798 386 : emit_cmp_and_jump_insns (iter, iter_limit, iter_cond, NULL_RTX, iter_mode,
2799 : true, top_label,
2800 772 : profile_probability::guessed_always ()
2801 : .apply_scale (9, 10));
2802 :
2803 : /* We got to the end without differences, so the result is zero. */
2804 403 : if (target == NULL_RTX
2805 403 : || !REG_P (target) || REGNO (target) < FIRST_PSEUDO_REGISTER)
2806 49 : target = gen_reg_rtx (TYPE_MODE (integer_type_node));
2807 :
2808 403 : emit_move_insn (target, const0_rtx);
2809 403 : emit_jump (res_label);
2810 :
2811 403 : emit_label (ne_label);
2812 :
2813 : /* Return nonzero, or pinpoint the difference to return the expected
2814 : result for non-equality tests. */
2815 403 : if (equality_only)
2816 0 : emit_move_insn (target, const1_rtx);
2817 : else
2818 : {
2819 403 : if (incr > UNITS_PER_WORD)
2820 : /* ??? Re-compare the block found to be different one word at a
2821 : time. */
2822 5 : part_res = emit_block_cmp_via_loop (x, y, GEN_INT (incr), len_type,
2823 : target, equality_only,
2824 : BITS_PER_WORD, 0);
2825 398 : else if (incr > 1)
2826 : /* ??? Re-compare the block found to be different one byte at a
2827 : time. We could do better using part_res, and being careful
2828 : about endianness. */
2829 37 : part_res = emit_block_cmp_via_loop (x, y, GEN_INT (incr), len_type,
2830 : target, equality_only,
2831 : BITS_PER_UNIT, 0);
2832 1083 : else if (known_gt (GET_MODE_BITSIZE (GET_MODE (target)),
2833 : GET_MODE_BITSIZE (cmp_mode)))
2834 361 : part_res = expand_binop (GET_MODE (target), sub_optab, x, y, target,
2835 : true, OPTAB_LIB_WIDEN);
2836 : else
2837 : {
2838 : /* In the odd chance target is QImode, we can't count on
2839 : widening subtract to capture the result of the unsigned
2840 : compares. */
2841 0 : rtx_code_label *ltu_label;
2842 0 : ltu_label = gen_label_rtx ();
2843 0 : emit_cmp_and_jump_insns (x, y, LTU, NULL_RTX,
2844 : cmp_mode, true, ltu_label,
2845 0 : profile_probability::guessed_always ()
2846 : .apply_scale (5, 10));
2847 :
2848 0 : emit_move_insn (target, const1_rtx);
2849 0 : emit_jump (res_label);
2850 :
2851 0 : emit_label (ltu_label);
2852 0 : emit_move_insn (target, constm1_rtx);
2853 0 : part_res = target;
2854 : }
2855 :
2856 403 : if (target != part_res)
2857 0 : convert_move (target, part_res, false);
2858 : }
2859 :
2860 403 : emit_label (res_label);
2861 :
2862 403 : return target;
2863 : }
2864 :
2865 :
2866 : /* Copy all or part of a value X into registers starting at REGNO.
2867 : The number of registers to be filled is NREGS. */
2868 :
2869 : void
2870 434 : move_block_to_reg (int regno, rtx x, int nregs, machine_mode mode)
2871 : {
2872 434 : if (nregs == 0)
2873 : return;
2874 :
2875 420 : if (CONSTANT_P (x) && !targetm.legitimate_constant_p (mode, x))
2876 0 : x = validize_mem (force_const_mem (mode, x));
2877 :
2878 : /* See if the machine can do this with a load multiple insn. */
2879 420 : if (targetm.have_load_multiple ())
2880 : {
2881 0 : rtx_insn *last = get_last_insn ();
2882 0 : rtx first = gen_rtx_REG (word_mode, regno);
2883 0 : if (rtx_insn *pat = targetm.gen_load_multiple (first, x,
2884 : GEN_INT (nregs)))
2885 : {
2886 0 : emit_insn (pat);
2887 0 : return;
2888 : }
2889 : else
2890 0 : delete_insns_since (last);
2891 : }
2892 :
2893 848 : for (int i = 0; i < nregs; i++)
2894 428 : emit_move_insn (gen_rtx_REG (word_mode, regno + i),
2895 428 : operand_subword_force (x, i, mode));
2896 : }
2897 :
2898 : /* Copy all or part of a BLKmode value X out of registers starting at REGNO.
2899 : The number of registers to be filled is NREGS. */
2900 :
2901 : void
2902 1216 : move_block_from_reg (int regno, rtx x, int nregs)
2903 : {
2904 1216 : if (nregs == 0)
2905 : return;
2906 :
2907 : /* See if the machine can do this with a store multiple insn. */
2908 1216 : if (targetm.have_store_multiple ())
2909 : {
2910 0 : rtx_insn *last = get_last_insn ();
2911 0 : rtx first = gen_rtx_REG (word_mode, regno);
2912 0 : if (rtx_insn *pat = targetm.gen_store_multiple (x, first,
2913 : GEN_INT (nregs)))
2914 : {
2915 0 : emit_insn (pat);
2916 0 : return;
2917 : }
2918 : else
2919 0 : delete_insns_since (last);
2920 : }
2921 :
2922 2436 : for (int i = 0; i < nregs; i++)
2923 : {
2924 1220 : rtx tem = operand_subword (x, i, 1, BLKmode);
2925 :
2926 1220 : gcc_assert (tem);
2927 :
2928 1220 : emit_move_insn (tem, gen_rtx_REG (word_mode, regno + i));
2929 : }
2930 : }
2931 :
2932 : /* Generate a PARALLEL rtx for a new non-consecutive group of registers from
2933 : ORIG, where ORIG is a non-consecutive group of registers represented by
2934 : a PARALLEL. The clone is identical to the original except in that the
2935 : original set of registers is replaced by a new set of pseudo registers.
2936 : The new set has the same modes as the original set. */
2937 :
2938 : rtx
2939 3063 : gen_group_rtx (rtx orig)
2940 : {
2941 3063 : int i, length;
2942 3063 : rtx *tmps;
2943 :
2944 3063 : gcc_assert (GET_CODE (orig) == PARALLEL);
2945 :
2946 3063 : length = XVECLEN (orig, 0);
2947 3063 : tmps = XALLOCAVEC (rtx, length);
2948 :
2949 : /* Skip a NULL entry in first slot. */
2950 3063 : i = XEXP (XVECEXP (orig, 0, 0), 0) ? 0 : 1;
2951 :
2952 3063 : if (i)
2953 0 : tmps[0] = 0;
2954 :
2955 7456 : for (; i < length; i++)
2956 : {
2957 4393 : machine_mode mode = GET_MODE (XEXP (XVECEXP (orig, 0, i), 0));
2958 4393 : rtx offset = XEXP (XVECEXP (orig, 0, i), 1);
2959 :
2960 4393 : tmps[i] = gen_rtx_EXPR_LIST (VOIDmode, gen_reg_rtx (mode), offset);
2961 : }
2962 :
2963 3063 : return gen_rtx_PARALLEL (GET_MODE (orig), gen_rtvec_v (length, tmps));
2964 : }
2965 :
2966 : /* A subroutine of emit_group_load. Arguments as for emit_group_load,
2967 : except that values are placed in TMPS[i], and must later be moved
2968 : into corresponding XEXP (XVECEXP (DST, 0, i), 0) element. */
2969 :
2970 : static void
2971 298475 : emit_group_load_1 (rtx *tmps, rtx dst, rtx orig_src, tree type,
2972 : poly_int64 ssize)
2973 : {
2974 298478 : rtx src;
2975 298478 : int start, i;
2976 298478 : machine_mode m = GET_MODE (orig_src);
2977 :
2978 298478 : gcc_assert (GET_CODE (dst) == PARALLEL);
2979 :
2980 298478 : if (m != VOIDmode
2981 279613 : && !SCALAR_INT_MODE_P (m)
2982 19155 : && !MEM_P (orig_src)
2983 5725 : && GET_CODE (orig_src) != CONCAT)
2984 : {
2985 3 : scalar_int_mode imode;
2986 3 : if (int_mode_for_mode (GET_MODE (orig_src)).exists (&imode))
2987 : {
2988 3 : src = gen_reg_rtx (imode);
2989 3 : emit_move_insn (gen_lowpart (GET_MODE (orig_src), src), orig_src);
2990 : }
2991 : else
2992 : {
2993 0 : src = assign_stack_temp (GET_MODE (orig_src), ssize);
2994 0 : emit_move_insn (src, orig_src);
2995 : }
2996 3 : emit_group_load_1 (tmps, dst, src, type, ssize);
2997 : return;
2998 : }
2999 :
3000 : /* Check for a NULL entry, used to indicate that the parameter goes
3001 : both on the stack and in registers. */
3002 298475 : if (XEXP (XVECEXP (dst, 0, 0), 0))
3003 : start = 0;
3004 : else
3005 0 : start = 1;
3006 :
3007 : /* Process the pieces. */
3008 883932 : for (i = start; i < XVECLEN (dst, 0); i++)
3009 : {
3010 585457 : machine_mode mode = GET_MODE (XEXP (XVECEXP (dst, 0, i), 0));
3011 585457 : poly_int64 bytepos = rtx_to_poly_int64 (XEXP (XVECEXP (dst, 0, i), 1));
3012 1170914 : poly_int64 bytelen = GET_MODE_SIZE (mode);
3013 585457 : poly_int64 shift = 0;
3014 :
3015 : /* Handle trailing fragments that run over the size of the struct.
3016 : It's the target's responsibility to make sure that the fragment
3017 : cannot be strictly smaller in some cases and strictly larger
3018 : in others. */
3019 585457 : gcc_checking_assert (ordered_p (bytepos + bytelen, ssize));
3020 585457 : if (known_size_p (ssize) && maybe_gt (bytepos + bytelen, ssize))
3021 : {
3022 : /* Arrange to shift the fragment to where it belongs.
3023 : extract_bit_field loads to the lsb of the reg. */
3024 : if (
3025 : #ifdef BLOCK_REG_PADDING
3026 : BLOCK_REG_PADDING (GET_MODE (orig_src), type, i == start)
3027 : == (BYTES_BIG_ENDIAN ? PAD_UPWARD : PAD_DOWNWARD)
3028 : #else
3029 : BYTES_BIG_ENDIAN
3030 : #endif
3031 : )
3032 : shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
3033 1836 : bytelen = ssize - bytepos;
3034 1836 : gcc_assert (maybe_gt (bytelen, 0));
3035 : }
3036 :
3037 : /* If we won't be loading directly from memory, protect the real source
3038 : from strange tricks we might play; but make sure that the source can
3039 : be loaded directly into the destination. */
3040 585457 : src = orig_src;
3041 585457 : if (!MEM_P (orig_src)
3042 484735 : && (!REG_P (orig_src) || HARD_REGISTER_P (orig_src))
3043 46417 : && GET_CODE (orig_src) != CONCAT
3044 623187 : && !CONSTANT_P (orig_src))
3045 : {
3046 0 : gcc_assert (GET_MODE (orig_src) != VOIDmode);
3047 0 : src = force_reg (GET_MODE (orig_src), orig_src);
3048 : }
3049 :
3050 : /* Optimize the access just a bit. */
3051 585457 : if (MEM_P (src)
3052 100722 : && (! targetm.slow_unaligned_access (mode, MEM_ALIGN (src))
3053 0 : || MEM_ALIGN (src) >= GET_MODE_ALIGNMENT (mode))
3054 201444 : && multiple_p (bytepos * BITS_PER_UNIT, GET_MODE_ALIGNMENT (mode))
3055 686179 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3056 : {
3057 98905 : tmps[i] = gen_reg_rtx (mode);
3058 98905 : emit_move_insn (tmps[i], adjust_address (src, mode, bytepos));
3059 : }
3060 486552 : else if (COMPLEX_MODE_P (mode)
3061 0 : && GET_MODE (src) == mode
3062 486552 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3063 : /* Let emit_move_complex do the bulk of the work. */
3064 0 : tmps[i] = src;
3065 486552 : else if (SCALAR_INT_MODE_P (mode)
3066 479995 : && COMPLEX_MODE_P (GET_MODE (src))
3067 9372 : && known_eq (GET_MODE_SIZE (mode),
3068 : GET_MODE_SIZE (GET_MODE (src)))
3069 489290 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3070 : {
3071 : /* When passing a complex value in an integer mode of the same
3072 : size, explicitly construct (highpart<<isize)+lowpart to
3073 : avoid spilling to memory before reload. */
3074 2738 : rtx tmp = read_complex_part (src, !BYTES_BIG_ENDIAN);
3075 2738 : scalar_int_mode imode = int_mode_for_mode (GET_MODE (tmp)).require();
3076 2738 : tmp = gen_lowpart (imode, tmp);
3077 2738 : tmp = simplify_gen_unary (ZERO_EXTEND, mode, tmp, imode);
3078 2738 : rtx result = force_reg (mode, tmp);
3079 5476 : result = expand_shift (LSHIFT_EXPR, mode, result,
3080 2738 : GET_MODE_BITSIZE (imode), NULL_RTX, 1);
3081 2738 : tmp = read_complex_part (src, BYTES_BIG_ENDIAN);
3082 2738 : tmp = gen_lowpart (imode, tmp);
3083 2738 : tmp = simplify_gen_unary (ZERO_EXTEND, mode, tmp, imode);
3084 2738 : result = simplify_gen_binary (PLUS, mode, result, tmp);
3085 2738 : tmps[i] = force_reg (mode, result);
3086 : }
3087 483814 : else if (GET_CODE (src) == CONCAT)
3088 : {
3089 11898 : poly_int64 slen = GET_MODE_SIZE (GET_MODE (src));
3090 11898 : poly_int64 slen0 = GET_MODE_SIZE (GET_MODE (XEXP (src, 0)));
3091 5949 : unsigned int elt;
3092 5949 : poly_int64 subpos;
3093 :
3094 5949 : if (can_div_trunc_p (bytepos, slen0, &elt, &subpos)
3095 5949 : && known_le (subpos + bytelen, slen0))
3096 : {
3097 : /* The following assumes that the concatenated objects all
3098 : have the same size. In this case, a simple calculation
3099 : can be used to determine the object and the bit field
3100 : to be extracted. */
3101 5930 : tmps[i] = XEXP (src, elt);
3102 5930 : if (maybe_ne (subpos, 0)
3103 5930 : || maybe_ne (subpos + bytelen, slen0)
3104 11860 : || (!CONSTANT_P (tmps[i])
3105 5930 : && (!REG_P (tmps[i]) || GET_MODE (tmps[i]) != mode)))
3106 0 : tmps[i] = extract_bit_field (tmps[i], bytelen * BITS_PER_UNIT,
3107 0 : subpos * BITS_PER_UNIT,
3108 : 1, NULL_RTX, mode, mode, false,
3109 : NULL);
3110 : }
3111 : else
3112 : {
3113 19 : rtx mem;
3114 :
3115 19 : gcc_assert (known_eq (bytepos, 0));
3116 19 : mem = assign_stack_temp (GET_MODE (src), slen);
3117 19 : emit_move_insn (mem, src);
3118 19 : tmps[i] = extract_bit_field (mem, bytelen * BITS_PER_UNIT,
3119 : 0, 1, NULL_RTX, mode, mode, false,
3120 : NULL);
3121 : }
3122 : }
3123 477865 : else if (CONSTANT_P (src) && GET_MODE (dst) != BLKmode
3124 37730 : && XVECLEN (dst, 0) > 1)
3125 37730 : tmps[i] = force_subreg (mode, src, GET_MODE (dst), bytepos);
3126 440135 : else if (CONSTANT_P (src))
3127 : {
3128 0 : if (known_eq (bytelen, ssize))
3129 0 : tmps[i] = src;
3130 : else
3131 : {
3132 : rtx first, second;
3133 :
3134 : /* TODO: const_wide_int can have sizes other than this... */
3135 0 : gcc_assert (known_eq (2 * bytelen, ssize));
3136 0 : split_double (src, &first, &second);
3137 0 : if (i)
3138 0 : tmps[i] = second;
3139 : else
3140 0 : tmps[i] = first;
3141 : }
3142 : }
3143 440135 : else if (REG_P (src) && GET_MODE (src) == mode)
3144 0 : tmps[i] = src;
3145 : else
3146 440135 : tmps[i] = extract_bit_field (src, bytelen * BITS_PER_UNIT,
3147 880270 : bytepos * BITS_PER_UNIT, 1, NULL_RTX,
3148 : mode, mode, false, NULL);
3149 :
3150 585457 : if (maybe_ne (shift, 0))
3151 : tmps[i] = expand_shift (LSHIFT_EXPR, mode, tmps[i],
3152 : shift, tmps[i], 0);
3153 : }
3154 : }
3155 :
3156 : /* Emit code to move a block SRC of type TYPE to a block DST,
3157 : where DST is non-consecutive registers represented by a PARALLEL.
3158 : SSIZE represents the total size of block ORIG_SRC in bytes, or -1
3159 : if not known. */
3160 :
3161 : void
3162 10915 : emit_group_load (rtx dst, rtx src, tree type, poly_int64 ssize)
3163 : {
3164 10915 : rtx *tmps;
3165 10915 : int i;
3166 :
3167 10915 : tmps = XALLOCAVEC (rtx, XVECLEN (dst, 0));
3168 10915 : emit_group_load_1 (tmps, dst, src, type, ssize);
3169 :
3170 : /* Copy the extracted pieces into the proper (probable) hard regs. */
3171 30473 : for (i = 0; i < XVECLEN (dst, 0); i++)
3172 : {
3173 19558 : rtx d = XEXP (XVECEXP (dst, 0, i), 0);
3174 19558 : if (d == NULL)
3175 0 : continue;
3176 19558 : emit_move_insn (d, tmps[i]);
3177 : }
3178 10915 : }
3179 :
3180 : /* Similar, but load SRC into new pseudos in a format that looks like
3181 : PARALLEL. This can later be fed to emit_group_move to get things
3182 : in the right place. */
3183 :
3184 : rtx
3185 287560 : emit_group_load_into_temps (rtx parallel, rtx src, tree type, poly_int64 ssize)
3186 : {
3187 287560 : rtvec vec;
3188 287560 : int i;
3189 :
3190 287560 : vec = rtvec_alloc (XVECLEN (parallel, 0));
3191 287560 : emit_group_load_1 (&RTVEC_ELT (vec, 0), parallel, src, type, ssize);
3192 :
3193 : /* Convert the vector to look just like the original PARALLEL, except
3194 : with the computed values. */
3195 853459 : for (i = 0; i < XVECLEN (parallel, 0); i++)
3196 : {
3197 565899 : rtx e = XVECEXP (parallel, 0, i);
3198 565899 : rtx d = XEXP (e, 0);
3199 :
3200 565899 : if (d)
3201 : {
3202 565899 : d = force_reg (GET_MODE (d), RTVEC_ELT (vec, i));
3203 565899 : e = alloc_EXPR_LIST (REG_NOTE_KIND (e), d, XEXP (e, 1));
3204 : }
3205 565899 : RTVEC_ELT (vec, i) = e;
3206 : }
3207 :
3208 287560 : return gen_rtx_PARALLEL (GET_MODE (parallel), vec);
3209 : }
3210 :
3211 : /* Emit code to move a block SRC to block DST, where SRC and DST are
3212 : non-consecutive groups of registers, each represented by a PARALLEL. */
3213 :
3214 : void
3215 290623 : emit_group_move (rtx dst, rtx src)
3216 : {
3217 290623 : int i;
3218 :
3219 290623 : gcc_assert (GET_CODE (src) == PARALLEL
3220 : && GET_CODE (dst) == PARALLEL
3221 : && XVECLEN (src, 0) == XVECLEN (dst, 0));
3222 :
3223 : /* Skip first entry if NULL. */
3224 860915 : for (i = XEXP (XVECEXP (src, 0, 0), 0) ? 0 : 1; i < XVECLEN (src, 0); i++)
3225 570292 : emit_move_insn (XEXP (XVECEXP (dst, 0, i), 0),
3226 570292 : XEXP (XVECEXP (src, 0, i), 0));
3227 290623 : }
3228 :
3229 : /* Move a group of registers represented by a PARALLEL into pseudos. */
3230 :
3231 : rtx
3232 5799 : emit_group_move_into_temps (rtx src)
3233 : {
3234 5799 : rtvec vec = rtvec_alloc (XVECLEN (src, 0));
3235 5799 : int i;
3236 :
3237 13629 : for (i = 0; i < XVECLEN (src, 0); i++)
3238 : {
3239 7830 : rtx e = XVECEXP (src, 0, i);
3240 7830 : rtx d = XEXP (e, 0);
3241 :
3242 7830 : if (d)
3243 7830 : e = alloc_EXPR_LIST (REG_NOTE_KIND (e), copy_to_reg (d), XEXP (e, 1));
3244 7830 : RTVEC_ELT (vec, i) = e;
3245 : }
3246 :
3247 5799 : return gen_rtx_PARALLEL (GET_MODE (src), vec);
3248 : }
3249 :
3250 : /* Emit code to move a block SRC to a block ORIG_DST of type TYPE,
3251 : where SRC is non-consecutive registers represented by a PARALLEL.
3252 : SSIZE represents the total size of block ORIG_DST, or -1 if not
3253 : known. */
3254 :
3255 : void
3256 65505 : emit_group_store (rtx orig_dst, rtx src, tree type ATTRIBUTE_UNUSED,
3257 : poly_int64 ssize)
3258 : {
3259 65505 : rtx *tmps, dst;
3260 65505 : int start, finish, i;
3261 65505 : machine_mode m = GET_MODE (orig_dst);
3262 :
3263 65505 : gcc_assert (GET_CODE (src) == PARALLEL);
3264 :
3265 65505 : if (!SCALAR_INT_MODE_P (m)
3266 14311 : && !MEM_P (orig_dst) && GET_CODE (orig_dst) != CONCAT)
3267 : {
3268 0 : scalar_int_mode imode;
3269 0 : if (int_mode_for_mode (GET_MODE (orig_dst)).exists (&imode))
3270 : {
3271 0 : dst = gen_reg_rtx (imode);
3272 0 : emit_group_store (dst, src, type, ssize);
3273 0 : dst = gen_lowpart (GET_MODE (orig_dst), dst);
3274 : }
3275 : else
3276 : {
3277 0 : dst = assign_stack_temp (GET_MODE (orig_dst), ssize);
3278 0 : emit_group_store (dst, src, type, ssize);
3279 : }
3280 0 : emit_move_insn (orig_dst, dst);
3281 0 : return;
3282 : }
3283 :
3284 : /* Check for a NULL entry, used to indicate that the parameter goes
3285 : both on the stack and in registers. */
3286 65505 : if (XEXP (XVECEXP (src, 0, 0), 0))
3287 : start = 0;
3288 : else
3289 0 : start = 1;
3290 65505 : finish = XVECLEN (src, 0);
3291 :
3292 65505 : tmps = XALLOCAVEC (rtx, finish);
3293 :
3294 : /* Copy the (probable) hard regs into pseudos. */
3295 188303 : for (i = start; i < finish; i++)
3296 : {
3297 122798 : rtx reg = XEXP (XVECEXP (src, 0, i), 0);
3298 122798 : if (!REG_P (reg) || REGNO (reg) < FIRST_PSEUDO_REGISTER)
3299 : {
3300 114968 : tmps[i] = gen_reg_rtx (GET_MODE (reg));
3301 114968 : emit_move_insn (tmps[i], reg);
3302 : }
3303 : else
3304 7830 : tmps[i] = reg;
3305 : }
3306 :
3307 : /* If we won't be storing directly into memory, protect the real destination
3308 : from strange tricks we might play. */
3309 65505 : dst = orig_dst;
3310 65505 : if (GET_CODE (dst) == PARALLEL)
3311 : {
3312 0 : rtx temp;
3313 :
3314 : /* We can get a PARALLEL dst if there is a conditional expression in
3315 : a return statement. In that case, the dst and src are the same,
3316 : so no action is necessary. */
3317 0 : if (rtx_equal_p (dst, src))
3318 : return;
3319 :
3320 : /* It is unclear if we can ever reach here, but we may as well handle
3321 : it. Allocate a temporary, and split this into a store/load to/from
3322 : the temporary. */
3323 0 : temp = assign_stack_temp (GET_MODE (dst), ssize);
3324 0 : emit_group_store (temp, src, type, ssize);
3325 0 : emit_group_load (dst, temp, type, ssize);
3326 0 : return;
3327 : }
3328 65505 : else if (!MEM_P (dst) && GET_CODE (dst) != CONCAT)
3329 : {
3330 50560 : machine_mode outer = GET_MODE (dst);
3331 50560 : machine_mode inner;
3332 50560 : poly_int64 bytepos;
3333 50560 : bool done = false;
3334 50560 : rtx temp;
3335 :
3336 50560 : if (!REG_P (dst) || REGNO (dst) < FIRST_PSEUDO_REGISTER)
3337 0 : dst = gen_reg_rtx (outer);
3338 :
3339 : /* Make life a bit easier for combine: if the first element of the
3340 : vector is the low part of the destination mode, use a paradoxical
3341 : subreg to initialize the destination. */
3342 50560 : if (start < finish)
3343 : {
3344 50560 : inner = GET_MODE (tmps[start]);
3345 50560 : bytepos = subreg_lowpart_offset (inner, outer);
3346 50560 : if (known_eq (rtx_to_poly_int64 (XEXP (XVECEXP (src, 0, start), 1)),
3347 : bytepos))
3348 : {
3349 50536 : temp = force_subreg (outer, tmps[start], inner, 0);
3350 50536 : if (temp)
3351 : {
3352 49864 : emit_move_insn (dst, temp);
3353 49864 : done = true;
3354 49864 : start++;
3355 : }
3356 : }
3357 : }
3358 :
3359 : /* If the first element wasn't the low part, try the last. */
3360 49864 : if (!done
3361 696 : && start < finish - 1)
3362 : {
3363 629 : inner = GET_MODE (tmps[finish - 1]);
3364 629 : bytepos = subreg_lowpart_offset (inner, outer);
3365 629 : if (known_eq (rtx_to_poly_int64 (XEXP (XVECEXP (src, 0,
3366 : finish - 1), 1)),
3367 : bytepos))
3368 : {
3369 0 : temp = force_subreg (outer, tmps[finish - 1], inner, 0);
3370 0 : if (temp)
3371 : {
3372 0 : emit_move_insn (dst, temp);
3373 0 : done = true;
3374 0 : finish--;
3375 : }
3376 : }
3377 : }
3378 :
3379 : /* Otherwise, simply initialize the result to zero. */
3380 49864 : if (!done)
3381 696 : emit_move_insn (dst, CONST0_RTX (outer));
3382 : }
3383 :
3384 : /* Process the pieces. */
3385 135007 : for (i = start; i < finish; i++)
3386 : {
3387 72934 : poly_int64 bytepos = rtx_to_poly_int64 (XEXP (XVECEXP (src, 0, i), 1));
3388 72934 : machine_mode mode = GET_MODE (tmps[i]);
3389 145868 : poly_int64 bytelen = GET_MODE_SIZE (mode);
3390 72934 : poly_uint64 adj_bytelen;
3391 72934 : rtx dest = dst;
3392 :
3393 : /* Handle trailing fragments that run over the size of the struct.
3394 : It's the target's responsibility to make sure that the fragment
3395 : cannot be strictly smaller in some cases and strictly larger
3396 : in others. */
3397 72934 : gcc_checking_assert (ordered_p (bytepos + bytelen, ssize));
3398 144918 : if (known_size_p (ssize) && maybe_gt (bytepos + bytelen, ssize))
3399 950 : adj_bytelen = ssize - bytepos;
3400 : else
3401 72934 : adj_bytelen = bytelen;
3402 :
3403 : /* Deal with destination CONCATs by either storing into one of the parts
3404 : or doing a copy after storing into a register or stack temporary. */
3405 72934 : if (GET_CODE (dst) == CONCAT)
3406 : {
3407 28280 : if (known_le (bytepos + adj_bytelen,
3408 : GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)))))
3409 : dest = XEXP (dst, 0);
3410 :
3411 17572 : else if (known_ge (bytepos, GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)))))
3412 : {
3413 10708 : bytepos -= GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)));
3414 5354 : dest = XEXP (dst, 1);
3415 : }
3416 :
3417 : else
3418 : {
3419 3432 : machine_mode dest_mode = GET_MODE (dest);
3420 3432 : machine_mode tmp_mode = GET_MODE (tmps[i]);
3421 3432 : scalar_int_mode dest_imode;
3422 :
3423 3432 : gcc_assert (known_eq (bytepos, 0) && XVECLEN (src, 0));
3424 :
3425 : /* If the source is a single scalar integer register, and the
3426 : destination has a complex mode for which a same-sized integer
3427 : mode exists, then we can take the left-justified part of the
3428 : source in the complex mode. */
3429 3432 : if (finish == start + 1
3430 3432 : && REG_P (tmps[i])
3431 3432 : && SCALAR_INT_MODE_P (tmp_mode)
3432 3432 : && COMPLEX_MODE_P (dest_mode)
3433 6864 : && int_mode_for_mode (dest_mode).exists (&dest_imode))
3434 : {
3435 3432 : const scalar_int_mode tmp_imode
3436 3432 : = as_a <scalar_int_mode> (tmp_mode);
3437 :
3438 6864 : if (GET_MODE_BITSIZE (dest_imode)
3439 3432 : < GET_MODE_BITSIZE (tmp_imode))
3440 : {
3441 59 : dest = gen_reg_rtx (dest_imode);
3442 59 : if (BYTES_BIG_ENDIAN)
3443 : tmps[i] = expand_shift (RSHIFT_EXPR, tmp_mode, tmps[i],
3444 : GET_MODE_BITSIZE (tmp_imode)
3445 : - GET_MODE_BITSIZE (dest_imode),
3446 : NULL_RTX, 1);
3447 59 : emit_move_insn (dest, gen_lowpart (dest_imode, tmps[i]));
3448 59 : dst = gen_lowpart (dest_mode, dest);
3449 : }
3450 : else
3451 3373 : dst = gen_lowpart (dest_mode, tmps[i]);
3452 : }
3453 :
3454 : /* Otherwise spill the source onto the stack using the more
3455 : aligned of the two modes. */
3456 0 : else if (GET_MODE_ALIGNMENT (dest_mode)
3457 0 : >= GET_MODE_ALIGNMENT (tmp_mode))
3458 : {
3459 0 : dest = assign_stack_temp (dest_mode,
3460 0 : GET_MODE_SIZE (dest_mode));
3461 0 : emit_move_insn (adjust_address (dest, tmp_mode, bytepos),
3462 : tmps[i]);
3463 0 : dst = dest;
3464 : }
3465 :
3466 : else
3467 : {
3468 0 : dest = assign_stack_temp (tmp_mode,
3469 0 : GET_MODE_SIZE (tmp_mode));
3470 0 : emit_move_insn (dest, tmps[i]);
3471 0 : dst = adjust_address (dest, dest_mode, bytepos);
3472 : }
3473 :
3474 : break;
3475 : }
3476 : }
3477 :
3478 : /* Handle trailing fragments that run over the size of the struct. */
3479 69502 : if (known_size_p (ssize) && maybe_gt (bytepos + bytelen, ssize))
3480 : {
3481 : /* store_bit_field always takes its value from the lsb.
3482 : Move the fragment to the lsb if it's not already there. */
3483 891 : if (
3484 : #ifdef BLOCK_REG_PADDING
3485 : BLOCK_REG_PADDING (GET_MODE (orig_dst), type, i == start)
3486 : == (BYTES_BIG_ENDIAN ? PAD_UPWARD : PAD_DOWNWARD)
3487 : #else
3488 : BYTES_BIG_ENDIAN
3489 : #endif
3490 : )
3491 : {
3492 : poly_int64 shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
3493 : tmps[i] = expand_shift (RSHIFT_EXPR, mode, tmps[i],
3494 : shift, tmps[i], 0);
3495 : }
3496 :
3497 : /* Make sure not to write past the end of the struct. */
3498 891 : store_bit_field (dest,
3499 891 : adj_bytelen * BITS_PER_UNIT, bytepos * BITS_PER_UNIT,
3500 2673 : bytepos * BITS_PER_UNIT, ssize * BITS_PER_UNIT - 1,
3501 : VOIDmode, tmps[i], false, false);
3502 : }
3503 :
3504 : /* Optimize the access just a bit. */
3505 68611 : else if (MEM_P (dest)
3506 7547 : && (!targetm.slow_unaligned_access (mode, MEM_ALIGN (dest))
3507 0 : || MEM_ALIGN (dest) >= GET_MODE_ALIGNMENT (mode))
3508 15094 : && multiple_p (bytepos * BITS_PER_UNIT,
3509 : GET_MODE_ALIGNMENT (mode))
3510 76158 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3511 7547 : emit_move_insn (adjust_address (dest, mode, bytepos), tmps[i]);
3512 :
3513 : else
3514 122128 : store_bit_field (dest, bytelen * BITS_PER_UNIT, bytepos * BITS_PER_UNIT,
3515 : 0, 0, mode, tmps[i], false, false);
3516 : }
3517 :
3518 : /* Copy from the pseudo into the (probable) hard reg. */
3519 65505 : if (orig_dst != dst)
3520 3432 : emit_move_insn (orig_dst, dst);
3521 : }
3522 :
3523 : /* Return a form of X that does not use a PARALLEL. TYPE is the type
3524 : of the value stored in X. */
3525 :
3526 : rtx
3527 335317 : maybe_emit_group_store (rtx x, tree type)
3528 : {
3529 335317 : machine_mode mode = TYPE_MODE (type);
3530 335317 : gcc_checking_assert (GET_MODE (x) == VOIDmode || GET_MODE (x) == mode);
3531 335317 : if (GET_CODE (x) == PARALLEL)
3532 : {
3533 0 : rtx result = gen_reg_rtx (mode);
3534 0 : emit_group_store (result, x, type, int_size_in_bytes (type));
3535 0 : return result;
3536 : }
3537 : return x;
3538 : }
3539 :
3540 : /* Copy a BLKmode object of TYPE out of a register SRCREG into TARGET.
3541 :
3542 : This is used on targets that return BLKmode values in registers. */
3543 :
3544 : static void
3545 256 : copy_blkmode_from_reg (rtx target, rtx srcreg, tree type)
3546 : {
3547 256 : unsigned HOST_WIDE_INT bytes = int_size_in_bytes (type);
3548 256 : rtx src = NULL, dst = NULL;
3549 256 : unsigned HOST_WIDE_INT bitsize = MIN (TYPE_ALIGN (type), BITS_PER_WORD);
3550 256 : unsigned HOST_WIDE_INT bitpos, xbitpos, padding_correction = 0;
3551 : /* No current ABI uses variable-sized modes to pass a BLKmnode type. */
3552 256 : fixed_size_mode mode = as_a <fixed_size_mode> (GET_MODE (srcreg));
3553 256 : fixed_size_mode tmode = as_a <fixed_size_mode> (GET_MODE (target));
3554 256 : fixed_size_mode copy_mode;
3555 :
3556 : /* BLKmode registers created in the back-end shouldn't have survived. */
3557 256 : gcc_assert (mode != BLKmode);
3558 :
3559 : /* If the structure doesn't take up a whole number of words, see whether
3560 : SRCREG is padded on the left or on the right. If it's on the left,
3561 : set PADDING_CORRECTION to the number of bits to skip.
3562 :
3563 : In most ABIs, the structure will be returned at the least end of
3564 : the register, which translates to right padding on little-endian
3565 : targets and left padding on big-endian targets. The opposite
3566 : holds if the structure is returned at the most significant
3567 : end of the register. */
3568 256 : if (bytes % UNITS_PER_WORD != 0
3569 256 : && (targetm.calls.return_in_msb (type)
3570 225 : ? !BYTES_BIG_ENDIAN
3571 : : BYTES_BIG_ENDIAN))
3572 0 : padding_correction
3573 0 : = (BITS_PER_WORD - ((bytes % UNITS_PER_WORD) * BITS_PER_UNIT));
3574 :
3575 : /* We can use a single move if we have an exact mode for the size. */
3576 256 : else if (MEM_P (target)
3577 256 : && (!targetm.slow_unaligned_access (mode, MEM_ALIGN (target))
3578 0 : || MEM_ALIGN (target) >= GET_MODE_ALIGNMENT (mode))
3579 512 : && bytes == GET_MODE_SIZE (mode))
3580 : {
3581 38 : emit_move_insn (adjust_address (target, mode, 0), srcreg);
3582 38 : return;
3583 : }
3584 :
3585 : /* And if we additionally have the same mode for a register. */
3586 218 : else if (REG_P (target)
3587 0 : && GET_MODE (target) == mode
3588 218 : && bytes == GET_MODE_SIZE (mode))
3589 : {
3590 0 : emit_move_insn (target, srcreg);
3591 0 : return;
3592 : }
3593 :
3594 : /* This code assumes srcreg is at least a full word. If it isn't, copy it
3595 : into a new pseudo which is a full word. */
3596 436 : if (GET_MODE_SIZE (mode) < UNITS_PER_WORD)
3597 : {
3598 77 : srcreg = convert_to_mode (word_mode, srcreg, TYPE_UNSIGNED (type));
3599 77 : mode = word_mode;
3600 : }
3601 :
3602 : /* Copy the structure BITSIZE bits at a time. If the target lives in
3603 : memory, take care of not reading/writing past its end by selecting
3604 : a copy mode suited to BITSIZE. This should always be possible given
3605 : how it is computed.
3606 :
3607 : If the target lives in register, make sure not to select a copy mode
3608 : larger than the mode of the register.
3609 :
3610 : We could probably emit more efficient code for machines which do not use
3611 : strict alignment, but it doesn't seem worth the effort at the current
3612 : time. */
3613 :
3614 218 : copy_mode = word_mode;
3615 218 : if (MEM_P (target))
3616 : {
3617 218 : opt_scalar_int_mode mem_mode = int_mode_for_size (bitsize, 1);
3618 218 : if (mem_mode.exists ())
3619 218 : copy_mode = mem_mode.require ();
3620 : }
3621 0 : else if (REG_P (target) && GET_MODE_BITSIZE (tmode) < BITS_PER_WORD)
3622 : copy_mode = tmode;
3623 :
3624 218 : for (bitpos = 0, xbitpos = padding_correction;
3625 1068 : bitpos < bytes * BITS_PER_UNIT;
3626 850 : bitpos += bitsize, xbitpos += bitsize)
3627 : {
3628 : /* We need a new source operand each time xbitpos is on a
3629 : word boundary and when xbitpos == padding_correction
3630 : (the first time through). */
3631 850 : if (xbitpos % BITS_PER_WORD == 0 || xbitpos == padding_correction)
3632 218 : src = operand_subword_force (srcreg, xbitpos / BITS_PER_WORD, mode);
3633 :
3634 : /* We need a new destination operand each time bitpos is on
3635 : a word boundary. */
3636 850 : if (REG_P (target) && GET_MODE_BITSIZE (tmode) < BITS_PER_WORD)
3637 : dst = target;
3638 850 : else if (bitpos % BITS_PER_WORD == 0)
3639 218 : dst = operand_subword (target, bitpos / BITS_PER_WORD, 1, tmode);
3640 :
3641 : /* Use xbitpos for the source extraction (right justified) and
3642 : bitpos for the destination store (left justified). */
3643 850 : store_bit_field (dst, bitsize, bitpos % BITS_PER_WORD, 0, 0, copy_mode,
3644 850 : extract_bit_field (src, bitsize,
3645 850 : xbitpos % BITS_PER_WORD, 1,
3646 : NULL_RTX, copy_mode, copy_mode,
3647 : false, NULL),
3648 : false, false);
3649 : }
3650 : }
3651 :
3652 : /* Copy BLKmode value SRC into a register of mode MODE_IN. Return the
3653 : register if it contains any data, otherwise return null.
3654 :
3655 : This is used on targets that return BLKmode values in registers. */
3656 :
3657 : rtx
3658 3511 : copy_blkmode_to_reg (machine_mode mode_in, tree src)
3659 : {
3660 3511 : int i, n_regs;
3661 3511 : unsigned HOST_WIDE_INT bitpos, xbitpos, padding_correction = 0, bytes;
3662 3511 : unsigned int bitsize;
3663 3511 : rtx *dst_words, dst, x, src_word = NULL_RTX, dst_word = NULL_RTX;
3664 : /* No current ABI uses variable-sized modes to pass a BLKmnode type. */
3665 3511 : fixed_size_mode mode = as_a <fixed_size_mode> (mode_in);
3666 3511 : fixed_size_mode dst_mode;
3667 3511 : scalar_int_mode min_mode;
3668 :
3669 3511 : gcc_assert (TYPE_MODE (TREE_TYPE (src)) == BLKmode);
3670 :
3671 3511 : x = expand_normal (src);
3672 :
3673 3511 : bytes = arg_int_size_in_bytes (TREE_TYPE (src));
3674 3511 : if (bytes == 0)
3675 : return NULL_RTX;
3676 :
3677 : /* If the structure doesn't take up a whole number of words, see
3678 : whether the register value should be padded on the left or on
3679 : the right. Set PADDING_CORRECTION to the number of padding
3680 : bits needed on the left side.
3681 :
3682 : In most ABIs, the structure will be returned at the least end of
3683 : the register, which translates to right padding on little-endian
3684 : targets and left padding on big-endian targets. The opposite
3685 : holds if the structure is returned at the most significant
3686 : end of the register. */
3687 1224 : if (bytes % UNITS_PER_WORD != 0
3688 1224 : && (targetm.calls.return_in_msb (TREE_TYPE (src))
3689 1188 : ? !BYTES_BIG_ENDIAN
3690 : : BYTES_BIG_ENDIAN))
3691 0 : padding_correction = (BITS_PER_WORD - ((bytes % UNITS_PER_WORD)
3692 0 : * BITS_PER_UNIT));
3693 :
3694 1224 : n_regs = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
3695 1224 : dst_words = XALLOCAVEC (rtx, n_regs);
3696 1224 : bitsize = MIN (TYPE_ALIGN (TREE_TYPE (src)), BITS_PER_WORD);
3697 1224 : min_mode = smallest_int_mode_for_size (bitsize).require ();
3698 :
3699 : /* Copy the structure BITSIZE bits at a time. */
3700 1224 : for (bitpos = 0, xbitpos = padding_correction;
3701 3974 : bitpos < bytes * BITS_PER_UNIT;
3702 2750 : bitpos += bitsize, xbitpos += bitsize)
3703 : {
3704 : /* We need a new destination pseudo each time xbitpos is
3705 : on a word boundary and when xbitpos == padding_correction
3706 : (the first time through). */
3707 2750 : if (xbitpos % BITS_PER_WORD == 0
3708 1523 : || xbitpos == padding_correction)
3709 : {
3710 : /* Generate an appropriate register. */
3711 1227 : dst_word = gen_reg_rtx (word_mode);
3712 1227 : dst_words[xbitpos / BITS_PER_WORD] = dst_word;
3713 :
3714 : /* Clear the destination before we move anything into it. */
3715 1227 : emit_move_insn (dst_word, CONST0_RTX (word_mode));
3716 : }
3717 :
3718 : /* Find the largest integer mode that can be used to copy all or as
3719 : many bits as possible of the structure if the target supports larger
3720 : copies. There are too many corner cases here w.r.t to alignments on
3721 : the read/writes. So if there is any padding just use single byte
3722 : operations. */
3723 2750 : opt_scalar_int_mode mode_iter;
3724 2750 : if (padding_correction == 0 && !STRICT_ALIGNMENT)
3725 : {
3726 7488 : FOR_EACH_MODE_FROM (mode_iter, min_mode)
3727 : {
3728 7488 : unsigned int msize = GET_MODE_BITSIZE (mode_iter.require ());
3729 7488 : if (msize <= ((bytes * BITS_PER_UNIT) - bitpos)
3730 4741 : && msize <= BITS_PER_WORD)
3731 4738 : bitsize = msize;
3732 : else
3733 : break;
3734 : }
3735 : }
3736 :
3737 : /* We need a new source operand each time bitpos is on a word
3738 : boundary. */
3739 2750 : if (bitpos % BITS_PER_WORD == 0)
3740 1227 : src_word = operand_subword_force (x, bitpos / BITS_PER_WORD, BLKmode);
3741 :
3742 : /* Use bitpos for the source extraction (left justified) and
3743 : xbitpos for the destination store (right justified). */
3744 5500 : store_bit_field (dst_word, bitsize, xbitpos % BITS_PER_WORD,
3745 : 0, 0, word_mode,
3746 2750 : extract_bit_field (src_word, bitsize,
3747 2750 : bitpos % BITS_PER_WORD, 1,
3748 : NULL_RTX, word_mode, word_mode,
3749 : false, NULL),
3750 : false, false);
3751 : }
3752 :
3753 1224 : if (mode == BLKmode)
3754 : {
3755 : /* Find the smallest integer mode large enough to hold the
3756 : entire structure. */
3757 0 : opt_scalar_int_mode mode_iter;
3758 0 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
3759 0 : if (GET_MODE_SIZE (mode_iter.require ()) >= bytes)
3760 : break;
3761 :
3762 : /* A suitable mode should have been found. */
3763 0 : mode = mode_iter.require ();
3764 : }
3765 :
3766 3672 : if (GET_MODE_SIZE (mode) < GET_MODE_SIZE (word_mode))
3767 : dst_mode = word_mode;
3768 : else
3769 498 : dst_mode = mode;
3770 1224 : dst = gen_reg_rtx (dst_mode);
3771 :
3772 3675 : for (i = 0; i < n_regs; i++)
3773 1227 : emit_move_insn (operand_subword (dst, i, 0, dst_mode), dst_words[i]);
3774 :
3775 1224 : if (mode != dst_mode)
3776 726 : dst = gen_lowpart (mode, dst);
3777 :
3778 : return dst;
3779 : }
3780 :
3781 : /* Add a USE expression for REG to the (possibly empty) list pointed
3782 : to by CALL_FUSAGE. REG must denote a hard register. */
3783 :
3784 : void
3785 11593819 : use_reg_mode (rtx *call_fusage, rtx reg, machine_mode mode)
3786 : {
3787 11593819 : gcc_assert (REG_P (reg));
3788 :
3789 11593819 : if (!HARD_REGISTER_P (reg))
3790 : return;
3791 :
3792 11593819 : *call_fusage
3793 11593819 : = gen_rtx_EXPR_LIST (mode, gen_rtx_USE (VOIDmode, reg), *call_fusage);
3794 : }
3795 :
3796 : /* Add a CLOBBER expression for REG to the (possibly empty) list pointed
3797 : to by CALL_FUSAGE. REG must denote a hard register. */
3798 :
3799 : void
3800 0 : clobber_reg_mode (rtx *call_fusage, rtx reg, machine_mode mode)
3801 : {
3802 0 : gcc_assert (REG_P (reg) && REGNO (reg) < FIRST_PSEUDO_REGISTER);
3803 :
3804 0 : *call_fusage
3805 0 : = gen_rtx_EXPR_LIST (mode, gen_rtx_CLOBBER (VOIDmode, reg), *call_fusage);
3806 0 : }
3807 :
3808 : /* Add USE expressions to *CALL_FUSAGE for each of NREGS consecutive regs,
3809 : starting at REGNO. All of these registers must be hard registers. */
3810 :
3811 : void
3812 1608 : use_regs (rtx *call_fusage, int regno, int nregs)
3813 : {
3814 1608 : int i;
3815 :
3816 1608 : gcc_assert (regno + nregs <= FIRST_PSEUDO_REGISTER);
3817 :
3818 3224 : for (i = 0; i < nregs; i++)
3819 1616 : use_reg (call_fusage, regno_reg_rtx[regno + i]);
3820 1608 : }
3821 :
3822 : /* Add USE expressions to *CALL_FUSAGE for each REG contained in the
3823 : PARALLEL REGS. This is for calls that pass values in multiple
3824 : non-contiguous locations. The Irix 6 ABI has examples of this. */
3825 :
3826 : void
3827 294333 : use_group_regs (rtx *call_fusage, rtx regs)
3828 : {
3829 294333 : int i;
3830 :
3831 873778 : for (i = 0; i < XVECLEN (regs, 0); i++)
3832 : {
3833 579445 : rtx reg = XEXP (XVECEXP (regs, 0, i), 0);
3834 :
3835 : /* A NULL entry means the parameter goes both on the stack and in
3836 : registers. This can also be a MEM for targets that pass values
3837 : partially on the stack and partially in registers. */
3838 579445 : if (reg != 0 && REG_P (reg))
3839 579445 : use_reg (call_fusage, reg);
3840 : }
3841 294333 : }
3842 :
3843 : /* Return the defining gimple statement for SSA_NAME NAME if it is an
3844 : assignment and the code of the expression on the RHS is CODE. Return
3845 : NULL otherwise. */
3846 :
3847 : static gimple *
3848 12022972 : get_def_for_expr (tree name, enum tree_code code)
3849 : {
3850 12022972 : gimple *def_stmt;
3851 :
3852 12022972 : if (TREE_CODE (name) != SSA_NAME)
3853 : return NULL;
3854 :
3855 9356964 : def_stmt = get_gimple_for_ssa_name (name);
3856 9356964 : if (!def_stmt
3857 2007370 : || !is_gimple_assign (def_stmt)
3858 11362875 : || gimple_assign_rhs_code (def_stmt) != code)
3859 : return NULL;
3860 :
3861 : return def_stmt;
3862 : }
3863 :
3864 : /* Return the defining gimple statement for SSA_NAME NAME if it is an
3865 : assignment and the class of the expression on the RHS is CLASS. Return
3866 : NULL otherwise. */
3867 :
3868 : static gimple *
3869 16656 : get_def_for_expr_class (tree name, enum tree_code_class tclass)
3870 : {
3871 16656 : gimple *def_stmt;
3872 :
3873 16656 : if (TREE_CODE (name) != SSA_NAME)
3874 : return NULL;
3875 :
3876 16656 : def_stmt = get_gimple_for_ssa_name (name);
3877 16656 : if (!def_stmt
3878 14563 : || !is_gimple_assign (def_stmt)
3879 31219 : || TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) != tclass)
3880 : return NULL;
3881 :
3882 : return def_stmt;
3883 : }
3884 :
3885 : /* Write zeros through the storage of OBJECT. If OBJECT has BLKmode, SIZE is
3886 : its length in bytes. */
3887 :
3888 : rtx
3889 157149 : clear_storage_hints (rtx object, rtx size, enum block_op_methods method,
3890 : unsigned int expected_align, HOST_WIDE_INT expected_size,
3891 : unsigned HOST_WIDE_INT min_size,
3892 : unsigned HOST_WIDE_INT max_size,
3893 : unsigned HOST_WIDE_INT probable_max_size,
3894 : unsigned ctz_size)
3895 : {
3896 157149 : machine_mode mode = GET_MODE (object);
3897 157149 : unsigned int align;
3898 :
3899 157149 : gcc_assert (method == BLOCK_OP_NORMAL || method == BLOCK_OP_TAILCALL);
3900 :
3901 : /* If OBJECT is not BLKmode and SIZE is the same size as its mode,
3902 : just move a zero. Otherwise, do this a piece at a time. */
3903 157149 : poly_int64 size_val;
3904 157149 : if (mode != BLKmode
3905 57337 : && poly_int_rtx_p (size, &size_val)
3906 214486 : && known_eq (size_val, GET_MODE_SIZE (mode)))
3907 : {
3908 57337 : rtx zero = CONST0_RTX (mode);
3909 57337 : if (zero != NULL)
3910 : {
3911 57337 : emit_move_insn (object, zero);
3912 57337 : return NULL;
3913 : }
3914 :
3915 0 : if (COMPLEX_MODE_P (mode))
3916 : {
3917 0 : zero = CONST0_RTX (GET_MODE_INNER (mode));
3918 0 : if (zero != NULL)
3919 : {
3920 0 : write_complex_part (object, zero, 0, true);
3921 0 : write_complex_part (object, zero, 1, false);
3922 0 : return NULL;
3923 : }
3924 : }
3925 : }
3926 :
3927 99812 : if (size == const0_rtx)
3928 : return NULL;
3929 :
3930 99811 : align = MEM_ALIGN (object);
3931 :
3932 99811 : if (CONST_INT_P (size)
3933 192290 : && targetm.use_by_pieces_infrastructure_p (INTVAL (size), align,
3934 : CLEAR_BY_PIECES,
3935 92479 : optimize_insn_for_speed_p ()))
3936 71584 : clear_by_pieces (object, INTVAL (size), align);
3937 28227 : else if (set_storage_via_setmem (object, size, const0_rtx, align,
3938 : expected_align, expected_size,
3939 : min_size, max_size, probable_max_size))
3940 : ;
3941 16069 : else if (try_store_by_multiple_pieces (object, size, ctz_size,
3942 : min_size, max_size,
3943 : NULL_RTX, 0, align))
3944 : ;
3945 15805 : else if (ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (object)))
3946 15805 : return set_storage_via_libcall (object, size, const0_rtx,
3947 15805 : method == BLOCK_OP_TAILCALL);
3948 : else
3949 0 : gcc_unreachable ();
3950 :
3951 : return NULL;
3952 : }
3953 :
3954 : rtx
3955 110308 : clear_storage (rtx object, rtx size, enum block_op_methods method)
3956 : {
3957 110308 : unsigned HOST_WIDE_INT max, min = 0;
3958 110308 : if (GET_CODE (size) == CONST_INT)
3959 110308 : min = max = UINTVAL (size);
3960 : else
3961 0 : max = GET_MODE_MASK (GET_MODE (size));
3962 110308 : return clear_storage_hints (object, size, method, 0, -1, min, max, max, 0);
3963 : }
3964 :
3965 :
3966 : /* A subroutine of clear_storage. Expand a call to memset.
3967 : Return the return value of memset, 0 otherwise. */
3968 :
3969 : rtx
3970 15806 : set_storage_via_libcall (rtx object, rtx size, rtx val, bool tailcall)
3971 : {
3972 15806 : tree call_expr, fn, object_tree, size_tree, val_tree;
3973 15806 : machine_mode size_mode;
3974 :
3975 15806 : object = copy_addr_to_reg (XEXP (object, 0));
3976 15806 : object_tree = make_tree (ptr_type_node, object);
3977 :
3978 15806 : if (!CONST_INT_P (val))
3979 0 : val = convert_to_mode (TYPE_MODE (integer_type_node), val, 1);
3980 15806 : val_tree = make_tree (integer_type_node, val);
3981 :
3982 15806 : size_mode = TYPE_MODE (sizetype);
3983 15806 : size = convert_to_mode (size_mode, size, 1);
3984 15806 : size = copy_to_mode_reg (size_mode, size);
3985 15806 : size_tree = make_tree (sizetype, size);
3986 :
3987 : /* It is incorrect to use the libcall calling conventions for calls to
3988 : memset because it can be provided by the user. */
3989 15806 : fn = builtin_decl_implicit (BUILT_IN_MEMSET);
3990 15806 : call_expr = build_call_expr (fn, 3, object_tree, val_tree, size_tree);
3991 15806 : CALL_EXPR_TAILCALL (call_expr) = tailcall;
3992 :
3993 15806 : return expand_call (call_expr, NULL_RTX, false);
3994 : }
3995 :
3996 : /* Expand a setmem pattern; return true if successful. */
3997 :
3998 : bool
3999 36670 : set_storage_via_setmem (rtx object, rtx size, rtx val, unsigned int align,
4000 : unsigned int expected_align, HOST_WIDE_INT expected_size,
4001 : unsigned HOST_WIDE_INT min_size,
4002 : unsigned HOST_WIDE_INT max_size,
4003 : unsigned HOST_WIDE_INT probable_max_size)
4004 : {
4005 : /* Try the most limited insn first, because there's no point
4006 : including more than one in the machine description unless
4007 : the more limited one has some advantage. */
4008 :
4009 36670 : if (expected_align < align)
4010 : expected_align = align;
4011 36670 : if (expected_size != -1)
4012 : {
4013 8 : if ((unsigned HOST_WIDE_INT)expected_size > max_size)
4014 0 : expected_size = max_size;
4015 8 : if ((unsigned HOST_WIDE_INT)expected_size < min_size)
4016 0 : expected_size = min_size;
4017 : }
4018 :
4019 36670 : opt_scalar_int_mode mode_iter;
4020 222095 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
4021 : {
4022 199827 : scalar_int_mode mode = mode_iter.require ();
4023 199827 : enum insn_code code = direct_optab_handler (setmem_optab, mode);
4024 :
4025 199827 : if (code != CODE_FOR_nothing
4026 : /* We don't need MODE to be narrower than BITS_PER_HOST_WIDE_INT
4027 : here because if SIZE is less than the mode mask, as it is
4028 : returned by the macro, it will definitely be less than the
4029 : actual mode mask. Since SIZE is within the Pmode address
4030 : space, we limit MODE to Pmode. */
4031 199827 : && ((CONST_INT_P (size)
4032 31697 : && ((unsigned HOST_WIDE_INT) INTVAL (size)
4033 31697 : <= (GET_MODE_MASK (mode) >> 1)))
4034 26364 : || max_size <= (GET_MODE_MASK (mode) >> 1)
4035 31368 : || GET_MODE_BITSIZE (mode) >= GET_MODE_BITSIZE (Pmode)))
4036 : {
4037 47555 : class expand_operand ops[9];
4038 47555 : unsigned int nops;
4039 :
4040 47555 : nops = insn_data[(int) code].n_generator_args;
4041 47555 : gcc_assert (nops == 4 || nops == 6 || nops == 8 || nops == 9);
4042 :
4043 47555 : create_fixed_operand (&ops[0], object);
4044 : /* The check above guarantees that this size conversion is valid. */
4045 47555 : create_convert_operand_to (&ops[1], size, mode, true);
4046 47555 : create_convert_operand_from (&ops[2], val, byte_mode, true);
4047 47555 : create_integer_operand (&ops[3], align / BITS_PER_UNIT);
4048 47555 : if (nops >= 6)
4049 : {
4050 47555 : create_integer_operand (&ops[4], expected_align / BITS_PER_UNIT);
4051 47555 : create_integer_operand (&ops[5], expected_size);
4052 : }
4053 47555 : if (nops >= 8)
4054 : {
4055 47555 : create_integer_operand (&ops[6], min_size);
4056 : /* If we cannot represent the maximal size,
4057 : make parameter NULL. */
4058 47555 : if ((HOST_WIDE_INT) max_size != -1)
4059 44420 : create_integer_operand (&ops[7], max_size);
4060 : else
4061 3135 : create_fixed_operand (&ops[7], NULL);
4062 : }
4063 47555 : if (nops == 9)
4064 : {
4065 : /* If we cannot represent the maximal size,
4066 : make parameter NULL. */
4067 47555 : if ((HOST_WIDE_INT) probable_max_size != -1)
4068 44580 : create_integer_operand (&ops[8], probable_max_size);
4069 : else
4070 2975 : create_fixed_operand (&ops[8], NULL);
4071 : }
4072 47555 : gcc_assert (min_size != max_size
4073 : || rtx_equal_p (ops[1].value, GEN_INT (min_size)));
4074 47555 : if (maybe_expand_insn (code, nops, ops))
4075 14402 : return true;
4076 : }
4077 : }
4078 :
4079 : return false;
4080 : }
4081 :
4082 :
4083 : /* Write to one of the components of the complex value CPLX. Write VAL to
4084 : the real part if IMAG_P is false, and the imaginary part if its true.
4085 : If UNDEFINED_P then the value in CPLX is currently undefined. */
4086 :
4087 : void
4088 571756 : write_complex_part (rtx cplx, rtx val, bool imag_p, bool undefined_p)
4089 : {
4090 571756 : machine_mode cmode;
4091 571756 : scalar_mode imode;
4092 571756 : unsigned ibitsize;
4093 :
4094 571756 : if (GET_CODE (cplx) == CONCAT)
4095 : {
4096 513910 : emit_move_insn (XEXP (cplx, imag_p), val);
4097 513910 : return;
4098 : }
4099 :
4100 57846 : cmode = GET_MODE (cplx);
4101 57846 : imode = GET_MODE_INNER (cmode);
4102 57846 : ibitsize = GET_MODE_BITSIZE (imode);
4103 :
4104 : /* For MEMs simplify_gen_subreg may generate an invalid new address
4105 : because, e.g., the original address is considered mode-dependent
4106 : by the target, which restricts simplify_subreg from invoking
4107 : adjust_address_nv. Instead of preparing fallback support for an
4108 : invalid address, we call adjust_address_nv directly. */
4109 57846 : if (MEM_P (cplx))
4110 : {
4111 68340 : emit_move_insn (adjust_address_nv (cplx, imode,
4112 : imag_p ? GET_MODE_SIZE (imode) : 0),
4113 : val);
4114 45560 : return;
4115 : }
4116 :
4117 : /* If the sub-object is at least word sized, then we know that subregging
4118 : will work. This special case is important, since store_bit_field
4119 : wants to operate on integer modes, and there's rarely an OImode to
4120 : correspond to TCmode. */
4121 12286 : if (ibitsize >= BITS_PER_WORD
4122 : /* For hard regs we have exact predicates. Assume we can split
4123 : the original object if it spans an even number of hard regs.
4124 : This special case is important for SCmode on 64-bit platforms
4125 : where the natural size of floating-point regs is 32-bit. */
4126 12286 : || (REG_P (cplx)
4127 7324 : && REGNO (cplx) < FIRST_PSEUDO_REGISTER
4128 7122 : && REG_NREGS (cplx) % 2 == 0))
4129 : {
4130 4978 : rtx part = simplify_gen_subreg (imode, cplx, cmode,
4131 7467 : imag_p ? GET_MODE_SIZE (imode) : 0);
4132 4978 : if (part)
4133 : {
4134 4978 : emit_move_insn (part, val);
4135 4978 : return;
4136 : }
4137 : else
4138 : /* simplify_gen_subreg may fail for sub-word MEMs. */
4139 0 : gcc_assert (MEM_P (cplx) && ibitsize < BITS_PER_WORD);
4140 : }
4141 :
4142 10962 : store_bit_field (cplx, ibitsize, imag_p ? ibitsize : 0, 0, 0, imode, val,
4143 : false, undefined_p);
4144 : }
4145 :
4146 : /* Extract one of the components of the complex value CPLX. Extract the
4147 : real part if IMAG_P is false, and the imaginary part if it's true. */
4148 :
4149 : rtx
4150 559215 : read_complex_part (rtx cplx, bool imag_p)
4151 : {
4152 559215 : machine_mode cmode;
4153 559215 : scalar_mode imode;
4154 559215 : unsigned ibitsize;
4155 :
4156 559215 : if (GET_CODE (cplx) == CONCAT)
4157 345638 : return XEXP (cplx, imag_p);
4158 :
4159 213577 : cmode = GET_MODE (cplx);
4160 213577 : imode = GET_MODE_INNER (cmode);
4161 213577 : ibitsize = GET_MODE_BITSIZE (imode);
4162 :
4163 : /* Special case reads from complex constants that got spilled to memory. */
4164 213577 : if (MEM_P (cplx) && GET_CODE (XEXP (cplx, 0)) == SYMBOL_REF)
4165 : {
4166 37949 : tree decl = SYMBOL_REF_DECL (XEXP (cplx, 0));
4167 37949 : if (decl && TREE_CODE (decl) == COMPLEX_CST)
4168 : {
4169 0 : tree part = imag_p ? TREE_IMAGPART (decl) : TREE_REALPART (decl);
4170 0 : if (CONSTANT_CLASS_P (part))
4171 0 : return expand_expr (part, NULL_RTX, imode, EXPAND_NORMAL);
4172 : }
4173 : }
4174 :
4175 : /* For MEMs simplify_gen_subreg may generate an invalid new address
4176 : because, e.g., the original address is considered mode-dependent
4177 : by the target, which restricts simplify_subreg from invoking
4178 : adjust_address_nv. Instead of preparing fallback support for an
4179 : invalid address, we call adjust_address_nv directly. */
4180 213577 : if (MEM_P (cplx))
4181 236989 : return adjust_address_nv (cplx, imode,
4182 : imag_p ? GET_MODE_SIZE (imode) : 0);
4183 :
4184 : /* If the sub-object is at least word sized, then we know that subregging
4185 : will work. This special case is important, since extract_bit_field
4186 : wants to operate on integer modes, and there's rarely an OImode to
4187 : correspond to TCmode. */
4188 55418 : if (ibitsize >= BITS_PER_WORD
4189 : /* For hard regs we have exact predicates. Assume we can split
4190 : the original object if it spans an even number of hard regs.
4191 : This special case is important for SCmode on 64-bit platforms
4192 : where the natural size of floating-point regs is 32-bit. */
4193 55418 : || (REG_P (cplx)
4194 34804 : && REGNO (cplx) < FIRST_PSEUDO_REGISTER
4195 292 : && REG_NREGS (cplx) % 2 == 0))
4196 : {
4197 10229 : rtx ret = simplify_gen_subreg (imode, cplx, cmode,
4198 15343 : imag_p ? GET_MODE_SIZE (imode) : 0);
4199 10229 : if (ret)
4200 : return ret;
4201 : else
4202 : /* simplify_gen_subreg may fail for sub-word MEMs. */
4203 0 : gcc_assert (MEM_P (cplx) && ibitsize < BITS_PER_WORD);
4204 : }
4205 :
4206 67781 : return extract_bit_field (cplx, ibitsize, imag_p ? ibitsize : 0,
4207 : true, NULL_RTX, imode, imode, false, NULL);
4208 : }
4209 :
4210 : /* A subroutine of emit_move_insn_1. Yet another lowpart generator.
4211 : NEW_MODE and OLD_MODE are the same size. Return NULL if X cannot be
4212 : represented in NEW_MODE. If FORCE is true, this will never happen, as
4213 : we'll force-create a SUBREG if needed. */
4214 :
4215 : static rtx
4216 277662 : emit_move_change_mode (machine_mode new_mode,
4217 : machine_mode old_mode, rtx x, bool force)
4218 : {
4219 277662 : rtx ret;
4220 :
4221 277662 : if (push_operand (x, GET_MODE (x)))
4222 : {
4223 1318 : ret = gen_rtx_MEM (new_mode, XEXP (x, 0));
4224 1318 : MEM_COPY_ATTRIBUTES (ret, x);
4225 : }
4226 276344 : else if (MEM_P (x))
4227 : {
4228 : /* We don't have to worry about changing the address since the
4229 : size in bytes is supposed to be the same. */
4230 52783 : if (reload_in_progress)
4231 : {
4232 : /* Copy the MEM to change the mode and move any
4233 : substitutions from the old MEM to the new one. */
4234 0 : ret = adjust_address_nv (x, new_mode, 0);
4235 0 : copy_replacements (x, ret);
4236 : }
4237 : else
4238 52783 : ret = adjust_address (x, new_mode, 0);
4239 : }
4240 : else
4241 : {
4242 : /* Note that we do want simplify_subreg's behavior of validating
4243 : that the new mode is ok for a hard register. If we were to use
4244 : simplify_gen_subreg, we would create the subreg, but would
4245 : probably run into the target not being able to implement it. */
4246 : /* Except, of course, when FORCE is true, when this is exactly what
4247 : we want. Which is needed for CCmodes on some targets. */
4248 223561 : if (force)
4249 223561 : ret = simplify_gen_subreg (new_mode, x, old_mode, 0);
4250 : else
4251 0 : ret = simplify_subreg (new_mode, x, old_mode, 0);
4252 : }
4253 :
4254 277662 : return ret;
4255 : }
4256 :
4257 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X using
4258 : an integer mode of the same size as MODE. Returns the instruction
4259 : emitted, or NULL if such a move could not be generated. */
4260 :
4261 : static rtx_insn *
4262 138831 : emit_move_via_integer (machine_mode mode, rtx x, rtx y, bool force)
4263 : {
4264 138831 : scalar_int_mode imode;
4265 138831 : enum insn_code code;
4266 :
4267 : /* There must exist a mode of the exact size we require. */
4268 138831 : if (!int_mode_for_mode (mode).exists (&imode))
4269 0 : return NULL;
4270 :
4271 : /* The target must support moves in this mode. */
4272 138831 : code = optab_handler (mov_optab, imode);
4273 138831 : if (code == CODE_FOR_nothing)
4274 : return NULL;
4275 :
4276 138831 : x = emit_move_change_mode (imode, mode, x, force);
4277 138831 : if (x == NULL_RTX)
4278 : return NULL;
4279 138831 : y = emit_move_change_mode (imode, mode, y, force);
4280 138831 : if (y == NULL_RTX)
4281 : return NULL;
4282 138831 : return emit_insn (GEN_FCN (code) (x, y));
4283 : }
4284 :
4285 : /* A subroutine of emit_move_insn_1. X is a push_operand in MODE.
4286 : Return an equivalent MEM that does not use an auto-increment. */
4287 :
4288 : rtx
4289 3636 : emit_move_resolve_push (machine_mode mode, rtx x)
4290 : {
4291 3636 : enum rtx_code code = GET_CODE (XEXP (x, 0));
4292 3636 : rtx temp;
4293 :
4294 7272 : poly_int64 adjust = GET_MODE_SIZE (mode);
4295 : #ifdef PUSH_ROUNDING
4296 3636 : adjust = PUSH_ROUNDING (adjust);
4297 : #endif
4298 3636 : if (code == PRE_DEC || code == POST_DEC)
4299 3240 : adjust = -adjust;
4300 396 : else if (code == PRE_MODIFY || code == POST_MODIFY)
4301 : {
4302 396 : rtx expr = XEXP (XEXP (x, 0), 1);
4303 :
4304 396 : gcc_assert (GET_CODE (expr) == PLUS || GET_CODE (expr) == MINUS);
4305 396 : poly_int64 val = rtx_to_poly_int64 (XEXP (expr, 1));
4306 396 : if (GET_CODE (expr) == MINUS)
4307 0 : val = -val;
4308 396 : gcc_assert (known_eq (adjust, val) || known_eq (adjust, -val));
4309 : adjust = val;
4310 : }
4311 :
4312 : /* Do not use anti_adjust_stack, since we don't want to update
4313 : stack_pointer_delta. */
4314 3640 : temp = expand_simple_binop (Pmode, PLUS, stack_pointer_rtx,
4315 3636 : gen_int_mode (adjust, Pmode), stack_pointer_rtx,
4316 : 0, OPTAB_LIB_WIDEN);
4317 3636 : if (temp != stack_pointer_rtx)
4318 0 : emit_move_insn (stack_pointer_rtx, temp);
4319 :
4320 3636 : switch (code)
4321 : {
4322 3636 : case PRE_INC:
4323 3636 : case PRE_DEC:
4324 3636 : case PRE_MODIFY:
4325 3636 : temp = stack_pointer_rtx;
4326 3636 : break;
4327 : case POST_INC:
4328 : case POST_DEC:
4329 : case POST_MODIFY:
4330 0 : temp = plus_constant (Pmode, stack_pointer_rtx, -adjust);
4331 0 : break;
4332 0 : default:
4333 0 : gcc_unreachable ();
4334 : }
4335 :
4336 3636 : return replace_equiv_address (x, temp);
4337 : }
4338 :
4339 : /* A subroutine of emit_move_complex. Generate a move from Y into X.
4340 : X is known to satisfy push_operand, and MODE is known to be complex.
4341 : Returns the last instruction emitted. */
4342 :
4343 : rtx_insn *
4344 5560 : emit_move_complex_push (machine_mode mode, rtx x, rtx y)
4345 : {
4346 5560 : scalar_mode submode = GET_MODE_INNER (mode);
4347 5560 : bool imag_first;
4348 :
4349 : #ifdef PUSH_ROUNDING
4350 11120 : poly_int64 submodesize = GET_MODE_SIZE (submode);
4351 :
4352 : /* In case we output to the stack, but the size is smaller than the
4353 : machine can push exactly, we need to use move instructions. */
4354 5560 : if (maybe_ne (PUSH_ROUNDING (submodesize), submodesize))
4355 : {
4356 718 : x = emit_move_resolve_push (mode, x);
4357 718 : return emit_move_insn (x, y);
4358 : }
4359 : #endif
4360 :
4361 : /* Note that the real part always precedes the imag part in memory
4362 : regardless of machine's endianness. */
4363 4842 : switch (GET_CODE (XEXP (x, 0)))
4364 : {
4365 : case PRE_DEC:
4366 : case POST_DEC:
4367 : imag_first = true;
4368 : break;
4369 0 : case PRE_INC:
4370 0 : case POST_INC:
4371 0 : imag_first = false;
4372 0 : break;
4373 0 : default:
4374 0 : gcc_unreachable ();
4375 : }
4376 :
4377 4842 : emit_move_insn (gen_rtx_MEM (submode, XEXP (x, 0)),
4378 : read_complex_part (y, imag_first));
4379 4842 : return emit_move_insn (gen_rtx_MEM (submode, XEXP (x, 0)),
4380 9684 : read_complex_part (y, !imag_first));
4381 : }
4382 :
4383 : /* A subroutine of emit_move_complex. Perform the move from Y to X
4384 : via two moves of the parts. Returns the last instruction emitted. */
4385 :
4386 : rtx_insn *
4387 87915 : emit_move_complex_parts (rtx x, rtx y)
4388 : {
4389 : /* Show the output dies here. This is necessary for SUBREGs
4390 : of pseudos since we cannot track their lifetimes correctly;
4391 : hard regs shouldn't appear here except as return values. */
4392 87915 : if (!reload_completed && !reload_in_progress
4393 175830 : && REG_P (x) && !reg_overlap_mentioned_p (x, y))
4394 6136 : emit_clobber (x);
4395 :
4396 87915 : write_complex_part (x, read_complex_part (y, false), false, true);
4397 87915 : write_complex_part (x, read_complex_part (y, true), true, false);
4398 :
4399 87915 : return get_last_insn ();
4400 : }
4401 :
4402 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X.
4403 : MODE is known to be complex. Returns the last instruction emitted. */
4404 :
4405 : static rtx_insn *
4406 110088 : emit_move_complex (machine_mode mode, rtx x, rtx y)
4407 : {
4408 110088 : bool try_int;
4409 :
4410 : /* Need to take special care for pushes, to maintain proper ordering
4411 : of the data, and possibly extra padding. */
4412 110088 : if (push_operand (x, mode))
4413 4952 : return emit_move_complex_push (mode, x, y);
4414 :
4415 : /* See if we can coerce the target into moving both values at once, except
4416 : for floating point where we favor moving as parts if this is easy. */
4417 105136 : if (GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT
4418 62431 : && optab_handler (mov_optab, GET_MODE_INNER (mode)) != CODE_FOR_nothing
4419 62431 : && !(REG_P (x)
4420 2213 : && HARD_REGISTER_P (x)
4421 564 : && REG_NREGS (x) == 1)
4422 167567 : && !(REG_P (y)
4423 3302 : && HARD_REGISTER_P (y)
4424 1651 : && REG_NREGS (y) == 1))
4425 : try_int = false;
4426 : /* Not possible if the values are inherently not adjacent. */
4427 42705 : else if (GET_CODE (x) == CONCAT || GET_CODE (y) == CONCAT)
4428 : try_int = false;
4429 : /* Is possible if both are registers (or subregs of registers). */
4430 17998 : else if (register_operand (x, mode) && register_operand (y, mode))
4431 : try_int = true;
4432 : /* If one of the operands is a memory, and alignment constraints
4433 : are friendly enough, we may be able to do combined memory operations.
4434 : We do not attempt this if Y is a constant because that combination is
4435 : usually better with the by-parts thing below. */
4436 635 : else if ((MEM_P (x) ? !CONSTANT_P (y) : MEM_P (y))
4437 : && (!STRICT_ALIGNMENT
4438 : || get_mode_alignment (mode) == BIGGEST_ALIGNMENT))
4439 : try_int = true;
4440 : else
4441 : try_int = false;
4442 :
4443 : if (try_int)
4444 : {
4445 17998 : rtx_insn *ret;
4446 :
4447 : /* For memory to memory moves, optimal behavior can be had with the
4448 : existing block move logic. But use normal expansion if optimizing
4449 : for size. */
4450 17998 : if (MEM_P (x) && MEM_P (y))
4451 : {
4452 726 : emit_block_move (x, y, gen_int_mode (GET_MODE_SIZE (mode), Pmode),
4453 360 : (optimize_insn_for_speed_p()
4454 : ? BLOCK_OP_NO_LIBCALL : BLOCK_OP_NORMAL));
4455 360 : return get_last_insn ();
4456 : }
4457 :
4458 17638 : ret = emit_move_via_integer (mode, x, y, true);
4459 17638 : if (ret)
4460 : return ret;
4461 : }
4462 :
4463 87138 : return emit_move_complex_parts (x, y);
4464 : }
4465 :
4466 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X.
4467 : MODE is known to be MODE_CC. Returns the last instruction emitted. */
4468 :
4469 : static rtx_insn *
4470 0 : emit_move_ccmode (machine_mode mode, rtx x, rtx y)
4471 : {
4472 0 : rtx_insn *ret;
4473 :
4474 : /* Assume all MODE_CC modes are equivalent; if we have movcc, use it. */
4475 0 : if (mode != CCmode)
4476 : {
4477 0 : enum insn_code code = optab_handler (mov_optab, CCmode);
4478 0 : if (code != CODE_FOR_nothing)
4479 : {
4480 0 : x = emit_move_change_mode (CCmode, mode, x, true);
4481 0 : y = emit_move_change_mode (CCmode, mode, y, true);
4482 0 : return emit_insn (GEN_FCN (code) (x, y));
4483 : }
4484 : }
4485 :
4486 : /* Otherwise, find the MODE_INT mode of the same width. */
4487 0 : ret = emit_move_via_integer (mode, x, y, false);
4488 0 : gcc_assert (ret != NULL);
4489 : return ret;
4490 : }
4491 :
4492 : /* Return true if word I of OP lies entirely in the
4493 : undefined bits of a paradoxical subreg. */
4494 :
4495 : static bool
4496 0 : undefined_operand_subword_p (const_rtx op, int i)
4497 : {
4498 0 : if (GET_CODE (op) != SUBREG)
4499 : return false;
4500 0 : machine_mode innermostmode = GET_MODE (SUBREG_REG (op));
4501 0 : poly_int64 offset = i * UNITS_PER_WORD + subreg_memory_offset (op);
4502 0 : return (known_ge (offset, GET_MODE_SIZE (innermostmode))
4503 0 : || known_le (offset, -UNITS_PER_WORD));
4504 : }
4505 :
4506 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X.
4507 : MODE is any multi-word or full-word mode that lacks a move_insn
4508 : pattern. Note that you will get better code if you define such
4509 : patterns, even if they must turn into multiple assembler instructions. */
4510 :
4511 : static rtx_insn *
4512 0 : emit_move_multi_word (machine_mode mode, rtx x, rtx y)
4513 : {
4514 0 : rtx_insn *last_insn = 0;
4515 0 : rtx_insn *seq;
4516 0 : rtx inner;
4517 0 : bool need_clobber;
4518 0 : int i, mode_size;
4519 :
4520 : /* This function can only handle cases where the number of words is
4521 : known at compile time. */
4522 0 : mode_size = GET_MODE_SIZE (mode).to_constant ();
4523 0 : gcc_assert (mode_size >= UNITS_PER_WORD);
4524 :
4525 : /* If X is a push on the stack, do the push now and replace
4526 : X with a reference to the stack pointer. */
4527 0 : if (push_operand (x, mode))
4528 0 : x = emit_move_resolve_push (mode, x);
4529 :
4530 : /* If we are in reload, see if either operand is a MEM whose address
4531 : is scheduled for replacement. */
4532 0 : if (reload_in_progress && MEM_P (x)
4533 0 : && (inner = find_replacement (&XEXP (x, 0))) != XEXP (x, 0))
4534 0 : x = replace_equiv_address_nv (x, inner);
4535 0 : if (reload_in_progress && MEM_P (y)
4536 0 : && (inner = find_replacement (&XEXP (y, 0))) != XEXP (y, 0))
4537 0 : y = replace_equiv_address_nv (y, inner);
4538 :
4539 0 : start_sequence ();
4540 :
4541 0 : need_clobber = false;
4542 0 : for (i = 0; i < CEIL (mode_size, UNITS_PER_WORD); i++)
4543 : {
4544 : /* Do not generate code for a move if it would go entirely
4545 : to the non-existing bits of a paradoxical subreg. */
4546 0 : if (undefined_operand_subword_p (x, i))
4547 0 : continue;
4548 :
4549 0 : rtx xpart = operand_subword (x, i, 1, mode);
4550 0 : rtx ypart;
4551 :
4552 : /* Do not generate code for a move if it would come entirely
4553 : from the undefined bits of a paradoxical subreg. */
4554 0 : if (undefined_operand_subword_p (y, i))
4555 0 : continue;
4556 :
4557 0 : ypart = operand_subword (y, i, 1, mode);
4558 :
4559 : /* If we can't get a part of Y, put Y into memory if it is a
4560 : constant. Otherwise, force it into a register. Then we must
4561 : be able to get a part of Y. */
4562 0 : if (ypart == 0 && CONSTANT_P (y))
4563 : {
4564 0 : y = use_anchored_address (force_const_mem (mode, y));
4565 0 : ypart = operand_subword (y, i, 1, mode);
4566 : }
4567 0 : else if (ypart == 0)
4568 0 : ypart = operand_subword_force (y, i, mode);
4569 :
4570 0 : gcc_assert (xpart && ypart);
4571 :
4572 0 : need_clobber |= (GET_CODE (xpart) == SUBREG);
4573 :
4574 0 : last_insn = emit_move_insn (xpart, ypart);
4575 : }
4576 :
4577 0 : seq = end_sequence ();
4578 :
4579 : /* Show the output dies here. This is necessary for SUBREGs
4580 : of pseudos since we cannot track their lifetimes correctly;
4581 : hard regs shouldn't appear here except as return values.
4582 : We never want to emit such a clobber after reload. */
4583 0 : if (x != y
4584 0 : && ! (reload_in_progress || reload_completed)
4585 0 : && need_clobber != 0)
4586 0 : emit_clobber (x);
4587 :
4588 0 : emit_insn (seq);
4589 :
4590 0 : return last_insn;
4591 : }
4592 :
4593 : /* Low level part of emit_move_insn.
4594 : Called just like emit_move_insn, but assumes X and Y
4595 : are basically valid. */
4596 :
4597 : rtx_insn *
4598 76944496 : emit_move_insn_1 (rtx x, rtx y)
4599 : {
4600 76944496 : machine_mode mode = GET_MODE (x);
4601 76944496 : enum insn_code code;
4602 :
4603 76944496 : gcc_assert ((unsigned int) mode < (unsigned int) MAX_MACHINE_MODE);
4604 :
4605 76944496 : code = optab_handler (mov_optab, mode);
4606 76944496 : if (code != CODE_FOR_nothing)
4607 76713215 : return emit_insn (GEN_FCN (code) (x, y));
4608 :
4609 : /* Expand complex moves by moving real part and imag part. */
4610 231281 : if (COMPLEX_MODE_P (mode))
4611 110088 : return emit_move_complex (mode, x, y);
4612 :
4613 : if (GET_MODE_CLASS (mode) == MODE_DECIMAL_FLOAT
4614 : || ALL_FIXED_POINT_MODE_P (mode))
4615 : {
4616 121193 : rtx_insn *result = emit_move_via_integer (mode, x, y, true);
4617 :
4618 : /* If we can't find an integer mode, use multi words. */
4619 121193 : if (result)
4620 : return result;
4621 : else
4622 0 : return emit_move_multi_word (mode, x, y);
4623 : }
4624 :
4625 : if (GET_MODE_CLASS (mode) == MODE_CC)
4626 0 : return emit_move_ccmode (mode, x, y);
4627 :
4628 : /* Try using a move pattern for the corresponding integer mode. This is
4629 : only safe when simplify_subreg can convert MODE constants into integer
4630 : constants. At present, it can only do this reliably if the value
4631 : fits within a HOST_WIDE_INT. */
4632 0 : if (!CONSTANT_P (y)
4633 0 : || known_le (GET_MODE_BITSIZE (mode), HOST_BITS_PER_WIDE_INT))
4634 : {
4635 0 : rtx_insn *ret = emit_move_via_integer (mode, x, y, lra_in_progress);
4636 :
4637 0 : if (ret)
4638 : {
4639 0 : if (! lra_in_progress || recog (PATTERN (ret), ret, 0) >= 0)
4640 0 : return ret;
4641 : }
4642 : }
4643 :
4644 0 : return emit_move_multi_word (mode, x, y);
4645 : }
4646 :
4647 : /* Generate code to copy Y into X.
4648 : Both Y and X must have the same mode, except that
4649 : Y can be a constant with VOIDmode.
4650 : This mode cannot be BLKmode; use emit_block_move for that.
4651 :
4652 : Return the last instruction emitted. */
4653 :
4654 : rtx_insn *
4655 71031787 : emit_move_insn (rtx x, rtx y)
4656 : {
4657 71031787 : machine_mode mode = GET_MODE (x);
4658 71031787 : rtx y_cst = NULL_RTX;
4659 71031787 : rtx_insn *last_insn;
4660 71031787 : rtx set;
4661 :
4662 71031787 : gcc_assert (mode != BLKmode
4663 : && (GET_MODE (y) == mode || GET_MODE (y) == VOIDmode));
4664 :
4665 : /* If we have a copy that looks like one of the following patterns:
4666 : (set (subreg:M1 (reg:M2 ...)) (subreg:M1 (reg:M2 ...)))
4667 : (set (subreg:M1 (reg:M2 ...)) (mem:M1 ADDR))
4668 : (set (mem:M1 ADDR) (subreg:M1 (reg:M2 ...)))
4669 : (set (subreg:M1 (reg:M2 ...)) (constant C))
4670 : where mode M1 is equal in size to M2, try to detect whether the
4671 : mode change involves an implicit round trip through memory.
4672 : If so, see if we can avoid that by removing the subregs and
4673 : doing the move in mode M2 instead. */
4674 :
4675 71031787 : rtx x_inner = NULL_RTX;
4676 71031787 : rtx y_inner = NULL_RTX;
4677 :
4678 73801015 : auto candidate_subreg_p = [&](rtx subreg) {
4679 2769228 : return (REG_P (SUBREG_REG (subreg))
4680 8305923 : && known_eq (GET_MODE_SIZE (GET_MODE (SUBREG_REG (subreg))),
4681 : GET_MODE_SIZE (GET_MODE (subreg)))
4682 3138737 : && optab_handler (mov_optab, GET_MODE (SUBREG_REG (subreg)))
4683 2769228 : != CODE_FOR_nothing);
4684 : };
4685 :
4686 71038016 : auto candidate_mem_p = [&](machine_mode innermode, rtx mem) {
4687 6229 : return (!targetm.can_change_mode_class (innermode, GET_MODE (mem), ALL_REGS)
4688 6229 : && !push_operand (mem, GET_MODE (mem))
4689 : /* Not a candidate if innermode requires too much alignment. */
4690 12398 : && (MEM_ALIGN (mem) >= GET_MODE_ALIGNMENT (innermode)
4691 232 : || targetm.slow_unaligned_access (GET_MODE (mem),
4692 116 : MEM_ALIGN (mem))
4693 232 : || !targetm.slow_unaligned_access (innermode,
4694 116 : MEM_ALIGN (mem))));
4695 : };
4696 :
4697 71031787 : if (SUBREG_P (x) && candidate_subreg_p (x))
4698 13185 : x_inner = SUBREG_REG (x);
4699 :
4700 71031787 : if (SUBREG_P (y) && candidate_subreg_p (y))
4701 320468 : y_inner = SUBREG_REG (y);
4702 :
4703 71031787 : if (x_inner != NULL_RTX
4704 71031787 : && y_inner != NULL_RTX
4705 1160 : && GET_MODE (x_inner) == GET_MODE (y_inner)
4706 71032524 : && !targetm.can_change_mode_class (GET_MODE (x_inner), mode, ALL_REGS))
4707 : {
4708 737 : x = x_inner;
4709 737 : y = y_inner;
4710 737 : mode = GET_MODE (x_inner);
4711 : }
4712 71031050 : else if (x_inner != NULL_RTX
4713 12448 : && MEM_P (y)
4714 71031374 : && candidate_mem_p (GET_MODE (x_inner), y))
4715 : {
4716 324 : x = x_inner;
4717 324 : y = adjust_address (y, GET_MODE (x_inner), 0);
4718 324 : mode = GET_MODE (x_inner);
4719 : }
4720 71030726 : else if (y_inner != NULL_RTX
4721 319731 : && MEM_P (x)
4722 71036631 : && candidate_mem_p (GET_MODE (y_inner), x))
4723 : {
4724 5845 : x = adjust_address (x, GET_MODE (y_inner), 0);
4725 5845 : y = y_inner;
4726 5845 : mode = GET_MODE (y_inner);
4727 : }
4728 71024881 : else if (x_inner != NULL_RTX
4729 12124 : && CONSTANT_P (y)
4730 475 : && !targetm.can_change_mode_class (GET_MODE (x_inner),
4731 : mode, ALL_REGS)
4732 71025356 : && (y_inner = simplify_subreg (GET_MODE (x_inner), y, mode, 0)))
4733 : {
4734 475 : x = x_inner;
4735 475 : y = y_inner;
4736 475 : mode = GET_MODE (x_inner);
4737 : }
4738 :
4739 71031787 : if (CONSTANT_P (y))
4740 : {
4741 16604823 : if (optimize
4742 12924945 : && SCALAR_FLOAT_MODE_P (GET_MODE (x))
4743 17378311 : && (last_insn = compress_float_constant (x, y)))
4744 : return last_insn;
4745 :
4746 16535701 : y_cst = y;
4747 :
4748 16535701 : if (!targetm.legitimate_constant_p (mode, y))
4749 : {
4750 510769 : y = force_const_mem (mode, y);
4751 :
4752 : /* If the target's cannot_force_const_mem prevented the spill,
4753 : assume that the target's move expanders will also take care
4754 : of the non-legitimate constant. */
4755 510769 : if (!y)
4756 : y = y_cst;
4757 : else
4758 490650 : y = use_anchored_address (y);
4759 : }
4760 : }
4761 :
4762 : /* If X or Y are memory references, verify that their addresses are valid
4763 : for the machine. */
4764 70962665 : if (MEM_P (x)
4765 87164579 : && (! memory_address_addr_space_p (GET_MODE (x), XEXP (x, 0),
4766 13941219 : MEM_ADDR_SPACE (x))
4767 2260869 : && ! push_operand (x, GET_MODE (x))))
4768 174 : x = validize_mem (x);
4769 :
4770 70962665 : if (MEM_P (y)
4771 88695556 : && ! memory_address_addr_space_p (GET_MODE (y), XEXP (y, 0),
4772 17732891 : MEM_ADDR_SPACE (y)))
4773 8702 : y = validize_mem (y);
4774 :
4775 70962665 : gcc_assert (mode != BLKmode);
4776 :
4777 70962665 : last_insn = emit_move_insn_1 (x, y);
4778 :
4779 16535701 : if (y_cst && REG_P (x)
4780 12116379 : && (set = single_set (last_insn)) != NULL_RTX
4781 12116379 : && SET_DEST (set) == x
4782 83064529 : && ! rtx_equal_p (y_cst, SET_SRC (set)))
4783 1243388 : set_unique_reg_note (last_insn, REG_EQUAL, copy_rtx (y_cst));
4784 :
4785 : return last_insn;
4786 : }
4787 :
4788 : /* Generate the body of an instruction to copy Y into X.
4789 : It may be a list of insns, if one insn isn't enough. */
4790 :
4791 : rtx_insn *
4792 5981751 : gen_move_insn (rtx x, rtx y)
4793 : {
4794 5981751 : start_sequence ();
4795 5981751 : emit_move_insn_1 (x, y);
4796 5981751 : return end_sequence ();
4797 : }
4798 :
4799 : /* If Y is representable exactly in a narrower mode, and the target can
4800 : perform the extension directly from constant or memory, then emit the
4801 : move as an extension. */
4802 :
4803 : static rtx_insn *
4804 773488 : compress_float_constant (rtx x, rtx y)
4805 : {
4806 773488 : machine_mode dstmode = GET_MODE (x);
4807 773488 : machine_mode orig_srcmode = GET_MODE (y);
4808 773488 : machine_mode srcmode;
4809 773488 : const REAL_VALUE_TYPE *r;
4810 773488 : int oldcost, newcost;
4811 773488 : bool speed = optimize_insn_for_speed_p ();
4812 :
4813 773488 : r = CONST_DOUBLE_REAL_VALUE (y);
4814 :
4815 773488 : if (targetm.legitimate_constant_p (dstmode, y))
4816 771870 : oldcost = set_src_cost (y, orig_srcmode, speed);
4817 : else
4818 1618 : oldcost = set_src_cost (force_const_mem (dstmode, y), dstmode, speed);
4819 :
4820 2740896 : FOR_EACH_MODE_UNTIL (srcmode, orig_srcmode)
4821 : {
4822 2036530 : enum insn_code ic;
4823 2036530 : rtx trunc_y;
4824 2036530 : rtx_insn *last_insn;
4825 :
4826 : /* Skip if the target can't extend this way. */
4827 2036530 : ic = can_extend_p (dstmode, srcmode, 0);
4828 2036530 : if (ic == CODE_FOR_nothing)
4829 1612731 : continue;
4830 :
4831 : /* Skip if the narrowed value isn't exact. */
4832 423799 : if (! exact_real_truncate (srcmode, r))
4833 52349 : continue;
4834 :
4835 371450 : trunc_y = const_double_from_real_value (*r, srcmode);
4836 :
4837 371450 : if (targetm.legitimate_constant_p (srcmode, trunc_y))
4838 : {
4839 : /* Skip if the target needs extra instructions to perform
4840 : the extension. */
4841 367806 : if (!insn_operand_matches (ic, 1, trunc_y))
4842 4080 : continue;
4843 : /* This is valid, but may not be cheaper than the original. */
4844 363726 : newcost = set_src_cost (gen_rtx_FLOAT_EXTEND (dstmode, trunc_y),
4845 : dstmode, speed);
4846 363726 : if (oldcost < newcost)
4847 294604 : continue;
4848 : }
4849 3644 : else if (float_extend_from_mem[dstmode][srcmode])
4850 : {
4851 0 : trunc_y = force_const_mem (srcmode, trunc_y);
4852 : /* This is valid, but may not be cheaper than the original. */
4853 0 : newcost = set_src_cost (gen_rtx_FLOAT_EXTEND (dstmode, trunc_y),
4854 : dstmode, speed);
4855 0 : if (oldcost < newcost)
4856 0 : continue;
4857 0 : trunc_y = validize_mem (trunc_y);
4858 : }
4859 : else
4860 3644 : continue;
4861 :
4862 : /* For CSE's benefit, force the compressed constant pool entry
4863 : into a new pseudo. This constant may be used in different modes,
4864 : and if not, combine will put things back together for us. */
4865 69122 : trunc_y = force_reg (srcmode, trunc_y);
4866 :
4867 : /* If x is a hard register, perform the extension into a pseudo,
4868 : so that e.g. stack realignment code is aware of it. */
4869 69122 : rtx target = x;
4870 69122 : if (REG_P (x) && HARD_REGISTER_P (x))
4871 1 : target = gen_reg_rtx (dstmode);
4872 :
4873 69122 : emit_unop_insn (ic, target, trunc_y, UNKNOWN);
4874 69122 : last_insn = get_last_insn ();
4875 :
4876 69122 : if (REG_P (target))
4877 58426 : set_unique_reg_note (last_insn, REG_EQUAL, y);
4878 :
4879 69122 : if (target != x)
4880 1 : return emit_move_insn (x, target);
4881 : return last_insn;
4882 : }
4883 :
4884 : return NULL;
4885 : }
4886 :
4887 : /* Pushing data onto the stack. */
4888 :
4889 : /* Push a block of length SIZE (perhaps variable)
4890 : and return an rtx to address the beginning of the block.
4891 : The value may be virtual_outgoing_args_rtx.
4892 :
4893 : EXTRA is the number of bytes of padding to push in addition to SIZE.
4894 : BELOW nonzero means this padding comes at low addresses;
4895 : otherwise, the padding comes at high addresses. */
4896 :
4897 : rtx
4898 269040 : push_block (rtx size, poly_int64 extra, int below)
4899 : {
4900 269040 : rtx temp;
4901 :
4902 334153 : size = convert_modes (Pmode, ptr_mode, size, 1);
4903 269040 : if (CONSTANT_P (size))
4904 334153 : anti_adjust_stack (plus_constant (Pmode, size, extra));
4905 0 : else if (REG_P (size) && known_eq (extra, 0))
4906 0 : anti_adjust_stack (size);
4907 : else
4908 : {
4909 0 : temp = copy_to_mode_reg (Pmode, size);
4910 0 : if (maybe_ne (extra, 0))
4911 0 : temp = expand_binop (Pmode, add_optab, temp,
4912 0 : gen_int_mode (extra, Pmode),
4913 : temp, 0, OPTAB_LIB_WIDEN);
4914 0 : anti_adjust_stack (temp);
4915 : }
4916 :
4917 269040 : if (STACK_GROWS_DOWNWARD)
4918 : {
4919 269040 : temp = virtual_outgoing_args_rtx;
4920 269040 : if (maybe_ne (extra, 0) && below)
4921 0 : temp = plus_constant (Pmode, temp, extra);
4922 : }
4923 : else
4924 : {
4925 : poly_int64 csize;
4926 : if (poly_int_rtx_p (size, &csize))
4927 : temp = plus_constant (Pmode, virtual_outgoing_args_rtx,
4928 : -csize - (below ? 0 : extra));
4929 : else if (maybe_ne (extra, 0) && !below)
4930 : temp = gen_rtx_PLUS (Pmode, virtual_outgoing_args_rtx,
4931 : negate_rtx (Pmode, plus_constant (Pmode, size,
4932 : extra)));
4933 : else
4934 : temp = gen_rtx_PLUS (Pmode, virtual_outgoing_args_rtx,
4935 : negate_rtx (Pmode, size));
4936 : }
4937 :
4938 269040 : return memory_address (NARROWEST_INT_MODE, temp);
4939 : }
4940 :
4941 : /* A utility routine that returns the base of an auto-inc memory, or NULL. */
4942 :
4943 : static rtx
4944 2585528 : mem_autoinc_base (rtx mem)
4945 : {
4946 0 : if (MEM_P (mem))
4947 : {
4948 1582528 : rtx addr = XEXP (mem, 0);
4949 1582528 : if (GET_RTX_CLASS (GET_CODE (addr)) == RTX_AUTOINC)
4950 1034033 : return XEXP (addr, 0);
4951 : }
4952 : return NULL;
4953 : }
4954 :
4955 : /* A utility routine used here, in reload, and in try_split. The insns
4956 : after PREV up to and including LAST are known to adjust the stack,
4957 : with a final value of END_ARGS_SIZE. Iterate backward from LAST
4958 : placing notes as appropriate. PREV may be NULL, indicating the
4959 : entire insn sequence prior to LAST should be scanned.
4960 :
4961 : The set of allowed stack pointer modifications is small:
4962 : (1) One or more auto-inc style memory references (aka pushes),
4963 : (2) One or more addition/subtraction with the SP as destination,
4964 : (3) A single move insn with the SP as destination,
4965 : (4) A call_pop insn,
4966 : (5) Noreturn call insns if !ACCUMULATE_OUTGOING_ARGS.
4967 :
4968 : Insns in the sequence that do not modify the SP are ignored,
4969 : except for noreturn calls.
4970 :
4971 : The return value is the amount of adjustment that can be trivially
4972 : verified, via immediate operand or auto-inc. If the adjustment
4973 : cannot be trivially extracted, the return value is HOST_WIDE_INT_MIN. */
4974 :
4975 : poly_int64
4976 4604184 : find_args_size_adjust (rtx_insn *insn)
4977 : {
4978 4604184 : rtx dest, set, pat;
4979 4604184 : int i;
4980 :
4981 4604184 : pat = PATTERN (insn);
4982 4604184 : set = NULL;
4983 :
4984 : /* Look for a call_pop pattern. */
4985 4604184 : if (CALL_P (insn))
4986 : {
4987 : /* We have to allow non-call_pop patterns for the case
4988 : of emit_single_push_insn of a TLS address. */
4989 2951212 : if (GET_CODE (pat) != PARALLEL)
4990 2936355 : return 0;
4991 :
4992 : /* All call_pop have a stack pointer adjust in the parallel.
4993 : The call itself is always first, and the stack adjust is
4994 : usually last, so search from the end. */
4995 14857 : for (i = XVECLEN (pat, 0) - 1; i > 0; --i)
4996 : {
4997 14857 : set = XVECEXP (pat, 0, i);
4998 14857 : if (GET_CODE (set) != SET)
4999 0 : continue;
5000 14857 : dest = SET_DEST (set);
5001 14857 : if (dest == stack_pointer_rtx)
5002 : break;
5003 : }
5004 : /* We'd better have found the stack pointer adjust. */
5005 14857 : if (i == 0)
5006 0 : return 0;
5007 : /* Fall through to process the extracted SET and DEST
5008 : as if it was a standalone insn. */
5009 : }
5010 1652972 : else if (GET_CODE (pat) == SET)
5011 : set = pat;
5012 257717 : else if ((set = single_set (insn)) != NULL)
5013 : ;
5014 46805 : else if (GET_CODE (pat) == PARALLEL)
5015 : {
5016 : /* ??? Some older ports use a parallel with a stack adjust
5017 : and a store for a PUSH_ROUNDING pattern, rather than a
5018 : PRE/POST_MODIFY rtx. Don't force them to update yet... */
5019 : /* ??? See h8300 and m68k, pushqi1. */
5020 0 : for (i = XVECLEN (pat, 0) - 1; i >= 0; --i)
5021 : {
5022 0 : set = XVECEXP (pat, 0, i);
5023 0 : if (GET_CODE (set) != SET)
5024 0 : continue;
5025 0 : dest = SET_DEST (set);
5026 0 : if (dest == stack_pointer_rtx)
5027 : break;
5028 :
5029 : /* We do not expect an auto-inc of the sp in the parallel. */
5030 0 : gcc_checking_assert (mem_autoinc_base (dest) != stack_pointer_rtx);
5031 0 : gcc_checking_assert (mem_autoinc_base (SET_SRC (set))
5032 : != stack_pointer_rtx);
5033 : }
5034 0 : if (i < 0)
5035 0 : return 0;
5036 : }
5037 : else
5038 46805 : return 0;
5039 :
5040 1621024 : dest = SET_DEST (set);
5041 :
5042 : /* Look for direct modifications of the stack pointer. */
5043 1621024 : if (REG_P (dest) && REGNO (dest) == STACK_POINTER_REGNUM)
5044 : {
5045 : /* Look for a trivial adjustment, otherwise assume nothing. */
5046 : /* Note that the SPU restore_stack_block pattern refers to
5047 : the stack pointer in V4SImode. Consider that non-trivial. */
5048 328260 : poly_int64 offset;
5049 328260 : if (SCALAR_INT_MODE_P (GET_MODE (dest))
5050 328260 : && strip_offset (SET_SRC (set), &offset) == stack_pointer_rtx)
5051 325827 : return offset;
5052 : /* ??? Reload can generate no-op moves, which will be cleaned
5053 : up later. Recognize it and continue searching. */
5054 2433 : else if (rtx_equal_p (dest, SET_SRC (set)))
5055 0 : return 0;
5056 : else
5057 2433 : return HOST_WIDE_INT_MIN;
5058 : }
5059 : else
5060 : {
5061 1292764 : rtx mem, addr;
5062 :
5063 : /* Otherwise only think about autoinc patterns. */
5064 2339351 : if (mem_autoinc_base (dest) == stack_pointer_rtx)
5065 : {
5066 925248 : mem = dest;
5067 1283792 : gcc_checking_assert (mem_autoinc_base (SET_SRC (set))
5068 : != stack_pointer_rtx);
5069 : }
5070 544913 : else if (mem_autoinc_base (SET_SRC (set)) == stack_pointer_rtx)
5071 : mem = SET_SRC (set);
5072 : else
5073 258731 : return 0;
5074 :
5075 1034033 : addr = XEXP (mem, 0);
5076 1034033 : switch (GET_CODE (addr))
5077 : {
5078 108785 : case PRE_INC:
5079 108785 : case POST_INC:
5080 217570 : return GET_MODE_SIZE (GET_MODE (mem));
5081 907185 : case PRE_DEC:
5082 907185 : case POST_DEC:
5083 1814370 : return -GET_MODE_SIZE (GET_MODE (mem));
5084 18063 : case PRE_MODIFY:
5085 18063 : case POST_MODIFY:
5086 18063 : addr = XEXP (addr, 1);
5087 18063 : gcc_assert (GET_CODE (addr) == PLUS);
5088 18063 : gcc_assert (XEXP (addr, 0) == stack_pointer_rtx);
5089 18063 : return rtx_to_poly_int64 (XEXP (addr, 1));
5090 0 : default:
5091 0 : gcc_unreachable ();
5092 : }
5093 : }
5094 : }
5095 :
5096 : poly_int64
5097 666620 : fixup_args_size_notes (rtx_insn *prev, rtx_insn *last,
5098 : poly_int64 end_args_size)
5099 : {
5100 666620 : poly_int64 args_size = end_args_size;
5101 666620 : bool saw_unknown = false;
5102 666620 : rtx_insn *insn;
5103 :
5104 1901568 : for (insn = last; insn != prev; insn = PREV_INSN (insn))
5105 : {
5106 1234948 : if (!NONDEBUG_INSN_P (insn))
5107 0 : continue;
5108 :
5109 : /* We might have existing REG_ARGS_SIZE notes, e.g. when pushing
5110 : a call argument containing a TLS address that itself requires
5111 : a call to __tls_get_addr. The handling of stack_pointer_delta
5112 : in emit_single_push_insn is supposed to ensure that any such
5113 : notes are already correct. */
5114 1234948 : rtx note = find_reg_note (insn, REG_ARGS_SIZE, NULL_RTX);
5115 1234948 : gcc_assert (!note || known_eq (args_size, get_args_size (note)));
5116 :
5117 1234948 : poly_int64 this_delta = find_args_size_adjust (insn);
5118 1234948 : if (known_eq (this_delta, 0))
5119 : {
5120 607183 : if (!CALL_P (insn)
5121 9 : || ACCUMULATE_OUTGOING_ARGS
5122 303605 : || find_reg_note (insn, REG_NORETURN, NULL_RTX) == NULL_RTX)
5123 303587 : continue;
5124 : }
5125 :
5126 931361 : gcc_assert (!saw_unknown);
5127 931361 : if (known_eq (this_delta, HOST_WIDE_INT_MIN))
5128 2411 : saw_unknown = true;
5129 :
5130 931361 : if (!note)
5131 931361 : add_args_size_note (insn, args_size);
5132 : if (STACK_GROWS_DOWNWARD)
5133 931361 : this_delta = -poly_uint64 (this_delta);
5134 :
5135 931361 : if (saw_unknown)
5136 : args_size = HOST_WIDE_INT_MIN;
5137 : else
5138 1234948 : args_size -= this_delta;
5139 : }
5140 :
5141 666620 : return args_size;
5142 : }
5143 :
5144 : #ifdef PUSH_ROUNDING
5145 : /* Emit single push insn. */
5146 :
5147 : static void
5148 1846418 : emit_single_push_insn_1 (machine_mode mode, rtx x, tree type)
5149 : {
5150 1846418 : rtx dest_addr;
5151 3692836 : poly_int64 rounded_size = PUSH_ROUNDING (GET_MODE_SIZE (mode));
5152 1846418 : rtx dest;
5153 1846418 : enum insn_code icode;
5154 :
5155 : /* If there is push pattern, use it. Otherwise try old way of throwing
5156 : MEM representing push operation to move expander. */
5157 1846418 : icode = optab_handler (push_optab, mode);
5158 1846418 : if (icode != CODE_FOR_nothing)
5159 : {
5160 0 : class expand_operand ops[1];
5161 :
5162 0 : create_input_operand (&ops[0], x, mode);
5163 0 : if (maybe_expand_insn (icode, 1, ops))
5164 0 : return;
5165 : }
5166 3692836 : if (known_eq (GET_MODE_SIZE (mode), rounded_size))
5167 3188815 : dest_addr = gen_rtx_fmt_e (STACK_PUSH_CODE, Pmode, stack_pointer_rtx);
5168 : /* If we are to pad downward, adjust the stack pointer first and
5169 : then store X into the stack location using an offset. This is
5170 : because emit_move_insn does not know how to pad; it does not have
5171 : access to type. */
5172 104591 : else if (targetm.calls.function_arg_padding (mode, type) == PAD_DOWNWARD)
5173 : {
5174 0 : emit_move_insn (stack_pointer_rtx,
5175 0 : expand_binop (Pmode,
5176 : STACK_GROWS_DOWNWARD ? sub_optab
5177 : : add_optab,
5178 : stack_pointer_rtx,
5179 0 : gen_int_mode (rounded_size, Pmode),
5180 : NULL_RTX, 0, OPTAB_LIB_WIDEN));
5181 :
5182 0 : poly_int64 offset = rounded_size - GET_MODE_SIZE (mode);
5183 0 : if (STACK_GROWS_DOWNWARD && STACK_PUSH_CODE == POST_DEC)
5184 : /* We have already decremented the stack pointer, so get the
5185 : previous value. */
5186 : offset += rounded_size;
5187 :
5188 0 : if (!STACK_GROWS_DOWNWARD && STACK_PUSH_CODE == POST_INC)
5189 : /* We have already incremented the stack pointer, so get the
5190 : previous value. */
5191 : offset -= rounded_size;
5192 :
5193 0 : dest_addr = plus_constant (Pmode, stack_pointer_rtx, offset);
5194 : }
5195 : else
5196 : {
5197 : if (STACK_GROWS_DOWNWARD)
5198 : /* ??? This seems wrong if STACK_PUSH_CODE == POST_DEC. */
5199 105139 : dest_addr = plus_constant (Pmode, stack_pointer_rtx, -rounded_size);
5200 : else
5201 : /* ??? This seems wrong if STACK_PUSH_CODE == POST_INC. */
5202 : dest_addr = plus_constant (Pmode, stack_pointer_rtx, rounded_size);
5203 :
5204 105139 : dest_addr = gen_rtx_PRE_MODIFY (Pmode, stack_pointer_rtx, dest_addr);
5205 : }
5206 :
5207 1846418 : dest = gen_rtx_MEM (mode, dest_addr);
5208 :
5209 1846418 : if (type != 0)
5210 : {
5211 1846371 : set_mem_attributes (dest, type, 1);
5212 :
5213 1846371 : if (cfun->tail_call_marked)
5214 : /* Function incoming arguments may overlap with sibling call
5215 : outgoing arguments and we cannot allow reordering of reads
5216 : from function arguments with stores to outgoing arguments
5217 : of sibling calls. */
5218 142564 : set_mem_alias_set (dest, 0);
5219 : }
5220 1846418 : emit_move_insn (dest, x);
5221 : }
5222 :
5223 : /* Emit and annotate a single push insn. */
5224 :
5225 : static void
5226 1846418 : emit_single_push_insn (machine_mode mode, rtx x, tree type)
5227 : {
5228 1846418 : poly_int64 delta, old_delta = stack_pointer_delta;
5229 1846418 : rtx_insn *prev = get_last_insn ();
5230 1846418 : rtx_insn *last;
5231 :
5232 1846418 : emit_single_push_insn_1 (mode, x, type);
5233 :
5234 : /* Adjust stack_pointer_delta to describe the situation after the push
5235 : we just performed. Note that we must do this after the push rather
5236 : than before the push in case calculating X needs pushes and pops of
5237 : its own (e.g. if calling __tls_get_addr). The REG_ARGS_SIZE notes
5238 : for such pushes and pops must not include the effect of the future
5239 : push of X. */
5240 3692836 : stack_pointer_delta += PUSH_ROUNDING (GET_MODE_SIZE (mode));
5241 :
5242 1846418 : last = get_last_insn ();
5243 :
5244 : /* Notice the common case where we emitted exactly one insn. */
5245 1846418 : if (PREV_INSN (last) == prev)
5246 : {
5247 1735770 : add_args_size_note (last, stack_pointer_delta);
5248 1735770 : return;
5249 : }
5250 :
5251 110648 : delta = fixup_args_size_notes (prev, last, stack_pointer_delta);
5252 110648 : gcc_assert (known_eq (delta, HOST_WIDE_INT_MIN)
5253 : || known_eq (delta, old_delta));
5254 : }
5255 : #endif
5256 :
5257 : /* If reading SIZE bytes from X will end up reading from
5258 : Y return the number of bytes that overlap. Return -1
5259 : if there is no overlap or -2 if we can't determine
5260 : (for example when X and Y have different base registers). */
5261 :
5262 : static int
5263 0 : memory_load_overlap (rtx x, rtx y, HOST_WIDE_INT size)
5264 : {
5265 0 : rtx tmp = plus_constant (Pmode, x, size);
5266 0 : rtx sub = simplify_gen_binary (MINUS, Pmode, tmp, y);
5267 :
5268 0 : if (!CONST_INT_P (sub))
5269 : return -2;
5270 :
5271 0 : HOST_WIDE_INT val = INTVAL (sub);
5272 :
5273 0 : return IN_RANGE (val, 1, size) ? val : -1;
5274 : }
5275 :
5276 : /* Generate code to push X onto the stack, assuming it has mode MODE and
5277 : type TYPE.
5278 : MODE is redundant except when X is a CONST_INT (since they don't
5279 : carry mode info).
5280 : SIZE is an rtx for the size of data to be copied (in bytes),
5281 : needed only if X is BLKmode.
5282 : Return true if successful. May return false if asked to push a
5283 : partial argument during a sibcall optimization (as specified by
5284 : SIBCALL_P) and the incoming and outgoing pointers cannot be shown
5285 : to not overlap.
5286 :
5287 : ALIGN (in bits) is maximum alignment we can assume.
5288 :
5289 : If PARTIAL and REG are both nonzero, then copy that many of the first
5290 : bytes of X into registers starting with REG, and push the rest of X.
5291 : The amount of space pushed is decreased by PARTIAL bytes.
5292 : REG must be a hard register in this case.
5293 : If REG is zero but PARTIAL is not, take any all others actions for an
5294 : argument partially in registers, but do not actually load any
5295 : registers.
5296 :
5297 : EXTRA is the amount in bytes of extra space to leave next to this arg.
5298 : This is ignored if an argument block has already been allocated.
5299 :
5300 : On a machine that lacks real push insns, ARGS_ADDR is the address of
5301 : the bottom of the argument block for this call. We use indexing off there
5302 : to store the arg. On machines with push insns, ARGS_ADDR is 0 when a
5303 : argument block has not been preallocated.
5304 :
5305 : ARGS_SO_FAR is the size of args previously pushed for this call.
5306 :
5307 : REG_PARM_STACK_SPACE is nonzero if functions require stack space
5308 : for arguments passed in registers. If nonzero, it will be the number
5309 : of bytes required. */
5310 :
5311 : bool
5312 2170848 : emit_push_insn (rtx x, machine_mode mode, tree type, rtx size,
5313 : unsigned int align, int partial, rtx reg, poly_int64 extra,
5314 : rtx args_addr, rtx args_so_far, int reg_parm_stack_space,
5315 : rtx alignment_pad, bool sibcall_p)
5316 : {
5317 2170848 : rtx xinner;
5318 2170848 : pad_direction stack_direction
5319 : = STACK_GROWS_DOWNWARD ? PAD_DOWNWARD : PAD_UPWARD;
5320 :
5321 : /* Decide where to pad the argument: PAD_DOWNWARD for below,
5322 : PAD_UPWARD for above, or PAD_NONE for don't pad it.
5323 : Default is below for small data on big-endian machines; else above. */
5324 2170848 : pad_direction where_pad = targetm.calls.function_arg_padding (mode, type);
5325 :
5326 : /* Invert direction if stack is post-decrement.
5327 : FIXME: why? */
5328 2170848 : if (STACK_PUSH_CODE == POST_DEC)
5329 : if (where_pad != PAD_NONE)
5330 : where_pad = (where_pad == PAD_DOWNWARD ? PAD_UPWARD : PAD_DOWNWARD);
5331 :
5332 2170848 : xinner = x;
5333 :
5334 2170848 : int nregs = partial / UNITS_PER_WORD;
5335 2170848 : rtx *tmp_regs = NULL;
5336 2170848 : int overlapping = 0;
5337 :
5338 2170848 : if (mode == BLKmode
5339 : || (STRICT_ALIGNMENT && align < GET_MODE_ALIGNMENT (mode)))
5340 : {
5341 : /* Copy a block into the stack, entirely or partially. */
5342 :
5343 269133 : rtx temp;
5344 269133 : int used;
5345 269133 : int offset;
5346 269133 : int skip;
5347 :
5348 269133 : offset = partial % (PARM_BOUNDARY / BITS_PER_UNIT);
5349 269133 : used = partial - offset;
5350 :
5351 269133 : if (mode != BLKmode)
5352 : {
5353 : /* A value is to be stored in an insufficiently aligned
5354 : stack slot; copy via a suitably aligned slot if
5355 : necessary. */
5356 : size = gen_int_mode (GET_MODE_SIZE (mode), Pmode);
5357 : if (!MEM_P (xinner))
5358 : {
5359 : temp = assign_temp (type, 1, 1);
5360 : emit_move_insn (temp, xinner);
5361 : xinner = temp;
5362 : }
5363 : }
5364 :
5365 269133 : gcc_assert (size);
5366 :
5367 : /* USED is now the # of bytes we need not copy to the stack
5368 : because registers will take care of them. */
5369 :
5370 269133 : if (partial != 0)
5371 0 : xinner = adjust_address (xinner, BLKmode, used);
5372 :
5373 : /* If the partial register-part of the arg counts in its stack size,
5374 : skip the part of stack space corresponding to the registers.
5375 : Otherwise, start copying to the beginning of the stack space,
5376 : by setting SKIP to 0. */
5377 269133 : skip = (reg_parm_stack_space == 0) ? 0 : used;
5378 :
5379 : #ifdef PUSH_ROUNDING
5380 : /* NB: Let the backend known the number of bytes to push and
5381 : decide if push insns should be generated. */
5382 269133 : unsigned int push_size;
5383 269133 : if (CONST_INT_P (size))
5384 269133 : push_size = INTVAL (size);
5385 : else
5386 : push_size = 0;
5387 :
5388 : /* Do it with several push insns if that doesn't take lots of insns
5389 : and if there is no difficulty with push insns that skip bytes
5390 : on the stack for alignment purposes. */
5391 269133 : if (args_addr == 0
5392 269079 : && targetm.calls.push_argument (push_size)
5393 3783 : && CONST_INT_P (size)
5394 3783 : && skip == 0
5395 3783 : && MEM_ALIGN (xinner) >= align
5396 2941 : && can_move_by_pieces ((unsigned) INTVAL (size) - used, align)
5397 : /* Here we avoid the case of a structure whose weak alignment
5398 : forces many pushes of a small amount of data,
5399 : and such small pushes do rounding that causes trouble. */
5400 2941 : && ((!targetm.slow_unaligned_access (word_mode, align))
5401 0 : || align >= BIGGEST_ALIGNMENT
5402 0 : || known_eq (PUSH_ROUNDING (align / BITS_PER_UNIT),
5403 : align / BITS_PER_UNIT))
5404 272074 : && known_eq (PUSH_ROUNDING (INTVAL (size)), INTVAL (size)))
5405 : {
5406 : /* Push padding now if padding above and stack grows down,
5407 : or if padding below and stack grows up.
5408 : But if space already allocated, this has already been done. */
5409 45 : if (maybe_ne (extra, 0)
5410 : && args_addr == 0
5411 0 : && where_pad != PAD_NONE
5412 45 : && where_pad != stack_direction)
5413 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5414 :
5415 45 : move_by_pieces (NULL, xinner, INTVAL (size) - used, align,
5416 : RETURN_BEGIN);
5417 : }
5418 : else
5419 : #endif /* PUSH_ROUNDING */
5420 : {
5421 269088 : rtx target;
5422 :
5423 : /* Otherwise make space on the stack and copy the data
5424 : to the address of that space. */
5425 :
5426 : /* Deduct words put into registers from the size we must copy. */
5427 269088 : if (partial != 0)
5428 : {
5429 0 : if (CONST_INT_P (size))
5430 0 : size = GEN_INT (INTVAL (size) - used);
5431 : else
5432 0 : size = expand_binop (GET_MODE (size), sub_optab, size,
5433 0 : gen_int_mode (used, GET_MODE (size)),
5434 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
5435 : }
5436 :
5437 : /* Get the address of the stack space.
5438 : In this case, we do not deal with EXTRA separately.
5439 : A single stack adjust will do. */
5440 269088 : poly_int64 const_args_so_far;
5441 269088 : if (! args_addr)
5442 : {
5443 269034 : temp = push_block (size, extra, where_pad == PAD_DOWNWARD);
5444 269034 : extra = 0;
5445 : }
5446 54 : else if (poly_int_rtx_p (args_so_far, &const_args_so_far))
5447 72 : temp = memory_address (BLKmode,
5448 : plus_constant (Pmode, args_addr,
5449 : skip + const_args_so_far));
5450 : else
5451 0 : temp = memory_address (BLKmode,
5452 : plus_constant (Pmode,
5453 : gen_rtx_PLUS (Pmode,
5454 : args_addr,
5455 : args_so_far),
5456 : skip));
5457 :
5458 269088 : if (!ACCUMULATE_OUTGOING_ARGS)
5459 : {
5460 : /* If the source is referenced relative to the stack pointer,
5461 : copy it to another register to stabilize it. We do not need
5462 : to do this if we know that we won't be changing sp. */
5463 :
5464 269088 : if (reg_mentioned_p (virtual_stack_dynamic_rtx, temp)
5465 269088 : || reg_mentioned_p (virtual_outgoing_args_rtx, temp))
5466 269034 : temp = copy_to_reg (temp);
5467 : }
5468 :
5469 269088 : target = gen_rtx_MEM (BLKmode, temp);
5470 :
5471 : /* We do *not* set_mem_attributes here, because incoming arguments
5472 : may overlap with sibling call outgoing arguments and we cannot
5473 : allow reordering of reads from function arguments with stores
5474 : to outgoing arguments of sibling calls. We do, however, want
5475 : to record the alignment of the stack slot. */
5476 : /* ALIGN may well be better aligned than TYPE, e.g. due to
5477 : PARM_BOUNDARY. Assume the caller isn't lying. */
5478 269088 : set_mem_align (target, align);
5479 :
5480 : /* If part should go in registers and pushing to that part would
5481 : overwrite some of the values that need to go into regs, load the
5482 : overlapping values into temporary pseudos to be moved into the hard
5483 : regs at the end after the stack pushing has completed.
5484 : We cannot load them directly into the hard regs here because
5485 : they can be clobbered by the block move expansions.
5486 : See PR 65358. */
5487 :
5488 269088 : if (partial > 0 && reg != 0 && mode == BLKmode
5489 0 : && GET_CODE (reg) != PARALLEL)
5490 : {
5491 0 : overlapping = memory_load_overlap (XEXP (x, 0), temp, partial);
5492 0 : if (overlapping > 0)
5493 : {
5494 0 : gcc_assert (overlapping % UNITS_PER_WORD == 0);
5495 0 : overlapping /= UNITS_PER_WORD;
5496 :
5497 0 : tmp_regs = XALLOCAVEC (rtx, overlapping);
5498 :
5499 0 : for (int i = 0; i < overlapping; i++)
5500 0 : tmp_regs[i] = gen_reg_rtx (word_mode);
5501 :
5502 0 : for (int i = 0; i < overlapping; i++)
5503 0 : emit_move_insn (tmp_regs[i],
5504 0 : operand_subword_force (target, i, mode));
5505 : }
5506 0 : else if (overlapping == -1)
5507 : overlapping = 0;
5508 : /* Could not determine whether there is overlap.
5509 : Fail the sibcall. */
5510 : else
5511 : {
5512 0 : overlapping = 0;
5513 0 : if (sibcall_p)
5514 0 : return false;
5515 : }
5516 : }
5517 :
5518 : /* If source is a constant VAR_DECL with a simple constructor,
5519 : store the constructor to the stack instead of moving it. */
5520 269088 : const_tree decl;
5521 269088 : HOST_WIDE_INT sz;
5522 269088 : if (partial == 0
5523 269088 : && MEM_P (xinner)
5524 269088 : && SYMBOL_REF_P (XEXP (xinner, 0))
5525 82554 : && (decl = SYMBOL_REF_DECL (XEXP (xinner, 0))) != NULL_TREE
5526 82554 : && VAR_P (decl)
5527 82554 : && TREE_READONLY (decl)
5528 5183 : && !TREE_SIDE_EFFECTS (decl)
5529 5183 : && immediate_const_ctor_p (DECL_INITIAL (decl), 2)
5530 9 : && (sz = int_expr_size (DECL_INITIAL (decl))) > 0
5531 9 : && CONST_INT_P (size)
5532 269097 : && INTVAL (size) == sz)
5533 3 : store_constructor (DECL_INITIAL (decl), target, 0, sz, false);
5534 : else
5535 269085 : emit_block_move (target, xinner, size, BLOCK_OP_CALL_PARM);
5536 : }
5537 : }
5538 1901715 : else if (partial > 0)
5539 : {
5540 : /* Scalar partly in registers. This case is only supported
5541 : for fixed-wdth modes. */
5542 0 : int num_words = GET_MODE_SIZE (mode).to_constant ();
5543 0 : num_words /= UNITS_PER_WORD;
5544 0 : int i;
5545 0 : int not_stack;
5546 : /* # bytes of start of argument
5547 : that we must make space for but need not store. */
5548 0 : int offset = partial % (PARM_BOUNDARY / BITS_PER_UNIT);
5549 0 : int args_offset = INTVAL (args_so_far);
5550 0 : int skip;
5551 :
5552 : /* Push padding now if padding above and stack grows down,
5553 : or if padding below and stack grows up.
5554 : But if space already allocated, this has already been done. */
5555 0 : if (maybe_ne (extra, 0)
5556 0 : && args_addr == 0
5557 0 : && where_pad != PAD_NONE
5558 0 : && where_pad != stack_direction)
5559 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5560 :
5561 : /* If we make space by pushing it, we might as well push
5562 : the real data. Otherwise, we can leave OFFSET nonzero
5563 : and leave the space uninitialized. */
5564 0 : if (args_addr == 0)
5565 0 : offset = 0;
5566 :
5567 : /* Now NOT_STACK gets the number of words that we don't need to
5568 : allocate on the stack. Convert OFFSET to words too. */
5569 0 : not_stack = (partial - offset) / UNITS_PER_WORD;
5570 0 : offset /= UNITS_PER_WORD;
5571 :
5572 : /* If the partial register-part of the arg counts in its stack size,
5573 : skip the part of stack space corresponding to the registers.
5574 : Otherwise, start copying to the beginning of the stack space,
5575 : by setting SKIP to 0. */
5576 0 : skip = (reg_parm_stack_space == 0) ? 0 : not_stack;
5577 :
5578 0 : if (CONSTANT_P (x) && !targetm.legitimate_constant_p (mode, x))
5579 0 : x = validize_mem (force_const_mem (mode, x));
5580 :
5581 : /* If X is a hard register in a non-integer mode, copy it into a pseudo;
5582 : SUBREGs of such registers are not allowed. */
5583 0 : if ((REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER
5584 0 : && GET_MODE_CLASS (GET_MODE (x)) != MODE_INT))
5585 0 : x = copy_to_reg (x);
5586 :
5587 : /* Loop over all the words allocated on the stack for this arg. */
5588 : /* We can do it by words, because any scalar bigger than a word
5589 : has a size a multiple of a word. */
5590 0 : tree word_mode_type = lang_hooks.types.type_for_mode (word_mode, 1);
5591 0 : for (i = num_words - 1; i >= not_stack; i--)
5592 0 : if (i >= not_stack + offset)
5593 0 : if (!emit_push_insn (operand_subword_force (x, i, mode),
5594 : word_mode, word_mode_type, NULL_RTX, align, 0,
5595 : NULL_RTX, 0, args_addr,
5596 0 : GEN_INT (args_offset + ((i - not_stack + skip)
5597 : * UNITS_PER_WORD)),
5598 : reg_parm_stack_space, alignment_pad, sibcall_p))
5599 0 : return false;
5600 : }
5601 : else
5602 : {
5603 1901715 : rtx addr;
5604 1901715 : rtx dest;
5605 :
5606 : /* Push padding now if padding above and stack grows down,
5607 : or if padding below and stack grows up.
5608 : But if space already allocated, this has already been done. */
5609 1901715 : if (maybe_ne (extra, 0)
5610 0 : && args_addr == 0
5611 0 : && where_pad != PAD_NONE
5612 1901715 : && where_pad != stack_direction)
5613 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5614 :
5615 : #ifdef PUSH_ROUNDING
5616 1901715 : if (args_addr == 0 && targetm.calls.push_argument (0))
5617 1846371 : emit_single_push_insn (mode, x, type);
5618 : else
5619 : #endif
5620 : {
5621 63583 : addr = simplify_gen_binary (PLUS, Pmode, args_addr, args_so_far);
5622 55344 : dest = gen_rtx_MEM (mode, memory_address (mode, addr));
5623 :
5624 : /* We do *not* set_mem_attributes here, because incoming arguments
5625 : may overlap with sibling call outgoing arguments and we cannot
5626 : allow reordering of reads from function arguments with stores
5627 : to outgoing arguments of sibling calls. We do, however, want
5628 : to record the alignment of the stack slot. */
5629 : /* ALIGN may well be better aligned than TYPE, e.g. due to
5630 : PARM_BOUNDARY. Assume the caller isn't lying. */
5631 55344 : set_mem_align (dest, align);
5632 :
5633 55344 : emit_move_insn (dest, x);
5634 : }
5635 : }
5636 :
5637 : /* Move the partial arguments into the registers and any overlapping
5638 : values that we moved into the pseudos in tmp_regs. */
5639 2170848 : if (partial > 0 && reg != 0)
5640 : {
5641 : /* Handle calls that pass values in multiple non-contiguous locations.
5642 : The Irix 6 ABI has examples of this. */
5643 0 : if (GET_CODE (reg) == PARALLEL)
5644 0 : emit_group_load (reg, x, type, -1);
5645 : else
5646 : {
5647 0 : gcc_assert (partial % UNITS_PER_WORD == 0);
5648 0 : move_block_to_reg (REGNO (reg), x, nregs - overlapping, mode);
5649 :
5650 0 : for (int i = 0; i < overlapping; i++)
5651 0 : emit_move_insn (gen_rtx_REG (word_mode, REGNO (reg)
5652 0 : + nregs - overlapping + i),
5653 0 : tmp_regs[i]);
5654 :
5655 : }
5656 : }
5657 :
5658 2170848 : if (maybe_ne (extra, 0) && args_addr == 0 && where_pad == stack_direction)
5659 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5660 :
5661 2170848 : if (alignment_pad && args_addr == 0)
5662 2115450 : anti_adjust_stack (alignment_pad);
5663 :
5664 : return true;
5665 : }
5666 :
5667 : /* Return X if X can be used as a subtarget in a sequence of arithmetic
5668 : operations. */
5669 :
5670 : static rtx
5671 203732050 : get_subtarget (rtx x)
5672 : {
5673 203732050 : return (optimize
5674 52897503 : || x == 0
5675 : /* Only registers can be subtargets. */
5676 17709164 : || !REG_P (x)
5677 : /* Don't use hard regs to avoid extending their life. */
5678 12042394 : || REGNO (x) < FIRST_PSEUDO_REGISTER
5679 203732050 : ? 0 : x);
5680 : }
5681 :
5682 : /* A subroutine of expand_assignment. Optimize FIELD op= VAL, where
5683 : FIELD is a bitfield. Returns true if the optimization was successful,
5684 : and there's nothing else to do. */
5685 :
5686 : static bool
5687 4774649 : optimize_bitfield_assignment_op (poly_uint64 pbitsize,
5688 : poly_uint64 pbitpos,
5689 : poly_uint64 pbitregion_start,
5690 : poly_uint64 pbitregion_end,
5691 : machine_mode mode1, rtx str_rtx,
5692 : tree to, tree src, bool reverse)
5693 : {
5694 : /* str_mode is not guaranteed to be a scalar type. */
5695 4774649 : machine_mode str_mode = GET_MODE (str_rtx);
5696 4774649 : unsigned int str_bitsize;
5697 4774649 : tree op0, op1;
5698 4774649 : rtx value, result;
5699 4774649 : optab binop;
5700 4774649 : gimple *srcstmt;
5701 4774649 : enum tree_code code;
5702 :
5703 4774649 : unsigned HOST_WIDE_INT bitsize, bitpos, bitregion_start, bitregion_end;
5704 4774649 : if (mode1 != VOIDmode
5705 67161 : || !pbitsize.is_constant (&bitsize)
5706 67161 : || !pbitpos.is_constant (&bitpos)
5707 67161 : || !pbitregion_start.is_constant (&bitregion_start)
5708 67161 : || !pbitregion_end.is_constant (&bitregion_end)
5709 67796 : || bitsize >= BITS_PER_WORD
5710 66890 : || !GET_MODE_BITSIZE (str_mode).is_constant (&str_bitsize)
5711 67525 : || str_bitsize > BITS_PER_WORD
5712 61480 : || TREE_SIDE_EFFECTS (to)
5713 4836000 : || TREE_THIS_VOLATILE (to))
5714 : return false;
5715 :
5716 61351 : STRIP_NOPS (src);
5717 61351 : if (TREE_CODE (src) != SSA_NAME)
5718 : return false;
5719 20194 : if (TREE_CODE (TREE_TYPE (src)) != INTEGER_TYPE)
5720 : return false;
5721 :
5722 18598 : srcstmt = get_gimple_for_ssa_name (src);
5723 18598 : if (!srcstmt
5724 4961 : || !is_gimple_assign (srcstmt)
5725 23559 : || TREE_CODE_CLASS (gimple_assign_rhs_code (srcstmt)) != tcc_binary)
5726 : return false;
5727 :
5728 851 : code = gimple_assign_rhs_code (srcstmt);
5729 :
5730 851 : op0 = gimple_assign_rhs1 (srcstmt);
5731 :
5732 : /* If OP0 is an SSA_NAME, then we want to walk the use-def chain
5733 : to find its initialization. Hopefully the initialization will
5734 : be from a bitfield load. */
5735 851 : if (TREE_CODE (op0) == SSA_NAME)
5736 : {
5737 851 : gimple *op0stmt = get_gimple_for_ssa_name (op0);
5738 :
5739 : /* We want to eventually have OP0 be the same as TO, which
5740 : should be a bitfield. */
5741 851 : if (!op0stmt
5742 785 : || !is_gimple_assign (op0stmt)
5743 1636 : || gimple_assign_rhs_code (op0stmt) != TREE_CODE (to))
5744 : return false;
5745 548 : op0 = gimple_assign_rhs1 (op0stmt);
5746 : }
5747 :
5748 548 : op1 = gimple_assign_rhs2 (srcstmt);
5749 :
5750 548 : if (!operand_equal_p (to, op0, 0))
5751 : return false;
5752 :
5753 463 : if (MEM_P (str_rtx))
5754 : {
5755 422 : unsigned HOST_WIDE_INT offset1;
5756 :
5757 422 : if (str_bitsize == 0 || str_bitsize > BITS_PER_WORD)
5758 82 : str_bitsize = BITS_PER_WORD;
5759 :
5760 422 : scalar_int_mode best_mode;
5761 422 : if (!get_best_mode (bitsize, bitpos, bitregion_start, bitregion_end,
5762 422 : MEM_ALIGN (str_rtx), str_bitsize, false, &best_mode))
5763 0 : return false;
5764 422 : str_mode = best_mode;
5765 422 : str_bitsize = GET_MODE_BITSIZE (best_mode);
5766 :
5767 422 : offset1 = bitpos;
5768 422 : bitpos %= str_bitsize;
5769 422 : offset1 = (offset1 - bitpos) / BITS_PER_UNIT;
5770 422 : str_rtx = adjust_address (str_rtx, str_mode, offset1);
5771 : }
5772 41 : else if (!REG_P (str_rtx) && GET_CODE (str_rtx) != SUBREG)
5773 : return false;
5774 :
5775 : /* If the bit field covers the whole REG/MEM, store_field
5776 : will likely generate better code. */
5777 463 : if (bitsize >= str_bitsize)
5778 : return false;
5779 :
5780 : /* We can't handle fields split across multiple entities. */
5781 463 : if (bitpos + bitsize > str_bitsize)
5782 : return false;
5783 :
5784 463 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
5785 0 : bitpos = str_bitsize - bitpos - bitsize;
5786 :
5787 463 : switch (code)
5788 : {
5789 140 : case PLUS_EXPR:
5790 140 : case MINUS_EXPR:
5791 : /* For now, just optimize the case of the topmost bitfield
5792 : where we don't need to do any masking and also
5793 : 1 bit bitfields where xor can be used.
5794 : We might win by one instruction for the other bitfields
5795 : too if insv/extv instructions aren't used, so that
5796 : can be added later. */
5797 140 : if ((reverse || bitpos + bitsize != str_bitsize)
5798 96 : && (bitsize != 1 || TREE_CODE (op1) != INTEGER_CST))
5799 : break;
5800 :
5801 69 : value = expand_expr (op1, NULL_RTX, str_mode, EXPAND_NORMAL);
5802 69 : value = convert_modes (str_mode,
5803 69 : TYPE_MODE (TREE_TYPE (op1)), value,
5804 69 : TYPE_UNSIGNED (TREE_TYPE (op1)));
5805 :
5806 : /* We may be accessing data outside the field, which means
5807 : we can alias adjacent data. */
5808 69 : if (MEM_P (str_rtx))
5809 : {
5810 69 : str_rtx = shallow_copy_rtx (str_rtx);
5811 69 : set_mem_alias_set (str_rtx, 0);
5812 69 : set_mem_expr (str_rtx, 0);
5813 : }
5814 :
5815 69 : if (bitsize == 1 && (reverse || bitpos + bitsize != str_bitsize))
5816 : {
5817 25 : value = expand_and (str_mode, value, const1_rtx, NULL);
5818 25 : binop = xor_optab;
5819 : }
5820 : else
5821 44 : binop = code == PLUS_EXPR ? add_optab : sub_optab;
5822 :
5823 69 : value = expand_shift (LSHIFT_EXPR, str_mode, value, bitpos, NULL_RTX, 1);
5824 69 : if (reverse)
5825 0 : value = flip_storage_order (str_mode, value);
5826 69 : result = expand_binop (str_mode, binop, str_rtx,
5827 : value, str_rtx, 1, OPTAB_WIDEN);
5828 69 : if (result != str_rtx)
5829 0 : emit_move_insn (str_rtx, result);
5830 : return true;
5831 :
5832 255 : case BIT_IOR_EXPR:
5833 255 : case BIT_XOR_EXPR:
5834 255 : if (TREE_CODE (op1) != INTEGER_CST)
5835 : break;
5836 97 : value = expand_expr (op1, NULL_RTX, str_mode, EXPAND_NORMAL);
5837 97 : value = convert_modes (str_mode,
5838 97 : TYPE_MODE (TREE_TYPE (op1)), value,
5839 97 : TYPE_UNSIGNED (TREE_TYPE (op1)));
5840 :
5841 : /* We may be accessing data outside the field, which means
5842 : we can alias adjacent data. */
5843 97 : if (MEM_P (str_rtx))
5844 : {
5845 61 : str_rtx = shallow_copy_rtx (str_rtx);
5846 61 : set_mem_alias_set (str_rtx, 0);
5847 61 : set_mem_expr (str_rtx, 0);
5848 : }
5849 :
5850 97 : binop = code == BIT_IOR_EXPR ? ior_optab : xor_optab;
5851 97 : if (bitpos + bitsize != str_bitsize)
5852 : {
5853 55 : rtx mask = gen_int_mode ((HOST_WIDE_INT_1U << bitsize) - 1,
5854 : str_mode);
5855 55 : value = expand_and (str_mode, value, mask, NULL_RTX);
5856 : }
5857 97 : value = expand_shift (LSHIFT_EXPR, str_mode, value, bitpos, NULL_RTX, 1);
5858 97 : if (reverse)
5859 0 : value = flip_storage_order (str_mode, value);
5860 97 : result = expand_binop (str_mode, binop, str_rtx,
5861 : value, str_rtx, 1, OPTAB_WIDEN);
5862 97 : if (result != str_rtx)
5863 0 : emit_move_insn (str_rtx, result);
5864 : return true;
5865 :
5866 : default:
5867 : break;
5868 : }
5869 :
5870 : return false;
5871 : }
5872 :
5873 : /* In the C++ memory model, consecutive bit fields in a structure are
5874 : considered one memory location.
5875 :
5876 : Given a COMPONENT_REF EXP at position (BITPOS, OFFSET), this function
5877 : returns the bit range of consecutive bits in which this COMPONENT_REF
5878 : belongs. The values are returned in *BITSTART and *BITEND. *BITPOS
5879 : and *OFFSET may be adjusted in the process.
5880 :
5881 : If the access does not need to be restricted, 0 is returned in both
5882 : *BITSTART and *BITEND. */
5883 :
5884 : void
5885 885565 : get_bit_range (poly_uint64 *bitstart, poly_uint64 *bitend, tree exp,
5886 : poly_int64 *bitpos, tree *offset)
5887 : {
5888 885565 : poly_int64 bitoffset;
5889 885565 : tree field, repr;
5890 :
5891 885565 : gcc_assert (TREE_CODE (exp) == COMPONENT_REF);
5892 :
5893 885565 : field = TREE_OPERAND (exp, 1);
5894 885565 : repr = DECL_BIT_FIELD_REPRESENTATIVE (field);
5895 : /* If we do not have a DECL_BIT_FIELD_REPRESENTATIVE there is no
5896 : need to limit the range we can access. */
5897 885565 : if (!repr)
5898 : {
5899 65 : *bitstart = *bitend = 0;
5900 65 : return;
5901 : }
5902 :
5903 : /* If we have a DECL_BIT_FIELD_REPRESENTATIVE but the enclosing record is
5904 : part of a larger bit field, then the representative does not serve any
5905 : useful purpose. This can occur in Ada. */
5906 885500 : if (handled_component_p (TREE_OPERAND (exp, 0)))
5907 : {
5908 458567 : machine_mode rmode;
5909 458567 : poly_int64 rbitsize, rbitpos;
5910 458567 : tree roffset;
5911 458567 : int unsignedp, reversep, volatilep = 0;
5912 458567 : get_inner_reference (TREE_OPERAND (exp, 0), &rbitsize, &rbitpos,
5913 : &roffset, &rmode, &unsignedp, &reversep,
5914 : &volatilep);
5915 917134 : if (!multiple_p (rbitpos, BITS_PER_UNIT))
5916 : {
5917 0 : *bitstart = *bitend = 0;
5918 0 : return;
5919 : }
5920 : }
5921 :
5922 : /* Compute the adjustment to bitpos from the offset of the field
5923 : relative to the representative. DECL_FIELD_OFFSET of field and
5924 : repr are the same by construction if they are not constants,
5925 : see finish_bitfield_layout. */
5926 885500 : poly_uint64 field_offset, repr_offset;
5927 885500 : if (poly_int_tree_p (DECL_FIELD_OFFSET (field), &field_offset)
5928 1770995 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
5929 885495 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
5930 : else
5931 : bitoffset = 0;
5932 885500 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (field))
5933 885500 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
5934 :
5935 : /* If the adjustment is larger than bitpos, we would have a negative bit
5936 : position for the lower bound and this may wreak havoc later. Adjust
5937 : offset and bitpos to make the lower bound non-negative in that case. */
5938 885500 : if (maybe_gt (bitoffset, *bitpos))
5939 : {
5940 9 : poly_int64 adjust_bits = upper_bound (bitoffset, *bitpos) - *bitpos;
5941 9 : poly_int64 adjust_bytes = exact_div (adjust_bits, BITS_PER_UNIT);
5942 :
5943 9 : *bitpos += adjust_bits;
5944 9 : if (*offset == NULL_TREE)
5945 5 : *offset = size_int (-adjust_bytes);
5946 : else
5947 4 : *offset = size_binop (MINUS_EXPR, *offset, size_int (adjust_bytes));
5948 9 : *bitstart = 0;
5949 : }
5950 : else
5951 885491 : *bitstart = *bitpos - bitoffset;
5952 :
5953 885500 : *bitend = *bitstart + tree_to_poly_uint64 (DECL_SIZE (repr)) - 1;
5954 : }
5955 :
5956 : /* Returns true if BASE is a DECL that does not reside in memory and
5957 : has non-BLKmode. DECL_RTL must not be a MEM; if
5958 : DECL_RTL was not set yet, return false. */
5959 :
5960 : bool
5961 5507086 : non_mem_decl_p (tree base)
5962 : {
5963 5507086 : if (!DECL_P (base)
5964 5506695 : || TREE_ADDRESSABLE (base)
5965 7958647 : || DECL_MODE (base) == BLKmode)
5966 : return false;
5967 :
5968 1589896 : if (!DECL_RTL_SET_P (base))
5969 : return false;
5970 :
5971 1471285 : return (!MEM_P (DECL_RTL (base)));
5972 : }
5973 :
5974 : /* Returns true if REF refers to an object that does not
5975 : reside in memory and has non-BLKmode. */
5976 :
5977 : bool
5978 12024528 : mem_ref_refers_to_non_mem_p (tree ref)
5979 : {
5980 12024528 : tree base;
5981 :
5982 12024528 : if (TREE_CODE (ref) == MEM_REF
5983 12024528 : || TREE_CODE (ref) == TARGET_MEM_REF)
5984 : {
5985 10497093 : tree addr = TREE_OPERAND (ref, 0);
5986 :
5987 10497093 : if (TREE_CODE (addr) != ADDR_EXPR)
5988 : return false;
5989 :
5990 3954455 : base = TREE_OPERAND (addr, 0);
5991 : }
5992 : else
5993 : base = ref;
5994 :
5995 5481890 : return non_mem_decl_p (base);
5996 : }
5997 :
5998 : /* Helper function of expand_assignment. Check if storing field of
5999 : size BITSIZE at position BITPOS overlaps with the most significant
6000 : bit of TO_RTX, known to be SUBREG_PROMOTED_VAR_P.
6001 : Updating this field requires an explicit extension. */
6002 : static bool
6003 0 : store_field_updates_msb_p (poly_int64 bitpos, poly_int64 bitsize, rtx to_rtx)
6004 : {
6005 0 : return (BYTES_BIG_ENDIAN
6006 : ? maybe_le (bitpos, 0)
6007 0 : : maybe_ge (bitpos + bitsize, GET_MODE_BITSIZE (GET_MODE (to_rtx))));
6008 : }
6009 :
6010 : /* Expand an assignment that stores the value of FROM into TO. If NONTEMPORAL
6011 : is true, try generating a nontemporal store. */
6012 :
6013 : void
6014 18619784 : expand_assignment (tree to, tree from, bool nontemporal)
6015 : {
6016 18619784 : rtx to_rtx = 0;
6017 18619784 : rtx result;
6018 18619784 : machine_mode mode;
6019 18619784 : unsigned int align;
6020 18619784 : enum insn_code icode;
6021 :
6022 : /* Don't crash if the lhs of the assignment was erroneous. */
6023 18619784 : if (TREE_CODE (to) == ERROR_MARK)
6024 : {
6025 0 : expand_normal (from);
6026 0 : return;
6027 : }
6028 :
6029 : /* Optimize away no-op moves without side-effects. */
6030 18619784 : if (operand_equal_p (to, from, 0))
6031 : return;
6032 :
6033 : /* Handle misaligned stores. */
6034 18619629 : mode = TYPE_MODE (TREE_TYPE (to));
6035 18619629 : if ((TREE_CODE (to) == MEM_REF
6036 18619629 : || TREE_CODE (to) == TARGET_MEM_REF
6037 16407989 : || DECL_P (to))
6038 4068640 : && mode != BLKmode
6039 3602295 : && !mem_ref_refers_to_non_mem_p (to)
6040 6458246 : && ((align = get_object_alignment (to))
6041 3229123 : < GET_MODE_ALIGNMENT (mode))
6042 19051319 : && (((icode = optab_handler (movmisalign_optab, mode))
6043 : != CODE_FOR_nothing)
6044 96787 : || targetm.slow_unaligned_access (mode, align)))
6045 : {
6046 334903 : rtx reg, mem;
6047 :
6048 334903 : reg = expand_expr (from, NULL_RTX, VOIDmode, EXPAND_NORMAL);
6049 : /* Handle PARALLEL. */
6050 334903 : reg = maybe_emit_group_store (reg, TREE_TYPE (from));
6051 334903 : reg = force_not_mem (reg);
6052 334903 : mem = expand_expr (to, NULL_RTX, VOIDmode, EXPAND_WRITE);
6053 334903 : if (TREE_CODE (to) == MEM_REF && REF_REVERSE_STORAGE_ORDER (to))
6054 0 : reg = flip_storage_order (mode, reg);
6055 :
6056 334903 : if (icode != CODE_FOR_nothing)
6057 : {
6058 334903 : class expand_operand ops[2];
6059 :
6060 334903 : create_fixed_operand (&ops[0], mem);
6061 334903 : create_input_operand (&ops[1], reg, mode);
6062 : /* The movmisalign<mode> pattern cannot fail, else the assignment
6063 : would silently be omitted. */
6064 334903 : expand_insn (icode, 2, ops);
6065 : }
6066 : else
6067 0 : store_bit_field (mem, GET_MODE_BITSIZE (mode), 0, 0, 0, mode, reg,
6068 : false, false);
6069 334903 : return;
6070 : }
6071 :
6072 : /* Assignment of a structure component needs special treatment
6073 : if the structure component's rtx is not simply a MEM.
6074 : Assignment of an array element at a constant index, and assignment of
6075 : an array element in an unaligned packed structure field, has the same
6076 : problem. Same for (partially) storing into a non-memory object. */
6077 18284726 : if (handled_component_p (to)
6078 13740592 : || (TREE_CODE (to) == MEM_REF
6079 1641196 : && (REF_REVERSE_STORAGE_ORDER (to)
6080 1641116 : || mem_ref_refers_to_non_mem_p (to)))
6081 13631059 : || TREE_CODE (TREE_TYPE (to)) == ARRAY_TYPE)
6082 : {
6083 4774780 : machine_mode mode1;
6084 4774780 : poly_int64 bitsize, bitpos;
6085 4774780 : poly_uint64 bitregion_start = 0;
6086 4774780 : poly_uint64 bitregion_end = 0;
6087 4774780 : tree offset;
6088 4774780 : int unsignedp, reversep, volatilep = 0;
6089 4774780 : tree tem;
6090 :
6091 4774780 : push_temp_slots ();
6092 4774780 : tem = get_inner_reference (to, &bitsize, &bitpos, &offset, &mode1,
6093 : &unsignedp, &reversep, &volatilep);
6094 :
6095 : /* Make sure bitpos is not negative, it can wreak havoc later. */
6096 4774780 : if (maybe_lt (bitpos, 0))
6097 : {
6098 199 : gcc_assert (offset == NULL_TREE);
6099 199 : offset = size_int (bits_to_bytes_round_down (bitpos));
6100 199 : bitpos = num_trailing_bits (bitpos);
6101 : }
6102 :
6103 4774780 : if (TREE_CODE (to) == COMPONENT_REF
6104 4774780 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (to, 1)))
6105 85340 : get_bit_range (&bitregion_start, &bitregion_end, to, &bitpos, &offset);
6106 : /* The C++ memory model naturally applies to byte-aligned fields.
6107 : However, if we do not have a DECL_BIT_FIELD_TYPE but BITPOS or
6108 : BITSIZE are not byte-aligned, there is no need to limit the range
6109 : we can access. This can occur with packed structures in Ada. */
6110 4689440 : else if (maybe_gt (bitsize, 0)
6111 4689425 : && multiple_p (bitsize, BITS_PER_UNIT)
6112 9378598 : && multiple_p (bitpos, BITS_PER_UNIT))
6113 : {
6114 4689158 : bitregion_start = bitpos;
6115 4689158 : bitregion_end = bitpos + bitsize - 1;
6116 : }
6117 :
6118 4774780 : to_rtx = expand_expr (tem, NULL_RTX, VOIDmode, EXPAND_WRITE);
6119 :
6120 : /* If the field has a mode, we want to access it in the
6121 : field's mode, not the computed mode.
6122 : If a MEM has VOIDmode (external with incomplete type),
6123 : use BLKmode for it instead. */
6124 4774780 : if (MEM_P (to_rtx))
6125 : {
6126 4082868 : if (mode1 != VOIDmode)
6127 4016444 : to_rtx = adjust_address (to_rtx, mode1, 0);
6128 66424 : else if (GET_MODE (to_rtx) == VOIDmode)
6129 0 : to_rtx = adjust_address (to_rtx, BLKmode, 0);
6130 : }
6131 :
6132 4774780 : rtx stemp = NULL_RTX, old_to_rtx = NULL_RTX;
6133 4774780 : if (offset != 0)
6134 : {
6135 179004 : machine_mode address_mode;
6136 179004 : rtx offset_rtx;
6137 :
6138 179004 : if (!MEM_P (to_rtx))
6139 : {
6140 : /* We can get constant negative offsets into arrays with broken
6141 : user code. Translate this to a trap instead of ICEing. */
6142 4 : if (TREE_CODE (offset) == INTEGER_CST)
6143 : {
6144 1 : expand_builtin_trap ();
6145 1 : to_rtx = gen_rtx_MEM (BLKmode, const0_rtx);
6146 : }
6147 : /* Else spill for variable offset to the destination. We expect
6148 : to run into this only for hard registers. */
6149 : else
6150 : {
6151 3 : gcc_assert (VAR_P (tem) && DECL_HARD_REGISTER (tem));
6152 3 : stemp = assign_stack_temp (GET_MODE (to_rtx),
6153 6 : GET_MODE_SIZE (GET_MODE (to_rtx)));
6154 3 : emit_move_insn (stemp, to_rtx);
6155 3 : old_to_rtx = to_rtx;
6156 3 : to_rtx = stemp;
6157 : }
6158 : }
6159 :
6160 179004 : offset_rtx = expand_expr (offset, NULL_RTX, VOIDmode, EXPAND_SUM);
6161 179004 : address_mode = get_address_mode (to_rtx);
6162 179004 : if (GET_MODE (offset_rtx) != address_mode)
6163 : {
6164 : /* We cannot be sure that the RTL in offset_rtx is valid outside
6165 : of a memory address context, so force it into a register
6166 : before attempting to convert it to the desired mode. */
6167 223 : offset_rtx = force_operand (offset_rtx, NULL_RTX);
6168 223 : offset_rtx = convert_to_mode (address_mode, offset_rtx, 0);
6169 : }
6170 :
6171 : /* If we have an expression in OFFSET_RTX and a non-zero
6172 : byte offset in BITPOS, adding the byte offset before the
6173 : OFFSET_RTX results in better intermediate code, which makes
6174 : later rtl optimization passes perform better.
6175 :
6176 : We prefer intermediate code like this:
6177 :
6178 : r124:DI=r123:DI+0x18
6179 : [r124:DI]=r121:DI
6180 :
6181 : ... instead of ...
6182 :
6183 : r124:DI=r123:DI+0x10
6184 : [r124:DI+0x8]=r121:DI
6185 :
6186 : This is only done for aligned data values, as these can
6187 : be expected to result in single move instructions. */
6188 179004 : poly_int64 bytepos;
6189 179004 : if (mode1 != VOIDmode
6190 178896 : && maybe_ne (bitpos, 0)
6191 47753 : && maybe_gt (bitsize, 0)
6192 226757 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
6193 226631 : && multiple_p (bitpos, bitsize)
6194 95254 : && multiple_p (bitsize, GET_MODE_ALIGNMENT (mode1))
6195 226631 : && MEM_ALIGN (to_rtx) >= GET_MODE_ALIGNMENT (mode1))
6196 : {
6197 47378 : to_rtx = adjust_address (to_rtx, mode1, bytepos);
6198 47378 : bitregion_start = 0;
6199 47378 : if (known_ge (bitregion_end, poly_uint64 (bitpos)))
6200 47378 : bitregion_end -= bitpos;
6201 47378 : bitpos = 0;
6202 : }
6203 :
6204 179004 : to_rtx = offset_address (to_rtx, offset_rtx,
6205 : highest_pow2_factor_for_target (to,
6206 : offset));
6207 : }
6208 :
6209 : /* No action is needed if the target is not a memory and the field
6210 : lies completely outside that target. This can occur if the source
6211 : code contains an out-of-bounds access to a small array. */
6212 4774780 : if (!MEM_P (to_rtx)
6213 691908 : && GET_MODE (to_rtx) != BLKmode
6214 5466688 : && known_ge (bitpos, GET_MODE_PRECISION (GET_MODE (to_rtx))))
6215 : {
6216 3 : expand_normal (from);
6217 3 : result = NULL;
6218 : }
6219 : /* Handle expand_expr of a complex value returning a CONCAT. */
6220 4774777 : else if (GET_CODE (to_rtx) == CONCAT)
6221 : {
6222 122 : machine_mode to_mode = GET_MODE (to_rtx);
6223 122 : gcc_checking_assert (COMPLEX_MODE_P (to_mode));
6224 244 : poly_int64 mode_bitsize = GET_MODE_BITSIZE (to_mode);
6225 122 : unsigned short inner_bitsize = GET_MODE_UNIT_BITSIZE (to_mode);
6226 122 : if (TYPE_MODE (TREE_TYPE (from)) == to_mode
6227 0 : && known_eq (bitpos, 0)
6228 122 : && known_eq (bitsize, mode_bitsize))
6229 0 : result = store_expr (from, to_rtx, false, nontemporal, reversep);
6230 122 : else if (TYPE_MODE (TREE_TYPE (from)) == GET_MODE_INNER (to_mode)
6231 70 : && known_eq (bitsize, inner_bitsize)
6232 192 : && (known_eq (bitpos, 0)
6233 31 : || known_eq (bitpos, inner_bitsize)))
6234 70 : result = store_expr (from, XEXP (to_rtx, maybe_ne (bitpos, 0)),
6235 : false, nontemporal, reversep);
6236 52 : else if (known_le (bitpos + bitsize, inner_bitsize))
6237 5 : result = store_field (XEXP (to_rtx, 0), bitsize, bitpos,
6238 : bitregion_start, bitregion_end,
6239 : mode1, from, get_alias_set (to),
6240 : nontemporal, reversep);
6241 47 : else if (known_ge (bitpos, inner_bitsize))
6242 3 : result = store_field (XEXP (to_rtx, 1), bitsize,
6243 : bitpos - inner_bitsize,
6244 : bitregion_start, bitregion_end,
6245 : mode1, from, get_alias_set (to),
6246 : nontemporal, reversep);
6247 44 : else if (known_eq (bitpos, 0) && known_eq (bitsize, mode_bitsize))
6248 : {
6249 37 : result = expand_normal (from);
6250 37 : if (GET_CODE (result) == CONCAT)
6251 : {
6252 0 : to_mode = GET_MODE_INNER (to_mode);
6253 0 : machine_mode from_mode = GET_MODE_INNER (GET_MODE (result));
6254 0 : rtx from_real
6255 0 : = force_subreg (to_mode, XEXP (result, 0), from_mode, 0);
6256 0 : rtx from_imag
6257 0 : = force_subreg (to_mode, XEXP (result, 1), from_mode, 0);
6258 0 : if (!from_real || !from_imag)
6259 0 : goto concat_store_slow;
6260 0 : emit_move_insn (XEXP (to_rtx, 0), from_real);
6261 0 : emit_move_insn (XEXP (to_rtx, 1), from_imag);
6262 : }
6263 : else
6264 : {
6265 37 : machine_mode from_mode
6266 37 : = GET_MODE (result) == VOIDmode
6267 37 : ? TYPE_MODE (TREE_TYPE (from))
6268 37 : : GET_MODE (result);
6269 37 : rtx from_rtx;
6270 37 : if (MEM_P (result))
6271 1 : from_rtx = change_address (result, to_mode, NULL_RTX);
6272 : else
6273 36 : from_rtx = force_subreg (to_mode, result, from_mode, 0);
6274 37 : if (from_rtx)
6275 : {
6276 37 : emit_move_insn (XEXP (to_rtx, 0),
6277 : read_complex_part (from_rtx, false));
6278 37 : emit_move_insn (XEXP (to_rtx, 1),
6279 : read_complex_part (from_rtx, true));
6280 : }
6281 : else
6282 : {
6283 0 : to_mode = GET_MODE_INNER (to_mode);
6284 0 : rtx from_real
6285 0 : = force_subreg (to_mode, result, from_mode, 0);
6286 0 : rtx from_imag
6287 0 : = force_subreg (to_mode, result, from_mode,
6288 0 : GET_MODE_SIZE (to_mode));
6289 0 : if (!from_real || !from_imag)
6290 0 : goto concat_store_slow;
6291 0 : emit_move_insn (XEXP (to_rtx, 0), from_real);
6292 0 : emit_move_insn (XEXP (to_rtx, 1), from_imag);
6293 : }
6294 : }
6295 : }
6296 : else
6297 : {
6298 7 : concat_store_slow:;
6299 7 : rtx temp = assign_stack_temp (GET_MODE (to_rtx),
6300 14 : GET_MODE_SIZE (GET_MODE (to_rtx)));
6301 7 : write_complex_part (temp, XEXP (to_rtx, 0), false, true);
6302 7 : write_complex_part (temp, XEXP (to_rtx, 1), true, false);
6303 7 : result = store_field (temp, bitsize, bitpos,
6304 : bitregion_start, bitregion_end,
6305 : mode1, from, get_alias_set (to),
6306 : nontemporal, reversep);
6307 7 : emit_move_insn (XEXP (to_rtx, 0), read_complex_part (temp, false));
6308 7 : emit_move_insn (XEXP (to_rtx, 1), read_complex_part (temp, true));
6309 : }
6310 : }
6311 : /* For calls to functions returning variable length structures, if TO_RTX
6312 : is not a MEM, go through a MEM because we must not create temporaries
6313 : of the VLA type. */
6314 4774655 : else if (!MEM_P (to_rtx)
6315 691783 : && TREE_CODE (from) == CALL_EXPR
6316 232 : && COMPLETE_TYPE_P (TREE_TYPE (from))
6317 4774887 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (from))) != INTEGER_CST)
6318 : {
6319 6 : rtx temp = assign_stack_temp (GET_MODE (to_rtx),
6320 12 : GET_MODE_SIZE (GET_MODE (to_rtx)));
6321 6 : result = store_field (temp, bitsize, bitpos, bitregion_start,
6322 : bitregion_end, mode1, from, get_alias_set (to),
6323 : nontemporal, reversep);
6324 6 : emit_move_insn (to_rtx, temp);
6325 : }
6326 : else
6327 : {
6328 4774649 : if (MEM_P (to_rtx))
6329 : {
6330 : /* If the field is at offset zero, we could have been given the
6331 : DECL_RTX of the parent struct. Don't munge it. */
6332 4082872 : to_rtx = shallow_copy_rtx (to_rtx);
6333 4082872 : set_mem_attributes_minus_bitpos (to_rtx, to, 0, bitpos);
6334 4082872 : if (volatilep)
6335 8693 : MEM_VOLATILE_P (to_rtx) = 1;
6336 : }
6337 :
6338 4774649 : gcc_checking_assert (known_ge (bitpos, 0));
6339 4774649 : if (optimize_bitfield_assignment_op (bitsize, bitpos,
6340 : bitregion_start, bitregion_end,
6341 : mode1, to_rtx, to, from,
6342 : reversep))
6343 : result = NULL;
6344 4774483 : else if (SUBREG_P (to_rtx)
6345 4774483 : && SUBREG_PROMOTED_VAR_P (to_rtx))
6346 : {
6347 : /* If to_rtx is a promoted subreg, we need to zero or sign
6348 : extend the value afterwards. */
6349 0 : if (TREE_CODE (to) == MEM_REF
6350 0 : && TYPE_MODE (TREE_TYPE (from)) != BLKmode
6351 0 : && !REF_REVERSE_STORAGE_ORDER (to)
6352 0 : && known_eq (bitpos, 0)
6353 0 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (to_rtx))))
6354 0 : result = store_expr (from, to_rtx, 0, nontemporal, false);
6355 : /* Check if the field overlaps the MSB, requiring extension. */
6356 0 : else if (store_field_updates_msb_p (bitpos, bitsize, to_rtx))
6357 : {
6358 0 : scalar_int_mode imode = subreg_unpromoted_mode (to_rtx);
6359 0 : scalar_int_mode omode = subreg_promoted_mode (to_rtx);
6360 0 : rtx to_rtx1 = lowpart_subreg (imode, SUBREG_REG (to_rtx),
6361 : omode);
6362 0 : result = store_field (to_rtx1, bitsize, bitpos,
6363 : bitregion_start, bitregion_end,
6364 : mode1, from, get_alias_set (to),
6365 : nontemporal, reversep);
6366 : /* If the target usually keeps IMODE appropriately
6367 : extended in OMODE it's unsafe to refer to it using
6368 : a SUBREG whilst this invariant doesn't hold. */
6369 0 : if (targetm.mode_rep_extended (imode, omode) != UNKNOWN)
6370 0 : to_rtx1 = simplify_gen_unary (TRUNCATE, imode,
6371 : SUBREG_REG (to_rtx), omode);
6372 0 : convert_move (SUBREG_REG (to_rtx), to_rtx1,
6373 0 : SUBREG_PROMOTED_SIGN (to_rtx));
6374 : }
6375 : else
6376 0 : result = store_field (to_rtx, bitsize, bitpos,
6377 : bitregion_start, bitregion_end,
6378 : mode1, from, get_alias_set (to),
6379 : nontemporal, reversep);
6380 : }
6381 : else
6382 4774483 : result = store_field (to_rtx, bitsize, bitpos,
6383 : bitregion_start, bitregion_end,
6384 : mode1, from, get_alias_set (to),
6385 : nontemporal, reversep);
6386 : /* Move the temporary storage back to the non-MEM_P. */
6387 4774649 : if (stemp)
6388 3 : emit_move_insn (old_to_rtx, stemp);
6389 : }
6390 :
6391 4774780 : if (result)
6392 833089 : preserve_temp_slots (result);
6393 4774780 : pop_temp_slots ();
6394 4774780 : return;
6395 : }
6396 :
6397 : /* If the rhs is a function call and its value is not an aggregate,
6398 : call the function before we start to compute the lhs.
6399 : This is needed for correct code for cases such as
6400 : val = setjmp (buf) on machines where reference to val
6401 : requires loading up part of an address in a separate insn.
6402 :
6403 : Don't do this if TO is a VAR_DECL or PARM_DECL whose DECL_RTL is REG
6404 : since it might be a promoted variable where the zero- or sign- extension
6405 : needs to be done. Handling this in the normal way is safe because no
6406 : computation is done before the call. The same is true for SSA names. */
6407 2345654 : if (TREE_CODE (from) == CALL_EXPR && ! aggregate_value_p (from, from)
6408 2146297 : && COMPLETE_TYPE_P (TREE_TYPE (from))
6409 2146297 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (from))) == INTEGER_CST
6410 15656243 : && ! (((VAR_P (to)
6411 : || TREE_CODE (to) == PARM_DECL
6412 : || TREE_CODE (to) == RESULT_DECL)
6413 121137 : && REG_P (DECL_RTL (to)))
6414 2035228 : || TREE_CODE (to) == SSA_NAME))
6415 : {
6416 10390 : rtx value;
6417 :
6418 10390 : push_temp_slots ();
6419 10390 : value = expand_normal (from);
6420 :
6421 10390 : if (to_rtx == 0)
6422 10390 : to_rtx = expand_expr (to, NULL_RTX, VOIDmode, EXPAND_WRITE);
6423 :
6424 : /* Handle calls that return values in multiple non-contiguous locations.
6425 : The Irix 6 ABI has examples of this. */
6426 10390 : if (GET_CODE (to_rtx) == PARALLEL)
6427 : {
6428 0 : if (GET_CODE (value) == PARALLEL)
6429 0 : emit_group_move (to_rtx, value);
6430 : else
6431 0 : emit_group_load (to_rtx, value, TREE_TYPE (from),
6432 0 : int_size_in_bytes (TREE_TYPE (from)));
6433 : }
6434 10390 : else if (GET_CODE (value) == PARALLEL)
6435 1956 : emit_group_store (to_rtx, value, TREE_TYPE (from),
6436 1956 : int_size_in_bytes (TREE_TYPE (from)));
6437 8434 : else if (GET_MODE (to_rtx) == BLKmode)
6438 : {
6439 : /* Handle calls that return BLKmode values in registers. */
6440 254 : if (REG_P (value))
6441 254 : copy_blkmode_from_reg (to_rtx, value, TREE_TYPE (from));
6442 : else
6443 0 : emit_block_move (to_rtx, value, expr_size (from), BLOCK_OP_NORMAL);
6444 : }
6445 : else
6446 : {
6447 8180 : if (POINTER_TYPE_P (TREE_TYPE (to)))
6448 0 : value = convert_memory_address_addr_space
6449 0 : (as_a <scalar_int_mode> (GET_MODE (to_rtx)), value,
6450 0 : TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (to))));
6451 :
6452 8180 : emit_move_insn (to_rtx, value);
6453 : }
6454 :
6455 10390 : preserve_temp_slots (to_rtx);
6456 10390 : pop_temp_slots ();
6457 10390 : return;
6458 : }
6459 :
6460 : /* Ordinary treatment. Expand TO to get a REG or MEM rtx. */
6461 13499556 : to_rtx = expand_expr (to, NULL_RTX, VOIDmode, EXPAND_WRITE);
6462 :
6463 : /* Don't move directly into a return register. */
6464 13499556 : if (TREE_CODE (to) == RESULT_DECL
6465 29219 : && (REG_P (to_rtx) || GET_CODE (to_rtx) == PARALLEL))
6466 : {
6467 0 : rtx temp;
6468 :
6469 0 : push_temp_slots ();
6470 :
6471 : /* If the source is itself a return value, it still is in a pseudo at
6472 : this point so we can move it back to the return register directly. */
6473 0 : if (REG_P (to_rtx)
6474 0 : && TYPE_MODE (TREE_TYPE (from)) == BLKmode
6475 0 : && TREE_CODE (from) != CALL_EXPR)
6476 0 : temp = copy_blkmode_to_reg (GET_MODE (to_rtx), from);
6477 : else
6478 0 : temp = expand_expr (from, NULL_RTX, GET_MODE (to_rtx), EXPAND_NORMAL);
6479 :
6480 : /* Handle calls that return values in multiple non-contiguous locations.
6481 : The Irix 6 ABI has examples of this. */
6482 0 : if (GET_CODE (to_rtx) == PARALLEL)
6483 : {
6484 0 : if (GET_CODE (temp) == PARALLEL)
6485 0 : emit_group_move (to_rtx, temp);
6486 : else
6487 0 : emit_group_load (to_rtx, temp, TREE_TYPE (from),
6488 0 : int_size_in_bytes (TREE_TYPE (from)));
6489 : }
6490 0 : else if (temp)
6491 0 : emit_move_insn (to_rtx, temp);
6492 :
6493 0 : preserve_temp_slots (to_rtx);
6494 0 : pop_temp_slots ();
6495 0 : return;
6496 : }
6497 :
6498 : /* In case we are returning the contents of an object which overlaps
6499 : the place the value is being stored, use a safe function when copying
6500 : a value through a pointer into a structure value return block. */
6501 13499556 : if (TREE_CODE (to) == RESULT_DECL
6502 29219 : && TREE_CODE (from) == INDIRECT_REF
6503 0 : && ADDR_SPACE_GENERIC_P
6504 : (TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (from, 0)))))
6505 0 : && refs_may_alias_p (to, from)
6506 0 : && cfun->returns_struct
6507 13499556 : && !cfun->returns_pcc_struct)
6508 : {
6509 0 : rtx from_rtx, size;
6510 :
6511 0 : push_temp_slots ();
6512 0 : size = expr_size (from);
6513 0 : from_rtx = expand_normal (from);
6514 :
6515 0 : emit_block_move_via_libcall (XEXP (to_rtx, 0), XEXP (from_rtx, 0), size);
6516 :
6517 0 : preserve_temp_slots (to_rtx);
6518 0 : pop_temp_slots ();
6519 0 : return;
6520 : }
6521 :
6522 : /* Compute FROM and store the value in the rtx we got. */
6523 :
6524 13499556 : push_temp_slots ();
6525 13499556 : result = store_expr (from, to_rtx, 0, nontemporal, false);
6526 13499554 : preserve_temp_slots (result);
6527 13499554 : pop_temp_slots ();
6528 13499554 : return;
6529 : }
6530 :
6531 : /* Emits nontemporal store insn that moves FROM to TO. Returns true if this
6532 : succeeded, false otherwise. */
6533 :
6534 : bool
6535 17 : emit_storent_insn (rtx to, rtx from)
6536 : {
6537 17 : class expand_operand ops[2];
6538 17 : machine_mode mode = GET_MODE (to);
6539 17 : enum insn_code code = optab_handler (storent_optab, mode);
6540 :
6541 17 : if (code == CODE_FOR_nothing)
6542 : return false;
6543 :
6544 17 : create_fixed_operand (&ops[0], to);
6545 17 : create_input_operand (&ops[1], from, mode);
6546 17 : return maybe_expand_insn (code, 2, ops);
6547 : }
6548 :
6549 : /* Helper function for store_expr storing of STRING_CST. */
6550 :
6551 : static rtx
6552 79395 : string_cst_read_str (void *data, void *, HOST_WIDE_INT offset,
6553 : fixed_size_mode mode)
6554 : {
6555 79395 : tree str = (tree) data;
6556 :
6557 79395 : gcc_assert (offset >= 0);
6558 79395 : if (offset >= TREE_STRING_LENGTH (str))
6559 3328 : return const0_rtx;
6560 :
6561 76067 : if ((unsigned HOST_WIDE_INT) offset + GET_MODE_SIZE (mode)
6562 76067 : > (unsigned HOST_WIDE_INT) TREE_STRING_LENGTH (str))
6563 : {
6564 2460 : char *p = XALLOCAVEC (char, GET_MODE_SIZE (mode));
6565 2460 : size_t l = TREE_STRING_LENGTH (str) - offset;
6566 2460 : memcpy (p, TREE_STRING_POINTER (str) + offset, l);
6567 2460 : memset (p + l, '\0', GET_MODE_SIZE (mode) - l);
6568 2460 : return c_readstr (p, mode, false);
6569 : }
6570 :
6571 73607 : return c_readstr (TREE_STRING_POINTER (str) + offset, mode, false);
6572 : }
6573 :
6574 : /* Helper function for store_expr storing of RAW_DATA_CST. */
6575 :
6576 : static rtx
6577 36 : raw_data_cst_read_str (void *data, void *, HOST_WIDE_INT offset,
6578 : fixed_size_mode mode)
6579 : {
6580 36 : tree cst = (tree) data;
6581 :
6582 36 : gcc_assert (offset >= 0);
6583 36 : if (offset >= RAW_DATA_LENGTH (cst))
6584 0 : return const0_rtx;
6585 :
6586 36 : if ((unsigned HOST_WIDE_INT) offset + GET_MODE_SIZE (mode)
6587 36 : > (unsigned HOST_WIDE_INT) RAW_DATA_LENGTH (cst))
6588 : {
6589 0 : char *p = XALLOCAVEC (char, GET_MODE_SIZE (mode));
6590 0 : size_t l = RAW_DATA_LENGTH (cst) - offset;
6591 0 : memcpy (p, RAW_DATA_POINTER (cst) + offset, l);
6592 0 : memset (p + l, '\0', GET_MODE_SIZE (mode) - l);
6593 0 : return c_readstr (p, mode, false);
6594 : }
6595 :
6596 36 : return c_readstr (RAW_DATA_POINTER (cst) + offset, mode, false);
6597 : }
6598 :
6599 : /* Generate code for computing expression EXP,
6600 : and storing the value into TARGET.
6601 :
6602 : If the mode is BLKmode then we may return TARGET itself.
6603 : It turns out that in BLKmode it doesn't cause a problem.
6604 : because C has no operators that could combine two different
6605 : assignments into the same BLKmode object with different values
6606 : with no sequence point. Will other languages need this to
6607 : be more thorough?
6608 :
6609 : If CALL_PARAM_P is nonzero, this is a store into a call param on the
6610 : stack, and block moves may need to be treated specially.
6611 :
6612 : If NONTEMPORAL is true, try using a nontemporal store instruction.
6613 :
6614 : If REVERSE is true, the store is to be done in reverse order. */
6615 :
6616 : rtx
6617 17483461 : store_expr (tree exp, rtx target, int call_param_p,
6618 : bool nontemporal, bool reverse)
6619 : {
6620 17483461 : rtx temp;
6621 17483461 : rtx alt_rtl = NULL_RTX;
6622 17483461 : location_t loc = curr_insn_location ();
6623 17483461 : bool shortened_string_cst = false;
6624 :
6625 17483461 : if (VOID_TYPE_P (TREE_TYPE (exp)))
6626 : {
6627 : /* C++ can generate ?: expressions with a throw expression in one
6628 : branch and an rvalue in the other. Here, we resolve attempts to
6629 : store the throw expression's nonexistent result. */
6630 0 : gcc_assert (!call_param_p);
6631 0 : expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
6632 0 : return NULL_RTX;
6633 : }
6634 17483461 : if (TREE_CODE (exp) == COMPOUND_EXPR)
6635 : {
6636 : /* Perform first part of compound expression, then assign from second
6637 : part. */
6638 0 : expand_expr (TREE_OPERAND (exp, 0), const0_rtx, VOIDmode,
6639 : call_param_p ? EXPAND_STACK_PARM : EXPAND_NORMAL);
6640 0 : return store_expr (TREE_OPERAND (exp, 1), target,
6641 0 : call_param_p, nontemporal, reverse);
6642 : }
6643 17483461 : else if (TREE_CODE (exp) == COND_EXPR && GET_MODE (target) == BLKmode)
6644 : {
6645 : /* For conditional expression, get safe form of the target. Then
6646 : test the condition, doing the appropriate assignment on either
6647 : side. This avoids the creation of unnecessary temporaries.
6648 : For non-BLKmode, it is more efficient not to do this. */
6649 :
6650 0 : rtx_code_label *lab1 = gen_label_rtx (), *lab2 = gen_label_rtx ();
6651 :
6652 0 : do_pending_stack_adjust ();
6653 0 : NO_DEFER_POP;
6654 0 : jumpifnot (TREE_OPERAND (exp, 0), lab1,
6655 : profile_probability::uninitialized ());
6656 0 : store_expr (TREE_OPERAND (exp, 1), target, call_param_p,
6657 : nontemporal, reverse);
6658 0 : emit_jump_insn (targetm.gen_jump (lab2));
6659 0 : emit_barrier ();
6660 0 : emit_label (lab1);
6661 0 : store_expr (TREE_OPERAND (exp, 2), target, call_param_p,
6662 : nontemporal, reverse);
6663 0 : emit_label (lab2);
6664 0 : OK_DEFER_POP;
6665 :
6666 0 : return NULL_RTX;
6667 : }
6668 17483461 : else if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
6669 : /* If this is a scalar in a register that is stored in a wider mode
6670 : than the declared mode, compute the result into its declared mode
6671 : and then convert to the wider mode. Our value is the computed
6672 : expression. */
6673 : {
6674 7 : rtx inner_target = 0;
6675 7 : scalar_int_mode outer_mode = subreg_unpromoted_mode (target);
6676 7 : scalar_int_mode inner_mode = subreg_promoted_mode (target);
6677 :
6678 : /* We can do the conversion inside EXP, which will often result
6679 : in some optimizations. Do the conversion in two steps: first
6680 : change the signedness, if needed, then the extend. But don't
6681 : do this if the type of EXP is a subtype of something else
6682 : since then the conversion might involve more than just
6683 : converting modes. */
6684 14 : if (INTEGRAL_TYPE_P (TREE_TYPE (exp))
6685 0 : && TREE_TYPE (TREE_TYPE (exp)) == 0
6686 7 : && GET_MODE_PRECISION (outer_mode)
6687 0 : == TYPE_PRECISION (TREE_TYPE (exp)))
6688 : {
6689 0 : if (!SUBREG_CHECK_PROMOTED_SIGN (target,
6690 : TYPE_UNSIGNED (TREE_TYPE (exp))))
6691 : {
6692 : /* Some types, e.g. Fortran's logical*4, won't have a signed
6693 : version, so use the mode instead. */
6694 0 : tree ntype
6695 : = (signed_or_unsigned_type_for
6696 0 : (SUBREG_PROMOTED_SIGN (target), TREE_TYPE (exp)));
6697 0 : if (ntype == NULL)
6698 0 : ntype = lang_hooks.types.type_for_mode
6699 0 : (TYPE_MODE (TREE_TYPE (exp)),
6700 0 : SUBREG_PROMOTED_SIGN (target));
6701 :
6702 0 : exp = fold_convert_loc (loc, ntype, exp);
6703 : }
6704 :
6705 0 : exp = fold_convert_loc (loc, lang_hooks.types.type_for_mode
6706 0 : (inner_mode, SUBREG_PROMOTED_SIGN (target)),
6707 : exp);
6708 :
6709 0 : inner_target = SUBREG_REG (target);
6710 : }
6711 :
6712 7 : temp = expand_expr (exp, inner_target, VOIDmode,
6713 : call_param_p ? EXPAND_STACK_PARM : EXPAND_NORMAL);
6714 :
6715 :
6716 : /* If TEMP is a VOIDmode constant, use convert_modes to make
6717 : sure that we properly convert it. */
6718 7 : if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode)
6719 : {
6720 0 : temp = convert_modes (outer_mode, TYPE_MODE (TREE_TYPE (exp)),
6721 0 : temp, SUBREG_PROMOTED_SIGN (target));
6722 0 : temp = convert_modes (inner_mode, outer_mode, temp,
6723 0 : SUBREG_PROMOTED_SIGN (target));
6724 0 : }
6725 7 : else if (!SCALAR_INT_MODE_P (GET_MODE (temp)))
6726 0 : temp = convert_modes (outer_mode, TYPE_MODE (TREE_TYPE (exp)),
6727 0 : temp, SUBREG_PROMOTED_SIGN (target));
6728 :
6729 21 : convert_move (SUBREG_REG (target), temp,
6730 7 : SUBREG_PROMOTED_SIGN (target));
6731 :
6732 7 : return NULL_RTX;
6733 : }
6734 17483454 : else if ((TREE_CODE (exp) == STRING_CST
6735 17476230 : || (TREE_CODE (exp) == MEM_REF
6736 1354693 : && TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR
6737 443077 : && TREE_CODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
6738 : == STRING_CST
6739 6401 : && integer_zerop (TREE_OPERAND (exp, 1))))
6740 13625 : && !nontemporal && !call_param_p
6741 17497079 : && MEM_P (target))
6742 : {
6743 : /* Optimize initialization of an array with a STRING_CST. */
6744 13607 : HOST_WIDE_INT exp_len, str_copy_len;
6745 13607 : rtx dest_mem;
6746 13607 : tree str = TREE_CODE (exp) == STRING_CST
6747 13607 : ? exp : TREE_OPERAND (TREE_OPERAND (exp, 0), 0);
6748 :
6749 13607 : exp_len = int_expr_size (exp);
6750 13607 : if (exp_len <= 0)
6751 0 : goto normal_expr;
6752 :
6753 13607 : if (TREE_STRING_LENGTH (str) <= 0)
6754 0 : goto normal_expr;
6755 :
6756 27214 : if (can_store_by_pieces (exp_len, string_cst_read_str, (void *) str,
6757 13607 : MEM_ALIGN (target), false))
6758 : {
6759 13415 : store_by_pieces (target, exp_len, string_cst_read_str, (void *) str,
6760 13415 : MEM_ALIGN (target), false, RETURN_BEGIN);
6761 13415 : return NULL_RTX;
6762 : }
6763 :
6764 192 : str_copy_len = TREE_STRING_LENGTH (str);
6765 :
6766 : /* Trailing NUL bytes in EXP will be handled by the call to
6767 : clear_storage, which is more efficient than copying them from
6768 : the STRING_CST, so trim those from STR_COPY_LEN. */
6769 325 : while (str_copy_len)
6770 : {
6771 272 : if (TREE_STRING_POINTER (str)[str_copy_len - 1])
6772 : break;
6773 133 : str_copy_len--;
6774 : }
6775 :
6776 192 : if ((STORE_MAX_PIECES & (STORE_MAX_PIECES - 1)) == 0)
6777 : {
6778 192 : str_copy_len += STORE_MAX_PIECES - 1;
6779 192 : str_copy_len &= ~(STORE_MAX_PIECES - 1);
6780 : }
6781 192 : if (str_copy_len >= exp_len)
6782 131 : goto normal_expr;
6783 :
6784 122 : if (!can_store_by_pieces (str_copy_len, string_cst_read_str,
6785 61 : (void *) str, MEM_ALIGN (target), false))
6786 2 : goto normal_expr;
6787 :
6788 59 : dest_mem = store_by_pieces (target, str_copy_len, string_cst_read_str,
6789 59 : (void *) str, MEM_ALIGN (target), false,
6790 : RETURN_END);
6791 59 : clear_storage (adjust_address_1 (dest_mem, BLKmode, 0, 1, 1, 0,
6792 59 : exp_len - str_copy_len),
6793 : GEN_INT (exp_len - str_copy_len), BLOCK_OP_NORMAL);
6794 59 : return NULL_RTX;
6795 : }
6796 : else
6797 : {
6798 17469980 : rtx tmp_target;
6799 :
6800 17469980 : normal_expr:
6801 : /* If we want to use a nontemporal or a reverse order store, force the
6802 : value into a register first. */
6803 17469980 : tmp_target = nontemporal || reverse ? NULL_RTX : target;
6804 17469980 : tree rexp = exp;
6805 17469980 : if (TREE_CODE (exp) == STRING_CST
6806 20 : && tmp_target == target
6807 20 : && GET_MODE (target) == BLKmode
6808 17470000 : && TYPE_MODE (TREE_TYPE (exp)) == BLKmode)
6809 : {
6810 20 : rtx size = expr_size (exp);
6811 20 : if (CONST_INT_P (size)
6812 20 : && size != const0_rtx
6813 40 : && (UINTVAL (size)
6814 20 : > ((unsigned HOST_WIDE_INT) TREE_STRING_LENGTH (exp) + 32)))
6815 : {
6816 : /* If the STRING_CST has much larger array type than
6817 : TREE_STRING_LENGTH, only emit the TREE_STRING_LENGTH part of
6818 : it into the rodata section as the code later on will use
6819 : memset zero for the remainder anyway. See PR95052. */
6820 2 : tmp_target = NULL_RTX;
6821 2 : rexp = copy_node (exp);
6822 2 : tree index
6823 2 : = build_index_type (size_int (TREE_STRING_LENGTH (exp) - 1));
6824 2 : TREE_TYPE (rexp) = build_array_type (TREE_TYPE (TREE_TYPE (exp)),
6825 : index);
6826 2 : shortened_string_cst = true;
6827 : }
6828 : }
6829 34939960 : temp = expand_expr_real (rexp, tmp_target, GET_MODE (target),
6830 : (call_param_p
6831 : ? EXPAND_STACK_PARM : EXPAND_NORMAL),
6832 : &alt_rtl, false);
6833 17469978 : if (shortened_string_cst)
6834 : {
6835 2 : gcc_assert (MEM_P (temp));
6836 2 : temp = change_address (temp, BLKmode, NULL_RTX);
6837 : }
6838 : }
6839 :
6840 : /* If TEMP is a VOIDmode constant and the mode of the type of EXP is not
6841 : the same as that of TARGET, adjust the constant. This is needed, for
6842 : example, in case it is a CONST_DOUBLE or CONST_WIDE_INT and we want
6843 : only a word-sized value. */
6844 3609572 : if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode
6845 2603138 : && TREE_CODE (exp) != ERROR_MARK
6846 20073116 : && GET_MODE (target) != TYPE_MODE (TREE_TYPE (exp)))
6847 : {
6848 0 : gcc_assert (!shortened_string_cst);
6849 0 : if (GET_MODE_CLASS (GET_MODE (target))
6850 0 : != GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (exp)))
6851 0 : && known_eq (GET_MODE_BITSIZE (GET_MODE (target)),
6852 : GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (exp)))))
6853 : {
6854 0 : rtx t = simplify_gen_subreg (GET_MODE (target), temp,
6855 0 : TYPE_MODE (TREE_TYPE (exp)), 0);
6856 0 : if (t)
6857 0 : temp = t;
6858 : }
6859 0 : if (GET_MODE (temp) == VOIDmode)
6860 0 : temp = convert_modes (GET_MODE (target), TYPE_MODE (TREE_TYPE (exp)),
6861 0 : temp, TYPE_UNSIGNED (TREE_TYPE (exp)));
6862 : }
6863 :
6864 : /* If value was not generated in the target, store it there.
6865 : Convert the value to TARGET's type first if necessary and emit the
6866 : pending incrementations that have been queued when expanding EXP.
6867 : Note that we cannot emit the whole queue blindly because this will
6868 : effectively disable the POST_INC optimization later.
6869 :
6870 : If TEMP and TARGET compare equal according to rtx_equal_p, but
6871 : one or both of them are volatile memory refs, we have to distinguish
6872 : two cases:
6873 : - expand_expr has used TARGET. In this case, we must not generate
6874 : another copy. This can be detected by TARGET being equal according
6875 : to == .
6876 : - expand_expr has not used TARGET - that means that the source just
6877 : happens to have the same RTX form. Since temp will have been created
6878 : by expand_expr, it will compare unequal according to == .
6879 : We must generate a copy in this case, to reach the correct number
6880 : of volatile memory references. */
6881 :
6882 17469978 : if ((! rtx_equal_p (temp, target)
6883 3109124 : || (temp != target && (side_effects_p (temp)
6884 46905 : || side_effects_p (target)
6885 46905 : || (MEM_P (temp)
6886 46426 : && !mems_same_for_tbaa_p (temp, target)))))
6887 14360858 : && TREE_CODE (exp) != ERROR_MARK
6888 : /* If store_expr stores a DECL whose DECL_RTL(exp) == TARGET,
6889 : but TARGET is not valid memory reference, TEMP will differ
6890 : from TARGET although it is really the same location. */
6891 14360858 : && !(alt_rtl
6892 1981347 : && rtx_equal_p (alt_rtl, target)
6893 1 : && !side_effects_p (alt_rtl)
6894 0 : && !side_effects_p (target))
6895 : /* If there's nothing to copy, don't bother. Don't call
6896 : expr_size unless necessary, because some front-ends (C++)
6897 : expr_size-hook must not be given objects that are not
6898 : supposed to be bit-copied or bit-initialized. */
6899 31830836 : && expr_size (exp) != const0_rtx)
6900 : {
6901 14360858 : if (GET_MODE (temp) != GET_MODE (target) && GET_MODE (temp) != VOIDmode)
6902 : {
6903 1633 : gcc_assert (!shortened_string_cst);
6904 1633 : if (GET_MODE (target) == BLKmode)
6905 : {
6906 : /* Handle calls that return BLKmode values in registers. */
6907 2 : if (REG_P (temp) && TREE_CODE (exp) == CALL_EXPR)
6908 2 : copy_blkmode_from_reg (target, temp, TREE_TYPE (exp));
6909 : else
6910 0 : store_bit_field (target,
6911 0 : rtx_to_poly_int64 (expr_size (exp))
6912 0 : * BITS_PER_UNIT,
6913 : 0, 0, 0, GET_MODE (temp), temp, reverse,
6914 : false);
6915 : }
6916 : else
6917 1631 : convert_move (target, temp, TYPE_UNSIGNED (TREE_TYPE (exp)));
6918 : }
6919 :
6920 14359225 : else if (GET_MODE (temp) == BLKmode && TREE_CODE (exp) == STRING_CST)
6921 : {
6922 : /* Handle copying a string constant into an array. The string
6923 : constant may be shorter than the array. So copy just the string's
6924 : actual length, and clear the rest. First get the size of the data
6925 : type of the string, which is actually the size of the target. */
6926 20 : rtx size = expr_size (exp);
6927 :
6928 20 : if (CONST_INT_P (size)
6929 20 : && INTVAL (size) < TREE_STRING_LENGTH (exp))
6930 0 : emit_block_move (target, temp, size,
6931 : (call_param_p
6932 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
6933 : else
6934 : {
6935 20 : machine_mode pointer_mode
6936 20 : = targetm.addr_space.pointer_mode (MEM_ADDR_SPACE (target));
6937 20 : machine_mode address_mode = get_address_mode (target);
6938 :
6939 : /* Compute the size of the data to copy from the string. */
6940 20 : tree copy_size
6941 20 : = size_binop_loc (loc, MIN_EXPR,
6942 : make_tree (sizetype, size),
6943 20 : size_int (TREE_STRING_LENGTH (exp)));
6944 20 : rtx copy_size_rtx
6945 20 : = expand_expr (copy_size, NULL_RTX, VOIDmode,
6946 : (call_param_p
6947 : ? EXPAND_STACK_PARM : EXPAND_NORMAL));
6948 20 : rtx_code_label *label = 0;
6949 :
6950 : /* Copy that much. */
6951 60 : copy_size_rtx = convert_to_mode (pointer_mode, copy_size_rtx,
6952 20 : TYPE_UNSIGNED (sizetype));
6953 40 : emit_block_move (target, temp, copy_size_rtx,
6954 : (call_param_p
6955 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
6956 :
6957 : /* Figure out how much is left in TARGET that we have to clear.
6958 : Do all calculations in pointer_mode. */
6959 20 : poly_int64 const_copy_size;
6960 20 : if (poly_int_rtx_p (copy_size_rtx, &const_copy_size))
6961 : {
6962 20 : size = plus_constant (address_mode, size, -const_copy_size);
6963 20 : target = adjust_address (target, BLKmode, const_copy_size);
6964 : }
6965 : else
6966 : {
6967 0 : size = expand_binop (TYPE_MODE (sizetype), sub_optab, size,
6968 : copy_size_rtx, NULL_RTX, 0,
6969 : OPTAB_LIB_WIDEN);
6970 :
6971 0 : if (GET_MODE (copy_size_rtx) != address_mode)
6972 0 : copy_size_rtx = convert_to_mode (address_mode,
6973 : copy_size_rtx,
6974 0 : TYPE_UNSIGNED (sizetype));
6975 :
6976 0 : target = offset_address (target, copy_size_rtx,
6977 : highest_pow2_factor (copy_size));
6978 0 : label = gen_label_rtx ();
6979 0 : emit_cmp_and_jump_insns (size, const0_rtx, LT, NULL_RTX,
6980 0 : GET_MODE (size), 0, label);
6981 : }
6982 :
6983 20 : if (size != const0_rtx)
6984 2 : clear_storage (target, size, BLOCK_OP_NORMAL);
6985 :
6986 20 : if (label)
6987 0 : emit_label (label);
6988 : }
6989 : }
6990 14359205 : else if (shortened_string_cst)
6991 0 : gcc_unreachable ();
6992 : /* Handle calls that return values in multiple non-contiguous locations.
6993 : The Irix 6 ABI has examples of this. */
6994 14359205 : else if (GET_CODE (target) == PARALLEL)
6995 : {
6996 0 : if (GET_CODE (temp) == PARALLEL)
6997 0 : emit_group_move (target, temp);
6998 : else
6999 0 : emit_group_load (target, temp, TREE_TYPE (exp),
7000 0 : int_size_in_bytes (TREE_TYPE (exp)));
7001 : }
7002 14359205 : else if (GET_CODE (temp) == PARALLEL)
7003 0 : emit_group_store (target, temp, TREE_TYPE (exp),
7004 0 : int_size_in_bytes (TREE_TYPE (exp)));
7005 14359205 : else if (GET_MODE (temp) == BLKmode)
7006 601838 : emit_block_move (target, temp, expr_size (exp),
7007 : (call_param_p
7008 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
7009 : /* If we emit a nontemporal store, there is nothing else to do. */
7010 14058286 : else if (nontemporal && emit_storent_insn (target, temp))
7011 : ;
7012 : else
7013 : {
7014 14058269 : if (reverse)
7015 722 : temp = flip_storage_order (GET_MODE (target), temp);
7016 14058269 : temp = force_operand (temp, target);
7017 14058269 : if (temp != target)
7018 14056762 : emit_move_insn (target, temp);
7019 : }
7020 : }
7021 : else
7022 3109120 : gcc_assert (!shortened_string_cst);
7023 :
7024 : return NULL_RTX;
7025 : }
7026 :
7027 : /* Return true if field F of structure TYPE is a flexible array. */
7028 :
7029 : static bool
7030 3882950 : flexible_array_member_p (const_tree f, const_tree type)
7031 : {
7032 3882950 : const_tree tf;
7033 :
7034 3882950 : tf = TREE_TYPE (f);
7035 3882950 : return (DECL_CHAIN (f) == NULL
7036 1080449 : && TREE_CODE (tf) == ARRAY_TYPE
7037 2525 : && TYPE_DOMAIN (tf)
7038 2525 : && TYPE_MIN_VALUE (TYPE_DOMAIN (tf))
7039 2525 : && integer_zerop (TYPE_MIN_VALUE (TYPE_DOMAIN (tf)))
7040 2525 : && !TYPE_MAX_VALUE (TYPE_DOMAIN (tf))
7041 3882989 : && int_size_in_bytes (type) >= 0);
7042 : }
7043 :
7044 : /* If FOR_CTOR_P, return the number of top-level elements that a constructor
7045 : must have in order for it to completely initialize a value of type TYPE.
7046 : Return -1 if the number isn't known.
7047 :
7048 : If !FOR_CTOR_P, return an estimate of the number of scalars in TYPE. */
7049 :
7050 : static HOST_WIDE_INT
7051 3707150 : count_type_elements (const_tree type, bool for_ctor_p)
7052 : {
7053 3707150 : switch (TREE_CODE (type))
7054 : {
7055 154924 : case ARRAY_TYPE:
7056 154924 : {
7057 154924 : tree nelts_minus_one;
7058 :
7059 154924 : nelts_minus_one = array_type_nelts_minus_one (type);
7060 154924 : if (nelts_minus_one && tree_fits_uhwi_p (nelts_minus_one))
7061 : {
7062 154913 : unsigned HOST_WIDE_INT n;
7063 :
7064 154913 : n = tree_to_uhwi (nelts_minus_one) + 1;
7065 154913 : if (n == 0 || for_ctor_p)
7066 153573 : return n;
7067 : else
7068 1340 : return n * count_type_elements (TREE_TYPE (type), false);
7069 : }
7070 11 : return for_ctor_p ? -1 : 1;
7071 : }
7072 :
7073 1802597 : case RECORD_TYPE:
7074 1802597 : {
7075 1802597 : unsigned HOST_WIDE_INT n;
7076 1802597 : tree f;
7077 :
7078 1802597 : n = 0;
7079 16944307 : for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
7080 15141710 : if (TREE_CODE (f) == FIELD_DECL)
7081 : {
7082 4346648 : if (!for_ctor_p)
7083 463698 : n += count_type_elements (TREE_TYPE (f), false);
7084 3882950 : else if (!flexible_array_member_p (f, type))
7085 : /* Don't count flexible arrays, which are not supposed
7086 : to be initialized. */
7087 3882911 : n += 1;
7088 : }
7089 :
7090 1802597 : return n;
7091 : }
7092 :
7093 3334 : case UNION_TYPE:
7094 3334 : case QUAL_UNION_TYPE:
7095 3334 : {
7096 3334 : tree f;
7097 3334 : HOST_WIDE_INT n, m;
7098 :
7099 3334 : gcc_assert (!for_ctor_p);
7100 : /* Estimate the number of scalars in each field and pick the
7101 : maximum. Other estimates would do instead; the idea is simply
7102 : to make sure that the estimate is not sensitive to the ordering
7103 : of the fields. */
7104 3334 : n = 1;
7105 75775 : for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
7106 72441 : if (TREE_CODE (f) == FIELD_DECL)
7107 : {
7108 67980 : m = count_type_elements (TREE_TYPE (f), false);
7109 : /* If the field doesn't span the whole union, add an extra
7110 : scalar for the rest. */
7111 67980 : if (simple_cst_equal (TYPE_SIZE (TREE_TYPE (f)),
7112 67980 : TYPE_SIZE (type)) != 1)
7113 48450 : m++;
7114 67980 : if (n < m)
7115 72441 : n = m;
7116 : }
7117 : return n;
7118 : }
7119 :
7120 : case COMPLEX_TYPE:
7121 : return 2;
7122 :
7123 421 : case VECTOR_TYPE:
7124 421 : {
7125 421 : unsigned HOST_WIDE_INT nelts;
7126 421 : if (TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts))
7127 421 : return nelts;
7128 : else
7129 : return -1;
7130 : }
7131 :
7132 : case INTEGER_TYPE:
7133 : case REAL_TYPE:
7134 : case FIXED_POINT_TYPE:
7135 : case ENUMERAL_TYPE:
7136 : case BOOLEAN_TYPE:
7137 : case POINTER_TYPE:
7138 : case OFFSET_TYPE:
7139 : case REFERENCE_TYPE:
7140 : case NULLPTR_TYPE:
7141 : case OPAQUE_TYPE:
7142 : case BITINT_TYPE:
7143 : return 1;
7144 :
7145 0 : case ERROR_MARK:
7146 0 : return 0;
7147 :
7148 0 : case VOID_TYPE:
7149 0 : case METHOD_TYPE:
7150 0 : case FUNCTION_TYPE:
7151 0 : case LANG_TYPE:
7152 0 : default:
7153 0 : gcc_unreachable ();
7154 : }
7155 : }
7156 :
7157 : /* Helper for categorize_ctor_elements. Identical interface. */
7158 :
7159 : static bool
7160 1543909 : categorize_ctor_elements_1 (const_tree ctor, HOST_WIDE_INT *p_nz_elts,
7161 : HOST_WIDE_INT *p_unique_nz_elts,
7162 : HOST_WIDE_INT *p_init_elts,
7163 : ctor_completeness *p_complete)
7164 : {
7165 1543909 : unsigned HOST_WIDE_INT idx;
7166 1543909 : HOST_WIDE_INT nz_elts, unique_nz_elts, init_elts, num_fields;
7167 1543909 : tree value, purpose, elt_type;
7168 :
7169 : /* Whether CTOR is a valid constant initializer, in accordance with what
7170 : initializer_constant_valid_p does. If inferred from the constructor
7171 : elements, true until proven otherwise. */
7172 1543909 : bool const_from_elts_p = constructor_static_from_elts_p (ctor);
7173 1543909 : bool const_p = const_from_elts_p ? true : TREE_STATIC (ctor);
7174 :
7175 1543909 : nz_elts = 0;
7176 1543909 : unique_nz_elts = 0;
7177 1543909 : init_elts = 0;
7178 1543909 : num_fields = 0;
7179 1543909 : elt_type = NULL_TREE;
7180 :
7181 5854991 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (ctor), idx, purpose, value)
7182 : {
7183 4311082 : HOST_WIDE_INT mult = 1;
7184 :
7185 4311082 : if (purpose && TREE_CODE (purpose) == RANGE_EXPR)
7186 : {
7187 692 : tree lo_index = TREE_OPERAND (purpose, 0);
7188 692 : tree hi_index = TREE_OPERAND (purpose, 1);
7189 :
7190 692 : if (tree_fits_uhwi_p (lo_index) && tree_fits_uhwi_p (hi_index))
7191 692 : mult = (tree_to_uhwi (hi_index)
7192 692 : - tree_to_uhwi (lo_index) + 1);
7193 : }
7194 4311082 : num_fields += mult;
7195 4311082 : elt_type = TREE_TYPE (value);
7196 :
7197 4311082 : switch (TREE_CODE (value))
7198 : {
7199 543884 : case CONSTRUCTOR:
7200 543884 : {
7201 543884 : HOST_WIDE_INT nz = 0, unz = 0, ic = 0;
7202 :
7203 543884 : bool const_elt_p = categorize_ctor_elements_1 (value, &nz, &unz,
7204 : &ic, p_complete);
7205 :
7206 543884 : nz_elts += mult * nz;
7207 543884 : unique_nz_elts += unz;
7208 543884 : init_elts += mult * ic;
7209 :
7210 543884 : if (const_from_elts_p && const_p)
7211 330726 : const_p = const_elt_p;
7212 : }
7213 543884 : break;
7214 :
7215 2298214 : case INTEGER_CST:
7216 2298214 : case REAL_CST:
7217 2298214 : case FIXED_CST:
7218 2298214 : if (!initializer_zerop (value))
7219 : {
7220 1685110 : nz_elts += mult;
7221 1685110 : unique_nz_elts++;
7222 : }
7223 2298214 : init_elts += mult;
7224 2298214 : break;
7225 :
7226 6298 : case STRING_CST:
7227 6298 : nz_elts += mult * TREE_STRING_LENGTH (value);
7228 6298 : unique_nz_elts += TREE_STRING_LENGTH (value);
7229 6298 : init_elts += mult * TREE_STRING_LENGTH (value);
7230 6298 : break;
7231 :
7232 166 : case RAW_DATA_CST:
7233 166 : nz_elts += mult * RAW_DATA_LENGTH (value);
7234 166 : unique_nz_elts += RAW_DATA_LENGTH (value);
7235 166 : init_elts += mult * RAW_DATA_LENGTH (value);
7236 166 : num_fields += mult * (RAW_DATA_LENGTH (value) - 1);
7237 166 : break;
7238 :
7239 3234 : case COMPLEX_CST:
7240 3234 : if (!initializer_zerop (TREE_REALPART (value)))
7241 : {
7242 2630 : nz_elts += mult;
7243 2630 : unique_nz_elts++;
7244 : }
7245 3234 : if (!initializer_zerop (TREE_IMAGPART (value)))
7246 : {
7247 2272 : nz_elts += mult;
7248 2272 : unique_nz_elts++;
7249 : }
7250 3234 : init_elts += 2 * mult;
7251 3234 : break;
7252 :
7253 6372 : case VECTOR_CST:
7254 6372 : {
7255 6372 : unsigned int nunits
7256 : = constant_lower_bound
7257 6372 : (TYPE_VECTOR_SUBPARTS (TREE_TYPE (value)));
7258 39739 : for (unsigned int i = 0; i < nunits; ++i)
7259 : {
7260 33367 : tree v = VECTOR_CST_ELT (value, i);
7261 33367 : if (!initializer_zerop (v))
7262 : {
7263 18925 : nz_elts += mult;
7264 18925 : unique_nz_elts++;
7265 : }
7266 33367 : init_elts += mult;
7267 : }
7268 : }
7269 : break;
7270 :
7271 1452914 : default:
7272 1452914 : {
7273 1452914 : HOST_WIDE_INT tc = count_type_elements (elt_type, false);
7274 1452914 : nz_elts += mult * tc;
7275 1452914 : unique_nz_elts += tc;
7276 1452914 : init_elts += mult * tc;
7277 :
7278 1452914 : if (const_from_elts_p && const_p)
7279 599569 : const_p
7280 599569 : = initializer_constant_valid_p (value,
7281 : elt_type,
7282 599569 : TYPE_REVERSE_STORAGE_ORDER
7283 : (TREE_TYPE (ctor)))
7284 : != NULL_TREE;
7285 : }
7286 : break;
7287 : }
7288 : }
7289 :
7290 1543909 : if (!p_complete->sparse && !complete_ctor_at_level_p (TREE_TYPE (ctor),
7291 : num_fields, elt_type))
7292 183770 : p_complete->sparse = true;
7293 1360139 : else if (TREE_CODE (TREE_TYPE (ctor)) == UNION_TYPE
7294 1360139 : || TREE_CODE (TREE_TYPE (ctor)) == QUAL_UNION_TYPE)
7295 : {
7296 12104 : if (!p_complete->sparse
7297 7871 : && CONSTRUCTOR_ZERO_PADDING_BITS (ctor)
7298 22193 : && (num_fields
7299 5046 : ? simple_cst_equal (TYPE_SIZE (TREE_TYPE (ctor)),
7300 5043 : TYPE_SIZE (elt_type)) != 1
7301 3 : : type_has_padding_at_level_p (TREE_TYPE (ctor))))
7302 3367 : p_complete->sparse = true;
7303 4504 : else if (!p_complete->sparse && !p_complete->padded_union
7304 17737 : && (num_fields
7305 4500 : ? simple_cst_equal (TYPE_SIZE (TREE_TYPE (ctor)),
7306 4500 : TYPE_SIZE (elt_type)) != 1
7307 0 : : type_has_padding_at_level_p (TREE_TYPE (ctor))))
7308 380 : p_complete->padded_union = true;
7309 : }
7310 1348035 : else if (!p_complete->sparse
7311 1232388 : && (CONSTRUCTOR_ZERO_PADDING_BITS (ctor)
7312 1212797 : || flag_zero_init_padding_bits == ZERO_INIT_PADDING_BITS_ALL)
7313 1367655 : && type_has_padding_at_level_p (TREE_TYPE (ctor)))
7314 2764 : p_complete->sparse = true;
7315 1229624 : else if (!p_complete->sparse && !p_complete->padded_non_union
7316 2527198 : && type_has_padding_at_level_p (TREE_TYPE (ctor)))
7317 15317 : p_complete->padded_non_union = true;
7318 :
7319 1543909 : *p_nz_elts += nz_elts;
7320 1543909 : *p_unique_nz_elts += unique_nz_elts;
7321 1543909 : *p_init_elts += init_elts;
7322 :
7323 1543909 : return const_p;
7324 : }
7325 :
7326 : /* Examine CTOR to discover:
7327 : * how many scalar fields are set to nonzero values,
7328 : and place it in *P_NZ_ELTS;
7329 : * the same, but counting RANGE_EXPRs as multiplier of 1 instead of
7330 : high - low + 1 (this can be useful for callers to determine ctors
7331 : that could be cheaply initialized with - perhaps nested - loops
7332 : compared to copied from huge read-only data),
7333 : and place it in *P_UNIQUE_NZ_ELTS;
7334 : * how many scalar fields in total are in CTOR,
7335 : and place it in *P_ELT_COUNT.
7336 : * whether the constructor is complete -- in the sense that every
7337 : meaningful byte is explicitly given a value --
7338 : and place it in *P_COMPLETE:
7339 :
7340 : Return whether or not CTOR is a valid static constant initializer, the same
7341 : as "initializer_constant_valid_p (CTOR, TREE_TYPE (CTOR)) != 0". */
7342 :
7343 : bool
7344 1000025 : categorize_ctor_elements (const_tree ctor, HOST_WIDE_INT *p_nz_elts,
7345 : HOST_WIDE_INT *p_unique_nz_elts,
7346 : HOST_WIDE_INT *p_init_elts,
7347 : ctor_completeness *p_complete)
7348 : {
7349 1000025 : *p_nz_elts = 0;
7350 1000025 : *p_unique_nz_elts = 0;
7351 1000025 : *p_init_elts = 0;
7352 1000025 : *p_complete = {};
7353 :
7354 1000025 : return categorize_ctor_elements_1 (ctor, p_nz_elts, p_unique_nz_elts,
7355 1000025 : p_init_elts, p_complete);
7356 : }
7357 :
7358 : /* Return true if constructor CTOR is simple enough to be materialized
7359 : in an integer mode register. Limit the size to WORDS words, which
7360 : is 1 by default. */
7361 :
7362 : bool
7363 23688 : immediate_const_ctor_p (const_tree ctor, unsigned int words)
7364 : {
7365 : /* Allow function to be called with a VAR_DECL's DECL_INITIAL. */
7366 23688 : if (!ctor || TREE_CODE (ctor) != CONSTRUCTOR)
7367 : return false;
7368 :
7369 2817 : return TREE_CONSTANT (ctor)
7370 2817 : && !TREE_ADDRESSABLE (ctor)
7371 2711 : && CONSTRUCTOR_NELTS (ctor)
7372 2688 : && TREE_CODE (TREE_TYPE (ctor)) != ARRAY_TYPE
7373 633 : && int_expr_size (ctor) <= words * UNITS_PER_WORD
7374 3006 : && initializer_constant_valid_for_bitfield_p (ctor);
7375 : }
7376 :
7377 : /* TYPE is initialized by a constructor with NUM_ELTS elements, the last
7378 : of which had type LAST_TYPE. Each element was itself a complete
7379 : initializer, in the sense that every meaningful byte was explicitly
7380 : given a value. Return true if the same is true for the constructor
7381 : as a whole. */
7382 :
7383 : bool
7384 1730309 : complete_ctor_at_level_p (const_tree type, HOST_WIDE_INT num_elts,
7385 : const_tree last_type)
7386 : {
7387 1730309 : if (TREE_CODE (type) == UNION_TYPE || TREE_CODE (type) == QUAL_UNION_TYPE)
7388 : {
7389 9091 : if (num_elts == 0)
7390 : {
7391 24 : if (flag_zero_init_padding_bits >= ZERO_INIT_PADDING_BITS_UNIONS)
7392 : return false;
7393 :
7394 : /* If the CONSTRUCTOR doesn't have any elts, it is
7395 : incomplete if the union has at least one field. */
7396 327 : for (tree f = TYPE_FIELDS (type); f; f = DECL_CHAIN (f))
7397 324 : if (TREE_CODE (f) == FIELD_DECL)
7398 : return false;
7399 :
7400 : return true;
7401 : }
7402 :
7403 9067 : gcc_assert (num_elts == 1 && last_type);
7404 :
7405 9067 : if (flag_zero_init_padding_bits >= ZERO_INIT_PADDING_BITS_UNIONS)
7406 : /* ??? We could look at each element of the union, and find the
7407 : largest element. Which would avoid comparing the size of the
7408 : initialized element against any tail padding in the union.
7409 : Doesn't seem worth the effort... */
7410 102 : return simple_cst_equal (TYPE_SIZE (type), TYPE_SIZE (last_type)) == 1;
7411 :
7412 : return true;
7413 : }
7414 :
7415 1721218 : return count_type_elements (type, true) == num_elts;
7416 : }
7417 :
7418 : /* Return true if EXP contains mostly (3/4) zeros. */
7419 :
7420 : static bool
7421 360079 : mostly_zeros_p (const_tree exp)
7422 : {
7423 360079 : if (TREE_CODE (exp) == CONSTRUCTOR)
7424 : {
7425 197 : HOST_WIDE_INT nz_elts, unz_elts, init_elts;
7426 197 : ctor_completeness complete_p;
7427 :
7428 197 : categorize_ctor_elements (exp, &nz_elts, &unz_elts, &init_elts,
7429 : &complete_p);
7430 197 : return complete_p.sparse || nz_elts < init_elts / 4;
7431 : }
7432 :
7433 359882 : return initializer_zerop (exp);
7434 : }
7435 :
7436 : /* Return true if EXP contains all zeros. */
7437 :
7438 : static bool
7439 1085 : all_zeros_p (const_tree exp)
7440 : {
7441 1085 : if (TREE_CODE (exp) == CONSTRUCTOR)
7442 : {
7443 1085 : HOST_WIDE_INT nz_elts, unz_elts, init_elts;
7444 1085 : ctor_completeness complete_p;
7445 :
7446 1085 : categorize_ctor_elements (exp, &nz_elts, &unz_elts, &init_elts,
7447 : &complete_p);
7448 1085 : return nz_elts == 0;
7449 : }
7450 :
7451 0 : return initializer_zerop (exp);
7452 : }
7453 :
7454 : /* Helper function for store_constructor.
7455 : TARGET, BITSIZE, BITPOS, MODE, EXP are as for store_field.
7456 : CLEARED is as for store_constructor.
7457 : ALIAS_SET is the alias set to use for any stores.
7458 : If REVERSE is true, the store is to be done in reverse order.
7459 :
7460 : This provides a recursive shortcut back to store_constructor when it isn't
7461 : necessary to go through store_field. This is so that we can pass through
7462 : the cleared field to let store_constructor know that we may not have to
7463 : clear a substructure if the outer structure has already been cleared. */
7464 :
7465 : static void
7466 36600 : store_constructor_field (rtx target, poly_uint64 bitsize, poly_int64 bitpos,
7467 : poly_uint64 bitregion_start,
7468 : poly_uint64 bitregion_end,
7469 : machine_mode mode,
7470 : tree exp, int cleared,
7471 : alias_set_type alias_set, bool reverse)
7472 : {
7473 36600 : poly_int64 bytepos;
7474 36600 : poly_uint64 bytesize;
7475 36600 : if (TREE_CODE (exp) == CONSTRUCTOR
7476 : /* We can only call store_constructor recursively if the size and
7477 : bit position are on a byte boundary. */
7478 82 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
7479 82 : && maybe_ne (bitsize, 0U)
7480 36600 : && multiple_p (bitsize, BITS_PER_UNIT, &bytesize)
7481 : /* If we have a nonzero bitpos for a register target, then we just
7482 : let store_field do the bitfield handling. This is unlikely to
7483 : generate unnecessary clear instructions anyways. */
7484 36682 : && (known_eq (bitpos, 0) || MEM_P (target)))
7485 : {
7486 82 : if (MEM_P (target))
7487 : {
7488 40 : machine_mode target_mode = GET_MODE (target);
7489 40 : if (target_mode != BLKmode
7490 40 : && !multiple_p (bitpos, GET_MODE_ALIGNMENT (target_mode)))
7491 : target_mode = BLKmode;
7492 40 : target = adjust_address (target, target_mode, bytepos);
7493 : }
7494 :
7495 :
7496 : /* Update the alias set, if required. */
7497 40 : if (MEM_P (target) && ! MEM_KEEP_ALIAS_SET_P (target)
7498 122 : && MEM_ALIAS_SET (target) != 0)
7499 : {
7500 36 : target = copy_rtx (target);
7501 36 : set_mem_alias_set (target, alias_set);
7502 : }
7503 :
7504 82 : store_constructor (exp, target, cleared, bytesize, reverse);
7505 : }
7506 : else
7507 36518 : store_field (target, bitsize, bitpos, bitregion_start, bitregion_end, mode,
7508 : exp, alias_set, false, reverse);
7509 36600 : }
7510 :
7511 :
7512 : /* Returns the number of FIELD_DECLs in TYPE. */
7513 :
7514 : static int
7515 68146 : fields_length (const_tree type)
7516 : {
7517 68146 : tree t = TYPE_FIELDS (type);
7518 68146 : int count = 0;
7519 :
7520 865487 : for (; t; t = DECL_CHAIN (t))
7521 797341 : if (TREE_CODE (t) == FIELD_DECL)
7522 297715 : ++count;
7523 :
7524 68146 : return count;
7525 : }
7526 :
7527 : /* Store the value of constructor EXP into the rtx TARGET.
7528 : TARGET is either a REG or a MEM; we know it cannot conflict, since
7529 : safe_from_p has been called.
7530 : CLEARED is true if TARGET is known to have been zero'd.
7531 : If the constructor EXP has a vector type then elements of TARGET for which
7532 : there is no corresponding element in EXP are zero'd. For a variable-length
7533 : vector type, only elements up to the minimum number of subparts of the type
7534 : are explicitly zero'd; any elements beyond that are implicitly zero.
7535 : SIZE is the number of bytes of TARGET we are allowed to modify: this
7536 : may not be the same as the size of EXP if we are assigning to a field
7537 : which has been packed to exclude padding bits.
7538 : If REVERSE is true, the store is to be done in reverse order. */
7539 :
7540 : void
7541 246140 : store_constructor (tree exp, rtx target, int cleared, poly_int64 size,
7542 : bool reverse)
7543 : {
7544 246140 : tree type = TREE_TYPE (exp);
7545 246140 : HOST_WIDE_INT exp_size = int_size_in_bytes (type);
7546 246140 : poly_int64 bitregion_end = known_gt (size, 0) ? size * BITS_PER_UNIT - 1 : 0;
7547 :
7548 246140 : switch (TREE_CODE (type))
7549 : {
7550 68605 : case RECORD_TYPE:
7551 68605 : case UNION_TYPE:
7552 68605 : case QUAL_UNION_TYPE:
7553 68605 : {
7554 68605 : unsigned HOST_WIDE_INT idx;
7555 68605 : tree field, value;
7556 :
7557 : /* The storage order is specified for every aggregate type. */
7558 68605 : reverse = TYPE_REVERSE_STORAGE_ORDER (type);
7559 :
7560 : /* If size is zero or the target is already cleared, do nothing. */
7561 68605 : if (known_eq (size, 0) || cleared)
7562 : cleared = 1;
7563 : /* We either clear the aggregate or indicate the value is dead. */
7564 68605 : else if ((TREE_CODE (type) == UNION_TYPE
7565 68605 : || TREE_CODE (type) == QUAL_UNION_TYPE)
7566 68909 : && ! CONSTRUCTOR_ELTS (exp))
7567 : /* If the constructor is empty, clear the union. */
7568 : {
7569 255 : clear_storage (target, expr_size (exp), BLOCK_OP_NORMAL);
7570 255 : cleared = 1;
7571 : }
7572 :
7573 : /* If we are building a static constructor into a register,
7574 : set the initial value as zero so we can fold the value into
7575 : a constant. But if more than one register is involved,
7576 : this probably loses. */
7577 4210 : else if (REG_P (target) && TREE_STATIC (exp)
7578 68880 : && known_le (GET_MODE_SIZE (GET_MODE (target)),
7579 : REGMODE_NATURAL_SIZE (GET_MODE (target))))
7580 : {
7581 204 : emit_move_insn (target, CONST0_RTX (GET_MODE (target)));
7582 204 : cleared = 1;
7583 : }
7584 :
7585 : /* If the constructor has fewer fields than the structure or
7586 : if we are initializing the structure to mostly zeros, clear
7587 : the whole structure first. Don't do this if TARGET is a
7588 : register whose mode size isn't equal to SIZE since
7589 : clear_storage can't handle this case. */
7590 68146 : else if (known_size_p (size)
7591 68315 : && (((int) CONSTRUCTOR_NELTS (exp) != fields_length (type))
7592 156 : || mostly_zeros_p (exp))
7593 136140 : && (!REG_P (target)
7594 8002 : || known_eq (GET_MODE_SIZE (GET_MODE (target)), size)))
7595 : {
7596 78707 : clear_storage (target, gen_int_mode (size, Pmode),
7597 : BLOCK_OP_NORMAL);
7598 67994 : cleared = 1;
7599 : }
7600 :
7601 68605 : if (REG_P (target) && !cleared)
7602 5 : emit_clobber (target);
7603 :
7604 : /* Store each element of the constructor into the
7605 : corresponding field of TARGET. */
7606 68951 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (exp), idx, field, value)
7607 : {
7608 346 : machine_mode mode;
7609 346 : HOST_WIDE_INT bitsize;
7610 346 : HOST_WIDE_INT bitpos = 0;
7611 346 : tree offset;
7612 346 : rtx to_rtx = target;
7613 :
7614 : /* Just ignore missing fields. We cleared the whole
7615 : structure, above, if any fields are missing. */
7616 346 : if (field == 0)
7617 346 : continue;
7618 :
7619 346 : if (cleared && initializer_zerop (value))
7620 17 : continue;
7621 :
7622 : /* Variable sized arrays are ignored. */
7623 329 : tree decl_size = DECL_SIZE (field);
7624 329 : if (!decl_size)
7625 4 : continue;
7626 :
7627 325 : if (tree_fits_uhwi_p (decl_size))
7628 325 : bitsize = tree_to_uhwi (decl_size);
7629 : else
7630 0 : gcc_unreachable ();
7631 :
7632 325 : mode = DECL_MODE (field);
7633 325 : if (DECL_BIT_FIELD (field))
7634 24 : mode = VOIDmode;
7635 :
7636 325 : offset = DECL_FIELD_OFFSET (field);
7637 325 : if (tree_fits_shwi_p (offset)
7638 325 : && tree_fits_shwi_p (bit_position (field)))
7639 : {
7640 325 : bitpos = int_bit_position (field);
7641 325 : offset = NULL_TREE;
7642 : }
7643 : else
7644 0 : gcc_unreachable ();
7645 :
7646 : /* If this initializes a field that is smaller than a
7647 : word, at the start of a word, try to widen it to a full
7648 : word. This special case allows us to output C++ member
7649 : function initializations in a form that the optimizers
7650 : can understand. */
7651 325 : if (WORD_REGISTER_OPERATIONS
7652 : && REG_P (target)
7653 : && bitsize < BITS_PER_WORD
7654 : && bitpos % BITS_PER_WORD == 0
7655 : && GET_MODE_CLASS (mode) == MODE_INT
7656 : && TREE_CODE (value) == INTEGER_CST
7657 : && exp_size >= 0
7658 : && bitpos + BITS_PER_WORD <= exp_size * BITS_PER_UNIT)
7659 : {
7660 : type = TREE_TYPE (value);
7661 :
7662 : if (TYPE_PRECISION (type) < BITS_PER_WORD)
7663 : {
7664 : type = lang_hooks.types.type_for_mode
7665 : (word_mode, TYPE_UNSIGNED (type));
7666 : value = fold_convert (type, value);
7667 : /* Make sure the bits beyond the original bitsize are zero
7668 : so that we can correctly avoid extra zeroing stores in
7669 : later constructor elements. */
7670 : tree bitsize_mask
7671 : = wide_int_to_tree (type, wi::mask (bitsize, false,
7672 : BITS_PER_WORD));
7673 : value = fold_build2 (BIT_AND_EXPR, type, value, bitsize_mask);
7674 : }
7675 :
7676 : if (BYTES_BIG_ENDIAN)
7677 : value
7678 : = fold_build2 (LSHIFT_EXPR, type, value,
7679 : build_int_cst (type,
7680 : BITS_PER_WORD - bitsize));
7681 : bitsize = BITS_PER_WORD;
7682 : mode = word_mode;
7683 : }
7684 :
7685 207 : if (MEM_P (to_rtx) && !MEM_KEEP_ALIAS_SET_P (to_rtx)
7686 532 : && DECL_NONADDRESSABLE_P (field))
7687 : {
7688 0 : to_rtx = copy_rtx (to_rtx);
7689 0 : MEM_KEEP_ALIAS_SET_P (to_rtx) = 1;
7690 : }
7691 :
7692 325 : store_constructor_field (to_rtx, bitsize, bitpos,
7693 325 : 0, bitregion_end, mode,
7694 : value, cleared,
7695 325 : get_alias_set (TREE_TYPE (field)),
7696 : reverse);
7697 : }
7698 : break;
7699 : }
7700 39523 : case ARRAY_TYPE:
7701 39523 : {
7702 39523 : tree value, index;
7703 39523 : unsigned HOST_WIDE_INT i, j = 0;
7704 39523 : bool need_to_clear;
7705 39523 : tree domain;
7706 39523 : tree elttype = TREE_TYPE (type);
7707 39523 : bool const_bounds_p;
7708 39523 : unsigned HOST_WIDE_INT minelt = 0;
7709 39523 : unsigned HOST_WIDE_INT maxelt = 0;
7710 :
7711 : /* The storage order is specified for every aggregate type. */
7712 39523 : reverse = TYPE_REVERSE_STORAGE_ORDER (type);
7713 :
7714 39523 : domain = TYPE_DOMAIN (type);
7715 39523 : const_bounds_p = (TYPE_MIN_VALUE (domain)
7716 39523 : && TYPE_MAX_VALUE (domain)
7717 39523 : && tree_fits_uhwi_p (TYPE_MIN_VALUE (domain))
7718 79046 : && tree_fits_uhwi_p (TYPE_MAX_VALUE (domain)));
7719 :
7720 : /* If we have constant bounds for the range of the type, get them. */
7721 39523 : if (const_bounds_p)
7722 : {
7723 39523 : minelt = tree_to_uhwi (TYPE_MIN_VALUE (domain));
7724 39523 : maxelt = tree_to_uhwi (TYPE_MAX_VALUE (domain));
7725 : }
7726 :
7727 : /* If the constructor has fewer elements than the array, clear
7728 : the whole array first. Similarly if this is static
7729 : constructor of a non-BLKmode object. */
7730 39523 : if (cleared)
7731 : need_to_clear = false;
7732 39477 : else if (REG_P (target) && TREE_STATIC (exp))
7733 : need_to_clear = true;
7734 : else
7735 : {
7736 39370 : unsigned HOST_WIDE_INT idx;
7737 39370 : unsigned HOST_WIDE_INT count = 0, zero_count = 0;
7738 39370 : need_to_clear = ! const_bounds_p;
7739 :
7740 : /* This loop is a more accurate version of the loop in
7741 : mostly_zeros_p (it handles RANGE_EXPR in an index). It
7742 : is also needed to check for missing elements. */
7743 39418 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (exp), idx, index, value)
7744 : {
7745 48 : unsigned HOST_WIDE_INT this_node_count;
7746 :
7747 48 : if (need_to_clear)
7748 : break;
7749 :
7750 48 : if (index != NULL_TREE && TREE_CODE (index) == RANGE_EXPR)
7751 : {
7752 0 : tree lo_index = TREE_OPERAND (index, 0);
7753 0 : tree hi_index = TREE_OPERAND (index, 1);
7754 :
7755 0 : if (! tree_fits_uhwi_p (lo_index)
7756 0 : || ! tree_fits_uhwi_p (hi_index))
7757 : {
7758 : need_to_clear = true;
7759 : break;
7760 : }
7761 :
7762 0 : this_node_count = (tree_to_uhwi (hi_index)
7763 0 : - tree_to_uhwi (lo_index) + 1);
7764 0 : }
7765 48 : else if (TREE_CODE (value) == RAW_DATA_CST)
7766 0 : this_node_count = RAW_DATA_LENGTH (value);
7767 : else
7768 : this_node_count = 1;
7769 :
7770 48 : count += this_node_count;
7771 48 : if (mostly_zeros_p (value))
7772 0 : zero_count += this_node_count;
7773 : }
7774 :
7775 : /* Clear the entire array first if there are any missing
7776 : elements, or if the incidence of zero elements is >=
7777 : 75%. */
7778 39370 : if (! need_to_clear
7779 39370 : && (count < maxelt - minelt + 1
7780 6 : || 4 * zero_count >= 3 * count))
7781 : need_to_clear = true;
7782 : }
7783 :
7784 39471 : if (need_to_clear && maybe_gt (size, 0))
7785 : {
7786 39471 : if (REG_P (target))
7787 1605 : emit_move_insn (target, CONST0_RTX (GET_MODE (target)));
7788 : else
7789 42630 : clear_storage (target, gen_int_mode (size, Pmode),
7790 : BLOCK_OP_NORMAL);
7791 : cleared = 1;
7792 : }
7793 :
7794 52 : if (!cleared && REG_P (target))
7795 : /* Inform later passes that the old value is dead. */
7796 0 : emit_clobber (target);
7797 :
7798 : /* Store each element of the constructor into the
7799 : corresponding element of TARGET, determined by counting the
7800 : elements. */
7801 40679 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (exp), i, index, value)
7802 : {
7803 1156 : machine_mode mode;
7804 1156 : poly_int64 bitsize;
7805 1156 : HOST_WIDE_INT bitpos;
7806 1156 : rtx xtarget = target;
7807 :
7808 1156 : if (cleared && initializer_zerop (value))
7809 : {
7810 672 : if (TREE_CODE (value) == RAW_DATA_CST)
7811 0 : j += RAW_DATA_LENGTH (value) - 1;
7812 674 : continue;
7813 : }
7814 :
7815 484 : mode = TYPE_MODE (elttype);
7816 484 : if (mode != BLKmode)
7817 968 : bitsize = GET_MODE_BITSIZE (mode);
7818 0 : else if (!poly_int_tree_p (TYPE_SIZE (elttype), &bitsize))
7819 0 : bitsize = -1;
7820 :
7821 484 : if (index != NULL_TREE && TREE_CODE (index) == RANGE_EXPR)
7822 : {
7823 0 : tree lo_index = TREE_OPERAND (index, 0);
7824 0 : tree hi_index = TREE_OPERAND (index, 1);
7825 0 : rtx index_r;
7826 0 : unsigned HOST_WIDE_INT lo, hi, count;
7827 0 : tree offset;
7828 :
7829 0 : gcc_assert (TREE_CODE (value) != RAW_DATA_CST);
7830 :
7831 : /* If the range is constant and "small", unroll the loop. */
7832 0 : if (const_bounds_p
7833 0 : && tree_fits_uhwi_p (lo_index)
7834 0 : && tree_fits_uhwi_p (hi_index)
7835 0 : && (lo = tree_to_uhwi (lo_index),
7836 0 : hi = tree_to_uhwi (hi_index),
7837 0 : count = hi - lo + 1,
7838 0 : (!MEM_P (target)
7839 0 : || count <= 2
7840 0 : || (tree_fits_uhwi_p (TYPE_SIZE (elttype))
7841 0 : && (tree_to_uhwi (TYPE_SIZE (elttype)) * count
7842 : <= 40 * 8)))))
7843 : {
7844 0 : lo -= minelt; hi -= minelt;
7845 0 : for (; lo <= hi; lo++)
7846 : {
7847 0 : bitpos = lo * tree_to_uhwi (TYPE_SIZE (elttype));
7848 :
7849 0 : if (MEM_P (target)
7850 0 : && !MEM_KEEP_ALIAS_SET_P (target)
7851 0 : && TREE_CODE (type) == ARRAY_TYPE
7852 0 : && TYPE_NONALIASED_COMPONENT (type))
7853 : {
7854 0 : target = copy_rtx (target);
7855 0 : MEM_KEEP_ALIAS_SET_P (target) = 1;
7856 : }
7857 :
7858 0 : store_constructor_field
7859 0 : (target, bitsize, bitpos, 0, bitregion_end,
7860 : mode, value, cleared,
7861 : get_alias_set (elttype), reverse);
7862 : }
7863 : }
7864 : else
7865 : {
7866 0 : rtx_code_label *loop_start = gen_label_rtx ();
7867 0 : rtx_code_label *loop_end = gen_label_rtx ();
7868 0 : tree exit_cond;
7869 :
7870 0 : expand_normal (hi_index);
7871 :
7872 0 : index = build_decl (EXPR_LOCATION (exp),
7873 : VAR_DECL, NULL_TREE, domain);
7874 0 : index_r = gen_reg_rtx (promote_decl_mode (index, NULL));
7875 0 : SET_DECL_RTL (index, index_r);
7876 0 : store_expr (lo_index, index_r, 0, false, reverse);
7877 :
7878 : /* Build the head of the loop. */
7879 0 : do_pending_stack_adjust ();
7880 0 : emit_label (loop_start);
7881 :
7882 : /* Assign value to element index. */
7883 0 : offset = fold_build2 (MINUS_EXPR,
7884 : TREE_TYPE (index),
7885 : index,
7886 : TYPE_MIN_VALUE (domain));
7887 :
7888 0 : offset = size_binop (MULT_EXPR,
7889 : fold_convert (sizetype, offset),
7890 : TYPE_SIZE_UNIT (elttype));
7891 :
7892 0 : xtarget = offset_address (target,
7893 : expand_normal (offset),
7894 : highest_pow2_factor (offset));
7895 0 : xtarget = adjust_address (xtarget, mode, 0);
7896 0 : if (TREE_CODE (value) == CONSTRUCTOR)
7897 0 : store_constructor (value, xtarget, cleared,
7898 : exact_div (bitsize, BITS_PER_UNIT),
7899 : reverse);
7900 : else
7901 0 : store_expr (value, xtarget, 0, false, reverse);
7902 :
7903 : /* Generate a conditional jump to exit the loop. */
7904 0 : exit_cond = build2 (LT_EXPR, integer_type_node,
7905 : index, hi_index);
7906 0 : jumpif (exit_cond, loop_end,
7907 : profile_probability::uninitialized ());
7908 :
7909 : /* Update the loop counter, and jump to the head of
7910 : the loop. */
7911 0 : expand_assignment (index,
7912 0 : build2 (PLUS_EXPR, TREE_TYPE (index),
7913 : index, integer_one_node),
7914 : false);
7915 :
7916 0 : emit_jump (loop_start);
7917 :
7918 : /* Build the end of the loop. */
7919 0 : emit_label (loop_end);
7920 : }
7921 : }
7922 484 : else if ((index && !tree_fits_uhwi_p (index))
7923 484 : || !tree_fits_uhwi_p (TYPE_SIZE (elttype)))
7924 : {
7925 0 : tree offset;
7926 :
7927 0 : gcc_assert (TREE_CODE (value) != RAW_DATA_CST);
7928 0 : if (index)
7929 0 : offset = fold_build2 (MINUS_EXPR,
7930 : TREE_TYPE (index),
7931 : index,
7932 : TYPE_MIN_VALUE (domain));
7933 : else
7934 0 : offset = size_int (i + j);
7935 :
7936 0 : offset = size_binop (MULT_EXPR,
7937 : fold_convert (sizetype, offset),
7938 : TYPE_SIZE_UNIT (elttype));
7939 0 : xtarget = offset_address (target,
7940 : expand_normal (offset),
7941 : highest_pow2_factor (offset));
7942 0 : xtarget = adjust_address (xtarget, mode, 0);
7943 0 : store_expr (value, xtarget, 0, false, reverse);
7944 : }
7945 : else
7946 : {
7947 484 : if (index)
7948 968 : bitpos = ((tree_to_uhwi (index) - minelt)
7949 484 : * tree_to_uhwi (TYPE_SIZE (elttype)));
7950 : else
7951 0 : bitpos = ((i + j) * tree_to_uhwi (TYPE_SIZE (elttype)));
7952 :
7953 52 : if (MEM_P (target) && !MEM_KEEP_ALIAS_SET_P (target)
7954 52 : && TREE_CODE (type) == ARRAY_TYPE
7955 536 : && TYPE_NONALIASED_COMPONENT (type))
7956 : {
7957 0 : target = copy_rtx (target);
7958 0 : MEM_KEEP_ALIAS_SET_P (target) = 1;
7959 : }
7960 484 : if (TREE_CODE (value) != RAW_DATA_CST)
7961 480 : store_constructor_field (target, bitsize, bitpos, 0,
7962 480 : bitregion_end, mode, value,
7963 : cleared, get_alias_set (elttype),
7964 : reverse);
7965 : else
7966 : {
7967 4 : j += RAW_DATA_LENGTH (value) - 1;
7968 4 : gcc_assert (known_eq (bitsize, BITS_PER_UNIT));
7969 4 : rtx to_rtx = adjust_address (target, mode,
7970 : bitpos / BITS_PER_UNIT);
7971 :
7972 4 : if (to_rtx == target)
7973 0 : to_rtx = copy_rtx (to_rtx);
7974 :
7975 4 : if (!MEM_KEEP_ALIAS_SET_P (to_rtx)
7976 8 : && MEM_ALIAS_SET (to_rtx) != 0)
7977 0 : set_mem_alias_set (to_rtx, get_alias_set (elttype));
7978 :
7979 8 : if (can_store_by_pieces (RAW_DATA_LENGTH (value),
7980 : raw_data_cst_read_str,
7981 : (void *) value,
7982 4 : MEM_ALIGN (target), false))
7983 : {
7984 4 : store_by_pieces (target, RAW_DATA_LENGTH (value),
7985 : raw_data_cst_read_str, (void *) value,
7986 2 : MEM_ALIGN (target), false,
7987 : RETURN_BEGIN);
7988 2 : continue;
7989 : }
7990 :
7991 2 : elttype
7992 2 : = build_array_type_nelts (TREE_TYPE (value),
7993 2 : RAW_DATA_LENGTH (value));
7994 2 : tree ctor = build_constructor_single (elttype, NULL_TREE,
7995 : value);
7996 2 : ctor = tree_output_constant_def (ctor);
7997 2 : mode = TYPE_MODE (type);
7998 2 : store_constructor_field (target,
7999 2 : bitsize * RAW_DATA_LENGTH (value),
8000 2 : bitpos, 0, bitregion_end, mode,
8001 : ctor, cleared,
8002 : get_alias_set (elttype), reverse);
8003 : }
8004 : }
8005 : }
8006 : break;
8007 : }
8008 :
8009 138012 : case VECTOR_TYPE:
8010 138012 : {
8011 138012 : unsigned HOST_WIDE_INT idx;
8012 138012 : constructor_elt *ce;
8013 138012 : bool need_to_clear;
8014 138012 : insn_code icode = CODE_FOR_nothing;
8015 138012 : tree elt;
8016 138012 : tree elttype = TREE_TYPE (type);
8017 138012 : int elt_size = vector_element_bits (type);
8018 138012 : machine_mode eltmode = TYPE_MODE (elttype);
8019 138012 : poly_int64 bitsize;
8020 138012 : poly_int64 bitpos;
8021 138012 : rtvec vector = NULL;
8022 138012 : poly_uint64 n_elts;
8023 138012 : unsigned HOST_WIDE_INT const_n_elts;
8024 138012 : alias_set_type alias;
8025 138012 : bool vec_vec_init_p = false;
8026 138012 : machine_mode mode = GET_MODE (target);
8027 :
8028 138012 : gcc_assert (eltmode != BLKmode);
8029 :
8030 : /* Try using vec_duplicate_optab for uniform vectors. */
8031 138012 : icode = optab_handler (vec_duplicate_optab, mode);
8032 138012 : if (!TREE_SIDE_EFFECTS (exp)
8033 138012 : && VECTOR_MODE_P (mode)
8034 134586 : && icode != CODE_FOR_nothing
8035 : /* If the vec_duplicate target pattern does not specify an element
8036 : mode check that eltmode is the normal inner mode of the
8037 : requested vector mode. But if the target allows eltmode
8038 : explicitly go ahead and use it. */
8039 92574 : && (eltmode == GET_MODE_INNER (mode)
8040 0 : || insn_data[icode].operand[1].mode == eltmode)
8041 92574 : && (elt = uniform_vector_p (exp))
8042 155688 : && !VECTOR_TYPE_P (TREE_TYPE (elt)))
8043 : {
8044 17676 : class expand_operand ops[2];
8045 17676 : create_output_operand (&ops[0], target, mode);
8046 17676 : create_input_operand (&ops[1], expand_normal (elt), eltmode);
8047 17676 : expand_insn (icode, 2, ops);
8048 17676 : if (!rtx_equal_p (target, ops[0].value))
8049 0 : emit_move_insn (target, ops[0].value);
8050 17676 : break;
8051 : }
8052 : /* Use sign-extension for uniform boolean vectors with
8053 : integer modes and single-bit mask entries.
8054 : Effectively "vec_duplicate" for bitmasks. */
8055 120336 : if (elt_size == 1
8056 146 : && !TREE_SIDE_EFFECTS (exp)
8057 146 : && VECTOR_BOOLEAN_TYPE_P (type)
8058 146 : && SCALAR_INT_MODE_P (TYPE_MODE (type))
8059 146 : && (elt = uniform_vector_p (exp))
8060 143 : && !VECTOR_TYPE_P (TREE_TYPE (elt))
8061 120336 : && !BYTES_BIG_ENDIAN)
8062 : {
8063 143 : rtx op0 = force_reg (TYPE_MODE (TREE_TYPE (elt)),
8064 : expand_normal (elt));
8065 143 : rtx tmp = gen_reg_rtx (mode);
8066 143 : convert_move (tmp, op0, 0);
8067 :
8068 : /* Ensure no excess bits are set.
8069 : GCN needs this for nunits < 64.
8070 : x86 needs this for nunits < 8. */
8071 143 : unsigned int nunits = TYPE_VECTOR_SUBPARTS (type).to_constant ();
8072 143 : if (maybe_ne (GET_MODE_PRECISION (mode), nunits))
8073 2 : tmp = expand_binop (mode, and_optab, tmp,
8074 2 : GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
8075 : target, true, OPTAB_WIDEN);
8076 143 : if (tmp != target)
8077 141 : emit_move_insn (target, tmp);
8078 : break;
8079 : }
8080 :
8081 120193 : n_elts = TYPE_VECTOR_SUBPARTS (type);
8082 120193 : if (REG_P (target)
8083 117070 : && VECTOR_MODE_P (mode))
8084 : {
8085 116910 : const_n_elts = 0;
8086 116910 : if (CONSTRUCTOR_NELTS (exp)
8087 116910 : && (TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (exp, 0)->value))
8088 : == VECTOR_TYPE))
8089 : {
8090 1438 : tree etype = TREE_TYPE (CONSTRUCTOR_ELT (exp, 0)->value);
8091 1438 : gcc_assert (known_eq (CONSTRUCTOR_NELTS (exp)
8092 : * TYPE_VECTOR_SUBPARTS (etype),
8093 : n_elts));
8094 :
8095 4314 : icode = convert_optab_handler (vec_init_optab, mode,
8096 1438 : TYPE_MODE (etype));
8097 1438 : const_n_elts = CONSTRUCTOR_NELTS (exp);
8098 1438 : vec_vec_init_p = icode != CODE_FOR_nothing;
8099 : }
8100 230944 : else if (exact_div (n_elts, GET_MODE_NUNITS (eltmode))
8101 115472 : .is_constant (&const_n_elts))
8102 : {
8103 : /* For a non-const type vector, we check it is made up of
8104 : similarly non-const type vectors. */
8105 115472 : icode = convert_optab_handler (vec_init_optab, mode, eltmode);
8106 : }
8107 : else
8108 : {
8109 : /* Handle variable-length vector types. */
8110 : icode = convert_optab_handler (vec_init_optab, mode, eltmode);
8111 : const_n_elts = constant_lower_bound (n_elts);
8112 : }
8113 :
8114 116910 : if (const_n_elts && icode != CODE_FOR_nothing)
8115 : {
8116 116179 : vector = rtvec_alloc (const_n_elts);
8117 433617 : for (unsigned int k = 0; k < const_n_elts; k++)
8118 317438 : RTVEC_ELT (vector, k) = CONST0_RTX (eltmode);
8119 : }
8120 : }
8121 : else
8122 : gcc_assert (n_elts.is_constant ());
8123 :
8124 : /* Compute the size of the elements in the CTOR. It differs
8125 : from the size of the vector type elements only when the
8126 : CTOR elements are vectors themselves. */
8127 120193 : tree val_type = (CONSTRUCTOR_NELTS (exp) != 0
8128 120193 : ? TREE_TYPE (CONSTRUCTOR_ELT (exp, 0)->value)
8129 120193 : : elttype);
8130 120193 : if (VECTOR_TYPE_P (val_type))
8131 1760 : bitsize = tree_to_poly_uint64 (TYPE_SIZE (val_type));
8132 : else
8133 118433 : bitsize = elt_size;
8134 :
8135 : /* If the constructor has fewer elements than the vector,
8136 : clear the whole array first. Similarly if this is static
8137 : constructor of a non-BLKmode object. */
8138 120193 : if (cleared)
8139 : need_to_clear = false;
8140 120193 : else if (REG_P (target) && TREE_STATIC (exp))
8141 : need_to_clear = true;
8142 : else
8143 : {
8144 : poly_uint64 count = 0, zero_count = 0;
8145 : tree value;
8146 :
8147 480027 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), idx, value)
8148 : {
8149 359834 : poly_int64 n_elts_here = exact_div (bitsize, elt_size);
8150 359834 : count += n_elts_here;
8151 359834 : if (mostly_zeros_p (value))
8152 359834 : zero_count += n_elts_here;
8153 : }
8154 :
8155 : /* Clear the entire vector first if there are any missing elements,
8156 : or if the incidence of zero elements is >= 75%. */
8157 120193 : need_to_clear = (maybe_lt (count, n_elts)
8158 120193 : || maybe_gt (4 * zero_count, 3 * count));
8159 : }
8160 :
8161 1164 : if (need_to_clear && maybe_gt (size, 0) && !vector)
8162 : {
8163 682 : if (REG_P (target))
8164 1 : emit_move_insn (target, CONST0_RTX (mode));
8165 : else
8166 681 : clear_storage (target, gen_int_mode (size, Pmode),
8167 : BLOCK_OP_NORMAL);
8168 : cleared = 1;
8169 : }
8170 :
8171 : /* Inform later passes that the old value is dead. */
8172 120193 : if (!cleared && !vector && REG_P (target) && maybe_gt (n_elts, 1u))
8173 : {
8174 573 : emit_move_insn (target, CONST0_RTX (mode));
8175 573 : cleared = 1;
8176 : }
8177 :
8178 120193 : if (MEM_P (target))
8179 3123 : alias = MEM_ALIAS_SET (target);
8180 : else
8181 117070 : alias = get_alias_set (elttype);
8182 :
8183 : /* Store each element of the constructor into the corresponding
8184 : element of TARGET, determined by counting the elements. */
8185 120193 : HOST_WIDE_INT chunk_size = 0;
8186 120193 : bool chunk_multiple_p = constant_multiple_p (bitsize, elt_size,
8187 : &chunk_size);
8188 120193 : gcc_assert (chunk_multiple_p || vec_vec_init_p);
8189 :
8190 480027 : for (idx = 0; vec_safe_iterate (CONSTRUCTOR_ELTS (exp), idx, &ce);
8191 : idx++)
8192 : {
8193 359834 : HOST_WIDE_INT eltpos;
8194 359834 : tree value = ce->value;
8195 :
8196 359834 : if (cleared && initializer_zerop (value))
8197 7683 : continue;
8198 :
8199 352151 : if (ce->index)
8200 48049 : eltpos = tree_to_uhwi (ce->index);
8201 : else
8202 304102 : eltpos = idx * chunk_size;
8203 :
8204 352151 : if (vector)
8205 : {
8206 316358 : if (vec_vec_init_p)
8207 : {
8208 2772 : gcc_assert (ce->index == NULL_TREE);
8209 2772 : gcc_assert (TREE_CODE (TREE_TYPE (value)) == VECTOR_TYPE);
8210 2772 : eltpos = idx;
8211 : }
8212 : else
8213 313586 : gcc_assert (TREE_CODE (TREE_TYPE (value)) != VECTOR_TYPE);
8214 316358 : RTVEC_ELT (vector, eltpos) = expand_normal (value);
8215 : }
8216 : else
8217 : {
8218 35793 : machine_mode value_mode
8219 35793 : = (TREE_CODE (TREE_TYPE (value)) == VECTOR_TYPE
8220 35793 : ? TYPE_MODE (TREE_TYPE (value)) : eltmode);
8221 35793 : bitpos = eltpos * elt_size;
8222 35793 : store_constructor_field (target, bitsize, bitpos, 0,
8223 35793 : bitregion_end, value_mode,
8224 : value, cleared, alias, reverse);
8225 : }
8226 : }
8227 :
8228 120193 : if (vector)
8229 116179 : emit_insn (GEN_FCN (icode) (target,
8230 : gen_rtx_PARALLEL (mode, vector)));
8231 : break;
8232 : }
8233 :
8234 0 : default:
8235 0 : gcc_unreachable ();
8236 : }
8237 246140 : }
8238 :
8239 : /* Store the value of EXP (an expression tree)
8240 : into a subfield of TARGET which has mode MODE and occupies
8241 : BITSIZE bits, starting BITPOS bits from the start of TARGET.
8242 : If MODE is VOIDmode, it means that we are storing into a bit-field.
8243 :
8244 : BITREGION_START is bitpos of the first bitfield in this region.
8245 : BITREGION_END is the bitpos of the ending bitfield in this region.
8246 : These two fields are 0, if the C++ memory model does not apply,
8247 : or we are not interested in keeping track of bitfield regions.
8248 :
8249 : Always return const0_rtx unless we have something particular to
8250 : return.
8251 :
8252 : ALIAS_SET is the alias set for the destination. This value will
8253 : (in general) be different from that for TARGET, since TARGET is a
8254 : reference to the containing structure.
8255 :
8256 : If NONTEMPORAL is true, try generating a nontemporal store.
8257 :
8258 : If REVERSE is true, the store is to be done in reverse order. */
8259 :
8260 : static rtx
8261 4811022 : store_field (rtx target, poly_int64 bitsize, poly_int64 bitpos,
8262 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
8263 : machine_mode mode, tree exp,
8264 : alias_set_type alias_set, bool nontemporal, bool reverse)
8265 : {
8266 4811022 : if (TREE_CODE (exp) == ERROR_MARK)
8267 0 : return const0_rtx;
8268 :
8269 : /* If we have nothing to store, do nothing unless the expression has
8270 : side-effects. Don't do that for zero sized addressable lhs of
8271 : calls. */
8272 4811022 : if (known_eq (bitsize, 0)
8273 4811022 : && (!TREE_ADDRESSABLE (TREE_TYPE (exp))
8274 3 : || TREE_CODE (exp) != CALL_EXPR))
8275 0 : return expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
8276 :
8277 4811022 : if (GET_CODE (target) == CONCAT)
8278 : {
8279 : /* We're storing into a struct containing a single __complex. */
8280 :
8281 0 : gcc_assert (known_eq (bitpos, 0));
8282 0 : return store_expr (exp, target, 0, nontemporal, reverse);
8283 : }
8284 :
8285 : /* If the structure is in a register or if the component
8286 : is a bit field, we cannot use addressing to access it.
8287 : Use bit-field techniques or SUBREG to store in it. */
8288 :
8289 4811022 : poly_int64 decl_bitsize;
8290 4811022 : if (mode == VOIDmode
8291 4744003 : || (mode != BLKmode && ! direct_store[(int) mode]
8292 5338 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_INT
8293 5336 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_FLOAT)
8294 4738696 : || REG_P (target)
8295 4045550 : || GET_CODE (target) == SUBREG
8296 : /* If the field isn't aligned enough to store as an ordinary memref,
8297 : store it as a bit field. */
8298 4045550 : || (mode != BLKmode
8299 3930142 : && ((((MEM_ALIGN (target) < GET_MODE_ALIGNMENT (mode))
8300 7772356 : || !multiple_p (bitpos, GET_MODE_ALIGNMENT (mode)))
8301 46733 : && targetm.slow_unaligned_access (mode, MEM_ALIGN (target)))
8302 3930142 : || !multiple_p (bitpos, BITS_PER_UNIT)))
8303 4045550 : || (known_size_p (bitsize)
8304 4045538 : && mode != BLKmode
8305 3930142 : && maybe_gt (GET_MODE_BITSIZE (mode), bitsize))
8306 : /* If the RHS and field are a constant size and the size of the
8307 : RHS isn't the same size as the bitfield, we must use bitfield
8308 : operations. */
8309 4045541 : || (known_size_p (bitsize)
8310 4045529 : && poly_int_tree_p (TYPE_SIZE (TREE_TYPE (exp)))
8311 4045529 : && maybe_ne (wi::to_poly_offset (TYPE_SIZE (TREE_TYPE (exp))),
8312 : bitsize)
8313 : /* Except for initialization of full bytes from a CONSTRUCTOR, which
8314 : we will handle specially below. */
8315 34 : && !(TREE_CODE (exp) == CONSTRUCTOR
8316 11 : && multiple_p (bitsize, BITS_PER_UNIT))
8317 : /* And except for bitwise copying of TREE_ADDRESSABLE types,
8318 : where the FIELD_DECL has the right bitsize, but TREE_TYPE (exp)
8319 : includes some extra padding. store_expr / expand_expr will in
8320 : that case call get_inner_reference that will have the bitsize
8321 : we check here and thus the block move will not clobber the
8322 : padding that shouldn't be clobbered. In the future we could
8323 : replace the TREE_ADDRESSABLE check with a check that
8324 : get_base_address needs to live in memory. */
8325 23 : && (!TREE_ADDRESSABLE (TREE_TYPE (exp))
8326 9 : || TREE_CODE (exp) != COMPONENT_REF
8327 6 : || !multiple_p (bitsize, BITS_PER_UNIT)
8328 6 : || !multiple_p (bitpos, BITS_PER_UNIT)
8329 6 : || !poly_int_tree_p (DECL_SIZE (TREE_OPERAND (exp, 1)),
8330 : &decl_bitsize)
8331 6 : || maybe_ne (decl_bitsize, bitsize))
8332 : /* A call with an addressable return type and return-slot
8333 : optimization must not need bitfield operations but we must
8334 : pass down the original target. */
8335 17 : && (TREE_CODE (exp) != CALL_EXPR
8336 6 : || !TREE_ADDRESSABLE (TREE_TYPE (exp))
8337 0 : || !CALL_EXPR_RETURN_SLOT_OPT (exp)))
8338 : /* If we are expanding a MEM_REF of a non-BLKmode non-addressable
8339 : decl we must use bitfield operations. */
8340 8856546 : || (known_size_p (bitsize)
8341 4045512 : && TREE_CODE (exp) == MEM_REF
8342 32751 : && TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR
8343 26740 : && DECL_P (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
8344 20371 : && !TREE_ADDRESSABLE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
8345 5646 : && DECL_MODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0)) != BLKmode))
8346 : {
8347 765767 : rtx temp;
8348 765767 : gimple *nop_def;
8349 :
8350 : /* If EXP is a NOP_EXPR of precision less than its mode, then that
8351 : implies a mask operation. If the precision is the same size as
8352 : the field we're storing into, that mask is redundant. This is
8353 : particularly common with bit field assignments generated by the
8354 : C front end. */
8355 765767 : nop_def = get_def_for_expr (exp, NOP_EXPR);
8356 765767 : if (nop_def)
8357 : {
8358 5626 : tree type = TREE_TYPE (exp);
8359 5626 : if (INTEGRAL_TYPE_P (type)
8360 5415 : && maybe_ne (TYPE_PRECISION (type),
8361 10830 : GET_MODE_BITSIZE (TYPE_MODE (type)))
8362 9019 : && known_eq (bitsize, TYPE_PRECISION (type)))
8363 : {
8364 3288 : tree op = gimple_assign_rhs1 (nop_def);
8365 3288 : type = TREE_TYPE (op);
8366 3288 : if (INTEGRAL_TYPE_P (type)
8367 3288 : && known_ge (TYPE_PRECISION (type), bitsize))
8368 : exp = op;
8369 : }
8370 : }
8371 :
8372 765767 : temp = expand_normal (exp);
8373 :
8374 : /* We don't support variable-sized BLKmode bitfields, since our
8375 : handling of BLKmode is bound up with the ability to break
8376 : things into words. */
8377 765767 : gcc_assert (mode != BLKmode || bitsize.is_constant ());
8378 :
8379 : /* Handle calls that return values in multiple non-contiguous locations.
8380 : The Irix 6 ABI has examples of this. */
8381 765767 : if (GET_CODE (temp) == PARALLEL)
8382 : {
8383 6 : HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));
8384 6 : machine_mode temp_mode = GET_MODE (temp);
8385 6 : if (temp_mode == BLKmode || temp_mode == VOIDmode)
8386 6 : temp_mode
8387 6 : = smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
8388 6 : rtx temp_target = gen_reg_rtx (temp_mode);
8389 6 : emit_group_store (temp_target, temp, TREE_TYPE (exp), size);
8390 6 : temp = temp_target;
8391 : }
8392 :
8393 : /* Handle calls that return BLKmode values in registers. */
8394 765761 : else if (mode == BLKmode && REG_P (temp) && TREE_CODE (exp) == CALL_EXPR)
8395 : {
8396 0 : rtx temp_target = gen_reg_rtx (GET_MODE (temp));
8397 0 : copy_blkmode_from_reg (temp_target, temp, TREE_TYPE (exp));
8398 0 : temp = temp_target;
8399 : }
8400 :
8401 : /* If the value has aggregate type and an integral mode then, if BITSIZE
8402 : is narrower than this mode and this is for big-endian data, we first
8403 : need to put the value into the low-order bits for store_bit_field,
8404 : except when MODE is BLKmode and BITSIZE larger than the word size
8405 : (see the handling of fields larger than a word in store_bit_field).
8406 : Moreover, the field may be not aligned on a byte boundary; in this
8407 : case, if it has reverse storage order, it needs to be accessed as a
8408 : scalar field with reverse storage order and we must first put the
8409 : value into target order. */
8410 765767 : scalar_int_mode temp_mode;
8411 1529506 : if (AGGREGATE_TYPE_P (TREE_TYPE (exp))
8412 766514 : && is_int_mode (GET_MODE (temp), &temp_mode))
8413 : {
8414 2448 : HOST_WIDE_INT size = GET_MODE_BITSIZE (temp_mode);
8415 :
8416 2448 : reverse = TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (exp));
8417 :
8418 2448 : if (reverse)
8419 0 : temp = flip_storage_order (temp_mode, temp);
8420 :
8421 2448 : gcc_checking_assert (known_le (bitsize, size));
8422 2448 : if (maybe_lt (bitsize, size)
8423 2448 : && reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN
8424 : /* Use of to_constant for BLKmode was checked above. */
8425 : && !(mode == BLKmode && bitsize.to_constant () > BITS_PER_WORD))
8426 0 : temp = expand_shift (RSHIFT_EXPR, temp_mode, temp,
8427 : size - bitsize, NULL_RTX, 1);
8428 : }
8429 :
8430 : /* Unless MODE is VOIDmode or BLKmode, convert TEMP to MODE. */
8431 698748 : if (mode != VOIDmode && mode != BLKmode
8432 1464238 : && mode != TYPE_MODE (TREE_TYPE (exp)))
8433 4 : temp = convert_modes (mode, TYPE_MODE (TREE_TYPE (exp)), temp, 1);
8434 :
8435 : /* If the mode of TEMP and TARGET is BLKmode, both must be in memory
8436 : and BITPOS must be aligned on a byte boundary. If so, we simply do
8437 : a block copy. Likewise for a BLKmode-like TARGET. */
8438 765767 : if (GET_MODE (temp) == BLKmode
8439 765767 : && (GET_MODE (target) == BLKmode
8440 182 : || (MEM_P (target)
8441 0 : && GET_MODE_CLASS (GET_MODE (target)) == MODE_INT
8442 0 : && multiple_p (bitpos, BITS_PER_UNIT)
8443 0 : && multiple_p (bitsize, BITS_PER_UNIT))))
8444 : {
8445 89 : gcc_assert (MEM_P (target) && MEM_P (temp));
8446 89 : poly_int64 bytepos = exact_div (bitpos, BITS_PER_UNIT);
8447 89 : poly_int64 bytesize = bits_to_bytes_round_up (bitsize);
8448 :
8449 89 : target = adjust_address (target, VOIDmode, bytepos);
8450 89 : emit_block_move (target, temp,
8451 89 : gen_int_mode (bytesize, Pmode),
8452 : BLOCK_OP_NORMAL);
8453 :
8454 89 : return const0_rtx;
8455 : }
8456 :
8457 : /* If the mode of TEMP is still BLKmode and BITSIZE not larger than the
8458 : word size, we need to load the value (see again store_bit_field). */
8459 765696 : if (GET_MODE (temp) == BLKmode && known_le (bitsize, BITS_PER_WORD))
8460 : {
8461 61 : temp_mode = smallest_int_mode_for_size (bitsize).require ();
8462 61 : temp = extract_bit_field (temp, bitsize, 0, 1, NULL_RTX, temp_mode,
8463 : temp_mode, false, NULL);
8464 : }
8465 :
8466 : /* Store the value in the bitfield. */
8467 765678 : gcc_checking_assert (known_ge (bitpos, 0));
8468 765678 : store_bit_field (target, bitsize, bitpos,
8469 : bitregion_start, bitregion_end,
8470 : mode, temp, reverse, false);
8471 :
8472 765678 : return const0_rtx;
8473 : }
8474 : else
8475 : {
8476 : /* Now build a reference to just the desired component. */
8477 4045255 : rtx to_rtx = adjust_address (target, mode,
8478 : exact_div (bitpos, BITS_PER_UNIT));
8479 :
8480 4045255 : if (to_rtx == target)
8481 1292279 : to_rtx = copy_rtx (to_rtx);
8482 :
8483 7815198 : if (!MEM_KEEP_ALIAS_SET_P (to_rtx) && MEM_ALIAS_SET (to_rtx) != 0)
8484 3511065 : set_mem_alias_set (to_rtx, alias_set);
8485 :
8486 : /* Above we avoided using bitfield operations for storing a CONSTRUCTOR
8487 : into a target smaller than its type; handle that case now. */
8488 4045255 : if (TREE_CODE (exp) == CONSTRUCTOR && known_size_p (bitsize))
8489 : {
8490 69726 : poly_int64 bytesize = exact_div (bitsize, BITS_PER_UNIT);
8491 69726 : store_constructor (exp, to_rtx, 0, bytesize, reverse);
8492 69726 : return to_rtx;
8493 : }
8494 :
8495 3975529 : return store_expr (exp, to_rtx, 0, nontemporal, reverse);
8496 : }
8497 : }
8498 :
8499 : /* Given an expression EXP that may be a COMPONENT_REF, a BIT_FIELD_REF,
8500 : an ARRAY_REF, or an ARRAY_RANGE_REF, look for nested operations of these
8501 : codes and find the ultimate containing object, which we return.
8502 :
8503 : We set *PBITSIZE to the size in bits that we want, *PBITPOS to the
8504 : bit position, *PUNSIGNEDP to the signedness and *PREVERSEP to the
8505 : storage order of the field.
8506 : If the position of the field is variable, we store a tree
8507 : giving the variable offset (in units) in *POFFSET.
8508 : This offset is in addition to the bit position.
8509 : If the position is not variable, we store 0 in *POFFSET.
8510 :
8511 : If any of the extraction expressions is volatile,
8512 : we store 1 in *PVOLATILEP. Otherwise we don't change that.
8513 :
8514 : If the field is a non-BLKmode bit-field, *PMODE is set to VOIDmode.
8515 : Otherwise, it is a mode that can be used to access the field.
8516 :
8517 : If the field describes a variable-sized object, *PMODE is set to
8518 : BLKmode and *PBITSIZE is set to -1. An access cannot be made in
8519 : this case, but the address of the object can be found. */
8520 :
8521 : tree
8522 288751956 : get_inner_reference (tree exp, poly_int64 *pbitsize,
8523 : poly_int64 *pbitpos, tree *poffset,
8524 : machine_mode *pmode, int *punsignedp,
8525 : int *preversep, int *pvolatilep)
8526 : {
8527 288751956 : tree size_tree = 0;
8528 288751956 : machine_mode mode = VOIDmode;
8529 288751956 : bool blkmode_bitfield = false;
8530 288751956 : tree offset = size_zero_node;
8531 288751956 : poly_offset_int bit_offset = 0;
8532 :
8533 : /* First get the mode, signedness, storage order and size. We do this from
8534 : just the outermost expression. */
8535 288751956 : *pbitsize = -1;
8536 288751956 : if (TREE_CODE (exp) == COMPONENT_REF)
8537 : {
8538 121495195 : tree field = TREE_OPERAND (exp, 1);
8539 121495195 : size_tree = DECL_SIZE (field);
8540 121495195 : if (flag_strict_volatile_bitfields > 0
8541 58 : && TREE_THIS_VOLATILE (exp)
8542 40 : && DECL_BIT_FIELD_TYPE (field)
8543 121495217 : && DECL_MODE (field) != BLKmode)
8544 : /* Volatile bitfields should be accessed in the mode of the
8545 : field's type, not the mode computed based on the bit
8546 : size. */
8547 22 : mode = TYPE_MODE (DECL_BIT_FIELD_TYPE (field));
8548 121495173 : else if (!DECL_BIT_FIELD (field))
8549 : {
8550 120208880 : mode = DECL_MODE (field);
8551 : /* For vector fields re-check the target flags, as DECL_MODE
8552 : could have been set with different target flags than
8553 : the current function has. */
8554 120208880 : if (VECTOR_TYPE_P (TREE_TYPE (field))
8555 120208880 : && VECTOR_MODE_P (TYPE_MODE_RAW (TREE_TYPE (field))))
8556 380853 : mode = TYPE_MODE (TREE_TYPE (field));
8557 : }
8558 1286293 : else if (DECL_MODE (field) == BLKmode)
8559 624 : blkmode_bitfield = true;
8560 :
8561 121495195 : *punsignedp = DECL_UNSIGNED (field);
8562 : }
8563 167256761 : else if (TREE_CODE (exp) == BIT_FIELD_REF)
8564 : {
8565 594946 : size_tree = TREE_OPERAND (exp, 1);
8566 1189473 : *punsignedp = (! INTEGRAL_TYPE_P (TREE_TYPE (exp))
8567 1043042 : || TYPE_UNSIGNED (TREE_TYPE (exp)));
8568 :
8569 : /* For vector element types with the correct size of access or for
8570 : vector typed accesses use the mode of the access type. */
8571 594946 : if ((TREE_CODE (TREE_TYPE (TREE_OPERAND (exp, 0))) == VECTOR_TYPE
8572 447115 : && TREE_TYPE (exp) == TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0)))
8573 402232 : && tree_int_cst_equal (size_tree, TYPE_SIZE (TREE_TYPE (exp))))
8574 639864 : || VECTOR_TYPE_P (TREE_TYPE (exp)))
8575 425378 : mode = TYPE_MODE (TREE_TYPE (exp));
8576 : }
8577 : else
8578 : {
8579 166661815 : mode = TYPE_MODE (TREE_TYPE (exp));
8580 166661815 : *punsignedp = TYPE_UNSIGNED (TREE_TYPE (exp));
8581 :
8582 166661815 : if (mode == BLKmode)
8583 73057798 : size_tree = TYPE_SIZE (TREE_TYPE (exp));
8584 : else
8585 187208034 : *pbitsize = GET_MODE_BITSIZE (mode);
8586 : }
8587 :
8588 288751956 : if (size_tree != 0)
8589 : {
8590 194320953 : if (!poly_int_tree_p (size_tree, pbitsize))
8591 474633 : mode = BLKmode, *pbitsize = -1;
8592 : }
8593 :
8594 288751956 : *preversep = reverse_storage_order_for_component_p (exp);
8595 :
8596 : /* Compute cumulative bit-offset for nested component-refs and array-refs,
8597 : and find the ultimate containing object. */
8598 732499506 : while (1)
8599 : {
8600 510625731 : switch (TREE_CODE (exp))
8601 : {
8602 594946 : case BIT_FIELD_REF:
8603 594946 : bit_offset += wi::to_poly_offset (TREE_OPERAND (exp, 2));
8604 594946 : break;
8605 :
8606 185476234 : case COMPONENT_REF:
8607 185476234 : {
8608 185476234 : tree field = TREE_OPERAND (exp, 1);
8609 185476234 : tree this_offset = component_ref_field_offset (exp);
8610 :
8611 : /* If this field hasn't been filled in yet, don't go past it.
8612 : This should only happen when folding expressions made during
8613 : type construction. */
8614 185476234 : if (this_offset == 0)
8615 : break;
8616 :
8617 185476234 : offset = size_binop (PLUS_EXPR, offset, this_offset);
8618 185476234 : bit_offset += wi::to_poly_offset (DECL_FIELD_BIT_OFFSET (field));
8619 :
8620 : /* ??? Right now we don't do anything with DECL_OFFSET_ALIGN. */
8621 : }
8622 185476234 : break;
8623 :
8624 32565710 : case ARRAY_REF:
8625 32565710 : case ARRAY_RANGE_REF:
8626 32565710 : {
8627 32565710 : tree index = TREE_OPERAND (exp, 1);
8628 32565710 : tree low_bound = array_ref_low_bound (exp);
8629 32565710 : tree unit_size = array_ref_element_size (exp);
8630 :
8631 : /* We assume all arrays have sizes that are a multiple of a byte.
8632 : First subtract the lower bound, if any, in the type of the
8633 : index, then convert to sizetype and multiply by the size of
8634 : the array element. */
8635 32565710 : if (! integer_zerop (low_bound))
8636 965197 : index = fold_build2 (MINUS_EXPR, TREE_TYPE (index),
8637 : index, low_bound);
8638 :
8639 32565710 : offset = size_binop (PLUS_EXPR, offset,
8640 : size_binop (MULT_EXPR,
8641 : fold_convert (sizetype, index),
8642 : unit_size));
8643 : }
8644 32565710 : break;
8645 :
8646 : case REALPART_EXPR:
8647 : break;
8648 :
8649 : case IMAGPART_EXPR:
8650 221873775 : bit_offset += *pbitsize;
8651 : break;
8652 :
8653 : case VIEW_CONVERT_EXPR:
8654 : break;
8655 :
8656 92233261 : case MEM_REF:
8657 : /* Hand back the decl for MEM[&decl, off]. */
8658 92233261 : if (TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR)
8659 : {
8660 21556191 : tree off = TREE_OPERAND (exp, 1);
8661 21556191 : if (!integer_zerop (off))
8662 : {
8663 12165171 : poly_offset_int boff = mem_ref_offset (exp);
8664 12165171 : boff <<= LOG2_BITS_PER_UNIT;
8665 12165171 : bit_offset += boff;
8666 : }
8667 21556191 : exp = TREE_OPERAND (TREE_OPERAND (exp, 0), 0);
8668 : }
8669 92233261 : goto done;
8670 :
8671 196518695 : default:
8672 196518695 : goto done;
8673 : }
8674 :
8675 : /* If any reference in the chain is volatile, the effect is volatile. */
8676 221873775 : if (TREE_THIS_VOLATILE (exp))
8677 564613 : *pvolatilep = 1;
8678 :
8679 221873775 : exp = TREE_OPERAND (exp, 0);
8680 221873775 : }
8681 288751956 : done:
8682 :
8683 : /* If OFFSET is constant, see if we can return the whole thing as a
8684 : constant bit position. Make sure to handle overflow during
8685 : this conversion. */
8686 288751956 : if (poly_int_tree_p (offset))
8687 : {
8688 275000851 : poly_offset_int tem = wi::sext (wi::to_poly_offset (offset),
8689 275000851 : TYPE_PRECISION (sizetype));
8690 275000851 : tem <<= LOG2_BITS_PER_UNIT;
8691 275000851 : tem += bit_offset;
8692 275000851 : if (tem.to_shwi (pbitpos))
8693 274999428 : *poffset = offset = NULL_TREE;
8694 : }
8695 :
8696 : /* Otherwise, split it up. */
8697 275000851 : if (offset)
8698 : {
8699 : /* Avoid returning a negative bitpos as this may wreak havoc later. */
8700 13752528 : if (!bit_offset.to_shwi (pbitpos) || maybe_lt (*pbitpos, 0))
8701 : {
8702 303 : *pbitpos = num_trailing_bits (bit_offset.force_shwi ());
8703 303 : poly_offset_int bytes = bits_to_bytes_round_down (bit_offset);
8704 303 : offset = size_binop (PLUS_EXPR, offset,
8705 : build_int_cst (sizetype, bytes.force_shwi ()));
8706 : }
8707 :
8708 13752528 : *poffset = offset;
8709 : }
8710 :
8711 : /* We can use BLKmode for a byte-aligned BLKmode bitfield. */
8712 288751956 : if (mode == VOIDmode
8713 5386425 : && blkmode_bitfield
8714 624 : && multiple_p (*pbitpos, BITS_PER_UNIT)
8715 288752444 : && multiple_p (*pbitsize, BITS_PER_UNIT))
8716 5 : *pmode = BLKmode;
8717 : else
8718 288751951 : *pmode = mode;
8719 :
8720 288751956 : return exp;
8721 : }
8722 :
8723 : /* Alignment in bits the TARGET of an assignment may be assumed to have. */
8724 :
8725 : static unsigned HOST_WIDE_INT
8726 514042 : target_align (const_tree target)
8727 : {
8728 : /* We might have a chain of nested references with intermediate misaligning
8729 : bitfields components, so need to recurse to find out. */
8730 :
8731 514042 : unsigned HOST_WIDE_INT this_align, outer_align;
8732 :
8733 514042 : switch (TREE_CODE (target))
8734 : {
8735 : case BIT_FIELD_REF:
8736 : return 1;
8737 :
8738 140086 : case COMPONENT_REF:
8739 140086 : this_align = DECL_ALIGN (TREE_OPERAND (target, 1));
8740 140086 : outer_align = target_align (TREE_OPERAND (target, 0));
8741 140086 : return MIN (this_align, outer_align);
8742 :
8743 190269 : case ARRAY_REF:
8744 190269 : case ARRAY_RANGE_REF:
8745 190269 : this_align = TYPE_ALIGN (TREE_TYPE (target));
8746 190269 : outer_align = target_align (TREE_OPERAND (target, 0));
8747 190269 : return MIN (this_align, outer_align);
8748 :
8749 4683 : CASE_CONVERT:
8750 4683 : case NON_LVALUE_EXPR:
8751 4683 : case VIEW_CONVERT_EXPR:
8752 4683 : this_align = TYPE_ALIGN (TREE_TYPE (target));
8753 4683 : outer_align = target_align (TREE_OPERAND (target, 0));
8754 4683 : return MAX (this_align, outer_align);
8755 :
8756 178999 : default:
8757 178999 : return TYPE_ALIGN (TREE_TYPE (target));
8758 : }
8759 : }
8760 :
8761 :
8762 : /* Given an rtx VALUE that may contain additions and multiplications, return
8763 : an equivalent value that just refers to a register, memory, or constant.
8764 : This is done by generating instructions to perform the arithmetic and
8765 : returning a pseudo-register containing the value.
8766 :
8767 : The returned value may be a REG, SUBREG, MEM or constant. */
8768 :
8769 : rtx
8770 31441005 : force_operand (rtx value, rtx target)
8771 : {
8772 31441005 : rtx op1, op2;
8773 : /* Use subtarget as the target for operand 0 of a binary operation. */
8774 31441005 : rtx subtarget = get_subtarget (target);
8775 31441005 : enum rtx_code code = GET_CODE (value);
8776 :
8777 : /* Check for subreg applied to an expression produced by loop optimizer. */
8778 31441005 : if (code == SUBREG
8779 305644 : && !REG_P (SUBREG_REG (value))
8780 172 : && !MEM_P (SUBREG_REG (value)))
8781 : {
8782 172 : value
8783 172 : = simplify_gen_subreg (GET_MODE (value),
8784 172 : force_reg (GET_MODE (SUBREG_REG (value)),
8785 : force_operand (SUBREG_REG (value),
8786 : NULL_RTX)),
8787 172 : GET_MODE (SUBREG_REG (value)),
8788 172 : SUBREG_BYTE (value));
8789 172 : code = GET_CODE (value);
8790 : }
8791 :
8792 : /* Check for a PIC address load. */
8793 31441005 : if ((code == PLUS || code == MINUS)
8794 3711832 : && XEXP (value, 0) == pic_offset_table_rtx
8795 1927 : && (GET_CODE (XEXP (value, 1)) == SYMBOL_REF
8796 1927 : || GET_CODE (XEXP (value, 1)) == LABEL_REF
8797 1927 : || GET_CODE (XEXP (value, 1)) == CONST))
8798 : {
8799 184 : if (!subtarget)
8800 184 : subtarget = gen_reg_rtx (GET_MODE (value));
8801 184 : emit_move_insn (subtarget, value);
8802 184 : return subtarget;
8803 : }
8804 :
8805 31440821 : if (ARITHMETIC_P (value))
8806 : {
8807 3824075 : op2 = XEXP (value, 1);
8808 3824075 : if (!CONSTANT_P (op2) && !(REG_P (op2) && op2 != subtarget))
8809 3824075 : subtarget = 0;
8810 3824075 : if (code == MINUS && CONST_INT_P (op2))
8811 : {
8812 0 : code = PLUS;
8813 0 : op2 = negate_rtx (GET_MODE (value), op2);
8814 : }
8815 :
8816 : /* Check for an addition with OP2 a constant integer and our first
8817 : operand a PLUS of a virtual register and something else. In that
8818 : case, we want to emit the sum of the virtual register and the
8819 : constant first and then add the other value. This allows virtual
8820 : register instantiation to simply modify the constant rather than
8821 : creating another one around this addition. */
8822 3651634 : if (code == PLUS && CONST_INT_P (op2)
8823 3280569 : && GET_CODE (XEXP (value, 0)) == PLUS
8824 64266 : && REG_P (XEXP (XEXP (value, 0), 0))
8825 3831339 : && VIRTUAL_REGISTER_P (XEXP (XEXP (value, 0), 0)))
8826 : {
8827 1132 : rtx temp = expand_simple_binop (GET_MODE (value), code,
8828 : XEXP (XEXP (value, 0), 0), op2,
8829 : subtarget, 0, OPTAB_LIB_WIDEN);
8830 1132 : return expand_simple_binop (GET_MODE (value), code, temp,
8831 1132 : force_operand (XEXP (XEXP (value,
8832 : 0), 1), 0),
8833 1132 : target, 0, OPTAB_LIB_WIDEN);
8834 : }
8835 :
8836 3822943 : op1 = force_operand (XEXP (value, 0), subtarget);
8837 3822943 : op2 = force_operand (op2, NULL_RTX);
8838 3822943 : switch (code)
8839 : {
8840 96686 : case MULT:
8841 96686 : return expand_mult (GET_MODE (value), op1, op2, target, 1);
8842 222 : case DIV:
8843 222 : if (!INTEGRAL_MODE_P (GET_MODE (value)))
8844 222 : return expand_simple_binop (GET_MODE (value), code, op1, op2,
8845 222 : target, 1, OPTAB_LIB_WIDEN);
8846 : else
8847 0 : return expand_divmod (0,
8848 : FLOAT_MODE_P (GET_MODE (value))
8849 : ? RDIV_EXPR : TRUNC_DIV_EXPR,
8850 0 : GET_MODE (value), op1, op2, target, 0);
8851 0 : case MOD:
8852 0 : return expand_divmod (1, TRUNC_MOD_EXPR, GET_MODE (value), op1, op2,
8853 0 : target, 0);
8854 348 : case UDIV:
8855 348 : return expand_divmod (0, TRUNC_DIV_EXPR, GET_MODE (value), op1, op2,
8856 348 : target, 1);
8857 0 : case UMOD:
8858 0 : return expand_divmod (1, TRUNC_MOD_EXPR, GET_MODE (value), op1, op2,
8859 0 : target, 1);
8860 46 : case ASHIFTRT:
8861 46 : return expand_simple_binop (GET_MODE (value), code, op1, op2,
8862 46 : target, 0, OPTAB_LIB_WIDEN);
8863 3725641 : default:
8864 3725641 : return expand_simple_binop (GET_MODE (value), code, op1, op2,
8865 3725641 : target, 1, OPTAB_LIB_WIDEN);
8866 : }
8867 : }
8868 27616746 : if (UNARY_P (value))
8869 : {
8870 21740 : if (!target)
8871 11523 : target = gen_reg_rtx (GET_MODE (value));
8872 : /* FIX or UNSIGNED_FIX with integral mode has unspecified rounding,
8873 : while FIX with floating point mode rounds toward zero. So, some
8874 : targets use expressions like (fix:SI (fix:DF (reg:DF ...)))
8875 : to express rounding toward zero during the conversion to int.
8876 : expand_fix isn't able to handle that, it can only handle
8877 : FIX/UNSIGNED_FIX from floating point mode to integral one. */
8878 21740 : if ((code == FIX || code == UNSIGNED_FIX)
8879 8 : && GET_CODE (XEXP (value, 0)) == FIX
8880 0 : && (GET_MODE (XEXP (value, 0))
8881 0 : == GET_MODE (XEXP (XEXP (value, 0), 0))))
8882 0 : op1 = force_operand (XEXP (XEXP (value, 0), 0), NULL_RTX);
8883 : else
8884 21740 : op1 = force_operand (XEXP (value, 0), NULL_RTX);
8885 21740 : switch (code)
8886 : {
8887 10135 : case ZERO_EXTEND:
8888 10135 : case SIGN_EXTEND:
8889 10135 : case TRUNCATE:
8890 10135 : case FLOAT_EXTEND:
8891 10135 : case FLOAT_TRUNCATE:
8892 10135 : convert_move (target, op1, code == ZERO_EXTEND);
8893 10135 : return target;
8894 :
8895 8 : case FIX:
8896 8 : case UNSIGNED_FIX:
8897 8 : expand_fix (target, op1, code == UNSIGNED_FIX);
8898 8 : return target;
8899 :
8900 120 : case FLOAT:
8901 120 : case UNSIGNED_FLOAT:
8902 120 : expand_float (target, op1, code == UNSIGNED_FLOAT);
8903 120 : return target;
8904 :
8905 11477 : default:
8906 11477 : return expand_simple_unop (GET_MODE (value), code, op1, target, 0);
8907 : }
8908 : }
8909 :
8910 : #ifdef INSN_SCHEDULING
8911 : /* On machines that have insn scheduling, we want all memory reference to be
8912 : explicit, so we need to deal with such paradoxical SUBREGs. */
8913 27595006 : if (paradoxical_subreg_p (value) && MEM_P (SUBREG_REG (value)))
8914 0 : value
8915 0 : = simplify_gen_subreg (GET_MODE (value),
8916 0 : force_reg (GET_MODE (SUBREG_REG (value)),
8917 : force_operand (SUBREG_REG (value),
8918 : NULL_RTX)),
8919 : GET_MODE (SUBREG_REG (value)),
8920 0 : SUBREG_BYTE (value));
8921 : #endif
8922 :
8923 : return value;
8924 : }
8925 :
8926 : /* Subroutine of expand_expr: return true iff there is no way that
8927 : EXP can reference X, which is being modified. TOP_P is nonzero if this
8928 : call is going to be used to determine whether we need a temporary
8929 : for EXP, as opposed to a recursive call to this function.
8930 :
8931 : It is always safe for this routine to return false since it merely
8932 : searches for optimization opportunities. */
8933 :
8934 : bool
8935 8861660 : safe_from_p (const_rtx x, tree exp, int top_p)
8936 : {
8937 8861676 : rtx exp_rtl = 0;
8938 8861676 : int i, nops;
8939 :
8940 8861676 : if (x == 0
8941 : /* If EXP has varying size, we MUST use a target since we currently
8942 : have no way of allocating temporaries of variable size
8943 : (except for arrays that have TYPE_ARRAY_MAX_SIZE set).
8944 : So we assume here that something at a higher level has prevented a
8945 : clash. This is somewhat bogus, but the best we can do. Only
8946 : do this when X is BLKmode and when we are at the top level. */
8947 2015187 : || (top_p && TREE_TYPE (exp) != 0 && COMPLETE_TYPE_P (TREE_TYPE (exp))
8948 1881023 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (exp))) != INTEGER_CST
8949 0 : && (TREE_CODE (TREE_TYPE (exp)) != ARRAY_TYPE
8950 0 : || TYPE_ARRAY_MAX_SIZE (TREE_TYPE (exp)) == NULL_TREE
8951 0 : || TREE_CODE (TYPE_ARRAY_MAX_SIZE (TREE_TYPE (exp)))
8952 : != INTEGER_CST)
8953 0 : && GET_MODE (x) == BLKmode)
8954 : /* If X is in the outgoing argument area, it is always safe. */
8955 10876863 : || (MEM_P (x)
8956 179758 : && (XEXP (x, 0) == virtual_outgoing_args_rtx
8957 179758 : || (GET_CODE (XEXP (x, 0)) == PLUS
8958 131783 : && XEXP (XEXP (x, 0), 0) == virtual_outgoing_args_rtx))))
8959 : return true;
8960 :
8961 : /* If this is a subreg of a hard register, declare it unsafe, otherwise,
8962 : find the underlying pseudo. */
8963 2015187 : if (GET_CODE (x) == SUBREG)
8964 : {
8965 0 : x = SUBREG_REG (x);
8966 0 : if (REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER)
8967 : return false;
8968 : }
8969 :
8970 : /* Now look at our tree code and possibly recurse. */
8971 2015187 : switch (TREE_CODE_CLASS (TREE_CODE (exp)))
8972 : {
8973 600 : case tcc_declaration:
8974 600 : exp_rtl = DECL_RTL_IF_SET (exp);
8975 : break;
8976 :
8977 : case tcc_constant:
8978 : return true;
8979 :
8980 875291 : case tcc_exceptional:
8981 875291 : if (TREE_CODE (exp) == TREE_LIST)
8982 : {
8983 0 : while (1)
8984 : {
8985 0 : if (TREE_VALUE (exp) && !safe_from_p (x, TREE_VALUE (exp), 0))
8986 : return false;
8987 0 : exp = TREE_CHAIN (exp);
8988 0 : if (!exp)
8989 : return true;
8990 0 : if (TREE_CODE (exp) != TREE_LIST)
8991 : return safe_from_p (x, exp, 0);
8992 : }
8993 : }
8994 875291 : else if (TREE_CODE (exp) == CONSTRUCTOR)
8995 : {
8996 : constructor_elt *ce;
8997 : unsigned HOST_WIDE_INT idx;
8998 :
8999 8154056 : FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (exp), idx, ce)
9000 4932 : if ((ce->index != NULL_TREE && !safe_from_p (x, ce->index, 0))
9001 133739 : || !safe_from_p (x, ce->value, 0))
9002 106125 : return false;
9003 : return true;
9004 : }
9005 731530 : else if (TREE_CODE (exp) == ERROR_MARK)
9006 : return true; /* An already-visited SAVE_EXPR? */
9007 : else
9008 : return false;
9009 :
9010 0 : case tcc_statement:
9011 : /* The only case we look at here is the DECL_INITIAL inside a
9012 : DECL_EXPR. */
9013 0 : return (TREE_CODE (exp) != DECL_EXPR
9014 0 : || TREE_CODE (DECL_EXPR_DECL (exp)) != VAR_DECL
9015 0 : || !DECL_INITIAL (DECL_EXPR_DECL (exp))
9016 0 : || safe_from_p (x, DECL_INITIAL (DECL_EXPR_DECL (exp)), 0));
9017 :
9018 0 : case tcc_binary:
9019 0 : case tcc_comparison:
9020 0 : if (!safe_from_p (x, TREE_OPERAND (exp, 1), 0))
9021 : return false;
9022 : /* Fall through. */
9023 :
9024 16 : case tcc_unary:
9025 16 : return safe_from_p (x, TREE_OPERAND (exp, 0), 0);
9026 :
9027 471 : case tcc_expression:
9028 471 : case tcc_reference:
9029 471 : case tcc_vl_exp:
9030 : /* Now do code-specific tests. EXP_RTL is set to any rtx we find in
9031 : the expression. If it is set, we conflict iff we are that rtx or
9032 : both are in memory. Otherwise, we check all operands of the
9033 : expression recursively. */
9034 :
9035 471 : switch (TREE_CODE (exp))
9036 : {
9037 383 : case ADDR_EXPR:
9038 : /* If the operand is static or we are static, we can't conflict.
9039 : Likewise if we don't conflict with the operand at all. */
9040 383 : if (staticp (TREE_OPERAND (exp, 0))
9041 233 : || TREE_STATIC (exp)
9042 616 : || safe_from_p (x, TREE_OPERAND (exp, 0), 0))
9043 383 : return true;
9044 :
9045 : /* Otherwise, the only way this can conflict is if we are taking
9046 : the address of a DECL a that address if part of X, which is
9047 : very rare. */
9048 0 : exp = TREE_OPERAND (exp, 0);
9049 0 : if (DECL_P (exp))
9050 : {
9051 0 : if (!DECL_RTL_SET_P (exp)
9052 0 : || !MEM_P (DECL_RTL (exp)))
9053 0 : return false;
9054 : else
9055 0 : exp_rtl = XEXP (DECL_RTL (exp), 0);
9056 : }
9057 : break;
9058 :
9059 73 : case MEM_REF:
9060 73 : if (MEM_P (x)
9061 73 : && alias_sets_conflict_p (MEM_ALIAS_SET (x),
9062 : get_alias_set (exp)))
9063 : return false;
9064 : break;
9065 :
9066 0 : case CALL_EXPR:
9067 : /* Assume that the call will clobber all hard registers and
9068 : all of memory. */
9069 0 : if ((REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER)
9070 0 : || MEM_P (x))
9071 : return false;
9072 : break;
9073 :
9074 0 : case WITH_CLEANUP_EXPR:
9075 0 : case CLEANUP_POINT_EXPR:
9076 : /* Lowered by gimplify.cc. */
9077 0 : gcc_unreachable ();
9078 :
9079 0 : case SAVE_EXPR:
9080 0 : return safe_from_p (x, TREE_OPERAND (exp, 0), 0);
9081 :
9082 : default:
9083 : break;
9084 : }
9085 :
9086 : /* If we have an rtx, we do not need to scan our operands. */
9087 0 : if (exp_rtl)
9088 : break;
9089 :
9090 88 : nops = TREE_OPERAND_LENGTH (exp);
9091 371 : for (i = 0; i < nops; i++)
9092 195 : if (TREE_OPERAND (exp, i) != 0
9093 195 : && ! safe_from_p (x, TREE_OPERAND (exp, i), 0))
9094 : return false;
9095 :
9096 : break;
9097 :
9098 0 : case tcc_type:
9099 : /* Should never get a type here. */
9100 0 : gcc_unreachable ();
9101 : }
9102 :
9103 : /* If we have an rtl, find any enclosed object. Then see if we conflict
9104 : with it. */
9105 392 : if (exp_rtl)
9106 : {
9107 304 : if (GET_CODE (exp_rtl) == SUBREG)
9108 : {
9109 0 : exp_rtl = SUBREG_REG (exp_rtl);
9110 0 : if (REG_P (exp_rtl)
9111 0 : && REGNO (exp_rtl) < FIRST_PSEUDO_REGISTER)
9112 : return false;
9113 : }
9114 :
9115 : /* If the rtl is X, then it is not safe. Otherwise, it is unless both
9116 : are memory and they conflict. */
9117 304 : return ! (rtx_equal_p (x, exp_rtl)
9118 304 : || (MEM_P (x) && MEM_P (exp_rtl)
9119 4 : && true_dependence (exp_rtl, VOIDmode, x)));
9120 : }
9121 :
9122 : /* If we reach here, it is safe. */
9123 : return true;
9124 : }
9125 :
9126 :
9127 : /* Return the highest power of two that EXP is known to be a multiple of.
9128 : This is used in updating alignment of MEMs in array references. */
9129 :
9130 : unsigned HOST_WIDE_INT
9131 33208475 : highest_pow2_factor (const_tree exp)
9132 : {
9133 33208475 : unsigned HOST_WIDE_INT ret;
9134 33208475 : int trailing_zeros = tree_ctz (exp);
9135 33208475 : if (trailing_zeros >= HOST_BITS_PER_WIDE_INT)
9136 45835721 : return BIGGEST_ALIGNMENT;
9137 9947425 : ret = HOST_WIDE_INT_1U << trailing_zeros;
9138 19429159 : if (ret > BIGGEST_ALIGNMENT)
9139 13115454 : return BIGGEST_ALIGNMENT;
9140 : return ret;
9141 : }
9142 :
9143 : /* Similar, except that the alignment requirements of TARGET are
9144 : taken into account. Assume it is at least as aligned as its
9145 : type, unless it is a COMPONENT_REF in which case the layout of
9146 : the structure gives the alignment. */
9147 :
9148 : static unsigned HOST_WIDE_INT
9149 179004 : highest_pow2_factor_for_target (const_tree target, const_tree exp)
9150 : {
9151 179004 : unsigned HOST_WIDE_INT talign = target_align (target) / BITS_PER_UNIT;
9152 179004 : unsigned HOST_WIDE_INT factor = highest_pow2_factor (exp);
9153 :
9154 179004 : return MAX (factor, talign);
9155 : }
9156 :
9157 : /* Convert the tree comparison code TCODE to the rtl one where the
9158 : signedness is UNSIGNEDP. */
9159 :
9160 : enum rtx_code
9161 21141 : convert_tree_comp_to_rtx (enum tree_code tcode, int unsignedp)
9162 : {
9163 21141 : enum rtx_code code;
9164 21141 : switch (tcode)
9165 : {
9166 : case EQ_EXPR:
9167 : code = EQ;
9168 : break;
9169 2395 : case NE_EXPR:
9170 2395 : code = NE;
9171 2395 : break;
9172 4086 : case LT_EXPR:
9173 4086 : code = unsignedp ? LTU : LT;
9174 : break;
9175 2716 : case LE_EXPR:
9176 2716 : code = unsignedp ? LEU : LE;
9177 : break;
9178 3185 : case GT_EXPR:
9179 3185 : code = unsignedp ? GTU : GT;
9180 : break;
9181 4371 : case GE_EXPR:
9182 4371 : code = unsignedp ? GEU : GE;
9183 : break;
9184 31 : case UNORDERED_EXPR:
9185 31 : code = UNORDERED;
9186 31 : break;
9187 31 : case ORDERED_EXPR:
9188 31 : code = ORDERED;
9189 31 : break;
9190 10 : case UNLT_EXPR:
9191 10 : code = UNLT;
9192 10 : break;
9193 19 : case UNLE_EXPR:
9194 19 : code = UNLE;
9195 19 : break;
9196 10 : case UNGT_EXPR:
9197 10 : code = UNGT;
9198 10 : break;
9199 11 : case UNGE_EXPR:
9200 11 : code = UNGE;
9201 11 : break;
9202 4 : case UNEQ_EXPR:
9203 4 : code = UNEQ;
9204 4 : break;
9205 4 : case LTGT_EXPR:
9206 4 : code = LTGT;
9207 4 : break;
9208 :
9209 0 : default:
9210 0 : gcc_unreachable ();
9211 : }
9212 21141 : return code;
9213 : }
9214 :
9215 : /* Subroutine of expand_expr. Expand the two operands of a binary
9216 : expression EXP0 and EXP1 placing the results in OP0 and OP1.
9217 : The value may be stored in TARGET if TARGET is nonzero. The
9218 : MODIFIER argument is as documented by expand_expr. */
9219 :
9220 : void
9221 8261224 : expand_operands (tree exp0, tree exp1, rtx target, rtx *op0, rtx *op1,
9222 : enum expand_modifier modifier)
9223 : {
9224 8261224 : if (! safe_from_p (target, exp1, 1))
9225 608162 : target = 0;
9226 8261224 : if (operand_equal_p (exp0, exp1, 0))
9227 : {
9228 47705 : *op0 = expand_expr (exp0, target, VOIDmode, modifier);
9229 47705 : *op1 = copy_rtx (*op0);
9230 : }
9231 : else
9232 : {
9233 8213519 : *op0 = expand_expr (exp0, target, VOIDmode, modifier);
9234 8213519 : *op1 = expand_expr (exp1, NULL_RTX, VOIDmode, modifier);
9235 : }
9236 8261224 : }
9237 :
9238 :
9239 : /* Return a MEM that contains constant EXP. DEFER is as for
9240 : output_constant_def and MODIFIER is as for expand_expr. */
9241 :
9242 : static rtx
9243 2977933 : expand_expr_constant (tree exp, int defer, enum expand_modifier modifier)
9244 : {
9245 2977933 : rtx mem;
9246 :
9247 2977933 : mem = output_constant_def (exp, defer);
9248 2977933 : if (modifier != EXPAND_INITIALIZER)
9249 1902756 : mem = use_anchored_address (mem);
9250 2977933 : return mem;
9251 : }
9252 :
9253 : /* A subroutine of expand_expr_addr_expr. Evaluate the address of EXP.
9254 : The TARGET, TMODE and MODIFIER arguments are as for expand_expr. */
9255 :
9256 : static rtx
9257 14521089 : expand_expr_addr_expr_1 (tree exp, rtx target, scalar_int_mode tmode,
9258 : enum expand_modifier modifier, addr_space_t as)
9259 : {
9260 14521089 : rtx result, subtarget;
9261 14521089 : tree inner, offset;
9262 14521089 : poly_int64 bitsize, bitpos;
9263 14521089 : int unsignedp, reversep, volatilep = 0;
9264 14521089 : machine_mode mode1;
9265 :
9266 : /* If we are taking the address of a constant and are at the top level,
9267 : we have to use output_constant_def since we can't call force_const_mem
9268 : at top level. */
9269 : /* ??? This should be considered a front-end bug. We should not be
9270 : generating ADDR_EXPR of something that isn't an LVALUE. The only
9271 : exception here is STRING_CST. */
9272 14521089 : if (CONSTANT_CLASS_P (exp))
9273 : {
9274 2717552 : result = XEXP (expand_expr_constant (exp, 0, modifier), 0);
9275 2717552 : if (modifier < EXPAND_SUM)
9276 1812196 : result = force_operand (result, target);
9277 2717552 : return result;
9278 : }
9279 :
9280 : /* Everything must be something allowed by is_gimple_addressable. */
9281 11803537 : switch (TREE_CODE (exp))
9282 : {
9283 46 : case INDIRECT_REF:
9284 : /* This case will happen via recursion for &a->b. */
9285 46 : return expand_expr (TREE_OPERAND (exp, 0), target, tmode, modifier);
9286 :
9287 571865 : case MEM_REF:
9288 571865 : {
9289 571865 : tree tem = TREE_OPERAND (exp, 0);
9290 571865 : if (!integer_zerop (TREE_OPERAND (exp, 1)))
9291 299193 : tem = fold_build_pointer_plus (tem, TREE_OPERAND (exp, 1));
9292 571865 : return expand_expr (tem, target, tmode, modifier);
9293 : }
9294 :
9295 1128 : case TARGET_MEM_REF:
9296 1128 : return addr_for_mem_ref (exp, as, true);
9297 :
9298 59662 : case CONST_DECL:
9299 : /* Expand the initializer like constants above. */
9300 59662 : result = XEXP (expand_expr_constant (DECL_INITIAL (exp),
9301 : 0, modifier), 0);
9302 59662 : if (modifier < EXPAND_SUM)
9303 59656 : result = force_operand (result, target);
9304 : return result;
9305 :
9306 199 : case REALPART_EXPR:
9307 : /* The real part of the complex number is always first, therefore
9308 : the address is the same as the address of the parent object. */
9309 199 : offset = 0;
9310 199 : bitpos = 0;
9311 199 : inner = TREE_OPERAND (exp, 0);
9312 199 : break;
9313 :
9314 116 : case IMAGPART_EXPR:
9315 : /* The imaginary part of the complex number is always second.
9316 : The expression is therefore always offset by the size of the
9317 : scalar type. */
9318 116 : offset = 0;
9319 232 : bitpos = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (exp)));
9320 116 : inner = TREE_OPERAND (exp, 0);
9321 116 : break;
9322 :
9323 13 : case COMPOUND_LITERAL_EXPR:
9324 : /* Allow COMPOUND_LITERAL_EXPR in initializers or coming from
9325 : initializers, if e.g. rtl_for_decl_init is called on DECL_INITIAL
9326 : with COMPOUND_LITERAL_EXPRs in it, or ARRAY_REF on a const static
9327 : array with address of COMPOUND_LITERAL_EXPR in DECL_INITIAL;
9328 : the initializers aren't gimplified. */
9329 13 : if (COMPOUND_LITERAL_EXPR_DECL (exp)
9330 13 : && is_global_var (COMPOUND_LITERAL_EXPR_DECL (exp)))
9331 13 : return expand_expr_addr_expr_1 (COMPOUND_LITERAL_EXPR_DECL (exp),
9332 13 : target, tmode, modifier, as);
9333 : /* FALLTHRU */
9334 11170508 : default:
9335 : /* If the object is a DECL, then expand it for its rtl. Don't bypass
9336 : expand_expr, as that can have various side effects; LABEL_DECLs for
9337 : example, may not have their DECL_RTL set yet. Expand the rtl of
9338 : CONSTRUCTORs too, which should yield a memory reference for the
9339 : constructor's contents. Assume language specific tree nodes can
9340 : be expanded in some interesting way. */
9341 11170508 : gcc_assert (TREE_CODE (exp) < LAST_AND_UNUSED_TREE_CODE);
9342 11170508 : if (DECL_P (exp)
9343 1329488 : || TREE_CODE (exp) == CONSTRUCTOR
9344 1329488 : || TREE_CODE (exp) == COMPOUND_LITERAL_EXPR)
9345 : {
9346 16170239 : result = expand_expr (exp, target, tmode,
9347 : modifier == EXPAND_INITIALIZER
9348 : ? EXPAND_INITIALIZER : EXPAND_CONST_ADDRESS);
9349 :
9350 : /* If the DECL isn't in memory, then the DECL wasn't properly
9351 : marked TREE_ADDRESSABLE, which will be either a front-end
9352 : or a tree optimizer bug. */
9353 :
9354 9841020 : gcc_assert (MEM_P (result));
9355 9841018 : result = XEXP (result, 0);
9356 :
9357 : /* ??? Is this needed anymore? */
9358 9841018 : if (DECL_P (exp))
9359 9841018 : TREE_USED (exp) = 1;
9360 :
9361 9841018 : if (modifier != EXPAND_INITIALIZER
9362 : && modifier != EXPAND_CONST_ADDRESS
9363 9841018 : && modifier != EXPAND_SUM)
9364 4290918 : result = force_operand (result, target);
9365 9841018 : return result;
9366 : }
9367 :
9368 : /* Pass FALSE as the last argument to get_inner_reference although
9369 : we are expanding to RTL. The rationale is that we know how to
9370 : handle "aligning nodes" here: we can just bypass them because
9371 : they won't change the final object whose address will be returned
9372 : (they actually exist only for that purpose). */
9373 1329488 : inner = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
9374 : &unsignedp, &reversep, &volatilep);
9375 1329488 : break;
9376 : }
9377 :
9378 : /* We must have made progress. */
9379 1329803 : gcc_assert (inner != exp);
9380 :
9381 1329803 : subtarget = offset || maybe_ne (bitpos, 0) ? NULL_RTX : target;
9382 : /* For VIEW_CONVERT_EXPR, where the outer alignment is bigger than
9383 : inner alignment, force the inner to be sufficiently aligned. */
9384 1329803 : if (CONSTANT_CLASS_P (inner)
9385 1329803 : && TYPE_ALIGN (TREE_TYPE (inner)) < TYPE_ALIGN (TREE_TYPE (exp)))
9386 : {
9387 0 : inner = copy_node (inner);
9388 0 : TREE_TYPE (inner) = copy_node (TREE_TYPE (inner));
9389 0 : SET_TYPE_ALIGN (TREE_TYPE (inner), TYPE_ALIGN (TREE_TYPE (exp)));
9390 0 : TYPE_USER_ALIGN (TREE_TYPE (inner)) = 1;
9391 : }
9392 1329803 : result = expand_expr_addr_expr_1 (inner, subtarget, tmode, modifier, as);
9393 :
9394 1329803 : if (offset)
9395 : {
9396 55370 : rtx tmp;
9397 :
9398 55370 : if (modifier != EXPAND_NORMAL)
9399 1625 : result = force_operand (result, NULL);
9400 56995 : tmp = expand_expr (offset, NULL_RTX, tmode,
9401 : modifier == EXPAND_INITIALIZER
9402 : ? EXPAND_INITIALIZER : EXPAND_NORMAL);
9403 :
9404 : /* expand_expr is allowed to return an object in a mode other
9405 : than TMODE. If it did, we need to convert. */
9406 55370 : if (GET_MODE (tmp) != VOIDmode && tmode != GET_MODE (tmp))
9407 0 : tmp = convert_modes (tmode, GET_MODE (tmp),
9408 0 : tmp, TYPE_UNSIGNED (TREE_TYPE (offset)));
9409 55370 : result = convert_memory_address_addr_space (tmode, result, as);
9410 55370 : tmp = convert_memory_address_addr_space (tmode, tmp, as);
9411 :
9412 55370 : if (modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER)
9413 1625 : result = simplify_gen_binary (PLUS, tmode, result, tmp);
9414 : else
9415 : {
9416 53745 : subtarget = maybe_ne (bitpos, 0) ? NULL_RTX : target;
9417 53745 : result = expand_simple_binop (tmode, PLUS, result, tmp, subtarget,
9418 : 1, OPTAB_LIB_WIDEN);
9419 : }
9420 : }
9421 :
9422 1329803 : if (maybe_ne (bitpos, 0))
9423 : {
9424 : /* Someone beforehand should have rejected taking the address
9425 : of an object that isn't byte-aligned. */
9426 629335 : poly_int64 bytepos = exact_div (bitpos, BITS_PER_UNIT);
9427 629335 : result = convert_memory_address_addr_space (tmode, result, as);
9428 629335 : result = plus_constant (tmode, result, bytepos);
9429 629335 : if (modifier < EXPAND_SUM)
9430 591807 : result = force_operand (result, target);
9431 : }
9432 :
9433 : return result;
9434 : }
9435 :
9436 : /* A subroutine of expand_expr. Evaluate EXP, which is an ADDR_EXPR.
9437 : The TARGET, TMODE and MODIFIER arguments are as for expand_expr. */
9438 :
9439 : static rtx
9440 13191271 : expand_expr_addr_expr (tree exp, rtx target, machine_mode tmode,
9441 : enum expand_modifier modifier)
9442 : {
9443 13191271 : addr_space_t as = ADDR_SPACE_GENERIC;
9444 13191271 : scalar_int_mode address_mode = Pmode;
9445 13191271 : scalar_int_mode pointer_mode = ptr_mode;
9446 13191271 : machine_mode rmode;
9447 13191271 : rtx result;
9448 :
9449 : /* Target mode of VOIDmode says "whatever's natural". */
9450 13191271 : if (tmode == VOIDmode)
9451 11230199 : tmode = TYPE_MODE (TREE_TYPE (exp));
9452 :
9453 13191271 : if (POINTER_TYPE_P (TREE_TYPE (exp)))
9454 : {
9455 13191271 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)));
9456 13191271 : address_mode = targetm.addr_space.address_mode (as);
9457 13191271 : pointer_mode = targetm.addr_space.pointer_mode (as);
9458 : }
9459 :
9460 : /* We can get called with some Weird Things if the user does silliness
9461 : like "(short) &a". In that case, convert_memory_address won't do
9462 : the right thing, so ignore the given target mode. */
9463 13191271 : scalar_int_mode new_tmode = (tmode == pointer_mode
9464 13191271 : ? pointer_mode
9465 13191271 : : address_mode);
9466 :
9467 13191271 : result = expand_expr_addr_expr_1 (TREE_OPERAND (exp, 0), target,
9468 : new_tmode, modifier, as);
9469 :
9470 : /* Despite expand_expr claims concerning ignoring TMODE when not
9471 : strictly convenient, stuff breaks if we don't honor it. Note
9472 : that combined with the above, we only do this for pointer modes. */
9473 13191269 : rmode = GET_MODE (result);
9474 13191269 : if (rmode == VOIDmode)
9475 6 : rmode = new_tmode;
9476 13191269 : if (rmode != new_tmode)
9477 74 : result = convert_memory_address_addr_space (new_tmode, result, as);
9478 :
9479 13191269 : return result;
9480 : }
9481 :
9482 : /* Generate code for computing CONSTRUCTOR EXP.
9483 : An rtx for the computed value is returned. If AVOID_TEMP_MEM
9484 : is TRUE, instead of creating a temporary variable in memory
9485 : NULL is returned and the caller needs to handle it differently. */
9486 :
9487 : static rtx
9488 177381 : expand_constructor (tree exp, rtx target, enum expand_modifier modifier,
9489 : bool avoid_temp_mem)
9490 : {
9491 177381 : tree type = TREE_TYPE (exp);
9492 177381 : machine_mode mode = TYPE_MODE (type);
9493 :
9494 : /* Try to avoid creating a temporary at all. This is possible
9495 : if all of the initializer is zero.
9496 : FIXME: try to handle all [0..255] initializers we can handle
9497 : with memset. */
9498 177381 : if (TREE_STATIC (exp)
9499 2176 : && !TREE_ADDRESSABLE (exp)
9500 2176 : && target != 0 && mode == BLKmode
9501 178466 : && all_zeros_p (exp))
9502 : {
9503 1076 : clear_storage (target, expr_size (exp), BLOCK_OP_NORMAL);
9504 1076 : return target;
9505 : }
9506 :
9507 : /* All elts simple constants => refer to a constant in memory. But
9508 : if this is a non-BLKmode mode, let it store a field at a time
9509 : since that should make a CONST_INT, CONST_WIDE_INT or
9510 : CONST_DOUBLE when we fold. Likewise, if we have a target we can
9511 : use, it is best to store directly into the target unless the type
9512 : is large enough that memcpy will be used. If we are making an
9513 : initializer and all operands are constant, put it in memory as
9514 : well.
9515 :
9516 : FIXME: Avoid trying to fill vector constructors piece-meal.
9517 : Output them with output_constant_def below unless we're sure
9518 : they're zeros. This should go away when vector initializers
9519 : are treated like VECTOR_CST instead of arrays. */
9520 176305 : if ((TREE_STATIC (exp)
9521 1100 : && ((mode == BLKmode
9522 46 : && ! (target != 0 && safe_from_p (target, exp, 1)))
9523 1063 : || TREE_ADDRESSABLE (exp)
9524 1063 : || (tree_fits_uhwi_p (TYPE_SIZE_UNIT (type))
9525 1063 : && (! can_move_by_pieces
9526 1063 : (tree_to_uhwi (TYPE_SIZE_UNIT (type)),
9527 1063 : TYPE_ALIGN (type)))
9528 4 : && ! mostly_zeros_p (exp))))
9529 177366 : || ((modifier == EXPAND_INITIALIZER || modifier == EXPAND_CONST_ADDRESS)
9530 0 : && TREE_CONSTANT (exp)))
9531 : {
9532 39 : rtx constructor;
9533 :
9534 39 : if (avoid_temp_mem)
9535 : return NULL_RTX;
9536 :
9537 37 : constructor = expand_expr_constant (exp, 1, modifier);
9538 :
9539 37 : if (modifier != EXPAND_CONST_ADDRESS
9540 : && modifier != EXPAND_INITIALIZER
9541 37 : && modifier != EXPAND_SUM)
9542 37 : constructor = validize_mem (constructor);
9543 :
9544 37 : return constructor;
9545 : }
9546 :
9547 : /* If the CTOR is available in static storage and not mostly
9548 : zeros and we can move it by pieces prefer to do so since
9549 : that's usually more efficient than performing a series of
9550 : stores from immediates. */
9551 176266 : if (avoid_temp_mem
9552 97 : && TREE_STATIC (exp)
9553 38 : && TREE_CONSTANT (exp)
9554 38 : && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type))
9555 38 : && can_move_by_pieces (tree_to_uhwi (TYPE_SIZE_UNIT (type)),
9556 38 : TYPE_ALIGN (type))
9557 176303 : && ! mostly_zeros_p (exp))
9558 : return NULL_RTX;
9559 :
9560 : /* Handle calls that pass values in multiple non-contiguous
9561 : locations. The Irix 6 ABI has examples of this. */
9562 143708 : if (target == 0 || ! safe_from_p (target, exp, 1)
9563 37583 : || GET_CODE (target) == PARALLEL || modifier == EXPAND_STACK_PARM
9564 : /* Also make a temporary if the store is to volatile memory, to
9565 : avoid individual accesses to aggregate members. */
9566 213807 : || (GET_CODE (target) == MEM
9567 33481 : && MEM_VOLATILE_P (target)
9568 134 : && !TREE_ADDRESSABLE (TREE_TYPE (exp))))
9569 : {
9570 138793 : if (avoid_temp_mem)
9571 : return NULL_RTX;
9572 :
9573 138782 : target = assign_temp (type, TREE_ADDRESSABLE (exp), 1);
9574 : }
9575 :
9576 176225 : store_constructor (exp, target, 0, int_expr_size (exp), false);
9577 176225 : return target;
9578 : }
9579 :
9580 :
9581 : /* expand_expr: generate code for computing expression EXP.
9582 : An rtx for the computed value is returned. The value is never null.
9583 : In the case of a void EXP, const0_rtx is returned.
9584 :
9585 : The value may be stored in TARGET if TARGET is nonzero.
9586 : TARGET is just a suggestion; callers must assume that
9587 : the rtx returned may not be the same as TARGET.
9588 :
9589 : If TARGET is CONST0_RTX, it means that the value will be ignored.
9590 :
9591 : If TMODE is not VOIDmode, it suggests generating the
9592 : result in mode TMODE. But this is done only when convenient.
9593 : Otherwise, TMODE is ignored and the value generated in its natural mode.
9594 : TMODE is just a suggestion; callers must assume that
9595 : the rtx returned may not have mode TMODE.
9596 :
9597 : Note that TARGET may have neither TMODE nor MODE. In that case, it
9598 : probably will not be used.
9599 :
9600 : If MODIFIER is EXPAND_SUM then when EXP is an addition
9601 : we can return an rtx of the form (MULT (REG ...) (CONST_INT ...))
9602 : or a nest of (PLUS ...) and (MINUS ...) where the terms are
9603 : products as above, or REG or MEM, or constant.
9604 : Ordinarily in such cases we would output mul or add instructions
9605 : and then return a pseudo reg containing the sum.
9606 :
9607 : EXPAND_INITIALIZER is much like EXPAND_SUM except that
9608 : it also marks a label as absolutely required (it can't be dead).
9609 : It also makes a ZERO_EXTEND or SIGN_EXTEND instead of emitting extend insns.
9610 : This is used for outputting expressions used in initializers.
9611 :
9612 : EXPAND_CONST_ADDRESS says that it is okay to return a MEM
9613 : with a constant address even if that address is not normally legitimate.
9614 : EXPAND_INITIALIZER and EXPAND_SUM also have this effect.
9615 :
9616 : EXPAND_STACK_PARM is used when expanding to a TARGET on the stack for
9617 : a call parameter. Such targets require special care as we haven't yet
9618 : marked TARGET so that it's safe from being trashed by libcalls. We
9619 : don't want to use TARGET for anything but the final result;
9620 : Intermediate values must go elsewhere. Additionally, calls to
9621 : emit_block_move will be flagged with BLOCK_OP_CALL_PARM.
9622 :
9623 : If EXP is a VAR_DECL whose DECL_RTL was a MEM with an invalid
9624 : address, and ALT_RTL is non-NULL, then *ALT_RTL is set to the
9625 : DECL_RTL of the VAR_DECL. *ALT_RTL is also set if EXP is a
9626 : COMPOUND_EXPR whose second argument is such a VAR_DECL, and so on
9627 : recursively.
9628 : If the result can be stored at TARGET, and ALT_RTL is non-NULL,
9629 : then *ALT_RTL is set to TARGET (before legitimziation).
9630 :
9631 : If INNER_REFERENCE_P is true, we are expanding an inner reference.
9632 : In this case, we don't adjust a returned MEM rtx that wouldn't be
9633 : sufficiently aligned for its mode; instead, it's up to the caller
9634 : to deal with it afterwards. This is used to make sure that unaligned
9635 : base objects for which out-of-bounds accesses are supported, for
9636 : example record types with trailing arrays, aren't realigned behind
9637 : the back of the caller.
9638 : The normal operating mode is to pass FALSE for this parameter. */
9639 :
9640 : rtx
9641 158139719 : expand_expr_real (tree exp, rtx target, machine_mode tmode,
9642 : enum expand_modifier modifier, rtx *alt_rtl,
9643 : bool inner_reference_p)
9644 : {
9645 158139719 : rtx ret;
9646 :
9647 : /* Handle ERROR_MARK before anybody tries to access its type. */
9648 158139719 : if (TREE_CODE (exp) == ERROR_MARK
9649 158139719 : || (TREE_CODE (TREE_TYPE (exp)) == ERROR_MARK))
9650 : {
9651 0 : ret = CONST0_RTX (tmode);
9652 0 : return ret ? ret : const0_rtx;
9653 : }
9654 :
9655 158139719 : ret = expand_expr_real_1 (exp, target, tmode, modifier, alt_rtl,
9656 : inner_reference_p);
9657 158139719 : return ret;
9658 : }
9659 :
9660 : /* Try to expand the conditional expression which is represented by
9661 : TREEOP0 ? TREEOP1 : TREEOP2 using conditional moves. If it succeeds
9662 : return the rtl reg which represents the result. Otherwise return
9663 : NULL_RTX. */
9664 :
9665 : static rtx
9666 18080 : expand_cond_expr_using_cmove (tree treeop0 ATTRIBUTE_UNUSED,
9667 : tree treeop1 ATTRIBUTE_UNUSED,
9668 : tree treeop2 ATTRIBUTE_UNUSED)
9669 : {
9670 18080 : rtx insn;
9671 18080 : rtx op00, op01, op1, op2;
9672 18080 : enum rtx_code comparison_code;
9673 18080 : machine_mode comparison_mode;
9674 18080 : gimple *srcstmt;
9675 18080 : rtx temp;
9676 18080 : tree type = TREE_TYPE (treeop1);
9677 18080 : int unsignedp = TYPE_UNSIGNED (type);
9678 18080 : machine_mode mode = TYPE_MODE (type);
9679 18080 : machine_mode orig_mode = mode;
9680 18080 : static bool expanding_cond_expr_using_cmove = false;
9681 :
9682 : /* Conditional move expansion can end up TERing two operands which,
9683 : when recursively hitting conditional expressions can result in
9684 : exponential behavior if the cmove expansion ultimatively fails.
9685 : It's hardly profitable to TER a cmove into a cmove so avoid doing
9686 : that by failing early if we end up recursing. */
9687 18080 : if (expanding_cond_expr_using_cmove)
9688 : return NULL_RTX;
9689 :
9690 : /* If we cannot do a conditional move on the mode, try doing it
9691 : with the promoted mode. */
9692 17094 : if (!can_conditionally_move_p (mode))
9693 : {
9694 287 : mode = promote_mode (type, mode, &unsignedp);
9695 287 : if (!can_conditionally_move_p (mode))
9696 : return NULL_RTX;
9697 0 : temp = assign_temp (type, 0, 0); /* Use promoted mode for temp. */
9698 : }
9699 : else
9700 16807 : temp = assign_temp (type, 0, 1);
9701 :
9702 16807 : expanding_cond_expr_using_cmove = true;
9703 16807 : start_sequence ();
9704 16807 : expand_operands (treeop1, treeop2,
9705 : mode == orig_mode ? temp : NULL_RTX, &op1, &op2,
9706 : EXPAND_NORMAL);
9707 :
9708 16807 : if (TREE_CODE (treeop0) == SSA_NAME
9709 16656 : && (srcstmt = get_def_for_expr_class (treeop0, tcc_comparison))
9710 31232 : && !VECTOR_TYPE_P (TREE_TYPE (gimple_assign_rhs1 (srcstmt))))
9711 : {
9712 14422 : type = TREE_TYPE (gimple_assign_rhs1 (srcstmt));
9713 14422 : enum tree_code cmpcode = gimple_assign_rhs_code (srcstmt);
9714 14422 : op00 = expand_normal (gimple_assign_rhs1 (srcstmt));
9715 14422 : op01 = expand_normal (gimple_assign_rhs2 (srcstmt));
9716 14422 : comparison_mode = TYPE_MODE (type);
9717 14422 : unsignedp = TYPE_UNSIGNED (type);
9718 14422 : comparison_code = convert_tree_comp_to_rtx (cmpcode, unsignedp);
9719 : }
9720 2385 : else if (COMPARISON_CLASS_P (treeop0)
9721 2385 : && !VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (treeop0, 0))))
9722 : {
9723 151 : type = TREE_TYPE (TREE_OPERAND (treeop0, 0));
9724 151 : enum tree_code cmpcode = TREE_CODE (treeop0);
9725 151 : op00 = expand_normal (TREE_OPERAND (treeop0, 0));
9726 151 : op01 = expand_normal (TREE_OPERAND (treeop0, 1));
9727 151 : unsignedp = TYPE_UNSIGNED (type);
9728 151 : comparison_mode = TYPE_MODE (type);
9729 151 : comparison_code = convert_tree_comp_to_rtx (cmpcode, unsignedp);
9730 : }
9731 : else
9732 : {
9733 2234 : op00 = expand_normal (treeop0);
9734 2234 : op01 = const0_rtx;
9735 2234 : comparison_code = NE;
9736 2234 : comparison_mode = GET_MODE (op00);
9737 2234 : if (comparison_mode == VOIDmode)
9738 0 : comparison_mode = TYPE_MODE (TREE_TYPE (treeop0));
9739 : }
9740 16807 : expanding_cond_expr_using_cmove = false;
9741 :
9742 16807 : if (GET_MODE (op1) != mode)
9743 2355 : op1 = gen_lowpart (mode, op1);
9744 :
9745 16807 : if (GET_MODE (op2) != mode)
9746 9061 : op2 = gen_lowpart (mode, op2);
9747 :
9748 : /* Try to emit the conditional move. */
9749 16807 : insn = emit_conditional_move (temp,
9750 : { comparison_code, op00, op01,
9751 : comparison_mode },
9752 : op1, op2, mode,
9753 : unsignedp);
9754 :
9755 : /* If we could do the conditional move, emit the sequence,
9756 : and return. */
9757 16807 : if (insn)
9758 : {
9759 14220 : rtx_insn *seq = end_sequence ();
9760 14220 : emit_insn (seq);
9761 14220 : return convert_modes (orig_mode, mode, temp, 0);
9762 : }
9763 :
9764 : /* Otherwise discard the sequence and fall back to code with
9765 : branches. */
9766 2587 : end_sequence ();
9767 2587 : return NULL_RTX;
9768 : }
9769 :
9770 : /* A helper function for expand_expr_real_2 to be used with a
9771 : misaligned mem_ref TEMP. Assume an unsigned type if UNSIGNEDP
9772 : is nonzero, with alignment ALIGN in bits.
9773 : Store the value at TARGET if possible (if TARGET is nonzero).
9774 : Regardless of TARGET, we return the rtx for where the value is placed.
9775 : If the result can be stored at TARGET, and ALT_RTL is non-NULL,
9776 : then *ALT_RTL is set to TARGET (before legitimziation). */
9777 :
9778 : static rtx
9779 209361 : expand_misaligned_mem_ref (rtx temp, machine_mode mode, int unsignedp,
9780 : unsigned int align, rtx target, rtx *alt_rtl)
9781 : {
9782 209361 : enum insn_code icode;
9783 :
9784 209361 : if ((icode = optab_handler (movmisalign_optab, mode))
9785 : != CODE_FOR_nothing)
9786 : {
9787 133799 : class expand_operand ops[2];
9788 :
9789 : /* We've already validated the memory, and we're creating a
9790 : new pseudo destination. The predicates really can't fail,
9791 : nor can the generator. */
9792 133799 : create_output_operand (&ops[0], NULL_RTX, mode);
9793 133799 : create_fixed_operand (&ops[1], temp);
9794 133799 : expand_insn (icode, 2, ops);
9795 133799 : temp = ops[0].value;
9796 : }
9797 75562 : else if (targetm.slow_unaligned_access (mode, align))
9798 0 : temp = extract_bit_field (temp, GET_MODE_BITSIZE (mode),
9799 : 0, unsignedp, target,
9800 : mode, mode, false, alt_rtl);
9801 209361 : return temp;
9802 : }
9803 :
9804 : /* Helper function of expand_expr_2, expand a division or modulo.
9805 : op0 and op1 should be already expanded treeop0 and treeop1, using
9806 : expand_operands. */
9807 :
9808 : static rtx
9809 173050 : expand_expr_divmod (tree_code code, machine_mode mode, tree treeop0,
9810 : tree treeop1, rtx op0, rtx op1, rtx target, int unsignedp)
9811 : {
9812 346100 : bool mod_p = (code == TRUNC_MOD_EXPR || code == FLOOR_MOD_EXPR
9813 173050 : || code == CEIL_MOD_EXPR || code == ROUND_MOD_EXPR);
9814 173050 : if (SCALAR_INT_MODE_P (mode)
9815 173050 : && optimize >= 2
9816 146154 : && get_range_pos_neg (treeop0, currently_expanding_gimple_stmt) == 1
9817 220226 : && get_range_pos_neg (treeop1, currently_expanding_gimple_stmt) == 1)
9818 : {
9819 : /* If both arguments are known to be positive when interpreted
9820 : as signed, we can expand it as both signed and unsigned
9821 : division or modulo. Choose the cheaper sequence in that case. */
9822 18750 : bool speed_p = optimize_insn_for_speed_p ();
9823 18750 : do_pending_stack_adjust ();
9824 18750 : start_sequence ();
9825 18750 : rtx uns_ret = expand_divmod (mod_p, code, mode, op0, op1, target, 1);
9826 18750 : rtx_insn *uns_insns = end_sequence ();
9827 18750 : start_sequence ();
9828 18750 : rtx sgn_ret = expand_divmod (mod_p, code, mode, op0, op1, target, 0);
9829 18750 : rtx_insn *sgn_insns = end_sequence ();
9830 18750 : unsigned uns_cost = seq_cost (uns_insns, speed_p);
9831 18750 : unsigned sgn_cost = seq_cost (sgn_insns, speed_p);
9832 18750 : bool was_tie = false;
9833 :
9834 : /* If costs are the same then use as tie breaker the other other
9835 : factor. */
9836 18750 : if (uns_cost == sgn_cost)
9837 : {
9838 8359 : uns_cost = seq_cost (uns_insns, !speed_p);
9839 8359 : sgn_cost = seq_cost (sgn_insns, !speed_p);
9840 8359 : was_tie = true;
9841 : }
9842 :
9843 18750 : if (dump_file && (dump_flags & TDF_DETAILS))
9844 0 : fprintf (dump_file, ";; positive division:%s unsigned cost: %u; "
9845 : "signed cost: %u\n",
9846 : was_tie ? " (needed tie breaker)" : "", uns_cost, sgn_cost);
9847 :
9848 18750 : if (uns_cost < sgn_cost || (uns_cost == sgn_cost && unsignedp))
9849 : {
9850 11199 : emit_insn (uns_insns);
9851 11199 : return uns_ret;
9852 : }
9853 7551 : emit_insn (sgn_insns);
9854 7551 : return sgn_ret;
9855 : }
9856 154300 : return expand_divmod (mod_p, code, mode, op0, op1, target, unsignedp);
9857 : }
9858 :
9859 : /* Return true if EXP has a range of values [0..1], false
9860 : otherwise. This works for constants and ssa names, calling back into the ranger. */
9861 : static bool
9862 1522562 : expr_has_boolean_range (tree exp, gimple *stmt)
9863 : {
9864 : /* An integral type with a single bit of precision. */
9865 3045111 : if (INTEGRAL_TYPE_P (TREE_TYPE (exp))
9866 1522562 : && TYPE_UNSIGNED (TREE_TYPE (exp))
9867 2533360 : && TYPE_PRECISION (TREE_TYPE (exp)) == 1)
9868 : return true;
9869 :
9870 : /* Signed 1 bit integers are not boolean ranges. */
9871 3045111 : if (!INTEGRAL_TYPE_P (TREE_TYPE (exp))
9872 3045111 : || TYPE_PRECISION (TREE_TYPE (exp)) <= 1)
9873 : return false;
9874 :
9875 1522518 : if (TREE_CODE (exp) == SSA_NAME)
9876 902338 : return ssa_name_has_boolean_range (exp, stmt);
9877 620180 : if (TREE_CODE (exp) == INTEGER_CST)
9878 546012 : return wi::leu_p (wi::to_wide (exp), 1);
9879 : return false;
9880 : }
9881 :
9882 : rtx
9883 13548419 : expand_expr_real_2 (const_sepops ops, rtx target, machine_mode tmode,
9884 : enum expand_modifier modifier)
9885 : {
9886 13548419 : rtx op0, op1, op2, temp;
9887 13548419 : rtx_code_label *lab;
9888 13548419 : tree type;
9889 13548419 : int unsignedp;
9890 13548419 : machine_mode mode;
9891 13548419 : scalar_int_mode int_mode;
9892 13548419 : enum tree_code code = ops->code;
9893 13548419 : optab this_optab;
9894 13548419 : rtx subtarget, original_target;
9895 13548419 : int ignore;
9896 13548419 : bool reduce_bit_field;
9897 13548419 : location_t loc = ops->location;
9898 13548419 : tree treeop0, treeop1, treeop2;
9899 : #define REDUCE_BIT_FIELD(expr) (reduce_bit_field \
9900 : ? reduce_to_bit_field_precision ((expr), \
9901 : target, \
9902 : type) \
9903 : : (expr))
9904 :
9905 13548419 : type = ops->type;
9906 13548419 : mode = TYPE_MODE (type);
9907 13548419 : unsignedp = TYPE_UNSIGNED (type);
9908 :
9909 13548419 : treeop0 = ops->op0;
9910 13548419 : treeop1 = ops->op1;
9911 13548419 : treeop2 = ops->op2;
9912 :
9913 : /* We should be called only on simple (binary or unary) expressions,
9914 : exactly those that are valid in gimple expressions that aren't
9915 : GIMPLE_SINGLE_RHS (or invalid). */
9916 13548419 : gcc_assert (get_gimple_rhs_class (code) == GIMPLE_UNARY_RHS
9917 : || get_gimple_rhs_class (code) == GIMPLE_BINARY_RHS
9918 : || get_gimple_rhs_class (code) == GIMPLE_TERNARY_RHS);
9919 :
9920 27096838 : ignore = (target == const0_rtx
9921 13548419 : || ((CONVERT_EXPR_CODE_P (code)
9922 9743560 : || code == COND_EXPR || code == VIEW_CONVERT_EXPR)
9923 3822939 : && TREE_CODE (type) == VOID_TYPE));
9924 :
9925 : /* We should be called only if we need the result. */
9926 0 : gcc_assert (!ignore);
9927 :
9928 : /* An operation in what may be a bit-field type needs the
9929 : result to be reduced to the precision of the bit-field type,
9930 : which is narrower than that of the type's mode. */
9931 27921419 : reduce_bit_field = (INTEGRAL_TYPE_P (type)
9932 13548419 : && !type_has_mode_precision_p (type));
9933 :
9934 824581 : if (reduce_bit_field
9935 824581 : && (modifier == EXPAND_STACK_PARM
9936 824057 : || (target && GET_MODE (target) != mode)))
9937 508015 : target = 0;
9938 :
9939 : /* Use subtarget as the target for operand 0 of a binary operation. */
9940 13548419 : subtarget = get_subtarget (target);
9941 13548419 : original_target = target;
9942 :
9943 13548419 : switch (code)
9944 : {
9945 3808057 : case NON_LVALUE_EXPR:
9946 3808057 : case PAREN_EXPR:
9947 3808057 : CASE_CONVERT:
9948 3808057 : if (treeop0 == error_mark_node)
9949 0 : return const0_rtx;
9950 :
9951 3808057 : if (TREE_CODE (type) == UNION_TYPE)
9952 : {
9953 0 : tree valtype = TREE_TYPE (treeop0);
9954 :
9955 : /* If both input and output are BLKmode, this conversion isn't doing
9956 : anything except possibly changing memory attribute. */
9957 0 : if (mode == BLKmode && TYPE_MODE (valtype) == BLKmode)
9958 : {
9959 0 : rtx result = expand_expr (treeop0, target, tmode,
9960 : modifier);
9961 :
9962 0 : result = copy_rtx (result);
9963 0 : set_mem_attributes (result, type, 0);
9964 0 : return result;
9965 : }
9966 :
9967 0 : if (target == 0)
9968 : {
9969 0 : if (TYPE_MODE (type) != BLKmode)
9970 0 : target = gen_reg_rtx (TYPE_MODE (type));
9971 : else
9972 0 : target = assign_temp (type, 1, 1);
9973 : }
9974 :
9975 0 : if (MEM_P (target))
9976 : /* Store data into beginning of memory target. */
9977 0 : store_expr (treeop0,
9978 0 : adjust_address (target, TYPE_MODE (valtype), 0),
9979 : modifier == EXPAND_STACK_PARM,
9980 0 : false, TYPE_REVERSE_STORAGE_ORDER (type));
9981 :
9982 : else
9983 : {
9984 0 : gcc_assert (REG_P (target)
9985 : && !TYPE_REVERSE_STORAGE_ORDER (type));
9986 :
9987 : /* Store this field into a union of the proper type. */
9988 0 : poly_uint64 op0_size
9989 0 : = tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (treeop0)));
9990 0 : poly_uint64 union_size = GET_MODE_BITSIZE (mode);
9991 0 : store_field (target,
9992 : /* The conversion must be constructed so that
9993 : we know at compile time how many bits
9994 : to preserve. */
9995 0 : ordered_min (op0_size, union_size),
9996 0 : 0, 0, 0, TYPE_MODE (valtype), treeop0, 0,
9997 : false, false);
9998 : }
9999 :
10000 : /* Return the entire union. */
10001 0 : return target;
10002 : }
10003 :
10004 3808057 : if (mode == TYPE_MODE (TREE_TYPE (treeop0)))
10005 : {
10006 2262307 : op0 = expand_expr (treeop0, target, VOIDmode,
10007 : modifier);
10008 :
10009 2262307 : return REDUCE_BIT_FIELD (op0);
10010 : }
10011 :
10012 1908088 : op0 = expand_expr (treeop0, NULL_RTX, mode,
10013 : modifier == EXPAND_SUM ? EXPAND_NORMAL : modifier);
10014 1545750 : if (GET_MODE (op0) == mode)
10015 : ;
10016 :
10017 : /* If OP0 is a constant, just convert it into the proper mode. */
10018 1505122 : else if (CONSTANT_P (op0))
10019 : {
10020 908 : tree inner_type = TREE_TYPE (treeop0);
10021 908 : machine_mode inner_mode = GET_MODE (op0);
10022 :
10023 908 : if (inner_mode == VOIDmode)
10024 15 : inner_mode = TYPE_MODE (inner_type);
10025 :
10026 908 : if (modifier == EXPAND_INITIALIZER)
10027 0 : op0 = force_lowpart_subreg (mode, op0, inner_mode);
10028 : else
10029 908 : op0 = convert_modes (mode, inner_mode, op0,
10030 908 : TYPE_UNSIGNED (inner_type));
10031 : }
10032 :
10033 1504214 : else if (modifier == EXPAND_INITIALIZER)
10034 24 : op0 = gen_rtx_fmt_e (TYPE_UNSIGNED (TREE_TYPE (treeop0))
10035 : ? ZERO_EXTEND : SIGN_EXTEND, mode, op0);
10036 :
10037 1504202 : else if (SCALAR_INT_MODE_P (GET_MODE (op0))
10038 1330935 : && optimize >= 2
10039 775641 : && SCALAR_INT_MODE_P (mode)
10040 775641 : && INTEGRAL_TYPE_P (TREE_TYPE (treeop0))
10041 775438 : && (GET_MODE_SIZE (as_a <scalar_int_mode> (mode))
10042 1550876 : > GET_MODE_SIZE (as_a <scalar_int_mode> (GET_MODE (op0))))
10043 2049184 : && get_range_pos_neg (treeop0,
10044 : currently_expanding_gimple_stmt) == 1)
10045 : {
10046 : /* If argument is known to be positive when interpreted
10047 : as signed, we can expand it as both sign and zero
10048 : extension. Choose the cheaper sequence in that case. */
10049 140385 : bool speed_p = optimize_insn_for_speed_p ();
10050 140385 : rtx uns_ret = NULL_RTX, sgn_ret = NULL_RTX;
10051 140385 : do_pending_stack_adjust ();
10052 140385 : start_sequence ();
10053 140385 : if (target == NULL_RTX)
10054 116371 : uns_ret = convert_to_mode (mode, op0, 1);
10055 : else
10056 24014 : convert_move (target, op0, 1);
10057 140385 : rtx_insn *uns_insns = end_sequence ();
10058 140385 : start_sequence ();
10059 140385 : if (target == NULL_RTX)
10060 116371 : sgn_ret = convert_to_mode (mode, op0, 0);
10061 : else
10062 24014 : convert_move (target, op0, 0);
10063 140385 : rtx_insn *sgn_insns = end_sequence ();
10064 140385 : unsigned uns_cost = seq_cost (uns_insns, speed_p);
10065 140385 : unsigned sgn_cost = seq_cost (sgn_insns, speed_p);
10066 140385 : bool was_tie = false;
10067 :
10068 : /* If costs are the same then use as tie breaker the other other
10069 : factor. */
10070 140385 : if (uns_cost == sgn_cost)
10071 : {
10072 38471 : uns_cost = seq_cost (uns_insns, !speed_p);
10073 38471 : sgn_cost = seq_cost (sgn_insns, !speed_p);
10074 38471 : was_tie = true;
10075 : }
10076 :
10077 140385 : if (dump_file && (dump_flags & TDF_DETAILS))
10078 0 : fprintf (dump_file, ";; positive extension:%s unsigned cost: %u; "
10079 : "signed cost: %u\n",
10080 : was_tie ? " (needed tie breaker)" : "",
10081 : uns_cost, sgn_cost);
10082 140385 : if (uns_cost < sgn_cost
10083 140385 : || (uns_cost == sgn_cost && TYPE_UNSIGNED (TREE_TYPE (treeop0))))
10084 : {
10085 125573 : emit_insn (uns_insns);
10086 125573 : sgn_ret = uns_ret;
10087 : }
10088 : else
10089 14812 : emit_insn (sgn_insns);
10090 140385 : if (target == NULL_RTX)
10091 116371 : op0 = sgn_ret;
10092 : else
10093 24014 : op0 = target;
10094 : }
10095 1363817 : else if (target == 0)
10096 863748 : op0 = convert_to_mode (mode, op0, TYPE_UNSIGNED (TREE_TYPE (treeop0)));
10097 : else
10098 : {
10099 500069 : convert_move (target, op0, TYPE_UNSIGNED (TREE_TYPE (treeop0)));
10100 500069 : op0 = target;
10101 : }
10102 :
10103 1545750 : return REDUCE_BIT_FIELD (op0);
10104 :
10105 0 : case ADDR_SPACE_CONVERT_EXPR:
10106 0 : {
10107 0 : tree treeop0_type = TREE_TYPE (treeop0);
10108 :
10109 0 : gcc_assert (POINTER_TYPE_P (type));
10110 0 : gcc_assert (POINTER_TYPE_P (treeop0_type));
10111 :
10112 0 : addr_space_t as_to = TYPE_ADDR_SPACE (TREE_TYPE (type));
10113 0 : addr_space_t as_from = TYPE_ADDR_SPACE (TREE_TYPE (treeop0_type));
10114 :
10115 : /* Conversions between pointers to the same address space should
10116 : have been implemented via CONVERT_EXPR / NOP_EXPR. */
10117 0 : gcc_assert (as_to != as_from);
10118 :
10119 0 : op0 = expand_expr (treeop0, NULL_RTX, VOIDmode, modifier);
10120 :
10121 : /* Ask target code to handle conversion between pointers
10122 : to overlapping address spaces. */
10123 0 : if (targetm.addr_space.subset_p (as_to, as_from)
10124 0 : || targetm.addr_space.subset_p (as_from, as_to))
10125 : {
10126 0 : op0 = targetm.addr_space.convert (op0, treeop0_type, type);
10127 : }
10128 : else
10129 : {
10130 : /* For disjoint address spaces, converting anything but a null
10131 : pointer invokes undefined behavior. We truncate or extend the
10132 : value as if we'd converted via integers, which handles 0 as
10133 : required, and all others as the programmer likely expects. */
10134 : #ifndef POINTERS_EXTEND_UNSIGNED
10135 : const int POINTERS_EXTEND_UNSIGNED = 1;
10136 : #endif
10137 0 : op0 = convert_modes (mode, TYPE_MODE (treeop0_type),
10138 : op0, POINTERS_EXTEND_UNSIGNED);
10139 : }
10140 0 : gcc_assert (op0);
10141 : return op0;
10142 : }
10143 :
10144 1358001 : case POINTER_PLUS_EXPR:
10145 : /* Even though the sizetype mode and the pointer's mode can be different
10146 : expand is able to handle this correctly and get the correct result out
10147 : of the PLUS_EXPR code. */
10148 : /* Make sure to sign-extend the sizetype offset in a POINTER_PLUS_EXPR
10149 : if sizetype precision is smaller than pointer precision. */
10150 1358001 : if (TYPE_PRECISION (sizetype) < TYPE_PRECISION (type))
10151 0 : treeop1 = fold_convert_loc (loc, type,
10152 : fold_convert_loc (loc, ssizetype,
10153 : treeop1));
10154 : /* If sizetype precision is larger than pointer precision, truncate the
10155 : offset to have matching modes. */
10156 1358001 : else if (TYPE_PRECISION (sizetype) > TYPE_PRECISION (type))
10157 0 : treeop1 = fold_convert_loc (loc, type, treeop1);
10158 : /* FALLTHRU */
10159 :
10160 5361693 : case PLUS_EXPR:
10161 : /* If we are adding a constant, a VAR_DECL that is sp, fp, or ap, and
10162 : something else, make sure we add the register to the constant and
10163 : then to the other thing. This case can occur during strength
10164 : reduction and doing it this way will produce better code if the
10165 : frame pointer or argument pointer is eliminated.
10166 :
10167 : fold-const.cc will ensure that the constant is always in the inner
10168 : PLUS_EXPR, so the only case we need to do anything about is if
10169 : sp, ap, or fp is our second argument, in which case we must swap
10170 : the innermost first argument and our second argument. */
10171 :
10172 5361693 : if (TREE_CODE (treeop0) == PLUS_EXPR
10173 6914 : && TREE_CODE (TREE_OPERAND (treeop0, 1)) == INTEGER_CST
10174 0 : && VAR_P (treeop1)
10175 5361693 : && (DECL_RTL (treeop1) == frame_pointer_rtx
10176 0 : || DECL_RTL (treeop1) == stack_pointer_rtx
10177 0 : || DECL_RTL (treeop1) == arg_pointer_rtx))
10178 : {
10179 0 : gcc_unreachable ();
10180 : }
10181 :
10182 : /* If the result is to be ptr_mode and we are adding an integer to
10183 : something, we might be forming a constant. So try to use
10184 : plus_constant. If it produces a sum and we can't accept it,
10185 : use force_operand. This allows P = &ARR[const] to generate
10186 : efficient code on machines where a SYMBOL_REF is not a valid
10187 : address.
10188 :
10189 : If this is an EXPAND_SUM call, always return the sum. */
10190 5361693 : if (modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER
10191 5361693 : || (mode == ptr_mode && (unsignedp || ! flag_trapv)))
10192 : {
10193 3500716 : if (modifier == EXPAND_STACK_PARM)
10194 18838 : target = 0;
10195 3500716 : if (TREE_CODE (treeop0) == INTEGER_CST
10196 3501723 : && HWI_COMPUTABLE_MODE_P (mode)
10197 3501727 : && TREE_CONSTANT (treeop1))
10198 : {
10199 4 : rtx constant_part;
10200 4 : HOST_WIDE_INT wc;
10201 4 : machine_mode wmode = TYPE_MODE (TREE_TYPE (treeop1));
10202 :
10203 4 : op1 = expand_expr (treeop1, subtarget, VOIDmode,
10204 : EXPAND_SUM);
10205 : /* Use wi::shwi to ensure that the constant is
10206 : truncated according to the mode of OP1, then sign extended
10207 : to a HOST_WIDE_INT. Using the constant directly can result
10208 : in non-canonical RTL in a 64x32 cross compile. */
10209 4 : wc = TREE_INT_CST_LOW (treeop0);
10210 4 : constant_part =
10211 4 : immed_wide_int_const (wi::shwi (wc, wmode), wmode);
10212 4 : op1 = plus_constant (mode, op1, INTVAL (constant_part));
10213 4 : if (modifier != EXPAND_SUM && modifier != EXPAND_INITIALIZER)
10214 1 : op1 = force_operand (op1, target);
10215 4 : return REDUCE_BIT_FIELD (op1);
10216 : }
10217 :
10218 3500712 : else if (TREE_CODE (treeop1) == INTEGER_CST
10219 7444607 : && HWI_COMPUTABLE_MODE_P (mode)
10220 5862840 : && TREE_CONSTANT (treeop0))
10221 : {
10222 279210 : rtx constant_part;
10223 279210 : HOST_WIDE_INT wc;
10224 279210 : machine_mode wmode = TYPE_MODE (TREE_TYPE (treeop0));
10225 :
10226 487019 : op0 = expand_expr (treeop0, subtarget, VOIDmode,
10227 : (modifier == EXPAND_INITIALIZER
10228 : ? EXPAND_INITIALIZER : EXPAND_SUM));
10229 279210 : if (! CONSTANT_P (op0))
10230 : {
10231 6 : op1 = expand_expr (treeop1, NULL_RTX,
10232 : VOIDmode, modifier);
10233 : /* Return a PLUS if modifier says it's OK. */
10234 6 : if (modifier == EXPAND_SUM
10235 : || modifier == EXPAND_INITIALIZER)
10236 6 : return simplify_gen_binary (PLUS, mode, op0, op1);
10237 0 : goto binop2;
10238 : }
10239 : /* Use wi::shwi to ensure that the constant is
10240 : truncated according to the mode of OP1, then sign extended
10241 : to a HOST_WIDE_INT. Using the constant directly can result
10242 : in non-canonical RTL in a 64x32 cross compile. */
10243 279204 : wc = TREE_INT_CST_LOW (treeop1);
10244 279204 : constant_part
10245 279204 : = immed_wide_int_const (wi::shwi (wc, wmode), wmode);
10246 279204 : op0 = plus_constant (mode, op0, INTVAL (constant_part));
10247 279204 : if (modifier != EXPAND_SUM && modifier != EXPAND_INITIALIZER)
10248 206985 : op0 = force_operand (op0, target);
10249 279204 : return REDUCE_BIT_FIELD (op0);
10250 : }
10251 : }
10252 :
10253 : /* Use TER to expand pointer addition of a negated value
10254 : as pointer subtraction. */
10255 9105172 : if ((POINTER_TYPE_P (TREE_TYPE (treeop0))
10256 4003679 : || (TREE_CODE (TREE_TYPE (treeop0)) == VECTOR_TYPE
10257 81153 : && POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (treeop0)))))
10258 1078800 : && TREE_CODE (treeop1) == SSA_NAME
10259 6208217 : && TYPE_MODE (TREE_TYPE (treeop0))
10260 562869 : == TYPE_MODE (TREE_TYPE (treeop1)))
10261 : {
10262 562869 : gimple *def = get_def_for_expr (treeop1, NEGATE_EXPR);
10263 562869 : if (def)
10264 : {
10265 2136 : treeop1 = gimple_assign_rhs1 (def);
10266 2136 : code = MINUS_EXPR;
10267 2136 : goto do_minus;
10268 : }
10269 : }
10270 :
10271 : /* No sense saving up arithmetic to be done
10272 : if it's all in the wrong mode to form part of an address.
10273 : And force_operand won't know whether to sign-extend or
10274 : zero-extend. */
10275 5080343 : if (modifier != EXPAND_INITIALIZER
10276 5080343 : && (modifier != EXPAND_SUM || mode != ptr_mode))
10277 : {
10278 4506157 : expand_operands (treeop0, treeop1,
10279 : subtarget, &op0, &op1, modifier);
10280 4506157 : if (op0 == const0_rtx)
10281 8670 : return op1;
10282 4497487 : if (op1 == const0_rtx)
10283 : return op0;
10284 4497236 : goto binop2;
10285 : }
10286 :
10287 574186 : expand_operands (treeop0, treeop1,
10288 : subtarget, &op0, &op1, modifier);
10289 574186 : return REDUCE_BIT_FIELD (simplify_gen_binary (PLUS, mode, op0, op1));
10290 :
10291 540004 : case MINUS_EXPR:
10292 540004 : case POINTER_DIFF_EXPR:
10293 540004 : do_minus:
10294 : /* For initializers, we are allowed to return a MINUS of two
10295 : symbolic constants. Here we handle all cases when both operands
10296 : are constant. */
10297 : /* Handle difference of two symbolic constants,
10298 : for the sake of an initializer. */
10299 540004 : if ((modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER)
10300 2657 : && really_constant_p (treeop0)
10301 540515 : && really_constant_p (treeop1))
10302 : {
10303 68 : expand_operands (treeop0, treeop1,
10304 : NULL_RTX, &op0, &op1, modifier);
10305 68 : return simplify_gen_binary (MINUS, mode, op0, op1);
10306 : }
10307 :
10308 : /* No sense saving up arithmetic to be done
10309 : if it's all in the wrong mode to form part of an address.
10310 : And force_operand won't know whether to sign-extend or
10311 : zero-extend. */
10312 539936 : if (modifier != EXPAND_INITIALIZER
10313 539936 : && (modifier != EXPAND_SUM || mode != ptr_mode))
10314 537347 : goto binop;
10315 :
10316 2589 : expand_operands (treeop0, treeop1,
10317 : subtarget, &op0, &op1, modifier);
10318 :
10319 : /* Convert A - const to A + (-const). */
10320 2589 : if (CONST_INT_P (op1))
10321 : {
10322 0 : op1 = negate_rtx (mode, op1);
10323 0 : return REDUCE_BIT_FIELD (simplify_gen_binary (PLUS, mode, op0, op1));
10324 : }
10325 :
10326 2589 : goto binop2;
10327 :
10328 0 : case WIDEN_MULT_PLUS_EXPR:
10329 0 : case WIDEN_MULT_MINUS_EXPR:
10330 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
10331 0 : op2 = expand_normal (treeop2);
10332 0 : target = expand_widen_pattern_expr (ops, op0, op1, op2,
10333 : target, unsignedp);
10334 0 : return target;
10335 :
10336 18516 : case WIDEN_MULT_EXPR:
10337 : /* If first operand is constant, swap them.
10338 : Thus the following special case checks need only
10339 : check the second operand. */
10340 18516 : if (TREE_CODE (treeop0) == INTEGER_CST)
10341 368 : std::swap (treeop0, treeop1);
10342 :
10343 : /* First, check if we have a multiplication of one signed and one
10344 : unsigned operand. */
10345 18516 : if (TREE_CODE (treeop1) != INTEGER_CST
10346 18516 : && (TYPE_UNSIGNED (TREE_TYPE (treeop0))
10347 14262 : != TYPE_UNSIGNED (TREE_TYPE (treeop1))))
10348 : {
10349 0 : machine_mode innermode = TYPE_MODE (TREE_TYPE (treeop0));
10350 0 : this_optab = usmul_widen_optab;
10351 0 : if (find_widening_optab_handler (this_optab, mode, innermode)
10352 : != CODE_FOR_nothing)
10353 : {
10354 0 : if (TYPE_UNSIGNED (TREE_TYPE (treeop0)))
10355 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1,
10356 : EXPAND_NORMAL);
10357 : else
10358 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op1, &op0,
10359 : EXPAND_NORMAL);
10360 : /* op0 and op1 might still be constant, despite the above
10361 : != INTEGER_CST check. Handle it. */
10362 0 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
10363 : {
10364 0 : op0 = convert_modes (mode, innermode, op0, true);
10365 0 : op1 = convert_modes (mode, innermode, op1, false);
10366 0 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1,
10367 : target, unsignedp));
10368 : }
10369 0 : goto binop3;
10370 : }
10371 : }
10372 : /* Check for a multiplication with matching signedness. */
10373 18516 : else if ((TREE_CODE (treeop1) == INTEGER_CST
10374 4254 : && int_fits_type_p (treeop1, TREE_TYPE (treeop0)))
10375 18520 : || (TYPE_UNSIGNED (TREE_TYPE (treeop1))
10376 14266 : == TYPE_UNSIGNED (TREE_TYPE (treeop0))))
10377 : {
10378 18516 : tree op0type = TREE_TYPE (treeop0);
10379 18516 : machine_mode innermode = TYPE_MODE (op0type);
10380 18516 : bool zextend_p = TYPE_UNSIGNED (op0type);
10381 18516 : optab other_optab = zextend_p ? smul_widen_optab : umul_widen_optab;
10382 2614 : this_optab = zextend_p ? umul_widen_optab : smul_widen_optab;
10383 :
10384 18516 : if (TREE_CODE (treeop0) != INTEGER_CST)
10385 : {
10386 18516 : if (find_widening_optab_handler (this_optab, mode, innermode)
10387 : != CODE_FOR_nothing)
10388 : {
10389 18516 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1,
10390 : EXPAND_NORMAL);
10391 : /* op0 and op1 might still be constant, despite the above
10392 : != INTEGER_CST check. Handle it. */
10393 18516 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
10394 : {
10395 0 : widen_mult_const:
10396 0 : op0 = convert_modes (mode, innermode, op0, zextend_p);
10397 0 : op1
10398 0 : = convert_modes (mode, innermode, op1,
10399 0 : TYPE_UNSIGNED (TREE_TYPE (treeop1)));
10400 0 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1,
10401 : target,
10402 : unsignedp));
10403 : }
10404 18516 : temp = expand_widening_mult (mode, op0, op1, target,
10405 : unsignedp, this_optab);
10406 18516 : return REDUCE_BIT_FIELD (temp);
10407 : }
10408 0 : if (find_widening_optab_handler (other_optab, mode, innermode)
10409 : != CODE_FOR_nothing
10410 0 : && innermode == word_mode)
10411 : {
10412 0 : rtx htem, hipart;
10413 0 : op0 = expand_normal (treeop0);
10414 0 : op1 = expand_normal (treeop1);
10415 : /* op0 and op1 might be constants, despite the above
10416 : != INTEGER_CST check. Handle it. */
10417 0 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
10418 0 : goto widen_mult_const;
10419 0 : temp = expand_binop (mode, other_optab, op0, op1, target,
10420 : unsignedp, OPTAB_LIB_WIDEN);
10421 0 : hipart = gen_highpart (word_mode, temp);
10422 0 : htem = expand_mult_highpart_adjust (word_mode, hipart,
10423 : op0, op1, hipart,
10424 : zextend_p);
10425 0 : if (htem != hipart)
10426 0 : emit_move_insn (hipart, htem);
10427 0 : return REDUCE_BIT_FIELD (temp);
10428 : }
10429 : }
10430 : }
10431 0 : treeop0 = fold_build1 (CONVERT_EXPR, type, treeop0);
10432 0 : treeop1 = fold_build1 (CONVERT_EXPR, type, treeop1);
10433 0 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10434 0 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1, target, unsignedp));
10435 :
10436 1423324 : case MULT_EXPR:
10437 : /* If this is a fixed-point operation, then we cannot use the code
10438 : below because "expand_mult" doesn't support sat/no-sat fixed-point
10439 : multiplications. */
10440 1423324 : if (ALL_FIXED_POINT_MODE_P (mode))
10441 0 : goto binop;
10442 :
10443 : /* If first operand is constant, swap them.
10444 : Thus the following special case checks need only
10445 : check the second operand. */
10446 1423324 : if (TREE_CODE (treeop0) == INTEGER_CST)
10447 686 : std::swap (treeop0, treeop1);
10448 :
10449 : /* Attempt to return something suitable for generating an
10450 : indexed address, for machines that support that. */
10451 :
10452 536039 : if (modifier == EXPAND_SUM && mode == ptr_mode
10453 1959363 : && tree_fits_shwi_p (treeop1))
10454 : {
10455 528674 : tree exp1 = treeop1;
10456 :
10457 528674 : op0 = expand_expr (treeop0, subtarget, VOIDmode,
10458 : EXPAND_SUM);
10459 :
10460 528674 : if (!REG_P (op0))
10461 244752 : op0 = force_operand (op0, NULL_RTX);
10462 528674 : if (!REG_P (op0))
10463 2344 : op0 = copy_to_mode_reg (mode, op0);
10464 :
10465 528674 : op1 = gen_int_mode (tree_to_shwi (exp1),
10466 528674 : TYPE_MODE (TREE_TYPE (exp1)));
10467 528674 : return REDUCE_BIT_FIELD (gen_rtx_MULT (mode, op0, op1));
10468 : }
10469 :
10470 894650 : if (modifier == EXPAND_STACK_PARM)
10471 2963 : target = 0;
10472 :
10473 894650 : if (SCALAR_INT_MODE_P (mode) && optimize >= 2)
10474 : {
10475 540713 : gimple *def_stmt0 = get_def_for_expr (treeop0, TRUNC_DIV_EXPR);
10476 540713 : gimple *def_stmt1 = get_def_for_expr (treeop1, TRUNC_DIV_EXPR);
10477 540713 : if (def_stmt0
10478 540713 : && !operand_equal_p (treeop1, gimple_assign_rhs2 (def_stmt0), 0))
10479 : def_stmt0 = NULL;
10480 540713 : if (def_stmt1
10481 540713 : && !operand_equal_p (treeop0, gimple_assign_rhs2 (def_stmt1), 0))
10482 : def_stmt1 = NULL;
10483 :
10484 540713 : if (def_stmt0 || def_stmt1)
10485 : {
10486 : /* X / Y * Y can be expanded as X - X % Y too.
10487 : Choose the cheaper sequence of those two. */
10488 250 : if (def_stmt0)
10489 250 : treeop0 = gimple_assign_rhs1 (def_stmt0);
10490 : else
10491 : {
10492 0 : treeop1 = treeop0;
10493 0 : treeop0 = gimple_assign_rhs1 (def_stmt1);
10494 : }
10495 250 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1,
10496 : EXPAND_NORMAL);
10497 250 : bool speed_p = optimize_insn_for_speed_p ();
10498 250 : do_pending_stack_adjust ();
10499 250 : start_sequence ();
10500 250 : rtx divmul_ret
10501 250 : = expand_expr_divmod (TRUNC_DIV_EXPR, mode, treeop0, treeop1,
10502 : op0, op1, NULL_RTX, unsignedp);
10503 250 : divmul_ret = expand_mult (mode, divmul_ret, op1, target,
10504 : unsignedp);
10505 250 : rtx_insn *divmul_insns = end_sequence ();
10506 250 : start_sequence ();
10507 250 : rtx modsub_ret
10508 250 : = expand_expr_divmod (TRUNC_MOD_EXPR, mode, treeop0, treeop1,
10509 : op0, op1, NULL_RTX, unsignedp);
10510 250 : this_optab = optab_for_tree_code (MINUS_EXPR, type,
10511 : optab_default);
10512 250 : modsub_ret = expand_binop (mode, this_optab, op0, modsub_ret,
10513 : target, unsignedp, OPTAB_LIB_WIDEN);
10514 250 : rtx_insn *modsub_insns = end_sequence ();
10515 250 : unsigned divmul_cost = seq_cost (divmul_insns, speed_p);
10516 250 : unsigned modsub_cost = seq_cost (modsub_insns, speed_p);
10517 : /* If costs are the same then use as tie breaker the other other
10518 : factor. */
10519 250 : if (divmul_cost == modsub_cost)
10520 : {
10521 22 : divmul_cost = seq_cost (divmul_insns, !speed_p);
10522 22 : modsub_cost = seq_cost (modsub_insns, !speed_p);
10523 : }
10524 :
10525 250 : if (divmul_cost <= modsub_cost)
10526 : {
10527 176 : emit_insn (divmul_insns);
10528 176 : return REDUCE_BIT_FIELD (divmul_ret);
10529 : }
10530 74 : emit_insn (modsub_insns);
10531 74 : return REDUCE_BIT_FIELD (modsub_ret);
10532 : }
10533 : }
10534 :
10535 894400 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10536 :
10537 : /* Expand X*Y as X&-Y when Y must be zero or one. */
10538 894400 : if (SCALAR_INT_MODE_P (mode))
10539 : {
10540 761281 : bool bit0_p = expr_has_boolean_range (treeop0, currently_expanding_gimple_stmt);
10541 761281 : bool bit1_p = expr_has_boolean_range (treeop1, currently_expanding_gimple_stmt);
10542 :
10543 : /* Expand X*Y as X&Y when both X and Y must be zero or one. */
10544 761281 : if (bit0_p && bit1_p)
10545 0 : return REDUCE_BIT_FIELD (expand_and (mode, op0, op1, target));
10546 :
10547 761281 : if (bit0_p || bit1_p)
10548 : {
10549 4006 : bool speed = optimize_insn_for_speed_p ();
10550 4006 : int cost = add_cost (speed, mode) + neg_cost (speed, mode);
10551 4006 : struct algorithm algorithm;
10552 4006 : enum mult_variant variant;
10553 4006 : if (CONST_INT_P (op1)
10554 4006 : ? !choose_mult_variant (mode, INTVAL (op1),
10555 : &algorithm, &variant, cost)
10556 625 : : cost < mul_cost (speed, mode))
10557 : {
10558 1970 : temp = bit0_p ? expand_and (mode, negate_rtx (mode, op0),
10559 : op1, target)
10560 244 : : expand_and (mode, op0,
10561 : negate_rtx (mode, op1),
10562 : target);
10563 1970 : return REDUCE_BIT_FIELD (temp);
10564 : }
10565 : }
10566 : }
10567 :
10568 892430 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1, target, unsignedp));
10569 :
10570 172550 : case TRUNC_MOD_EXPR:
10571 172550 : case FLOOR_MOD_EXPR:
10572 172550 : case CEIL_MOD_EXPR:
10573 172550 : case ROUND_MOD_EXPR:
10574 :
10575 172550 : case TRUNC_DIV_EXPR:
10576 172550 : case FLOOR_DIV_EXPR:
10577 172550 : case CEIL_DIV_EXPR:
10578 172550 : case ROUND_DIV_EXPR:
10579 172550 : case EXACT_DIV_EXPR:
10580 : /* If this is a fixed-point operation, then we cannot use the code
10581 : below because "expand_divmod" doesn't support sat/no-sat fixed-point
10582 : divisions. */
10583 172550 : if (ALL_FIXED_POINT_MODE_P (mode))
10584 0 : goto binop;
10585 :
10586 172550 : if (modifier == EXPAND_STACK_PARM)
10587 325 : target = 0;
10588 : /* Possible optimization: compute the dividend with EXPAND_SUM
10589 : then if the divisor is constant can optimize the case
10590 : where some terms of the dividend have coeffs divisible by it. */
10591 172550 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10592 172550 : return expand_expr_divmod (code, mode, treeop0, treeop1, op0, op1,
10593 172550 : target, unsignedp);
10594 :
10595 31200 : case RDIV_EXPR:
10596 31200 : goto binop;
10597 :
10598 1771 : case MULT_HIGHPART_EXPR:
10599 1771 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10600 1771 : temp = expand_mult_highpart (mode, op0, op1, target, unsignedp);
10601 1771 : gcc_assert (temp);
10602 : return temp;
10603 :
10604 0 : case FIXED_CONVERT_EXPR:
10605 0 : op0 = expand_normal (treeop0);
10606 0 : if (target == 0 || modifier == EXPAND_STACK_PARM)
10607 0 : target = gen_reg_rtx (mode);
10608 :
10609 0 : if ((TREE_CODE (TREE_TYPE (treeop0)) == INTEGER_TYPE
10610 0 : && TYPE_UNSIGNED (TREE_TYPE (treeop0)))
10611 0 : || (TREE_CODE (type) == INTEGER_TYPE && TYPE_UNSIGNED (type)))
10612 0 : expand_fixed_convert (target, op0, 1, TYPE_SATURATING (type));
10613 : else
10614 0 : expand_fixed_convert (target, op0, 0, TYPE_SATURATING (type));
10615 : return target;
10616 :
10617 48095 : case FIX_TRUNC_EXPR:
10618 48095 : op0 = expand_normal (treeop0);
10619 48095 : if (target == 0 || modifier == EXPAND_STACK_PARM)
10620 3202 : target = gen_reg_rtx (mode);
10621 48095 : expand_fix (target, op0, unsignedp);
10622 48095 : return target;
10623 :
10624 133489 : case FLOAT_EXPR:
10625 133489 : op0 = expand_normal (treeop0);
10626 133489 : if (target == 0 || modifier == EXPAND_STACK_PARM)
10627 63033 : target = gen_reg_rtx (mode);
10628 : /* expand_float can't figure out what to do if FROM has VOIDmode.
10629 : So give it the correct mode. With -O, cse will optimize this. */
10630 133489 : if (GET_MODE (op0) == VOIDmode)
10631 109 : op0 = copy_to_mode_reg (TYPE_MODE (TREE_TYPE (treeop0)),
10632 : op0);
10633 266978 : expand_float (target, op0,
10634 133489 : TYPE_UNSIGNED (TREE_TYPE (treeop0)));
10635 133489 : return target;
10636 :
10637 59546 : case NEGATE_EXPR:
10638 59546 : op0 = expand_expr (treeop0, subtarget,
10639 : VOIDmode, EXPAND_NORMAL);
10640 59546 : if (modifier == EXPAND_STACK_PARM)
10641 241 : target = 0;
10642 59546 : temp = expand_unop (mode,
10643 : optab_for_tree_code (NEGATE_EXPR, type,
10644 : optab_default),
10645 : op0, target, 0);
10646 59546 : gcc_assert (temp);
10647 59546 : return REDUCE_BIT_FIELD (temp);
10648 :
10649 27252 : case ABS_EXPR:
10650 27252 : case ABSU_EXPR:
10651 27252 : op0 = expand_expr (treeop0, subtarget,
10652 : VOIDmode, EXPAND_NORMAL);
10653 27252 : if (modifier == EXPAND_STACK_PARM)
10654 63 : target = 0;
10655 :
10656 : /* ABS_EXPR is not valid for complex arguments. */
10657 27252 : gcc_assert (GET_MODE_CLASS (mode) != MODE_COMPLEX_INT
10658 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_FLOAT);
10659 :
10660 : /* Unsigned abs is simply the operand. Testing here means we don't
10661 : risk generating incorrect code below. */
10662 27252 : if (TYPE_UNSIGNED (TREE_TYPE (treeop0)))
10663 : return op0;
10664 :
10665 27252 : return expand_abs (mode, op0, target, unsignedp,
10666 54504 : safe_from_p (target, treeop0, 1));
10667 :
10668 134552 : case MAX_EXPR:
10669 134552 : case MIN_EXPR:
10670 134552 : target = original_target;
10671 134552 : if (target == 0
10672 134552 : || modifier == EXPAND_STACK_PARM
10673 69985 : || (MEM_P (target) && MEM_VOLATILE_P (target))
10674 69985 : || GET_MODE (target) != mode
10675 204537 : || (REG_P (target)
10676 63376 : && REGNO (target) < FIRST_PSEUDO_REGISTER))
10677 64567 : target = gen_reg_rtx (mode);
10678 134552 : expand_operands (treeop0, treeop1,
10679 : target, &op0, &op1, EXPAND_NORMAL);
10680 :
10681 : /* First try to do it with a special MIN or MAX instruction.
10682 : If that does not win, use a conditional jump to select the proper
10683 : value. */
10684 134552 : this_optab = optab_for_tree_code (code, type, optab_default);
10685 134552 : temp = expand_binop (mode, this_optab, op0, op1, target, unsignedp,
10686 : OPTAB_WIDEN);
10687 134552 : if (temp != 0)
10688 : return temp;
10689 :
10690 122 : if (VECTOR_TYPE_P (type))
10691 0 : gcc_unreachable ();
10692 :
10693 : /* At this point, a MEM target is no longer useful; we will get better
10694 : code without it. */
10695 :
10696 122 : if (! REG_P (target))
10697 1 : target = gen_reg_rtx (mode);
10698 :
10699 : /* If op1 was placed in target, swap op0 and op1. */
10700 122 : if (target != op0 && target == op1)
10701 0 : std::swap (op0, op1);
10702 :
10703 : /* We generate better code and avoid problems with op1 mentioning
10704 : target by forcing op1 into a pseudo if it isn't a constant. */
10705 122 : if (! CONSTANT_P (op1))
10706 42 : op1 = force_reg (mode, op1);
10707 :
10708 122 : {
10709 122 : enum rtx_code comparison_code;
10710 122 : rtx cmpop1 = op1;
10711 :
10712 122 : if (code == MAX_EXPR)
10713 62 : comparison_code = unsignedp ? GEU : GE;
10714 : else
10715 60 : comparison_code = unsignedp ? LEU : LE;
10716 :
10717 : /* Canonicalize to comparisons against 0. */
10718 122 : if (op1 == const1_rtx)
10719 : {
10720 : /* Converting (a >= 1 ? a : 1) into (a > 0 ? a : 1)
10721 : or (a != 0 ? a : 1) for unsigned.
10722 : For MIN we are safe converting (a <= 1 ? a : 1)
10723 : into (a <= 0 ? a : 1) */
10724 0 : cmpop1 = const0_rtx;
10725 0 : if (code == MAX_EXPR)
10726 0 : comparison_code = unsignedp ? NE : GT;
10727 : }
10728 122 : if (op1 == constm1_rtx && !unsignedp)
10729 : {
10730 : /* Converting (a >= -1 ? a : -1) into (a >= 0 ? a : -1)
10731 : and (a <= -1 ? a : -1) into (a < 0 ? a : -1) */
10732 0 : cmpop1 = const0_rtx;
10733 0 : if (code == MIN_EXPR)
10734 0 : comparison_code = LT;
10735 : }
10736 :
10737 : /* Use a conditional move if possible. */
10738 122 : if (can_conditionally_move_p (mode))
10739 : {
10740 75 : rtx insn;
10741 :
10742 75 : start_sequence ();
10743 :
10744 : /* Try to emit the conditional move. */
10745 75 : insn = emit_conditional_move (target,
10746 : { comparison_code,
10747 : op0, cmpop1, mode },
10748 : op0, op1, mode,
10749 : unsignedp);
10750 :
10751 : /* If we could do the conditional move, emit the sequence,
10752 : and return. */
10753 75 : if (insn)
10754 : {
10755 41 : rtx_insn *seq = end_sequence ();
10756 41 : emit_insn (seq);
10757 41 : return target;
10758 : }
10759 :
10760 : /* Otherwise discard the sequence and fall back to code with
10761 : branches. */
10762 34 : end_sequence ();
10763 : }
10764 :
10765 81 : if (target != op0)
10766 52 : emit_move_insn (target, op0);
10767 :
10768 81 : lab = gen_label_rtx ();
10769 81 : do_compare_rtx_and_jump (target, cmpop1, comparison_code,
10770 : unsignedp, mode, NULL_RTX, NULL, lab,
10771 : profile_probability::uninitialized ());
10772 : }
10773 81 : emit_move_insn (target, op1);
10774 81 : emit_label (lab);
10775 81 : return target;
10776 :
10777 62395 : case BIT_NOT_EXPR:
10778 62395 : op0 = expand_expr (treeop0, subtarget,
10779 : VOIDmode, EXPAND_NORMAL);
10780 62395 : if (modifier == EXPAND_STACK_PARM)
10781 88 : target = 0;
10782 : /* In case we have to reduce the result to bitfield precision
10783 : for unsigned bitfield expand this as XOR with a proper constant
10784 : instead. */
10785 62395 : if (reduce_bit_field && TYPE_UNSIGNED (type))
10786 : {
10787 22989 : int_mode = SCALAR_INT_TYPE_MODE (type);
10788 22989 : wide_int mask = wi::mask (TYPE_PRECISION (type),
10789 45978 : false, GET_MODE_PRECISION (int_mode));
10790 :
10791 45978 : temp = expand_binop (int_mode, xor_optab, op0,
10792 45978 : immed_wide_int_const (mask, int_mode),
10793 : target, 1, OPTAB_LIB_WIDEN);
10794 22989 : }
10795 : else
10796 39406 : temp = expand_unop (mode, one_cmpl_optab, op0, target, 1);
10797 62395 : gcc_assert (temp);
10798 : return temp;
10799 :
10800 : /* ??? Can optimize bitwise operations with one arg constant.
10801 : Can optimize (a bitwise1 n) bitwise2 (a bitwise3 b)
10802 : and (a bitwise1 b) bitwise2 b (etc)
10803 : but that is probably not worth while. */
10804 :
10805 615510 : case BIT_AND_EXPR:
10806 615510 : case BIT_IOR_EXPR:
10807 615510 : case BIT_XOR_EXPR:
10808 615510 : goto binop;
10809 :
10810 8423 : case LROTATE_EXPR:
10811 8423 : case RROTATE_EXPR:
10812 8423 : gcc_assert (VECTOR_MODE_P (TYPE_MODE (type))
10813 : || type_has_mode_precision_p (type));
10814 : /* fall through */
10815 :
10816 293034 : case LSHIFT_EXPR:
10817 293034 : case RSHIFT_EXPR:
10818 293034 : {
10819 : /* If this is a fixed-point operation, then we cannot use the code
10820 : below because "expand_shift" doesn't support sat/no-sat fixed-point
10821 : shifts. */
10822 293034 : if (ALL_FIXED_POINT_MODE_P (mode))
10823 0 : goto binop;
10824 :
10825 293034 : if (! safe_from_p (subtarget, treeop1, 1))
10826 4972 : subtarget = 0;
10827 293034 : if (modifier == EXPAND_STACK_PARM)
10828 2580 : target = 0;
10829 293034 : op0 = expand_expr (treeop0, subtarget,
10830 : VOIDmode, EXPAND_NORMAL);
10831 :
10832 : /* Left shift optimization when shifting across word_size boundary.
10833 :
10834 : If mode == GET_MODE_WIDER_MODE (word_mode), then normally
10835 : there isn't native instruction to support this wide mode
10836 : left shift. Given below scenario:
10837 :
10838 : Type A = (Type) B << C
10839 :
10840 : |< T >|
10841 : | dest_high | dest_low |
10842 :
10843 : | word_size |
10844 :
10845 : If the shift amount C caused we shift B to across the word
10846 : size boundary, i.e part of B shifted into high half of
10847 : destination register, and part of B remains in the low
10848 : half, then GCC will use the following left shift expand
10849 : logic:
10850 :
10851 : 1. Initialize dest_low to B.
10852 : 2. Initialize every bit of dest_high to the sign bit of B.
10853 : 3. Logic left shift dest_low by C bit to finalize dest_low.
10854 : The value of dest_low before this shift is kept in a temp D.
10855 : 4. Logic left shift dest_high by C.
10856 : 5. Logic right shift D by (word_size - C).
10857 : 6. Or the result of 4 and 5 to finalize dest_high.
10858 :
10859 : While, by checking gimple statements, if operand B is
10860 : coming from signed extension, then we can simplify above
10861 : expand logic into:
10862 :
10863 : 1. dest_high = src_low >> (word_size - C).
10864 : 2. dest_low = src_low << C.
10865 :
10866 : We can use one arithmetic right shift to finish all the
10867 : purpose of steps 2, 4, 5, 6, thus we reduce the steps
10868 : needed from 6 into 2.
10869 :
10870 : The case is similar for zero extension, except that we
10871 : initialize dest_high to zero rather than copies of the sign
10872 : bit from B. Furthermore, we need to use a logical right shift
10873 : in this case.
10874 :
10875 : The choice of sign-extension versus zero-extension is
10876 : determined entirely by whether or not B is signed and is
10877 : independent of the current setting of unsignedp. */
10878 :
10879 293034 : temp = NULL_RTX;
10880 293034 : if (code == LSHIFT_EXPR
10881 293034 : && target
10882 33144 : && REG_P (target)
10883 323153 : && GET_MODE_2XWIDER_MODE (word_mode).exists (&int_mode)
10884 30186 : && mode == int_mode
10885 1491 : && TREE_CONSTANT (treeop1)
10886 293865 : && TREE_CODE (treeop0) == SSA_NAME)
10887 : {
10888 831 : gimple *def = SSA_NAME_DEF_STMT (treeop0);
10889 831 : if (is_gimple_assign (def)
10890 831 : && gimple_assign_rhs_code (def) == NOP_EXPR)
10891 : {
10892 327 : scalar_int_mode rmode = SCALAR_INT_TYPE_MODE
10893 : (TREE_TYPE (gimple_assign_rhs1 (def)));
10894 :
10895 654 : if (GET_MODE_SIZE (rmode) < GET_MODE_SIZE (int_mode)
10896 608 : && TREE_INT_CST_LOW (treeop1) < GET_MODE_BITSIZE (word_mode)
10897 469 : && ((TREE_INT_CST_LOW (treeop1) + GET_MODE_BITSIZE (rmode))
10898 71 : >= GET_MODE_BITSIZE (word_mode)))
10899 : {
10900 67 : rtx_insn *seq, *seq_old;
10901 67 : poly_uint64 high_off = subreg_highpart_offset (word_mode,
10902 : int_mode);
10903 67 : bool extend_unsigned
10904 67 : = TYPE_UNSIGNED (TREE_TYPE (gimple_assign_rhs1 (def)));
10905 67 : rtx low = lowpart_subreg (word_mode, op0, int_mode);
10906 67 : rtx dest_low = lowpart_subreg (word_mode, target, int_mode);
10907 67 : rtx dest_high = simplify_gen_subreg (word_mode, target,
10908 : int_mode, high_off);
10909 67 : HOST_WIDE_INT ramount = (BITS_PER_WORD
10910 67 : - TREE_INT_CST_LOW (treeop1));
10911 67 : tree rshift = build_int_cst (TREE_TYPE (treeop1), ramount);
10912 :
10913 67 : start_sequence ();
10914 : /* dest_high = src_low >> (word_size - C). */
10915 67 : temp = expand_variable_shift (RSHIFT_EXPR, word_mode, low,
10916 : rshift, dest_high,
10917 : extend_unsigned);
10918 67 : if (temp != dest_high)
10919 0 : emit_move_insn (dest_high, temp);
10920 :
10921 : /* dest_low = src_low << C. */
10922 67 : temp = expand_variable_shift (LSHIFT_EXPR, word_mode, low,
10923 : treeop1, dest_low, unsignedp);
10924 67 : if (temp != dest_low)
10925 2 : emit_move_insn (dest_low, temp);
10926 :
10927 67 : seq = end_sequence ();
10928 67 : temp = target ;
10929 :
10930 67 : if (have_insn_for (ASHIFT, int_mode))
10931 : {
10932 67 : bool speed_p = optimize_insn_for_speed_p ();
10933 67 : start_sequence ();
10934 67 : rtx ret_old = expand_variable_shift (code, int_mode,
10935 : op0, treeop1,
10936 : target,
10937 : unsignedp);
10938 :
10939 67 : seq_old = end_sequence ();
10940 134 : if (seq_cost (seq, speed_p)
10941 67 : >= seq_cost (seq_old, speed_p))
10942 : {
10943 67 : seq = seq_old;
10944 67 : temp = ret_old;
10945 : }
10946 : }
10947 67 : emit_insn (seq);
10948 : }
10949 : }
10950 : }
10951 :
10952 67 : if (temp == NULL_RTX)
10953 292967 : temp = expand_variable_shift (code, mode, op0, treeop1, target,
10954 : unsignedp);
10955 293034 : if (code == LSHIFT_EXPR)
10956 77991 : temp = REDUCE_BIT_FIELD (temp);
10957 : return temp;
10958 : }
10959 :
10960 : /* Could determine the answer when only additive constants differ. Also,
10961 : the addition of one can be handled by changing the condition. */
10962 637470 : case LT_EXPR:
10963 637470 : case LE_EXPR:
10964 637470 : case GT_EXPR:
10965 637470 : case GE_EXPR:
10966 637470 : case EQ_EXPR:
10967 637470 : case NE_EXPR:
10968 637470 : case UNORDERED_EXPR:
10969 637470 : case ORDERED_EXPR:
10970 637470 : case UNLT_EXPR:
10971 637470 : case UNLE_EXPR:
10972 637470 : case UNGT_EXPR:
10973 637470 : case UNGE_EXPR:
10974 637470 : case UNEQ_EXPR:
10975 637470 : case LTGT_EXPR:
10976 637470 : {
10977 1048189 : temp = do_store_flag (ops,
10978 : modifier != EXPAND_STACK_PARM ? target : NULL_RTX,
10979 : tmode != VOIDmode ? tmode : mode);
10980 637470 : if (temp)
10981 : return temp;
10982 :
10983 : /* Use a compare and a jump for BLKmode comparisons, or for function
10984 : type comparisons is have_canonicalize_funcptr_for_compare. */
10985 :
10986 0 : if ((target == 0
10987 0 : || modifier == EXPAND_STACK_PARM
10988 0 : || ! safe_from_p (target, treeop0, 1)
10989 0 : || ! safe_from_p (target, treeop1, 1)
10990 : /* Make sure we don't have a hard reg (such as function's return
10991 : value) live across basic blocks, if not optimizing. */
10992 0 : || (!optimize && REG_P (target)
10993 0 : && REGNO (target) < FIRST_PSEUDO_REGISTER)))
10994 0 : target = gen_reg_rtx (tmode != VOIDmode ? tmode : mode);
10995 :
10996 0 : emit_move_insn (target, const0_rtx);
10997 :
10998 0 : rtx_code_label *lab1 = gen_label_rtx ();
10999 0 : jumpifnot_1 (code, treeop0, treeop1, lab1,
11000 : profile_probability::uninitialized ());
11001 :
11002 0 : if (TYPE_PRECISION (type) == 1 && !TYPE_UNSIGNED (type))
11003 0 : emit_move_insn (target, constm1_rtx);
11004 : else
11005 0 : emit_move_insn (target, const1_rtx);
11006 :
11007 0 : emit_label (lab1);
11008 0 : return target;
11009 : }
11010 54928 : case COMPLEX_EXPR:
11011 : /* Get the rtx code of the operands. */
11012 54928 : op0 = expand_normal (treeop0);
11013 54928 : op1 = expand_normal (treeop1);
11014 :
11015 54928 : if (!target)
11016 3097 : target = gen_reg_rtx (TYPE_MODE (type));
11017 : else
11018 : /* If target overlaps with op1, then either we need to force
11019 : op1 into a pseudo (if target also overlaps with op0),
11020 : or write the complex parts in reverse order. */
11021 51831 : switch (GET_CODE (target))
11022 : {
11023 49077 : case CONCAT:
11024 49077 : if (reg_overlap_mentioned_p (XEXP (target, 0), op1))
11025 : {
11026 0 : if (reg_overlap_mentioned_p (XEXP (target, 1), op0))
11027 : {
11028 0 : complex_expr_force_op1:
11029 1724 : temp = gen_reg_rtx (GET_MODE_INNER (GET_MODE (target)));
11030 862 : emit_move_insn (temp, op1);
11031 862 : op1 = temp;
11032 862 : break;
11033 : }
11034 0 : complex_expr_swap_order:
11035 : /* Move the imaginary (op1) and real (op0) parts to their
11036 : location. */
11037 1 : write_complex_part (target, op1, true, true);
11038 1 : write_complex_part (target, op0, false, false);
11039 :
11040 1 : return target;
11041 : }
11042 : break;
11043 2754 : case MEM:
11044 5508 : temp = adjust_address_nv (target,
11045 : GET_MODE_INNER (GET_MODE (target)), 0);
11046 2754 : if (reg_overlap_mentioned_p (temp, op1))
11047 : {
11048 863 : scalar_mode imode = GET_MODE_INNER (GET_MODE (target));
11049 1726 : temp = adjust_address_nv (target, imode,
11050 : GET_MODE_SIZE (imode));
11051 863 : if (reg_overlap_mentioned_p (temp, op0))
11052 862 : goto complex_expr_force_op1;
11053 1 : goto complex_expr_swap_order;
11054 : }
11055 : break;
11056 0 : default:
11057 0 : if (reg_overlap_mentioned_p (target, op1))
11058 : {
11059 0 : if (reg_overlap_mentioned_p (target, op0))
11060 0 : goto complex_expr_force_op1;
11061 0 : goto complex_expr_swap_order;
11062 : }
11063 : break;
11064 : }
11065 :
11066 : /* Move the real (op0) and imaginary (op1) parts to their location. */
11067 54927 : write_complex_part (target, op0, false, true);
11068 54927 : write_complex_part (target, op1, true, false);
11069 :
11070 54927 : return target;
11071 :
11072 0 : case WIDEN_SUM_EXPR:
11073 0 : {
11074 0 : tree oprnd0 = treeop0;
11075 0 : tree oprnd1 = treeop1;
11076 :
11077 0 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11078 0 : target = expand_widen_pattern_expr (ops, op0, NULL_RTX, op1,
11079 : target, unsignedp);
11080 0 : return target;
11081 : }
11082 :
11083 18166 : case VEC_UNPACK_HI_EXPR:
11084 18166 : case VEC_UNPACK_LO_EXPR:
11085 18166 : case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
11086 18166 : case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
11087 18166 : {
11088 18166 : op0 = expand_normal (treeop0);
11089 18166 : temp = expand_widen_pattern_expr (ops, op0, NULL_RTX, NULL_RTX,
11090 : target, unsignedp);
11091 18166 : gcc_assert (temp);
11092 : return temp;
11093 : }
11094 :
11095 1754 : case VEC_UNPACK_FLOAT_HI_EXPR:
11096 1754 : case VEC_UNPACK_FLOAT_LO_EXPR:
11097 1754 : {
11098 1754 : op0 = expand_normal (treeop0);
11099 : /* The signedness is determined from input operand. */
11100 1754 : temp = expand_widen_pattern_expr
11101 3508 : (ops, op0, NULL_RTX, NULL_RTX,
11102 1754 : target, TYPE_UNSIGNED (TREE_TYPE (treeop0)));
11103 :
11104 1754 : gcc_assert (temp);
11105 : return temp;
11106 : }
11107 :
11108 993 : case VEC_WIDEN_MULT_HI_EXPR:
11109 993 : case VEC_WIDEN_MULT_LO_EXPR:
11110 993 : case VEC_WIDEN_MULT_EVEN_EXPR:
11111 993 : case VEC_WIDEN_MULT_ODD_EXPR:
11112 993 : case VEC_WIDEN_LSHIFT_HI_EXPR:
11113 993 : case VEC_WIDEN_LSHIFT_LO_EXPR:
11114 993 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11115 993 : target = expand_widen_pattern_expr (ops, op0, op1, NULL_RTX,
11116 : target, unsignedp);
11117 993 : gcc_assert (target);
11118 : return target;
11119 :
11120 362 : case VEC_PACK_SAT_EXPR:
11121 362 : case VEC_PACK_FIX_TRUNC_EXPR:
11122 362 : mode = TYPE_MODE (TREE_TYPE (treeop0));
11123 362 : subtarget = NULL_RTX;
11124 362 : goto binop;
11125 :
11126 11035 : case VEC_PACK_TRUNC_EXPR:
11127 11035 : if (VECTOR_BOOLEAN_TYPE_P (type)
11128 2481 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (treeop0))
11129 2481 : && mode == TYPE_MODE (TREE_TYPE (treeop0))
11130 11739 : && SCALAR_INT_MODE_P (mode))
11131 : {
11132 704 : class expand_operand eops[4];
11133 704 : machine_mode imode = TYPE_MODE (TREE_TYPE (treeop0));
11134 704 : expand_operands (treeop0, treeop1,
11135 : subtarget, &op0, &op1, EXPAND_NORMAL);
11136 704 : this_optab = vec_pack_sbool_trunc_optab;
11137 704 : enum insn_code icode = optab_handler (this_optab, imode);
11138 704 : create_output_operand (&eops[0], target, mode);
11139 704 : create_convert_operand_from (&eops[1], op0, imode, false);
11140 704 : create_convert_operand_from (&eops[2], op1, imode, false);
11141 704 : temp = GEN_INT (TYPE_VECTOR_SUBPARTS (type).to_constant ());
11142 704 : create_input_operand (&eops[3], temp, imode);
11143 704 : expand_insn (icode, 4, eops);
11144 704 : return eops[0].value;
11145 : }
11146 10331 : mode = TYPE_MODE (TREE_TYPE (treeop0));
11147 10331 : subtarget = NULL_RTX;
11148 10331 : goto binop;
11149 :
11150 27 : case VEC_PACK_FLOAT_EXPR:
11151 27 : mode = TYPE_MODE (TREE_TYPE (treeop0));
11152 27 : expand_operands (treeop0, treeop1,
11153 : subtarget, &op0, &op1, EXPAND_NORMAL);
11154 27 : this_optab = optab_for_tree_code (code, TREE_TYPE (treeop0),
11155 : optab_default);
11156 81 : target = expand_binop (mode, this_optab, op0, op1, target,
11157 27 : TYPE_UNSIGNED (TREE_TYPE (treeop0)),
11158 : OPTAB_LIB_WIDEN);
11159 27 : gcc_assert (target);
11160 : return target;
11161 :
11162 75233 : case VEC_PERM_EXPR:
11163 75233 : {
11164 75233 : expand_operands (treeop0, treeop1, target, &op0, &op1, EXPAND_NORMAL);
11165 75233 : vec_perm_builder sel;
11166 75233 : if (TREE_CODE (treeop2) == VECTOR_CST
11167 75233 : && tree_to_vec_perm_builder (&sel, treeop2))
11168 : {
11169 75215 : machine_mode sel_mode = TYPE_MODE (TREE_TYPE (treeop2));
11170 75215 : temp = expand_vec_perm_const (mode, op0, op1, sel,
11171 : sel_mode, target);
11172 : }
11173 : else
11174 : {
11175 18 : op2 = expand_normal (treeop2);
11176 18 : temp = expand_vec_perm_var (mode, op0, op1, op2, target);
11177 : }
11178 75233 : gcc_assert (temp);
11179 75233 : return temp;
11180 75233 : }
11181 :
11182 429 : case DOT_PROD_EXPR:
11183 429 : {
11184 429 : tree oprnd0 = treeop0;
11185 429 : tree oprnd1 = treeop1;
11186 429 : tree oprnd2 = treeop2;
11187 :
11188 429 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11189 429 : op2 = expand_normal (oprnd2);
11190 429 : target = expand_widen_pattern_expr (ops, op0, op1, op2,
11191 : target, unsignedp);
11192 429 : return target;
11193 : }
11194 :
11195 113 : case SAD_EXPR:
11196 113 : {
11197 113 : tree oprnd0 = treeop0;
11198 113 : tree oprnd1 = treeop1;
11199 113 : tree oprnd2 = treeop2;
11200 :
11201 113 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11202 113 : op2 = expand_normal (oprnd2);
11203 113 : target = expand_widen_pattern_expr (ops, op0, op1, op2,
11204 : target, unsignedp);
11205 113 : return target;
11206 : }
11207 :
11208 0 : case REALIGN_LOAD_EXPR:
11209 0 : {
11210 0 : tree oprnd0 = treeop0;
11211 0 : tree oprnd1 = treeop1;
11212 0 : tree oprnd2 = treeop2;
11213 :
11214 0 : this_optab = optab_for_tree_code (code, type, optab_default);
11215 0 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11216 0 : op2 = expand_normal (oprnd2);
11217 0 : temp = expand_ternary_op (mode, this_optab, op0, op1, op2,
11218 : target, unsignedp);
11219 0 : gcc_assert (temp);
11220 : return temp;
11221 : }
11222 :
11223 18080 : case COND_EXPR:
11224 18080 : {
11225 : /* A COND_EXPR with its type being VOID_TYPE represents a
11226 : conditional jump and is handled in
11227 : expand_gimple_cond_expr. */
11228 18080 : gcc_assert (!VOID_TYPE_P (type));
11229 :
11230 : /* Note that COND_EXPRs whose type is a structure or union
11231 : are required to be constructed to contain assignments of
11232 : a temporary variable, so that we can evaluate them here
11233 : for side effect only. If type is void, we must do likewise. */
11234 :
11235 18080 : gcc_assert (!TREE_ADDRESSABLE (type)
11236 : && !ignore
11237 : && TREE_TYPE (treeop1) != void_type_node
11238 : && TREE_TYPE (treeop2) != void_type_node);
11239 :
11240 18080 : temp = expand_cond_expr_using_cmove (treeop0, treeop1, treeop2);
11241 18080 : if (temp)
11242 : return temp;
11243 :
11244 : /* If we are not to produce a result, we have no target. Otherwise,
11245 : if a target was specified use it; it will not be used as an
11246 : intermediate target unless it is safe. If no target, use a
11247 : temporary. */
11248 :
11249 3860 : if (modifier != EXPAND_STACK_PARM
11250 3860 : && original_target
11251 2285 : && safe_from_p (original_target, treeop0, 1)
11252 0 : && GET_MODE (original_target) == mode
11253 3860 : && !MEM_P (original_target))
11254 : temp = original_target;
11255 : else
11256 3860 : temp = assign_temp (type, 0, 1);
11257 :
11258 3860 : do_pending_stack_adjust ();
11259 3860 : NO_DEFER_POP;
11260 3860 : rtx_code_label *lab0 = gen_label_rtx ();
11261 3860 : rtx_code_label *lab1 = gen_label_rtx ();
11262 3860 : jumpifnot (treeop0, lab0,
11263 : profile_probability::uninitialized ());
11264 3860 : store_expr (treeop1, temp,
11265 : modifier == EXPAND_STACK_PARM,
11266 : false, false);
11267 :
11268 3860 : emit_jump_insn (targetm.gen_jump (lab1));
11269 3860 : emit_barrier ();
11270 3860 : emit_label (lab0);
11271 3860 : store_expr (treeop2, temp,
11272 : modifier == EXPAND_STACK_PARM,
11273 : false, false);
11274 :
11275 3860 : emit_label (lab1);
11276 3860 : OK_DEFER_POP;
11277 3860 : return temp;
11278 : }
11279 :
11280 0 : case VEC_DUPLICATE_EXPR:
11281 0 : op0 = expand_expr (treeop0, NULL_RTX, VOIDmode, modifier);
11282 0 : target = expand_vector_broadcast (mode, op0);
11283 0 : gcc_assert (target);
11284 : return target;
11285 :
11286 0 : case VEC_SERIES_EXPR:
11287 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, modifier);
11288 0 : return expand_vec_series_expr (mode, op0, op1, target);
11289 :
11290 977 : case BIT_INSERT_EXPR:
11291 977 : {
11292 977 : unsigned bitpos = tree_to_uhwi (treeop2);
11293 977 : unsigned bitsize;
11294 977 : if (INTEGRAL_TYPE_P (TREE_TYPE (treeop1)))
11295 658 : bitsize = TYPE_PRECISION (TREE_TYPE (treeop1));
11296 : else
11297 319 : bitsize = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (treeop1)));
11298 977 : op0 = expand_normal (treeop0);
11299 977 : op1 = expand_normal (treeop1);
11300 977 : rtx dst = gen_reg_rtx (mode);
11301 977 : emit_move_insn (dst, op0);
11302 977 : store_bit_field (dst, bitsize, bitpos, 0, 0,
11303 977 : TYPE_MODE (TREE_TYPE (treeop1)), op1, false, false);
11304 977 : return dst;
11305 : }
11306 :
11307 0 : default:
11308 0 : gcc_unreachable ();
11309 : }
11310 :
11311 : /* Here to do an ordinary binary operator. */
11312 1194750 : binop:
11313 1194750 : expand_operands (treeop0, treeop1,
11314 : subtarget, &op0, &op1, EXPAND_NORMAL);
11315 5694575 : binop2:
11316 5694575 : this_optab = optab_for_tree_code (code, type, optab_default);
11317 5694575 : binop3:
11318 5694575 : if (modifier == EXPAND_STACK_PARM)
11319 27201 : target = 0;
11320 5694575 : temp = expand_binop (mode, this_optab, op0, op1, target,
11321 : unsignedp, OPTAB_LIB_WIDEN);
11322 5694575 : gcc_assert (temp);
11323 : /* Bitwise operations do not need bitfield reduction as we expect their
11324 : operands being properly truncated. */
11325 5694575 : if (code == BIT_XOR_EXPR
11326 : || code == BIT_AND_EXPR
11327 5694575 : || code == BIT_IOR_EXPR)
11328 : return temp;
11329 5079065 : return REDUCE_BIT_FIELD (temp);
11330 : }
11331 : #undef REDUCE_BIT_FIELD
11332 :
11333 :
11334 : /* Return TRUE if expression STMT is suitable for replacement.
11335 : Never consider memory loads as replaceable, because those don't ever lead
11336 : into constant expressions. */
11337 :
11338 : static bool
11339 8 : stmt_is_replaceable_p (gimple *stmt)
11340 : {
11341 8 : if (ssa_is_replaceable_p (stmt))
11342 : {
11343 : /* Don't move around loads. */
11344 7 : if (!gimple_assign_single_p (stmt)
11345 7 : || is_gimple_val (gimple_assign_rhs1 (stmt)))
11346 6 : return true;
11347 : }
11348 : return false;
11349 : }
11350 :
11351 : /* A subroutine of expand_expr_real_1. Expand gimple assignment G,
11352 : which is known to set an SSA_NAME result. The other arguments are
11353 : as for expand_expr_real_1. */
11354 :
11355 : rtx
11356 15111854 : expand_expr_real_gassign (gassign *g, rtx target, machine_mode tmode,
11357 : enum expand_modifier modifier, rtx *alt_rtl,
11358 : bool inner_reference_p)
11359 : {
11360 15111854 : separate_ops ops;
11361 15111854 : rtx r;
11362 15111854 : location_t saved_loc = curr_insn_location ();
11363 15111854 : auto loc = gimple_location (g);
11364 15111854 : if (loc != UNKNOWN_LOCATION)
11365 12092607 : set_curr_insn_location (loc);
11366 15111854 : tree lhs = gimple_assign_lhs (g);
11367 15111854 : ops.code = gimple_assign_rhs_code (g);
11368 15111854 : ops.type = TREE_TYPE (lhs);
11369 15111854 : switch (get_gimple_rhs_class (ops.code))
11370 : {
11371 94681 : case GIMPLE_TERNARY_RHS:
11372 189362 : ops.op2 = gimple_assign_rhs3 (g);
11373 : /* Fallthru */
11374 8113015 : case GIMPLE_BINARY_RHS:
11375 8113015 : ops.op1 = gimple_assign_rhs2 (g);
11376 :
11377 : /* Try to expand conditional compare. */
11378 8113015 : if (targetm.have_ccmp ())
11379 : {
11380 164948 : gcc_checking_assert (targetm.gen_ccmp_next != NULL);
11381 164948 : r = expand_ccmp_expr (g, TYPE_MODE (ops.type));
11382 164948 : if (r)
11383 : break;
11384 : }
11385 : /* Fallthru */
11386 11662061 : case GIMPLE_UNARY_RHS:
11387 11662061 : ops.op0 = gimple_assign_rhs1 (g);
11388 11662061 : ops.location = loc;
11389 11662061 : r = expand_expr_real_2 (&ops, target, tmode, modifier);
11390 11662061 : break;
11391 3449775 : case GIMPLE_SINGLE_RHS:
11392 3449775 : {
11393 3449775 : r = expand_expr_real (gimple_assign_rhs1 (g), target,
11394 : tmode, modifier, alt_rtl,
11395 : inner_reference_p);
11396 3449775 : break;
11397 : }
11398 0 : default:
11399 0 : gcc_unreachable ();
11400 : }
11401 15111854 : set_curr_insn_location (saved_loc);
11402 15111854 : if (REG_P (r) && !REG_EXPR (r))
11403 3960683 : set_reg_attrs_for_decl_rtl (lhs, r);
11404 15111854 : return r;
11405 : }
11406 :
11407 : /* A subroutine of expand_expr_real_1. Attempt to VIEW_CONVERT_EXPR
11408 : the complex expression OP0 to the vector mode MODE. Store the
11409 : result at TARGET if possible (if TARGET is nonzero). Returns
11410 : NULL_RTX on failure. */
11411 : static rtx
11412 6660 : try_expand_complex_as_vector (machine_mode mode, rtx op0, rtx target)
11413 : {
11414 6660 : if (COMPLEX_MODE_P (GET_MODE (op0))
11415 1404 : && VECTOR_MODE_P (mode)
11416 58 : && known_eq (GET_MODE_NUNITS (mode), 2)
11417 6718 : && GET_MODE_INNER (mode) == GET_MODE_INNER (GET_MODE (op0)))
11418 : {
11419 29 : enum insn_code icode = convert_optab_handler (vec_init_optab, mode,
11420 29 : GET_MODE_INNER (mode));
11421 29 : if (icode != CODE_FOR_nothing)
11422 : {
11423 29 : if (!target || !REG_P (target))
11424 16 : target = gen_reg_rtx (mode);
11425 29 : rtx rpart = read_complex_part (op0, false);
11426 29 : rtx ipart = read_complex_part (op0, true);
11427 29 : if (!REG_P (rpart) && !CONSTANT_P (rpart))
11428 0 : rpart = force_reg (GET_MODE_INNER (mode), rpart);
11429 29 : if (!REG_P (ipart) && !CONSTANT_P (ipart))
11430 0 : ipart = force_reg (GET_MODE_INNER (mode), ipart);
11431 29 : rtvec vec = rtvec_alloc (2);
11432 29 : RTVEC_ELT (vec, 0) = rpart;
11433 29 : RTVEC_ELT (vec, 1) = ipart;
11434 29 : rtx par = gen_rtx_PARALLEL (mode, vec);
11435 29 : rtx_insn *insn = GEN_FCN (icode) (target, par);
11436 29 : if (insn)
11437 : {
11438 29 : emit_insn (insn);
11439 29 : return target;
11440 : }
11441 : }
11442 : }
11443 : return NULL_RTX;
11444 : }
11445 :
11446 : rtx
11447 158141343 : expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
11448 : enum expand_modifier modifier, rtx *alt_rtl,
11449 : bool inner_reference_p)
11450 : {
11451 158141343 : rtx op0, op1, temp, decl_rtl;
11452 158141343 : tree type;
11453 158141343 : int unsignedp;
11454 158141343 : machine_mode mode, dmode;
11455 158141343 : enum tree_code code = TREE_CODE (exp);
11456 158141343 : rtx subtarget, original_target;
11457 158141343 : int ignore;
11458 158141343 : bool reduce_bit_field;
11459 158141343 : location_t loc = EXPR_LOCATION (exp);
11460 158141343 : struct separate_ops ops;
11461 158141343 : tree treeop0, treeop1, treeop2;
11462 158141343 : tree ssa_name = NULL_TREE;
11463 158141343 : gimple *g;
11464 :
11465 : /* Some ABIs define padding bits in _BitInt uninitialized. Normally, RTL
11466 : expansion sign/zero extends integral types with less than mode precision
11467 : when reading from bit-fields and after arithmetic operations (see
11468 : REDUCE_BIT_FIELD in expand_expr_real_2) and on subsequent loads relies
11469 : on those extensions to have been already performed, but because of the
11470 : above for _BitInt they need to be sign/zero extended when reading from
11471 : locations that could be exposed to ABI boundaries (when loading from
11472 : objects in memory, or function arguments, return value). Because we
11473 : internally extend after arithmetic operations, we can avoid doing that
11474 : when reading from SSA_NAMEs of vars. */
11475 : #define EXTEND_BITINT(expr) \
11476 : ((BITINT_TYPE_P (type) \
11477 : && !bitint_extended \
11478 : && reduce_bit_field \
11479 : && mode != BLKmode \
11480 : && modifier != EXPAND_MEMORY \
11481 : && modifier != EXPAND_WRITE \
11482 : && modifier != EXPAND_INITIALIZER \
11483 : && modifier != EXPAND_CONST_ADDRESS) \
11484 : ? reduce_to_bit_field_precision ((expr), NULL_RTX, type) : (expr))
11485 :
11486 158141343 : type = TREE_TYPE (exp);
11487 158141343 : mode = TYPE_MODE (type);
11488 158141343 : unsignedp = TYPE_UNSIGNED (type);
11489 158141343 : if (BITINT_TYPE_P (type) && bitint_extended == -1)
11490 : {
11491 8798 : struct bitint_info info;
11492 8798 : bool ok = targetm.c.bitint_type_info (TYPE_PRECISION (type), &info);
11493 8798 : gcc_assert (ok);
11494 8798 : bitint_extended = info.extended;
11495 : }
11496 :
11497 158141343 : treeop0 = treeop1 = treeop2 = NULL_TREE;
11498 158141343 : if (!VL_EXP_CLASS_P (exp))
11499 151369580 : switch (TREE_CODE_LENGTH (code))
11500 : {
11501 5545846 : default:
11502 5545846 : case 3: treeop2 = TREE_OPERAND (exp, 2); /* FALLTHRU */
11503 13601928 : case 2: treeop1 = TREE_OPERAND (exp, 1); /* FALLTHRU */
11504 28120335 : case 1: treeop0 = TREE_OPERAND (exp, 0); /* FALLTHRU */
11505 : case 0: break;
11506 : }
11507 158141343 : ops.code = code;
11508 158141343 : ops.type = type;
11509 158141343 : ops.op0 = treeop0;
11510 158141343 : ops.op1 = treeop1;
11511 158141343 : ops.op2 = treeop2;
11512 158141343 : ops.location = loc;
11513 :
11514 316282686 : ignore = (target == const0_rtx
11515 158141343 : || ((CONVERT_EXPR_CODE_P (code)
11516 153058261 : || code == COND_EXPR || code == VIEW_CONVERT_EXPR)
11517 956789 : && TREE_CODE (type) == VOID_TYPE));
11518 :
11519 : /* An operation in what may be a bit-field type needs the
11520 : result to be reduced to the precision of the bit-field type,
11521 : which is narrower than that of the type's mode. */
11522 316282450 : reduce_bit_field = (!ignore
11523 153667068 : && INTEGRAL_TYPE_P (type)
11524 80797869 : && !type_has_mode_precision_p (type));
11525 :
11526 : /* If we are going to ignore this result, we need only do something
11527 : if there is a side-effect somewhere in the expression. If there
11528 : is, short-circuit the most common cases here. Note that we must
11529 : not call expand_expr with anything but const0_rtx in case this
11530 : is an initial expansion of a size that contains a PLACEHOLDER_EXPR. */
11531 :
11532 4474275 : if (ignore)
11533 : {
11534 4474275 : if (! TREE_SIDE_EFFECTS (exp))
11535 : return const0_rtx;
11536 :
11537 : /* Ensure we reference a volatile object even if value is ignored, but
11538 : don't do this if all we are doing is taking its address. */
11539 4474039 : if (TREE_THIS_VOLATILE (exp)
11540 0 : && TREE_CODE (exp) != FUNCTION_DECL
11541 0 : && mode != VOIDmode && mode != BLKmode
11542 0 : && modifier != EXPAND_CONST_ADDRESS)
11543 : {
11544 0 : temp = expand_expr (exp, NULL_RTX, VOIDmode, modifier);
11545 0 : if (MEM_P (temp))
11546 0 : copy_to_reg (temp);
11547 0 : return const0_rtx;
11548 : }
11549 :
11550 4474039 : if (TREE_CODE_CLASS (code) == tcc_unary
11551 : || code == BIT_FIELD_REF
11552 4474039 : || code == COMPONENT_REF
11553 4474039 : || code == INDIRECT_REF)
11554 0 : return expand_expr (treeop0, const0_rtx, VOIDmode,
11555 0 : modifier);
11556 :
11557 4474039 : else if (TREE_CODE_CLASS (code) == tcc_binary
11558 4474039 : || TREE_CODE_CLASS (code) == tcc_comparison
11559 4474039 : || code == ARRAY_REF || code == ARRAY_RANGE_REF)
11560 : {
11561 0 : expand_expr (treeop0, const0_rtx, VOIDmode, modifier);
11562 0 : expand_expr (treeop1, const0_rtx, VOIDmode, modifier);
11563 0 : return const0_rtx;
11564 : }
11565 :
11566 : target = 0;
11567 : }
11568 :
11569 158141107 : if (reduce_bit_field && modifier == EXPAND_STACK_PARM)
11570 39464 : target = 0;
11571 :
11572 : /* Use subtarget as the target for operand 0 of a binary operation. */
11573 158141107 : subtarget = get_subtarget (target);
11574 158141107 : original_target = target;
11575 :
11576 158141107 : switch (code)
11577 : {
11578 10841 : case LABEL_DECL:
11579 10841 : {
11580 10841 : tree function = decl_function_context (exp);
11581 :
11582 10841 : temp = label_rtx (exp);
11583 14785 : temp = gen_rtx_LABEL_REF (Pmode, temp);
11584 :
11585 10841 : if (function != current_function_decl
11586 1270 : && function != 0)
11587 1270 : LABEL_REF_NONLOCAL_P (temp) = 1;
11588 :
11589 10841 : temp = gen_rtx_MEM (FUNCTION_MODE, temp);
11590 10841 : return temp;
11591 : }
11592 :
11593 57207075 : case SSA_NAME:
11594 : /* ??? ivopts calls expander, without any preparation from
11595 : out-of-ssa. So fake instructions as if this was an access to the
11596 : base variable. This unnecessarily allocates a pseudo, see how we can
11597 : reuse it, if partition base vars have it set already. */
11598 57207075 : if (!currently_expanding_to_rtl)
11599 : {
11600 0 : tree var = SSA_NAME_VAR (exp);
11601 0 : if (var && DECL_RTL_SET_P (var))
11602 0 : return DECL_RTL (var);
11603 0 : return gen_raw_REG (TYPE_MODE (TREE_TYPE (exp)),
11604 0 : LAST_VIRTUAL_REGISTER + 1);
11605 : }
11606 :
11607 57207075 : g = get_gimple_for_ssa_name (exp);
11608 : /* For EXPAND_INITIALIZER try harder to get something simpler. */
11609 57207075 : if (g == NULL
11610 57207075 : && modifier == EXPAND_INITIALIZER
11611 26 : && !SSA_NAME_IS_DEFAULT_DEF (exp)
11612 11 : && (optimize || !SSA_NAME_VAR (exp)
11613 1 : || DECL_IGNORED_P (SSA_NAME_VAR (exp)))
11614 10 : && is_gimple_assign (SSA_NAME_DEF_STMT (exp))
11615 57207083 : && stmt_is_replaceable_p (SSA_NAME_DEF_STMT (exp)))
11616 6 : g = SSA_NAME_DEF_STMT (exp);
11617 57207075 : if (safe_is_a <gassign *> (g))
11618 9126221 : return expand_expr_real_gassign (as_a<gassign *> (g), target, tmode,
11619 9126221 : modifier, alt_rtl, inner_reference_p);
11620 48080854 : else if (safe_is_a <gcall *> (g))
11621 : {
11622 : /* ??? internal call expansion doesn't follow the usual API
11623 : of returning the destination RTX and being passed a desired
11624 : target. */
11625 72886 : if (modifier == EXPAND_WRITE)
11626 36447 : return DECL_RTL (SSA_NAME_VAR (exp));
11627 36439 : rtx dest = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
11628 36439 : tree tmplhs = make_tree (TREE_TYPE (exp), dest);
11629 36439 : tree var_or_id = SSA_NAME_VAR (exp);
11630 : if (!var_or_id)
11631 26452 : var_or_id = SSA_NAME_IDENTIFIER (exp);
11632 36439 : SET_SSA_NAME_VAR_OR_IDENTIFIER (exp, tmplhs);
11633 36439 : expand_internal_call (as_a <gcall *> (g));
11634 36439 : SET_SSA_NAME_VAR_OR_IDENTIFIER (exp, var_or_id);
11635 36439 : return dest;
11636 : }
11637 :
11638 48007968 : ssa_name = exp;
11639 48007968 : decl_rtl = get_rtx_for_ssa_name (ssa_name);
11640 48007968 : exp = SSA_NAME_VAR (ssa_name);
11641 : /* Optimize and avoid to EXTEND_BITINIT doing anything if it is an
11642 : SSA_NAME computed within the current function. In such case the
11643 : value have been already extended before. While if it is a function
11644 : parameter, result or some memory location, we need to be prepared
11645 : for some other compiler leaving the bits uninitialized. */
11646 18812383 : if (!exp || VAR_P (exp))
11647 : reduce_bit_field = false;
11648 48007968 : goto expand_decl_rtl;
11649 :
11650 19780629 : case VAR_DECL:
11651 : /* Allow accel compiler to handle variables that require special
11652 : treatment, e.g. if they have been modified in some way earlier in
11653 : compilation by the adjust_private_decl OpenACC hook. */
11654 19780629 : if (flag_openacc && targetm.goacc.expand_var_decl)
11655 : {
11656 0 : temp = targetm.goacc.expand_var_decl (exp);
11657 0 : if (temp)
11658 : return temp;
11659 : }
11660 : /* Expand const VAR_DECLs with CONSTRUCTOR initializers that
11661 : have scalar integer modes to a reg via store_constructor. */
11662 19780629 : if (TREE_READONLY (exp)
11663 3376022 : && !TREE_SIDE_EFFECTS (exp)
11664 3353614 : && (modifier == EXPAND_NORMAL || modifier == EXPAND_STACK_PARM)
11665 18489 : && immediate_const_ctor_p (DECL_INITIAL (exp))
11666 163 : && SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (exp)))
11667 127 : && crtl->emit.regno_pointer_align_length
11668 19780756 : && !target)
11669 : {
11670 88 : target = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
11671 88 : store_constructor (DECL_INITIAL (exp), target, 0,
11672 88 : int_expr_size (DECL_INITIAL (exp)), false);
11673 88 : return target;
11674 : }
11675 : /* ... fall through ... */
11676 :
11677 20325377 : case PARM_DECL:
11678 : /* If a static var's type was incomplete when the decl was written,
11679 : but the type is complete now, lay out the decl now. */
11680 20325377 : if (DECL_SIZE (exp) == 0
11681 44930 : && COMPLETE_OR_UNBOUND_ARRAY_TYPE_P (TREE_TYPE (exp))
11682 20370195 : && (TREE_STATIC (exp) || DECL_EXTERNAL (exp)))
11683 44818 : layout_decl (exp, 0);
11684 :
11685 : /* fall through */
11686 :
11687 21791013 : case FUNCTION_DECL:
11688 21791013 : case RESULT_DECL:
11689 21791013 : decl_rtl = DECL_RTL (exp);
11690 69798981 : expand_decl_rtl:
11691 69798981 : gcc_assert (decl_rtl);
11692 :
11693 : /* DECL_MODE might change when TYPE_MODE depends on attribute target
11694 : settings for VECTOR_TYPE_P that might switch for the function. */
11695 69798981 : if (currently_expanding_to_rtl
11696 66121570 : && code == VAR_DECL && MEM_P (decl_rtl)
11697 84558896 : && VECTOR_TYPE_P (type) && exp && DECL_MODE (exp) != mode)
11698 57 : decl_rtl = change_address (decl_rtl, TYPE_MODE (type), 0);
11699 : else
11700 69798924 : decl_rtl = copy_rtx (decl_rtl);
11701 :
11702 : /* Record writes to register variables. */
11703 69798981 : if (modifier == EXPAND_WRITE
11704 21922982 : && REG_P (decl_rtl)
11705 85824782 : && HARD_REGISTER_P (decl_rtl))
11706 2027 : add_to_hard_reg_set (&crtl->asm_clobbers,
11707 2027 : GET_MODE (decl_rtl), REGNO (decl_rtl));
11708 :
11709 : /* Ensure variable marked as used even if it doesn't go through
11710 : a parser. If it hasn't be used yet, write out an external
11711 : definition. */
11712 69798981 : if (exp)
11713 40603396 : TREE_USED (exp) = 1;
11714 :
11715 : /* Show we haven't gotten RTL for this yet. */
11716 110402377 : temp = 0;
11717 :
11718 : /* Variables inherited from containing functions should have
11719 : been lowered by this point. */
11720 40603396 : if (exp)
11721 : {
11722 40603396 : tree context = decl_function_context (exp);
11723 40603396 : gcc_assert (SCOPE_FILE_SCOPE_P (context)
11724 : || context == current_function_decl
11725 : || TREE_STATIC (exp)
11726 : || DECL_EXTERNAL (exp)
11727 : /* ??? C++ creates functions that are not
11728 : TREE_STATIC. */
11729 : || TREE_CODE (exp) == FUNCTION_DECL);
11730 : }
11731 :
11732 : /* This is the case of an array whose size is to be determined
11733 : from its initializer, while the initializer is still being parsed.
11734 : ??? We aren't parsing while expanding anymore. */
11735 :
11736 69798981 : if (MEM_P (decl_rtl) && REG_P (XEXP (decl_rtl, 0)))
11737 393551 : temp = validize_mem (decl_rtl);
11738 :
11739 : /* If DECL_RTL is memory, we are in the normal case and the
11740 : address is not valid, get the address into a register. */
11741 :
11742 69405430 : else if (MEM_P (decl_rtl) && modifier != EXPAND_INITIALIZER)
11743 : {
11744 18833446 : if (alt_rtl)
11745 2027702 : *alt_rtl = decl_rtl;
11746 18833446 : decl_rtl = use_anchored_address (decl_rtl);
11747 18833446 : if (modifier != EXPAND_CONST_ADDRESS
11748 18833446 : && modifier != EXPAND_SUM
11749 31183378 : && !memory_address_addr_space_p (exp ? DECL_MODE (exp)
11750 17391 : : GET_MODE (decl_rtl),
11751 : XEXP (decl_rtl, 0),
11752 12349932 : MEM_ADDR_SPACE (decl_rtl)))
11753 137192 : temp = replace_equiv_address (decl_rtl,
11754 : copy_rtx (XEXP (decl_rtl, 0)));
11755 : }
11756 :
11757 : /* If we got something, return it. But first, set the alignment
11758 : if the address is a register. */
11759 19226997 : if (temp != 0)
11760 : {
11761 530743 : if (exp && MEM_P (temp) && REG_P (XEXP (temp, 0)))
11762 498763 : mark_reg_pointer (XEXP (temp, 0), DECL_ALIGN (exp));
11763 : }
11764 69268238 : else if (MEM_P (decl_rtl))
11765 : temp = decl_rtl;
11766 :
11767 47559695 : if (temp != 0)
11768 : {
11769 22738059 : if (MEM_P (temp)
11770 : && modifier != EXPAND_WRITE
11771 22738059 : && modifier != EXPAND_MEMORY
11772 : && modifier != EXPAND_INITIALIZER
11773 16778754 : && modifier != EXPAND_CONST_ADDRESS
11774 6948570 : && modifier != EXPAND_SUM
11775 6948570 : && !inner_reference_p
11776 4569627 : && mode != BLKmode
11777 26940596 : && MEM_ALIGN (temp) < GET_MODE_ALIGNMENT (mode))
11778 19902 : temp = expand_misaligned_mem_ref (temp, mode, unsignedp,
11779 19902 : MEM_ALIGN (temp), NULL_RTX, NULL);
11780 :
11781 22738059 : return EXTEND_BITINT (temp);
11782 : }
11783 :
11784 47060922 : if (exp)
11785 17882738 : dmode = DECL_MODE (exp);
11786 : else
11787 29178184 : dmode = TYPE_MODE (TREE_TYPE (ssa_name));
11788 :
11789 : /* If the mode of DECL_RTL does not match that of the decl,
11790 : there are two cases: we are dealing with a BLKmode value
11791 : that is returned in a register, or we are dealing with
11792 : a promoted value. In the latter case, return a SUBREG
11793 : of the wanted mode, but mark it so that we know that it
11794 : was already extended. */
11795 47060922 : if (REG_P (decl_rtl)
11796 46527498 : && dmode != BLKmode
11797 46527498 : && GET_MODE (decl_rtl) != dmode)
11798 : {
11799 82 : machine_mode pmode;
11800 :
11801 : /* Get the signedness to be used for this variable. Ensure we get
11802 : the same mode we got when the variable was declared. */
11803 82 : if (code != SSA_NAME)
11804 0 : pmode = promote_decl_mode (exp, &unsignedp);
11805 82 : else if ((g = SSA_NAME_DEF_STMT (ssa_name))
11806 82 : && gimple_code (g) == GIMPLE_CALL
11807 84 : && !gimple_call_internal_p (g))
11808 2 : pmode = promote_function_mode (type, mode, &unsignedp,
11809 2 : gimple_call_fntype (g),
11810 : 2);
11811 : else
11812 80 : pmode = promote_ssa_mode (ssa_name, &unsignedp);
11813 82 : gcc_assert (GET_MODE (decl_rtl) == pmode);
11814 :
11815 : /* Some ABIs require scalar floating point modes to be passed
11816 : in a wider scalar integer mode. We need to explicitly
11817 : truncate to an integer mode of the correct precision before
11818 : using a SUBREG to reinterpret as a floating point value. */
11819 82 : if (SCALAR_FLOAT_MODE_P (mode)
11820 0 : && SCALAR_INT_MODE_P (pmode)
11821 82 : && known_lt (GET_MODE_SIZE (mode), GET_MODE_SIZE (pmode)))
11822 0 : return convert_wider_int_to_float (mode, pmode, decl_rtl);
11823 :
11824 82 : temp = gen_lowpart_SUBREG (mode, decl_rtl);
11825 82 : SUBREG_PROMOTED_VAR_P (temp) = 1;
11826 82 : SUBREG_PROMOTED_SET (temp, unsignedp);
11827 82 : return EXTEND_BITINT (temp);
11828 : }
11829 :
11830 47060840 : return EXTEND_BITINT (decl_rtl);
11831 :
11832 42429593 : case INTEGER_CST:
11833 42429593 : {
11834 42429593 : if (BITINT_TYPE_P (type))
11835 : {
11836 11582 : unsigned int prec = TYPE_PRECISION (type);
11837 11582 : struct bitint_info info;
11838 11582 : bool ok = targetm.c.bitint_type_info (prec, &info);
11839 11582 : gcc_assert (ok);
11840 11582 : scalar_int_mode limb_mode
11841 11582 : = as_a <scalar_int_mode> (info.limb_mode);
11842 11582 : unsigned int limb_prec = GET_MODE_PRECISION (limb_mode);
11843 19154 : if (prec > limb_prec && prec > MAX_FIXED_MODE_SIZE)
11844 : {
11845 : /* Emit large/huge _BitInt INTEGER_CSTs into memory. */
11846 5113 : exp = tree_output_constant_def (exp);
11847 5113 : return expand_expr (exp, target, VOIDmode, modifier);
11848 : }
11849 : }
11850 :
11851 : /* Given that TYPE_PRECISION (type) is not always equal to
11852 : GET_MODE_PRECISION (TYPE_MODE (type)), we need to extend from
11853 : the former to the latter according to the signedness of the
11854 : type. */
11855 42424480 : scalar_int_mode int_mode = SCALAR_INT_TYPE_MODE (type);
11856 42424480 : temp = immed_wide_int_const
11857 42424480 : (wi::to_wide (exp, GET_MODE_PRECISION (int_mode)), int_mode);
11858 42424480 : return temp;
11859 : }
11860 :
11861 569403 : case VECTOR_CST:
11862 569403 : {
11863 569403 : tree tmp = NULL_TREE;
11864 569403 : if (VECTOR_MODE_P (mode))
11865 566604 : return const_vector_from_tree (exp);
11866 2799 : scalar_int_mode int_mode;
11867 2799 : if (is_int_mode (mode, &int_mode))
11868 : {
11869 911 : tree type_for_mode = lang_hooks.types.type_for_mode (int_mode, 1);
11870 911 : if (type_for_mode)
11871 911 : tmp = fold_unary_loc (loc, VIEW_CONVERT_EXPR,
11872 : type_for_mode, exp);
11873 : }
11874 911 : if (!tmp)
11875 : {
11876 1888 : vec<constructor_elt, va_gc> *v;
11877 : /* Constructors need to be fixed-length. FIXME. */
11878 1888 : unsigned int nunits = VECTOR_CST_NELTS (exp).to_constant ();
11879 1888 : vec_alloc (v, nunits);
11880 28739 : for (unsigned int i = 0; i < nunits; ++i)
11881 26851 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, VECTOR_CST_ELT (exp, i));
11882 1888 : tmp = build_constructor (type, v);
11883 : }
11884 2799 : return expand_expr (tmp, ignore ? const0_rtx : target,
11885 2799 : tmode, modifier);
11886 : }
11887 :
11888 141 : case CONST_DECL:
11889 141 : if (modifier == EXPAND_WRITE)
11890 : {
11891 : /* Writing into CONST_DECL is always invalid, but handle it
11892 : gracefully. */
11893 2 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (exp));
11894 2 : scalar_int_mode address_mode = targetm.addr_space.address_mode (as);
11895 2 : op0 = expand_expr_addr_expr_1 (exp, NULL_RTX, address_mode,
11896 : EXPAND_NORMAL, as);
11897 2 : op0 = memory_address_addr_space (mode, op0, as);
11898 2 : temp = gen_rtx_MEM (mode, op0);
11899 2 : set_mem_addr_space (temp, as);
11900 2 : return temp;
11901 : }
11902 139 : return expand_expr (DECL_INITIAL (exp), target, VOIDmode, modifier);
11903 :
11904 862901 : case REAL_CST:
11905 : /* If optimized, generate immediate CONST_DOUBLE
11906 : which will be turned into memory by reload if necessary.
11907 :
11908 : We used to force a register so that loop.c could see it. But
11909 : this does not allow gen_* patterns to perform optimizations with
11910 : the constants. It also produces two insns in cases like "x = 1.0;".
11911 : On most machines, floating-point constants are not permitted in
11912 : many insns, so we'd end up copying it to a register in any case.
11913 :
11914 : Now, we do the copying in expand_binop, if appropriate. */
11915 862901 : return const_double_from_real_value (TREE_REAL_CST (exp),
11916 1725802 : TYPE_MODE (TREE_TYPE (exp)));
11917 :
11918 0 : case FIXED_CST:
11919 0 : return CONST_FIXED_FROM_FIXED_VALUE (TREE_FIXED_CST (exp),
11920 : TYPE_MODE (TREE_TYPE (exp)));
11921 :
11922 18396 : case COMPLEX_CST:
11923 : /* Handle evaluating a complex constant in a CONCAT target. */
11924 18396 : if (original_target && GET_CODE (original_target) == CONCAT)
11925 : {
11926 219 : rtx rtarg, itarg;
11927 :
11928 219 : mode = TYPE_MODE (TREE_TYPE (TREE_TYPE (exp)));
11929 219 : rtarg = XEXP (original_target, 0);
11930 219 : itarg = XEXP (original_target, 1);
11931 :
11932 : /* Move the real and imaginary parts separately. */
11933 219 : op0 = expand_expr (TREE_REALPART (exp), rtarg, mode, EXPAND_NORMAL);
11934 219 : op1 = expand_expr (TREE_IMAGPART (exp), itarg, mode, EXPAND_NORMAL);
11935 :
11936 219 : if (op0 != rtarg)
11937 219 : emit_move_insn (rtarg, op0);
11938 219 : if (op1 != itarg)
11939 219 : emit_move_insn (itarg, op1);
11940 :
11941 219 : return original_target;
11942 : }
11943 :
11944 : /* fall through */
11945 :
11946 200682 : case STRING_CST:
11947 200682 : temp = expand_expr_constant (exp, 1, modifier);
11948 :
11949 : /* temp contains a constant address.
11950 : On RISC machines where a constant address isn't valid,
11951 : make some insns to get that address into a register. */
11952 200682 : if (modifier != EXPAND_CONST_ADDRESS
11953 : && modifier != EXPAND_INITIALIZER
11954 200682 : && modifier != EXPAND_SUM
11955 219239 : && ! memory_address_addr_space_p (mode, XEXP (temp, 0),
11956 18557 : MEM_ADDR_SPACE (temp)))
11957 12 : return replace_equiv_address (temp,
11958 12 : copy_rtx (XEXP (temp, 0)));
11959 : return temp;
11960 :
11961 0 : case POLY_INT_CST:
11962 0 : return immed_wide_int_const (poly_int_cst_value (exp), mode);
11963 :
11964 1624 : case SAVE_EXPR:
11965 1624 : {
11966 1624 : tree val = treeop0;
11967 1624 : rtx ret = expand_expr_real_1 (val, target, tmode, modifier, alt_rtl,
11968 : inner_reference_p);
11969 :
11970 1624 : if (!SAVE_EXPR_RESOLVED_P (exp))
11971 : {
11972 : /* We can indeed still hit this case, typically via builtin
11973 : expanders calling save_expr immediately before expanding
11974 : something. Assume this means that we only have to deal
11975 : with non-BLKmode values. */
11976 1577 : gcc_assert (GET_MODE (ret) != BLKmode);
11977 :
11978 1577 : val = build_decl (curr_insn_location (),
11979 1577 : VAR_DECL, NULL, TREE_TYPE (exp));
11980 1577 : DECL_ARTIFICIAL (val) = 1;
11981 1577 : DECL_IGNORED_P (val) = 1;
11982 1577 : treeop0 = val;
11983 1577 : TREE_OPERAND (exp, 0) = treeop0;
11984 1577 : SAVE_EXPR_RESOLVED_P (exp) = 1;
11985 :
11986 1577 : if (!CONSTANT_P (ret))
11987 1577 : ret = copy_to_reg (ret);
11988 1577 : SET_DECL_RTL (val, ret);
11989 : }
11990 :
11991 : return ret;
11992 : }
11993 :
11994 :
11995 177281 : case CONSTRUCTOR:
11996 : /* If we don't need the result, just ensure we evaluate any
11997 : subexpressions. */
11998 177281 : if (ignore)
11999 : {
12000 : unsigned HOST_WIDE_INT idx;
12001 : tree value;
12002 :
12003 0 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), idx, value)
12004 0 : expand_expr (value, const0_rtx, VOIDmode, EXPAND_NORMAL);
12005 :
12006 0 : return const0_rtx;
12007 : }
12008 :
12009 177281 : return expand_constructor (exp, target, modifier, false);
12010 :
12011 855609 : case TARGET_MEM_REF:
12012 855609 : {
12013 855609 : addr_space_t as
12014 855609 : = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
12015 855609 : unsigned int align;
12016 :
12017 855609 : op0 = addr_for_mem_ref (exp, as, true);
12018 855609 : op0 = memory_address_addr_space (mode, op0, as);
12019 855609 : temp = gen_rtx_MEM (mode, op0);
12020 855609 : set_mem_attributes (temp, exp, 0);
12021 855609 : set_mem_addr_space (temp, as);
12022 855609 : align = get_object_alignment (exp);
12023 855609 : if (modifier != EXPAND_WRITE
12024 855609 : && modifier != EXPAND_MEMORY
12025 589042 : && mode != BLKmode
12026 1438739 : && align < GET_MODE_ALIGNMENT (mode))
12027 52989 : temp = expand_misaligned_mem_ref (temp, mode, unsignedp,
12028 : align, NULL_RTX, NULL);
12029 855609 : return EXTEND_BITINT (temp);
12030 : }
12031 :
12032 6754131 : case MEM_REF:
12033 6754131 : {
12034 6754131 : const bool reverse = REF_REVERSE_STORAGE_ORDER (exp);
12035 6754131 : addr_space_t as
12036 6754131 : = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
12037 6754131 : machine_mode address_mode;
12038 6754131 : tree base = TREE_OPERAND (exp, 0);
12039 6754131 : gimple *def_stmt;
12040 6754131 : unsigned align;
12041 : /* Handle expansion of non-aliased memory with non-BLKmode. That
12042 : might end up in a register. */
12043 6754131 : if (mem_ref_refers_to_non_mem_p (exp))
12044 : {
12045 79391 : poly_int64 offset = mem_ref_offset (exp).force_shwi ();
12046 79391 : base = TREE_OPERAND (base, 0);
12047 79391 : poly_uint64 type_size;
12048 79391 : if (known_eq (offset, 0)
12049 45758 : && !reverse
12050 45758 : && poly_int_tree_p (TYPE_SIZE (type), &type_size)
12051 170907 : && known_eq (GET_MODE_BITSIZE (DECL_MODE (base)), type_size))
12052 19291 : return expand_expr (build1 (VIEW_CONVERT_EXPR, type, base),
12053 19291 : target, tmode, modifier);
12054 60100 : unsigned align;
12055 60100 : if (TYPE_MODE (type) == BLKmode || maybe_lt (offset, 0))
12056 : {
12057 218 : temp = assign_stack_temp (DECL_MODE (base),
12058 436 : GET_MODE_SIZE (DECL_MODE (base)));
12059 218 : store_expr (base, temp, 0, false, false);
12060 218 : temp = adjust_address (temp, TYPE_MODE (type), offset);
12061 218 : if (TYPE_MODE (type) == BLKmode)
12062 197 : set_mem_size (temp, int_size_in_bytes (type));
12063 : /* When the original ref was misaligned so will be the
12064 : access to the stack temporary. Not all targets handle
12065 : this correctly, some will ICE in sanity checking.
12066 : Handle this by doing bitfield extraction when necessary. */
12067 42 : else if ((align = get_object_alignment (exp))
12068 21 : < GET_MODE_ALIGNMENT (TYPE_MODE (type)))
12069 3 : temp
12070 3 : = expand_misaligned_mem_ref (temp, TYPE_MODE (type),
12071 : unsignedp, align,
12072 : modifier == EXPAND_STACK_PARM
12073 : ? NULL_RTX : target, NULL);
12074 218 : return temp;
12075 : }
12076 : /* When the access is fully outside of the underlying object
12077 : expand the offset as zero. This avoids out-of-bound
12078 : BIT_FIELD_REFs and generates smaller code for these cases
12079 : with UB. */
12080 59882 : type_size = tree_to_poly_uint64 (TYPE_SIZE_UNIT (type));
12081 119764 : if (!ranges_maybe_overlap_p (offset, type_size, 0,
12082 119764 : GET_MODE_SIZE (DECL_MODE (base))))
12083 37 : offset = 0;
12084 59882 : exp = build3 (BIT_FIELD_REF, type, base, TYPE_SIZE (type),
12085 59882 : bitsize_int (offset * BITS_PER_UNIT));
12086 59882 : REF_REVERSE_STORAGE_ORDER (exp) = reverse;
12087 59882 : return expand_expr (exp, target, tmode, modifier);
12088 : }
12089 6674740 : address_mode = targetm.addr_space.address_mode (as);
12090 6674740 : if ((def_stmt = get_def_for_expr (base, BIT_AND_EXPR)))
12091 : {
12092 44 : tree mask = gimple_assign_rhs2 (def_stmt);
12093 44 : base = build2 (BIT_AND_EXPR, TREE_TYPE (base),
12094 : gimple_assign_rhs1 (def_stmt), mask);
12095 44 : TREE_OPERAND (exp, 0) = base;
12096 : }
12097 6674740 : align = get_object_alignment (exp);
12098 6674740 : op0 = expand_expr (base, NULL_RTX, VOIDmode, EXPAND_SUM);
12099 6674740 : op0 = memory_address_addr_space (mode, op0, as);
12100 6674740 : if (!integer_zerop (TREE_OPERAND (exp, 1)))
12101 : {
12102 2028906 : rtx off = immed_wide_int_const (mem_ref_offset (exp), address_mode);
12103 2028906 : op0 = simplify_gen_binary (PLUS, address_mode, op0, off);
12104 2028906 : op0 = memory_address_addr_space (mode, op0, as);
12105 : }
12106 6674740 : temp = gen_rtx_MEM (mode, op0);
12107 6674740 : set_mem_attributes (temp, exp, 0);
12108 6674740 : set_mem_addr_space (temp, as);
12109 6674740 : if (TREE_THIS_VOLATILE (exp))
12110 12405 : MEM_VOLATILE_P (temp) = 1;
12111 6674740 : if (modifier == EXPAND_WRITE || modifier == EXPAND_MEMORY)
12112 : return temp;
12113 3963419 : if (!inner_reference_p
12114 2053507 : && mode != BLKmode
12115 5955588 : && align < GET_MODE_ALIGNMENT (mode))
12116 141804 : temp = expand_misaligned_mem_ref (temp, mode, unsignedp, align,
12117 : modifier == EXPAND_STACK_PARM
12118 : ? NULL_RTX : target, alt_rtl);
12119 3963419 : if (reverse)
12120 7 : temp = flip_storage_order (mode, temp);
12121 3963419 : return EXTEND_BITINT (temp);
12122 : }
12123 :
12124 625372 : case ARRAY_REF:
12125 :
12126 625372 : {
12127 625372 : tree array = treeop0;
12128 625372 : tree index = treeop1;
12129 625372 : tree init;
12130 :
12131 : /* Fold an expression like: "foo"[2].
12132 : This is not done in fold so it won't happen inside &.
12133 : Don't fold if this is for wide characters since it's too
12134 : difficult to do correctly and this is a very rare case. */
12135 :
12136 625372 : if (modifier != EXPAND_CONST_ADDRESS
12137 625372 : && modifier != EXPAND_INITIALIZER
12138 625372 : && modifier != EXPAND_MEMORY)
12139 : {
12140 625267 : tree t = fold_read_from_constant_string (exp);
12141 :
12142 625267 : if (t)
12143 0 : return expand_expr (t, target, tmode, modifier);
12144 : }
12145 :
12146 : /* If this is a constant index into a constant array,
12147 : just get the value from the array. Handle both the cases when
12148 : we have an explicit constructor and when our operand is a variable
12149 : that was declared const. */
12150 :
12151 625372 : if (modifier != EXPAND_CONST_ADDRESS
12152 : && modifier != EXPAND_INITIALIZER
12153 : && modifier != EXPAND_MEMORY
12154 625267 : && TREE_CODE (array) == CONSTRUCTOR
12155 0 : && ! TREE_SIDE_EFFECTS (array)
12156 625372 : && TREE_CODE (index) == INTEGER_CST)
12157 : {
12158 : unsigned HOST_WIDE_INT ix;
12159 : tree field, value;
12160 :
12161 0 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (array), ix,
12162 : field, value)
12163 0 : if (tree_int_cst_equal (field, index))
12164 : {
12165 0 : if (!TREE_SIDE_EFFECTS (value)
12166 0 : && TREE_CODE (value) != RAW_DATA_CST)
12167 0 : return expand_expr (fold (value), target, tmode, modifier);
12168 : break;
12169 : }
12170 : }
12171 :
12172 625372 : else if (optimize >= 1
12173 : && modifier != EXPAND_CONST_ADDRESS
12174 357119 : && modifier != EXPAND_INITIALIZER
12175 357119 : && modifier != EXPAND_MEMORY
12176 357018 : && TREE_READONLY (array) && ! TREE_SIDE_EFFECTS (array)
12177 9061 : && TREE_CODE (index) == INTEGER_CST
12178 2051 : && (VAR_P (array) || TREE_CODE (array) == CONST_DECL)
12179 625521 : && (init = ctor_for_folding (array)) != error_mark_node)
12180 : {
12181 108 : if (init == NULL_TREE)
12182 : {
12183 5 : tree value = build_zero_cst (type);
12184 5 : if (TREE_CODE (value) == CONSTRUCTOR)
12185 : {
12186 : /* If VALUE is a CONSTRUCTOR, this optimization is only
12187 : useful if this doesn't store the CONSTRUCTOR into
12188 : memory. If it does, it is more efficient to just
12189 : load the data from the array directly. */
12190 5 : rtx ret = expand_constructor (value, target,
12191 : modifier, true);
12192 5 : if (ret == NULL_RTX)
12193 : value = NULL_TREE;
12194 : }
12195 :
12196 : if (value)
12197 0 : return expand_expr (value, target, tmode, modifier);
12198 : }
12199 103 : else if (TREE_CODE (init) == CONSTRUCTOR)
12200 : {
12201 : unsigned HOST_WIDE_INT ix;
12202 : tree field, value;
12203 :
12204 168 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (init), ix,
12205 : field, value)
12206 166 : if (tree_int_cst_equal (field, index))
12207 : {
12208 100 : if (TREE_SIDE_EFFECTS (value)
12209 100 : || TREE_CODE (value) == RAW_DATA_CST)
12210 : break;
12211 :
12212 100 : if (TREE_CODE (value) == CONSTRUCTOR)
12213 : {
12214 : /* If VALUE is a CONSTRUCTOR, this
12215 : optimization is only useful if
12216 : this doesn't store the CONSTRUCTOR
12217 : into memory. If it does, it is more
12218 : efficient to just load the data from
12219 : the array directly. */
12220 95 : rtx ret = expand_constructor (value, target,
12221 : modifier, true);
12222 95 : if (ret == NULL_RTX)
12223 : break;
12224 : }
12225 :
12226 62 : return expand_expr (fold (value), target, tmode,
12227 62 : modifier);
12228 : }
12229 : }
12230 1 : else if (TREE_CODE (init) == STRING_CST)
12231 : {
12232 1 : tree low_bound = array_ref_low_bound (exp);
12233 1 : tree index1 = fold_convert_loc (loc, sizetype, treeop1);
12234 :
12235 : /* Optimize the special case of a zero lower bound.
12236 :
12237 : We convert the lower bound to sizetype to avoid problems
12238 : with constant folding. E.g. suppose the lower bound is
12239 : 1 and its mode is QI. Without the conversion
12240 : (ARRAY + (INDEX - (unsigned char)1))
12241 : becomes
12242 : (ARRAY + (-(unsigned char)1) + INDEX)
12243 : which becomes
12244 : (ARRAY + 255 + INDEX). Oops! */
12245 1 : if (!integer_zerop (low_bound))
12246 0 : index1 = size_diffop_loc (loc, index1,
12247 : fold_convert_loc (loc, sizetype,
12248 : low_bound));
12249 :
12250 1 : if (tree_fits_uhwi_p (index1)
12251 2 : && compare_tree_int (index1, TREE_STRING_LENGTH (init)) < 0)
12252 : {
12253 0 : tree char_type = TREE_TYPE (TREE_TYPE (init));
12254 0 : scalar_int_mode char_mode;
12255 :
12256 625310 : if (is_int_mode (TYPE_MODE (char_type), &char_mode)
12257 0 : && GET_MODE_SIZE (char_mode) == 1)
12258 0 : return gen_int_mode (TREE_STRING_POINTER (init)
12259 0 : [TREE_INT_CST_LOW (index1)],
12260 : char_mode);
12261 : }
12262 : }
12263 : }
12264 : }
12265 625310 : goto normal_inner_ref;
12266 :
12267 3723700 : case COMPONENT_REF:
12268 3723700 : gcc_assert (TREE_CODE (treeop0) != CONSTRUCTOR);
12269 : /* Fall through. */
12270 4605537 : case BIT_FIELD_REF:
12271 4605537 : case ARRAY_RANGE_REF:
12272 3723700 : normal_inner_ref:
12273 4605537 : {
12274 4605537 : machine_mode mode1, mode2;
12275 4605537 : poly_int64 bitsize, bitpos, bytepos;
12276 4605537 : tree offset;
12277 4605537 : int reversep, volatilep = 0;
12278 4605537 : tree tem
12279 4605537 : = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
12280 : &unsignedp, &reversep, &volatilep);
12281 4605537 : rtx orig_op0, memloc;
12282 4605537 : bool clear_mem_expr = false;
12283 4605537 : bool must_force_mem;
12284 :
12285 : /* If we got back the original object, something is wrong. Perhaps
12286 : we are evaluating an expression too early. In any event, don't
12287 : infinitely recurse. */
12288 4605537 : gcc_assert (tem != exp);
12289 :
12290 : /* Make sure bitpos is not negative, this can wreak havoc later. */
12291 4605537 : if (maybe_lt (bitpos, 0))
12292 : {
12293 249 : gcc_checking_assert (offset == NULL_TREE);
12294 249 : offset = size_int (bits_to_bytes_round_down (bitpos));
12295 249 : bitpos = num_trailing_bits (bitpos);
12296 : }
12297 :
12298 : /* If we have either an offset, a BLKmode result, or a reference
12299 : outside the underlying object, we must force it to memory.
12300 : Such a case can occur in Ada if we have unchecked conversion
12301 : of an expression from a scalar type to an aggregate type or
12302 : for an ARRAY_RANGE_REF whose type is BLKmode, or if we were
12303 : passed a partially uninitialized object or a view-conversion
12304 : to a larger size. */
12305 9211074 : must_force_mem = offset != NULL_TREE
12306 4326923 : || mode1 == BLKmode
12307 8860424 : || (mode == BLKmode
12308 0 : && !int_mode_for_size (bitsize, 1).exists ());
12309 :
12310 524433 : const enum expand_modifier tem_modifier
12311 : = must_force_mem
12312 : ? EXPAND_MEMORY
12313 4254887 : : modifier == EXPAND_SUM ? EXPAND_NORMAL : modifier;
12314 :
12315 : /* If TEM's type is a union of variable size, pass TARGET to the inner
12316 : computation, since it will need a temporary and TARGET is known
12317 : to have to do. This occurs in unchecked conversion in Ada. */
12318 4605537 : const rtx tem_target
12319 4605537 : = TREE_CODE (TREE_TYPE (tem)) == UNION_TYPE
12320 36700 : && COMPLETE_TYPE_P (TREE_TYPE (tem))
12321 36695 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (tem))) != INTEGER_CST
12322 0 : && modifier != EXPAND_STACK_PARM
12323 4605537 : ? target
12324 : : NULL_RTX;
12325 :
12326 9211074 : orig_op0 = op0
12327 4605537 : = expand_expr_real (tem, tem_target, VOIDmode, tem_modifier, NULL,
12328 : true);
12329 :
12330 : /* If the field has a mode, we want to access it in the
12331 : field's mode, not the computed mode.
12332 : If a MEM has VOIDmode (external with incomplete type),
12333 : use BLKmode for it instead. */
12334 4605537 : if (MEM_P (op0))
12335 : {
12336 4266241 : if (mode1 != VOIDmode)
12337 4111014 : op0 = adjust_address (op0, mode1, 0);
12338 155227 : else if (GET_MODE (op0) == VOIDmode)
12339 0 : op0 = adjust_address (op0, BLKmode, 0);
12340 : }
12341 :
12342 4605537 : mode2
12343 4605537 : = CONSTANT_P (op0) ? TYPE_MODE (TREE_TYPE (tem)) : GET_MODE (op0);
12344 :
12345 : /* See above for the rationale. */
12346 9211074 : if (maybe_gt (bitpos + bitsize, GET_MODE_BITSIZE (mode2)))
12347 2907841 : must_force_mem = true;
12348 :
12349 : /* Handle CONCAT first. */
12350 4605537 : if (GET_CODE (op0) == CONCAT && !must_force_mem)
12351 : {
12352 114 : if (known_eq (bitpos, 0)
12353 206 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0)))
12354 98 : && COMPLEX_MODE_P (mode1)
12355 93 : && COMPLEX_MODE_P (GET_MODE (op0))
12356 393 : && (GET_MODE_PRECISION (GET_MODE_INNER (mode1))
12357 186 : == GET_MODE_PRECISION (GET_MODE_INNER (GET_MODE (op0)))))
12358 : {
12359 93 : if (reversep)
12360 0 : op0 = flip_storage_order (GET_MODE (op0), op0);
12361 93 : if (mode1 != GET_MODE (op0))
12362 : {
12363 : rtx parts[2];
12364 0 : for (int i = 0; i < 2; i++)
12365 : {
12366 0 : rtx op = read_complex_part (op0, i != 0);
12367 0 : if (GET_CODE (op) == SUBREG)
12368 0 : op = force_reg (GET_MODE (op), op);
12369 0 : temp = gen_lowpart_common (GET_MODE_INNER (mode1), op);
12370 0 : if (temp)
12371 : op = temp;
12372 : else
12373 : {
12374 0 : if (!REG_P (op) && !MEM_P (op))
12375 0 : op = force_reg (GET_MODE (op), op);
12376 0 : op = gen_lowpart (GET_MODE_INNER (mode1), op);
12377 : }
12378 0 : parts[i] = op;
12379 : }
12380 0 : op0 = gen_rtx_CONCAT (mode1, parts[0], parts[1]);
12381 : }
12382 93 : return op0;
12383 : }
12384 21 : if (known_eq (bitpos, 0)
12385 20 : && known_eq (bitsize,
12386 : GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 0))))
12387 23 : && maybe_ne (bitsize, 0))
12388 : {
12389 : op0 = XEXP (op0, 0);
12390 : mode2 = GET_MODE (op0);
12391 : }
12392 19 : else if (known_eq (bitpos,
12393 : GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 0))))
12394 4 : && known_eq (bitsize,
12395 : GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 1))))
12396 0 : && maybe_ne (bitpos, 0)
12397 19 : && maybe_ne (bitsize, 0))
12398 : {
12399 0 : op0 = XEXP (op0, 1);
12400 0 : bitpos = 0;
12401 0 : mode2 = GET_MODE (op0);
12402 : }
12403 : else
12404 : /* Otherwise force into memory. */
12405 : must_force_mem = true;
12406 : }
12407 :
12408 : /* If this is a constant, put it in a register if it is a legitimate
12409 : constant and we don't need a memory reference. */
12410 4605444 : if (CONSTANT_P (op0)
12411 33 : && mode2 != BLKmode
12412 33 : && targetm.legitimate_constant_p (mode2, op0)
12413 4605465 : && !must_force_mem)
12414 21 : op0 = force_reg (mode2, op0);
12415 :
12416 : /* Otherwise, if this is a constant, try to force it to the constant
12417 : pool. Note that back-ends, e.g. MIPS, may refuse to do so if it
12418 : is a legitimate constant. */
12419 4605423 : else if (CONSTANT_P (op0) && (memloc = force_const_mem (mode2, op0)))
12420 12 : op0 = validize_mem (memloc);
12421 :
12422 : /* Otherwise, if this is a constant or the object is not in memory
12423 : and need be, put it there. */
12424 4605411 : else if (CONSTANT_P (op0) || (!MEM_P (op0) && must_force_mem))
12425 : {
12426 1095 : machine_mode tem_mode = TYPE_MODE (TREE_TYPE (tem));
12427 1095 : poly_int64 size;
12428 1095 : if (!poly_int_tree_p (TYPE_SIZE_UNIT (TREE_TYPE (tem)), &size))
12429 0 : size = max_int_size_in_bytes (TREE_TYPE (tem));
12430 1095 : unsigned int align = TREE_CODE (tem) == SSA_NAME
12431 1095 : ? TYPE_ALIGN (TREE_TYPE (tem))
12432 1095 : : get_object_alignment (tem);
12433 1095 : if (STRICT_ALIGNMENT)
12434 : {
12435 : /* For STRICT_ALIGNMENT targets, when we force the operand to
12436 : memory, we may need to increase the alignment to meet the
12437 : expectation in later RTL lowering passes. The increased
12438 : alignment is capped by MAX_SUPPORTED_STACK_ALIGNMENT. */
12439 : if (tem_mode != BLKmode)
12440 : align = MAX (align, GET_MODE_ALIGNMENT (tem_mode));
12441 : else
12442 : align = MAX (align, TYPE_ALIGN (TREE_TYPE (tem)));
12443 : align = MIN (align, (unsigned) MAX_SUPPORTED_STACK_ALIGNMENT);
12444 : }
12445 1095 : memloc = assign_stack_local (tem_mode, size, align);
12446 1095 : emit_move_insn (memloc, op0);
12447 1095 : op0 = memloc;
12448 1095 : clear_mem_expr = true;
12449 : }
12450 :
12451 4605444 : if (offset)
12452 : {
12453 278614 : machine_mode address_mode;
12454 278614 : rtx offset_rtx = expand_expr (offset, NULL_RTX, VOIDmode,
12455 : EXPAND_SUM);
12456 :
12457 278614 : gcc_assert (MEM_P (op0));
12458 :
12459 278614 : address_mode = get_address_mode (op0);
12460 278614 : if (GET_MODE (offset_rtx) != address_mode)
12461 : {
12462 : /* We cannot be sure that the RTL in offset_rtx is valid outside
12463 : of a memory address context, so force it into a register
12464 : before attempting to convert it to the desired mode. */
12465 422 : offset_rtx = force_operand (offset_rtx, NULL_RTX);
12466 422 : offset_rtx = convert_to_mode (address_mode, offset_rtx, 0);
12467 : }
12468 :
12469 : /* See the comment in expand_assignment for the rationale. */
12470 278614 : if (mode1 != VOIDmode
12471 278398 : && maybe_ne (bitpos, 0)
12472 74431 : && maybe_gt (bitsize, 0)
12473 353045 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
12474 352506 : && multiple_p (bitpos, bitsize)
12475 147784 : && multiple_p (bitsize, GET_MODE_ALIGNMENT (mode1))
12476 352506 : && MEM_ALIGN (op0) >= GET_MODE_ALIGNMENT (mode1))
12477 : {
12478 73537 : op0 = adjust_address (op0, mode1, bytepos);
12479 73537 : bitpos = 0;
12480 : }
12481 :
12482 278614 : op0 = offset_address (op0, offset_rtx,
12483 : highest_pow2_factor (offset));
12484 : }
12485 :
12486 : /* If OFFSET is making OP0 more aligned than BIGGEST_ALIGNMENT,
12487 : record its alignment as BIGGEST_ALIGNMENT. */
12488 4605444 : if (MEM_P (op0)
12489 4267348 : && known_eq (bitpos, 0)
12490 1444371 : && offset != 0
12491 4883014 : && is_aligning_offset (offset, tem))
12492 0 : set_mem_align (op0, BIGGEST_ALIGNMENT);
12493 :
12494 : /* Don't forget about volatility even if this is a bitfield. */
12495 4605444 : if (MEM_P (op0) && volatilep && ! MEM_VOLATILE_P (op0))
12496 : {
12497 528 : if (op0 == orig_op0)
12498 158 : op0 = copy_rtx (op0);
12499 :
12500 528 : MEM_VOLATILE_P (op0) = 1;
12501 : }
12502 :
12503 4605444 : if (MEM_P (op0) && TREE_CODE (tem) == FUNCTION_DECL)
12504 : {
12505 6 : if (op0 == orig_op0)
12506 0 : op0 = copy_rtx (op0);
12507 :
12508 6 : set_mem_align (op0, BITS_PER_UNIT);
12509 : }
12510 :
12511 : /* In cases where an aligned union has an unaligned object
12512 : as a field, we might be extracting a BLKmode value from
12513 : an integer-mode (e.g., SImode) object. Handle this case
12514 : by doing the extract into an object as wide as the field
12515 : (which we know to be the width of a basic mode), then
12516 : storing into memory, and changing the mode to BLKmode. */
12517 4605444 : if (mode1 == VOIDmode
12518 4378452 : || REG_P (op0) || GET_CODE (op0) == SUBREG
12519 4112087 : || (mode1 != BLKmode && ! direct_load[(int) mode1]
12520 27257 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_INT
12521 23098 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_FLOAT
12522 : && modifier != EXPAND_CONST_ADDRESS
12523 15389 : && modifier != EXPAND_INITIALIZER
12524 15389 : && modifier != EXPAND_MEMORY)
12525 : /* If the bitfield is volatile and the bitsize
12526 : is narrower than the access size of the bitfield,
12527 : we need to extract bitfields from the access. */
12528 4096698 : || (volatilep && TREE_CODE (exp) == COMPONENT_REF
12529 1462 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (exp, 1))
12530 10 : && mode1 != BLKmode
12531 20 : && maybe_lt (bitsize, GET_MODE_SIZE (mode1) * BITS_PER_UNIT))
12532 : /* If the field isn't aligned enough to fetch as a memref,
12533 : fetch it as a bit field. */
12534 4096689 : || (mode1 != BLKmode
12535 4023791 : && (((MEM_P (op0)
12536 4023791 : ? MEM_ALIGN (op0) < GET_MODE_ALIGNMENT (mode1)
12537 7903623 : || !multiple_p (bitpos, GET_MODE_ALIGNMENT (mode1))
12538 0 : : TYPE_ALIGN (TREE_TYPE (tem)) < GET_MODE_ALIGNMENT (mode)
12539 0 : || !multiple_p (bitpos, GET_MODE_ALIGNMENT (mode)))
12540 75299 : && modifier != EXPAND_MEMORY
12541 75299 : && ((modifier == EXPAND_CONST_ADDRESS
12542 75299 : || modifier == EXPAND_INITIALIZER)
12543 75299 : ? STRICT_ALIGNMENT
12544 75299 : : targetm.slow_unaligned_access (mode1,
12545 75299 : MEM_ALIGN (op0))))
12546 4023791 : || !multiple_p (bitpos, BITS_PER_UNIT)))
12547 : /* If the type and the field are a constant size and the
12548 : size of the type isn't the same size as the bitfield,
12549 : we must use bitfield operations. */
12550 12725924 : || (known_size_p (bitsize)
12551 4096689 : && TYPE_SIZE (TREE_TYPE (exp))
12552 4096689 : && poly_int_tree_p (TYPE_SIZE (TREE_TYPE (exp)))
12553 4096689 : && maybe_ne (wi::to_poly_offset (TYPE_SIZE (TREE_TYPE (exp))),
12554 : bitsize)))
12555 : {
12556 508784 : machine_mode ext_mode = mode;
12557 :
12558 508784 : if (ext_mode == BLKmode
12559 508784 : && ! (target != 0 && MEM_P (op0)
12560 22 : && MEM_P (target)
12561 22 : && multiple_p (bitpos, BITS_PER_UNIT)))
12562 3 : ext_mode = int_mode_for_size (bitsize, 1).else_blk ();
12563 :
12564 508784 : if (ext_mode == BLKmode)
12565 : {
12566 25 : if (target == 0)
12567 3 : target = assign_temp (type, 1, 1);
12568 :
12569 : /* ??? Unlike the similar test a few lines below, this one is
12570 : very likely obsolete. */
12571 25 : if (known_eq (bitsize, 0))
12572 : return target;
12573 :
12574 : /* In this case, BITPOS must start at a byte boundary and
12575 : TARGET, if specified, must be a MEM. */
12576 25 : gcc_assert (MEM_P (op0)
12577 : && (!target || MEM_P (target)));
12578 :
12579 25 : bytepos = exact_div (bitpos, BITS_PER_UNIT);
12580 25 : poly_int64 bytesize = bits_to_bytes_round_up (bitsize);
12581 50 : emit_block_move (target,
12582 : adjust_address (op0, VOIDmode, bytepos),
12583 25 : gen_int_mode (bytesize, Pmode),
12584 : (modifier == EXPAND_STACK_PARM
12585 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
12586 :
12587 25 : return target;
12588 : }
12589 :
12590 : /* If we have nothing to extract, the result will be 0 for targets
12591 : with SHIFT_COUNT_TRUNCATED == 0 and garbage otherwise. Always
12592 : return 0 for the sake of consistency, as reading a zero-sized
12593 : bitfield is valid in Ada and the value is fully specified. */
12594 508759 : if (known_eq (bitsize, 0))
12595 0 : return const0_rtx;
12596 :
12597 508759 : op0 = validize_mem (op0);
12598 :
12599 508759 : if (MEM_P (op0) && REG_P (XEXP (op0, 0)))
12600 47437 : mark_reg_pointer (XEXP (op0, 0), MEM_ALIGN (op0));
12601 :
12602 : /* If the result has aggregate type and the extraction is done in
12603 : an integral mode, then the field may be not aligned on a byte
12604 : boundary; in this case, if it has reverse storage order, it
12605 : needs to be extracted as a scalar field with reverse storage
12606 : order and put back into memory order afterwards. */
12607 508759 : if (AGGREGATE_TYPE_P (type)
12608 4834 : && GET_MODE_CLASS (ext_mode) == MODE_INT)
12609 4752 : reversep = TYPE_REVERSE_STORAGE_ORDER (type);
12610 :
12611 508759 : gcc_checking_assert (known_ge (bitpos, 0));
12612 521391 : op0 = extract_bit_field (op0, bitsize, bitpos, unsignedp,
12613 : (modifier == EXPAND_STACK_PARM
12614 : ? NULL_RTX : target),
12615 : ext_mode, ext_mode, reversep, alt_rtl);
12616 :
12617 : /* If the result has aggregate type and the mode of OP0 is an
12618 : integral mode then, if BITSIZE is narrower than this mode
12619 : and this is for big-endian data, we must put the field
12620 : into the high-order bits. And we must also put it back
12621 : into memory order if it has been previously reversed. */
12622 508759 : scalar_int_mode op0_mode;
12623 508759 : if (AGGREGATE_TYPE_P (type)
12624 508759 : && is_int_mode (GET_MODE (op0), &op0_mode))
12625 : {
12626 4752 : HOST_WIDE_INT size = GET_MODE_BITSIZE (op0_mode);
12627 :
12628 4752 : gcc_checking_assert (known_le (bitsize, size));
12629 4752 : if (maybe_lt (bitsize, size)
12630 4752 : && reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
12631 0 : op0 = expand_shift (LSHIFT_EXPR, op0_mode, op0,
12632 : size - bitsize, op0, 1);
12633 :
12634 4752 : if (reversep)
12635 0 : op0 = flip_storage_order (op0_mode, op0);
12636 : }
12637 :
12638 : /* If the result type is BLKmode, store the data into a temporary
12639 : of the appropriate type, but with the mode corresponding to the
12640 : mode for the data we have (op0's mode). */
12641 508759 : if (mode == BLKmode)
12642 : {
12643 0 : rtx new_rtx
12644 0 : = assign_stack_temp_for_type (ext_mode,
12645 0 : GET_MODE_BITSIZE (ext_mode),
12646 : type);
12647 0 : emit_move_insn (new_rtx, op0);
12648 0 : op0 = copy_rtx (new_rtx);
12649 0 : PUT_MODE (op0, BLKmode);
12650 : }
12651 :
12652 508759 : return op0;
12653 : }
12654 :
12655 : /* If the result is BLKmode, use that to access the object
12656 : now as well. */
12657 4096660 : if (mode == BLKmode)
12658 72873 : mode1 = BLKmode;
12659 :
12660 : /* Get a reference to just this component. */
12661 4096660 : bytepos = bits_to_bytes_round_down (bitpos);
12662 4096660 : if (modifier == EXPAND_CONST_ADDRESS
12663 4096660 : || modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER)
12664 164352 : op0 = adjust_address_nv (op0, mode1, bytepos);
12665 : else
12666 3932308 : op0 = adjust_address (op0, mode1, bytepos);
12667 :
12668 4096660 : if (op0 == orig_op0)
12669 126748 : op0 = copy_rtx (op0);
12670 :
12671 : /* Don't set memory attributes if the base expression is
12672 : SSA_NAME that got expanded as a MEM or a CONSTANT. In that case,
12673 : we should just honor its original memory attributes. */
12674 4096660 : if (!(TREE_CODE (tem) == SSA_NAME
12675 8884 : && (MEM_P (orig_op0) || CONSTANT_P (orig_op0))))
12676 4087776 : set_mem_attributes (op0, exp, 0);
12677 :
12678 4096660 : if (REG_P (XEXP (op0, 0)))
12679 662722 : mark_reg_pointer (XEXP (op0, 0), MEM_ALIGN (op0));
12680 :
12681 : /* If op0 is a temporary because the original expressions was forced
12682 : to memory, clear MEM_EXPR so that the original expression cannot
12683 : be marked as addressable through MEM_EXPR of the temporary. */
12684 4096660 : if (clear_mem_expr)
12685 1061 : set_mem_expr (op0, NULL_TREE);
12686 :
12687 4096660 : MEM_VOLATILE_P (op0) |= volatilep;
12688 :
12689 4096660 : if (reversep
12690 : && modifier != EXPAND_MEMORY
12691 491 : && modifier != EXPAND_WRITE)
12692 491 : op0 = flip_storage_order (mode1, op0);
12693 :
12694 4096660 : op0 = EXTEND_BITINT (op0);
12695 :
12696 4096660 : if (mode == mode1 || mode1 == BLKmode || mode1 == tmode
12697 : || modifier == EXPAND_CONST_ADDRESS
12698 0 : || modifier == EXPAND_INITIALIZER)
12699 : return op0;
12700 :
12701 0 : if (target == 0)
12702 0 : target = gen_reg_rtx (tmode != VOIDmode ? tmode : mode);
12703 :
12704 0 : convert_move (target, op0, unsignedp);
12705 0 : return target;
12706 : }
12707 :
12708 84487 : case OBJ_TYPE_REF:
12709 84487 : return expand_expr (OBJ_TYPE_REF_EXPR (exp), target, tmode, modifier);
12710 :
12711 6771535 : case CALL_EXPR:
12712 : /* All valid uses of __builtin_va_arg_pack () are removed during
12713 : inlining. */
12714 6771535 : if (CALL_EXPR_VA_ARG_PACK (exp))
12715 6 : error ("invalid use of %<__builtin_va_arg_pack ()%>");
12716 6771535 : {
12717 6771535 : tree fndecl = get_callee_fndecl (exp), attr;
12718 :
12719 6771535 : if (fndecl
12720 : /* Don't diagnose the error attribute in thunks, those are
12721 : artificially created. */
12722 6585464 : && !CALL_FROM_THUNK_P (exp)
12723 13353430 : && (attr = lookup_attribute ("error",
12724 6581895 : DECL_ATTRIBUTES (fndecl))) != NULL)
12725 : {
12726 5 : const char *ident = lang_hooks.decl_printable_name (fndecl, 1);
12727 10 : error ("call to %qs declared with attribute error: %s",
12728 : identifier_to_locale (ident),
12729 5 : TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr))));
12730 : }
12731 6771535 : if (fndecl
12732 : /* Don't diagnose the warning attribute in thunks, those are
12733 : artificially created. */
12734 6585464 : && !CALL_FROM_THUNK_P (exp)
12735 13353430 : && (attr = lookup_attribute ("warning",
12736 6581895 : DECL_ATTRIBUTES (fndecl))) != NULL)
12737 : {
12738 17 : const char *ident = lang_hooks.decl_printable_name (fndecl, 1);
12739 34 : warning_at (EXPR_LOCATION (exp),
12740 17 : OPT_Wattribute_warning,
12741 : "call to %qs declared with attribute warning: %s",
12742 : identifier_to_locale (ident),
12743 17 : TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr))));
12744 : }
12745 :
12746 : /* Check for a built-in function. */
12747 6771535 : if (fndecl && fndecl_built_in_p (fndecl))
12748 : {
12749 2003033 : gcc_assert (DECL_BUILT_IN_CLASS (fndecl) != BUILT_IN_FRONTEND);
12750 2003033 : return expand_builtin (exp, target, subtarget, tmode, ignore);
12751 : }
12752 : }
12753 4768502 : temp = expand_call (exp, target, ignore);
12754 4768500 : return EXTEND_BITINT (temp);
12755 :
12756 347831 : case VIEW_CONVERT_EXPR:
12757 347831 : op0 = NULL_RTX;
12758 :
12759 : /* If we are converting to BLKmode, try to avoid an intermediate
12760 : temporary by fetching an inner memory reference. */
12761 347831 : if (mode == BLKmode
12762 74203 : && poly_int_tree_p (TYPE_SIZE (type))
12763 74203 : && TYPE_MODE (TREE_TYPE (treeop0)) != BLKmode
12764 347831 : && handled_component_p (treeop0))
12765 : {
12766 0 : machine_mode mode1;
12767 0 : poly_int64 bitsize, bitpos, bytepos;
12768 0 : tree offset;
12769 0 : int reversep, volatilep = 0;
12770 0 : tree tem
12771 0 : = get_inner_reference (treeop0, &bitsize, &bitpos, &offset, &mode1,
12772 : &unsignedp, &reversep, &volatilep);
12773 :
12774 : /* ??? We should work harder and deal with non-zero offsets. */
12775 0 : if (!offset
12776 0 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
12777 0 : && !reversep
12778 0 : && known_size_p (bitsize)
12779 0 : && known_eq (wi::to_poly_offset (TYPE_SIZE (type)), bitsize))
12780 : {
12781 : /* See the normal_inner_ref case for the rationale. */
12782 0 : rtx orig_op0
12783 0 : = expand_expr_real (tem,
12784 0 : (TREE_CODE (TREE_TYPE (tem)) == UNION_TYPE
12785 0 : && (TREE_CODE (TYPE_SIZE (TREE_TYPE (tem)))
12786 : != INTEGER_CST)
12787 0 : && modifier != EXPAND_STACK_PARM
12788 : ? target : NULL_RTX),
12789 : VOIDmode,
12790 : modifier == EXPAND_SUM ? EXPAND_NORMAL : modifier,
12791 : NULL, true);
12792 :
12793 0 : if (MEM_P (orig_op0))
12794 : {
12795 0 : op0 = orig_op0;
12796 :
12797 : /* Get a reference to just this component. */
12798 0 : if (modifier == EXPAND_CONST_ADDRESS
12799 : || modifier == EXPAND_SUM
12800 0 : || modifier == EXPAND_INITIALIZER)
12801 0 : op0 = adjust_address_nv (op0, mode, bytepos);
12802 : else
12803 0 : op0 = adjust_address (op0, mode, bytepos);
12804 :
12805 0 : if (op0 == orig_op0)
12806 0 : op0 = copy_rtx (op0);
12807 :
12808 0 : set_mem_attributes (op0, treeop0, 0);
12809 0 : if (REG_P (XEXP (op0, 0)))
12810 0 : mark_reg_pointer (XEXP (op0, 0), MEM_ALIGN (op0));
12811 :
12812 0 : MEM_VOLATILE_P (op0) |= volatilep;
12813 : }
12814 : }
12815 : }
12816 :
12817 0 : if (!op0)
12818 347831 : op0 = expand_expr_real (treeop0, NULL_RTX, VOIDmode, modifier,
12819 347831 : NULL, inner_reference_p || mode == BLKmode);
12820 :
12821 : /* If the input and output modes are both the same, we are done. */
12822 347831 : if (mode == GET_MODE (op0))
12823 : ;
12824 : /* Similarly if the output mode is BLKmode and input is a MEM,
12825 : adjust_address done below is all we need. */
12826 148805 : else if (mode == BLKmode && MEM_P (op0))
12827 : ;
12828 : /* If neither mode is BLKmode, and both modes are the same size
12829 : then we can use gen_lowpart. */
12830 : else if (mode != BLKmode
12831 148771 : && GET_MODE (op0) != BLKmode
12832 147444 : && known_eq (GET_MODE_PRECISION (mode),
12833 : GET_MODE_PRECISION (GET_MODE (op0)))
12834 143547 : && !COMPLEX_MODE_P (GET_MODE (op0)))
12835 : {
12836 142143 : if (GET_CODE (op0) == SUBREG)
12837 108 : op0 = force_reg (GET_MODE (op0), op0);
12838 142143 : temp = gen_lowpart_common (mode, op0);
12839 142143 : if (temp)
12840 : op0 = temp;
12841 : else
12842 : {
12843 27368 : if (!REG_P (op0) && !MEM_P (op0))
12844 18 : op0 = force_reg (GET_MODE (op0), op0);
12845 27368 : op0 = gen_lowpart (mode, op0);
12846 : }
12847 : }
12848 : /* If both types are integral, convert from one mode to the other. */
12849 6660 : else if (INTEGRAL_TYPE_P (type)
12850 3129 : && INTEGRAL_TYPE_P (TREE_TYPE (treeop0))
12851 0 : && mode != BLKmode
12852 6660 : && GET_MODE (op0) != BLKmode)
12853 0 : op0 = convert_modes (mode, GET_MODE (op0), op0,
12854 0 : TYPE_UNSIGNED (TREE_TYPE (treeop0)));
12855 : /* If the output type is a bit-field type, do an extraction. */
12856 6660 : else if (reduce_bit_field
12857 2 : && mode != BLKmode
12858 2 : && (MEM_P (op0) || !COMPLEX_MODE_P (GET_MODE (op0))))
12859 0 : return extract_bit_field (op0, TYPE_PRECISION (type), 0,
12860 0 : TYPE_UNSIGNED (type), NULL_RTX,
12861 : mode, mode, false, NULL);
12862 : /* If source is a complex number and destination is a
12863 : two-component vector with same inner type, try to use
12864 : vector initialization. */
12865 6660 : else if ((temp = try_expand_complex_as_vector (mode, op0, target))
12866 : != NULL_RTX)
12867 : return temp;
12868 : /* As a last resort, spill op0 to memory, and reload it in a
12869 : different mode. */
12870 6631 : else if (!MEM_P (op0))
12871 : {
12872 : /* If the operand is not a MEM, force it into memory. Since we
12873 : are going to be changing the mode of the MEM, don't call
12874 : force_const_mem for constants because we don't allow pool
12875 : constants to change mode. */
12876 5294 : tree inner_type = TREE_TYPE (treeop0);
12877 :
12878 5294 : gcc_assert (!TREE_ADDRESSABLE (exp));
12879 :
12880 5294 : if (target == 0 || GET_MODE (target) != TYPE_MODE (inner_type))
12881 5290 : target
12882 : = assign_stack_temp_for_type
12883 5290 : (TYPE_MODE (inner_type),
12884 10580 : GET_MODE_SIZE (TYPE_MODE (inner_type)), inner_type);
12885 :
12886 5294 : emit_move_insn (target, op0);
12887 5294 : op0 = target;
12888 :
12889 5294 : if (reduce_bit_field && mode != BLKmode)
12890 2 : return extract_bit_field (op0, TYPE_PRECISION (type), 0,
12891 2 : TYPE_UNSIGNED (type), NULL_RTX,
12892 : mode, mode, false, NULL);
12893 : }
12894 :
12895 : /* If OP0 is (now) a MEM, we need to deal with alignment issues. If the
12896 : output type is such that the operand is known to be aligned, indicate
12897 : that it is. Otherwise, we need only be concerned about alignment for
12898 : non-BLKmode results. */
12899 347800 : if (MEM_P (op0))
12900 : {
12901 127580 : enum insn_code icode;
12902 :
12903 127580 : if (modifier != EXPAND_WRITE
12904 127580 : && modifier != EXPAND_MEMORY
12905 127580 : && !inner_reference_p
12906 127579 : && mode != BLKmode
12907 180956 : && MEM_ALIGN (op0) < GET_MODE_ALIGNMENT (mode))
12908 : {
12909 : /* If the target does have special handling for unaligned
12910 : loads of mode then use them. */
12911 15648 : if ((icode = optab_handler (movmisalign_optab, mode))
12912 : != CODE_FOR_nothing)
12913 : {
12914 1774 : rtx reg;
12915 :
12916 1774 : op0 = adjust_address (op0, mode, 0);
12917 : /* We've already validated the memory, and we're creating a
12918 : new pseudo destination. The predicates really can't
12919 : fail. */
12920 1774 : reg = gen_reg_rtx (mode);
12921 :
12922 : /* Nor can the insn generator. */
12923 1774 : rtx_insn *insn = GEN_FCN (icode) (reg, op0);
12924 1774 : emit_insn (insn);
12925 1774 : return reg;
12926 : }
12927 : else if (STRICT_ALIGNMENT)
12928 : {
12929 : poly_uint64 mode_size = GET_MODE_SIZE (mode);
12930 : poly_uint64 temp_size = mode_size;
12931 : if (GET_MODE (op0) != BLKmode)
12932 : temp_size = upper_bound (temp_size,
12933 : GET_MODE_SIZE (GET_MODE (op0)));
12934 : rtx new_rtx
12935 : = assign_stack_temp_for_type (mode, temp_size, type);
12936 : rtx new_with_op0_mode
12937 : = adjust_address (new_rtx, GET_MODE (op0), 0);
12938 :
12939 : gcc_assert (!TREE_ADDRESSABLE (exp));
12940 :
12941 : if (GET_MODE (op0) == BLKmode)
12942 : {
12943 : rtx size_rtx = gen_int_mode (mode_size, Pmode);
12944 : emit_block_move (new_with_op0_mode, op0, size_rtx,
12945 : (modifier == EXPAND_STACK_PARM
12946 : ? BLOCK_OP_CALL_PARM
12947 : : BLOCK_OP_NORMAL));
12948 : }
12949 : else
12950 : emit_move_insn (new_with_op0_mode, op0);
12951 :
12952 : op0 = new_rtx;
12953 : }
12954 : }
12955 :
12956 125806 : op0 = adjust_address (op0, mode, 0);
12957 : }
12958 :
12959 : return op0;
12960 :
12961 63582 : case MODIFY_EXPR:
12962 63582 : {
12963 63582 : tree lhs = treeop0;
12964 63582 : tree rhs = treeop1;
12965 63582 : gcc_assert (ignore);
12966 :
12967 : /* Check for |= or &= of a bitfield of size one into another bitfield
12968 : of size 1. In this case, (unless we need the result of the
12969 : assignment) we can do this more efficiently with a
12970 : test followed by an assignment, if necessary.
12971 :
12972 : ??? At this point, we can't get a BIT_FIELD_REF here. But if
12973 : things change so we do, this code should be enhanced to
12974 : support it. */
12975 63582 : if (TREE_CODE (lhs) == COMPONENT_REF
12976 60510 : && (TREE_CODE (rhs) == BIT_IOR_EXPR
12977 60510 : || TREE_CODE (rhs) == BIT_AND_EXPR)
12978 0 : && TREE_OPERAND (rhs, 0) == lhs
12979 0 : && TREE_CODE (TREE_OPERAND (rhs, 1)) == COMPONENT_REF
12980 0 : && integer_onep (DECL_SIZE (TREE_OPERAND (lhs, 1)))
12981 63582 : && integer_onep (DECL_SIZE (TREE_OPERAND (TREE_OPERAND (rhs, 1), 1))))
12982 : {
12983 0 : rtx_code_label *label = gen_label_rtx ();
12984 0 : int value = TREE_CODE (rhs) == BIT_IOR_EXPR;
12985 0 : profile_probability prob = profile_probability::uninitialized ();
12986 0 : if (value)
12987 0 : jumpifnot (TREE_OPERAND (rhs, 1), label, prob);
12988 : else
12989 0 : jumpif (TREE_OPERAND (rhs, 1), label, prob);
12990 0 : expand_assignment (lhs, build_int_cst (TREE_TYPE (rhs), value),
12991 : false);
12992 0 : do_pending_stack_adjust ();
12993 0 : emit_label (label);
12994 0 : return const0_rtx;
12995 : }
12996 :
12997 63582 : expand_assignment (lhs, rhs, false);
12998 63582 : return const0_rtx;
12999 : }
13000 :
13001 13191271 : case ADDR_EXPR:
13002 13191271 : return expand_expr_addr_expr (exp, target, tmode, modifier);
13003 :
13004 172083 : case REALPART_EXPR:
13005 172083 : op0 = expand_normal (treeop0);
13006 172083 : return read_complex_part (op0, false);
13007 :
13008 195908 : case IMAGPART_EXPR:
13009 195908 : op0 = expand_normal (treeop0);
13010 195908 : return read_complex_part (op0, true);
13011 :
13012 0 : case RETURN_EXPR:
13013 0 : case LABEL_EXPR:
13014 0 : case GOTO_EXPR:
13015 0 : case SWITCH_EXPR:
13016 0 : case ASM_EXPR:
13017 : /* Expanded in cfgexpand.cc. */
13018 0 : gcc_unreachable ();
13019 :
13020 0 : case TRY_CATCH_EXPR:
13021 0 : case CATCH_EXPR:
13022 0 : case EH_FILTER_EXPR:
13023 0 : case TRY_FINALLY_EXPR:
13024 0 : case EH_ELSE_EXPR:
13025 : /* Lowered by tree-eh.cc. */
13026 0 : gcc_unreachable ();
13027 :
13028 0 : case WITH_CLEANUP_EXPR:
13029 0 : case CLEANUP_POINT_EXPR:
13030 0 : case TARGET_EXPR:
13031 0 : case CASE_LABEL_EXPR:
13032 0 : case VA_ARG_EXPR:
13033 0 : case BIND_EXPR:
13034 0 : case INIT_EXPR:
13035 0 : case CONJ_EXPR:
13036 0 : case COMPOUND_EXPR:
13037 0 : case PREINCREMENT_EXPR:
13038 0 : case PREDECREMENT_EXPR:
13039 0 : case POSTINCREMENT_EXPR:
13040 0 : case POSTDECREMENT_EXPR:
13041 0 : case LOOP_EXPR:
13042 0 : case EXIT_EXPR:
13043 0 : case COMPOUND_LITERAL_EXPR:
13044 : /* Lowered by gimplify.cc. */
13045 0 : gcc_unreachable ();
13046 :
13047 0 : case FDESC_EXPR:
13048 : /* Function descriptors are not valid except for as
13049 : initialization constants, and should not be expanded. */
13050 0 : gcc_unreachable ();
13051 :
13052 258 : case WITH_SIZE_EXPR:
13053 : /* WITH_SIZE_EXPR expands to its first argument. The caller should
13054 : have pulled out the size to use in whatever context it needed. */
13055 258 : return expand_expr_real (treeop0, original_target, tmode,
13056 258 : modifier, alt_rtl, inner_reference_p);
13057 :
13058 1847952 : default:
13059 1847952 : return expand_expr_real_2 (&ops, target, tmode, modifier);
13060 : }
13061 : }
13062 : #undef EXTEND_BITINT
13063 :
13064 : /* Subroutine of above: reduce EXP to the precision of TYPE (in the
13065 : signedness of TYPE), possibly returning the result in TARGET.
13066 : TYPE is known to be a partial integer type. */
13067 : static rtx
13068 88720 : reduce_to_bit_field_precision (rtx exp, rtx target, tree type)
13069 : {
13070 88720 : scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
13071 88720 : HOST_WIDE_INT prec = TYPE_PRECISION (type);
13072 88720 : gcc_assert ((GET_MODE (exp) == VOIDmode || GET_MODE (exp) == mode)
13073 : && (!target || GET_MODE (target) == mode));
13074 :
13075 : /* For constant values, reduce using wide_int_to_tree. */
13076 88720 : if (poly_int_rtx_p (exp))
13077 : {
13078 0 : auto value = wi::to_poly_wide (exp, mode);
13079 0 : tree t = wide_int_to_tree (type, value);
13080 0 : return expand_expr (t, target, VOIDmode, EXPAND_NORMAL);
13081 : }
13082 88720 : else if (TYPE_UNSIGNED (type))
13083 : {
13084 69784 : rtx mask = immed_wide_int_const
13085 69784 : (wi::mask (prec, false, GET_MODE_PRECISION (mode)), mode);
13086 69784 : return expand_and (mode, exp, mask, target);
13087 : }
13088 : else
13089 : {
13090 18936 : int count = GET_MODE_PRECISION (mode) - prec;
13091 18936 : exp = expand_shift (LSHIFT_EXPR, mode, exp, count, target, 0);
13092 18936 : return expand_shift (RSHIFT_EXPR, mode, exp, count, target, 0);
13093 : }
13094 : }
13095 :
13096 : /* Subroutine of above: returns true if OFFSET corresponds to an offset that
13097 : when applied to the address of EXP produces an address known to be
13098 : aligned more than BIGGEST_ALIGNMENT. */
13099 :
13100 : static bool
13101 277570 : is_aligning_offset (const_tree offset, const_tree exp)
13102 : {
13103 : /* Strip off any conversions. */
13104 295002 : while (CONVERT_EXPR_P (offset))
13105 17432 : offset = TREE_OPERAND (offset, 0);
13106 :
13107 : /* We must now have a BIT_AND_EXPR with a constant that is one less than
13108 : power of 2 and which is larger than BIGGEST_ALIGNMENT. */
13109 277570 : if (TREE_CODE (offset) != BIT_AND_EXPR
13110 0 : || !tree_fits_uhwi_p (TREE_OPERAND (offset, 1))
13111 0 : || compare_tree_int (TREE_OPERAND (offset, 1),
13112 0 : BIGGEST_ALIGNMENT / BITS_PER_UNIT) <= 0
13113 277570 : || !pow2p_hwi (tree_to_uhwi (TREE_OPERAND (offset, 1)) + 1))
13114 277570 : return false;
13115 :
13116 : /* Look at the first operand of BIT_AND_EXPR and strip any conversion.
13117 : It must be NEGATE_EXPR. Then strip any more conversions. */
13118 0 : offset = TREE_OPERAND (offset, 0);
13119 0 : while (CONVERT_EXPR_P (offset))
13120 0 : offset = TREE_OPERAND (offset, 0);
13121 :
13122 0 : if (TREE_CODE (offset) != NEGATE_EXPR)
13123 : return false;
13124 :
13125 0 : offset = TREE_OPERAND (offset, 0);
13126 0 : while (CONVERT_EXPR_P (offset))
13127 0 : offset = TREE_OPERAND (offset, 0);
13128 :
13129 : /* This must now be the address of EXP. */
13130 0 : return TREE_CODE (offset) == ADDR_EXPR && TREE_OPERAND (offset, 0) == exp;
13131 : }
13132 :
13133 : /* Return a STRING_CST corresponding to ARG's constant initializer either
13134 : if it's a string constant, or, when VALREP is set, any other constant,
13135 : or null otherwise.
13136 : On success, set *PTR_OFFSET to the (possibly non-constant) byte offset
13137 : within the byte string that ARG is references. If nonnull set *MEM_SIZE
13138 : to the size of the byte string. If nonnull, set *DECL to the constant
13139 : declaration ARG refers to. */
13140 :
13141 : static tree
13142 15615482 : constant_byte_string (tree arg, tree *ptr_offset, tree *mem_size, tree *decl,
13143 : bool valrep = false)
13144 : {
13145 15615482 : tree dummy = NULL_TREE;
13146 15615482 : if (!mem_size)
13147 11013 : mem_size = &dummy;
13148 :
13149 : /* Store the type of the original expression before conversions
13150 : via NOP_EXPR or POINTER_PLUS_EXPR to other types have been
13151 : removed. */
13152 15615482 : tree argtype = TREE_TYPE (arg);
13153 :
13154 15615482 : tree array;
13155 15615482 : STRIP_NOPS (arg);
13156 :
13157 : /* Non-constant index into the character array in an ARRAY_REF
13158 : expression or null. */
13159 15615482 : tree varidx = NULL_TREE;
13160 :
13161 15615482 : poly_int64 base_off = 0;
13162 :
13163 15615482 : if (TREE_CODE (arg) == ADDR_EXPR)
13164 : {
13165 6427910 : arg = TREE_OPERAND (arg, 0);
13166 6427910 : tree ref = arg;
13167 6427910 : if (TREE_CODE (arg) == ARRAY_REF)
13168 : {
13169 565231 : tree idx = TREE_OPERAND (arg, 1);
13170 565231 : if (TREE_CODE (idx) != INTEGER_CST)
13171 : {
13172 : /* From a pointer (but not array) argument extract the variable
13173 : index to prevent get_addr_base_and_unit_offset() from failing
13174 : due to it. Use it later to compute the non-constant offset
13175 : into the string and return it to the caller. */
13176 166989 : varidx = idx;
13177 166989 : ref = TREE_OPERAND (arg, 0);
13178 :
13179 166989 : if (TREE_CODE (TREE_TYPE (arg)) == ARRAY_TYPE)
13180 : return NULL_TREE;
13181 :
13182 28115 : if (!integer_zerop (array_ref_low_bound (arg)))
13183 : return NULL_TREE;
13184 :
13185 27355 : if (!integer_onep (array_ref_element_size (arg)))
13186 : return NULL_TREE;
13187 : }
13188 : }
13189 6287173 : array = get_addr_base_and_unit_offset (ref, &base_off);
13190 6287173 : if (!array
13191 6239739 : || (TREE_CODE (array) != VAR_DECL
13192 6239739 : && TREE_CODE (array) != CONST_DECL
13193 3917092 : && TREE_CODE (array) != STRING_CST))
13194 : return NULL_TREE;
13195 : }
13196 9187572 : else if (TREE_CODE (arg) == PLUS_EXPR || TREE_CODE (arg) == POINTER_PLUS_EXPR)
13197 : {
13198 26634 : tree arg0 = TREE_OPERAND (arg, 0);
13199 26634 : tree arg1 = TREE_OPERAND (arg, 1);
13200 :
13201 26634 : tree offset;
13202 26634 : tree str = string_constant (arg0, &offset, mem_size, decl);
13203 26634 : if (!str)
13204 : {
13205 17329 : str = string_constant (arg1, &offset, mem_size, decl);
13206 17329 : arg1 = arg0;
13207 : }
13208 :
13209 17329 : if (str)
13210 : {
13211 : /* Avoid pointers to arrays (see bug 86622). */
13212 9305 : if (POINTER_TYPE_P (TREE_TYPE (arg))
13213 9305 : && TREE_CODE (TREE_TYPE (TREE_TYPE (arg))) == ARRAY_TYPE
13214 1890 : && !(decl && !*decl)
13215 11825 : && !(decl && tree_fits_uhwi_p (DECL_SIZE_UNIT (*decl))
13216 1260 : && tree_fits_uhwi_p (*mem_size)
13217 1260 : && tree_int_cst_equal (*mem_size, DECL_SIZE_UNIT (*decl))))
13218 1890 : return NULL_TREE;
13219 :
13220 7415 : tree type = TREE_TYPE (offset);
13221 7415 : arg1 = fold_convert (type, arg1);
13222 7415 : *ptr_offset = fold_build2 (PLUS_EXPR, type, offset, arg1);
13223 7415 : return str;
13224 : }
13225 : return NULL_TREE;
13226 : }
13227 9160938 : else if (TREE_CODE (arg) == SSA_NAME)
13228 : {
13229 3906174 : gimple *stmt = SSA_NAME_DEF_STMT (arg);
13230 3906174 : if (!is_gimple_assign (stmt))
13231 : return NULL_TREE;
13232 :
13233 1043944 : tree rhs1 = gimple_assign_rhs1 (stmt);
13234 1043944 : tree_code code = gimple_assign_rhs_code (stmt);
13235 1043944 : if (code == ADDR_EXPR)
13236 234010 : return string_constant (rhs1, ptr_offset, mem_size, decl);
13237 809934 : else if (code != POINTER_PLUS_EXPR)
13238 : return NULL_TREE;
13239 :
13240 190303 : tree offset;
13241 190303 : if (tree str = string_constant (rhs1, &offset, mem_size, decl))
13242 : {
13243 : /* Avoid pointers to arrays (see bug 86622). */
13244 21604 : if (POINTER_TYPE_P (TREE_TYPE (rhs1))
13245 21604 : && TREE_CODE (TREE_TYPE (TREE_TYPE (rhs1))) == ARRAY_TYPE
13246 18907 : && !(decl && !*decl)
13247 37852 : && !(decl && tree_fits_uhwi_p (DECL_SIZE_UNIT (*decl))
13248 8124 : && tree_fits_uhwi_p (*mem_size)
13249 8124 : && tree_int_cst_equal (*mem_size, DECL_SIZE_UNIT (*decl))))
13250 15949 : return NULL_TREE;
13251 :
13252 5655 : tree rhs2 = gimple_assign_rhs2 (stmt);
13253 5655 : tree type = TREE_TYPE (offset);
13254 5655 : rhs2 = fold_convert (type, rhs2);
13255 5655 : *ptr_offset = fold_build2 (PLUS_EXPR, type, offset, rhs2);
13256 5655 : return str;
13257 : }
13258 : return NULL_TREE;
13259 : }
13260 5254764 : else if (DECL_P (arg))
13261 : array = arg;
13262 : else
13263 : return NULL_TREE;
13264 :
13265 8645443 : tree offset = wide_int_to_tree (sizetype, base_off);
13266 8645443 : if (varidx)
13267 : {
13268 2258 : if (TREE_CODE (TREE_TYPE (array)) != ARRAY_TYPE)
13269 : return NULL_TREE;
13270 :
13271 1747 : gcc_assert (TREE_CODE (arg) == ARRAY_REF);
13272 1747 : tree chartype = TREE_TYPE (TREE_TYPE (TREE_OPERAND (arg, 0)));
13273 1747 : if (TREE_CODE (chartype) != INTEGER_TYPE)
13274 : return NULL;
13275 :
13276 1591 : offset = fold_convert (sizetype, varidx);
13277 : }
13278 :
13279 8644776 : if (TREE_CODE (array) == STRING_CST)
13280 : {
13281 3526205 : *ptr_offset = fold_convert (sizetype, offset);
13282 3526205 : *mem_size = TYPE_SIZE_UNIT (TREE_TYPE (array));
13283 3526205 : if (decl)
13284 758852 : *decl = NULL_TREE;
13285 3526205 : gcc_checking_assert (tree_to_shwi (TYPE_SIZE_UNIT (TREE_TYPE (array)))
13286 : >= TREE_STRING_LENGTH (array));
13287 : return array;
13288 : }
13289 :
13290 5118571 : tree init = ctor_for_folding (array);
13291 5118571 : if (!init || init == error_mark_node)
13292 : return NULL_TREE;
13293 :
13294 158920 : if (valrep)
13295 : {
13296 58360 : HOST_WIDE_INT cstoff;
13297 58360 : if (!base_off.is_constant (&cstoff))
13298 : return NULL_TREE;
13299 :
13300 : /* Check that the host and target are sane. */
13301 58360 : if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
13302 : return NULL_TREE;
13303 :
13304 58360 : HOST_WIDE_INT typesz = int_size_in_bytes (TREE_TYPE (init));
13305 58360 : if (typesz <= 0 || (int) typesz != typesz)
13306 : return NULL_TREE;
13307 :
13308 58207 : HOST_WIDE_INT size = typesz;
13309 58207 : if (VAR_P (array)
13310 58195 : && DECL_SIZE_UNIT (array)
13311 116402 : && tree_fits_shwi_p (DECL_SIZE_UNIT (array)))
13312 : {
13313 58195 : size = tree_to_shwi (DECL_SIZE_UNIT (array));
13314 58195 : gcc_checking_assert (size >= typesz);
13315 : }
13316 :
13317 : /* If value representation was requested convert the initializer
13318 : for the whole array or object into a string of bytes forming
13319 : its value representation and return it. */
13320 58207 : unsigned char *bytes = XNEWVEC (unsigned char, size);
13321 58207 : int r = native_encode_initializer (init, bytes, size);
13322 58207 : if (r < typesz)
13323 : {
13324 60 : XDELETEVEC (bytes);
13325 60 : return NULL_TREE;
13326 : }
13327 :
13328 58147 : if (r < size)
13329 0 : memset (bytes + r, '\0', size - r);
13330 :
13331 58147 : const char *p = reinterpret_cast<const char *>(bytes);
13332 58147 : init = build_string_literal (size, p, char_type_node);
13333 58147 : init = TREE_OPERAND (init, 0);
13334 58147 : init = TREE_OPERAND (init, 0);
13335 58147 : XDELETE (bytes);
13336 :
13337 58147 : *mem_size = size_int (TREE_STRING_LENGTH (init));
13338 58147 : *ptr_offset = wide_int_to_tree (ssizetype, base_off);
13339 :
13340 58147 : if (decl)
13341 0 : *decl = array;
13342 :
13343 58147 : return init;
13344 : }
13345 :
13346 100560 : if (TREE_CODE (init) == CONSTRUCTOR)
13347 : {
13348 : /* Convert the 64-bit constant offset to a wider type to avoid
13349 : overflow and use it to obtain the initializer for the subobject
13350 : it points into. */
13351 80926 : offset_int wioff;
13352 80926 : if (!base_off.is_constant (&wioff))
13353 155 : return NULL_TREE;
13354 :
13355 80926 : wioff *= BITS_PER_UNIT;
13356 80926 : if (!wi::fits_uhwi_p (wioff))
13357 : return NULL_TREE;
13358 :
13359 80818 : base_off = wioff.to_uhwi ();
13360 80818 : unsigned HOST_WIDE_INT fieldoff = 0;
13361 80818 : init = fold_ctor_reference (TREE_TYPE (arg), init, base_off, 0, array,
13362 : &fieldoff);
13363 80818 : if (!init || init == error_mark_node)
13364 : return NULL_TREE;
13365 :
13366 80771 : HOST_WIDE_INT cstoff;
13367 80771 : if (!base_off.is_constant (&cstoff))
13368 : return NULL_TREE;
13369 :
13370 80771 : cstoff = (cstoff - fieldoff) / BITS_PER_UNIT;
13371 80771 : tree off = build_int_cst (sizetype, cstoff);
13372 80771 : if (varidx)
13373 944 : offset = fold_build2 (PLUS_EXPR, TREE_TYPE (offset), offset, off);
13374 : else
13375 : offset = off;
13376 : }
13377 :
13378 100405 : *ptr_offset = offset;
13379 :
13380 100405 : tree inittype = TREE_TYPE (init);
13381 :
13382 100405 : if (TREE_CODE (init) == INTEGER_CST
13383 100405 : && (TREE_CODE (TREE_TYPE (array)) == INTEGER_TYPE
13384 1108 : || TYPE_MAIN_VARIANT (inittype) == char_type_node))
13385 : {
13386 : /* Check that the host and target are sane. */
13387 901 : if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
13388 : return NULL_TREE;
13389 :
13390 : /* For a reference to (address of) a single constant character,
13391 : store the native representation of the character in CHARBUF.
13392 : If the reference is to an element of an array or a member
13393 : of a struct, only consider narrow characters until ctors
13394 : for wide character arrays are transformed to STRING_CSTs
13395 : like those for narrow arrays. */
13396 901 : unsigned char charbuf[MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT];
13397 901 : int len = native_encode_expr (init, charbuf, sizeof charbuf, 0);
13398 901 : if (len > 0)
13399 : {
13400 : /* Construct a string literal with elements of INITTYPE and
13401 : the representation above. Then strip
13402 : the ADDR_EXPR (ARRAY_REF (...)) around the STRING_CST. */
13403 901 : init = build_string_literal (len, (char *)charbuf, inittype);
13404 901 : init = TREE_OPERAND (TREE_OPERAND (init, 0), 0);
13405 : }
13406 : }
13407 :
13408 100405 : tree initsize = TYPE_SIZE_UNIT (inittype);
13409 :
13410 100405 : if (TREE_CODE (init) == CONSTRUCTOR && initializer_zerop (init))
13411 : {
13412 : /* Fold an empty/zero constructor for an implicitly initialized
13413 : object or subobject into the empty string. */
13414 :
13415 : /* Determine the character type from that of the original
13416 : expression. */
13417 9337 : tree chartype = argtype;
13418 9337 : if (POINTER_TYPE_P (chartype))
13419 9330 : chartype = TREE_TYPE (chartype);
13420 14958 : while (TREE_CODE (chartype) == ARRAY_TYPE)
13421 5621 : chartype = TREE_TYPE (chartype);
13422 :
13423 9337 : if (INTEGRAL_TYPE_P (chartype)
13424 9337 : && TYPE_PRECISION (chartype) == TYPE_PRECISION (char_type_node))
13425 : {
13426 : /* Convert a char array to an empty STRING_CST having an array
13427 : of the expected type and size. */
13428 9133 : if (!initsize)
13429 3090 : initsize = integer_zero_node;
13430 6043 : else if (!tree_fits_uhwi_p (initsize))
13431 : return NULL_TREE;
13432 :
13433 9127 : unsigned HOST_WIDE_INT size = tree_to_uhwi (initsize);
13434 9127 : if (size > (unsigned HOST_WIDE_INT) INT_MAX)
13435 : return NULL_TREE;
13436 :
13437 9127 : init = build_string_literal (size, NULL, chartype, size);
13438 9127 : init = TREE_OPERAND (init, 0);
13439 9127 : init = TREE_OPERAND (init, 0);
13440 :
13441 9127 : *ptr_offset = integer_zero_node;
13442 : }
13443 : }
13444 :
13445 100399 : if (decl)
13446 54390 : *decl = array;
13447 :
13448 100399 : if (TREE_CODE (init) != STRING_CST)
13449 : return NULL_TREE;
13450 :
13451 68386 : *mem_size = initsize;
13452 :
13453 68386 : gcc_checking_assert (tree_to_shwi (initsize) >= TREE_STRING_LENGTH (init));
13454 :
13455 : return init;
13456 : }
13457 :
13458 : /* Return STRING_CST if an ARG corresponds to a string constant or zero
13459 : if it doesn't. If we return nonzero, set *PTR_OFFSET to the (possibly
13460 : non-constant) offset in bytes within the string that ARG is accessing.
13461 : If MEM_SIZE is non-zero the storage size of the memory is returned.
13462 : If DECL is non-zero the constant declaration is returned if available. */
13463 :
13464 : tree
13465 10938913 : string_constant (tree arg, tree *ptr_offset, tree *mem_size, tree *decl)
13466 : {
13467 10938913 : return constant_byte_string (arg, ptr_offset, mem_size, decl, false);
13468 : }
13469 :
13470 : /* Similar to string_constant, return a STRING_CST corresponding
13471 : to the value representation of the first argument if it's
13472 : a constant. */
13473 :
13474 : tree
13475 4676569 : byte_representation (tree arg, tree *ptr_offset, tree *mem_size, tree *decl)
13476 : {
13477 4676569 : return constant_byte_string (arg, ptr_offset, mem_size, decl, true);
13478 : }
13479 :
13480 : /* Optimize x % C1 == C2 for signed modulo if C1 is a power of two and C2
13481 : is non-zero and C3 ((1<<(prec-1)) | (C1 - 1)):
13482 : for C2 > 0 to x & C3 == C2
13483 : for C2 < 0 to x & C3 == (C2 & C3). */
13484 : enum tree_code
13485 35 : maybe_optimize_pow2p_mod_cmp (enum tree_code code, tree *arg0, tree *arg1)
13486 : {
13487 35 : gimple *stmt = get_def_for_expr (*arg0, TRUNC_MOD_EXPR);
13488 35 : tree treeop0 = gimple_assign_rhs1 (stmt);
13489 35 : tree treeop1 = gimple_assign_rhs2 (stmt);
13490 35 : tree type = TREE_TYPE (*arg0);
13491 35 : scalar_int_mode mode;
13492 35 : if (!is_a <scalar_int_mode> (TYPE_MODE (type), &mode))
13493 : return code;
13494 70 : if (GET_MODE_BITSIZE (mode) != TYPE_PRECISION (type)
13495 35 : || TYPE_PRECISION (type) <= 1
13496 35 : || TYPE_UNSIGNED (type)
13497 : /* Signed x % c == 0 should have been optimized into unsigned modulo
13498 : earlier. */
13499 35 : || integer_zerop (*arg1)
13500 : /* If c is known to be non-negative, modulo will be expanded as unsigned
13501 : modulo. */
13502 70 : || get_range_pos_neg (treeop0, currently_expanding_gimple_stmt) == 1)
13503 0 : return code;
13504 :
13505 : /* x % c == d where d < 0 && d <= -c should be always false. */
13506 35 : if (tree_int_cst_sgn (*arg1) == -1
13507 51 : && -wi::to_widest (treeop1) >= wi::to_widest (*arg1))
13508 : return code;
13509 :
13510 35 : int prec = TYPE_PRECISION (type);
13511 35 : wide_int w = wi::to_wide (treeop1) - 1;
13512 35 : w |= wi::shifted_mask (0, prec - 1, true, prec);
13513 35 : tree c3 = wide_int_to_tree (type, w);
13514 35 : tree c4 = *arg1;
13515 35 : if (tree_int_cst_sgn (*arg1) == -1)
13516 16 : c4 = wide_int_to_tree (type, w & wi::to_wide (*arg1));
13517 :
13518 35 : rtx op0 = expand_normal (treeop0);
13519 35 : treeop0 = make_tree (TREE_TYPE (treeop0), op0);
13520 :
13521 35 : bool speed_p = optimize_insn_for_speed_p ();
13522 :
13523 35 : do_pending_stack_adjust ();
13524 :
13525 35 : location_t loc = gimple_location (stmt);
13526 35 : struct separate_ops ops;
13527 35 : ops.code = TRUNC_MOD_EXPR;
13528 35 : ops.location = loc;
13529 35 : ops.type = TREE_TYPE (treeop0);
13530 35 : ops.op0 = treeop0;
13531 35 : ops.op1 = treeop1;
13532 35 : ops.op2 = NULL_TREE;
13533 35 : start_sequence ();
13534 35 : rtx mor = expand_expr_real_2 (&ops, NULL_RTX, TYPE_MODE (ops.type),
13535 : EXPAND_NORMAL);
13536 35 : rtx_insn *moinsns = end_sequence ();
13537 :
13538 35 : unsigned mocost = seq_cost (moinsns, speed_p);
13539 35 : mocost += rtx_cost (mor, mode, EQ, 0, speed_p);
13540 35 : mocost += rtx_cost (expand_normal (*arg1), mode, EQ, 1, speed_p);
13541 :
13542 35 : ops.code = BIT_AND_EXPR;
13543 35 : ops.location = loc;
13544 35 : ops.type = TREE_TYPE (treeop0);
13545 35 : ops.op0 = treeop0;
13546 35 : ops.op1 = c3;
13547 35 : ops.op2 = NULL_TREE;
13548 35 : start_sequence ();
13549 35 : rtx mur = expand_expr_real_2 (&ops, NULL_RTX, TYPE_MODE (ops.type),
13550 : EXPAND_NORMAL);
13551 35 : rtx_insn *muinsns = end_sequence ();
13552 :
13553 35 : unsigned mucost = seq_cost (muinsns, speed_p);
13554 35 : mucost += rtx_cost (mur, mode, EQ, 0, speed_p);
13555 35 : mucost += rtx_cost (expand_normal (c4), mode, EQ, 1, speed_p);
13556 :
13557 35 : if (mocost <= mucost)
13558 : {
13559 0 : emit_insn (moinsns);
13560 0 : *arg0 = make_tree (TREE_TYPE (*arg0), mor);
13561 0 : return code;
13562 : }
13563 :
13564 35 : emit_insn (muinsns);
13565 35 : *arg0 = make_tree (TREE_TYPE (*arg0), mur);
13566 35 : *arg1 = c4;
13567 35 : return code;
13568 35 : }
13569 :
13570 : /* Attempt to optimize unsigned (X % C1) == C2 (or (X % C1) != C2).
13571 : If C1 is odd to:
13572 : (X - C2) * C3 <= C4 (or >), where
13573 : C3 is modular multiplicative inverse of C1 and 1<<prec and
13574 : C4 is ((1<<prec) - 1) / C1 or ((1<<prec) - 1) / C1 - 1 (the latter
13575 : if C2 > ((1<<prec) - 1) % C1).
13576 : If C1 is even, S = ctz (C1) and C2 is 0, use
13577 : ((X * C3) r>> S) <= C4, where C3 is modular multiplicative
13578 : inverse of C1>>S and 1<<prec and C4 is (((1<<prec) - 1) / (C1>>S)) >> S.
13579 :
13580 : For signed (X % C1) == 0 if C1 is odd to (all operations in it
13581 : unsigned):
13582 : (X * C3) + C4 <= 2 * C4, where
13583 : C3 is modular multiplicative inverse of (unsigned) C1 and 1<<prec and
13584 : C4 is ((1<<(prec - 1) - 1) / C1).
13585 : If C1 is even, S = ctz(C1), use
13586 : ((X * C3) + C4) r>> S <= (C4 >> (S - 1))
13587 : where C3 is modular multiplicative inverse of (unsigned)(C1>>S) and 1<<prec
13588 : and C4 is ((1<<(prec - 1) - 1) / (C1>>S)) & (-1<<S).
13589 :
13590 : See the Hacker's Delight book, section 10-17. */
13591 : enum tree_code
13592 3348964 : maybe_optimize_mod_cmp (enum tree_code code, tree *arg0, tree *arg1)
13593 : {
13594 3348964 : gcc_checking_assert (code == EQ_EXPR || code == NE_EXPR);
13595 3348964 : gcc_checking_assert (TREE_CODE (*arg1) == INTEGER_CST);
13596 :
13597 3348964 : if (optimize < 2)
13598 : return code;
13599 :
13600 2534489 : gimple *stmt = get_def_for_expr (*arg0, TRUNC_MOD_EXPR);
13601 2534489 : if (stmt == NULL)
13602 : return code;
13603 :
13604 2429 : tree treeop0 = gimple_assign_rhs1 (stmt);
13605 2429 : tree treeop1 = gimple_assign_rhs2 (stmt);
13606 2429 : if (TREE_CODE (treeop0) != SSA_NAME
13607 2321 : || TREE_CODE (treeop1) != INTEGER_CST
13608 : /* Don't optimize the undefined behavior case x % 0;
13609 : x % 1 should have been optimized into zero, punt if
13610 : it makes it here for whatever reason;
13611 : x % -c should have been optimized into x % c. */
13612 1966 : || compare_tree_int (treeop1, 2) <= 0
13613 : /* Likewise x % c == d where d >= c should be always false. */
13614 4319 : || tree_int_cst_le (treeop1, *arg1))
13615 539 : return code;
13616 :
13617 : /* Unsigned x % pow2 is handled right already, for signed
13618 : modulo handle it in maybe_optimize_pow2p_mod_cmp. */
13619 1890 : if (integer_pow2p (treeop1))
13620 35 : return maybe_optimize_pow2p_mod_cmp (code, arg0, arg1);
13621 :
13622 1855 : tree type = TREE_TYPE (*arg0);
13623 1855 : scalar_int_mode mode;
13624 3348936 : if (!is_a <scalar_int_mode> (TYPE_MODE (type), &mode))
13625 : return code;
13626 3710 : if (GET_MODE_BITSIZE (mode) != TYPE_PRECISION (type)
13627 1855 : || TYPE_PRECISION (type) <= 1)
13628 : return code;
13629 :
13630 1855 : signop sgn = UNSIGNED;
13631 : /* If both operands are known to have the sign bit clear, handle
13632 : even the signed modulo case as unsigned. treeop1 is always
13633 : positive >= 2, checked above. */
13634 1855 : if (!TYPE_UNSIGNED (type)
13635 1855 : && get_range_pos_neg (treeop0, currently_expanding_gimple_stmt) != 1)
13636 : sgn = SIGNED;
13637 :
13638 1855 : if (!TYPE_UNSIGNED (type))
13639 : {
13640 1627 : if (tree_int_cst_sgn (*arg1) == -1)
13641 : return code;
13642 1620 : type = unsigned_type_for (type);
13643 1620 : if (!type || TYPE_MODE (type) != TYPE_MODE (TREE_TYPE (*arg0)))
13644 0 : return code;
13645 : }
13646 :
13647 1848 : int prec = TYPE_PRECISION (type);
13648 1848 : wide_int w = wi::to_wide (treeop1);
13649 1848 : int shift = wi::ctz (w);
13650 : /* Unsigned (X % C1) == C2 is equivalent to (X - C2) % C1 == 0 if
13651 : C2 <= -1U % C1, because for any Z >= 0U - C2 in that case (Z % C1) != 0.
13652 : If C1 is odd, we can handle all cases by subtracting
13653 : C4 below. We could handle even the even C1 and C2 > -1U % C1 cases
13654 : e.g. by testing for overflow on the subtraction, punt on that for now
13655 : though. */
13656 1848 : if ((sgn == SIGNED || shift) && !integer_zerop (*arg1))
13657 : {
13658 189 : if (sgn == SIGNED)
13659 174 : return code;
13660 26 : wide_int x = wi::umod_trunc (wi::mask (prec, false, prec), w);
13661 26 : if (wi::gtu_p (wi::to_wide (*arg1), x))
13662 11 : return code;
13663 26 : }
13664 :
13665 1674 : imm_use_iterator imm_iter;
13666 1674 : use_operand_p use_p;
13667 12406 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, treeop0)
13668 : {
13669 9068 : gimple *use_stmt = USE_STMT (use_p);
13670 : /* Punt if treeop0 is used in the same bb in a division
13671 : or another modulo with the same divisor. We should expect
13672 : the division and modulo combined together. */
13673 17700 : if (use_stmt == stmt
13674 9068 : || gimple_bb (use_stmt) != gimple_bb (stmt))
13675 8632 : continue;
13676 436 : if (!is_gimple_assign (use_stmt)
13677 436 : || (gimple_assign_rhs_code (use_stmt) != TRUNC_DIV_EXPR
13678 420 : && gimple_assign_rhs_code (use_stmt) != TRUNC_MOD_EXPR))
13679 411 : continue;
13680 25 : if (gimple_assign_rhs1 (use_stmt) != treeop0
13681 25 : || !operand_equal_p (gimple_assign_rhs2 (use_stmt), treeop1, 0))
13682 15 : continue;
13683 10 : return code;
13684 10 : }
13685 :
13686 1664 : w = wi::lrshift (w, shift);
13687 1664 : wide_int a = wide_int::from (w, prec + 1, UNSIGNED);
13688 1664 : wide_int b = wi::shifted_mask (prec, 1, false, prec + 1);
13689 1664 : wide_int m = wide_int::from (wi::mod_inv (a, b), prec, UNSIGNED);
13690 1664 : tree c3 = wide_int_to_tree (type, m);
13691 1664 : tree c5 = NULL_TREE;
13692 1664 : wide_int d, e;
13693 1664 : if (sgn == UNSIGNED)
13694 : {
13695 1059 : d = wi::divmod_trunc (wi::mask (prec, false, prec), w, UNSIGNED, &e);
13696 : /* Use <= floor ((1<<prec) - 1) / C1 only if C2 <= ((1<<prec) - 1) % C1,
13697 : otherwise use < or subtract one from C4. E.g. for
13698 : x % 3U == 0 we transform this into x * 0xaaaaaaab <= 0x55555555, but
13699 : x % 3U == 1 already needs to be
13700 : (x - 1) * 0xaaaaaaabU <= 0x55555554. */
13701 1059 : if (!shift && wi::gtu_p (wi::to_wide (*arg1), e))
13702 15 : d -= 1;
13703 1059 : if (shift)
13704 442 : d = wi::lrshift (d, shift);
13705 : }
13706 : else
13707 : {
13708 605 : e = wi::udiv_trunc (wi::mask (prec - 1, false, prec), w);
13709 605 : if (!shift)
13710 406 : d = wi::lshift (e, 1);
13711 : else
13712 : {
13713 199 : e = wi::bit_and (e, wi::mask (shift, true, prec));
13714 199 : d = wi::lrshift (e, shift - 1);
13715 : }
13716 605 : c5 = wide_int_to_tree (type, e);
13717 : }
13718 1664 : tree c4 = wide_int_to_tree (type, d);
13719 :
13720 1664 : rtx op0 = expand_normal (treeop0);
13721 1664 : treeop0 = make_tree (TREE_TYPE (treeop0), op0);
13722 :
13723 1664 : bool speed_p = optimize_insn_for_speed_p ();
13724 :
13725 1664 : do_pending_stack_adjust ();
13726 :
13727 1664 : location_t loc = gimple_location (stmt);
13728 1664 : struct separate_ops ops;
13729 1664 : ops.code = TRUNC_MOD_EXPR;
13730 1664 : ops.location = loc;
13731 1664 : ops.type = TREE_TYPE (treeop0);
13732 1664 : ops.op0 = treeop0;
13733 1664 : ops.op1 = treeop1;
13734 1664 : ops.op2 = NULL_TREE;
13735 1664 : start_sequence ();
13736 1664 : rtx mor = expand_expr_real_2 (&ops, NULL_RTX, TYPE_MODE (ops.type),
13737 : EXPAND_NORMAL);
13738 1664 : rtx_insn *moinsns = end_sequence ();
13739 :
13740 1664 : unsigned mocost = seq_cost (moinsns, speed_p);
13741 1664 : mocost += rtx_cost (mor, mode, EQ, 0, speed_p);
13742 1664 : mocost += rtx_cost (expand_normal (*arg1), mode, EQ, 1, speed_p);
13743 :
13744 1664 : tree t = fold_convert_loc (loc, type, treeop0);
13745 1664 : if (!integer_zerop (*arg1))
13746 36 : t = fold_build2_loc (loc, MINUS_EXPR, type, t, fold_convert (type, *arg1));
13747 1664 : t = fold_build2_loc (loc, MULT_EXPR, type, t, c3);
13748 1664 : if (sgn == SIGNED)
13749 605 : t = fold_build2_loc (loc, PLUS_EXPR, type, t, c5);
13750 1664 : if (shift)
13751 : {
13752 641 : tree s = build_int_cst (NULL_TREE, shift);
13753 641 : t = fold_build2_loc (loc, RROTATE_EXPR, type, t, s);
13754 : }
13755 :
13756 1664 : start_sequence ();
13757 1664 : rtx mur = expand_normal (t);
13758 1664 : rtx_insn *muinsns = end_sequence ();
13759 :
13760 1664 : unsigned mucost = seq_cost (muinsns, speed_p);
13761 1664 : mucost += rtx_cost (mur, mode, LE, 0, speed_p);
13762 1664 : mucost += rtx_cost (expand_normal (c4), mode, LE, 1, speed_p);
13763 :
13764 1664 : if (mocost <= mucost)
13765 : {
13766 266 : emit_insn (moinsns);
13767 266 : *arg0 = make_tree (TREE_TYPE (*arg0), mor);
13768 266 : return code;
13769 : }
13770 :
13771 1398 : emit_insn (muinsns);
13772 1398 : *arg0 = make_tree (type, mur);
13773 1398 : *arg1 = c4;
13774 1398 : return code == EQ_EXPR ? LE_EXPR : GT_EXPR;
13775 3512 : }
13776 :
13777 : /* Optimize x - y < 0 into x < 0 if x - y has undefined overflow. */
13778 :
13779 : void
13780 240705 : maybe_optimize_sub_cmp_0 (enum tree_code code, tree *arg0, tree *arg1)
13781 : {
13782 240705 : gcc_checking_assert (code == GT_EXPR || code == GE_EXPR
13783 : || code == LT_EXPR || code == LE_EXPR);
13784 240705 : gcc_checking_assert (integer_zerop (*arg1));
13785 :
13786 240705 : if (!optimize)
13787 : return;
13788 :
13789 207085 : gimple *stmt = get_def_for_expr (*arg0, MINUS_EXPR);
13790 207085 : if (stmt == NULL)
13791 : return;
13792 :
13793 1351 : tree treeop0 = gimple_assign_rhs1 (stmt);
13794 1351 : tree treeop1 = gimple_assign_rhs2 (stmt);
13795 1351 : if (!TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (treeop0)))
13796 : return;
13797 :
13798 1254 : *arg0 = treeop0;
13799 1254 : *arg1 = treeop1;
13800 : }
13801 :
13802 :
13803 : /* Expand CODE with arguments INNER & (1<<BITNUM) and 0 that represents
13804 : a single bit equality/inequality test, returns where the result is located. */
13805 :
13806 : static rtx
13807 9792 : expand_single_bit_test (location_t loc, enum tree_code code,
13808 : tree inner, int bitnum,
13809 : tree result_type, rtx target,
13810 : machine_mode mode)
13811 : {
13812 9792 : gcc_assert (code == NE_EXPR || code == EQ_EXPR);
13813 :
13814 9792 : tree type = TREE_TYPE (inner);
13815 9792 : scalar_int_mode operand_mode = SCALAR_INT_TYPE_MODE (type);
13816 9792 : int ops_unsigned;
13817 9792 : tree signed_type, unsigned_type, intermediate_type;
13818 9792 : gimple *inner_def;
13819 :
13820 : /* First, see if we can fold the single bit test into a sign-bit
13821 : test. */
13822 9792 : if (bitnum == TYPE_PRECISION (type) - 1
13823 9792 : && type_has_mode_precision_p (type))
13824 : {
13825 241 : tree stype = signed_type_for (type);
13826 444 : tree tmp = fold_build2_loc (loc, code == EQ_EXPR ? GE_EXPR : LT_EXPR,
13827 : result_type,
13828 : fold_convert_loc (loc, stype, inner),
13829 : build_int_cst (stype, 0));
13830 241 : return expand_expr (tmp, target, VOIDmode, EXPAND_NORMAL);
13831 : }
13832 :
13833 : /* Otherwise we have (A & C) != 0 where C is a single bit,
13834 : convert that into ((A >> C2) & 1). Where C2 = log2(C).
13835 : Similarly for (A & C) == 0. */
13836 :
13837 : /* If INNER is a right shift of a constant and it plus BITNUM does
13838 : not overflow, adjust BITNUM and INNER. */
13839 9551 : if ((inner_def = get_def_for_expr (inner, RSHIFT_EXPR))
13840 93 : && TREE_CODE (gimple_assign_rhs2 (inner_def)) == INTEGER_CST
13841 16 : && bitnum < TYPE_PRECISION (type)
13842 9567 : && wi::ltu_p (wi::to_wide (gimple_assign_rhs2 (inner_def)),
13843 9551 : TYPE_PRECISION (type) - bitnum))
13844 : {
13845 16 : bitnum += tree_to_uhwi (gimple_assign_rhs2 (inner_def));
13846 16 : inner = gimple_assign_rhs1 (inner_def);
13847 : }
13848 :
13849 : /* If we are going to be able to omit the AND below, we must do our
13850 : operations as unsigned. If we must use the AND, we have a choice.
13851 : Normally unsigned is faster, but for some machines signed is. */
13852 9551 : ops_unsigned = (load_extend_op (operand_mode) == SIGN_EXTEND
13853 : && !flag_syntax_only) ? 0 : 1;
13854 :
13855 9551 : signed_type = lang_hooks.types.type_for_mode (operand_mode, 0);
13856 9551 : unsigned_type = lang_hooks.types.type_for_mode (operand_mode, 1);
13857 9551 : intermediate_type = ops_unsigned ? unsigned_type : signed_type;
13858 9551 : inner = fold_convert_loc (loc, intermediate_type, inner);
13859 :
13860 9551 : rtx inner0 = expand_expr (inner, NULL_RTX, VOIDmode, EXPAND_NORMAL);
13861 :
13862 9551 : if (CONST_SCALAR_INT_P (inner0))
13863 : {
13864 0 : wide_int t = rtx_mode_t (inner0, operand_mode);
13865 0 : bool setp = (wi::lrshift (t, bitnum) & 1) != 0;
13866 0 : return (setp ^ (code == EQ_EXPR)) ? const1_rtx : const0_rtx;
13867 0 : }
13868 9551 : int bitpos = bitnum;
13869 :
13870 9551 : if (BYTES_BIG_ENDIAN)
13871 : bitpos = GET_MODE_BITSIZE (operand_mode) - 1 - bitpos;
13872 :
13873 9551 : inner0 = extract_bit_field (inner0, 1, bitpos, 1, target,
13874 : operand_mode, mode, 0, NULL);
13875 :
13876 9551 : if (code == EQ_EXPR)
13877 2672 : inner0 = expand_binop (GET_MODE (inner0), xor_optab, inner0, const1_rtx,
13878 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
13879 9551 : if (GET_MODE (inner0) != mode)
13880 : {
13881 0 : rtx t = gen_reg_rtx (mode);
13882 0 : convert_move (t, inner0, 0);
13883 0 : return t;
13884 : }
13885 : return inner0;
13886 : }
13887 :
13888 : /* Generate code to calculate OPS, and exploded expression
13889 : using a store-flag instruction and return an rtx for the result.
13890 : OPS reflects a comparison.
13891 :
13892 : If TARGET is nonzero, store the result there if convenient.
13893 :
13894 : Return zero if there is no suitable set-flag instruction
13895 : available on this machine.
13896 :
13897 : Once expand_expr has been called on the arguments of the comparison,
13898 : we are committed to doing the store flag, since it is not safe to
13899 : re-evaluate the expression. We emit the store-flag insn by calling
13900 : emit_store_flag, but only expand the arguments if we have a reason
13901 : to believe that emit_store_flag will be successful. If we think that
13902 : it will, but it isn't, we have to simulate the store-flag with a
13903 : set/jump/set sequence. */
13904 :
13905 : static rtx
13906 637576 : do_store_flag (const_sepops ops, rtx target, machine_mode mode)
13907 : {
13908 637576 : enum rtx_code code;
13909 637576 : tree arg0, arg1, type;
13910 637576 : machine_mode operand_mode;
13911 637576 : int unsignedp;
13912 637576 : rtx op0, op1;
13913 637576 : rtx subtarget = target;
13914 637576 : location_t loc = ops->location;
13915 637576 : unsigned HOST_WIDE_INT nunits;
13916 :
13917 637576 : arg0 = ops->op0;
13918 637576 : arg1 = ops->op1;
13919 :
13920 : /* Don't crash if the comparison was erroneous. */
13921 637576 : if (arg0 == error_mark_node || arg1 == error_mark_node)
13922 0 : return const0_rtx;
13923 :
13924 637576 : type = TREE_TYPE (arg0);
13925 637576 : operand_mode = TYPE_MODE (type);
13926 637576 : unsignedp = TYPE_UNSIGNED (type);
13927 :
13928 : /* We won't bother with BLKmode store-flag operations because it would mean
13929 : passing a lot of information to emit_store_flag. */
13930 637576 : if (operand_mode == BLKmode)
13931 : return 0;
13932 :
13933 : /* We won't bother with store-flag operations involving function pointers
13934 : when function pointers must be canonicalized before comparisons. */
13935 637576 : if (targetm.have_canonicalize_funcptr_for_compare ()
13936 637576 : && ((POINTER_TYPE_P (TREE_TYPE (arg0))
13937 0 : && FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (arg0))))
13938 0 : || (POINTER_TYPE_P (TREE_TYPE (arg1))
13939 0 : && FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (arg1))))))
13940 : return 0;
13941 :
13942 637576 : STRIP_NOPS (arg0);
13943 637576 : STRIP_NOPS (arg1);
13944 :
13945 : /* For vector typed comparisons emit code to generate the desired
13946 : all-ones or all-zeros mask. */
13947 637576 : if (VECTOR_TYPE_P (ops->type))
13948 : {
13949 26159 : tree ifexp = build2 (ops->code, ops->type, arg0, arg1);
13950 26159 : if (VECTOR_BOOLEAN_TYPE_P (ops->type)
13951 52318 : && expand_vec_cmp_expr_p (TREE_TYPE (arg0), ops->type, ops->code))
13952 26159 : return expand_vec_cmp_expr (ops->type, ifexp, target);
13953 : else
13954 0 : gcc_unreachable ();
13955 : }
13956 :
13957 : /* Optimize (x % C1) == C2 or (x % C1) != C2 if it is beneficial
13958 : into (x - C2) * C3 < C4. */
13959 611417 : if ((ops->code == EQ_EXPR || ops->code == NE_EXPR)
13960 412695 : && TREE_CODE (arg0) == SSA_NAME
13961 412120 : && TREE_CODE (arg1) == INTEGER_CST)
13962 : {
13963 246410 : enum tree_code new_code = maybe_optimize_mod_cmp (ops->code,
13964 : &arg0, &arg1);
13965 246410 : if (new_code != ops->code)
13966 : {
13967 106 : struct separate_ops nops = *ops;
13968 106 : nops.code = new_code;
13969 106 : nops.op0 = arg0;
13970 106 : nops.op1 = arg1;
13971 106 : nops.type = TREE_TYPE (arg0);
13972 106 : return do_store_flag (&nops, target, mode);
13973 : }
13974 : }
13975 :
13976 : /* Optimize (x - y) < 0 into x < y if x - y has undefined overflow. */
13977 611311 : if (!unsignedp
13978 439340 : && (ops->code == LT_EXPR || ops->code == LE_EXPR
13979 439340 : || ops->code == GT_EXPR || ops->code == GE_EXPR)
13980 126678 : && integer_zerop (arg1)
13981 649811 : && TREE_CODE (arg0) == SSA_NAME)
13982 38496 : maybe_optimize_sub_cmp_0 (ops->code, &arg0, &arg1);
13983 :
13984 : /* Get the rtx comparison code to use. We know that EXP is a comparison
13985 : operation of some type. Some comparisons against 1 and -1 can be
13986 : converted to comparisons with zero. Do so here so that the tests
13987 : below will be aware that we have a comparison with zero. These
13988 : tests will not catch constants in the first operand, but constants
13989 : are rarely passed as the first operand. */
13990 :
13991 611311 : switch (ops->code)
13992 : {
13993 : case EQ_EXPR:
13994 : code = EQ;
13995 : break;
13996 234678 : case NE_EXPR:
13997 234678 : code = NE;
13998 234678 : break;
13999 40006 : case LT_EXPR:
14000 40006 : if (integer_onep (arg1))
14001 0 : arg1 = integer_zero_node, code = unsignedp ? LEU : LE;
14002 : else
14003 40006 : code = unsignedp ? LTU : LT;
14004 : break;
14005 33485 : case LE_EXPR:
14006 33485 : if (! unsignedp && integer_all_onesp (arg1))
14007 0 : arg1 = integer_zero_node, code = LT;
14008 : else
14009 33485 : code = unsignedp ? LEU : LE;
14010 : break;
14011 53847 : case GT_EXPR:
14012 53847 : if (! unsignedp && integer_all_onesp (arg1))
14013 0 : arg1 = integer_zero_node, code = GE;
14014 : else
14015 53847 : code = unsignedp ? GTU : GT;
14016 : break;
14017 46945 : case GE_EXPR:
14018 46945 : if (integer_onep (arg1))
14019 29246 : arg1 = integer_zero_node, code = unsignedp ? GTU : GT;
14020 : else
14021 46945 : code = unsignedp ? GEU : GE;
14022 : break;
14023 :
14024 1016 : case UNORDERED_EXPR:
14025 1016 : code = UNORDERED;
14026 1016 : break;
14027 712 : case ORDERED_EXPR:
14028 712 : code = ORDERED;
14029 712 : break;
14030 523 : case UNLT_EXPR:
14031 523 : code = UNLT;
14032 523 : break;
14033 11121 : case UNLE_EXPR:
14034 11121 : code = UNLE;
14035 11121 : break;
14036 942 : case UNGT_EXPR:
14037 942 : code = UNGT;
14038 942 : break;
14039 9916 : case UNGE_EXPR:
14040 9916 : code = UNGE;
14041 9916 : break;
14042 135 : case UNEQ_EXPR:
14043 135 : code = UNEQ;
14044 135 : break;
14045 74 : case LTGT_EXPR:
14046 74 : code = LTGT;
14047 74 : break;
14048 :
14049 0 : default:
14050 0 : gcc_unreachable ();
14051 : }
14052 :
14053 : /* Put a constant second. */
14054 611311 : if (TREE_CODE (arg0) == REAL_CST || TREE_CODE (arg0) == INTEGER_CST
14055 609511 : || TREE_CODE (arg0) == FIXED_CST)
14056 : {
14057 1800 : std::swap (arg0, arg1);
14058 1800 : code = swap_condition (code);
14059 : }
14060 :
14061 : /* If this is an equality or inequality test of a single bit, we can
14062 : do this by shifting the bit being tested to the low-order bit and
14063 : masking the result with the constant 1. If the condition was EQ,
14064 : we xor it with 1. This does not require an scc insn and is faster
14065 : than an scc insn even if we have it. */
14066 :
14067 611311 : if ((code == NE || code == EQ)
14068 412589 : && (integer_zerop (arg1)
14069 258497 : || integer_pow2p (arg1))
14070 : /* vector types are not handled here. */
14071 187050 : && TREE_CODE (TREE_TYPE (arg1)) != VECTOR_TYPE
14072 798321 : && (TYPE_PRECISION (ops->type) != 1 || TYPE_UNSIGNED (ops->type)))
14073 : {
14074 187010 : tree narg0 = arg0;
14075 187010 : wide_int nz = tree_nonzero_bits (narg0);
14076 187010 : gimple *srcstmt = get_def_for_expr (narg0, BIT_AND_EXPR);
14077 : /* If the defining statement was (x & POW2), then use that instead of
14078 : the non-zero bits. */
14079 187010 : if (srcstmt && integer_pow2p (gimple_assign_rhs2 (srcstmt)))
14080 : {
14081 5470 : nz = wi::to_wide (gimple_assign_rhs2 (srcstmt));
14082 5470 : narg0 = gimple_assign_rhs1 (srcstmt);
14083 : }
14084 :
14085 187010 : if (wi::popcount (nz) == 1
14086 187010 : && (integer_zerop (arg1)
14087 177250 : || wi::to_wide (arg1) == nz))
14088 : {
14089 9792 : int bitnum = wi::exact_log2 (nz);
14090 9792 : enum tree_code tcode = EQ_EXPR;
14091 9792 : if ((code == NE) ^ !integer_zerop (arg1))
14092 7082 : tcode = NE_EXPR;
14093 :
14094 9792 : type = lang_hooks.types.type_for_mode (mode, unsignedp);
14095 9792 : return expand_single_bit_test (loc, tcode,
14096 : narg0,
14097 : bitnum, type, target, mode);
14098 : }
14099 187010 : }
14100 :
14101 :
14102 601519 : if (! get_subtarget (target)
14103 601519 : || GET_MODE (subtarget) != operand_mode)
14104 : subtarget = 0;
14105 :
14106 601519 : expand_operands (arg0, arg1, subtarget, &op0, &op1, EXPAND_NORMAL);
14107 :
14108 : /* For boolean vectors with less than mode precision
14109 : make sure to fill padding with consistent values. */
14110 40 : if (VECTOR_BOOLEAN_TYPE_P (type)
14111 27 : && SCALAR_INT_MODE_P (operand_mode)
14112 601519 : && TYPE_VECTOR_SUBPARTS (type).is_constant (&nunits)
14113 601519 : && maybe_ne (GET_MODE_PRECISION (operand_mode), nunits))
14114 : {
14115 0 : gcc_assert (code == EQ || code == NE);
14116 0 : op0 = expand_binop (mode, and_optab, op0,
14117 0 : GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
14118 : NULL_RTX, true, OPTAB_WIDEN);
14119 0 : op1 = expand_binop (mode, and_optab, op1,
14120 : GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
14121 : NULL_RTX, true, OPTAB_WIDEN);
14122 : }
14123 :
14124 601519 : if (target == 0)
14125 410733 : target = gen_reg_rtx (mode);
14126 :
14127 : /* Try a cstore if possible. */
14128 601519 : return emit_store_flag_force (target, code, op0, op1,
14129 : operand_mode, unsignedp,
14130 601519 : (TYPE_PRECISION (ops->type) == 1
14131 1202367 : && !TYPE_UNSIGNED (ops->type)) ? -1 : 1);
14132 : }
14133 :
14134 : /* Attempt to generate a casesi instruction. Returns true if successful,
14135 : false otherwise (i.e. if there is no casesi instruction).
14136 :
14137 : DEFAULT_PROBABILITY is the probability of jumping to the default
14138 : label. */
14139 : bool
14140 6594 : try_casesi (tree index_type, tree index_expr, tree minval, tree range,
14141 : rtx table_label, rtx default_label, rtx fallback_label,
14142 : profile_probability default_probability)
14143 : {
14144 6594 : class expand_operand ops[5];
14145 6594 : scalar_int_mode index_mode = SImode;
14146 6594 : rtx op1, op2, index;
14147 :
14148 6594 : if (! targetm.have_casesi ())
14149 : return false;
14150 :
14151 : /* The index must be some form of integer. Convert it to SImode. */
14152 0 : scalar_int_mode omode = SCALAR_INT_TYPE_MODE (index_type);
14153 0 : if (GET_MODE_BITSIZE (omode) > GET_MODE_BITSIZE (index_mode))
14154 : {
14155 0 : rtx rangertx = expand_normal (range);
14156 :
14157 : /* We must handle the endpoints in the original mode. */
14158 0 : index_expr = build2 (MINUS_EXPR, index_type,
14159 : index_expr, minval);
14160 0 : minval = integer_zero_node;
14161 0 : index = expand_normal (index_expr);
14162 0 : if (default_label)
14163 0 : emit_cmp_and_jump_insns (rangertx, index, LTU, NULL_RTX,
14164 : omode, 1, default_label,
14165 : default_probability);
14166 : /* Now we can safely truncate. */
14167 0 : index = convert_to_mode (index_mode, index, 0);
14168 : }
14169 : else
14170 : {
14171 0 : if (omode != index_mode)
14172 : {
14173 0 : index_type = lang_hooks.types.type_for_mode (index_mode, 0);
14174 0 : index_expr = fold_convert (index_type, index_expr);
14175 : }
14176 :
14177 0 : index = expand_normal (index_expr);
14178 : }
14179 :
14180 0 : do_pending_stack_adjust ();
14181 :
14182 0 : op1 = expand_normal (minval);
14183 0 : op2 = expand_normal (range);
14184 :
14185 0 : create_input_operand (&ops[0], index, index_mode);
14186 0 : create_convert_operand_from_type (&ops[1], op1, TREE_TYPE (minval));
14187 0 : create_convert_operand_from_type (&ops[2], op2, TREE_TYPE (range));
14188 0 : create_fixed_operand (&ops[3], table_label);
14189 0 : create_fixed_operand (&ops[4], (default_label
14190 : ? default_label
14191 : : fallback_label));
14192 0 : expand_jump_insn (targetm.code_for_casesi, 5, ops);
14193 0 : return true;
14194 : }
14195 :
14196 : /* Attempt to generate a tablejump instruction; same concept. */
14197 : /* Subroutine of the next function.
14198 :
14199 : INDEX is the value being switched on, with the lowest value
14200 : in the table already subtracted.
14201 : MODE is its expected mode (needed if INDEX is constant).
14202 : RANGE is the length of the jump table.
14203 : TABLE_LABEL is a CODE_LABEL rtx for the table itself.
14204 :
14205 : DEFAULT_LABEL is a CODE_LABEL rtx to jump to if the
14206 : index value is out of range.
14207 : DEFAULT_PROBABILITY is the probability of jumping to
14208 : the default label. */
14209 :
14210 : static void
14211 6594 : do_tablejump (rtx index, machine_mode mode, rtx range, rtx table_label,
14212 : rtx default_label, profile_probability default_probability)
14213 : {
14214 6594 : rtx temp, vector;
14215 :
14216 6594 : if (INTVAL (range) > cfun->cfg->max_jumptable_ents)
14217 5713 : cfun->cfg->max_jumptable_ents = INTVAL (range);
14218 :
14219 : /* Do an unsigned comparison (in the proper mode) between the index
14220 : expression and the value which represents the length of the range.
14221 : Since we just finished subtracting the lower bound of the range
14222 : from the index expression, this comparison allows us to simultaneously
14223 : check that the original index expression value is both greater than
14224 : or equal to the minimum value of the range and less than or equal to
14225 : the maximum value of the range. */
14226 :
14227 6594 : if (default_label)
14228 6548 : emit_cmp_and_jump_insns (index, range, GTU, NULL_RTX, mode, 1,
14229 : default_label, default_probability);
14230 :
14231 : /* If index is in range, it must fit in Pmode.
14232 : Convert to Pmode so we can index with it. */
14233 7649 : if (mode != Pmode)
14234 : {
14235 5597 : unsigned int width;
14236 :
14237 : /* We know the value of INDEX is between 0 and RANGE. If we have a
14238 : sign-extended subreg, and RANGE does not have the sign bit set, then
14239 : we have a value that is valid for both sign and zero extension. In
14240 : this case, we get better code if we sign extend. */
14241 5597 : if (GET_CODE (index) == SUBREG
14242 9 : && SUBREG_PROMOTED_VAR_P (index)
14243 0 : && SUBREG_PROMOTED_SIGNED_P (index)
14244 0 : && ((width = GET_MODE_PRECISION (as_a <scalar_int_mode> (mode)))
14245 : <= HOST_BITS_PER_WIDE_INT)
14246 5597 : && ! (UINTVAL (range) & (HOST_WIDE_INT_1U << (width - 1))))
14247 0 : index = convert_to_mode (Pmode, index, 0);
14248 : else
14249 5597 : index = convert_to_mode (Pmode, index, 1);
14250 : }
14251 :
14252 : /* Don't let a MEM slip through, because then INDEX that comes
14253 : out of PIC_CASE_VECTOR_ADDRESS won't be a valid address,
14254 : and break_out_memory_refs will go to work on it and mess it up. */
14255 : #ifdef PIC_CASE_VECTOR_ADDRESS
14256 : if (flag_pic && !REG_P (index))
14257 : index = copy_to_mode_reg (Pmode, index);
14258 : #endif
14259 :
14260 : /* ??? The only correct use of CASE_VECTOR_MODE is the one inside the
14261 : GET_MODE_SIZE, because this indicates how large insns are. The other
14262 : uses should all be Pmode, because they are addresses. This code
14263 : could fail if addresses and insns are not the same size. */
14264 7649 : index = simplify_gen_binary (MULT, Pmode, index,
14265 13188 : gen_int_mode (GET_MODE_SIZE (CASE_VECTOR_MODE),
14266 6594 : Pmode));
14267 7649 : index = simplify_gen_binary (PLUS, Pmode, index,
14268 6594 : gen_rtx_LABEL_REF (Pmode, table_label));
14269 :
14270 : #ifdef PIC_CASE_VECTOR_ADDRESS
14271 : if (flag_pic)
14272 : index = PIC_CASE_VECTOR_ADDRESS (index);
14273 : else
14274 : #endif
14275 8252 : index = memory_address (CASE_VECTOR_MODE, index);
14276 8252 : temp = gen_reg_rtx (CASE_VECTOR_MODE);
14277 8252 : vector = gen_const_mem (CASE_VECTOR_MODE, index);
14278 6594 : convert_move (temp, vector, 0);
14279 :
14280 6594 : emit_jump_insn (targetm.gen_tablejump (temp, table_label));
14281 :
14282 : /* If we are generating PIC code or if the table is PC-relative, the
14283 : table and JUMP_INSN must be adjacent, so don't output a BARRIER. */
14284 6594 : if (! CASE_VECTOR_PC_RELATIVE && ! flag_pic)
14285 5435 : emit_barrier ();
14286 6594 : }
14287 :
14288 : bool
14289 6594 : try_tablejump (tree index_type, tree index_expr, tree minval, tree range,
14290 : rtx table_label, rtx default_label,
14291 : profile_probability default_probability)
14292 : {
14293 6594 : rtx index;
14294 :
14295 6594 : if (! targetm.have_tablejump ())
14296 : return false;
14297 :
14298 6594 : index_expr = fold_build2 (MINUS_EXPR, index_type,
14299 : fold_convert (index_type, index_expr),
14300 : fold_convert (index_type, minval));
14301 6594 : index = expand_normal (index_expr);
14302 6594 : do_pending_stack_adjust ();
14303 :
14304 6594 : do_tablejump (index, TYPE_MODE (index_type),
14305 6594 : convert_modes (TYPE_MODE (index_type),
14306 6594 : TYPE_MODE (TREE_TYPE (range)),
14307 : expand_normal (range),
14308 6594 : TYPE_UNSIGNED (TREE_TYPE (range))),
14309 : table_label, default_label, default_probability);
14310 6594 : return true;
14311 : }
14312 :
14313 : /* Return a CONST_VECTOR rtx representing vector mask for
14314 : a VECTOR_CST of booleans. */
14315 : static rtx
14316 684 : const_vector_mask_from_tree (tree exp)
14317 : {
14318 684 : machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
14319 684 : machine_mode inner = GET_MODE_INNER (mode);
14320 :
14321 684 : rtx_vector_builder builder (mode, VECTOR_CST_NPATTERNS (exp),
14322 684 : VECTOR_CST_NELTS_PER_PATTERN (exp));
14323 684 : unsigned int count = builder.encoded_nelts ();
14324 3680 : for (unsigned int i = 0; i < count; ++i)
14325 : {
14326 2996 : tree elt = VECTOR_CST_ELT (exp, i);
14327 2996 : gcc_assert (TREE_CODE (elt) == INTEGER_CST);
14328 2996 : if (integer_zerop (elt))
14329 1177 : builder.quick_push (CONST0_RTX (inner));
14330 1819 : else if (integer_onep (elt)
14331 1819 : || integer_minus_onep (elt))
14332 1819 : builder.quick_push (CONSTM1_RTX (inner));
14333 : else
14334 0 : gcc_unreachable ();
14335 : }
14336 684 : return builder.build ();
14337 684 : }
14338 :
14339 : /* Return a CONST_VECTOR rtx for a VECTOR_CST tree. */
14340 : static rtx
14341 566604 : const_vector_from_tree (tree exp)
14342 : {
14343 566604 : machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
14344 :
14345 566604 : if (initializer_zerop (exp))
14346 168215 : return CONST0_RTX (mode);
14347 :
14348 398389 : if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (exp)))
14349 684 : return const_vector_mask_from_tree (exp);
14350 :
14351 397705 : machine_mode inner = GET_MODE_INNER (mode);
14352 :
14353 397705 : rtx_vector_builder builder (mode, VECTOR_CST_NPATTERNS (exp),
14354 397705 : VECTOR_CST_NELTS_PER_PATTERN (exp));
14355 397705 : unsigned int count = builder.encoded_nelts ();
14356 1283067 : for (unsigned int i = 0; i < count; ++i)
14357 : {
14358 885362 : tree elt = VECTOR_CST_ELT (exp, i);
14359 885362 : if (TREE_CODE (elt) == REAL_CST)
14360 134896 : builder.quick_push (const_double_from_real_value (TREE_REAL_CST (elt),
14361 : inner));
14362 750466 : else if (TREE_CODE (elt) == FIXED_CST)
14363 0 : builder.quick_push (CONST_FIXED_FROM_FIXED_VALUE (TREE_FIXED_CST (elt),
14364 : inner));
14365 : else
14366 750466 : builder.quick_push (immed_wide_int_const (wi::to_poly_wide (elt),
14367 : inner));
14368 : }
14369 397705 : return builder.build ();
14370 397705 : }
14371 :
14372 : /* Build a decl for a personality function given a language prefix. */
14373 :
14374 : tree
14375 32928 : build_personality_function (const char *lang)
14376 : {
14377 32928 : const char *unwind_and_version;
14378 32928 : tree decl, type;
14379 32928 : char *name;
14380 :
14381 32928 : switch (targetm_common.except_unwind_info (&global_options))
14382 : {
14383 : case UI_NONE:
14384 : return NULL;
14385 : case UI_SJLJ:
14386 : unwind_and_version = "_sj0";
14387 : break;
14388 32928 : case UI_DWARF2:
14389 32928 : case UI_TARGET:
14390 32928 : unwind_and_version = "_v0";
14391 32928 : break;
14392 0 : case UI_SEH:
14393 0 : unwind_and_version = "_seh0";
14394 0 : break;
14395 0 : default:
14396 0 : gcc_unreachable ();
14397 : }
14398 :
14399 32928 : name = ACONCAT (("__", lang, "_personality", unwind_and_version, NULL));
14400 :
14401 32928 : type = build_function_type_list (unsigned_type_node,
14402 : integer_type_node, integer_type_node,
14403 : long_long_unsigned_type_node,
14404 : ptr_type_node, ptr_type_node, NULL_TREE);
14405 32928 : decl = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL,
14406 : get_identifier (name), type);
14407 32928 : DECL_ARTIFICIAL (decl) = 1;
14408 32928 : DECL_EXTERNAL (decl) = 1;
14409 32928 : TREE_PUBLIC (decl) = 1;
14410 :
14411 : /* Zap the nonsensical SYMBOL_REF_DECL for this. What we're left with
14412 : are the flags assigned by targetm.encode_section_info. */
14413 32928 : SET_SYMBOL_REF_DECL (XEXP (DECL_RTL (decl), 0), NULL);
14414 :
14415 32928 : return decl;
14416 : }
14417 :
14418 : /* Extracts the personality function of DECL and returns the corresponding
14419 : libfunc. */
14420 :
14421 : rtx
14422 1654337 : get_personality_function (tree decl)
14423 : {
14424 1654337 : tree personality = DECL_FUNCTION_PERSONALITY (decl);
14425 1654337 : enum eh_personality_kind pk;
14426 :
14427 1654337 : pk = function_needs_eh_personality (DECL_STRUCT_FUNCTION (decl));
14428 1654337 : if (pk == eh_personality_none)
14429 : return NULL;
14430 :
14431 157204 : if (!personality
14432 157204 : && pk == eh_personality_any)
14433 71838 : personality = lang_hooks.eh_personality ();
14434 :
14435 157204 : if (pk == eh_personality_lang)
14436 85366 : gcc_assert (personality != NULL_TREE);
14437 :
14438 157204 : return XEXP (DECL_RTL (personality), 0);
14439 : }
14440 :
14441 : /* Returns a tree for the size of EXP in bytes. */
14442 :
14443 : static tree
14444 14853213 : tree_expr_size (const_tree exp)
14445 : {
14446 14853213 : if (DECL_P (exp)
14447 14853213 : && DECL_SIZE_UNIT (exp) != 0)
14448 1949341 : return DECL_SIZE_UNIT (exp);
14449 : else
14450 12903872 : return size_in_bytes (TREE_TYPE (exp));
14451 : }
14452 :
14453 : /* Return an rtx for the size in bytes of the value of EXP. */
14454 :
14455 : rtx
14456 14663415 : expr_size (tree exp)
14457 : {
14458 14663415 : tree size;
14459 :
14460 14663415 : if (TREE_CODE (exp) == WITH_SIZE_EXPR)
14461 773 : size = TREE_OPERAND (exp, 1);
14462 : else
14463 : {
14464 14662642 : size = tree_expr_size (exp);
14465 14662642 : gcc_assert (size);
14466 14662642 : gcc_assert (size == SUBSTITUTE_PLACEHOLDER_IN_EXPR (size, exp));
14467 : }
14468 :
14469 14663415 : return expand_expr (size, NULL_RTX, TYPE_MODE (sizetype), EXPAND_NORMAL);
14470 : }
14471 :
14472 : /* Return a wide integer for the size in bytes of the value of EXP, or -1
14473 : if the size can vary or is larger than an integer. */
14474 :
14475 : HOST_WIDE_INT
14476 190571 : int_expr_size (const_tree exp)
14477 : {
14478 190571 : tree size;
14479 :
14480 190571 : if (TREE_CODE (exp) == WITH_SIZE_EXPR)
14481 0 : size = TREE_OPERAND (exp, 1);
14482 : else
14483 : {
14484 190571 : size = tree_expr_size (exp);
14485 190571 : gcc_assert (size);
14486 : }
14487 :
14488 190571 : if (size == 0 || !tree_fits_shwi_p (size))
14489 0 : return -1;
14490 :
14491 190571 : return tree_to_shwi (size);
14492 : }
14493 :
14494 : /* Return the quotient of polynomial long division of x^2N by POLYNOMIAL
14495 : in GF (2^N).
14496 : Author: Richard Sandiford <richard.sandiford@arm.com> */
14497 :
14498 : unsigned HOST_WIDE_INT
14499 0 : gf2n_poly_long_div_quotient (unsigned HOST_WIDE_INT polynomial,
14500 : unsigned short n)
14501 : {
14502 : /* The result has degree N, so needs N + 1 bits. */
14503 0 : gcc_assert (n < 64);
14504 :
14505 : /* Perform a division step for the x^2N coefficient. At this point the
14506 : quotient and remainder have N implicit trailing zeros. */
14507 : unsigned HOST_WIDE_INT quotient = 1;
14508 : unsigned HOST_WIDE_INT remainder = polynomial;
14509 :
14510 : /* Process the coefficients for x^(2N-1) down to x^N, with each step
14511 : reducing the number of implicit trailing zeros by one. */
14512 0 : for (unsigned int i = 0; i < n; ++i)
14513 : {
14514 0 : bool coeff = remainder & (HOST_WIDE_INT_1U << (n - 1));
14515 0 : quotient = (quotient << 1) | coeff;
14516 0 : remainder = (remainder << 1) ^ (coeff ? polynomial : 0);
14517 : }
14518 0 : return quotient;
14519 : }
14520 :
14521 : /* Calculate CRC for the initial CRC and given POLYNOMIAL.
14522 : CRC_BITS is CRC size. */
14523 :
14524 : static unsigned HOST_WIDE_INT
14525 34560 : calculate_crc (unsigned HOST_WIDE_INT crc,
14526 : unsigned HOST_WIDE_INT polynomial,
14527 : unsigned short crc_bits)
14528 : {
14529 34560 : unsigned HOST_WIDE_INT msb = HOST_WIDE_INT_1U << (crc_bits - 1);
14530 34560 : crc = crc << (crc_bits - 8);
14531 311040 : for (short i = 8; i > 0; --i)
14532 : {
14533 276480 : if (crc & msb)
14534 138240 : crc = (crc << 1) ^ polynomial;
14535 : else
14536 138240 : crc <<= 1;
14537 : }
14538 : /* Zero out bits in crc beyond the specified number of crc_bits. */
14539 34560 : if (crc_bits < sizeof (crc) * CHAR_BIT)
14540 31488 : crc &= (HOST_WIDE_INT_1U << crc_bits) - 1;
14541 34560 : return crc;
14542 : }
14543 :
14544 : /* Assemble CRC table with 256 elements for the given POLYNOM and CRC_BITS.
14545 : POLYNOM is the polynomial used to calculate the CRC table's elements.
14546 : CRC_BITS is the size of CRC, may be 8, 16, ... . */
14547 :
14548 : static rtx
14549 135 : assemble_crc_table (unsigned HOST_WIDE_INT polynom, unsigned short crc_bits)
14550 : {
14551 135 : unsigned table_el_n = 0x100;
14552 135 : tree ar = build_array_type (make_unsigned_type (crc_bits),
14553 135 : build_index_type (size_int (table_el_n - 1)));
14554 :
14555 : /* Initialize the table. */
14556 135 : vec<tree, va_gc> *initial_values;
14557 135 : vec_alloc (initial_values, table_el_n);
14558 34695 : for (size_t i = 0; i < table_el_n; ++i)
14559 : {
14560 34560 : unsigned HOST_WIDE_INT crc = calculate_crc (i, polynom, crc_bits);
14561 34560 : tree element = build_int_cstu (make_unsigned_type (crc_bits), crc);
14562 34560 : vec_safe_push (initial_values, element);
14563 : }
14564 135 : tree ctor = build_constructor_from_vec (ar, initial_values);
14565 135 : rtx mem = output_constant_def (ctor, 1);
14566 135 : gcc_assert (MEM_P (mem));
14567 135 : if (dump_file && (dump_flags & TDF_DETAILS))
14568 : {
14569 16 : fprintf (dump_file,
14570 : ";; emitting crc table crc_%u_polynomial_"
14571 : HOST_WIDE_INT_PRINT_HEX " ",
14572 : crc_bits, polynom);
14573 16 : print_rtl_single (dump_file, XEXP (mem, 0));
14574 16 : fprintf (dump_file, "\n");
14575 : }
14576 :
14577 135 : return XEXP (mem, 0);
14578 : }
14579 :
14580 : /* Generate CRC lookup table by calculating CRC for all possible
14581 : 8-bit data values. The table is stored with a specific name in the read-only
14582 : static data section.
14583 : POLYNOM is the polynomial used to calculate the CRC table's elements.
14584 : CRC_BITS is the size of CRC, may be 8, 16, ... . */
14585 :
14586 : static rtx
14587 135 : generate_crc_table (unsigned HOST_WIDE_INT polynom, unsigned short crc_bits)
14588 : {
14589 135 : gcc_assert (crc_bits <= 64);
14590 :
14591 135 : return assemble_crc_table (polynom, crc_bits);
14592 : }
14593 :
14594 : /* Calculate CRC for the initial CRC and given POLYNOMIAL.
14595 : CRC_BITS is CRC size. */
14596 :
14597 : static unsigned HOST_WIDE_INT
14598 35328 : calculate_reversed_crc (unsigned HOST_WIDE_INT crc,
14599 : unsigned HOST_WIDE_INT polynomial,
14600 : unsigned short crc_bits)
14601 : {
14602 35328 : unsigned HOST_WIDE_INT rev_polynom = reflect_hwi (polynomial, crc_bits);
14603 317952 : for (int j = 0; j < 8; j++)
14604 : {
14605 282624 : if (crc & 1)
14606 141312 : crc = (crc >> 1) ^ rev_polynom;
14607 : else
14608 141312 : crc >>= 1;
14609 : }
14610 : /* Zero out bits in crc beyond the specified number of crc_bits. */
14611 35328 : if (crc_bits < sizeof (crc) * CHAR_BIT)
14612 29952 : crc &= (HOST_WIDE_INT_1U << crc_bits) - 1;
14613 35328 : return crc;
14614 : }
14615 :
14616 : /* Assemble CRC table with 256 elements for the given POLYNOM and CRC_BITS.
14617 : POLYNOM is the polynomial used to calculate the CRC table's elements.
14618 : CRC_BITS is the size of CRC, may be 8, 16, ... . */
14619 :
14620 : static rtx
14621 138 : assemble_reversed_crc_table (unsigned HOST_WIDE_INT polynom, unsigned short crc_bits)
14622 : {
14623 138 : unsigned table_el_n = 0x100;
14624 138 : tree ar = build_array_type (make_unsigned_type (crc_bits),
14625 138 : build_index_type (size_int (table_el_n - 1)));
14626 :
14627 : /* Initialize the table. */
14628 138 : vec<tree, va_gc> *initial_values;
14629 138 : vec_alloc (initial_values, table_el_n);
14630 35466 : for (size_t i = 0; i < table_el_n; ++i)
14631 : {
14632 35328 : unsigned HOST_WIDE_INT crc = calculate_reversed_crc (i, polynom, crc_bits);
14633 35328 : tree element = build_int_cstu (make_unsigned_type (crc_bits), crc);
14634 35328 : vec_safe_push (initial_values, element);
14635 : }
14636 138 : tree ctor = build_constructor_from_vec (ar, initial_values);
14637 138 : rtx mem = output_constant_def (ctor, 1);
14638 138 : gcc_assert (MEM_P (mem));
14639 138 : if (dump_file && (dump_flags & TDF_DETAILS))
14640 : {
14641 16 : fprintf (dump_file,
14642 : ";; emitting reversed crc table crc_%u_polynomial_"
14643 : HOST_WIDE_INT_PRINT_HEX " ",
14644 : crc_bits, polynom);
14645 16 : print_rtl_single (dump_file, XEXP (mem, 0));
14646 16 : fprintf (dump_file, "\n");
14647 : }
14648 :
14649 138 : return XEXP (mem, 0);
14650 : }
14651 :
14652 : /* Generate reversed CRC table for the given POLYNOM and CRC_BITS. */
14653 :
14654 : static rtx
14655 138 : generate_reversed_crc_table (unsigned HOST_WIDE_INT polynom,
14656 : unsigned short crc_bits)
14657 : {
14658 138 : gcc_assert (crc_bits <= 64);
14659 :
14660 138 : return assemble_reversed_crc_table (polynom, crc_bits);
14661 : }
14662 :
14663 : /* Generate table-based CRC code for the given CRC, INPUT_DATA and the
14664 : POLYNOMIAL (without leading 1).
14665 :
14666 : First, using POLYNOMIAL's value generates CRC table of 256 elements,
14667 : then generates the assembly for the following code,
14668 : where crc_bit_size and data_bit_size may be 8, 16, 32, 64, depending on CRC:
14669 :
14670 : for (int i = 0; i < data_bit_size / 8; i++)
14671 : crc = (crc << 8) ^ crc_table[(crc >> (crc_bit_size - 8))
14672 : ^ (data >> (data_bit_size - (i + 1) * 8)
14673 : & 0xFF))];
14674 :
14675 : So to take values from the table, we need 8-bit data.
14676 : If input data size is not 8, then first we extract upper 8 bits,
14677 : then the other 8 bits, and so on. */
14678 :
14679 : static void
14680 135 : calculate_table_based_CRC (rtx *crc, const rtx &input_data,
14681 : const rtx &polynomial,
14682 : machine_mode data_mode)
14683 : {
14684 135 : machine_mode mode = GET_MODE (*crc);
14685 135 : unsigned short crc_bit_size = GET_MODE_BITSIZE (mode).to_constant ();
14686 135 : unsigned short data_size = GET_MODE_SIZE (data_mode).to_constant ();
14687 135 : rtx tab = generate_crc_table (UINTVAL (polynomial), crc_bit_size);
14688 :
14689 394 : for (unsigned short i = 0; i < data_size; i++)
14690 : {
14691 : /* crc >> (crc_bit_size - 8). */
14692 259 : *crc = force_reg (mode, *crc);
14693 259 : rtx op1 = expand_shift (RSHIFT_EXPR, mode, *crc, crc_bit_size - 8,
14694 : NULL_RTX, 1);
14695 :
14696 : /* data >> (8 * (GET_MODE_SIZE (data_mode).to_constant () - i - 1)). */
14697 259 : unsigned range_8 = 8 * (data_size - i - 1);
14698 : /* CRC's mode is always at least as wide as INPUT_DATA. Convert
14699 : INPUT_DATA into CRC's mode. */
14700 259 : rtx data = gen_reg_rtx (mode);
14701 259 : convert_move (data, input_data, 1);
14702 259 : data = expand_shift (RSHIFT_EXPR, mode, data, range_8, NULL_RTX, 1);
14703 :
14704 : /* data >> (8 * (GET_MODE_SIZE (mode)
14705 : .to_constant () - i - 1)) & 0xFF. */
14706 259 : rtx data_final = expand_and (mode, data,
14707 : gen_int_mode (255, mode), NULL_RTX);
14708 :
14709 : /* (crc >> (crc_bit_size - 8)) ^ data_8bit. */
14710 259 : rtx in = expand_binop (mode, xor_optab, op1, data_final,
14711 : NULL_RTX, 1, OPTAB_WIDEN);
14712 :
14713 : /* ((crc >> (crc_bit_size - 8)) ^ data_8bit) & 0xFF. */
14714 259 : rtx index = expand_and (mode, in, gen_int_mode (255, mode),
14715 : NULL_RTX);
14716 518 : int log_crc_size = exact_log2 (GET_MODE_SIZE (mode).to_constant ());
14717 259 : index = expand_shift (LSHIFT_EXPR, mode, index,
14718 259 : log_crc_size, NULL_RTX, 0);
14719 :
14720 264 : rtx addr = gen_reg_rtx (Pmode);
14721 259 : convert_move (addr, index, 1);
14722 264 : addr = expand_binop (Pmode, add_optab, addr, tab, NULL_RTX,
14723 : 0, OPTAB_DIRECT);
14724 :
14725 : /* crc_table[(crc >> (crc_bit_size - 8)) ^ data_8bit] */
14726 259 : rtx tab_el = validize_mem (gen_rtx_MEM (mode, addr));
14727 :
14728 : /* (crc << 8) if CRC is larger than 8, otherwise crc = 0. */
14729 259 : rtx high = NULL_RTX;
14730 259 : if (crc_bit_size != 8)
14731 229 : high = expand_shift (LSHIFT_EXPR, mode, *crc, 8, NULL_RTX, 0);
14732 : else
14733 30 : high = gen_int_mode (0, mode);
14734 :
14735 : /* crc = (crc << 8)
14736 : ^ crc_table[(crc >> (crc_bit_size - 8)) ^ data_8bit]; */
14737 259 : *crc = expand_binop (mode, xor_optab, tab_el, high, NULL_RTX, 1,
14738 : OPTAB_WIDEN);
14739 : }
14740 135 : }
14741 :
14742 : /* Generate table-based reversed CRC code for the given CRC, INPUT_DATA
14743 : and the POLYNOMIAL (without leading 1).
14744 :
14745 : This function generates code for reversed (bit-reflected) CRC calculation
14746 : using a pre-computed lookup table. Unlike the standard CRC calculation,
14747 : this processes data from LSB to MSB, eliminating the need for explicit
14748 : bit reflection before and after the CRC computation. */
14749 :
14750 : static void
14751 138 : calculate_table_based_reversed_CRC (rtx *crc, const rtx &input_data,
14752 : const rtx &polynomial,
14753 : machine_mode data_mode)
14754 : {
14755 138 : machine_mode mode = GET_MODE (*crc);
14756 138 : unsigned short crc_bit_size = GET_MODE_BITSIZE (mode).to_constant ();
14757 138 : unsigned short data_size = GET_MODE_SIZE (data_mode).to_constant ();
14758 138 : rtx tab = generate_reversed_crc_table (UINTVAL (polynomial), crc_bit_size);
14759 :
14760 : /* CRC's mode is always at least as wide as INPUT_DATA. Convert
14761 : INPUT_DATA into CRC's mode once outside the loop since INPUT_DATA
14762 : is loop-invariant. */
14763 138 : rtx data_in_crc_mode = gen_reg_rtx (mode);
14764 138 : convert_move (data_in_crc_mode, input_data, 1);
14765 :
14766 364 : for (unsigned short i = 0; i < data_size; i++)
14767 : {
14768 226 : *crc = force_reg (mode, *crc);
14769 :
14770 : /* data >> (8 * i). */
14771 226 : unsigned range_8 = 8 * i;
14772 226 : rtx data = expand_shift (RSHIFT_EXPR, mode, data_in_crc_mode,
14773 226 : range_8, NULL_RTX, 1);
14774 :
14775 : /* crc ^ (data >> (8 * i)). */
14776 226 : rtx in = expand_binop (mode, xor_optab, *crc, data,
14777 : NULL_RTX, 1, OPTAB_WIDEN);
14778 :
14779 : /* (crc ^ data) & 0xFF. */
14780 226 : rtx index = expand_and (mode, in, gen_int_mode (255, mode),
14781 : NULL_RTX);
14782 452 : int log_crc_size = exact_log2 (GET_MODE_SIZE (mode).to_constant ());
14783 226 : index = expand_shift (LSHIFT_EXPR, mode, index,
14784 226 : log_crc_size, NULL_RTX, 0);
14785 :
14786 239 : rtx addr = gen_reg_rtx (Pmode);
14787 226 : convert_move (addr, index, 1);
14788 239 : addr = expand_binop (Pmode, add_optab, addr, tab, NULL_RTX,
14789 : 0, OPTAB_DIRECT);
14790 :
14791 : /* crc_table[(crc ^ data) & 0xFF]. */
14792 226 : rtx tab_el = validize_mem (gen_rtx_MEM (mode, addr));
14793 :
14794 : /* (crc >> 8) if CRC is larger than 8, otherwise 0. */
14795 226 : rtx high = NULL_RTX;
14796 226 : if (crc_bit_size != 8)
14797 217 : high = expand_shift (RSHIFT_EXPR, mode, *crc, 8, NULL_RTX, 1);
14798 : else
14799 9 : high = gen_int_mode (0, mode);
14800 :
14801 : /* crc = (crc >> 8) ^ crc_table[(crc ^ data) & 0xFF]. */
14802 226 : *crc = expand_binop (mode, xor_optab, tab_el, high, NULL_RTX, 1,
14803 : OPTAB_WIDEN);
14804 : }
14805 138 : }
14806 :
14807 : /* Generate table-based CRC code for the given CRC, INPUT_DATA and the
14808 : POLYNOMIAL (without leading 1).
14809 :
14810 : CRC is OP1, data is OP2 and the polynomial is OP3.
14811 : This must generate a CRC table and an assembly for the following code,
14812 : where crc_bit_size and data_bit_size may be 8, 16, 32, 64:
14813 : uint_crc_bit_size_t
14814 : crc_crc_bit_size (uint_crc_bit_size_t crc_init,
14815 : uint_data_bit_size_t data, size_t size)
14816 : {
14817 : uint_crc_bit_size_t crc = crc_init;
14818 : for (int i = 0; i < data_bit_size / 8; i++)
14819 : crc = (crc << 8) ^ crc_table[(crc >> (crc_bit_size - 8))
14820 : ^ (data >> (data_bit_size - (i + 1) * 8)
14821 : & 0xFF))];
14822 : return crc;
14823 : } */
14824 :
14825 : void
14826 135 : expand_crc_table_based (rtx op0, rtx op1, rtx op2, rtx op3,
14827 : machine_mode data_mode)
14828 : {
14829 135 : gcc_assert (!CONST_INT_P (op0));
14830 135 : gcc_assert (CONST_INT_P (op3));
14831 135 : machine_mode crc_mode = GET_MODE (op0);
14832 135 : rtx crc = gen_reg_rtx (crc_mode);
14833 135 : convert_move (crc, op1, 0);
14834 135 : calculate_table_based_CRC (&crc, op2, op3, data_mode);
14835 135 : convert_move (op0, crc, 0);
14836 135 : }
14837 :
14838 : /* Generate the common operation for reflecting values:
14839 : *OP = (*OP & AND1_VALUE) << SHIFT_VAL | (*OP & AND2_VALUE) >> SHIFT_VAL; */
14840 :
14841 : void
14842 0 : gen_common_operation_to_reflect (rtx *op,
14843 : unsigned HOST_WIDE_INT and1_value,
14844 : unsigned HOST_WIDE_INT and2_value,
14845 : unsigned shift_val)
14846 : {
14847 0 : rtx op1 = expand_and (GET_MODE (*op), *op,
14848 0 : gen_int_mode (and1_value, GET_MODE (*op)), NULL_RTX);
14849 0 : op1 = expand_shift (LSHIFT_EXPR, GET_MODE (*op), op1, shift_val, op1, 0);
14850 0 : rtx op2 = expand_and (GET_MODE (*op), *op,
14851 0 : gen_int_mode (and2_value, GET_MODE (*op)), NULL_RTX);
14852 0 : op2 = expand_shift (RSHIFT_EXPR, GET_MODE (*op), op2, shift_val, op2, 1);
14853 0 : *op = expand_binop (GET_MODE (*op), ior_optab, op1,
14854 : op2, *op, 0, OPTAB_LIB_WIDEN);
14855 0 : }
14856 :
14857 : /* Reflect 64-bit value for the 64-bit target. */
14858 :
14859 : void
14860 0 : reflect_64_bit_value (rtx *op)
14861 : {
14862 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x00000000FFFFFFFF),
14863 : HOST_WIDE_INT_C (0xFFFFFFFF00000000), 32);
14864 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0000FFFF0000FFFF),
14865 : HOST_WIDE_INT_C (0xFFFF0000FFFF0000), 16);
14866 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x00FF00FF00FF00FF),
14867 : HOST_WIDE_INT_C (0xFF00FF00FF00FF00), 8);
14868 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0F0F0F0F0F0F0F0F),
14869 : HOST_WIDE_INT_C (0xF0F0F0F0F0F0F0F0), 4);
14870 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x3333333333333333),
14871 : HOST_WIDE_INT_C (0xCCCCCCCCCCCCCCCC), 2);
14872 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x5555555555555555),
14873 : HOST_WIDE_INT_C (0xAAAAAAAAAAAAAAAA), 1);
14874 0 : }
14875 :
14876 : /* Reflect 32-bit value for the 32-bit target. */
14877 :
14878 : void
14879 0 : reflect_32_bit_value (rtx *op)
14880 : {
14881 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0000FFFF),
14882 : HOST_WIDE_INT_C (0xFFFF0000), 16);
14883 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x00FF00FF),
14884 : HOST_WIDE_INT_C (0xFF00FF00), 8);
14885 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0F0F0F0F),
14886 : HOST_WIDE_INT_C (0xF0F0F0F0), 4);
14887 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x33333333),
14888 : HOST_WIDE_INT_C (0xCCCCCCCC), 2);
14889 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x55555555),
14890 : HOST_WIDE_INT_C (0xAAAAAAAA), 1);
14891 0 : }
14892 :
14893 : /* Reflect 16-bit value for the 16-bit target. */
14894 :
14895 : void
14896 0 : reflect_16_bit_value (rtx *op)
14897 : {
14898 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x00FF),
14899 : HOST_WIDE_INT_C (0xFF00), 8);
14900 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0F0F),
14901 : HOST_WIDE_INT_C (0xF0F0), 4);
14902 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x3333),
14903 : HOST_WIDE_INT_C (0xCCCC), 2);
14904 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x5555),
14905 : HOST_WIDE_INT_C (0xAAAA), 1);
14906 0 : }
14907 :
14908 : /* Reflect 8-bit value for the 8-bit target. */
14909 :
14910 : void
14911 0 : reflect_8_bit_value (rtx *op)
14912 : {
14913 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0F),
14914 : HOST_WIDE_INT_C (0xF0), 4);
14915 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x33),
14916 : HOST_WIDE_INT_C (0xCC), 2);
14917 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x55),
14918 : HOST_WIDE_INT_C (0xAA), 1);
14919 0 : }
14920 :
14921 : /* Generate instruction sequence which reflects the value of the OP
14922 : using shift, and, or operations. OP's mode may be less than word_mode. */
14923 :
14924 : void
14925 0 : generate_reflecting_code_standard (rtx *op)
14926 : {
14927 0 : gcc_assert (GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () >= 8
14928 : && GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () <= 64);
14929 :
14930 0 : if (GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () == 64)
14931 0 : reflect_64_bit_value (op);
14932 0 : else if (GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () == 32)
14933 0 : reflect_32_bit_value (op);
14934 0 : else if (GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () == 16)
14935 0 : reflect_16_bit_value (op);
14936 : else
14937 0 : reflect_8_bit_value (op);
14938 0 : }
14939 :
14940 : /* Generate table-based reversed CRC code for the given CRC, INPUT_DATA and
14941 : the POLYNOMIAL (without leading 1).
14942 :
14943 : CRC is OP1, data is OP2 and the polynomial is OP3.
14944 : This generates a reversed CRC table and assembly for the following code,
14945 : where crc_bit_size and data_bit_size may be 8, 16, 32, 64:
14946 : uint_crc_bit_size_t
14947 : crc_crc_bit_size (uint_crc_bit_size_t crc_init,
14948 : uint_data_bit_size_t data)
14949 : {
14950 : uint_crc_bit_size_t crc = crc_init;
14951 : for (int i = 0; i < data_bit_size / 8; i++)
14952 : crc = (crc >> 8) ^ crc_table[(crc ^ (data >> (i * 8))) & 0xFF];
14953 : return crc;
14954 : }
14955 :
14956 : This approach uses a pre-computed reversed polynomial table, eliminating
14957 : the need for explicit bit reflection before and after the CRC computation. */
14958 :
14959 : void
14960 138 : expand_reversed_crc_table_based (rtx op0, rtx op1, rtx op2, rtx op3,
14961 : machine_mode data_mode)
14962 : {
14963 138 : gcc_assert (!CONST_INT_P (op0));
14964 138 : gcc_assert (CONST_INT_P (op3));
14965 138 : machine_mode crc_mode = GET_MODE (op0);
14966 138 : rtx crc = gen_reg_rtx (crc_mode);
14967 138 : convert_move (crc, op1, 0);
14968 138 : calculate_table_based_reversed_CRC (&crc, op2, op3, data_mode);
14969 138 : convert_move (op0, crc, 0);
14970 138 : }
|
