Line data Source code
1 : /* Convert tree expression to rtl instructions, for GNU compiler.
2 : Copyright (C) 1988-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "predict.h"
29 : #include "memmodel.h"
30 : #include "tm_p.h"
31 : #include "ssa.h"
32 : #include "optabs.h"
33 : #include "expmed.h"
34 : #include "regs.h"
35 : #include "emit-rtl.h"
36 : #include "recog.h"
37 : #include "cgraph.h"
38 : #include "diagnostic.h"
39 : #include "alias.h"
40 : #include "fold-const.h"
41 : #include "stor-layout.h"
42 : #include "attribs.h"
43 : #include "varasm.h"
44 : #include "except.h"
45 : #include "insn-attr.h"
46 : #include "dojump.h"
47 : #include "explow.h"
48 : #include "calls.h"
49 : #include "stmt.h"
50 : /* Include expr.h after insn-config.h so we get HAVE_conditional_move. */
51 : #include "expr.h"
52 : #include "optabs-tree.h"
53 : #include "libfuncs.h"
54 : #include "reload.h"
55 : #include "langhooks.h"
56 : #include "common/common-target.h"
57 : #include "tree-dfa.h"
58 : #include "tree-ssa-live.h"
59 : #include "tree-outof-ssa.h"
60 : #include "tree-ssa-address.h"
61 : #include "builtins.h"
62 : #include "ccmp.h"
63 : #include "gimple-iterator.h"
64 : #include "gimple-fold.h"
65 : #include "rtx-vector-builder.h"
66 : #include "tree-pretty-print.h"
67 : #include "flags.h"
68 : #include "internal-fn.h"
69 :
70 :
71 : /* If this is nonzero, we do not bother generating VOLATILE
72 : around volatile memory references, and we are willing to
73 : output indirect addresses. If cse is to follow, we reject
74 : indirect addresses so a useful potential cse is generated;
75 : if it is used only once, instruction combination will produce
76 : the same indirect address eventually. */
77 : int cse_not_expected;
78 :
79 : /* Cache of the "extended" flag in the target's _BitInt description
80 : for use during expand. */
81 : int bitint_extended = -1;
82 :
83 : static bool block_move_libcall_safe_for_call_parm (void);
84 : static bool emit_block_move_via_pattern (rtx, rtx, rtx, unsigned, unsigned,
85 : HOST_WIDE_INT, unsigned HOST_WIDE_INT,
86 : unsigned HOST_WIDE_INT,
87 : unsigned HOST_WIDE_INT, bool);
88 : static void emit_block_move_via_loop (rtx, rtx, rtx, unsigned, int);
89 : static void emit_block_move_via_sized_loop (rtx, rtx, rtx, unsigned, unsigned);
90 : static void emit_block_move_via_oriented_loop (rtx, rtx, rtx, unsigned, unsigned);
91 : static rtx emit_block_cmp_via_loop (rtx, rtx, rtx, tree, rtx, bool,
92 : unsigned, unsigned);
93 : static rtx_insn *compress_float_constant (rtx, rtx);
94 : static rtx get_subtarget (rtx);
95 : static rtx store_field (rtx, poly_int64, poly_int64, poly_uint64, poly_uint64,
96 : machine_mode, tree, alias_set_type, bool, bool);
97 :
98 : static unsigned HOST_WIDE_INT highest_pow2_factor_for_target (const_tree, const_tree);
99 :
100 : static bool is_aligning_offset (const_tree, const_tree);
101 : static rtx reduce_to_bit_field_precision (rtx, rtx, tree);
102 : static rtx do_store_flag (const_sepops, rtx, machine_mode);
103 : #ifdef PUSH_ROUNDING
104 : static void emit_single_push_insn (machine_mode, rtx, tree);
105 : #endif
106 : static void do_tablejump (rtx, machine_mode, rtx, rtx, rtx,
107 : profile_probability);
108 : static rtx const_vector_from_tree (tree);
109 : static tree tree_expr_size (const_tree);
110 : static void convert_mode_scalar (rtx, rtx, int);
111 :
112 :
113 : /* This is run to set up which modes can be used
114 : directly in memory and to initialize the block move optab. It is run
115 : at the beginning of compilation and when the target is reinitialized. */
116 :
117 : void
118 215998 : init_expr_target (void)
119 : {
120 215998 : rtx pat;
121 215998 : int num_clobbers;
122 215998 : rtx mem, mem1;
123 215998 : rtx reg;
124 :
125 : /* Try indexing by frame ptr and try by stack ptr.
126 : It is known that on the Convex the stack ptr isn't a valid index.
127 : With luck, one or the other is valid on any machine. */
128 215998 : mem = gen_rtx_MEM (word_mode, stack_pointer_rtx);
129 215998 : mem1 = gen_rtx_MEM (word_mode, frame_pointer_rtx);
130 :
131 : /* A scratch register we can modify in-place below to avoid
132 : useless RTL allocations. */
133 215998 : reg = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
134 :
135 215998 : rtx_insn *insn = as_a<rtx_insn *> (rtx_alloc (INSN));
136 215998 : pat = gen_rtx_SET (NULL_RTX, NULL_RTX);
137 215998 : PATTERN (insn) = pat;
138 :
139 26999750 : for (machine_mode mode = VOIDmode; (int) mode < NUM_MACHINE_MODES;
140 26783752 : mode = (machine_mode) ((int) mode + 1))
141 : {
142 26783752 : int regno;
143 :
144 26783752 : direct_load[(int) mode] = direct_store[(int) mode] = 0;
145 26783752 : PUT_MODE (mem, mode);
146 26783752 : PUT_MODE (mem1, mode);
147 :
148 : /* See if there is some register that can be used in this mode and
149 : directly loaded or stored from memory. */
150 :
151 26783752 : if (mode != VOIDmode && mode != BLKmode)
152 1805266765 : for (regno = 0; regno < FIRST_PSEUDO_REGISTER
153 1831618521 : && (direct_load[(int) mode] == 0 || direct_store[(int) mode] == 0);
154 : regno++)
155 : {
156 1805266765 : if (!targetm.hard_regno_mode_ok (regno, mode))
157 1687053562 : continue;
158 :
159 118213203 : set_mode_and_regno (reg, mode, regno);
160 :
161 118213203 : SET_SRC (pat) = mem;
162 118213203 : SET_DEST (pat) = reg;
163 118213203 : if (recog (pat, insn, &num_clobbers) >= 0)
164 7263697 : direct_load[(int) mode] = 1;
165 :
166 118213203 : SET_SRC (pat) = mem1;
167 118213203 : SET_DEST (pat) = reg;
168 118213203 : if (recog (pat, insn, &num_clobbers) >= 0)
169 7263697 : direct_load[(int) mode] = 1;
170 :
171 118213203 : SET_SRC (pat) = reg;
172 118213203 : SET_DEST (pat) = mem;
173 118213203 : if (recog (pat, insn, &num_clobbers) >= 0)
174 7263697 : direct_store[(int) mode] = 1;
175 :
176 118213203 : SET_SRC (pat) = reg;
177 118213203 : SET_DEST (pat) = mem1;
178 118213203 : if (recog (pat, insn, &num_clobbers) >= 0)
179 7263697 : direct_store[(int) mode] = 1;
180 : }
181 : }
182 :
183 221703 : mem = gen_rtx_MEM (VOIDmode, gen_raw_REG (Pmode, LAST_VIRTUAL_REGISTER + 1));
184 :
185 215998 : opt_scalar_float_mode mode_iter;
186 1511986 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_FLOAT)
187 : {
188 1295988 : scalar_float_mode mode = mode_iter.require ();
189 1295988 : scalar_float_mode srcmode;
190 4535958 : FOR_EACH_MODE_UNTIL (srcmode, mode)
191 : {
192 3239970 : enum insn_code ic;
193 :
194 3239970 : ic = can_extend_p (mode, srcmode, 0);
195 3239970 : if (ic == CODE_FOR_nothing)
196 2587338 : continue;
197 :
198 652632 : PUT_MODE (mem, srcmode);
199 :
200 652632 : if (insn_operand_matches (ic, 1, mem))
201 650505 : float_extend_from_mem[mode][srcmode] = true;
202 : }
203 : }
204 215998 : }
205 :
206 : /* This is run at the start of compiling a function. */
207 :
208 : void
209 1703805 : init_expr (void)
210 : {
211 1703805 : memset (&crtl->expr, 0, sizeof (crtl->expr));
212 1703805 : }
213 :
214 : /* Copy data from FROM to TO, where the machine modes are not the same.
215 : Both modes may be integer, or both may be floating, or both may be
216 : fixed-point.
217 : UNSIGNEDP should be nonzero if FROM is an unsigned type.
218 : This causes zero-extension instead of sign-extension. */
219 :
220 : void
221 2139976 : convert_move (rtx to, rtx from, int unsignedp)
222 : {
223 2139976 : machine_mode to_mode = GET_MODE (to);
224 2139976 : machine_mode from_mode = GET_MODE (from);
225 :
226 2139976 : gcc_assert (to_mode != BLKmode);
227 2139976 : gcc_assert (from_mode != BLKmode);
228 :
229 : /* If the source and destination are already the same, then there's
230 : nothing to do. */
231 2139976 : if (to == from)
232 2139976 : return;
233 :
234 : /* If FROM is a SUBREG that indicates that we have already done at least
235 : the required extension, strip it. We don't handle such SUBREGs as
236 : TO here. */
237 :
238 2139976 : scalar_int_mode to_int_mode;
239 2139976 : if (GET_CODE (from) == SUBREG
240 113082 : && SUBREG_PROMOTED_VAR_P (from)
241 2139976 : && is_a <scalar_int_mode> (to_mode, &to_int_mode)
242 2139976 : && (GET_MODE_PRECISION (subreg_promoted_mode (from))
243 0 : >= GET_MODE_PRECISION (to_int_mode))
244 2139976 : && SUBREG_CHECK_PROMOTED_SIGN (from, unsignedp))
245 : {
246 0 : scalar_int_mode int_orig_mode;
247 0 : scalar_int_mode int_inner_mode;
248 0 : machine_mode orig_mode = GET_MODE (from);
249 :
250 0 : from = gen_lowpart (to_int_mode, SUBREG_REG (from));
251 0 : from_mode = to_int_mode;
252 :
253 : /* Preserve SUBREG_PROMOTED_VAR_P if the new mode is wider than
254 : the original mode, but narrower than the inner mode. */
255 0 : if (GET_CODE (from) == SUBREG
256 0 : && is_a <scalar_int_mode> (orig_mode, &int_orig_mode)
257 0 : && GET_MODE_PRECISION (to_int_mode)
258 0 : > GET_MODE_PRECISION (int_orig_mode)
259 0 : && is_a <scalar_int_mode> (GET_MODE (SUBREG_REG (from)),
260 : &int_inner_mode)
261 0 : && GET_MODE_PRECISION (int_inner_mode)
262 0 : > GET_MODE_PRECISION (to_int_mode))
263 : {
264 0 : SUBREG_PROMOTED_VAR_P (from) = 1;
265 0 : SUBREG_PROMOTED_SET (from, unsignedp);
266 : }
267 : }
268 :
269 2139976 : gcc_assert (GET_CODE (to) != SUBREG || !SUBREG_PROMOTED_VAR_P (to));
270 :
271 2139976 : if (to_mode == from_mode
272 2093377 : || (from_mode == VOIDmode && CONSTANT_P (from)))
273 : {
274 46870 : emit_move_insn (to, from);
275 46870 : return;
276 : }
277 :
278 2093106 : if (VECTOR_MODE_P (to_mode) || VECTOR_MODE_P (from_mode))
279 : {
280 27212 : if (GET_MODE_UNIT_PRECISION (to_mode)
281 13606 : > GET_MODE_UNIT_PRECISION (from_mode))
282 : {
283 4460 : optab op = unsignedp ? zext_optab : sext_optab;
284 4460 : insn_code icode = convert_optab_handler (op, to_mode, from_mode);
285 4460 : if (icode != CODE_FOR_nothing)
286 : {
287 642 : emit_unop_insn (icode, to, from,
288 : unsignedp ? ZERO_EXTEND : SIGN_EXTEND);
289 642 : return;
290 : }
291 : }
292 :
293 25928 : if (GET_MODE_UNIT_PRECISION (to_mode)
294 12964 : < GET_MODE_UNIT_PRECISION (from_mode))
295 : {
296 1883 : insn_code icode = convert_optab_handler (trunc_optab,
297 : to_mode, from_mode);
298 1883 : if (icode != CODE_FOR_nothing)
299 : {
300 870 : emit_unop_insn (icode, to, from, TRUNCATE);
301 870 : return;
302 : }
303 : }
304 :
305 36282 : gcc_assert (known_eq (GET_MODE_BITSIZE (from_mode),
306 : GET_MODE_BITSIZE (to_mode)));
307 :
308 12094 : if (VECTOR_MODE_P (to_mode))
309 12094 : from = force_subreg (to_mode, from, GET_MODE (from), 0);
310 : else
311 0 : to = simplify_gen_subreg (from_mode, to, GET_MODE (to), 0);
312 :
313 12094 : emit_move_insn (to, from);
314 12094 : return;
315 : }
316 :
317 2079500 : if (GET_CODE (to) == CONCAT && GET_CODE (from) == CONCAT)
318 : {
319 0 : convert_move (XEXP (to, 0), XEXP (from, 0), unsignedp);
320 0 : convert_move (XEXP (to, 1), XEXP (from, 1), unsignedp);
321 0 : return;
322 : }
323 :
324 2079500 : convert_mode_scalar (to, from, unsignedp);
325 : }
326 :
327 : /* Like convert_move, but deals only with scalar modes. */
328 :
329 : static void
330 2079550 : convert_mode_scalar (rtx to, rtx from, int unsignedp)
331 : {
332 : /* Both modes should be scalar types. */
333 2079555 : scalar_mode from_mode = as_a <scalar_mode> (GET_MODE (from));
334 2079555 : scalar_mode to_mode = as_a <scalar_mode> (GET_MODE (to));
335 2079555 : bool to_real = SCALAR_FLOAT_MODE_P (to_mode);
336 2079555 : bool from_real = SCALAR_FLOAT_MODE_P (from_mode);
337 2079555 : enum insn_code code;
338 2079555 : rtx libcall;
339 :
340 2079555 : gcc_assert (to_real == from_real);
341 :
342 : /* rtx code for making an equivalent value. */
343 3158860 : enum rtx_code equiv_code = (unsignedp < 0 ? UNKNOWN
344 2079555 : : (unsignedp ? ZERO_EXTEND : SIGN_EXTEND));
345 :
346 2079555 : auto acceptable_same_precision_modes
347 580 : = [] (scalar_mode from_mode, scalar_mode to_mode) -> bool
348 : {
349 580 : if (DECIMAL_FLOAT_MODE_P (from_mode) != DECIMAL_FLOAT_MODE_P (to_mode))
350 : return true;
351 :
352 : /* arm_bfloat_half_format <-> ieee_half_format */
353 2 : if ((REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
354 1 : && REAL_MODE_FORMAT (to_mode) == &ieee_half_format)
355 2 : || (REAL_MODE_FORMAT (to_mode) == &arm_bfloat_half_format
356 1 : && REAL_MODE_FORMAT (from_mode) == &ieee_half_format))
357 : return true;
358 :
359 : /* ibm_extended_format <-> ieee_quad_format */
360 0 : if ((REAL_MODE_FORMAT (from_mode) == &ibm_extended_format
361 0 : && REAL_MODE_FORMAT (to_mode) == &ieee_quad_format)
362 0 : || (REAL_MODE_FORMAT (from_mode) == &ieee_quad_format
363 0 : && REAL_MODE_FORMAT (to_mode) == &ibm_extended_format))
364 0 : return true;
365 :
366 : return false;
367 : };
368 :
369 2079555 : if (to_real)
370 : {
371 183196 : rtx value;
372 183196 : rtx_insn *insns;
373 183196 : convert_optab tab;
374 :
375 183196 : gcc_assert ((GET_MODE_PRECISION (from_mode)
376 : != GET_MODE_PRECISION (to_mode))
377 : || acceptable_same_precision_modes (from_mode, to_mode));
378 :
379 183196 : if (GET_MODE_PRECISION (from_mode) == GET_MODE_PRECISION (to_mode))
380 : {
381 580 : if ((REAL_MODE_FORMAT (to_mode) == &arm_bfloat_half_format
382 1 : && REAL_MODE_FORMAT (from_mode) == &ieee_half_format)
383 580 : || (REAL_MODE_FORMAT (to_mode) == &ieee_quad_format
384 0 : && REAL_MODE_FORMAT (from_mode) == &ibm_extended_format))
385 : /* libgcc implements just __trunchfbf2, not __extendhfbf2;
386 : and __trunctfkf2, not __extendtfkf2. */
387 : tab = trunc_optab;
388 : else
389 : /* Conversion between decimal float and binary float, same
390 : size. */
391 579 : tab = DECIMAL_FLOAT_MODE_P (from_mode) ? trunc_optab : sext_optab;
392 : }
393 182616 : else if (GET_MODE_PRECISION (from_mode) < GET_MODE_PRECISION (to_mode))
394 : tab = sext_optab;
395 : else
396 18333 : tab = trunc_optab;
397 :
398 : /* Try converting directly if the insn is supported. */
399 :
400 183196 : code = convert_optab_handler (tab, to_mode, from_mode);
401 183196 : if (code != CODE_FOR_nothing)
402 : {
403 161017 : emit_unop_insn (code, to, from,
404 : tab == sext_optab ? FLOAT_EXTEND : FLOAT_TRUNCATE);
405 161017 : return;
406 : }
407 :
408 : #ifdef HAVE_SFmode
409 22179 : if (REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
410 22179 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
411 : {
412 2417 : if (GET_MODE_PRECISION (to_mode) > GET_MODE_PRECISION (SFmode))
413 : {
414 : /* To cut down on libgcc size, implement
415 : BFmode -> {DF,XF,TF}mode conversions by
416 : BFmode -> SFmode -> {DF,XF,TF}mode conversions. */
417 4 : rtx temp = gen_reg_rtx (SFmode);
418 4 : convert_mode_scalar (temp, from, unsignedp);
419 4 : convert_mode_scalar (to, temp, unsignedp);
420 4 : return;
421 : }
422 2413 : if (REAL_MODE_FORMAT (to_mode) == &ieee_half_format)
423 : {
424 : /* Similarly, implement BFmode -> HFmode as
425 : BFmode -> SFmode -> HFmode conversion where SFmode
426 : has superset of BFmode values. We don't need
427 : to handle sNaNs by raising exception and turning
428 : it into qNaN though, as that can be done in the
429 : SFmode -> HFmode conversion too. */
430 1 : rtx temp = gen_reg_rtx (SFmode);
431 1 : int save_flag_finite_math_only = flag_finite_math_only;
432 1 : flag_finite_math_only = true;
433 1 : convert_mode_scalar (temp, from, unsignedp);
434 1 : flag_finite_math_only = save_flag_finite_math_only;
435 1 : convert_mode_scalar (to, temp, unsignedp);
436 1 : return;
437 : }
438 2412 : if (to_mode == SFmode
439 2412 : && !HONOR_NANS (from_mode)
440 46 : && !HONOR_NANS (to_mode)
441 2458 : && optimize_insn_for_speed_p ())
442 : {
443 : /* If we don't expect sNaNs, for BFmode -> SFmode we can just
444 : shift the bits up. */
445 45 : machine_mode fromi_mode, toi_mode;
446 90 : if (int_mode_for_size (GET_MODE_BITSIZE (from_mode),
447 45 : 0).exists (&fromi_mode)
448 45 : && int_mode_for_size (GET_MODE_BITSIZE (to_mode),
449 0 : 0).exists (&toi_mode))
450 : {
451 45 : start_sequence ();
452 45 : rtx fromi = force_lowpart_subreg (fromi_mode, from,
453 : from_mode);
454 45 : rtx tof = NULL_RTX;
455 45 : if (fromi)
456 : {
457 45 : rtx toi;
458 45 : if (GET_MODE (fromi) == VOIDmode)
459 0 : toi = simplify_unary_operation (ZERO_EXTEND, toi_mode,
460 : fromi, fromi_mode);
461 : else
462 : {
463 45 : toi = gen_reg_rtx (toi_mode);
464 45 : convert_mode_scalar (toi, fromi, 1);
465 : }
466 45 : toi
467 90 : = maybe_expand_shift (LSHIFT_EXPR, toi_mode, toi,
468 45 : GET_MODE_PRECISION (to_mode)
469 45 : - GET_MODE_PRECISION (from_mode),
470 : NULL_RTX, 1);
471 45 : if (toi)
472 : {
473 45 : tof = force_lowpart_subreg (to_mode, toi, toi_mode);
474 45 : if (tof)
475 45 : emit_move_insn (to, tof);
476 : }
477 : }
478 45 : insns = end_sequence ();
479 45 : if (tof)
480 : {
481 45 : emit_insn (insns);
482 45 : return;
483 : }
484 : }
485 : }
486 : }
487 22129 : if (REAL_MODE_FORMAT (from_mode) == &ieee_single_format
488 4035 : && REAL_MODE_FORMAT (to_mode) == &arm_bfloat_half_format
489 1544 : && !HONOR_NANS (from_mode)
490 0 : && !HONOR_NANS (to_mode)
491 0 : && !flag_rounding_math
492 22129 : && optimize_insn_for_speed_p ())
493 : {
494 : /* If we don't expect qNaNs nor sNaNs and can assume rounding
495 : to nearest, we can expand the conversion inline as
496 : (fromi + 0x7fff + ((fromi >> 16) & 1)) >> 16. */
497 0 : machine_mode fromi_mode, toi_mode;
498 0 : if (int_mode_for_size (GET_MODE_BITSIZE (from_mode),
499 0 : 0).exists (&fromi_mode)
500 0 : && int_mode_for_size (GET_MODE_BITSIZE (to_mode),
501 0 : 0).exists (&toi_mode))
502 : {
503 0 : start_sequence ();
504 0 : rtx fromi = force_lowpart_subreg (fromi_mode, from, from_mode);
505 0 : rtx tof = NULL_RTX;
506 0 : do
507 : {
508 0 : if (!fromi)
509 : break;
510 0 : int shift = (GET_MODE_PRECISION (from_mode)
511 0 : - GET_MODE_PRECISION (to_mode));
512 0 : rtx temp1
513 0 : = maybe_expand_shift (RSHIFT_EXPR, fromi_mode, fromi,
514 : shift, NULL_RTX, 1);
515 0 : if (!temp1)
516 : break;
517 0 : rtx temp2
518 0 : = expand_binop (fromi_mode, and_optab, temp1, const1_rtx,
519 : NULL_RTX, 1, OPTAB_DIRECT);
520 0 : if (!temp2)
521 : break;
522 0 : rtx temp3
523 0 : = expand_binop (fromi_mode, add_optab, fromi,
524 : gen_int_mode ((HOST_WIDE_INT_1U
525 0 : << (shift - 1)) - 1,
526 : fromi_mode), NULL_RTX,
527 : 1, OPTAB_DIRECT);
528 0 : if (!temp3)
529 : break;
530 0 : rtx temp4
531 0 : = expand_binop (fromi_mode, add_optab, temp3, temp2,
532 : NULL_RTX, 1, OPTAB_DIRECT);
533 0 : if (!temp4)
534 : break;
535 0 : rtx temp5 = maybe_expand_shift (RSHIFT_EXPR, fromi_mode,
536 : temp4, shift, NULL_RTX, 1);
537 0 : if (!temp5)
538 : break;
539 0 : rtx temp6 = force_lowpart_subreg (toi_mode, temp5,
540 : fromi_mode);
541 0 : if (!temp6)
542 : break;
543 0 : tof = force_lowpart_subreg (to_mode, temp6, toi_mode);
544 0 : if (tof)
545 0 : emit_move_insn (to, tof);
546 : }
547 : while (0);
548 0 : insns = end_sequence ();
549 0 : if (tof)
550 : {
551 0 : emit_insn (insns);
552 0 : return;
553 : }
554 : }
555 : }
556 : #endif
557 :
558 : /* Otherwise use a libcall. */
559 22129 : libcall = convert_optab_libfunc (tab, to_mode, from_mode);
560 :
561 : /* Is this conversion implemented yet? */
562 22129 : gcc_assert (libcall);
563 :
564 22129 : start_sequence ();
565 22129 : value = emit_library_call_value (libcall, NULL_RTX, LCT_CONST, to_mode,
566 : from, from_mode);
567 22129 : insns = end_sequence ();
568 22129 : emit_libcall_block (insns, to, value,
569 6508 : tab == trunc_optab ? gen_rtx_FLOAT_TRUNCATE (to_mode,
570 : from)
571 15621 : : gen_rtx_FLOAT_EXTEND (to_mode, from));
572 22129 : return;
573 : }
574 :
575 : /* Handle pointer conversion. */ /* SPEE 900220. */
576 : /* If the target has a converter from FROM_MODE to TO_MODE, use it. */
577 1896359 : {
578 1896359 : convert_optab ctab;
579 :
580 1896359 : if (GET_MODE_PRECISION (from_mode) > GET_MODE_PRECISION (to_mode))
581 : ctab = trunc_optab;
582 1709156 : else if (unsignedp)
583 : ctab = zext_optab;
584 : else
585 815404 : ctab = sext_optab;
586 :
587 1896359 : if (convert_optab_handler (ctab, to_mode, from_mode)
588 : != CODE_FOR_nothing)
589 : {
590 1638725 : emit_unop_insn (convert_optab_handler (ctab, to_mode, from_mode),
591 : to, from, UNKNOWN);
592 1638725 : return;
593 : }
594 : }
595 :
596 : /* Targets are expected to provide conversion insns between PxImode and
597 : xImode for all MODE_PARTIAL_INT modes they use, but no others. */
598 257634 : if (GET_MODE_CLASS (to_mode) == MODE_PARTIAL_INT)
599 : {
600 0 : scalar_int_mode full_mode
601 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE (to_mode)).require ();
602 :
603 0 : gcc_assert (convert_optab_handler (trunc_optab, to_mode, full_mode)
604 : != CODE_FOR_nothing);
605 :
606 0 : if (full_mode != from_mode)
607 0 : from = convert_to_mode (full_mode, from, unsignedp);
608 0 : emit_unop_insn (convert_optab_handler (trunc_optab, to_mode, full_mode),
609 : to, from, UNKNOWN);
610 0 : return;
611 : }
612 257634 : if (GET_MODE_CLASS (from_mode) == MODE_PARTIAL_INT)
613 : {
614 0 : rtx new_from;
615 0 : scalar_int_mode full_mode
616 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE (from_mode)).require ();
617 0 : convert_optab ctab = unsignedp ? zext_optab : sext_optab;
618 0 : enum insn_code icode;
619 :
620 0 : icode = convert_optab_handler (ctab, full_mode, from_mode);
621 0 : gcc_assert (icode != CODE_FOR_nothing);
622 :
623 0 : if (to_mode == full_mode)
624 : {
625 0 : emit_unop_insn (icode, to, from, UNKNOWN);
626 0 : return;
627 : }
628 :
629 0 : new_from = gen_reg_rtx (full_mode);
630 0 : emit_unop_insn (icode, new_from, from, UNKNOWN);
631 :
632 : /* else proceed to integer conversions below. */
633 0 : from_mode = full_mode;
634 0 : from = new_from;
635 : }
636 :
637 : /* Make sure both are fixed-point modes or both are not. */
638 257634 : gcc_assert (ALL_SCALAR_FIXED_POINT_MODE_P (from_mode) ==
639 : ALL_SCALAR_FIXED_POINT_MODE_P (to_mode));
640 257634 : if (ALL_SCALAR_FIXED_POINT_MODE_P (from_mode))
641 : {
642 : /* If we widen from_mode to to_mode and they are in the same class,
643 : we won't saturate the result.
644 : Otherwise, always saturate the result to play safe. */
645 0 : if (GET_MODE_CLASS (from_mode) == GET_MODE_CLASS (to_mode)
646 0 : && GET_MODE_SIZE (from_mode) < GET_MODE_SIZE (to_mode))
647 0 : expand_fixed_convert (to, from, 0, 0);
648 : else
649 0 : expand_fixed_convert (to, from, 0, 1);
650 0 : return;
651 : }
652 :
653 : /* Now both modes are integers. */
654 :
655 : /* Handle expanding beyond a word. */
656 257634 : if (GET_MODE_PRECISION (from_mode) < GET_MODE_PRECISION (to_mode)
657 260075 : && GET_MODE_PRECISION (to_mode) > BITS_PER_WORD)
658 : {
659 70431 : rtx_insn *insns;
660 70431 : rtx lowpart;
661 70431 : rtx fill_value;
662 70431 : rtx lowfrom;
663 70431 : int i;
664 70431 : scalar_mode lowpart_mode;
665 140862 : int nwords = CEIL (GET_MODE_SIZE (to_mode), UNITS_PER_WORD);
666 :
667 : /* Try converting directly if the insn is supported. */
668 70431 : if ((code = can_extend_p (to_mode, from_mode, unsignedp))
669 : != CODE_FOR_nothing)
670 : {
671 : /* If FROM is a SUBREG, put it into a register. Do this
672 : so that we always generate the same set of insns for
673 : better cse'ing; if an intermediate assignment occurred,
674 : we won't be doing the operation directly on the SUBREG. */
675 0 : if (optimize > 0 && GET_CODE (from) == SUBREG)
676 0 : from = force_reg (from_mode, from);
677 0 : emit_unop_insn (code, to, from, equiv_code);
678 0 : return;
679 : }
680 : /* Next, try converting via full word. */
681 70431 : else if (GET_MODE_PRECISION (from_mode) < BITS_PER_WORD
682 70431 : && ((code = can_extend_p (to_mode, word_mode, unsignedp))
683 : != CODE_FOR_nothing))
684 : {
685 70431 : rtx word_to = gen_reg_rtx (word_mode);
686 70431 : if (REG_P (to))
687 : {
688 70358 : if (reg_overlap_mentioned_p (to, from))
689 0 : from = force_reg (from_mode, from);
690 70358 : emit_clobber (to);
691 : }
692 70431 : convert_move (word_to, from, unsignedp);
693 70431 : emit_unop_insn (code, to, word_to, equiv_code);
694 70431 : return;
695 : }
696 :
697 : /* No special multiword conversion insn; do it by hand. */
698 0 : start_sequence ();
699 :
700 : /* Since we will turn this into a no conflict block, we must ensure
701 : the source does not overlap the target so force it into an isolated
702 : register when maybe so. Likewise for any MEM input, since the
703 : conversion sequence might require several references to it and we
704 : must ensure we're getting the same value every time. */
705 :
706 0 : if (MEM_P (from) || reg_overlap_mentioned_p (to, from))
707 0 : from = force_reg (from_mode, from);
708 :
709 : /* Get a copy of FROM widened to a word, if necessary. */
710 0 : if (GET_MODE_PRECISION (from_mode) < BITS_PER_WORD)
711 0 : lowpart_mode = word_mode;
712 : else
713 : lowpart_mode = from_mode;
714 :
715 0 : lowfrom = convert_to_mode (lowpart_mode, from, unsignedp);
716 :
717 0 : lowpart = gen_lowpart (lowpart_mode, to);
718 0 : emit_move_insn (lowpart, lowfrom);
719 :
720 : /* Compute the value to put in each remaining word. */
721 0 : if (unsignedp)
722 0 : fill_value = const0_rtx;
723 : else
724 0 : fill_value = emit_store_flag_force (gen_reg_rtx (word_mode),
725 : LT, lowfrom, const0_rtx,
726 : lowpart_mode, 0, -1);
727 :
728 : /* Fill the remaining words. */
729 0 : for (i = GET_MODE_SIZE (lowpart_mode) / UNITS_PER_WORD; i < nwords; i++)
730 : {
731 0 : int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
732 0 : rtx subword = operand_subword (to, index, 1, to_mode);
733 :
734 0 : gcc_assert (subword);
735 :
736 0 : if (fill_value != subword)
737 0 : emit_move_insn (subword, fill_value);
738 : }
739 :
740 0 : insns = end_sequence ();
741 :
742 0 : emit_insn (insns);
743 0 : return;
744 : }
745 :
746 : /* Truncating multi-word to a word or less. */
747 187203 : if (GET_MODE_PRECISION (from_mode) > BITS_PER_WORD
748 187203 : && GET_MODE_PRECISION (to_mode) <= BITS_PER_WORD)
749 : {
750 69057 : if (!((MEM_P (from)
751 539 : && ! MEM_VOLATILE_P (from)
752 539 : && direct_load[(int) to_mode]
753 539 : && ! mode_dependent_address_p (XEXP (from, 0),
754 539 : MEM_ADDR_SPACE (from)))
755 38855 : || REG_P (from)
756 : || GET_CODE (from) == SUBREG))
757 0 : from = force_reg (from_mode, from);
758 39394 : convert_move (to, gen_lowpart (word_mode, from), 0);
759 39394 : return;
760 : }
761 :
762 : /* Now follow all the conversions between integers
763 : no more than a word long. */
764 :
765 : /* For truncation, usually we can just refer to FROM in a narrower mode. */
766 295618 : if (GET_MODE_BITSIZE (to_mode) < GET_MODE_BITSIZE (from_mode)
767 147809 : && TRULY_NOOP_TRUNCATION_MODES_P (to_mode, from_mode))
768 : {
769 159626 : if (!((MEM_P (from)
770 13555 : && ! MEM_VOLATILE_P (from)
771 13496 : && direct_load[(int) to_mode]
772 13496 : && ! mode_dependent_address_p (XEXP (from, 0),
773 13496 : MEM_ADDR_SPACE (from)))
774 134313 : || REG_P (from)
775 : || GET_CODE (from) == SUBREG))
776 924 : from = force_reg (from_mode, from);
777 123420 : if (REG_P (from) && REGNO (from) < FIRST_PSEUDO_REGISTER
778 147810 : && !targetm.hard_regno_mode_ok (REGNO (from), to_mode))
779 0 : from = copy_to_reg (from);
780 147809 : emit_move_insn (to, gen_lowpart (to_mode, from));
781 147809 : return;
782 : }
783 :
784 : /* Handle extension. */
785 0 : if (GET_MODE_PRECISION (to_mode) > GET_MODE_PRECISION (from_mode))
786 : {
787 : /* Convert directly if that works. */
788 0 : if ((code = can_extend_p (to_mode, from_mode, unsignedp))
789 : != CODE_FOR_nothing)
790 : {
791 0 : emit_unop_insn (code, to, from, equiv_code);
792 0 : return;
793 : }
794 : else
795 : {
796 0 : rtx tmp;
797 0 : int shift_amount;
798 :
799 : /* Search for a mode to convert via. */
800 0 : opt_scalar_mode intermediate_iter;
801 0 : FOR_EACH_MODE_FROM (intermediate_iter, from_mode)
802 : {
803 0 : scalar_mode intermediate = intermediate_iter.require ();
804 0 : if (((can_extend_p (to_mode, intermediate, unsignedp)
805 : != CODE_FOR_nothing)
806 0 : || (GET_MODE_SIZE (to_mode) < GET_MODE_SIZE (intermediate)
807 0 : && TRULY_NOOP_TRUNCATION_MODES_P (to_mode,
808 : intermediate)))
809 0 : && (can_extend_p (intermediate, from_mode, unsignedp)
810 : != CODE_FOR_nothing))
811 : {
812 0 : convert_move (to, convert_to_mode (intermediate, from,
813 : unsignedp), unsignedp);
814 0 : return;
815 : }
816 : }
817 :
818 : /* No suitable intermediate mode.
819 : Generate what we need with shifts. */
820 0 : shift_amount = (GET_MODE_PRECISION (to_mode)
821 0 : - GET_MODE_PRECISION (from_mode));
822 0 : from = gen_lowpart (to_mode, force_reg (from_mode, from));
823 0 : tmp = expand_shift (LSHIFT_EXPR, to_mode, from, shift_amount,
824 : to, unsignedp);
825 0 : tmp = expand_shift (RSHIFT_EXPR, to_mode, tmp, shift_amount,
826 : to, unsignedp);
827 0 : if (tmp != to)
828 0 : emit_move_insn (to, tmp);
829 0 : return;
830 : }
831 : }
832 :
833 : /* Support special truncate insns for certain modes. */
834 0 : if (convert_optab_handler (trunc_optab, to_mode,
835 : from_mode) != CODE_FOR_nothing)
836 : {
837 0 : emit_unop_insn (convert_optab_handler (trunc_optab, to_mode, from_mode),
838 : to, from, UNKNOWN);
839 0 : return;
840 : }
841 :
842 : /* Handle truncation of volatile memrefs, and so on;
843 : the things that couldn't be truncated directly,
844 : and for which there was no special instruction.
845 :
846 : ??? Code above formerly short-circuited this, for most integer
847 : mode pairs, with a force_reg in from_mode followed by a recursive
848 : call to this routine. Appears always to have been wrong. */
849 0 : if (GET_MODE_PRECISION (to_mode) < GET_MODE_PRECISION (from_mode))
850 : {
851 0 : rtx temp = force_reg (to_mode, gen_lowpart (to_mode, from));
852 0 : emit_move_insn (to, temp);
853 0 : return;
854 : }
855 :
856 : /* Mode combination is not recognized. */
857 0 : gcc_unreachable ();
858 : }
859 :
860 : /* Return an rtx for a value that would result
861 : from converting X to mode MODE.
862 : Both X and MODE may be floating, or both integer.
863 : UNSIGNEDP is nonzero if X is an unsigned value.
864 : This can be done by referring to a part of X in place
865 : or by copying to a new temporary with conversion. */
866 :
867 : rtx
868 1934270 : convert_to_mode (machine_mode mode, rtx x, int unsignedp)
869 : {
870 1934270 : return convert_modes (mode, VOIDmode, x, unsignedp);
871 : }
872 :
873 : /* Return an rtx for a value that would result
874 : from converting X from mode OLDMODE to mode MODE.
875 : Both modes may be floating, or both integer.
876 : UNSIGNEDP is nonzero if X is an unsigned value.
877 :
878 : This can be done by referring to a part of X in place
879 : or by copying to a new temporary with conversion.
880 :
881 : You can give VOIDmode for OLDMODE, if you are sure X has a nonvoid mode. */
882 :
883 : rtx
884 4747005 : convert_modes (machine_mode mode, machine_mode oldmode, rtx x, int unsignedp)
885 : {
886 4747005 : rtx temp;
887 4747005 : scalar_int_mode int_mode;
888 :
889 : /* If FROM is a SUBREG that indicates that we have already done at least
890 : the required extension, strip it. */
891 :
892 4747005 : if (GET_CODE (x) == SUBREG
893 80921 : && SUBREG_PROMOTED_VAR_P (x)
894 4747005 : && is_a <scalar_int_mode> (mode, &int_mode)
895 4747005 : && (GET_MODE_PRECISION (subreg_promoted_mode (x))
896 7 : >= GET_MODE_PRECISION (int_mode))
897 4747012 : && SUBREG_CHECK_PROMOTED_SIGN (x, unsignedp))
898 : {
899 7 : scalar_int_mode int_orig_mode;
900 7 : scalar_int_mode int_inner_mode;
901 7 : machine_mode orig_mode = GET_MODE (x);
902 7 : x = gen_lowpart (int_mode, SUBREG_REG (x));
903 :
904 : /* Preserve SUBREG_PROMOTED_VAR_P if the new mode is wider than
905 : the original mode, but narrower than the inner mode. */
906 7 : if (GET_CODE (x) == SUBREG
907 0 : && is_a <scalar_int_mode> (orig_mode, &int_orig_mode)
908 0 : && GET_MODE_PRECISION (int_mode)
909 0 : > GET_MODE_PRECISION (int_orig_mode)
910 0 : && is_a <scalar_int_mode> (GET_MODE (SUBREG_REG (x)),
911 : &int_inner_mode)
912 7 : && GET_MODE_PRECISION (int_inner_mode)
913 0 : > GET_MODE_PRECISION (int_mode))
914 : {
915 0 : SUBREG_PROMOTED_VAR_P (x) = 1;
916 0 : SUBREG_PROMOTED_SET (x, unsignedp);
917 : }
918 : }
919 :
920 4747005 : if (GET_MODE (x) != VOIDmode)
921 2468783 : oldmode = GET_MODE (x);
922 :
923 4747005 : if (mode == oldmode)
924 : return x;
925 :
926 3696860 : if (CONST_SCALAR_INT_P (x)
927 3696860 : && is_a <scalar_int_mode> (mode, &int_mode))
928 : {
929 : /* If the caller did not tell us the old mode, then there is not
930 : much to do with respect to canonicalization. We have to
931 : assume that all the bits are significant. */
932 1958228 : if (!is_a <scalar_int_mode> (oldmode))
933 1644693 : oldmode = MAX_MODE_INT;
934 1958228 : wide_int w = wide_int::from (rtx_mode_t (x, oldmode),
935 1958228 : GET_MODE_PRECISION (int_mode),
936 2594185 : unsignedp ? UNSIGNED : SIGNED);
937 1958228 : return immed_wide_int_const (w, int_mode);
938 1958228 : }
939 :
940 : /* We can do this with a gen_lowpart if both desired and current modes
941 : are integer, and this is either a constant integer, a register, or a
942 : non-volatile MEM. */
943 1738632 : scalar_int_mode int_oldmode;
944 1738632 : if (is_int_mode (mode, &int_mode)
945 1648631 : && is_int_mode (oldmode, &int_oldmode)
946 1648631 : && GET_MODE_PRECISION (int_mode) <= GET_MODE_PRECISION (int_oldmode)
947 438934 : && ((MEM_P (x) && !MEM_VOLATILE_P (x) && direct_load[(int) int_mode])
948 11978 : || CONST_POLY_INT_P (x)
949 426956 : || (REG_P (x)
950 391295 : && (!HARD_REGISTER_P (x)
951 1041 : || targetm.hard_regno_mode_ok (REGNO (x), int_mode))
952 391295 : && TRULY_NOOP_TRUNCATION_MODES_P (int_mode, GET_MODE (x)))))
953 403273 : return gen_lowpart (int_mode, x);
954 :
955 : /* Converting from integer constant into mode is always equivalent to an
956 : subreg operation. */
957 1335359 : if (VECTOR_MODE_P (mode) && GET_MODE (x) == VOIDmode)
958 : {
959 0 : gcc_assert (known_eq (GET_MODE_BITSIZE (mode),
960 : GET_MODE_BITSIZE (oldmode)));
961 0 : return force_subreg (mode, x, oldmode, 0);
962 : }
963 :
964 1335359 : temp = gen_reg_rtx (mode);
965 1335359 : convert_move (temp, x, unsignedp);
966 1335359 : return temp;
967 : }
968 :
969 : /* Variant of convert_modes for ABI parameter passing/return.
970 : Return an rtx for a value that would result from converting X from
971 : a floating point mode FMODE to wider integer mode MODE. */
972 :
973 : rtx
974 0 : convert_float_to_wider_int (machine_mode mode, machine_mode fmode, rtx x)
975 : {
976 0 : gcc_assert (SCALAR_INT_MODE_P (mode) && SCALAR_FLOAT_MODE_P (fmode));
977 0 : scalar_int_mode tmp_mode = int_mode_for_mode (fmode).require ();
978 0 : rtx tmp = force_reg (tmp_mode, gen_lowpart (tmp_mode, x));
979 0 : return convert_modes (mode, tmp_mode, tmp, 1);
980 : }
981 :
982 : /* Variant of convert_modes for ABI parameter passing/return.
983 : Return an rtx for a value that would result from converting X from
984 : an integer mode IMODE to a narrower floating point mode MODE. */
985 :
986 : rtx
987 0 : convert_wider_int_to_float (machine_mode mode, machine_mode imode, rtx x)
988 : {
989 0 : gcc_assert (SCALAR_FLOAT_MODE_P (mode) && SCALAR_INT_MODE_P (imode));
990 0 : scalar_int_mode tmp_mode = int_mode_for_mode (mode).require ();
991 0 : rtx tmp = force_reg (tmp_mode, gen_lowpart (tmp_mode, x));
992 0 : return gen_lowpart_SUBREG (mode, tmp);
993 : }
994 :
995 : /* Return the largest alignment we can use for doing a move (or store)
996 : of MAX_PIECES. ALIGN is the largest alignment we could use. */
997 :
998 : static unsigned int
999 2734254 : alignment_for_piecewise_move (unsigned int max_pieces, unsigned int align)
1000 : {
1001 2734254 : scalar_int_mode tmode
1002 2734254 : = int_mode_for_size (max_pieces * BITS_PER_UNIT, 0).require ();
1003 :
1004 2734254 : if (align >= GET_MODE_ALIGNMENT (tmode))
1005 2156122 : align = GET_MODE_ALIGNMENT (tmode);
1006 : else
1007 : {
1008 578132 : scalar_int_mode xmode = NARROWEST_INT_MODE;
1009 578132 : opt_scalar_int_mode mode_iter;
1010 3484855 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
1011 : {
1012 3479194 : tmode = mode_iter.require ();
1013 3479194 : if (GET_MODE_SIZE (tmode) > max_pieces
1014 3479194 : || targetm.slow_unaligned_access (tmode, align))
1015 : break;
1016 2906723 : xmode = tmode;
1017 : }
1018 :
1019 578132 : align = MAX (align, GET_MODE_ALIGNMENT (xmode));
1020 : }
1021 :
1022 2734254 : return align;
1023 : }
1024 :
1025 : /* Return true if we know how to implement OP using vectors of bytes. */
1026 : static bool
1027 5785328 : can_use_qi_vectors (by_pieces_operation op)
1028 : {
1029 5785328 : return (op == COMPARE_BY_PIECES
1030 0 : || op == SET_BY_PIECES
1031 0 : || op == CLEAR_BY_PIECES);
1032 : }
1033 :
1034 : /* Return true if optabs exists for the mode and certain by pieces
1035 : operations. */
1036 : static bool
1037 25663355 : by_pieces_mode_supported_p (fixed_size_mode mode, by_pieces_operation op)
1038 : {
1039 25663355 : if (optab_handler (mov_optab, mode) == CODE_FOR_nothing)
1040 : return false;
1041 :
1042 24372880 : if ((op == SET_BY_PIECES || op == CLEAR_BY_PIECES)
1043 636107 : && VECTOR_MODE_P (mode)
1044 24835050 : && optab_handler (vec_duplicate_optab, mode) == CODE_FOR_nothing)
1045 : return false;
1046 :
1047 24324347 : if (op == COMPARE_BY_PIECES
1048 24324347 : && !can_compare_p (EQ, mode, ccp_jump))
1049 : return false;
1050 :
1051 : return true;
1052 : }
1053 :
1054 : /* Return the widest mode that can be used to perform part of an
1055 : operation OP on SIZE bytes. Try to use QI vector modes where
1056 : possible. */
1057 : static fixed_size_mode
1058 5295587 : widest_fixed_size_mode_for_size (unsigned int size, by_pieces_operation op)
1059 : {
1060 5295587 : fixed_size_mode result = NARROWEST_INT_MODE;
1061 :
1062 5295587 : gcc_checking_assert (size > 1);
1063 :
1064 : /* Use QI vector only if size is wider than a WORD. */
1065 5295587 : if (can_use_qi_vectors (op))
1066 : {
1067 689521 : machine_mode mode;
1068 689521 : fixed_size_mode candidate;
1069 11416948 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_INT)
1070 11416948 : if (is_a<fixed_size_mode> (mode, &candidate)
1071 12522101 : && GET_MODE_SIZE (candidate) > UNITS_PER_WORD
1072 12350024 : && GET_MODE_INNER (candidate) == QImode)
1073 : {
1074 5386928 : if (GET_MODE_SIZE (candidate) >= size)
1075 : break;
1076 2003943 : if (by_pieces_mode_supported_p (candidate, op))
1077 10727427 : result = candidate;
1078 : }
1079 :
1080 689521 : if (result != NARROWEST_INT_MODE)
1081 441295 : return result;
1082 : }
1083 :
1084 4854292 : opt_scalar_int_mode tmode;
1085 4854292 : scalar_int_mode mode;
1086 38834336 : FOR_EACH_MODE_IN_CLASS (tmode, MODE_INT)
1087 : {
1088 33980044 : mode = tmode.require ();
1089 33980044 : if (GET_MODE_SIZE (mode) < size
1090 33980044 : && by_pieces_mode_supported_p (mode, op))
1091 23618279 : result = mode;
1092 : }
1093 :
1094 4854292 : return result;
1095 : }
1096 :
1097 : /* Determine whether an operation OP on LEN bytes with alignment ALIGN can
1098 : and should be performed piecewise. */
1099 :
1100 : static bool
1101 958761 : can_do_by_pieces (unsigned HOST_WIDE_INT len, unsigned int align,
1102 : enum by_pieces_operation op)
1103 : {
1104 958761 : return targetm.use_by_pieces_infrastructure_p (len, align, op,
1105 958761 : optimize_insn_for_speed_p ());
1106 : }
1107 :
1108 : /* Determine whether the LEN bytes can be moved by using several move
1109 : instructions. Return nonzero if a call to move_by_pieces should
1110 : succeed. */
1111 :
1112 : bool
1113 889613 : can_move_by_pieces (unsigned HOST_WIDE_INT len, unsigned int align)
1114 : {
1115 889613 : return can_do_by_pieces (len, align, MOVE_BY_PIECES);
1116 : }
1117 :
1118 : /* Return number of insns required to perform operation OP by pieces
1119 : for L bytes. ALIGN (in bits) is maximum alignment we can assume. */
1120 :
1121 : unsigned HOST_WIDE_INT
1122 1931632 : by_pieces_ninsns (unsigned HOST_WIDE_INT l, unsigned int align,
1123 : unsigned int max_size, by_pieces_operation op)
1124 : {
1125 1931632 : unsigned HOST_WIDE_INT n_insns = 0;
1126 1931632 : fixed_size_mode mode;
1127 :
1128 1931632 : if (targetm.overlap_op_by_pieces_p ())
1129 : {
1130 : /* NB: Round up L and ALIGN to the widest integer mode for
1131 : MAX_SIZE. */
1132 1931632 : mode = widest_fixed_size_mode_for_size (max_size, op);
1133 1931632 : gcc_assert (optab_handler (mov_optab, mode) != CODE_FOR_nothing);
1134 3863264 : unsigned HOST_WIDE_INT up = ROUND_UP (l, GET_MODE_SIZE (mode));
1135 1931632 : if (up > l)
1136 : l = up;
1137 1931632 : align = GET_MODE_ALIGNMENT (mode);
1138 : }
1139 :
1140 1931632 : align = alignment_for_piecewise_move (MOVE_MAX_PIECES, align);
1141 :
1142 5798124 : while (max_size > 1 && l > 0)
1143 : {
1144 1934860 : mode = widest_fixed_size_mode_for_size (max_size, op);
1145 1934860 : gcc_assert (optab_handler (mov_optab, mode) != CODE_FOR_nothing);
1146 :
1147 1934860 : unsigned int modesize = GET_MODE_SIZE (mode);
1148 :
1149 1934860 : if (align >= GET_MODE_ALIGNMENT (mode))
1150 : {
1151 1934860 : unsigned HOST_WIDE_INT n_pieces = l / modesize;
1152 1934860 : l %= modesize;
1153 1934860 : switch (op)
1154 : {
1155 1865712 : default:
1156 1865712 : n_insns += n_pieces;
1157 1865712 : break;
1158 :
1159 69148 : case COMPARE_BY_PIECES:
1160 69148 : int batch = targetm.compare_by_pieces_branch_ratio (mode);
1161 69148 : int batch_ops = 4 * batch - 1;
1162 69148 : unsigned HOST_WIDE_INT full = n_pieces / batch;
1163 69148 : n_insns += full * batch_ops;
1164 69148 : if (n_pieces % batch != 0)
1165 0 : n_insns++;
1166 : break;
1167 :
1168 : }
1169 : }
1170 : max_size = modesize;
1171 : }
1172 :
1173 1931632 : gcc_assert (!l);
1174 1931632 : return n_insns;
1175 : }
1176 :
1177 : /* Used when performing piecewise block operations, holds information
1178 : about one of the memory objects involved. The member functions
1179 : can be used to generate code for loading from the object and
1180 : updating the address when iterating. */
1181 :
1182 : class pieces_addr
1183 : {
1184 : /* The object being referenced, a MEM. Can be NULL_RTX to indicate
1185 : stack pushes. */
1186 : rtx m_obj;
1187 : /* The address of the object. Can differ from that seen in the
1188 : MEM rtx if we copied the address to a register. */
1189 : rtx m_addr;
1190 : /* Nonzero if the address on the object has an autoincrement already,
1191 : signifies whether that was an increment or decrement. */
1192 : signed char m_addr_inc;
1193 : /* Nonzero if we intend to use autoinc without the address already
1194 : having autoinc form. We will insert add insns around each memory
1195 : reference, expecting later passes to form autoinc addressing modes.
1196 : The only supported options are predecrement and postincrement. */
1197 : signed char m_explicit_inc;
1198 : /* True if we have either of the two possible cases of using
1199 : autoincrement. */
1200 : bool m_auto;
1201 : /* True if this is an address to be used for load operations rather
1202 : than stores. */
1203 : bool m_is_load;
1204 :
1205 : /* Optionally, a function to obtain constants for any given offset into
1206 : the objects, and data associated with it. */
1207 : by_pieces_constfn m_constfn;
1208 : void *m_cfndata;
1209 : public:
1210 : pieces_addr (rtx, bool, by_pieces_constfn, void *);
1211 : rtx adjust (fixed_size_mode, HOST_WIDE_INT, by_pieces_prev * = nullptr);
1212 : void increment_address (HOST_WIDE_INT);
1213 : void maybe_predec (HOST_WIDE_INT);
1214 : void maybe_postinc (HOST_WIDE_INT);
1215 : void decide_autoinc (machine_mode, bool, HOST_WIDE_INT);
1216 749055 : int get_addr_inc ()
1217 : {
1218 749055 : return m_addr_inc;
1219 : }
1220 : };
1221 :
1222 : /* Initialize a pieces_addr structure from an object OBJ. IS_LOAD is
1223 : true if the operation to be performed on this object is a load
1224 : rather than a store. For stores, OBJ can be NULL, in which case we
1225 : assume the operation is a stack push. For loads, the optional
1226 : CONSTFN and its associated CFNDATA can be used in place of the
1227 : memory load. */
1228 :
1229 1498110 : pieces_addr::pieces_addr (rtx obj, bool is_load, by_pieces_constfn constfn,
1230 1498110 : void *cfndata)
1231 1498110 : : m_obj (obj), m_is_load (is_load), m_constfn (constfn), m_cfndata (cfndata)
1232 : {
1233 1498110 : m_addr_inc = 0;
1234 1498110 : m_auto = false;
1235 1498110 : if (obj)
1236 : {
1237 1379003 : rtx addr = XEXP (obj, 0);
1238 1379003 : rtx_code code = GET_CODE (addr);
1239 1379003 : m_addr = addr;
1240 1379003 : bool dec = code == PRE_DEC || code == POST_DEC;
1241 1379003 : bool inc = code == PRE_INC || code == POST_INC;
1242 1379003 : m_auto = inc || dec;
1243 1379003 : if (m_auto)
1244 0 : m_addr_inc = dec ? -1 : 1;
1245 :
1246 : /* While we have always looked for these codes here, the code
1247 : implementing the memory operation has never handled them.
1248 : Support could be added later if necessary or beneficial. */
1249 1379003 : gcc_assert (code != PRE_INC && code != POST_DEC);
1250 : }
1251 : else
1252 : {
1253 119107 : m_addr = NULL_RTX;
1254 119107 : if (!is_load)
1255 : {
1256 45 : m_auto = true;
1257 45 : if (STACK_GROWS_DOWNWARD)
1258 45 : m_addr_inc = -1;
1259 : else
1260 : m_addr_inc = 1;
1261 : }
1262 : else
1263 119062 : gcc_assert (constfn != NULL);
1264 : }
1265 1498110 : m_explicit_inc = 0;
1266 1498110 : if (constfn)
1267 144924 : gcc_assert (is_load);
1268 1498110 : }
1269 :
1270 : /* Decide whether to use autoinc for an address involved in a memory op.
1271 : MODE is the mode of the accesses, REVERSE is true if we've decided to
1272 : perform the operation starting from the end, and LEN is the length of
1273 : the operation. Don't override an earlier decision to set m_auto. */
1274 :
1275 : void
1276 371098 : pieces_addr::decide_autoinc (machine_mode ARG_UNUSED (mode), bool reverse,
1277 : HOST_WIDE_INT len)
1278 : {
1279 371098 : if (m_auto || m_obj == NULL_RTX)
1280 : return;
1281 :
1282 341536 : bool use_predec = (m_is_load
1283 : ? USE_LOAD_PRE_DECREMENT (mode)
1284 : : USE_STORE_PRE_DECREMENT (mode));
1285 341536 : bool use_postinc = (m_is_load
1286 : ? USE_LOAD_POST_INCREMENT (mode)
1287 : : USE_STORE_POST_INCREMENT (mode));
1288 341536 : machine_mode addr_mode = get_address_mode (m_obj);
1289 :
1290 341536 : if (use_predec && reverse)
1291 : {
1292 : m_addr = copy_to_mode_reg (addr_mode,
1293 : plus_constant (addr_mode,
1294 : m_addr, len));
1295 : m_auto = true;
1296 : m_explicit_inc = -1;
1297 : }
1298 341536 : else if (use_postinc && !reverse)
1299 : {
1300 : m_addr = copy_to_mode_reg (addr_mode, m_addr);
1301 : m_auto = true;
1302 : m_explicit_inc = 1;
1303 : }
1304 341536 : else if (CONSTANT_P (m_addr))
1305 71783 : m_addr = copy_to_mode_reg (addr_mode, m_addr);
1306 : }
1307 :
1308 : /* Adjust the address to refer to the data at OFFSET in MODE. If we
1309 : are using autoincrement for this address, we don't add the offset,
1310 : but we still modify the MEM's properties. */
1311 :
1312 : rtx
1313 4088676 : pieces_addr::adjust (fixed_size_mode mode, HOST_WIDE_INT offset,
1314 : by_pieces_prev *prev)
1315 : {
1316 4088676 : if (m_constfn)
1317 : /* Pass the previous data to m_constfn. */
1318 354250 : return m_constfn (m_cfndata, prev, offset, mode);
1319 3734426 : if (m_obj == NULL_RTX)
1320 : return NULL_RTX;
1321 3734379 : if (m_auto)
1322 0 : return adjust_automodify_address (m_obj, mode, m_addr, offset);
1323 : else
1324 3734379 : return adjust_address (m_obj, mode, offset);
1325 : }
1326 :
1327 : /* Emit an add instruction to increment the address by SIZE. */
1328 :
1329 : void
1330 0 : pieces_addr::increment_address (HOST_WIDE_INT size)
1331 : {
1332 0 : rtx amount = gen_int_mode (size, GET_MODE (m_addr));
1333 0 : emit_insn (gen_add2_insn (m_addr, amount));
1334 0 : }
1335 :
1336 : /* If we are supposed to decrement the address after each access, emit code
1337 : to do so now. Increment by SIZE (which has should have the correct sign
1338 : already). */
1339 :
1340 : void
1341 4088024 : pieces_addr::maybe_predec (HOST_WIDE_INT size)
1342 : {
1343 4088024 : if (m_explicit_inc >= 0)
1344 4088024 : return;
1345 0 : gcc_assert (HAVE_PRE_DECREMENT);
1346 : increment_address (size);
1347 : }
1348 :
1349 : /* If we are supposed to decrement the address after each access, emit code
1350 : to do so now. Increment by SIZE. */
1351 :
1352 : void
1353 4088047 : pieces_addr::maybe_postinc (HOST_WIDE_INT size)
1354 : {
1355 4088047 : if (m_explicit_inc <= 0)
1356 4088047 : return;
1357 0 : gcc_assert (HAVE_POST_INCREMENT);
1358 : increment_address (size);
1359 : }
1360 :
1361 : /* This structure is used by do_op_by_pieces to describe the operation
1362 : to be performed. */
1363 :
1364 : class op_by_pieces_d
1365 : {
1366 : private:
1367 : fixed_size_mode get_usable_mode (fixed_size_mode, unsigned int);
1368 : fixed_size_mode smallest_fixed_size_mode_for_size (unsigned int);
1369 :
1370 : protected:
1371 : pieces_addr m_to, m_from;
1372 : /* Make m_len read-only so that smallest_fixed_size_mode_for_size can
1373 : use it to check the valid mode size. */
1374 : const unsigned HOST_WIDE_INT m_len;
1375 : HOST_WIDE_INT m_offset;
1376 : unsigned int m_align;
1377 : unsigned int m_max_size;
1378 : bool m_reverse;
1379 : /* True if this is a stack push. */
1380 : bool m_push;
1381 : /* True if targetm.overlap_op_by_pieces_p () returns true. */
1382 : bool m_overlap_op_by_pieces;
1383 : /* The type of operation that we're performing. */
1384 : by_pieces_operation m_op;
1385 :
1386 : /* Virtual functions, overriden by derived classes for the specific
1387 : operation. */
1388 : virtual void generate (rtx, rtx, machine_mode) = 0;
1389 : virtual bool prepare_mode (machine_mode, unsigned int) = 0;
1390 1138929 : virtual void finish_mode (machine_mode)
1391 : {
1392 1138929 : }
1393 :
1394 : public:
1395 : op_by_pieces_d (unsigned int, rtx, bool, rtx, bool, by_pieces_constfn,
1396 : void *, unsigned HOST_WIDE_INT, unsigned int, bool,
1397 : by_pieces_operation);
1398 : void run ();
1399 : };
1400 :
1401 : /* The constructor for an op_by_pieces_d structure. We require two
1402 : objects named TO and FROM, which are identified as loads or stores
1403 : by TO_LOAD and FROM_LOAD. If FROM is a load, the optional FROM_CFN
1404 : and its associated FROM_CFN_DATA can be used to replace loads with
1405 : constant values. MAX_PIECES describes the maximum number of bytes
1406 : at a time which can be moved efficiently. LEN describes the length
1407 : of the operation. */
1408 :
1409 749055 : op_by_pieces_d::op_by_pieces_d (unsigned int max_pieces, rtx to,
1410 : bool to_load, rtx from, bool from_load,
1411 : by_pieces_constfn from_cfn,
1412 : void *from_cfn_data,
1413 : unsigned HOST_WIDE_INT len,
1414 : unsigned int align, bool push,
1415 749055 : by_pieces_operation op)
1416 749055 : : m_to (to, to_load, NULL, NULL),
1417 749055 : m_from (from, from_load, from_cfn, from_cfn_data),
1418 749055 : m_len (len), m_max_size (max_pieces + 1),
1419 749055 : m_push (push), m_op (op)
1420 : {
1421 749055 : int toi = m_to.get_addr_inc ();
1422 749055 : int fromi = m_from.get_addr_inc ();
1423 749055 : if (toi >= 0 && fromi >= 0)
1424 749010 : m_reverse = false;
1425 45 : else if (toi <= 0 && fromi <= 0)
1426 45 : m_reverse = true;
1427 : else
1428 0 : gcc_unreachable ();
1429 :
1430 749055 : m_offset = m_reverse ? len : 0;
1431 2758061 : align = MIN (to ? MEM_ALIGN (to) : align,
1432 : from ? MEM_ALIGN (from) : align);
1433 :
1434 : /* If copying requires more than two move insns,
1435 : copy addresses to registers (to make displacements shorter)
1436 : and use post-increment if available. */
1437 749055 : if (by_pieces_ninsns (len, align, m_max_size, MOVE_BY_PIECES) > 2)
1438 : {
1439 : /* Find the mode of the largest comparison. */
1440 185549 : fixed_size_mode mode
1441 185549 : = widest_fixed_size_mode_for_size (m_max_size, m_op);
1442 :
1443 185549 : m_from.decide_autoinc (mode, m_reverse, len);
1444 185549 : m_to.decide_autoinc (mode, m_reverse, len);
1445 : }
1446 :
1447 749065 : align = alignment_for_piecewise_move (MOVE_MAX_PIECES, align);
1448 749055 : m_align = align;
1449 :
1450 749055 : m_overlap_op_by_pieces = targetm.overlap_op_by_pieces_p ();
1451 749055 : }
1452 :
1453 : /* This function returns the largest usable integer mode for LEN bytes
1454 : whose size is no bigger than size of MODE. */
1455 :
1456 : fixed_size_mode
1457 1232892 : op_by_pieces_d::get_usable_mode (fixed_size_mode mode, unsigned int len)
1458 : {
1459 1531903 : unsigned int size;
1460 1830914 : do
1461 : {
1462 1531903 : size = GET_MODE_SIZE (mode);
1463 1531903 : if (len >= size && prepare_mode (mode, m_align))
1464 : break;
1465 : /* widest_fixed_size_mode_for_size checks SIZE > 1. */
1466 299011 : mode = widest_fixed_size_mode_for_size (size, m_op);
1467 : }
1468 : while (1);
1469 1232892 : return mode;
1470 : }
1471 :
1472 : /* Return the smallest integer or QI vector mode that is not narrower
1473 : than SIZE bytes. */
1474 :
1475 : fixed_size_mode
1476 489741 : op_by_pieces_d::smallest_fixed_size_mode_for_size (unsigned int size)
1477 : {
1478 : /* Use QI vector only for > size of WORD. */
1479 501985 : if (can_use_qi_vectors (m_op) && size > UNITS_PER_WORD)
1480 : {
1481 7958 : machine_mode mode;
1482 7958 : fixed_size_mode candidate;
1483 107990 : FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_INT)
1484 107990 : if (is_a<fixed_size_mode> (mode, &candidate)
1485 215980 : && GET_MODE_INNER (candidate) == QImode)
1486 : {
1487 : /* Don't return a mode wider than M_LEN. */
1488 100236 : if (GET_MODE_SIZE (candidate) > m_len)
1489 : break;
1490 :
1491 48204 : if (GET_MODE_SIZE (candidate) >= size
1492 48204 : && by_pieces_mode_supported_p (candidate, m_op))
1493 6044 : return candidate;
1494 : }
1495 : }
1496 :
1497 483697 : return smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
1498 : }
1499 :
1500 : /* This function contains the main loop used for expanding a block
1501 : operation. First move what we can in the largest integer mode,
1502 : then go to successively smaller modes. For every access, call
1503 : GENFUN with the two operands and the EXTRA_DATA. */
1504 :
1505 : void
1506 749055 : op_by_pieces_d::run ()
1507 : {
1508 749055 : if (m_len == 0)
1509 : return;
1510 :
1511 743151 : unsigned HOST_WIDE_INT length = m_len;
1512 :
1513 : /* widest_fixed_size_mode_for_size checks M_MAX_SIZE > 1. */
1514 743151 : fixed_size_mode mode
1515 743151 : = widest_fixed_size_mode_for_size (m_max_size, m_op);
1516 743151 : mode = get_usable_mode (mode, length);
1517 :
1518 743151 : by_pieces_prev to_prev = { nullptr, mode };
1519 743151 : by_pieces_prev from_prev = { nullptr, mode };
1520 :
1521 1232892 : do
1522 : {
1523 1232892 : unsigned int size = GET_MODE_SIZE (mode);
1524 1232892 : rtx to1 = NULL_RTX, from1;
1525 :
1526 3276904 : while (length >= size)
1527 : {
1528 2044012 : if (m_reverse)
1529 47 : m_offset -= size;
1530 :
1531 2044012 : to1 = m_to.adjust (mode, m_offset, &to_prev);
1532 2044012 : to_prev.data = to1;
1533 2044012 : to_prev.mode = mode;
1534 2044012 : from1 = m_from.adjust (mode, m_offset, &from_prev);
1535 2044012 : from_prev.data = from1;
1536 2044012 : from_prev.mode = mode;
1537 :
1538 2044012 : m_to.maybe_predec (-(HOST_WIDE_INT)size);
1539 2044012 : m_from.maybe_predec (-(HOST_WIDE_INT)size);
1540 :
1541 2044012 : generate (to1, from1, mode);
1542 :
1543 2044012 : m_to.maybe_postinc (size);
1544 2044012 : m_from.maybe_postinc (size);
1545 :
1546 2044012 : if (!m_reverse)
1547 2043965 : m_offset += size;
1548 :
1549 2044012 : length -= size;
1550 : }
1551 :
1552 1232892 : finish_mode (mode);
1553 :
1554 1232892 : if (length == 0)
1555 : return;
1556 :
1557 489741 : if (!m_push && m_overlap_op_by_pieces)
1558 : {
1559 : /* NB: Generate overlapping operations if it is not a stack
1560 : push since stack push must not overlap. Get the smallest
1561 : fixed size mode for M_LEN bytes. */
1562 489741 : mode = smallest_fixed_size_mode_for_size (length);
1563 979482 : mode = get_usable_mode (mode, GET_MODE_SIZE (mode));
1564 489741 : int gap = GET_MODE_SIZE (mode) - length;
1565 489741 : if (gap > 0)
1566 : {
1567 : /* If size of MODE > M_LEN, generate the last operation
1568 : in MODE for the remaining bytes with ovelapping memory
1569 : from the previois operation. */
1570 47681 : if (m_reverse)
1571 0 : m_offset += gap;
1572 : else
1573 47681 : m_offset -= gap;
1574 47681 : length += gap;
1575 : }
1576 : }
1577 : else
1578 : {
1579 : /* widest_fixed_size_mode_for_size checks SIZE > 1. */
1580 0 : mode = widest_fixed_size_mode_for_size (size, m_op);
1581 0 : mode = get_usable_mode (mode, length);
1582 : }
1583 : }
1584 : while (1);
1585 : }
1586 :
1587 : /* Derived class from op_by_pieces_d, providing support for block move
1588 : operations. */
1589 :
1590 : #ifdef PUSH_ROUNDING
1591 : #define PUSHG_P(to) ((to) == nullptr)
1592 : #else
1593 : #define PUSHG_P(to) false
1594 : #endif
1595 :
1596 : class move_by_pieces_d : public op_by_pieces_d
1597 : {
1598 : insn_gen_fn m_gen_fun;
1599 : void generate (rtx, rtx, machine_mode) final override;
1600 : bool prepare_mode (machine_mode, unsigned int) final override;
1601 :
1602 : public:
1603 567582 : move_by_pieces_d (rtx to, rtx from, unsigned HOST_WIDE_INT len,
1604 : unsigned int align)
1605 567582 : : op_by_pieces_d (MOVE_MAX_PIECES, to, false, from, true, NULL,
1606 1135164 : NULL, len, align, PUSHG_P (to), MOVE_BY_PIECES)
1607 : {
1608 567582 : }
1609 : rtx finish_retmode (memop_ret);
1610 : };
1611 :
1612 : /* Return true if MODE can be used for a set of copies, given an
1613 : alignment ALIGN. Prepare whatever data is necessary for later
1614 : calls to generate. */
1615 :
1616 : bool
1617 947530 : move_by_pieces_d::prepare_mode (machine_mode mode, unsigned int align)
1618 : {
1619 947530 : insn_code icode = optab_handler (mov_optab, mode);
1620 947530 : m_gen_fun = GEN_FCN (icode);
1621 947530 : return icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode);
1622 : }
1623 :
1624 : /* A callback used when iterating for a compare_by_pieces_operation.
1625 : OP0 and OP1 are the values that have been loaded and should be
1626 : compared in MODE. If OP0 is NULL, this means we should generate a
1627 : push; otherwise EXTRA_DATA holds a pointer to a pointer to the insn
1628 : gen function that should be used to generate the mode. */
1629 :
1630 : void
1631 1644341 : move_by_pieces_d::generate (rtx op0, rtx op1,
1632 : machine_mode mode ATTRIBUTE_UNUSED)
1633 : {
1634 : #ifdef PUSH_ROUNDING
1635 1644341 : if (op0 == NULL_RTX)
1636 : {
1637 47 : emit_single_push_insn (mode, op1, NULL);
1638 47 : return;
1639 : }
1640 : #endif
1641 1644294 : emit_insn (m_gen_fun (op0, op1));
1642 : }
1643 :
1644 : /* Perform the final adjustment at the end of a string to obtain the
1645 : correct return value for the block operation.
1646 : Return value is based on RETMODE argument. */
1647 :
1648 : rtx
1649 0 : move_by_pieces_d::finish_retmode (memop_ret retmode)
1650 : {
1651 0 : gcc_assert (!m_reverse);
1652 0 : if (retmode == RETURN_END_MINUS_ONE)
1653 : {
1654 0 : m_to.maybe_postinc (-1);
1655 0 : --m_offset;
1656 : }
1657 0 : return m_to.adjust (QImode, m_offset);
1658 : }
1659 :
1660 : /* Generate several move instructions to copy LEN bytes from block FROM to
1661 : block TO. (These are MEM rtx's with BLKmode).
1662 :
1663 : If PUSH_ROUNDING is defined and TO is NULL, emit_single_push_insn is
1664 : used to push FROM to the stack.
1665 :
1666 : ALIGN is maximum stack alignment we can assume.
1667 :
1668 : Return value is based on RETMODE argument. */
1669 :
1670 : rtx
1671 567582 : move_by_pieces (rtx to, rtx from, unsigned HOST_WIDE_INT len,
1672 : unsigned int align, memop_ret retmode)
1673 : {
1674 : #ifndef PUSH_ROUNDING
1675 : if (to == NULL)
1676 : gcc_unreachable ();
1677 : #endif
1678 :
1679 567582 : move_by_pieces_d data (to, from, len, align);
1680 :
1681 567582 : data.run ();
1682 :
1683 567582 : if (retmode != RETURN_BEGIN)
1684 0 : return data.finish_retmode (retmode);
1685 : else
1686 : return to;
1687 : }
1688 :
1689 : /* Derived class from op_by_pieces_d, providing support for block move
1690 : operations. */
1691 :
1692 : class store_by_pieces_d : public op_by_pieces_d
1693 : {
1694 : insn_gen_fn m_gen_fun;
1695 :
1696 : void generate (rtx, rtx, machine_mode) final override;
1697 : bool prepare_mode (machine_mode, unsigned int) final override;
1698 :
1699 : public:
1700 119062 : store_by_pieces_d (rtx to, by_pieces_constfn cfn, void *cfn_data,
1701 : unsigned HOST_WIDE_INT len, unsigned int align,
1702 : by_pieces_operation op)
1703 119062 : : op_by_pieces_d (STORE_MAX_PIECES, to, false, NULL_RTX, true, cfn,
1704 238124 : cfn_data, len, align, false, op)
1705 : {
1706 119062 : }
1707 : rtx finish_retmode (memop_ret);
1708 : };
1709 :
1710 : /* Return true if MODE can be used for a set of stores, given an
1711 : alignment ALIGN. Prepare whatever data is necessary for later
1712 : calls to generate. */
1713 :
1714 : bool
1715 191399 : store_by_pieces_d::prepare_mode (machine_mode mode, unsigned int align)
1716 : {
1717 191399 : insn_code icode = optab_handler (mov_optab, mode);
1718 191399 : m_gen_fun = GEN_FCN (icode);
1719 191399 : return icode != CODE_FOR_nothing && align >= GET_MODE_ALIGNMENT (mode);
1720 : }
1721 :
1722 : /* A callback used when iterating for a store_by_pieces_operation.
1723 : OP0 and OP1 are the values that have been loaded and should be
1724 : compared in MODE. If OP0 is NULL, this means we should generate a
1725 : push; otherwise EXTRA_DATA holds a pointer to a pointer to the insn
1726 : gen function that should be used to generate the mode. */
1727 :
1728 : void
1729 302478 : store_by_pieces_d::generate (rtx op0, rtx op1, machine_mode)
1730 : {
1731 302478 : emit_insn (m_gen_fun (op0, op1));
1732 302478 : }
1733 :
1734 : /* Perform the final adjustment at the end of a string to obtain the
1735 : correct return value for the block operation.
1736 : Return value is based on RETMODE argument. */
1737 :
1738 : rtx
1739 652 : store_by_pieces_d::finish_retmode (memop_ret retmode)
1740 : {
1741 652 : gcc_assert (!m_reverse);
1742 652 : if (retmode == RETURN_END_MINUS_ONE)
1743 : {
1744 23 : m_to.maybe_postinc (-1);
1745 23 : --m_offset;
1746 : }
1747 652 : return m_to.adjust (QImode, m_offset);
1748 : }
1749 :
1750 : /* Determine whether the LEN bytes generated by CONSTFUN can be
1751 : stored to memory using several move instructions. CONSTFUNDATA is
1752 : a pointer which will be passed as argument in every CONSTFUN call.
1753 : ALIGN is maximum alignment we can assume. MEMSETP is true if this is
1754 : a memset operation and false if it's a copy of a constant string.
1755 : Return true if a call to store_by_pieces should succeed. */
1756 :
1757 : bool
1758 88193 : can_store_by_pieces (unsigned HOST_WIDE_INT len,
1759 : by_pieces_constfn constfun,
1760 : void *constfundata, unsigned int align, bool memsetp)
1761 : {
1762 88193 : unsigned HOST_WIDE_INT l;
1763 88193 : unsigned int max_size;
1764 88193 : HOST_WIDE_INT offset = 0;
1765 88193 : enum insn_code icode;
1766 88193 : int reverse;
1767 : /* cst is set but not used if LEGITIMATE_CONSTANT doesn't use it. */
1768 88193 : rtx cst ATTRIBUTE_UNUSED;
1769 :
1770 88193 : if (len == 0)
1771 : return true;
1772 :
1773 134893 : if (!targetm.use_by_pieces_infrastructure_p (len, align,
1774 : memsetp
1775 : ? SET_BY_PIECES
1776 : : STORE_BY_PIECES,
1777 88149 : optimize_insn_for_speed_p ()))
1778 : return false;
1779 :
1780 53567 : align = alignment_for_piecewise_move (STORE_MAX_PIECES, align);
1781 :
1782 : /* We would first store what we can in the largest integer mode, then go to
1783 : successively smaller modes. */
1784 :
1785 53567 : for (reverse = 0;
1786 107134 : reverse <= (HAVE_PRE_DECREMENT || HAVE_POST_DECREMENT);
1787 : reverse++)
1788 : {
1789 53567 : l = len;
1790 53567 : max_size = STORE_MAX_PIECES + 1;
1791 254951 : while (max_size > 1 && l > 0)
1792 : {
1793 201384 : auto op = memsetp ? SET_BY_PIECES : STORE_BY_PIECES;
1794 201384 : auto mode = widest_fixed_size_mode_for_size (max_size, op);
1795 :
1796 201384 : icode = optab_handler (mov_optab, mode);
1797 201384 : if (icode != CODE_FOR_nothing
1798 201384 : && align >= GET_MODE_ALIGNMENT (mode))
1799 : {
1800 201384 : unsigned int size = GET_MODE_SIZE (mode);
1801 :
1802 344016 : while (l >= size)
1803 : {
1804 142632 : if (reverse)
1805 : offset -= size;
1806 :
1807 142632 : cst = (*constfun) (constfundata, nullptr, offset, mode);
1808 : /* All CONST_VECTORs can be loaded for memset since
1809 : vec_duplicate_optab is a precondition to pick a
1810 : vector mode for the memset expander. */
1811 266783 : if (!((memsetp && VECTOR_MODE_P (mode))
1812 124151 : || targetm.legitimate_constant_p (mode, cst)))
1813 34582 : return false;
1814 :
1815 142632 : if (!reverse)
1816 142632 : offset += size;
1817 :
1818 142632 : l -= size;
1819 : }
1820 : }
1821 :
1822 402768 : max_size = GET_MODE_SIZE (mode);
1823 : }
1824 :
1825 : /* The code above should have handled everything. */
1826 53567 : gcc_assert (!l);
1827 : }
1828 :
1829 : return true;
1830 : }
1831 :
1832 : /* Generate several move instructions to store LEN bytes generated by
1833 : CONSTFUN to block TO. (A MEM rtx with BLKmode). CONSTFUNDATA is a
1834 : pointer which will be passed as argument in every CONSTFUN call.
1835 : ALIGN is maximum alignment we can assume. MEMSETP is true if this is
1836 : a memset operation and false if it's a copy of a constant string.
1837 : Return value is based on RETMODE argument. */
1838 :
1839 : rtx
1840 58555 : store_by_pieces (rtx to, unsigned HOST_WIDE_INT len,
1841 : by_pieces_constfn constfun,
1842 : void *constfundata, unsigned int align, bool memsetp,
1843 : memop_ret retmode)
1844 : {
1845 58555 : if (len == 0)
1846 : {
1847 5537 : gcc_assert (retmode != RETURN_END_MINUS_ONE);
1848 : return to;
1849 : }
1850 :
1851 95831 : gcc_assert (targetm.use_by_pieces_infrastructure_p
1852 : (len, align,
1853 : memsetp ? SET_BY_PIECES : STORE_BY_PIECES,
1854 : optimize_insn_for_speed_p ()));
1855 :
1856 53018 : store_by_pieces_d data (to, constfun, constfundata, len, align,
1857 53018 : memsetp ? SET_BY_PIECES : STORE_BY_PIECES);
1858 53018 : data.run ();
1859 :
1860 53018 : if (retmode != RETURN_BEGIN)
1861 652 : return data.finish_retmode (retmode);
1862 : else
1863 : return to;
1864 : }
1865 :
1866 : void
1867 66044 : clear_by_pieces (rtx to, unsigned HOST_WIDE_INT len, unsigned int align)
1868 : {
1869 66044 : if (len == 0)
1870 0 : return;
1871 :
1872 : /* Use builtin_memset_read_str to support vector mode broadcast. */
1873 66044 : char c = 0;
1874 66044 : store_by_pieces_d data (to, builtin_memset_read_str, &c, len, align,
1875 66044 : CLEAR_BY_PIECES);
1876 66044 : data.run ();
1877 : }
1878 :
1879 : /* Context used by compare_by_pieces_genfn. It stores the fail label
1880 : to jump to in case of miscomparison, and for branch ratios greater than 1,
1881 : it stores an accumulator and the current and maximum counts before
1882 : emitting another branch. */
1883 :
1884 : class compare_by_pieces_d : public op_by_pieces_d
1885 : {
1886 : rtx_code_label *m_fail_label;
1887 : rtx m_accumulator;
1888 : int m_count, m_batch;
1889 :
1890 : void generate (rtx, rtx, machine_mode) final override;
1891 : bool prepare_mode (machine_mode, unsigned int) final override;
1892 : void finish_mode (machine_mode) final override;
1893 :
1894 : public:
1895 62411 : compare_by_pieces_d (rtx op0, rtx op1, by_pieces_constfn op1_cfn,
1896 : void *op1_cfn_data, HOST_WIDE_INT len, int align,
1897 : rtx_code_label *fail_label)
1898 62411 : : op_by_pieces_d (COMPARE_MAX_PIECES, op0, true, op1, true, op1_cfn,
1899 124822 : op1_cfn_data, len, align, false, COMPARE_BY_PIECES)
1900 : {
1901 62411 : m_fail_label = fail_label;
1902 62411 : }
1903 : };
1904 :
1905 : /* A callback used when iterating for a compare_by_pieces_operation.
1906 : OP0 and OP1 are the values that have been loaded and should be
1907 : compared in MODE. DATA holds a pointer to the compare_by_pieces_data
1908 : context structure. */
1909 :
1910 : void
1911 97193 : compare_by_pieces_d::generate (rtx op0, rtx op1, machine_mode mode)
1912 : {
1913 97193 : if (m_batch > 1)
1914 : {
1915 0 : rtx temp = expand_binop (mode, sub_optab, op0, op1, NULL_RTX,
1916 : true, OPTAB_LIB_WIDEN);
1917 0 : if (m_count != 0)
1918 0 : temp = expand_binop (mode, ior_optab, m_accumulator, temp, temp,
1919 : true, OPTAB_LIB_WIDEN);
1920 0 : m_accumulator = temp;
1921 :
1922 0 : if (++m_count < m_batch)
1923 : return;
1924 :
1925 0 : m_count = 0;
1926 0 : op0 = m_accumulator;
1927 0 : op1 = const0_rtx;
1928 0 : m_accumulator = NULL_RTX;
1929 : }
1930 97193 : do_compare_rtx_and_jump (op0, op1, NE, true, mode, NULL_RTX, NULL,
1931 : m_fail_label, profile_probability::uninitialized ());
1932 : }
1933 :
1934 : /* Return true if MODE can be used for a set of moves and comparisons,
1935 : given an alignment ALIGN. Prepare whatever data is necessary for
1936 : later calls to generate. */
1937 :
1938 : bool
1939 93963 : compare_by_pieces_d::prepare_mode (machine_mode mode, unsigned int align)
1940 : {
1941 93963 : insn_code icode = optab_handler (mov_optab, mode);
1942 93963 : if (icode == CODE_FOR_nothing
1943 93963 : || align < GET_MODE_ALIGNMENT (mode)
1944 187926 : || !can_compare_p (EQ, mode, ccp_jump))
1945 0 : return false;
1946 93963 : m_batch = targetm.compare_by_pieces_branch_ratio (mode);
1947 93963 : if (m_batch < 0)
1948 : return false;
1949 93963 : m_accumulator = NULL_RTX;
1950 93963 : m_count = 0;
1951 93963 : return true;
1952 : }
1953 :
1954 : /* Called after expanding a series of comparisons in MODE. If we have
1955 : accumulated results for which we haven't emitted a branch yet, do
1956 : so now. */
1957 :
1958 : void
1959 93963 : compare_by_pieces_d::finish_mode (machine_mode mode)
1960 : {
1961 93963 : if (m_accumulator != NULL_RTX)
1962 0 : do_compare_rtx_and_jump (m_accumulator, const0_rtx, NE, true, mode,
1963 : NULL_RTX, NULL, m_fail_label,
1964 : profile_probability::uninitialized ());
1965 93963 : }
1966 :
1967 : /* Generate several move instructions to compare LEN bytes from blocks
1968 : ARG0 and ARG1. (These are MEM rtx's with BLKmode).
1969 :
1970 : If PUSH_ROUNDING is defined and TO is NULL, emit_single_push_insn is
1971 : used to push FROM to the stack.
1972 :
1973 : ALIGN is maximum stack alignment we can assume.
1974 :
1975 : Optionally, the caller can pass a constfn and associated data in A1_CFN
1976 : and A1_CFN_DATA. describing that the second operand being compared is a
1977 : known constant and how to obtain its data. */
1978 :
1979 : static rtx
1980 62411 : compare_by_pieces (rtx arg0, rtx arg1, unsigned HOST_WIDE_INT len,
1981 : rtx target, unsigned int align,
1982 : by_pieces_constfn a1_cfn, void *a1_cfn_data)
1983 : {
1984 62411 : rtx_code_label *fail_label = gen_label_rtx ();
1985 62411 : rtx_code_label *end_label = gen_label_rtx ();
1986 :
1987 62411 : if (target == NULL_RTX
1988 62411 : || !REG_P (target) || REGNO (target) < FIRST_PSEUDO_REGISTER)
1989 2 : target = gen_reg_rtx (TYPE_MODE (integer_type_node));
1990 :
1991 62411 : compare_by_pieces_d data (arg0, arg1, a1_cfn, a1_cfn_data, len, align,
1992 62411 : fail_label);
1993 :
1994 62411 : data.run ();
1995 :
1996 62411 : emit_move_insn (target, const0_rtx);
1997 62411 : emit_jump (end_label);
1998 62411 : emit_barrier ();
1999 62411 : emit_label (fail_label);
2000 62411 : emit_move_insn (target, const1_rtx);
2001 62411 : emit_label (end_label);
2002 :
2003 62411 : return target;
2004 : }
2005 :
2006 : /* Emit code to move a block Y to a block X. This may be done with
2007 : string-move instructions, with multiple scalar move instructions,
2008 : or with a library call.
2009 :
2010 : Both X and Y must be MEM rtx's (perhaps inside VOLATILE) with mode BLKmode.
2011 : SIZE is an rtx that says how long they are.
2012 : ALIGN is the maximum alignment we can assume they have.
2013 : METHOD describes what kind of copy this is, and what mechanisms may be used.
2014 : MIN_SIZE is the minimal size of block to move
2015 : MAX_SIZE is the maximal size of block to move, if it cannot be represented
2016 : in unsigned HOST_WIDE_INT, than it is mask of all ones.
2017 : CTZ_SIZE is the trailing-zeros count of SIZE; even a nonconstant SIZE is
2018 : known to be a multiple of 1<<CTZ_SIZE.
2019 :
2020 : Return the address of the new block, if memcpy is called and returns it,
2021 : 0 otherwise. */
2022 :
2023 : rtx
2024 665100 : emit_block_move_hints (rtx x, rtx y, rtx size, enum block_op_methods method,
2025 : unsigned int expected_align, HOST_WIDE_INT expected_size,
2026 : unsigned HOST_WIDE_INT min_size,
2027 : unsigned HOST_WIDE_INT max_size,
2028 : unsigned HOST_WIDE_INT probable_max_size,
2029 : bool bail_out_libcall, bool *is_move_done,
2030 : bool might_overlap, unsigned ctz_size)
2031 : {
2032 665100 : int may_use_call;
2033 665100 : rtx retval = 0;
2034 665100 : unsigned int align;
2035 :
2036 665100 : if (is_move_done)
2037 95287 : *is_move_done = true;
2038 :
2039 665100 : gcc_assert (size);
2040 665100 : if (CONST_INT_P (size) && INTVAL (size) == 0)
2041 : return 0;
2042 :
2043 664823 : switch (method)
2044 : {
2045 : case BLOCK_OP_NORMAL:
2046 : case BLOCK_OP_TAILCALL:
2047 : may_use_call = 1;
2048 : break;
2049 :
2050 268919 : case BLOCK_OP_CALL_PARM:
2051 268919 : may_use_call = block_move_libcall_safe_for_call_parm ();
2052 :
2053 : /* Make inhibit_defer_pop nonzero around the library call
2054 : to force it to pop the arguments right away. */
2055 268919 : NO_DEFER_POP;
2056 268919 : break;
2057 :
2058 354 : case BLOCK_OP_NO_LIBCALL:
2059 354 : may_use_call = 0;
2060 354 : break;
2061 :
2062 1074 : case BLOCK_OP_NO_LIBCALL_RET:
2063 1074 : may_use_call = -1;
2064 1074 : break;
2065 :
2066 0 : default:
2067 0 : gcc_unreachable ();
2068 : }
2069 :
2070 664823 : gcc_assert (MEM_P (x) && MEM_P (y));
2071 664908 : align = MIN (MEM_ALIGN (x), MEM_ALIGN (y));
2072 664823 : gcc_assert (align >= BITS_PER_UNIT);
2073 :
2074 : /* Make sure we've got BLKmode addresses; store_one_arg can decide that
2075 : block copy is more efficient for other large modes, e.g. DCmode. */
2076 664823 : x = adjust_address (x, BLKmode, 0);
2077 664823 : y = adjust_address (y, BLKmode, 0);
2078 :
2079 : /* If source and destination are the same, no need to copy anything. */
2080 664823 : if (rtx_equal_p (x, y)
2081 13 : && !MEM_VOLATILE_P (x)
2082 664836 : && !MEM_VOLATILE_P (y))
2083 : return 0;
2084 :
2085 : /* Set MEM_SIZE as appropriate for this block copy. The main place this
2086 : can be incorrect is coming from __builtin_memcpy. */
2087 664810 : poly_int64 const_size;
2088 664810 : if (poly_int_rtx_p (size, &const_size))
2089 : {
2090 587743 : x = shallow_copy_rtx (x);
2091 587743 : y = shallow_copy_rtx (y);
2092 587743 : set_mem_size (x, const_size);
2093 587743 : set_mem_size (y, const_size);
2094 : }
2095 :
2096 664810 : bool pieces_ok = CONST_INT_P (size)
2097 664810 : && can_move_by_pieces (INTVAL (size), align);
2098 664810 : bool pattern_ok = false;
2099 :
2100 664810 : if (!pieces_ok || might_overlap)
2101 : {
2102 103674 : pattern_ok
2103 103674 : = emit_block_move_via_pattern (x, y, size, align,
2104 : expected_align, expected_size,
2105 : min_size, max_size, probable_max_size,
2106 : might_overlap);
2107 103674 : if (!pattern_ok && might_overlap)
2108 : {
2109 : /* Do not try any of the other methods below as they are not safe
2110 : for overlapping moves. */
2111 11021 : *is_move_done = false;
2112 11021 : return retval;
2113 : }
2114 : }
2115 :
2116 92653 : bool dynamic_direction = false;
2117 92653 : if (!pattern_ok && !pieces_ok && may_use_call
2118 129996 : && (flag_inline_stringops & (might_overlap ? ILSOP_MEMMOVE : ILSOP_MEMCPY)))
2119 : {
2120 653789 : may_use_call = 0;
2121 653789 : dynamic_direction = might_overlap;
2122 : }
2123 :
2124 653789 : if (pattern_ok)
2125 : ;
2126 626134 : else if (pieces_ok)
2127 561136 : move_by_pieces (x, y, INTVAL (size), align, RETURN_BEGIN);
2128 64998 : else if (may_use_call && !might_overlap
2129 64950 : && ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (x))
2130 129948 : && ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (y)))
2131 : {
2132 64950 : if (bail_out_libcall)
2133 : {
2134 259 : if (is_move_done)
2135 259 : *is_move_done = false;
2136 259 : return retval;
2137 : }
2138 :
2139 64691 : if (may_use_call < 0)
2140 0 : return pc_rtx;
2141 :
2142 64691 : retval = emit_block_copy_via_libcall (x, y, size,
2143 : method == BLOCK_OP_TAILCALL);
2144 : }
2145 48 : else if (dynamic_direction)
2146 0 : emit_block_move_via_oriented_loop (x, y, size, align, ctz_size);
2147 48 : else if (might_overlap)
2148 0 : *is_move_done = false;
2149 : else
2150 48 : emit_block_move_via_sized_loop (x, y, size, align, ctz_size);
2151 :
2152 653530 : if (method == BLOCK_OP_CALL_PARM)
2153 268908 : OK_DEFER_POP;
2154 :
2155 : return retval;
2156 : }
2157 :
2158 : rtx
2159 569813 : emit_block_move (rtx x, rtx y, rtx size, enum block_op_methods method,
2160 : unsigned int ctz_size)
2161 : {
2162 569813 : unsigned HOST_WIDE_INT max, min = 0;
2163 569813 : if (GET_CODE (size) == CONST_INT)
2164 569593 : min = max = UINTVAL (size);
2165 : else
2166 220 : max = GET_MODE_MASK (GET_MODE (size));
2167 569813 : return emit_block_move_hints (x, y, size, method, 0, -1,
2168 : min, max, max,
2169 569813 : false, NULL, false, ctz_size);
2170 : }
2171 :
2172 : /* A subroutine of emit_block_move. Returns true if calling the
2173 : block move libcall will not clobber any parameters which may have
2174 : already been placed on the stack. */
2175 :
2176 : static bool
2177 268919 : block_move_libcall_safe_for_call_parm (void)
2178 : {
2179 268919 : tree fn;
2180 :
2181 : /* If arguments are pushed on the stack, then they're safe. */
2182 268919 : if (targetm.calls.push_argument (0))
2183 : return true;
2184 :
2185 : /* If registers go on the stack anyway, any argument is sure to clobber
2186 : an outgoing argument. */
2187 : #if defined (REG_PARM_STACK_SPACE)
2188 3 : fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
2189 : /* Avoid set but not used warning if *REG_PARM_STACK_SPACE doesn't
2190 : depend on its argument. */
2191 3 : (void) fn;
2192 3 : if (OUTGOING_REG_PARM_STACK_SPACE ((!fn ? NULL_TREE : TREE_TYPE (fn)))
2193 3 : && REG_PARM_STACK_SPACE (fn) != 0)
2194 : return false;
2195 : #endif
2196 :
2197 : /* If any argument goes in memory, then it might clobber an outgoing
2198 : argument. */
2199 3 : {
2200 3 : CUMULATIVE_ARGS args_so_far_v;
2201 3 : cumulative_args_t args_so_far;
2202 3 : tree arg;
2203 :
2204 3 : fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
2205 3 : INIT_CUMULATIVE_ARGS (args_so_far_v, TREE_TYPE (fn), NULL_RTX, 0, 3);
2206 3 : args_so_far = pack_cumulative_args (&args_so_far_v);
2207 :
2208 3 : arg = TYPE_ARG_TYPES (TREE_TYPE (fn));
2209 12 : for ( ; arg != void_list_node ; arg = TREE_CHAIN (arg))
2210 : {
2211 9 : machine_mode mode = TYPE_MODE (TREE_VALUE (arg));
2212 9 : function_arg_info arg_info (mode, /*named=*/true);
2213 9 : rtx tmp = targetm.calls.function_arg (args_so_far, arg_info);
2214 9 : if (!tmp || !REG_P (tmp))
2215 0 : return false;
2216 9 : if (targetm.calls.arg_partial_bytes (args_so_far, arg_info))
2217 : return false;
2218 9 : targetm.calls.function_arg_advance (args_so_far, arg_info);
2219 : }
2220 : }
2221 3 : return true;
2222 : }
2223 :
2224 : /* A subroutine of emit_block_move. Expand a cpymem or movmem pattern;
2225 : return true if successful.
2226 :
2227 : X is the destination of the copy or move.
2228 : Y is the source of the copy or move.
2229 : SIZE is the size of the block to be moved.
2230 :
2231 : MIGHT_OVERLAP indicates this originated with expansion of a
2232 : builtin_memmove() and the source and destination blocks may
2233 : overlap.
2234 : */
2235 :
2236 : static bool
2237 103674 : emit_block_move_via_pattern (rtx x, rtx y, rtx size, unsigned int align,
2238 : unsigned int expected_align,
2239 : HOST_WIDE_INT expected_size,
2240 : unsigned HOST_WIDE_INT min_size,
2241 : unsigned HOST_WIDE_INT max_size,
2242 : unsigned HOST_WIDE_INT probable_max_size,
2243 : bool might_overlap)
2244 : {
2245 103674 : if (expected_align < align)
2246 : expected_align = align;
2247 103674 : if (expected_size != -1)
2248 : {
2249 22 : if ((unsigned HOST_WIDE_INT)expected_size > probable_max_size)
2250 0 : expected_size = probable_max_size;
2251 22 : if ((unsigned HOST_WIDE_INT)expected_size < min_size)
2252 0 : expected_size = min_size;
2253 : }
2254 :
2255 : /* Since this is a move insn, we don't care about volatility. */
2256 103674 : temporary_volatile_ok v (true);
2257 :
2258 : /* Try the most limited insn first, because there's no point
2259 : including more than one in the machine description unless
2260 : the more limited one has some advantage. */
2261 :
2262 103674 : opt_scalar_int_mode mode_iter;
2263 695228 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
2264 : {
2265 619209 : scalar_int_mode mode = mode_iter.require ();
2266 619209 : enum insn_code code;
2267 619209 : if (might_overlap)
2268 94608 : code = direct_optab_handler (movmem_optab, mode);
2269 : else
2270 524601 : code = direct_optab_handler (cpymem_optab, mode);
2271 :
2272 619209 : if (code != CODE_FOR_nothing
2273 : /* We don't need MODE to be narrower than BITS_PER_HOST_WIDE_INT
2274 : here because if SIZE is less than the mode mask, as it is
2275 : returned by the macro, it will definitely be less than the
2276 : actual mode mask. Since SIZE is within the Pmode address
2277 : space, we limit MODE to Pmode. */
2278 619209 : && ((CONST_INT_P (size)
2279 41627 : && ((unsigned HOST_WIDE_INT) INTVAL (size)
2280 41627 : <= (GET_MODE_MASK (mode) >> 1)))
2281 137116 : || max_size <= (GET_MODE_MASK (mode) >> 1)
2282 162374 : || GET_MODE_BITSIZE (mode) >= GET_MODE_BITSIZE (Pmode)))
2283 : {
2284 120710 : class expand_operand ops[9];
2285 120710 : unsigned int nops;
2286 :
2287 : /* ??? When called via emit_block_move_for_call, it'd be
2288 : nice if there were some way to inform the backend, so
2289 : that it doesn't fail the expansion because it thinks
2290 : emitting the libcall would be more efficient. */
2291 120710 : nops = insn_data[(int) code].n_generator_args;
2292 120710 : gcc_assert (nops == 4 || nops == 6 || nops == 8 || nops == 9);
2293 :
2294 120710 : create_fixed_operand (&ops[0], x);
2295 120710 : create_fixed_operand (&ops[1], y);
2296 : /* The check above guarantees that this size conversion is valid. */
2297 120710 : create_convert_operand_to (&ops[2], size, mode, true);
2298 120710 : create_integer_operand (&ops[3], align / BITS_PER_UNIT);
2299 120710 : if (nops >= 6)
2300 : {
2301 120710 : create_integer_operand (&ops[4], expected_align / BITS_PER_UNIT);
2302 120710 : create_integer_operand (&ops[5], expected_size);
2303 : }
2304 120710 : if (nops >= 8)
2305 : {
2306 120710 : create_integer_operand (&ops[6], min_size);
2307 : /* If we cannot represent the maximal size,
2308 : make parameter NULL. */
2309 120710 : if ((HOST_WIDE_INT) max_size != -1)
2310 108195 : create_integer_operand (&ops[7], max_size);
2311 : else
2312 12515 : create_fixed_operand (&ops[7], NULL);
2313 : }
2314 120710 : if (nops == 9)
2315 : {
2316 : /* If we cannot represent the maximal size,
2317 : make parameter NULL. */
2318 120710 : if ((HOST_WIDE_INT) probable_max_size != -1)
2319 110774 : create_integer_operand (&ops[8], probable_max_size);
2320 : else
2321 9936 : create_fixed_operand (&ops[8], NULL);
2322 : }
2323 120710 : if (maybe_expand_insn (code, nops, ops))
2324 27655 : return true;
2325 : }
2326 : }
2327 :
2328 : return false;
2329 103674 : }
2330 :
2331 : /* Like emit_block_move_via_loop, but choose a suitable INCR based on
2332 : ALIGN and CTZ_SIZE. */
2333 :
2334 : static void
2335 48 : emit_block_move_via_sized_loop (rtx x, rtx y, rtx size,
2336 : unsigned int align,
2337 : unsigned int ctz_size)
2338 : {
2339 48 : int incr = align / BITS_PER_UNIT;
2340 :
2341 48 : if (CONST_INT_P (size))
2342 0 : ctz_size = MAX (ctz_size, (unsigned) wi::ctz (UINTVAL (size)));
2343 :
2344 48 : if (HOST_WIDE_INT_1U << ctz_size < (unsigned HOST_WIDE_INT) incr)
2345 0 : incr = HOST_WIDE_INT_1U << ctz_size;
2346 :
2347 48 : while (incr > 1 && !can_move_by_pieces (incr, align))
2348 0 : incr >>= 1;
2349 :
2350 48 : gcc_checking_assert (incr);
2351 :
2352 48 : return emit_block_move_via_loop (x, y, size, align, incr);
2353 : }
2354 :
2355 : /* Like emit_block_move_via_sized_loop, but besides choosing INCR so
2356 : as to ensure safe moves even in case of overlap, output dynamic
2357 : tests to choose between two loops, one moving downwards, another
2358 : moving upwards. */
2359 :
2360 : static void
2361 0 : emit_block_move_via_oriented_loop (rtx x, rtx y, rtx size,
2362 : unsigned int align,
2363 : unsigned int ctz_size)
2364 : {
2365 0 : int incr = align / BITS_PER_UNIT;
2366 :
2367 0 : if (CONST_INT_P (size))
2368 0 : ctz_size = MAX (ctz_size, (unsigned) wi::ctz (UINTVAL (size)));
2369 :
2370 0 : if (HOST_WIDE_INT_1U << ctz_size < (unsigned HOST_WIDE_INT) incr)
2371 0 : incr = HOST_WIDE_INT_1U << ctz_size;
2372 :
2373 0 : while (incr > 1 && !int_mode_for_size (incr, 0).exists ())
2374 0 : incr >>= 1;
2375 :
2376 0 : gcc_checking_assert (incr);
2377 :
2378 0 : rtx_code_label *upw_label, *end_label;
2379 0 : upw_label = gen_label_rtx ();
2380 0 : end_label = gen_label_rtx ();
2381 :
2382 0 : rtx x_addr = force_operand (XEXP (x, 0), NULL_RTX);
2383 0 : rtx y_addr = force_operand (XEXP (y, 0), NULL_RTX);
2384 0 : do_pending_stack_adjust ();
2385 :
2386 0 : machine_mode mode = GET_MODE (x_addr);
2387 0 : if (mode != GET_MODE (y_addr))
2388 : {
2389 0 : scalar_int_mode xmode
2390 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE (mode)).require ();
2391 0 : scalar_int_mode ymode
2392 0 : = smallest_int_mode_for_size (GET_MODE_BITSIZE
2393 0 : (GET_MODE (y_addr))).require ();
2394 0 : if (GET_MODE_BITSIZE (xmode) < GET_MODE_BITSIZE (ymode))
2395 : mode = ymode;
2396 : else
2397 0 : mode = xmode;
2398 :
2399 : #ifndef POINTERS_EXTEND_UNSIGNED
2400 : const int POINTERS_EXTEND_UNSIGNED = 1;
2401 : #endif
2402 0 : x_addr = convert_modes (mode, GET_MODE (x_addr), x_addr,
2403 : POINTERS_EXTEND_UNSIGNED);
2404 0 : y_addr = convert_modes (mode, GET_MODE (y_addr), y_addr,
2405 : POINTERS_EXTEND_UNSIGNED);
2406 : }
2407 :
2408 : /* Test for overlap: if (x >= y || x + size <= y) goto upw_label. */
2409 0 : emit_cmp_and_jump_insns (x_addr, y_addr, GEU, NULL_RTX, mode,
2410 : true, upw_label,
2411 0 : profile_probability::guessed_always ()
2412 : .apply_scale (5, 10));
2413 0 : rtx tmp = convert_modes (GET_MODE (x_addr), GET_MODE (size), size, true);
2414 0 : tmp = simplify_gen_binary (PLUS, GET_MODE (x_addr), x_addr, tmp);
2415 :
2416 0 : emit_cmp_and_jump_insns (tmp, y_addr, LEU, NULL_RTX, mode,
2417 : true, upw_label,
2418 0 : profile_probability::guessed_always ()
2419 : .apply_scale (8, 10));
2420 :
2421 0 : emit_block_move_via_loop (x, y, size, align, -incr);
2422 :
2423 0 : emit_jump (end_label);
2424 0 : emit_label (upw_label);
2425 :
2426 0 : emit_block_move_via_loop (x, y, size, align, incr);
2427 :
2428 0 : emit_label (end_label);
2429 0 : }
2430 :
2431 : /* A subroutine of emit_block_move. Copy the data via an explicit
2432 : loop. This is used only when libcalls are forbidden, or when
2433 : inlining is required. INCR is the block size to be copied in each
2434 : loop iteration. If it is negative, the absolute value is used, and
2435 : the block is copied backwards. INCR must be a power of two, an
2436 : exact divisor for SIZE and ALIGN, and imply a mode that can be
2437 : safely copied per iteration assuming no overlap. */
2438 :
2439 : static void
2440 48 : emit_block_move_via_loop (rtx x, rtx y, rtx size,
2441 : unsigned int align, int incr)
2442 : {
2443 48 : rtx_code_label *cmp_label, *top_label;
2444 48 : rtx iter, x_addr, y_addr, tmp;
2445 48 : machine_mode x_addr_mode = get_address_mode (x);
2446 48 : machine_mode y_addr_mode = get_address_mode (y);
2447 48 : machine_mode iter_mode;
2448 :
2449 48 : iter_mode = GET_MODE (size);
2450 48 : if (iter_mode == VOIDmode)
2451 0 : iter_mode = word_mode;
2452 :
2453 48 : top_label = gen_label_rtx ();
2454 48 : cmp_label = gen_label_rtx ();
2455 48 : iter = gen_reg_rtx (iter_mode);
2456 :
2457 48 : bool downwards = incr < 0;
2458 48 : rtx iter_init;
2459 48 : rtx_code iter_cond;
2460 48 : rtx iter_limit;
2461 48 : rtx iter_incr;
2462 48 : machine_mode move_mode;
2463 48 : if (downwards)
2464 : {
2465 0 : incr = -incr;
2466 0 : iter_init = size;
2467 0 : iter_cond = GEU;
2468 0 : iter_limit = const0_rtx;
2469 0 : iter_incr = GEN_INT (incr);
2470 : }
2471 : else
2472 : {
2473 48 : iter_init = const0_rtx;
2474 48 : iter_cond = LTU;
2475 48 : iter_limit = size;
2476 48 : iter_incr = GEN_INT (incr);
2477 : }
2478 48 : emit_move_insn (iter, iter_init);
2479 :
2480 48 : opt_scalar_int_mode int_move_mode
2481 48 : = int_mode_for_size (incr * BITS_PER_UNIT, 1);
2482 48 : if (!int_move_mode.exists (&move_mode)
2483 96 : || GET_MODE_BITSIZE (int_move_mode.require ()) != incr * BITS_PER_UNIT)
2484 : {
2485 0 : move_mode = BLKmode;
2486 0 : gcc_checking_assert (can_move_by_pieces (incr, align));
2487 : }
2488 :
2489 48 : x_addr = force_operand (XEXP (x, 0), NULL_RTX);
2490 48 : y_addr = force_operand (XEXP (y, 0), NULL_RTX);
2491 48 : do_pending_stack_adjust ();
2492 :
2493 48 : emit_jump (cmp_label);
2494 48 : emit_label (top_label);
2495 :
2496 48 : tmp = convert_modes (x_addr_mode, iter_mode, iter, true);
2497 48 : x_addr = simplify_gen_binary (PLUS, x_addr_mode, x_addr, tmp);
2498 :
2499 48 : if (x_addr_mode != y_addr_mode)
2500 0 : tmp = convert_modes (y_addr_mode, iter_mode, iter, true);
2501 48 : y_addr = simplify_gen_binary (PLUS, y_addr_mode, y_addr, tmp);
2502 :
2503 48 : x = change_address (x, move_mode, x_addr);
2504 48 : y = change_address (y, move_mode, y_addr);
2505 :
2506 48 : if (move_mode == BLKmode)
2507 : {
2508 0 : bool done;
2509 0 : emit_block_move_hints (x, y, iter_incr, BLOCK_OP_NO_LIBCALL,
2510 : align, incr, incr, incr, incr,
2511 : false, &done, false);
2512 0 : gcc_checking_assert (done);
2513 : }
2514 : else
2515 48 : emit_move_insn (x, y);
2516 :
2517 48 : if (downwards)
2518 0 : emit_label (cmp_label);
2519 :
2520 48 : tmp = expand_simple_binop (iter_mode, PLUS, iter, iter_incr, iter,
2521 : true, OPTAB_LIB_WIDEN);
2522 48 : if (tmp != iter)
2523 0 : emit_move_insn (iter, tmp);
2524 :
2525 48 : if (!downwards)
2526 48 : emit_label (cmp_label);
2527 :
2528 48 : emit_cmp_and_jump_insns (iter, iter_limit, iter_cond, NULL_RTX, iter_mode,
2529 : true, top_label,
2530 48 : profile_probability::guessed_always ()
2531 : .apply_scale (9, 10));
2532 48 : }
2533 :
2534 : /* Expand a call to memcpy or memmove or memcmp, and return the result.
2535 : TAILCALL is true if this is a tail call. */
2536 :
2537 : rtx
2538 64694 : emit_block_op_via_libcall (enum built_in_function fncode, rtx dst, rtx src,
2539 : rtx size, bool tailcall)
2540 : {
2541 64694 : rtx dst_addr, src_addr;
2542 64694 : tree call_expr, dst_tree, src_tree, size_tree;
2543 64694 : machine_mode size_mode;
2544 :
2545 : /* Since dst and src are passed to a libcall, mark the corresponding
2546 : tree EXPR as addressable. */
2547 64694 : tree dst_expr = MEM_EXPR (dst);
2548 64694 : tree src_expr = MEM_EXPR (src);
2549 64694 : if (dst_expr)
2550 57977 : mark_addressable (dst_expr);
2551 64694 : if (src_expr)
2552 63289 : mark_addressable (src_expr);
2553 :
2554 64694 : dst_addr = copy_addr_to_reg (XEXP (dst, 0));
2555 64694 : dst_addr = convert_memory_address (ptr_mode, dst_addr);
2556 64694 : dst_tree = make_tree (ptr_type_node, dst_addr);
2557 :
2558 64694 : src_addr = copy_addr_to_reg (XEXP (src, 0));
2559 64694 : src_addr = convert_memory_address (ptr_mode, src_addr);
2560 64694 : src_tree = make_tree (ptr_type_node, src_addr);
2561 :
2562 64694 : size_mode = TYPE_MODE (sizetype);
2563 64694 : size = convert_to_mode (size_mode, size, 1);
2564 64694 : size = copy_to_mode_reg (size_mode, size);
2565 64694 : size_tree = make_tree (sizetype, size);
2566 :
2567 : /* It is incorrect to use the libcall calling conventions for calls to
2568 : memcpy/memmove/memcmp because they can be provided by the user. */
2569 64694 : tree fn = builtin_decl_implicit (fncode);
2570 64694 : call_expr = build_call_expr (fn, 3, dst_tree, src_tree, size_tree);
2571 64694 : CALL_EXPR_TAILCALL (call_expr) = tailcall;
2572 :
2573 64694 : return expand_call (call_expr, NULL_RTX, false);
2574 : }
2575 :
2576 : /* Try to expand cmpstrn or cmpmem operation ICODE with the given operands.
2577 : ARG3_TYPE is the type of ARG3_RTX. Return the result rtx on success,
2578 : otherwise return null. */
2579 :
2580 : rtx
2581 170798 : expand_cmpstrn_or_cmpmem (insn_code icode, rtx target, rtx arg1_rtx,
2582 : rtx arg2_rtx, tree arg3_type, rtx arg3_rtx,
2583 : HOST_WIDE_INT align)
2584 : {
2585 170798 : machine_mode insn_mode = insn_data[icode].operand[0].mode;
2586 :
2587 170798 : if (target && (!REG_P (target) || HARD_REGISTER_P (target)))
2588 : target = NULL_RTX;
2589 :
2590 170798 : class expand_operand ops[5];
2591 170798 : create_output_operand (&ops[0], target, insn_mode);
2592 170798 : create_fixed_operand (&ops[1], arg1_rtx);
2593 170798 : create_fixed_operand (&ops[2], arg2_rtx);
2594 170798 : create_convert_operand_from (&ops[3], arg3_rtx, TYPE_MODE (arg3_type),
2595 170798 : TYPE_UNSIGNED (arg3_type));
2596 170798 : create_integer_operand (&ops[4], align);
2597 170798 : if (maybe_expand_insn (icode, 5, ops))
2598 5796 : return ops[0].value;
2599 : return NULL_RTX;
2600 : }
2601 :
2602 : /* Expand a block compare between X and Y with length LEN using the
2603 : cmpmem optab, placing the result in TARGET. LEN_TYPE is the type
2604 : of the expression that was used to calculate the length. ALIGN
2605 : gives the known minimum common alignment. */
2606 :
2607 : static rtx
2608 42861 : emit_block_cmp_via_cmpmem (rtx x, rtx y, rtx len, tree len_type, rtx target,
2609 : unsigned align)
2610 : {
2611 : /* Note: The cmpstrnsi pattern, if it exists, is not suitable for
2612 : implementing memcmp because it will stop if it encounters two
2613 : zero bytes. */
2614 42861 : insn_code icode = direct_optab_handler (cmpmem_optab, SImode);
2615 :
2616 42861 : if (icode == CODE_FOR_nothing)
2617 : return NULL_RTX;
2618 :
2619 42861 : return expand_cmpstrn_or_cmpmem (icode, target, x, y, len_type, len, align);
2620 : }
2621 :
2622 : /* Emit code to compare a block Y to a block X. This may be done with
2623 : string-compare instructions, with multiple scalar instructions,
2624 : or with a library call.
2625 :
2626 : Both X and Y must be MEM rtx's. LEN is an rtx that says how long
2627 : they are. LEN_TYPE is the type of the expression that was used to
2628 : calculate it, and CTZ_LEN is the known trailing-zeros count of LEN,
2629 : so LEN must be a multiple of 1<<CTZ_LEN even if it's not constant.
2630 :
2631 : If EQUALITY_ONLY is true, it means we don't have to return the tri-state
2632 : value of a normal memcmp call, instead we can just compare for equality.
2633 : If FORCE_LIBCALL is true, we should emit a call to memcmp rather than
2634 : returning NULL_RTX.
2635 :
2636 : Optionally, the caller can pass a constfn and associated data in Y_CFN
2637 : and Y_CFN_DATA. describing that the second operand being compared is a
2638 : known constant and how to obtain its data.
2639 : Return the result of the comparison, or NULL_RTX if we failed to
2640 : perform the operation. */
2641 :
2642 : rtx
2643 105271 : emit_block_cmp_hints (rtx x, rtx y, rtx len, tree len_type, rtx target,
2644 : bool equality_only, by_pieces_constfn y_cfn,
2645 : void *y_cfndata, unsigned ctz_len)
2646 : {
2647 105271 : rtx result = 0;
2648 :
2649 105271 : if (CONST_INT_P (len) && INTVAL (len) == 0)
2650 0 : return const0_rtx;
2651 :
2652 105271 : gcc_assert (MEM_P (x) && MEM_P (y));
2653 105271 : unsigned int align = MIN (MEM_ALIGN (x), MEM_ALIGN (y));
2654 105271 : gcc_assert (align >= BITS_PER_UNIT);
2655 :
2656 105271 : x = adjust_address (x, BLKmode, 0);
2657 105271 : y = adjust_address (y, BLKmode, 0);
2658 :
2659 105271 : if (equality_only
2660 90598 : && CONST_INT_P (len)
2661 174373 : && can_do_by_pieces (INTVAL (len), align, COMPARE_BY_PIECES))
2662 62410 : result = compare_by_pieces (x, y, INTVAL (len), target, align,
2663 : y_cfn, y_cfndata);
2664 : else
2665 42861 : result = emit_block_cmp_via_cmpmem (x, y, len, len_type, target, align);
2666 :
2667 105271 : if (!result && (flag_inline_stringops & ILSOP_MEMCMP))
2668 361 : result = emit_block_cmp_via_loop (x, y, len, len_type,
2669 : target, equality_only,
2670 : align, ctz_len);
2671 :
2672 : return result;
2673 : }
2674 :
2675 : /* Like emit_block_cmp_hints, but with known alignment and no support
2676 : for constats. Always expand to a loop with iterations that compare
2677 : blocks of the largest compare-by-pieces size that divides both len
2678 : and align, and then, if !EQUALITY_ONLY, identify the word and then
2679 : the unit that first differs to return the result. */
2680 :
2681 : rtx
2682 403 : emit_block_cmp_via_loop (rtx x, rtx y, rtx len, tree len_type, rtx target,
2683 : bool equality_only, unsigned align, unsigned ctz_len)
2684 : {
2685 403 : unsigned incr = align / BITS_PER_UNIT;
2686 :
2687 403 : if (CONST_INT_P (len))
2688 319 : ctz_len = MAX (ctz_len, (unsigned) wi::ctz (UINTVAL (len)));
2689 :
2690 403 : if (HOST_WIDE_INT_1U << ctz_len < (unsigned HOST_WIDE_INT) incr)
2691 148 : incr = HOST_WIDE_INT_1U << ctz_len;
2692 :
2693 : while (incr > 1
2694 407 : && !can_do_by_pieces (incr, align, COMPARE_BY_PIECES))
2695 4 : incr >>= 1;
2696 :
2697 403 : rtx_code_label *cmp_label, *top_label, *ne_label, *res_label;
2698 403 : rtx iter, x_addr, y_addr, tmp;
2699 403 : machine_mode x_addr_mode = get_address_mode (x);
2700 403 : machine_mode y_addr_mode = get_address_mode (y);
2701 403 : machine_mode iter_mode;
2702 :
2703 403 : iter_mode = GET_MODE (len);
2704 403 : if (iter_mode == VOIDmode)
2705 319 : iter_mode = word_mode;
2706 :
2707 403 : rtx iter_init = const0_rtx;
2708 403 : rtx_code iter_cond = LTU;
2709 403 : rtx_code entry_cond = GEU;
2710 403 : rtx iter_limit = len;
2711 403 : rtx iter_incr = GEN_INT (incr);
2712 403 : machine_mode cmp_mode;
2713 :
2714 : /* We can drop the loop back edge if we know there's exactly one
2715 : iteration. */
2716 403 : top_label = (!rtx_equal_p (len, iter_incr)
2717 403 : ? gen_label_rtx ()
2718 : : NULL);
2719 : /* We need not test before entering the loop if len is known
2720 : nonzero. ??? This could be even stricter, testing whether a
2721 : nonconstant LEN could possibly be zero. */
2722 403 : cmp_label = (!CONSTANT_P (len) || rtx_equal_p (len, iter_init)
2723 403 : ? gen_label_rtx ()
2724 : : NULL);
2725 403 : ne_label = gen_label_rtx ();
2726 403 : res_label = gen_label_rtx ();
2727 :
2728 403 : iter = gen_reg_rtx (iter_mode);
2729 403 : emit_move_insn (iter, iter_init);
2730 :
2731 403 : opt_scalar_int_mode int_cmp_mode
2732 403 : = int_mode_for_size (incr * BITS_PER_UNIT, 1);
2733 403 : if (!int_cmp_mode.exists (&cmp_mode)
2734 1206 : || GET_MODE_BITSIZE (int_cmp_mode.require ()) != incr * BITS_PER_UNIT
2735 402 : || !can_compare_p (NE, cmp_mode, ccp_jump))
2736 : {
2737 1 : cmp_mode = BLKmode;
2738 1 : gcc_checking_assert (incr != 1);
2739 : }
2740 :
2741 : /* Save the base addresses. */
2742 403 : x_addr = force_operand (XEXP (x, 0), NULL_RTX);
2743 403 : y_addr = force_operand (XEXP (y, 0), NULL_RTX);
2744 403 : do_pending_stack_adjust ();
2745 :
2746 403 : if (cmp_label)
2747 : {
2748 84 : if (top_label)
2749 84 : emit_jump (cmp_label);
2750 : else
2751 0 : emit_cmp_and_jump_insns (iter, iter_limit, entry_cond,
2752 : NULL_RTX, iter_mode,
2753 : true, cmp_label,
2754 0 : profile_probability::guessed_always ()
2755 : .apply_scale (1, 10));
2756 : }
2757 403 : if (top_label)
2758 386 : emit_label (top_label);
2759 :
2760 : /* Offset the base addresses by ITER. */
2761 403 : tmp = convert_modes (x_addr_mode, iter_mode, iter, true);
2762 403 : x_addr = simplify_gen_binary (PLUS, x_addr_mode, x_addr, tmp);
2763 :
2764 403 : if (x_addr_mode != y_addr_mode)
2765 0 : tmp = convert_modes (y_addr_mode, iter_mode, iter, true);
2766 403 : y_addr = simplify_gen_binary (PLUS, y_addr_mode, y_addr, tmp);
2767 :
2768 403 : x = change_address (x, cmp_mode, x_addr);
2769 403 : y = change_address (y, cmp_mode, y_addr);
2770 :
2771 : /* Compare one block. */
2772 403 : rtx part_res;
2773 403 : if (cmp_mode == BLKmode)
2774 1 : part_res = compare_by_pieces (x, y, incr, target, align, 0, 0);
2775 : else
2776 402 : part_res = expand_binop (cmp_mode, sub_optab, x, y, NULL_RTX,
2777 : true, OPTAB_LIB_WIDEN);
2778 :
2779 : /* Stop if we found a difference. */
2780 806 : emit_cmp_and_jump_insns (part_res, GEN_INT (0), NE, NULL_RTX,
2781 403 : GET_MODE (part_res), true, ne_label,
2782 403 : profile_probability::guessed_always ()
2783 : .apply_scale (1, 10));
2784 :
2785 : /* Increment ITER. */
2786 403 : tmp = expand_simple_binop (iter_mode, PLUS, iter, iter_incr, iter,
2787 : true, OPTAB_LIB_WIDEN);
2788 403 : if (tmp != iter)
2789 0 : emit_move_insn (iter, tmp);
2790 :
2791 403 : if (cmp_label)
2792 84 : emit_label (cmp_label);
2793 : /* Loop until we reach the limit. */
2794 :
2795 403 : if (top_label)
2796 386 : emit_cmp_and_jump_insns (iter, iter_limit, iter_cond, NULL_RTX, iter_mode,
2797 : true, top_label,
2798 772 : profile_probability::guessed_always ()
2799 : .apply_scale (9, 10));
2800 :
2801 : /* We got to the end without differences, so the result is zero. */
2802 403 : if (target == NULL_RTX
2803 403 : || !REG_P (target) || REGNO (target) < FIRST_PSEUDO_REGISTER)
2804 49 : target = gen_reg_rtx (TYPE_MODE (integer_type_node));
2805 :
2806 403 : emit_move_insn (target, const0_rtx);
2807 403 : emit_jump (res_label);
2808 :
2809 403 : emit_label (ne_label);
2810 :
2811 : /* Return nonzero, or pinpoint the difference to return the expected
2812 : result for non-equality tests. */
2813 403 : if (equality_only)
2814 0 : emit_move_insn (target, const1_rtx);
2815 : else
2816 : {
2817 403 : if (incr > UNITS_PER_WORD)
2818 : /* ??? Re-compare the block found to be different one word at a
2819 : time. */
2820 5 : part_res = emit_block_cmp_via_loop (x, y, GEN_INT (incr), len_type,
2821 : target, equality_only,
2822 : BITS_PER_WORD, 0);
2823 398 : else if (incr > 1)
2824 : /* ??? Re-compare the block found to be different one byte at a
2825 : time. We could do better using part_res, and being careful
2826 : about endianness. */
2827 37 : part_res = emit_block_cmp_via_loop (x, y, GEN_INT (incr), len_type,
2828 : target, equality_only,
2829 : BITS_PER_UNIT, 0);
2830 1083 : else if (known_gt (GET_MODE_BITSIZE (GET_MODE (target)),
2831 : GET_MODE_BITSIZE (cmp_mode)))
2832 361 : part_res = expand_binop (GET_MODE (target), sub_optab, x, y, target,
2833 : true, OPTAB_LIB_WIDEN);
2834 : else
2835 : {
2836 : /* In the odd chance target is QImode, we can't count on
2837 : widening subtract to capture the result of the unsigned
2838 : compares. */
2839 0 : rtx_code_label *ltu_label;
2840 0 : ltu_label = gen_label_rtx ();
2841 0 : emit_cmp_and_jump_insns (x, y, LTU, NULL_RTX,
2842 : cmp_mode, true, ltu_label,
2843 0 : profile_probability::guessed_always ()
2844 : .apply_scale (5, 10));
2845 :
2846 0 : emit_move_insn (target, const1_rtx);
2847 0 : emit_jump (res_label);
2848 :
2849 0 : emit_label (ltu_label);
2850 0 : emit_move_insn (target, constm1_rtx);
2851 0 : part_res = target;
2852 : }
2853 :
2854 403 : if (target != part_res)
2855 0 : convert_move (target, part_res, false);
2856 : }
2857 :
2858 403 : emit_label (res_label);
2859 :
2860 403 : return target;
2861 : }
2862 :
2863 :
2864 : /* Copy all or part of a value X into registers starting at REGNO.
2865 : The number of registers to be filled is NREGS. */
2866 :
2867 : void
2868 434 : move_block_to_reg (int regno, rtx x, int nregs, machine_mode mode)
2869 : {
2870 434 : if (nregs == 0)
2871 : return;
2872 :
2873 420 : if (CONSTANT_P (x) && !targetm.legitimate_constant_p (mode, x))
2874 0 : x = validize_mem (force_const_mem (mode, x));
2875 :
2876 : /* See if the machine can do this with a load multiple insn. */
2877 420 : if (targetm.have_load_multiple ())
2878 : {
2879 0 : rtx_insn *last = get_last_insn ();
2880 0 : rtx first = gen_rtx_REG (word_mode, regno);
2881 0 : if (rtx_insn *pat = targetm.gen_load_multiple (first, x,
2882 : GEN_INT (nregs)))
2883 : {
2884 0 : emit_insn (pat);
2885 0 : return;
2886 : }
2887 : else
2888 0 : delete_insns_since (last);
2889 : }
2890 :
2891 848 : for (int i = 0; i < nregs; i++)
2892 428 : emit_move_insn (gen_rtx_REG (word_mode, regno + i),
2893 428 : operand_subword_force (x, i, mode));
2894 : }
2895 :
2896 : /* Copy all or part of a BLKmode value X out of registers starting at REGNO.
2897 : The number of registers to be filled is NREGS. */
2898 :
2899 : void
2900 1216 : move_block_from_reg (int regno, rtx x, int nregs)
2901 : {
2902 1216 : if (nregs == 0)
2903 : return;
2904 :
2905 : /* See if the machine can do this with a store multiple insn. */
2906 1216 : if (targetm.have_store_multiple ())
2907 : {
2908 0 : rtx_insn *last = get_last_insn ();
2909 0 : rtx first = gen_rtx_REG (word_mode, regno);
2910 0 : if (rtx_insn *pat = targetm.gen_store_multiple (x, first,
2911 : GEN_INT (nregs)))
2912 : {
2913 0 : emit_insn (pat);
2914 0 : return;
2915 : }
2916 : else
2917 0 : delete_insns_since (last);
2918 : }
2919 :
2920 2436 : for (int i = 0; i < nregs; i++)
2921 : {
2922 1220 : rtx tem = operand_subword (x, i, 1, BLKmode);
2923 :
2924 1220 : gcc_assert (tem);
2925 :
2926 1220 : emit_move_insn (tem, gen_rtx_REG (word_mode, regno + i));
2927 : }
2928 : }
2929 :
2930 : /* Generate a PARALLEL rtx for a new non-consecutive group of registers from
2931 : ORIG, where ORIG is a non-consecutive group of registers represented by
2932 : a PARALLEL. The clone is identical to the original except in that the
2933 : original set of registers is replaced by a new set of pseudo registers.
2934 : The new set has the same modes as the original set. */
2935 :
2936 : rtx
2937 3039 : gen_group_rtx (rtx orig)
2938 : {
2939 3039 : int i, length;
2940 3039 : rtx *tmps;
2941 :
2942 3039 : gcc_assert (GET_CODE (orig) == PARALLEL);
2943 :
2944 3039 : length = XVECLEN (orig, 0);
2945 3039 : tmps = XALLOCAVEC (rtx, length);
2946 :
2947 : /* Skip a NULL entry in first slot. */
2948 3039 : i = XEXP (XVECEXP (orig, 0, 0), 0) ? 0 : 1;
2949 :
2950 3039 : if (i)
2951 0 : tmps[0] = 0;
2952 :
2953 7385 : for (; i < length; i++)
2954 : {
2955 4346 : machine_mode mode = GET_MODE (XEXP (XVECEXP (orig, 0, i), 0));
2956 4346 : rtx offset = XEXP (XVECEXP (orig, 0, i), 1);
2957 :
2958 4346 : tmps[i] = gen_rtx_EXPR_LIST (VOIDmode, gen_reg_rtx (mode), offset);
2959 : }
2960 :
2961 3039 : return gen_rtx_PARALLEL (GET_MODE (orig), gen_rtvec_v (length, tmps));
2962 : }
2963 :
2964 : /* A subroutine of emit_group_load. Arguments as for emit_group_load,
2965 : except that values are placed in TMPS[i], and must later be moved
2966 : into corresponding XEXP (XVECEXP (DST, 0, i), 0) element. */
2967 :
2968 : static void
2969 293904 : emit_group_load_1 (rtx *tmps, rtx dst, rtx orig_src, tree type,
2970 : poly_int64 ssize)
2971 : {
2972 293907 : rtx src;
2973 293907 : int start, i;
2974 293907 : machine_mode m = GET_MODE (orig_src);
2975 :
2976 293907 : gcc_assert (GET_CODE (dst) == PARALLEL);
2977 :
2978 293907 : if (m != VOIDmode
2979 275108 : && !SCALAR_INT_MODE_P (m)
2980 19124 : && !MEM_P (orig_src)
2981 5720 : && GET_CODE (orig_src) != CONCAT)
2982 : {
2983 3 : scalar_int_mode imode;
2984 3 : if (int_mode_for_mode (GET_MODE (orig_src)).exists (&imode))
2985 : {
2986 3 : src = gen_reg_rtx (imode);
2987 3 : emit_move_insn (gen_lowpart (GET_MODE (orig_src), src), orig_src);
2988 : }
2989 : else
2990 : {
2991 0 : src = assign_stack_temp (GET_MODE (orig_src), ssize);
2992 0 : emit_move_insn (src, orig_src);
2993 : }
2994 3 : emit_group_load_1 (tmps, dst, src, type, ssize);
2995 : return;
2996 : }
2997 :
2998 : /* Check for a NULL entry, used to indicate that the parameter goes
2999 : both on the stack and in registers. */
3000 293904 : if (XEXP (XVECEXP (dst, 0, 0), 0))
3001 : start = 0;
3002 : else
3003 0 : start = 1;
3004 :
3005 : /* Process the pieces. */
3006 870237 : for (i = start; i < XVECLEN (dst, 0); i++)
3007 : {
3008 576333 : machine_mode mode = GET_MODE (XEXP (XVECEXP (dst, 0, i), 0));
3009 576333 : poly_int64 bytepos = rtx_to_poly_int64 (XEXP (XVECEXP (dst, 0, i), 1));
3010 1152666 : poly_int64 bytelen = GET_MODE_SIZE (mode);
3011 576333 : poly_int64 shift = 0;
3012 :
3013 : /* Handle trailing fragments that run over the size of the struct.
3014 : It's the target's responsibility to make sure that the fragment
3015 : cannot be strictly smaller in some cases and strictly larger
3016 : in others. */
3017 576333 : gcc_checking_assert (ordered_p (bytepos + bytelen, ssize));
3018 576333 : if (known_size_p (ssize) && maybe_gt (bytepos + bytelen, ssize))
3019 : {
3020 : /* Arrange to shift the fragment to where it belongs.
3021 : extract_bit_field loads to the lsb of the reg. */
3022 : if (
3023 : #ifdef BLOCK_REG_PADDING
3024 : BLOCK_REG_PADDING (GET_MODE (orig_src), type, i == start)
3025 : == (BYTES_BIG_ENDIAN ? PAD_UPWARD : PAD_DOWNWARD)
3026 : #else
3027 : BYTES_BIG_ENDIAN
3028 : #endif
3029 : )
3030 : shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
3031 1822 : bytelen = ssize - bytepos;
3032 1822 : gcc_assert (maybe_gt (bytelen, 0));
3033 : }
3034 :
3035 : /* If we won't be loading directly from memory, protect the real source
3036 : from strange tricks we might play; but make sure that the source can
3037 : be loaded directly into the destination. */
3038 576333 : src = orig_src;
3039 576333 : if (!MEM_P (orig_src)
3040 477087 : && (!REG_P (orig_src) || HARD_REGISTER_P (orig_src))
3041 46277 : && GET_CODE (orig_src) != CONCAT
3042 613931 : && !CONSTANT_P (orig_src))
3043 : {
3044 0 : gcc_assert (GET_MODE (orig_src) != VOIDmode);
3045 0 : src = force_reg (GET_MODE (orig_src), orig_src);
3046 : }
3047 :
3048 : /* Optimize the access just a bit. */
3049 576333 : if (MEM_P (src)
3050 99246 : && (! targetm.slow_unaligned_access (mode, MEM_ALIGN (src))
3051 0 : || MEM_ALIGN (src) >= GET_MODE_ALIGNMENT (mode))
3052 198492 : && multiple_p (bytepos * BITS_PER_UNIT, GET_MODE_ALIGNMENT (mode))
3053 675579 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3054 : {
3055 97443 : tmps[i] = gen_reg_rtx (mode);
3056 97443 : emit_move_insn (tmps[i], adjust_address (src, mode, bytepos));
3057 : }
3058 478890 : else if (COMPLEX_MODE_P (mode)
3059 0 : && GET_MODE (src) == mode
3060 478890 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3061 : /* Let emit_move_complex do the bulk of the work. */
3062 0 : tmps[i] = src;
3063 478890 : else if (GET_CODE (src) == CONCAT)
3064 : {
3065 17358 : poly_int64 slen = GET_MODE_SIZE (GET_MODE (src));
3066 17358 : poly_int64 slen0 = GET_MODE_SIZE (GET_MODE (XEXP (src, 0)));
3067 8679 : unsigned int elt;
3068 8679 : poly_int64 subpos;
3069 :
3070 8679 : if (can_div_trunc_p (bytepos, slen0, &elt, &subpos)
3071 8679 : && known_le (subpos + bytelen, slen0))
3072 : {
3073 : /* The following assumes that the concatenated objects all
3074 : have the same size. In this case, a simple calculation
3075 : can be used to determine the object and the bit field
3076 : to be extracted. */
3077 5924 : tmps[i] = XEXP (src, elt);
3078 5924 : if (maybe_ne (subpos, 0)
3079 5924 : || maybe_ne (subpos + bytelen, slen0)
3080 11848 : || (!CONSTANT_P (tmps[i])
3081 5924 : && (!REG_P (tmps[i]) || GET_MODE (tmps[i]) != mode)))
3082 0 : tmps[i] = extract_bit_field (tmps[i], bytelen * BITS_PER_UNIT,
3083 0 : subpos * BITS_PER_UNIT,
3084 : 1, NULL_RTX, mode, mode, false,
3085 : NULL);
3086 : }
3087 : else
3088 : {
3089 2755 : rtx mem;
3090 :
3091 2755 : gcc_assert (known_eq (bytepos, 0));
3092 2755 : mem = assign_stack_temp (GET_MODE (src), slen);
3093 2755 : emit_move_insn (mem, src);
3094 2755 : tmps[i] = extract_bit_field (mem, bytelen * BITS_PER_UNIT,
3095 : 0, 1, NULL_RTX, mode, mode, false,
3096 : NULL);
3097 : }
3098 : }
3099 470211 : else if (CONSTANT_P (src) && GET_MODE (dst) != BLKmode
3100 37598 : && XVECLEN (dst, 0) > 1)
3101 37598 : tmps[i] = force_subreg (mode, src, GET_MODE (dst), bytepos);
3102 432613 : else if (CONSTANT_P (src))
3103 : {
3104 0 : if (known_eq (bytelen, ssize))
3105 0 : tmps[i] = src;
3106 : else
3107 : {
3108 : rtx first, second;
3109 :
3110 : /* TODO: const_wide_int can have sizes other than this... */
3111 0 : gcc_assert (known_eq (2 * bytelen, ssize));
3112 0 : split_double (src, &first, &second);
3113 0 : if (i)
3114 0 : tmps[i] = second;
3115 : else
3116 0 : tmps[i] = first;
3117 : }
3118 : }
3119 432613 : else if (REG_P (src) && GET_MODE (src) == mode)
3120 0 : tmps[i] = src;
3121 : else
3122 432613 : tmps[i] = extract_bit_field (src, bytelen * BITS_PER_UNIT,
3123 865226 : bytepos * BITS_PER_UNIT, 1, NULL_RTX,
3124 : mode, mode, false, NULL);
3125 :
3126 576333 : if (maybe_ne (shift, 0))
3127 : tmps[i] = expand_shift (LSHIFT_EXPR, mode, tmps[i],
3128 : shift, tmps[i], 0);
3129 : }
3130 : }
3131 :
3132 : /* Emit code to move a block SRC of type TYPE to a block DST,
3133 : where DST is non-consecutive registers represented by a PARALLEL.
3134 : SSIZE represents the total size of block ORIG_SRC in bytes, or -1
3135 : if not known. */
3136 :
3137 : void
3138 10845 : emit_group_load (rtx dst, rtx src, tree type, poly_int64 ssize)
3139 : {
3140 10845 : rtx *tmps;
3141 10845 : int i;
3142 :
3143 10845 : tmps = XALLOCAVEC (rtx, XVECLEN (dst, 0));
3144 10845 : emit_group_load_1 (tmps, dst, src, type, ssize);
3145 :
3146 : /* Copy the extracted pieces into the proper (probable) hard regs. */
3147 30266 : for (i = 0; i < XVECLEN (dst, 0); i++)
3148 : {
3149 19421 : rtx d = XEXP (XVECEXP (dst, 0, i), 0);
3150 19421 : if (d == NULL)
3151 0 : continue;
3152 19421 : emit_move_insn (d, tmps[i]);
3153 : }
3154 10845 : }
3155 :
3156 : /* Similar, but load SRC into new pseudos in a format that looks like
3157 : PARALLEL. This can later be fed to emit_group_move to get things
3158 : in the right place. */
3159 :
3160 : rtx
3161 283059 : emit_group_load_into_temps (rtx parallel, rtx src, tree type, poly_int64 ssize)
3162 : {
3163 283059 : rtvec vec;
3164 283059 : int i;
3165 :
3166 283059 : vec = rtvec_alloc (XVECLEN (parallel, 0));
3167 283059 : emit_group_load_1 (&RTVEC_ELT (vec, 0), parallel, src, type, ssize);
3168 :
3169 : /* Convert the vector to look just like the original PARALLEL, except
3170 : with the computed values. */
3171 839971 : for (i = 0; i < XVECLEN (parallel, 0); i++)
3172 : {
3173 556912 : rtx e = XVECEXP (parallel, 0, i);
3174 556912 : rtx d = XEXP (e, 0);
3175 :
3176 556912 : if (d)
3177 : {
3178 556912 : d = force_reg (GET_MODE (d), RTVEC_ELT (vec, i));
3179 556912 : e = alloc_EXPR_LIST (REG_NOTE_KIND (e), d, XEXP (e, 1));
3180 : }
3181 556912 : RTVEC_ELT (vec, i) = e;
3182 : }
3183 :
3184 283059 : return gen_rtx_PARALLEL (GET_MODE (parallel), vec);
3185 : }
3186 :
3187 : /* Emit code to move a block SRC to block DST, where SRC and DST are
3188 : non-consecutive groups of registers, each represented by a PARALLEL. */
3189 :
3190 : void
3191 286098 : emit_group_move (rtx dst, rtx src)
3192 : {
3193 286098 : int i;
3194 :
3195 286098 : gcc_assert (GET_CODE (src) == PARALLEL
3196 : && GET_CODE (dst) == PARALLEL
3197 : && XVECLEN (src, 0) == XVECLEN (dst, 0));
3198 :
3199 : /* Skip first entry if NULL. */
3200 847356 : for (i = XEXP (XVECEXP (src, 0, 0), 0) ? 0 : 1; i < XVECLEN (src, 0); i++)
3201 561258 : emit_move_insn (XEXP (XVECEXP (dst, 0, i), 0),
3202 561258 : XEXP (XVECEXP (src, 0, i), 0));
3203 286098 : }
3204 :
3205 : /* Move a group of registers represented by a PARALLEL into pseudos. */
3206 :
3207 : rtx
3208 5769 : emit_group_move_into_temps (rtx src)
3209 : {
3210 5769 : rtvec vec = rtvec_alloc (XVECLEN (src, 0));
3211 5769 : int i;
3212 :
3213 13551 : for (i = 0; i < XVECLEN (src, 0); i++)
3214 : {
3215 7782 : rtx e = XVECEXP (src, 0, i);
3216 7782 : rtx d = XEXP (e, 0);
3217 :
3218 7782 : if (d)
3219 7782 : e = alloc_EXPR_LIST (REG_NOTE_KIND (e), copy_to_reg (d), XEXP (e, 1));
3220 7782 : RTVEC_ELT (vec, i) = e;
3221 : }
3222 :
3223 5769 : return gen_rtx_PARALLEL (GET_MODE (src), vec);
3224 : }
3225 :
3226 : /* Emit code to move a block SRC to a block ORIG_DST of type TYPE,
3227 : where SRC is non-consecutive registers represented by a PARALLEL.
3228 : SSIZE represents the total size of block ORIG_DST, or -1 if not
3229 : known. */
3230 :
3231 : void
3232 64598 : emit_group_store (rtx orig_dst, rtx src, tree type ATTRIBUTE_UNUSED,
3233 : poly_int64 ssize)
3234 : {
3235 64598 : rtx *tmps, dst;
3236 64598 : int start, finish, i;
3237 64598 : machine_mode m = GET_MODE (orig_dst);
3238 :
3239 64598 : gcc_assert (GET_CODE (src) == PARALLEL);
3240 :
3241 64598 : if (!SCALAR_INT_MODE_P (m)
3242 14180 : && !MEM_P (orig_dst) && GET_CODE (orig_dst) != CONCAT)
3243 : {
3244 0 : scalar_int_mode imode;
3245 0 : if (int_mode_for_mode (GET_MODE (orig_dst)).exists (&imode))
3246 : {
3247 0 : dst = gen_reg_rtx (imode);
3248 0 : emit_group_store (dst, src, type, ssize);
3249 0 : dst = gen_lowpart (GET_MODE (orig_dst), dst);
3250 : }
3251 : else
3252 : {
3253 0 : dst = assign_stack_temp (GET_MODE (orig_dst), ssize);
3254 0 : emit_group_store (dst, src, type, ssize);
3255 : }
3256 0 : emit_move_insn (orig_dst, dst);
3257 0 : return;
3258 : }
3259 :
3260 : /* Check for a NULL entry, used to indicate that the parameter goes
3261 : both on the stack and in registers. */
3262 64598 : if (XEXP (XVECEXP (src, 0, 0), 0))
3263 : start = 0;
3264 : else
3265 0 : start = 1;
3266 64598 : finish = XVECLEN (src, 0);
3267 :
3268 64598 : tmps = XALLOCAVEC (rtx, finish);
3269 :
3270 : /* Copy the (probable) hard regs into pseudos. */
3271 185651 : for (i = start; i < finish; i++)
3272 : {
3273 121053 : rtx reg = XEXP (XVECEXP (src, 0, i), 0);
3274 121053 : if (!REG_P (reg) || REGNO (reg) < FIRST_PSEUDO_REGISTER)
3275 : {
3276 113271 : tmps[i] = gen_reg_rtx (GET_MODE (reg));
3277 113271 : emit_move_insn (tmps[i], reg);
3278 : }
3279 : else
3280 7782 : tmps[i] = reg;
3281 : }
3282 :
3283 : /* If we won't be storing directly into memory, protect the real destination
3284 : from strange tricks we might play. */
3285 64598 : dst = orig_dst;
3286 64598 : if (GET_CODE (dst) == PARALLEL)
3287 : {
3288 0 : rtx temp;
3289 :
3290 : /* We can get a PARALLEL dst if there is a conditional expression in
3291 : a return statement. In that case, the dst and src are the same,
3292 : so no action is necessary. */
3293 0 : if (rtx_equal_p (dst, src))
3294 : return;
3295 :
3296 : /* It is unclear if we can ever reach here, but we may as well handle
3297 : it. Allocate a temporary, and split this into a store/load to/from
3298 : the temporary. */
3299 0 : temp = assign_stack_temp (GET_MODE (dst), ssize);
3300 0 : emit_group_store (temp, src, type, ssize);
3301 0 : emit_group_load (dst, temp, type, ssize);
3302 0 : return;
3303 : }
3304 64598 : else if (!MEM_P (dst) && GET_CODE (dst) != CONCAT)
3305 : {
3306 49794 : machine_mode outer = GET_MODE (dst);
3307 49794 : machine_mode inner;
3308 49794 : poly_int64 bytepos;
3309 49794 : bool done = false;
3310 49794 : rtx temp;
3311 :
3312 49794 : if (!REG_P (dst) || REGNO (dst) < FIRST_PSEUDO_REGISTER)
3313 0 : dst = gen_reg_rtx (outer);
3314 :
3315 : /* Make life a bit easier for combine: if the first element of the
3316 : vector is the low part of the destination mode, use a paradoxical
3317 : subreg to initialize the destination. */
3318 49794 : if (start < finish)
3319 : {
3320 49794 : inner = GET_MODE (tmps[start]);
3321 49794 : bytepos = subreg_lowpart_offset (inner, outer);
3322 49794 : if (known_eq (rtx_to_poly_int64 (XEXP (XVECEXP (src, 0, start), 1)),
3323 : bytepos))
3324 : {
3325 49770 : temp = force_subreg (outer, tmps[start], inner, 0);
3326 49770 : if (temp)
3327 : {
3328 49117 : emit_move_insn (dst, temp);
3329 49117 : done = true;
3330 49117 : start++;
3331 : }
3332 : }
3333 : }
3334 :
3335 : /* If the first element wasn't the low part, try the last. */
3336 49117 : if (!done
3337 677 : && start < finish - 1)
3338 : {
3339 610 : inner = GET_MODE (tmps[finish - 1]);
3340 610 : bytepos = subreg_lowpart_offset (inner, outer);
3341 610 : if (known_eq (rtx_to_poly_int64 (XEXP (XVECEXP (src, 0,
3342 : finish - 1), 1)),
3343 : bytepos))
3344 : {
3345 0 : temp = force_subreg (outer, tmps[finish - 1], inner, 0);
3346 0 : if (temp)
3347 : {
3348 0 : emit_move_insn (dst, temp);
3349 0 : done = true;
3350 0 : finish--;
3351 : }
3352 : }
3353 : }
3354 :
3355 : /* Otherwise, simply initialize the result to zero. */
3356 49117 : if (!done)
3357 677 : emit_move_insn (dst, CONST0_RTX (outer));
3358 : }
3359 :
3360 : /* Process the pieces. */
3361 133159 : for (i = start; i < finish; i++)
3362 : {
3363 71936 : poly_int64 bytepos = rtx_to_poly_int64 (XEXP (XVECEXP (src, 0, i), 1));
3364 71936 : machine_mode mode = GET_MODE (tmps[i]);
3365 143872 : poly_int64 bytelen = GET_MODE_SIZE (mode);
3366 71936 : poly_uint64 adj_bytelen;
3367 71936 : rtx dest = dst;
3368 :
3369 : /* Handle trailing fragments that run over the size of the struct.
3370 : It's the target's responsibility to make sure that the fragment
3371 : cannot be strictly smaller in some cases and strictly larger
3372 : in others. */
3373 71936 : gcc_checking_assert (ordered_p (bytepos + bytelen, ssize));
3374 142932 : if (known_size_p (ssize) && maybe_gt (bytepos + bytelen, ssize))
3375 940 : adj_bytelen = ssize - bytepos;
3376 : else
3377 71936 : adj_bytelen = bytelen;
3378 :
3379 : /* Deal with destination CONCATs by either storing into one of the parts
3380 : or doing a copy after storing into a register or stack temporary. */
3381 71936 : if (GET_CODE (dst) == CONCAT)
3382 : {
3383 27950 : if (known_le (bytepos + adj_bytelen,
3384 : GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)))))
3385 : dest = XEXP (dst, 0);
3386 :
3387 17350 : else if (known_ge (bytepos, GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)))))
3388 : {
3389 10600 : bytepos -= GET_MODE_SIZE (GET_MODE (XEXP (dst, 0)));
3390 5300 : dest = XEXP (dst, 1);
3391 : }
3392 :
3393 : else
3394 : {
3395 3375 : machine_mode dest_mode = GET_MODE (dest);
3396 3375 : machine_mode tmp_mode = GET_MODE (tmps[i]);
3397 3375 : scalar_int_mode dest_imode;
3398 :
3399 3375 : gcc_assert (known_eq (bytepos, 0) && XVECLEN (src, 0));
3400 :
3401 : /* If the source is a single scalar integer register, and the
3402 : destination has a complex mode for which a same-sized integer
3403 : mode exists, then we can take the left-justified part of the
3404 : source in the complex mode. */
3405 3375 : if (finish == start + 1
3406 3375 : && REG_P (tmps[i])
3407 3375 : && SCALAR_INT_MODE_P (tmp_mode)
3408 3375 : && COMPLEX_MODE_P (dest_mode)
3409 6750 : && int_mode_for_mode (dest_mode).exists (&dest_imode))
3410 : {
3411 3375 : const scalar_int_mode tmp_imode
3412 3375 : = as_a <scalar_int_mode> (tmp_mode);
3413 :
3414 6750 : if (GET_MODE_BITSIZE (dest_imode)
3415 3375 : < GET_MODE_BITSIZE (tmp_imode))
3416 : {
3417 59 : dest = gen_reg_rtx (dest_imode);
3418 59 : if (BYTES_BIG_ENDIAN)
3419 : tmps[i] = expand_shift (RSHIFT_EXPR, tmp_mode, tmps[i],
3420 : GET_MODE_BITSIZE (tmp_imode)
3421 : - GET_MODE_BITSIZE (dest_imode),
3422 : NULL_RTX, 1);
3423 59 : emit_move_insn (dest, gen_lowpart (dest_imode, tmps[i]));
3424 59 : dst = gen_lowpart (dest_mode, dest);
3425 : }
3426 : else
3427 3316 : dst = gen_lowpart (dest_mode, tmps[i]);
3428 : }
3429 :
3430 : /* Otherwise spill the source onto the stack using the more
3431 : aligned of the two modes. */
3432 0 : else if (GET_MODE_ALIGNMENT (dest_mode)
3433 0 : >= GET_MODE_ALIGNMENT (tmp_mode))
3434 : {
3435 0 : dest = assign_stack_temp (dest_mode,
3436 0 : GET_MODE_SIZE (dest_mode));
3437 0 : emit_move_insn (adjust_address (dest, tmp_mode, bytepos),
3438 : tmps[i]);
3439 0 : dst = dest;
3440 : }
3441 :
3442 : else
3443 : {
3444 0 : dest = assign_stack_temp (tmp_mode,
3445 0 : GET_MODE_SIZE (tmp_mode));
3446 0 : emit_move_insn (dest, tmps[i]);
3447 0 : dst = adjust_address (dest, dest_mode, bytepos);
3448 : }
3449 :
3450 : break;
3451 : }
3452 : }
3453 :
3454 : /* Handle trailing fragments that run over the size of the struct. */
3455 68561 : if (known_size_p (ssize) && maybe_gt (bytepos + bytelen, ssize))
3456 : {
3457 : /* store_bit_field always takes its value from the lsb.
3458 : Move the fragment to the lsb if it's not already there. */
3459 881 : if (
3460 : #ifdef BLOCK_REG_PADDING
3461 : BLOCK_REG_PADDING (GET_MODE (orig_dst), type, i == start)
3462 : == (BYTES_BIG_ENDIAN ? PAD_UPWARD : PAD_DOWNWARD)
3463 : #else
3464 : BYTES_BIG_ENDIAN
3465 : #endif
3466 : )
3467 : {
3468 : poly_int64 shift = (bytelen - (ssize - bytepos)) * BITS_PER_UNIT;
3469 : tmps[i] = expand_shift (RSHIFT_EXPR, mode, tmps[i],
3470 : shift, tmps[i], 0);
3471 : }
3472 :
3473 : /* Make sure not to write past the end of the struct. */
3474 881 : store_bit_field (dest,
3475 881 : adj_bytelen * BITS_PER_UNIT, bytepos * BITS_PER_UNIT,
3476 2643 : bytepos * BITS_PER_UNIT, ssize * BITS_PER_UNIT - 1,
3477 : VOIDmode, tmps[i], false, false);
3478 : }
3479 :
3480 : /* Optimize the access just a bit. */
3481 67680 : else if (MEM_P (dest)
3482 7509 : && (!targetm.slow_unaligned_access (mode, MEM_ALIGN (dest))
3483 0 : || MEM_ALIGN (dest) >= GET_MODE_ALIGNMENT (mode))
3484 15018 : && multiple_p (bytepos * BITS_PER_UNIT,
3485 : GET_MODE_ALIGNMENT (mode))
3486 75189 : && known_eq (bytelen, GET_MODE_SIZE (mode)))
3487 7509 : emit_move_insn (adjust_address (dest, mode, bytepos), tmps[i]);
3488 :
3489 : else
3490 120342 : store_bit_field (dest, bytelen * BITS_PER_UNIT, bytepos * BITS_PER_UNIT,
3491 : 0, 0, mode, tmps[i], false, false);
3492 : }
3493 :
3494 : /* Copy from the pseudo into the (probable) hard reg. */
3495 64598 : if (orig_dst != dst)
3496 3375 : emit_move_insn (orig_dst, dst);
3497 : }
3498 :
3499 : /* Return a form of X that does not use a PARALLEL. TYPE is the type
3500 : of the value stored in X. */
3501 :
3502 : rtx
3503 333027 : maybe_emit_group_store (rtx x, tree type)
3504 : {
3505 333027 : machine_mode mode = TYPE_MODE (type);
3506 333027 : gcc_checking_assert (GET_MODE (x) == VOIDmode || GET_MODE (x) == mode);
3507 333027 : if (GET_CODE (x) == PARALLEL)
3508 : {
3509 0 : rtx result = gen_reg_rtx (mode);
3510 0 : emit_group_store (result, x, type, int_size_in_bytes (type));
3511 0 : return result;
3512 : }
3513 : return x;
3514 : }
3515 :
3516 : /* Copy a BLKmode object of TYPE out of a register SRCREG into TARGET.
3517 :
3518 : This is used on targets that return BLKmode values in registers. */
3519 :
3520 : static void
3521 256 : copy_blkmode_from_reg (rtx target, rtx srcreg, tree type)
3522 : {
3523 256 : unsigned HOST_WIDE_INT bytes = int_size_in_bytes (type);
3524 256 : rtx src = NULL, dst = NULL;
3525 256 : unsigned HOST_WIDE_INT bitsize = MIN (TYPE_ALIGN (type), BITS_PER_WORD);
3526 256 : unsigned HOST_WIDE_INT bitpos, xbitpos, padding_correction = 0;
3527 : /* No current ABI uses variable-sized modes to pass a BLKmnode type. */
3528 256 : fixed_size_mode mode = as_a <fixed_size_mode> (GET_MODE (srcreg));
3529 256 : fixed_size_mode tmode = as_a <fixed_size_mode> (GET_MODE (target));
3530 256 : fixed_size_mode copy_mode;
3531 :
3532 : /* BLKmode registers created in the back-end shouldn't have survived. */
3533 256 : gcc_assert (mode != BLKmode);
3534 :
3535 : /* If the structure doesn't take up a whole number of words, see whether
3536 : SRCREG is padded on the left or on the right. If it's on the left,
3537 : set PADDING_CORRECTION to the number of bits to skip.
3538 :
3539 : In most ABIs, the structure will be returned at the least end of
3540 : the register, which translates to right padding on little-endian
3541 : targets and left padding on big-endian targets. The opposite
3542 : holds if the structure is returned at the most significant
3543 : end of the register. */
3544 256 : if (bytes % UNITS_PER_WORD != 0
3545 256 : && (targetm.calls.return_in_msb (type)
3546 225 : ? !BYTES_BIG_ENDIAN
3547 : : BYTES_BIG_ENDIAN))
3548 0 : padding_correction
3549 0 : = (BITS_PER_WORD - ((bytes % UNITS_PER_WORD) * BITS_PER_UNIT));
3550 :
3551 : /* We can use a single move if we have an exact mode for the size. */
3552 256 : else if (MEM_P (target)
3553 256 : && (!targetm.slow_unaligned_access (mode, MEM_ALIGN (target))
3554 0 : || MEM_ALIGN (target) >= GET_MODE_ALIGNMENT (mode))
3555 512 : && bytes == GET_MODE_SIZE (mode))
3556 : {
3557 38 : emit_move_insn (adjust_address (target, mode, 0), srcreg);
3558 38 : return;
3559 : }
3560 :
3561 : /* And if we additionally have the same mode for a register. */
3562 218 : else if (REG_P (target)
3563 0 : && GET_MODE (target) == mode
3564 218 : && bytes == GET_MODE_SIZE (mode))
3565 : {
3566 0 : emit_move_insn (target, srcreg);
3567 0 : return;
3568 : }
3569 :
3570 : /* This code assumes srcreg is at least a full word. If it isn't, copy it
3571 : into a new pseudo which is a full word. */
3572 436 : if (GET_MODE_SIZE (mode) < UNITS_PER_WORD)
3573 : {
3574 77 : srcreg = convert_to_mode (word_mode, srcreg, TYPE_UNSIGNED (type));
3575 77 : mode = word_mode;
3576 : }
3577 :
3578 : /* Copy the structure BITSIZE bits at a time. If the target lives in
3579 : memory, take care of not reading/writing past its end by selecting
3580 : a copy mode suited to BITSIZE. This should always be possible given
3581 : how it is computed.
3582 :
3583 : If the target lives in register, make sure not to select a copy mode
3584 : larger than the mode of the register.
3585 :
3586 : We could probably emit more efficient code for machines which do not use
3587 : strict alignment, but it doesn't seem worth the effort at the current
3588 : time. */
3589 :
3590 218 : copy_mode = word_mode;
3591 218 : if (MEM_P (target))
3592 : {
3593 218 : opt_scalar_int_mode mem_mode = int_mode_for_size (bitsize, 1);
3594 218 : if (mem_mode.exists ())
3595 218 : copy_mode = mem_mode.require ();
3596 : }
3597 0 : else if (REG_P (target) && GET_MODE_BITSIZE (tmode) < BITS_PER_WORD)
3598 : copy_mode = tmode;
3599 :
3600 218 : for (bitpos = 0, xbitpos = padding_correction;
3601 1068 : bitpos < bytes * BITS_PER_UNIT;
3602 850 : bitpos += bitsize, xbitpos += bitsize)
3603 : {
3604 : /* We need a new source operand each time xbitpos is on a
3605 : word boundary and when xbitpos == padding_correction
3606 : (the first time through). */
3607 850 : if (xbitpos % BITS_PER_WORD == 0 || xbitpos == padding_correction)
3608 218 : src = operand_subword_force (srcreg, xbitpos / BITS_PER_WORD, mode);
3609 :
3610 : /* We need a new destination operand each time bitpos is on
3611 : a word boundary. */
3612 850 : if (REG_P (target) && GET_MODE_BITSIZE (tmode) < BITS_PER_WORD)
3613 : dst = target;
3614 850 : else if (bitpos % BITS_PER_WORD == 0)
3615 218 : dst = operand_subword (target, bitpos / BITS_PER_WORD, 1, tmode);
3616 :
3617 : /* Use xbitpos for the source extraction (right justified) and
3618 : bitpos for the destination store (left justified). */
3619 850 : store_bit_field (dst, bitsize, bitpos % BITS_PER_WORD, 0, 0, copy_mode,
3620 850 : extract_bit_field (src, bitsize,
3621 850 : xbitpos % BITS_PER_WORD, 1,
3622 : NULL_RTX, copy_mode, copy_mode,
3623 : false, NULL),
3624 : false, false);
3625 : }
3626 : }
3627 :
3628 : /* Copy BLKmode value SRC into a register of mode MODE_IN. Return the
3629 : register if it contains any data, otherwise return null.
3630 :
3631 : This is used on targets that return BLKmode values in registers. */
3632 :
3633 : rtx
3634 3462 : copy_blkmode_to_reg (machine_mode mode_in, tree src)
3635 : {
3636 3462 : int i, n_regs;
3637 3462 : unsigned HOST_WIDE_INT bitpos, xbitpos, padding_correction = 0, bytes;
3638 3462 : unsigned int bitsize;
3639 3462 : rtx *dst_words, dst, x, src_word = NULL_RTX, dst_word = NULL_RTX;
3640 : /* No current ABI uses variable-sized modes to pass a BLKmnode type. */
3641 3462 : fixed_size_mode mode = as_a <fixed_size_mode> (mode_in);
3642 3462 : fixed_size_mode dst_mode;
3643 3462 : scalar_int_mode min_mode;
3644 :
3645 3462 : gcc_assert (TYPE_MODE (TREE_TYPE (src)) == BLKmode);
3646 :
3647 3462 : x = expand_normal (src);
3648 :
3649 3462 : bytes = arg_int_size_in_bytes (TREE_TYPE (src));
3650 3462 : if (bytes == 0)
3651 : return NULL_RTX;
3652 :
3653 : /* If the structure doesn't take up a whole number of words, see
3654 : whether the register value should be padded on the left or on
3655 : the right. Set PADDING_CORRECTION to the number of padding
3656 : bits needed on the left side.
3657 :
3658 : In most ABIs, the structure will be returned at the least end of
3659 : the register, which translates to right padding on little-endian
3660 : targets and left padding on big-endian targets. The opposite
3661 : holds if the structure is returned at the most significant
3662 : end of the register. */
3663 1224 : if (bytes % UNITS_PER_WORD != 0
3664 1224 : && (targetm.calls.return_in_msb (TREE_TYPE (src))
3665 1188 : ? !BYTES_BIG_ENDIAN
3666 : : BYTES_BIG_ENDIAN))
3667 0 : padding_correction = (BITS_PER_WORD - ((bytes % UNITS_PER_WORD)
3668 0 : * BITS_PER_UNIT));
3669 :
3670 1224 : n_regs = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
3671 1224 : dst_words = XALLOCAVEC (rtx, n_regs);
3672 1224 : bitsize = MIN (TYPE_ALIGN (TREE_TYPE (src)), BITS_PER_WORD);
3673 1224 : min_mode = smallest_int_mode_for_size (bitsize).require ();
3674 :
3675 : /* Copy the structure BITSIZE bits at a time. */
3676 1224 : for (bitpos = 0, xbitpos = padding_correction;
3677 3974 : bitpos < bytes * BITS_PER_UNIT;
3678 2750 : bitpos += bitsize, xbitpos += bitsize)
3679 : {
3680 : /* We need a new destination pseudo each time xbitpos is
3681 : on a word boundary and when xbitpos == padding_correction
3682 : (the first time through). */
3683 2750 : if (xbitpos % BITS_PER_WORD == 0
3684 1523 : || xbitpos == padding_correction)
3685 : {
3686 : /* Generate an appropriate register. */
3687 1227 : dst_word = gen_reg_rtx (word_mode);
3688 1227 : dst_words[xbitpos / BITS_PER_WORD] = dst_word;
3689 :
3690 : /* Clear the destination before we move anything into it. */
3691 1227 : emit_move_insn (dst_word, CONST0_RTX (word_mode));
3692 : }
3693 :
3694 : /* Find the largest integer mode that can be used to copy all or as
3695 : many bits as possible of the structure if the target supports larger
3696 : copies. There are too many corner cases here w.r.t to alignments on
3697 : the read/writes. So if there is any padding just use single byte
3698 : operations. */
3699 2750 : opt_scalar_int_mode mode_iter;
3700 2750 : if (padding_correction == 0 && !STRICT_ALIGNMENT)
3701 : {
3702 7488 : FOR_EACH_MODE_FROM (mode_iter, min_mode)
3703 : {
3704 7488 : unsigned int msize = GET_MODE_BITSIZE (mode_iter.require ());
3705 7488 : if (msize <= ((bytes * BITS_PER_UNIT) - bitpos)
3706 4741 : && msize <= BITS_PER_WORD)
3707 4738 : bitsize = msize;
3708 : else
3709 : break;
3710 : }
3711 : }
3712 :
3713 : /* We need a new source operand each time bitpos is on a word
3714 : boundary. */
3715 2750 : if (bitpos % BITS_PER_WORD == 0)
3716 1227 : src_word = operand_subword_force (x, bitpos / BITS_PER_WORD, BLKmode);
3717 :
3718 : /* Use bitpos for the source extraction (left justified) and
3719 : xbitpos for the destination store (right justified). */
3720 5500 : store_bit_field (dst_word, bitsize, xbitpos % BITS_PER_WORD,
3721 : 0, 0, word_mode,
3722 2750 : extract_bit_field (src_word, bitsize,
3723 2750 : bitpos % BITS_PER_WORD, 1,
3724 : NULL_RTX, word_mode, word_mode,
3725 : false, NULL),
3726 : false, false);
3727 : }
3728 :
3729 1224 : if (mode == BLKmode)
3730 : {
3731 : /* Find the smallest integer mode large enough to hold the
3732 : entire structure. */
3733 0 : opt_scalar_int_mode mode_iter;
3734 0 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
3735 0 : if (GET_MODE_SIZE (mode_iter.require ()) >= bytes)
3736 : break;
3737 :
3738 : /* A suitable mode should have been found. */
3739 0 : mode = mode_iter.require ();
3740 : }
3741 :
3742 3672 : if (GET_MODE_SIZE (mode) < GET_MODE_SIZE (word_mode))
3743 : dst_mode = word_mode;
3744 : else
3745 498 : dst_mode = mode;
3746 1224 : dst = gen_reg_rtx (dst_mode);
3747 :
3748 3675 : for (i = 0; i < n_regs; i++)
3749 1227 : emit_move_insn (operand_subword (dst, i, 0, dst_mode), dst_words[i]);
3750 :
3751 1224 : if (mode != dst_mode)
3752 726 : dst = gen_lowpart (mode, dst);
3753 :
3754 : return dst;
3755 : }
3756 :
3757 : /* Add a USE expression for REG to the (possibly empty) list pointed
3758 : to by CALL_FUSAGE. REG must denote a hard register. */
3759 :
3760 : void
3761 11271177 : use_reg_mode (rtx *call_fusage, rtx reg, machine_mode mode)
3762 : {
3763 11271177 : gcc_assert (REG_P (reg));
3764 :
3765 11271177 : if (!HARD_REGISTER_P (reg))
3766 : return;
3767 :
3768 11271177 : *call_fusage
3769 11271177 : = gen_rtx_EXPR_LIST (mode, gen_rtx_USE (VOIDmode, reg), *call_fusage);
3770 : }
3771 :
3772 : /* Add a CLOBBER expression for REG to the (possibly empty) list pointed
3773 : to by CALL_FUSAGE. REG must denote a hard register. */
3774 :
3775 : void
3776 798007 : clobber_reg_mode (rtx *call_fusage, rtx reg, machine_mode mode)
3777 : {
3778 798007 : gcc_assert (REG_P (reg) && REGNO (reg) < FIRST_PSEUDO_REGISTER);
3779 :
3780 798007 : *call_fusage
3781 798007 : = gen_rtx_EXPR_LIST (mode, gen_rtx_CLOBBER (VOIDmode, reg), *call_fusage);
3782 798007 : }
3783 :
3784 : /* Add USE expressions to *CALL_FUSAGE for each of NREGS consecutive regs,
3785 : starting at REGNO. All of these registers must be hard registers. */
3786 :
3787 : void
3788 1608 : use_regs (rtx *call_fusage, int regno, int nregs)
3789 : {
3790 1608 : int i;
3791 :
3792 1608 : gcc_assert (regno + nregs <= FIRST_PSEUDO_REGISTER);
3793 :
3794 3224 : for (i = 0; i < nregs; i++)
3795 1616 : use_reg (call_fusage, regno_reg_rtx[regno + i]);
3796 1608 : }
3797 :
3798 : /* Add USE expressions to *CALL_FUSAGE for each REG contained in the
3799 : PARALLEL REGS. This is for calls that pass values in multiple
3800 : non-contiguous locations. The Irix 6 ABI has examples of this. */
3801 :
3802 : void
3803 289789 : use_group_regs (rtx *call_fusage, rtx regs)
3804 : {
3805 289789 : int i;
3806 :
3807 860161 : for (i = 0; i < XVECLEN (regs, 0); i++)
3808 : {
3809 570372 : rtx reg = XEXP (XVECEXP (regs, 0, i), 0);
3810 :
3811 : /* A NULL entry means the parameter goes both on the stack and in
3812 : registers. This can also be a MEM for targets that pass values
3813 : partially on the stack and partially in registers. */
3814 570372 : if (reg != 0 && REG_P (reg))
3815 570372 : use_reg (call_fusage, reg);
3816 : }
3817 289789 : }
3818 :
3819 : /* Return the defining gimple statement for SSA_NAME NAME if it is an
3820 : assigment and the code of the expresion on the RHS is CODE. Return
3821 : NULL otherwise. */
3822 :
3823 : static gimple *
3824 11675759 : get_def_for_expr (tree name, enum tree_code code)
3825 : {
3826 11675759 : gimple *def_stmt;
3827 :
3828 11675759 : if (TREE_CODE (name) != SSA_NAME)
3829 : return NULL;
3830 :
3831 9060917 : def_stmt = get_gimple_for_ssa_name (name);
3832 9060917 : if (!def_stmt
3833 1950348 : || !is_gimple_assign (def_stmt)
3834 11009836 : || gimple_assign_rhs_code (def_stmt) != code)
3835 : return NULL;
3836 :
3837 : return def_stmt;
3838 : }
3839 :
3840 : /* Return the defining gimple statement for SSA_NAME NAME if it is an
3841 : assigment and the class of the expresion on the RHS is CLASS. Return
3842 : NULL otherwise. */
3843 :
3844 : static gimple *
3845 16606 : get_def_for_expr_class (tree name, enum tree_code_class tclass)
3846 : {
3847 16606 : gimple *def_stmt;
3848 :
3849 16606 : if (TREE_CODE (name) != SSA_NAME)
3850 : return NULL;
3851 :
3852 16606 : def_stmt = get_gimple_for_ssa_name (name);
3853 16606 : if (!def_stmt
3854 14532 : || !is_gimple_assign (def_stmt)
3855 31138 : || TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) != tclass)
3856 : return NULL;
3857 :
3858 : return def_stmt;
3859 : }
3860 :
3861 : /* Write zeros through the storage of OBJECT. If OBJECT has BLKmode, SIZE is
3862 : its length in bytes. */
3863 :
3864 : rtx
3865 147996 : clear_storage_hints (rtx object, rtx size, enum block_op_methods method,
3866 : unsigned int expected_align, HOST_WIDE_INT expected_size,
3867 : unsigned HOST_WIDE_INT min_size,
3868 : unsigned HOST_WIDE_INT max_size,
3869 : unsigned HOST_WIDE_INT probable_max_size,
3870 : unsigned ctz_size)
3871 : {
3872 147996 : machine_mode mode = GET_MODE (object);
3873 147996 : unsigned int align;
3874 :
3875 147996 : gcc_assert (method == BLOCK_OP_NORMAL || method == BLOCK_OP_TAILCALL);
3876 :
3877 : /* If OBJECT is not BLKmode and SIZE is the same size as its mode,
3878 : just move a zero. Otherwise, do this a piece at a time. */
3879 147996 : poly_int64 size_val;
3880 147996 : if (mode != BLKmode
3881 56044 : && poly_int_rtx_p (size, &size_val)
3882 204040 : && known_eq (size_val, GET_MODE_SIZE (mode)))
3883 : {
3884 56044 : rtx zero = CONST0_RTX (mode);
3885 56044 : if (zero != NULL)
3886 : {
3887 56044 : emit_move_insn (object, zero);
3888 56044 : return NULL;
3889 : }
3890 :
3891 0 : if (COMPLEX_MODE_P (mode))
3892 : {
3893 0 : zero = CONST0_RTX (GET_MODE_INNER (mode));
3894 0 : if (zero != NULL)
3895 : {
3896 0 : write_complex_part (object, zero, 0, true);
3897 0 : write_complex_part (object, zero, 1, false);
3898 0 : return NULL;
3899 : }
3900 : }
3901 : }
3902 :
3903 91952 : if (size == const0_rtx)
3904 : return NULL;
3905 :
3906 91952 : align = MEM_ALIGN (object);
3907 :
3908 91952 : if (CONST_INT_P (size)
3909 176711 : && targetm.use_by_pieces_infrastructure_p (INTVAL (size), align,
3910 : CLEAR_BY_PIECES,
3911 84759 : optimize_insn_for_speed_p ()))
3912 66044 : clear_by_pieces (object, INTVAL (size), align);
3913 25908 : else if (set_storage_via_setmem (object, size, const0_rtx, align,
3914 : expected_align, expected_size,
3915 : min_size, max_size, probable_max_size))
3916 : ;
3917 15617 : else if (try_store_by_multiple_pieces (object, size, ctz_size,
3918 : min_size, max_size,
3919 : NULL_RTX, 0, align))
3920 : ;
3921 15354 : else if (ADDR_SPACE_GENERIC_P (MEM_ADDR_SPACE (object)))
3922 15354 : return set_storage_via_libcall (object, size, const0_rtx,
3923 15354 : method == BLOCK_OP_TAILCALL);
3924 : else
3925 0 : gcc_unreachable ();
3926 :
3927 : return NULL;
3928 : }
3929 :
3930 : rtx
3931 107910 : clear_storage (rtx object, rtx size, enum block_op_methods method)
3932 : {
3933 107910 : unsigned HOST_WIDE_INT max, min = 0;
3934 107910 : if (GET_CODE (size) == CONST_INT)
3935 107910 : min = max = UINTVAL (size);
3936 : else
3937 0 : max = GET_MODE_MASK (GET_MODE (size));
3938 107910 : return clear_storage_hints (object, size, method, 0, -1, min, max, max, 0);
3939 : }
3940 :
3941 :
3942 : /* A subroutine of clear_storage. Expand a call to memset.
3943 : Return the return value of memset, 0 otherwise. */
3944 :
3945 : rtx
3946 15358 : set_storage_via_libcall (rtx object, rtx size, rtx val, bool tailcall)
3947 : {
3948 15358 : tree call_expr, fn, object_tree, size_tree, val_tree;
3949 15358 : machine_mode size_mode;
3950 :
3951 15358 : object = copy_addr_to_reg (XEXP (object, 0));
3952 15358 : object_tree = make_tree (ptr_type_node, object);
3953 :
3954 15358 : if (!CONST_INT_P (val))
3955 3 : val = convert_to_mode (TYPE_MODE (integer_type_node), val, 1);
3956 15358 : val_tree = make_tree (integer_type_node, val);
3957 :
3958 15358 : size_mode = TYPE_MODE (sizetype);
3959 15358 : size = convert_to_mode (size_mode, size, 1);
3960 15358 : size = copy_to_mode_reg (size_mode, size);
3961 15358 : size_tree = make_tree (sizetype, size);
3962 :
3963 : /* It is incorrect to use the libcall calling conventions for calls to
3964 : memset because it can be provided by the user. */
3965 15358 : fn = builtin_decl_implicit (BUILT_IN_MEMSET);
3966 15358 : call_expr = build_call_expr (fn, 3, object_tree, val_tree, size_tree);
3967 15358 : CALL_EXPR_TAILCALL (call_expr) = tailcall;
3968 :
3969 15358 : return expand_call (call_expr, NULL_RTX, false);
3970 : }
3971 :
3972 : /* Expand a setmem pattern; return true if successful. */
3973 :
3974 : bool
3975 33715 : set_storage_via_setmem (rtx object, rtx size, rtx val, unsigned int align,
3976 : unsigned int expected_align, HOST_WIDE_INT expected_size,
3977 : unsigned HOST_WIDE_INT min_size,
3978 : unsigned HOST_WIDE_INT max_size,
3979 : unsigned HOST_WIDE_INT probable_max_size)
3980 : {
3981 : /* Try the most limited insn first, because there's no point
3982 : including more than one in the machine description unless
3983 : the more limited one has some advantage. */
3984 :
3985 33715 : if (expected_align < align)
3986 : expected_align = align;
3987 33715 : if (expected_size != -1)
3988 : {
3989 8 : if ((unsigned HOST_WIDE_INT)expected_size > max_size)
3990 0 : expected_size = max_size;
3991 8 : if ((unsigned HOST_WIDE_INT)expected_size < min_size)
3992 0 : expected_size = min_size;
3993 : }
3994 :
3995 33715 : opt_scalar_int_mode mode_iter;
3996 208812 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
3997 : {
3998 187424 : scalar_int_mode mode = mode_iter.require ();
3999 187424 : enum insn_code code = direct_optab_handler (setmem_optab, mode);
4000 :
4001 187424 : if (code != CODE_FOR_nothing
4002 : /* We don't need MODE to be narrower than BITS_PER_HOST_WIDE_INT
4003 : here because if SIZE is less than the mode mask, as it is
4004 : returned by the macro, it will definitely be less than the
4005 : actual mode mask. Since SIZE is within the Pmode address
4006 : space, we limit MODE to Pmode. */
4007 187424 : && ((CONST_INT_P (size)
4008 28715 : && ((unsigned HOST_WIDE_INT) INTVAL (size)
4009 28715 : <= (GET_MODE_MASK (mode) >> 1)))
4010 25494 : || max_size <= (GET_MODE_MASK (mode) >> 1)
4011 27342 : || GET_MODE_BITSIZE (mode) >= GET_MODE_BITSIZE (Pmode)))
4012 : {
4013 44426 : class expand_operand ops[9];
4014 44426 : unsigned int nops;
4015 :
4016 44426 : nops = insn_data[(int) code].n_generator_args;
4017 44426 : gcc_assert (nops == 4 || nops == 6 || nops == 8 || nops == 9);
4018 :
4019 44426 : create_fixed_operand (&ops[0], object);
4020 : /* The check above guarantees that this size conversion is valid. */
4021 44426 : create_convert_operand_to (&ops[1], size, mode, true);
4022 44426 : create_convert_operand_from (&ops[2], val, byte_mode, true);
4023 44426 : create_integer_operand (&ops[3], align / BITS_PER_UNIT);
4024 44426 : if (nops >= 6)
4025 : {
4026 44426 : create_integer_operand (&ops[4], expected_align / BITS_PER_UNIT);
4027 44426 : create_integer_operand (&ops[5], expected_size);
4028 : }
4029 44426 : if (nops >= 8)
4030 : {
4031 44426 : create_integer_operand (&ops[6], min_size);
4032 : /* If we cannot represent the maximal size,
4033 : make parameter NULL. */
4034 44426 : if ((HOST_WIDE_INT) max_size != -1)
4035 42712 : create_integer_operand (&ops[7], max_size);
4036 : else
4037 1714 : create_fixed_operand (&ops[7], NULL);
4038 : }
4039 44426 : if (nops == 9)
4040 : {
4041 : /* If we cannot represent the maximal size,
4042 : make parameter NULL. */
4043 44426 : if ((HOST_WIDE_INT) probable_max_size != -1)
4044 43239 : create_integer_operand (&ops[8], probable_max_size);
4045 : else
4046 1187 : create_fixed_operand (&ops[8], NULL);
4047 : }
4048 44426 : if (maybe_expand_insn (code, nops, ops))
4049 12327 : return true;
4050 : }
4051 : }
4052 :
4053 : return false;
4054 : }
4055 :
4056 :
4057 : /* Write to one of the components of the complex value CPLX. Write VAL to
4058 : the real part if IMAG_P is false, and the imaginary part if its true.
4059 : If UNDEFINED_P then the value in CPLX is currently undefined. */
4060 :
4061 : void
4062 540317 : write_complex_part (rtx cplx, rtx val, bool imag_p, bool undefined_p)
4063 : {
4064 540317 : machine_mode cmode;
4065 540317 : scalar_mode imode;
4066 540317 : unsigned ibitsize;
4067 :
4068 540317 : if (GET_CODE (cplx) == CONCAT)
4069 : {
4070 476967 : emit_move_insn (XEXP (cplx, imag_p), val);
4071 476967 : return;
4072 : }
4073 :
4074 63350 : cmode = GET_MODE (cplx);
4075 63350 : imode = GET_MODE_INNER (cmode);
4076 63350 : ibitsize = GET_MODE_BITSIZE (imode);
4077 :
4078 : /* For MEMs simplify_gen_subreg may generate an invalid new address
4079 : because, e.g., the original address is considered mode-dependent
4080 : by the target, which restricts simplify_subreg from invoking
4081 : adjust_address_nv. Instead of preparing fallback support for an
4082 : invalid address, we call adjust_address_nv directly. */
4083 63350 : if (MEM_P (cplx))
4084 : {
4085 76644 : emit_move_insn (adjust_address_nv (cplx, imode,
4086 : imag_p ? GET_MODE_SIZE (imode) : 0),
4087 : val);
4088 51096 : return;
4089 : }
4090 :
4091 : /* If the sub-object is at least word sized, then we know that subregging
4092 : will work. This special case is important, since store_bit_field
4093 : wants to operate on integer modes, and there's rarely an OImode to
4094 : correspond to TCmode. */
4095 12254 : if (ibitsize >= BITS_PER_WORD
4096 : /* For hard regs we have exact predicates. Assume we can split
4097 : the original object if it spans an even number of hard regs.
4098 : This special case is important for SCmode on 64-bit platforms
4099 : where the natural size of floating-point regs is 32-bit. */
4100 12254 : || (REG_P (cplx)
4101 7308 : && REGNO (cplx) < FIRST_PSEUDO_REGISTER
4102 7122 : && REG_NREGS (cplx) % 2 == 0))
4103 : {
4104 4962 : rtx part = simplify_gen_subreg (imode, cplx, cmode,
4105 7443 : imag_p ? GET_MODE_SIZE (imode) : 0);
4106 4962 : if (part)
4107 : {
4108 4962 : emit_move_insn (part, val);
4109 4962 : return;
4110 : }
4111 : else
4112 : /* simplify_gen_subreg may fail for sub-word MEMs. */
4113 0 : gcc_assert (MEM_P (cplx) && ibitsize < BITS_PER_WORD);
4114 : }
4115 :
4116 10938 : store_bit_field (cplx, ibitsize, imag_p ? ibitsize : 0, 0, 0, imode, val,
4117 : false, undefined_p);
4118 : }
4119 :
4120 : /* Extract one of the components of the complex value CPLX. Extract the
4121 : real part if IMAG_P is false, and the imaginary part if it's true. */
4122 :
4123 : rtx
4124 488818 : read_complex_part (rtx cplx, bool imag_p)
4125 : {
4126 488818 : machine_mode cmode;
4127 488818 : scalar_mode imode;
4128 488818 : unsigned ibitsize;
4129 :
4130 488818 : if (GET_CODE (cplx) == CONCAT)
4131 310787 : return XEXP (cplx, imag_p);
4132 :
4133 178031 : cmode = GET_MODE (cplx);
4134 178031 : imode = GET_MODE_INNER (cmode);
4135 178031 : ibitsize = GET_MODE_BITSIZE (imode);
4136 :
4137 : /* Special case reads from complex constants that got spilled to memory. */
4138 178031 : if (MEM_P (cplx) && GET_CODE (XEXP (cplx, 0)) == SYMBOL_REF)
4139 : {
4140 37945 : tree decl = SYMBOL_REF_DECL (XEXP (cplx, 0));
4141 37945 : if (decl && TREE_CODE (decl) == COMPLEX_CST)
4142 : {
4143 0 : tree part = imag_p ? TREE_IMAGPART (decl) : TREE_REALPART (decl);
4144 0 : if (CONSTANT_CLASS_P (part))
4145 0 : return expand_expr (part, NULL_RTX, imode, EXPAND_NORMAL);
4146 : }
4147 : }
4148 :
4149 : /* For MEMs simplify_gen_subreg may generate an invalid new address
4150 : because, e.g., the original address is considered mode-dependent
4151 : by the target, which restricts simplify_subreg from invoking
4152 : adjust_address_nv. Instead of preparing fallback support for an
4153 : invalid address, we call adjust_address_nv directly. */
4154 178031 : if (MEM_P (cplx))
4155 235645 : return adjust_address_nv (cplx, imode,
4156 : imag_p ? GET_MODE_SIZE (imode) : 0);
4157 :
4158 : /* If the sub-object is at least word sized, then we know that subregging
4159 : will work. This special case is important, since extract_bit_field
4160 : wants to operate on integer modes, and there's rarely an OImode to
4161 : correspond to TCmode. */
4162 20834 : if (ibitsize >= BITS_PER_WORD
4163 : /* For hard regs we have exact predicates. Assume we can split
4164 : the original object if it spans an even number of hard regs.
4165 : This special case is important for SCmode on 64-bit platforms
4166 : where the natural size of floating-point regs is 32-bit. */
4167 20834 : || (REG_P (cplx)
4168 338 : && REGNO (cplx) < FIRST_PSEUDO_REGISTER
4169 292 : && REG_NREGS (cplx) % 2 == 0))
4170 : {
4171 10225 : rtx ret = simplify_gen_subreg (imode, cplx, cmode,
4172 15337 : imag_p ? GET_MODE_SIZE (imode) : 0);
4173 10225 : if (ret)
4174 : return ret;
4175 : else
4176 : /* simplify_gen_subreg may fail for sub-word MEMs. */
4177 0 : gcc_assert (MEM_P (cplx) && ibitsize < BITS_PER_WORD);
4178 : }
4179 :
4180 15911 : return extract_bit_field (cplx, ibitsize, imag_p ? ibitsize : 0,
4181 : true, NULL_RTX, imode, imode, false, NULL);
4182 : }
4183 :
4184 : /* A subroutine of emit_move_insn_1. Yet another lowpart generator.
4185 : NEW_MODE and OLD_MODE are the same size. Return NULL if X cannot be
4186 : represented in NEW_MODE. If FORCE is true, this will never happen, as
4187 : we'll force-create a SUBREG if needed. */
4188 :
4189 : static rtx
4190 243292 : emit_move_change_mode (machine_mode new_mode,
4191 : machine_mode old_mode, rtx x, bool force)
4192 : {
4193 243292 : rtx ret;
4194 :
4195 243292 : if (push_operand (x, GET_MODE (x)))
4196 : {
4197 1318 : ret = gen_rtx_MEM (new_mode, XEXP (x, 0));
4198 1318 : MEM_COPY_ATTRIBUTES (ret, x);
4199 : }
4200 241974 : else if (MEM_P (x))
4201 : {
4202 : /* We don't have to worry about changing the address since the
4203 : size in bytes is supposed to be the same. */
4204 52810 : if (reload_in_progress)
4205 : {
4206 : /* Copy the MEM to change the mode and move any
4207 : substitutions from the old MEM to the new one. */
4208 0 : ret = adjust_address_nv (x, new_mode, 0);
4209 0 : copy_replacements (x, ret);
4210 : }
4211 : else
4212 52810 : ret = adjust_address (x, new_mode, 0);
4213 : }
4214 : else
4215 : {
4216 : /* Note that we do want simplify_subreg's behavior of validating
4217 : that the new mode is ok for a hard register. If we were to use
4218 : simplify_gen_subreg, we would create the subreg, but would
4219 : probably run into the target not being able to implement it. */
4220 : /* Except, of course, when FORCE is true, when this is exactly what
4221 : we want. Which is needed for CCmodes on some targets. */
4222 189164 : if (force)
4223 189164 : ret = simplify_gen_subreg (new_mode, x, old_mode, 0);
4224 : else
4225 0 : ret = simplify_subreg (new_mode, x, old_mode, 0);
4226 : }
4227 :
4228 243292 : return ret;
4229 : }
4230 :
4231 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X using
4232 : an integer mode of the same size as MODE. Returns the instruction
4233 : emitted, or NULL if such a move could not be generated. */
4234 :
4235 : static rtx_insn *
4236 121646 : emit_move_via_integer (machine_mode mode, rtx x, rtx y, bool force)
4237 : {
4238 121646 : scalar_int_mode imode;
4239 121646 : enum insn_code code;
4240 :
4241 : /* There must exist a mode of the exact size we require. */
4242 121646 : if (!int_mode_for_mode (mode).exists (&imode))
4243 0 : return NULL;
4244 :
4245 : /* The target must support moves in this mode. */
4246 121646 : code = optab_handler (mov_optab, imode);
4247 121646 : if (code == CODE_FOR_nothing)
4248 : return NULL;
4249 :
4250 121646 : x = emit_move_change_mode (imode, mode, x, force);
4251 121646 : if (x == NULL_RTX)
4252 : return NULL;
4253 121646 : y = emit_move_change_mode (imode, mode, y, force);
4254 121646 : if (y == NULL_RTX)
4255 : return NULL;
4256 121646 : return emit_insn (GEN_FCN (code) (x, y));
4257 : }
4258 :
4259 : /* A subroutine of emit_move_insn_1. X is a push_operand in MODE.
4260 : Return an equivalent MEM that does not use an auto-increment. */
4261 :
4262 : rtx
4263 3617 : emit_move_resolve_push (machine_mode mode, rtx x)
4264 : {
4265 3617 : enum rtx_code code = GET_CODE (XEXP (x, 0));
4266 3617 : rtx temp;
4267 :
4268 7234 : poly_int64 adjust = GET_MODE_SIZE (mode);
4269 : #ifdef PUSH_ROUNDING
4270 3617 : adjust = PUSH_ROUNDING (adjust);
4271 : #endif
4272 3617 : if (code == PRE_DEC || code == POST_DEC)
4273 3221 : adjust = -adjust;
4274 396 : else if (code == PRE_MODIFY || code == POST_MODIFY)
4275 : {
4276 396 : rtx expr = XEXP (XEXP (x, 0), 1);
4277 :
4278 396 : gcc_assert (GET_CODE (expr) == PLUS || GET_CODE (expr) == MINUS);
4279 396 : poly_int64 val = rtx_to_poly_int64 (XEXP (expr, 1));
4280 396 : if (GET_CODE (expr) == MINUS)
4281 0 : val = -val;
4282 396 : gcc_assert (known_eq (adjust, val) || known_eq (adjust, -val));
4283 : adjust = val;
4284 : }
4285 :
4286 : /* Do not use anti_adjust_stack, since we don't want to update
4287 : stack_pointer_delta. */
4288 3621 : temp = expand_simple_binop (Pmode, PLUS, stack_pointer_rtx,
4289 3617 : gen_int_mode (adjust, Pmode), stack_pointer_rtx,
4290 : 0, OPTAB_LIB_WIDEN);
4291 3617 : if (temp != stack_pointer_rtx)
4292 0 : emit_move_insn (stack_pointer_rtx, temp);
4293 :
4294 3617 : switch (code)
4295 : {
4296 3617 : case PRE_INC:
4297 3617 : case PRE_DEC:
4298 3617 : case PRE_MODIFY:
4299 3617 : temp = stack_pointer_rtx;
4300 3617 : break;
4301 : case POST_INC:
4302 : case POST_DEC:
4303 : case POST_MODIFY:
4304 0 : temp = plus_constant (Pmode, stack_pointer_rtx, -adjust);
4305 0 : break;
4306 0 : default:
4307 0 : gcc_unreachable ();
4308 : }
4309 :
4310 3617 : return replace_equiv_address (x, temp);
4311 : }
4312 :
4313 : /* A subroutine of emit_move_complex. Generate a move from Y into X.
4314 : X is known to satisfy push_operand, and MODE is known to be complex.
4315 : Returns the last instruction emitted. */
4316 :
4317 : rtx_insn *
4318 5561 : emit_move_complex_push (machine_mode mode, rtx x, rtx y)
4319 : {
4320 5561 : scalar_mode submode = GET_MODE_INNER (mode);
4321 5561 : bool imag_first;
4322 :
4323 : #ifdef PUSH_ROUNDING
4324 11122 : poly_int64 submodesize = GET_MODE_SIZE (submode);
4325 :
4326 : /* In case we output to the stack, but the size is smaller than the
4327 : machine can push exactly, we need to use move instructions. */
4328 5561 : if (maybe_ne (PUSH_ROUNDING (submodesize), submodesize))
4329 : {
4330 718 : x = emit_move_resolve_push (mode, x);
4331 718 : return emit_move_insn (x, y);
4332 : }
4333 : #endif
4334 :
4335 : /* Note that the real part always precedes the imag part in memory
4336 : regardless of machine's endianness. */
4337 4843 : switch (GET_CODE (XEXP (x, 0)))
4338 : {
4339 : case PRE_DEC:
4340 : case POST_DEC:
4341 : imag_first = true;
4342 : break;
4343 0 : case PRE_INC:
4344 0 : case POST_INC:
4345 0 : imag_first = false;
4346 0 : break;
4347 0 : default:
4348 0 : gcc_unreachable ();
4349 : }
4350 :
4351 4843 : emit_move_insn (gen_rtx_MEM (submode, XEXP (x, 0)),
4352 : read_complex_part (y, imag_first));
4353 4843 : return emit_move_insn (gen_rtx_MEM (submode, XEXP (x, 0)),
4354 9686 : read_complex_part (y, !imag_first));
4355 : }
4356 :
4357 : /* A subroutine of emit_move_complex. Perform the move from Y to X
4358 : via two moves of the parts. Returns the last instruction emitted. */
4359 :
4360 : rtx_insn *
4361 73363 : emit_move_complex_parts (rtx x, rtx y)
4362 : {
4363 : /* Show the output dies here. This is necessary for SUBREGs
4364 : of pseudos since we cannot track their lifetimes correctly;
4365 : hard regs shouldn't appear here except as return values. */
4366 73363 : if (!reload_completed && !reload_in_progress
4367 146726 : && REG_P (x) && !reg_overlap_mentioned_p (x, y))
4368 6120 : emit_clobber (x);
4369 :
4370 73363 : write_complex_part (x, read_complex_part (y, false), false, true);
4371 73363 : write_complex_part (x, read_complex_part (y, true), true, false);
4372 :
4373 73363 : return get_last_insn ();
4374 : }
4375 :
4376 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X.
4377 : MODE is known to be complex. Returns the last instruction emitted. */
4378 :
4379 : static rtx_insn *
4380 78280 : emit_move_complex (machine_mode mode, rtx x, rtx y)
4381 : {
4382 78280 : bool try_int;
4383 :
4384 : /* Need to take special care for pushes, to maintain proper ordering
4385 : of the data, and possibly extra padding. */
4386 78280 : if (push_operand (x, mode))
4387 4953 : return emit_move_complex_push (mode, x, y);
4388 :
4389 : /* See if we can coerce the target into moving both values at once, except
4390 : for floating point where we favor moving as parts if this is easy. */
4391 73327 : if (GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT
4392 65112 : && optab_handler (mov_optab, GET_MODE_INNER (mode)) != CODE_FOR_nothing
4393 65112 : && !(REG_P (x)
4394 2213 : && HARD_REGISTER_P (x)
4395 564 : && REG_NREGS (x) == 1)
4396 138439 : && !(REG_P (y)
4397 3302 : && HARD_REGISTER_P (y)
4398 1651 : && REG_NREGS (y) == 1))
4399 : try_int = false;
4400 : /* Not possible if the values are inherently not adjacent. */
4401 8215 : else if (GET_CODE (x) == CONCAT || GET_CODE (y) == CONCAT)
4402 : try_int = false;
4403 : /* Is possible if both are registers (or subregs of registers). */
4404 757 : else if (register_operand (x, mode) && register_operand (y, mode))
4405 : try_int = true;
4406 : /* If one of the operands is a memory, and alignment constraints
4407 : are friendly enough, we may be able to do combined memory operations.
4408 : We do not attempt this if Y is a constant because that combination is
4409 : usually better with the by-parts thing below. */
4410 635 : else if ((MEM_P (x) ? !CONSTANT_P (y) : MEM_P (y))
4411 : && (!STRICT_ALIGNMENT
4412 : || get_mode_alignment (mode) == BIGGEST_ALIGNMENT))
4413 : try_int = true;
4414 : else
4415 : try_int = false;
4416 :
4417 : if (try_int)
4418 : {
4419 757 : rtx_insn *ret;
4420 :
4421 : /* For memory to memory moves, optimal behavior can be had with the
4422 : existing block move logic. But use normal expansion if optimizing
4423 : for size. */
4424 757 : if (MEM_P (x) && MEM_P (y))
4425 : {
4426 726 : emit_block_move (x, y, gen_int_mode (GET_MODE_SIZE (mode), Pmode),
4427 360 : (optimize_insn_for_speed_p()
4428 : ? BLOCK_OP_NO_LIBCALL : BLOCK_OP_NORMAL));
4429 360 : return get_last_insn ();
4430 : }
4431 :
4432 397 : ret = emit_move_via_integer (mode, x, y, true);
4433 397 : if (ret)
4434 : return ret;
4435 : }
4436 :
4437 72570 : return emit_move_complex_parts (x, y);
4438 : }
4439 :
4440 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X.
4441 : MODE is known to be MODE_CC. Returns the last instruction emitted. */
4442 :
4443 : static rtx_insn *
4444 0 : emit_move_ccmode (machine_mode mode, rtx x, rtx y)
4445 : {
4446 0 : rtx_insn *ret;
4447 :
4448 : /* Assume all MODE_CC modes are equivalent; if we have movcc, use it. */
4449 0 : if (mode != CCmode)
4450 : {
4451 0 : enum insn_code code = optab_handler (mov_optab, CCmode);
4452 0 : if (code != CODE_FOR_nothing)
4453 : {
4454 0 : x = emit_move_change_mode (CCmode, mode, x, true);
4455 0 : y = emit_move_change_mode (CCmode, mode, y, true);
4456 0 : return emit_insn (GEN_FCN (code) (x, y));
4457 : }
4458 : }
4459 :
4460 : /* Otherwise, find the MODE_INT mode of the same width. */
4461 0 : ret = emit_move_via_integer (mode, x, y, false);
4462 0 : gcc_assert (ret != NULL);
4463 : return ret;
4464 : }
4465 :
4466 : /* Return true if word I of OP lies entirely in the
4467 : undefined bits of a paradoxical subreg. */
4468 :
4469 : static bool
4470 0 : undefined_operand_subword_p (const_rtx op, int i)
4471 : {
4472 0 : if (GET_CODE (op) != SUBREG)
4473 : return false;
4474 0 : machine_mode innermostmode = GET_MODE (SUBREG_REG (op));
4475 0 : poly_int64 offset = i * UNITS_PER_WORD + subreg_memory_offset (op);
4476 0 : return (known_ge (offset, GET_MODE_SIZE (innermostmode))
4477 0 : || known_le (offset, -UNITS_PER_WORD));
4478 : }
4479 :
4480 : /* A subroutine of emit_move_insn_1. Generate a move from Y into X.
4481 : MODE is any multi-word or full-word mode that lacks a move_insn
4482 : pattern. Note that you will get better code if you define such
4483 : patterns, even if they must turn into multiple assembler instructions. */
4484 :
4485 : static rtx_insn *
4486 0 : emit_move_multi_word (machine_mode mode, rtx x, rtx y)
4487 : {
4488 0 : rtx_insn *last_insn = 0;
4489 0 : rtx_insn *seq;
4490 0 : rtx inner;
4491 0 : bool need_clobber;
4492 0 : int i, mode_size;
4493 :
4494 : /* This function can only handle cases where the number of words is
4495 : known at compile time. */
4496 0 : mode_size = GET_MODE_SIZE (mode).to_constant ();
4497 0 : gcc_assert (mode_size >= UNITS_PER_WORD);
4498 :
4499 : /* If X is a push on the stack, do the push now and replace
4500 : X with a reference to the stack pointer. */
4501 0 : if (push_operand (x, mode))
4502 0 : x = emit_move_resolve_push (mode, x);
4503 :
4504 : /* If we are in reload, see if either operand is a MEM whose address
4505 : is scheduled for replacement. */
4506 0 : if (reload_in_progress && MEM_P (x)
4507 0 : && (inner = find_replacement (&XEXP (x, 0))) != XEXP (x, 0))
4508 0 : x = replace_equiv_address_nv (x, inner);
4509 0 : if (reload_in_progress && MEM_P (y)
4510 0 : && (inner = find_replacement (&XEXP (y, 0))) != XEXP (y, 0))
4511 0 : y = replace_equiv_address_nv (y, inner);
4512 :
4513 0 : start_sequence ();
4514 :
4515 0 : need_clobber = false;
4516 0 : for (i = 0; i < CEIL (mode_size, UNITS_PER_WORD); i++)
4517 : {
4518 : /* Do not generate code for a move if it would go entirely
4519 : to the non-existing bits of a paradoxical subreg. */
4520 0 : if (undefined_operand_subword_p (x, i))
4521 0 : continue;
4522 :
4523 0 : rtx xpart = operand_subword (x, i, 1, mode);
4524 0 : rtx ypart;
4525 :
4526 : /* Do not generate code for a move if it would come entirely
4527 : from the undefined bits of a paradoxical subreg. */
4528 0 : if (undefined_operand_subword_p (y, i))
4529 0 : continue;
4530 :
4531 0 : ypart = operand_subword (y, i, 1, mode);
4532 :
4533 : /* If we can't get a part of Y, put Y into memory if it is a
4534 : constant. Otherwise, force it into a register. Then we must
4535 : be able to get a part of Y. */
4536 0 : if (ypart == 0 && CONSTANT_P (y))
4537 : {
4538 0 : y = use_anchored_address (force_const_mem (mode, y));
4539 0 : ypart = operand_subword (y, i, 1, mode);
4540 : }
4541 0 : else if (ypart == 0)
4542 0 : ypart = operand_subword_force (y, i, mode);
4543 :
4544 0 : gcc_assert (xpart && ypart);
4545 :
4546 0 : need_clobber |= (GET_CODE (xpart) == SUBREG);
4547 :
4548 0 : last_insn = emit_move_insn (xpart, ypart);
4549 : }
4550 :
4551 0 : seq = end_sequence ();
4552 :
4553 : /* Show the output dies here. This is necessary for SUBREGs
4554 : of pseudos since we cannot track their lifetimes correctly;
4555 : hard regs shouldn't appear here except as return values.
4556 : We never want to emit such a clobber after reload. */
4557 0 : if (x != y
4558 0 : && ! (reload_in_progress || reload_completed)
4559 0 : && need_clobber != 0)
4560 0 : emit_clobber (x);
4561 :
4562 0 : emit_insn (seq);
4563 :
4564 0 : return last_insn;
4565 : }
4566 :
4567 : /* Low level part of emit_move_insn.
4568 : Called just like emit_move_insn, but assumes X and Y
4569 : are basically valid. */
4570 :
4571 : rtx_insn *
4572 75737591 : emit_move_insn_1 (rtx x, rtx y)
4573 : {
4574 75737591 : machine_mode mode = GET_MODE (x);
4575 75737591 : enum insn_code code;
4576 :
4577 75737591 : gcc_assert ((unsigned int) mode < (unsigned int) MAX_MACHINE_MODE);
4578 :
4579 75737591 : code = optab_handler (mov_optab, mode);
4580 75737591 : if (code != CODE_FOR_nothing)
4581 75538062 : return emit_insn (GEN_FCN (code) (x, y));
4582 :
4583 : /* Expand complex moves by moving real part and imag part. */
4584 199529 : if (COMPLEX_MODE_P (mode))
4585 78280 : return emit_move_complex (mode, x, y);
4586 :
4587 : if (GET_MODE_CLASS (mode) == MODE_DECIMAL_FLOAT
4588 : || ALL_FIXED_POINT_MODE_P (mode))
4589 : {
4590 121249 : rtx_insn *result = emit_move_via_integer (mode, x, y, true);
4591 :
4592 : /* If we can't find an integer mode, use multi words. */
4593 121249 : if (result)
4594 : return result;
4595 : else
4596 0 : return emit_move_multi_word (mode, x, y);
4597 : }
4598 :
4599 : if (GET_MODE_CLASS (mode) == MODE_CC)
4600 0 : return emit_move_ccmode (mode, x, y);
4601 :
4602 : /* Try using a move pattern for the corresponding integer mode. This is
4603 : only safe when simplify_subreg can convert MODE constants into integer
4604 : constants. At present, it can only do this reliably if the value
4605 : fits within a HOST_WIDE_INT. */
4606 0 : if (!CONSTANT_P (y)
4607 0 : || known_le (GET_MODE_BITSIZE (mode), HOST_BITS_PER_WIDE_INT))
4608 : {
4609 0 : rtx_insn *ret = emit_move_via_integer (mode, x, y, lra_in_progress);
4610 :
4611 0 : if (ret)
4612 : {
4613 0 : if (! lra_in_progress || recog (PATTERN (ret), ret, 0) >= 0)
4614 0 : return ret;
4615 : }
4616 : }
4617 :
4618 0 : return emit_move_multi_word (mode, x, y);
4619 : }
4620 :
4621 : /* Generate code to copy Y into X.
4622 : Both Y and X must have the same mode, except that
4623 : Y can be a constant with VOIDmode.
4624 : This mode cannot be BLKmode; use emit_block_move for that.
4625 :
4626 : Return the last instruction emitted. */
4627 :
4628 : rtx_insn *
4629 69913810 : emit_move_insn (rtx x, rtx y)
4630 : {
4631 69913810 : machine_mode mode = GET_MODE (x);
4632 69913810 : rtx y_cst = NULL_RTX;
4633 69913810 : rtx_insn *last_insn;
4634 69913810 : rtx set;
4635 :
4636 69913810 : gcc_assert (mode != BLKmode
4637 : && (GET_MODE (y) == mode || GET_MODE (y) == VOIDmode));
4638 :
4639 : /* If we have a copy that looks like one of the following patterns:
4640 : (set (subreg:M1 (reg:M2 ...)) (subreg:M1 (reg:M2 ...)))
4641 : (set (subreg:M1 (reg:M2 ...)) (mem:M1 ADDR))
4642 : (set (mem:M1 ADDR) (subreg:M1 (reg:M2 ...)))
4643 : (set (subreg:M1 (reg:M2 ...)) (constant C))
4644 : where mode M1 is equal in size to M2, try to detect whether the
4645 : mode change involves an implicit round trip through memory.
4646 : If so, see if we can avoid that by removing the subregs and
4647 : doing the move in mode M2 instead. */
4648 :
4649 69913810 : rtx x_inner = NULL_RTX;
4650 69913810 : rtx y_inner = NULL_RTX;
4651 :
4652 72566454 : auto candidate_subreg_p = [&](rtx subreg) {
4653 2652644 : return (REG_P (SUBREG_REG (subreg))
4654 7956171 : && known_eq (GET_MODE_SIZE (GET_MODE (SUBREG_REG (subreg))),
4655 : GET_MODE_SIZE (GET_MODE (subreg)))
4656 2972509 : && optab_handler (mov_optab, GET_MODE (SUBREG_REG (subreg)))
4657 2652644 : != CODE_FOR_nothing);
4658 : };
4659 :
4660 69919975 : auto candidate_mem_p = [&](machine_mode innermode, rtx mem) {
4661 6165 : return (!targetm.can_change_mode_class (innermode, GET_MODE (mem), ALL_REGS)
4662 6165 : && !push_operand (mem, GET_MODE (mem))
4663 : /* Not a candiate if innermode requires too much alignment. */
4664 12270 : && (MEM_ALIGN (mem) >= GET_MODE_ALIGNMENT (innermode)
4665 232 : || targetm.slow_unaligned_access (GET_MODE (mem),
4666 116 : MEM_ALIGN (mem))
4667 232 : || !targetm.slow_unaligned_access (innermode,
4668 116 : MEM_ALIGN (mem))));
4669 : };
4670 :
4671 69913810 : if (SUBREG_P (x) && candidate_subreg_p (x))
4672 12638 : x_inner = SUBREG_REG (x);
4673 :
4674 69913810 : if (SUBREG_P (y) && candidate_subreg_p (y))
4675 305897 : y_inner = SUBREG_REG (y);
4676 :
4677 69913810 : if (x_inner != NULL_RTX
4678 69913810 : && y_inner != NULL_RTX
4679 1116 : && GET_MODE (x_inner) == GET_MODE (y_inner)
4680 69914521 : && !targetm.can_change_mode_class (GET_MODE (x_inner), mode, ALL_REGS))
4681 : {
4682 711 : x = x_inner;
4683 711 : y = y_inner;
4684 711 : mode = GET_MODE (x_inner);
4685 : }
4686 69913099 : else if (x_inner != NULL_RTX
4687 11927 : && MEM_P (y)
4688 69913423 : && candidate_mem_p (GET_MODE (x_inner), y))
4689 : {
4690 324 : x = x_inner;
4691 324 : y = adjust_address (y, GET_MODE (x_inner), 0);
4692 324 : mode = GET_MODE (x_inner);
4693 : }
4694 69912775 : else if (y_inner != NULL_RTX
4695 305186 : && MEM_P (x)
4696 69918616 : && candidate_mem_p (GET_MODE (y_inner), x))
4697 : {
4698 5781 : x = adjust_address (x, GET_MODE (y_inner), 0);
4699 5781 : y = y_inner;
4700 5781 : mode = GET_MODE (y_inner);
4701 : }
4702 69906994 : else if (x_inner != NULL_RTX
4703 11603 : && CONSTANT_P (y)
4704 470 : && !targetm.can_change_mode_class (GET_MODE (x_inner),
4705 : mode, ALL_REGS)
4706 69907464 : && (y_inner = simplify_subreg (GET_MODE (x_inner), y, mode, 0)))
4707 : {
4708 470 : x = x_inner;
4709 470 : y = y_inner;
4710 470 : mode = GET_MODE (x_inner);
4711 : }
4712 :
4713 69913810 : if (CONSTANT_P (y))
4714 : {
4715 16335525 : if (optimize
4716 12716242 : && SCALAR_FLOAT_MODE_P (GET_MODE (x))
4717 17102121 : && (last_insn = compress_float_constant (x, y)))
4718 : return last_insn;
4719 :
4720 16266602 : y_cst = y;
4721 :
4722 16266602 : if (!targetm.legitimate_constant_p (mode, y))
4723 : {
4724 503348 : y = force_const_mem (mode, y);
4725 :
4726 : /* If the target's cannot_force_const_mem prevented the spill,
4727 : assume that the target's move expanders will also take care
4728 : of the non-legitimate constant. */
4729 503348 : if (!y)
4730 : y = y_cst;
4731 : else
4732 483314 : y = use_anchored_address (y);
4733 : }
4734 : }
4735 :
4736 : /* If X or Y are memory references, verify that their addresses are valid
4737 : for the machine. */
4738 69844887 : if (MEM_P (x)
4739 85937509 : && (! memory_address_addr_space_p (GET_MODE (x), XEXP (x, 0),
4740 13834569 : MEM_ADDR_SPACE (x))
4741 2258227 : && ! push_operand (x, GET_MODE (x))))
4742 174 : x = validize_mem (x);
4743 :
4744 69844887 : if (MEM_P (y)
4745 87545552 : && ! memory_address_addr_space_p (GET_MODE (y), XEXP (y, 0),
4746 17700665 : MEM_ADDR_SPACE (y)))
4747 8702 : y = validize_mem (y);
4748 :
4749 69844887 : gcc_assert (mode != BLKmode);
4750 :
4751 69844887 : last_insn = emit_move_insn_1 (x, y);
4752 :
4753 16266602 : if (y_cst && REG_P (x)
4754 11912890 : && (set = single_set (last_insn)) != NULL_RTX
4755 11912890 : && SET_DEST (set) == x
4756 81743326 : && ! rtx_equal_p (y_cst, SET_SRC (set)))
4757 1231907 : set_unique_reg_note (last_insn, REG_EQUAL, copy_rtx (y_cst));
4758 :
4759 : return last_insn;
4760 : }
4761 :
4762 : /* Generate the body of an instruction to copy Y into X.
4763 : It may be a list of insns, if one insn isn't enough. */
4764 :
4765 : rtx_insn *
4766 5892624 : gen_move_insn (rtx x, rtx y)
4767 : {
4768 5892624 : start_sequence ();
4769 5892624 : emit_move_insn_1 (x, y);
4770 5892624 : return end_sequence ();
4771 : }
4772 :
4773 : /* If Y is representable exactly in a narrower mode, and the target can
4774 : perform the extension directly from constant or memory, then emit the
4775 : move as an extension. */
4776 :
4777 : static rtx_insn *
4778 766596 : compress_float_constant (rtx x, rtx y)
4779 : {
4780 766596 : machine_mode dstmode = GET_MODE (x);
4781 766596 : machine_mode orig_srcmode = GET_MODE (y);
4782 766596 : machine_mode srcmode;
4783 766596 : const REAL_VALUE_TYPE *r;
4784 766596 : int oldcost, newcost;
4785 766596 : bool speed = optimize_insn_for_speed_p ();
4786 :
4787 766596 : r = CONST_DOUBLE_REAL_VALUE (y);
4788 :
4789 766596 : if (targetm.legitimate_constant_p (dstmode, y))
4790 764982 : oldcost = set_src_cost (y, orig_srcmode, speed);
4791 : else
4792 1614 : oldcost = set_src_cost (force_const_mem (dstmode, y), dstmode, speed);
4793 :
4794 2715803 : FOR_EACH_MODE_UNTIL (srcmode, orig_srcmode)
4795 : {
4796 2018130 : enum insn_code ic;
4797 2018130 : rtx trunc_y;
4798 2018130 : rtx_insn *last_insn;
4799 :
4800 : /* Skip if the target can't extend this way. */
4801 2018130 : ic = can_extend_p (dstmode, srcmode, 0);
4802 2018130 : if (ic == CODE_FOR_nothing)
4803 1597633 : continue;
4804 :
4805 : /* Skip if the narrowed value isn't exact. */
4806 420497 : if (! exact_real_truncate (srcmode, r))
4807 50586 : continue;
4808 :
4809 369911 : trunc_y = const_double_from_real_value (*r, srcmode);
4810 :
4811 369911 : if (targetm.legitimate_constant_p (srcmode, trunc_y))
4812 : {
4813 : /* Skip if the target needs extra instructions to perform
4814 : the extension. */
4815 366274 : if (!insn_operand_matches (ic, 1, trunc_y))
4816 4070 : continue;
4817 : /* This is valid, but may not be cheaper than the original. */
4818 362204 : newcost = set_src_cost (gen_rtx_FLOAT_EXTEND (dstmode, trunc_y),
4819 : dstmode, speed);
4820 362204 : if (oldcost < newcost)
4821 293281 : continue;
4822 : }
4823 3637 : else if (float_extend_from_mem[dstmode][srcmode])
4824 : {
4825 0 : trunc_y = force_const_mem (srcmode, trunc_y);
4826 : /* This is valid, but may not be cheaper than the original. */
4827 0 : newcost = set_src_cost (gen_rtx_FLOAT_EXTEND (dstmode, trunc_y),
4828 : dstmode, speed);
4829 0 : if (oldcost < newcost)
4830 0 : continue;
4831 0 : trunc_y = validize_mem (trunc_y);
4832 : }
4833 : else
4834 3637 : continue;
4835 :
4836 : /* For CSE's benefit, force the compressed constant pool entry
4837 : into a new pseudo. This constant may be used in different modes,
4838 : and if not, combine will put things back together for us. */
4839 68923 : trunc_y = force_reg (srcmode, trunc_y);
4840 :
4841 : /* If x is a hard register, perform the extension into a pseudo,
4842 : so that e.g. stack realignment code is aware of it. */
4843 68923 : rtx target = x;
4844 68923 : if (REG_P (x) && HARD_REGISTER_P (x))
4845 1 : target = gen_reg_rtx (dstmode);
4846 :
4847 68923 : emit_unop_insn (ic, target, trunc_y, UNKNOWN);
4848 68923 : last_insn = get_last_insn ();
4849 :
4850 68923 : if (REG_P (target))
4851 58263 : set_unique_reg_note (last_insn, REG_EQUAL, y);
4852 :
4853 68923 : if (target != x)
4854 1 : return emit_move_insn (x, target);
4855 : return last_insn;
4856 : }
4857 :
4858 : return NULL;
4859 : }
4860 :
4861 : /* Pushing data onto the stack. */
4862 :
4863 : /* Push a block of length SIZE (perhaps variable)
4864 : and return an rtx to address the beginning of the block.
4865 : The value may be virtual_outgoing_args_rtx.
4866 :
4867 : EXTRA is the number of bytes of padding to push in addition to SIZE.
4868 : BELOW nonzero means this padding comes at low addresses;
4869 : otherwise, the padding comes at high addresses. */
4870 :
4871 : rtx
4872 268870 : push_block (rtx size, poly_int64 extra, int below)
4873 : {
4874 268870 : rtx temp;
4875 :
4876 333994 : size = convert_modes (Pmode, ptr_mode, size, 1);
4877 268870 : if (CONSTANT_P (size))
4878 333994 : anti_adjust_stack (plus_constant (Pmode, size, extra));
4879 0 : else if (REG_P (size) && known_eq (extra, 0))
4880 0 : anti_adjust_stack (size);
4881 : else
4882 : {
4883 0 : temp = copy_to_mode_reg (Pmode, size);
4884 0 : if (maybe_ne (extra, 0))
4885 0 : temp = expand_binop (Pmode, add_optab, temp,
4886 0 : gen_int_mode (extra, Pmode),
4887 : temp, 0, OPTAB_LIB_WIDEN);
4888 0 : anti_adjust_stack (temp);
4889 : }
4890 :
4891 268870 : if (STACK_GROWS_DOWNWARD)
4892 : {
4893 268870 : temp = virtual_outgoing_args_rtx;
4894 268870 : if (maybe_ne (extra, 0) && below)
4895 0 : temp = plus_constant (Pmode, temp, extra);
4896 : }
4897 : else
4898 : {
4899 : poly_int64 csize;
4900 : if (poly_int_rtx_p (size, &csize))
4901 : temp = plus_constant (Pmode, virtual_outgoing_args_rtx,
4902 : -csize - (below ? 0 : extra));
4903 : else if (maybe_ne (extra, 0) && !below)
4904 : temp = gen_rtx_PLUS (Pmode, virtual_outgoing_args_rtx,
4905 : negate_rtx (Pmode, plus_constant (Pmode, size,
4906 : extra)));
4907 : else
4908 : temp = gen_rtx_PLUS (Pmode, virtual_outgoing_args_rtx,
4909 : negate_rtx (Pmode, size));
4910 : }
4911 :
4912 268870 : return memory_address (NARROWEST_INT_MODE, temp);
4913 : }
4914 :
4915 : /* A utility routine that returns the base of an auto-inc memory, or NULL. */
4916 :
4917 : static rtx
4918 2581602 : mem_autoinc_base (rtx mem)
4919 : {
4920 0 : if (MEM_P (mem))
4921 : {
4922 1579051 : rtx addr = XEXP (mem, 0);
4923 1579051 : if (GET_RTX_CLASS (GET_CODE (addr)) == RTX_AUTOINC)
4924 1032468 : return XEXP (addr, 0);
4925 : }
4926 : return NULL;
4927 : }
4928 :
4929 : /* A utility routine used here, in reload, and in try_split. The insns
4930 : after PREV up to and including LAST are known to adjust the stack,
4931 : with a final value of END_ARGS_SIZE. Iterate backward from LAST
4932 : placing notes as appropriate. PREV may be NULL, indicating the
4933 : entire insn sequence prior to LAST should be scanned.
4934 :
4935 : The set of allowed stack pointer modifications is small:
4936 : (1) One or more auto-inc style memory references (aka pushes),
4937 : (2) One or more addition/subtraction with the SP as destination,
4938 : (3) A single move insn with the SP as destination,
4939 : (4) A call_pop insn,
4940 : (5) Noreturn call insns if !ACCUMULATE_OUTGOING_ARGS.
4941 :
4942 : Insns in the sequence that do not modify the SP are ignored,
4943 : except for noreturn calls.
4944 :
4945 : The return value is the amount of adjustment that can be trivially
4946 : verified, via immediate operand or auto-inc. If the adjustment
4947 : cannot be trivially extracted, the return value is HOST_WIDE_INT_MIN. */
4948 :
4949 : poly_int64
4950 4510422 : find_args_size_adjust (rtx_insn *insn)
4951 : {
4952 4510422 : rtx dest, set, pat;
4953 4510422 : int i;
4954 :
4955 4510422 : pat = PATTERN (insn);
4956 4510422 : set = NULL;
4957 :
4958 : /* Look for a call_pop pattern. */
4959 4510422 : if (CALL_P (insn))
4960 : {
4961 : /* We have to allow non-call_pop patterns for the case
4962 : of emit_single_push_insn of a TLS address. */
4963 2860549 : if (GET_CODE (pat) != PARALLEL)
4964 2845644 : return 0;
4965 :
4966 : /* All call_pop have a stack pointer adjust in the parallel.
4967 : The call itself is always first, and the stack adjust is
4968 : usually last, so search from the end. */
4969 14905 : for (i = XVECLEN (pat, 0) - 1; i > 0; --i)
4970 : {
4971 14905 : set = XVECEXP (pat, 0, i);
4972 14905 : if (GET_CODE (set) != SET)
4973 0 : continue;
4974 14905 : dest = SET_DEST (set);
4975 14905 : if (dest == stack_pointer_rtx)
4976 : break;
4977 : }
4978 : /* We'd better have found the stack pointer adjust. */
4979 14905 : if (i == 0)
4980 0 : return 0;
4981 : /* Fall through to process the extracted SET and DEST
4982 : as if it was a standalone insn. */
4983 : }
4984 1649873 : else if (GET_CODE (pat) == SET)
4985 : set = pat;
4986 256871 : else if ((set = single_set (insn)) != NULL)
4987 : ;
4988 46720 : else if (GET_CODE (pat) == PARALLEL)
4989 : {
4990 : /* ??? Some older ports use a parallel with a stack adjust
4991 : and a store for a PUSH_ROUNDING pattern, rather than a
4992 : PRE/POST_MODIFY rtx. Don't force them to update yet... */
4993 : /* ??? See h8300 and m68k, pushqi1. */
4994 0 : for (i = XVECLEN (pat, 0) - 1; i >= 0; --i)
4995 : {
4996 0 : set = XVECEXP (pat, 0, i);
4997 0 : if (GET_CODE (set) != SET)
4998 0 : continue;
4999 0 : dest = SET_DEST (set);
5000 0 : if (dest == stack_pointer_rtx)
5001 : break;
5002 :
5003 : /* We do not expect an auto-inc of the sp in the parallel. */
5004 0 : gcc_checking_assert (mem_autoinc_base (dest) != stack_pointer_rtx);
5005 0 : gcc_checking_assert (mem_autoinc_base (SET_SRC (set))
5006 : != stack_pointer_rtx);
5007 : }
5008 0 : if (i < 0)
5009 0 : return 0;
5010 : }
5011 : else
5012 46720 : return 0;
5013 :
5014 1618058 : dest = SET_DEST (set);
5015 :
5016 : /* Look for direct modifications of the stack pointer. */
5017 1618058 : if (REG_P (dest) && REGNO (dest) == STACK_POINTER_REGNUM)
5018 : {
5019 : /* Look for a trivial adjustment, otherwise assume nothing. */
5020 : /* Note that the SPU restore_stack_block pattern refers to
5021 : the stack pointer in V4SImode. Consider that non-trivial. */
5022 327257 : poly_int64 offset;
5023 327257 : if (SCALAR_INT_MODE_P (GET_MODE (dest))
5024 327257 : && strip_offset (SET_SRC (set), &offset) == stack_pointer_rtx)
5025 324829 : return offset;
5026 : /* ??? Reload can generate no-op moves, which will be cleaned
5027 : up later. Recognize it and continue searching. */
5028 2428 : else if (rtx_equal_p (dest, SET_SRC (set)))
5029 0 : return 0;
5030 : else
5031 2428 : return HOST_WIDE_INT_MIN;
5032 : }
5033 : else
5034 : {
5035 1290801 : rtx mem, addr;
5036 :
5037 : /* Otherwise only think about autoinc patterns. */
5038 2335934 : if (mem_autoinc_base (dest) == stack_pointer_rtx)
5039 : {
5040 924006 : mem = dest;
5041 1280927 : gcc_checking_assert (mem_autoinc_base (SET_SRC (set))
5042 : != stack_pointer_rtx);
5043 : }
5044 543792 : else if (mem_autoinc_base (SET_SRC (set)) == stack_pointer_rtx)
5045 : mem = SET_SRC (set);
5046 : else
5047 258333 : return 0;
5048 :
5049 1032468 : addr = XEXP (mem, 0);
5050 1032468 : switch (GET_CODE (addr))
5051 : {
5052 108462 : case PRE_INC:
5053 108462 : case POST_INC:
5054 216924 : return GET_MODE_SIZE (GET_MODE (mem));
5055 906009 : case PRE_DEC:
5056 906009 : case POST_DEC:
5057 1812018 : return -GET_MODE_SIZE (GET_MODE (mem));
5058 17997 : case PRE_MODIFY:
5059 17997 : case POST_MODIFY:
5060 17997 : addr = XEXP (addr, 1);
5061 17997 : gcc_assert (GET_CODE (addr) == PLUS);
5062 17997 : gcc_assert (XEXP (addr, 0) == stack_pointer_rtx);
5063 17997 : return rtx_to_poly_int64 (XEXP (addr, 1));
5064 0 : default:
5065 0 : gcc_unreachable ();
5066 : }
5067 : }
5068 : }
5069 :
5070 : poly_int64
5071 665707 : fixup_args_size_notes (rtx_insn *prev, rtx_insn *last,
5072 : poly_int64 end_args_size)
5073 : {
5074 665707 : poly_int64 args_size = end_args_size;
5075 665707 : bool saw_unknown = false;
5076 665707 : rtx_insn *insn;
5077 :
5078 1899093 : for (insn = last; insn != prev; insn = PREV_INSN (insn))
5079 : {
5080 1233386 : if (!NONDEBUG_INSN_P (insn))
5081 0 : continue;
5082 :
5083 : /* We might have existing REG_ARGS_SIZE notes, e.g. when pushing
5084 : a call argument containing a TLS address that itself requires
5085 : a call to __tls_get_addr. The handling of stack_pointer_delta
5086 : in emit_single_push_insn is supposed to ensure that any such
5087 : notes are already correct. */
5088 1233386 : rtx note = find_reg_note (insn, REG_ARGS_SIZE, NULL_RTX);
5089 1233386 : gcc_assert (!note || known_eq (args_size, get_args_size (note)));
5090 :
5091 1233386 : poly_int64 this_delta = find_args_size_adjust (insn);
5092 1233386 : if (known_eq (this_delta, 0))
5093 : {
5094 606349 : if (!CALL_P (insn)
5095 9 : || ACCUMULATE_OUTGOING_ARGS
5096 303188 : || find_reg_note (insn, REG_NORETURN, NULL_RTX) == NULL_RTX)
5097 303170 : continue;
5098 : }
5099 :
5100 930216 : gcc_assert (!saw_unknown);
5101 930216 : if (known_eq (this_delta, HOST_WIDE_INT_MIN))
5102 2406 : saw_unknown = true;
5103 :
5104 930216 : if (!note)
5105 930216 : add_args_size_note (insn, args_size);
5106 : if (STACK_GROWS_DOWNWARD)
5107 930216 : this_delta = -poly_uint64 (this_delta);
5108 :
5109 930216 : if (saw_unknown)
5110 : args_size = HOST_WIDE_INT_MIN;
5111 : else
5112 1233386 : args_size -= this_delta;
5113 : }
5114 :
5115 665707 : return args_size;
5116 : }
5117 :
5118 : #ifdef PUSH_ROUNDING
5119 : /* Emit single push insn. */
5120 :
5121 : static void
5122 1844075 : emit_single_push_insn_1 (machine_mode mode, rtx x, tree type)
5123 : {
5124 1844075 : rtx dest_addr;
5125 3688150 : poly_int64 rounded_size = PUSH_ROUNDING (GET_MODE_SIZE (mode));
5126 1844075 : rtx dest;
5127 1844075 : enum insn_code icode;
5128 :
5129 : /* If there is push pattern, use it. Otherwise try old way of throwing
5130 : MEM representing push operation to move expander. */
5131 1844075 : icode = optab_handler (push_optab, mode);
5132 1844075 : if (icode != CODE_FOR_nothing)
5133 : {
5134 0 : class expand_operand ops[1];
5135 :
5136 0 : create_input_operand (&ops[0], x, mode);
5137 0 : if (maybe_expand_insn (icode, 1, ops))
5138 0 : return;
5139 : }
5140 3688150 : if (known_eq (GET_MODE_SIZE (mode), rounded_size))
5141 3185830 : dest_addr = gen_rtx_fmt_e (STACK_PUSH_CODE, Pmode, stack_pointer_rtx);
5142 : /* If we are to pad downward, adjust the stack pointer first and
5143 : then store X into the stack location using an offset. This is
5144 : because emit_move_insn does not know how to pad; it does not have
5145 : access to type. */
5146 104143 : else if (targetm.calls.function_arg_padding (mode, type) == PAD_DOWNWARD)
5147 : {
5148 0 : emit_move_insn (stack_pointer_rtx,
5149 0 : expand_binop (Pmode,
5150 : STACK_GROWS_DOWNWARD ? sub_optab
5151 : : add_optab,
5152 : stack_pointer_rtx,
5153 0 : gen_int_mode (rounded_size, Pmode),
5154 : NULL_RTX, 0, OPTAB_LIB_WIDEN));
5155 :
5156 0 : poly_int64 offset = rounded_size - GET_MODE_SIZE (mode);
5157 0 : if (STACK_GROWS_DOWNWARD && STACK_PUSH_CODE == POST_DEC)
5158 : /* We have already decremented the stack pointer, so get the
5159 : previous value. */
5160 : offset += rounded_size;
5161 :
5162 0 : if (!STACK_GROWS_DOWNWARD && STACK_PUSH_CODE == POST_INC)
5163 : /* We have already incremented the stack pointer, so get the
5164 : previous value. */
5165 : offset -= rounded_size;
5166 :
5167 0 : dest_addr = plus_constant (Pmode, stack_pointer_rtx, offset);
5168 : }
5169 : else
5170 : {
5171 : if (STACK_GROWS_DOWNWARD)
5172 : /* ??? This seems wrong if STACK_PUSH_CODE == POST_DEC. */
5173 104697 : dest_addr = plus_constant (Pmode, stack_pointer_rtx, -rounded_size);
5174 : else
5175 : /* ??? This seems wrong if STACK_PUSH_CODE == POST_INC. */
5176 : dest_addr = plus_constant (Pmode, stack_pointer_rtx, rounded_size);
5177 :
5178 104697 : dest_addr = gen_rtx_PRE_MODIFY (Pmode, stack_pointer_rtx, dest_addr);
5179 : }
5180 :
5181 1844075 : dest = gen_rtx_MEM (mode, dest_addr);
5182 :
5183 1844075 : if (type != 0)
5184 : {
5185 1844028 : set_mem_attributes (dest, type, 1);
5186 :
5187 1844028 : if (cfun->tail_call_marked)
5188 : /* Function incoming arguments may overlap with sibling call
5189 : outgoing arguments and we cannot allow reordering of reads
5190 : from function arguments with stores to outgoing arguments
5191 : of sibling calls. */
5192 141935 : set_mem_alias_set (dest, 0);
5193 : }
5194 1844075 : emit_move_insn (dest, x);
5195 : }
5196 :
5197 : /* Emit and annotate a single push insn. */
5198 :
5199 : static void
5200 1844075 : emit_single_push_insn (machine_mode mode, rtx x, tree type)
5201 : {
5202 1844075 : poly_int64 delta, old_delta = stack_pointer_delta;
5203 1844075 : rtx_insn *prev = get_last_insn ();
5204 1844075 : rtx_insn *last;
5205 :
5206 1844075 : emit_single_push_insn_1 (mode, x, type);
5207 :
5208 : /* Adjust stack_pointer_delta to describe the situation after the push
5209 : we just performed. Note that we must do this after the push rather
5210 : than before the push in case calculating X needs pushes and pops of
5211 : its own (e.g. if calling __tls_get_addr). The REG_ARGS_SIZE notes
5212 : for such pushes and pops must not include the effect of the future
5213 : push of X. */
5214 3688150 : stack_pointer_delta += PUSH_ROUNDING (GET_MODE_SIZE (mode));
5215 :
5216 1844075 : last = get_last_insn ();
5217 :
5218 : /* Notice the common case where we emitted exactly one insn. */
5219 1844075 : if (PREV_INSN (last) == prev)
5220 : {
5221 1733638 : add_args_size_note (last, stack_pointer_delta);
5222 1733638 : return;
5223 : }
5224 :
5225 110437 : delta = fixup_args_size_notes (prev, last, stack_pointer_delta);
5226 110437 : gcc_assert (known_eq (delta, HOST_WIDE_INT_MIN)
5227 : || known_eq (delta, old_delta));
5228 : }
5229 : #endif
5230 :
5231 : /* If reading SIZE bytes from X will end up reading from
5232 : Y return the number of bytes that overlap. Return -1
5233 : if there is no overlap or -2 if we can't determine
5234 : (for example when X and Y have different base registers). */
5235 :
5236 : static int
5237 0 : memory_load_overlap (rtx x, rtx y, HOST_WIDE_INT size)
5238 : {
5239 0 : rtx tmp = plus_constant (Pmode, x, size);
5240 0 : rtx sub = simplify_gen_binary (MINUS, Pmode, tmp, y);
5241 :
5242 0 : if (!CONST_INT_P (sub))
5243 : return -2;
5244 :
5245 0 : HOST_WIDE_INT val = INTVAL (sub);
5246 :
5247 0 : return IN_RANGE (val, 1, size) ? val : -1;
5248 : }
5249 :
5250 : /* Generate code to push X onto the stack, assuming it has mode MODE and
5251 : type TYPE.
5252 : MODE is redundant except when X is a CONST_INT (since they don't
5253 : carry mode info).
5254 : SIZE is an rtx for the size of data to be copied (in bytes),
5255 : needed only if X is BLKmode.
5256 : Return true if successful. May return false if asked to push a
5257 : partial argument during a sibcall optimization (as specified by
5258 : SIBCALL_P) and the incoming and outgoing pointers cannot be shown
5259 : to not overlap.
5260 :
5261 : ALIGN (in bits) is maximum alignment we can assume.
5262 :
5263 : If PARTIAL and REG are both nonzero, then copy that many of the first
5264 : bytes of X into registers starting with REG, and push the rest of X.
5265 : The amount of space pushed is decreased by PARTIAL bytes.
5266 : REG must be a hard register in this case.
5267 : If REG is zero but PARTIAL is not, take any all others actions for an
5268 : argument partially in registers, but do not actually load any
5269 : registers.
5270 :
5271 : EXTRA is the amount in bytes of extra space to leave next to this arg.
5272 : This is ignored if an argument block has already been allocated.
5273 :
5274 : On a machine that lacks real push insns, ARGS_ADDR is the address of
5275 : the bottom of the argument block for this call. We use indexing off there
5276 : to store the arg. On machines with push insns, ARGS_ADDR is 0 when a
5277 : argument block has not been preallocated.
5278 :
5279 : ARGS_SO_FAR is the size of args previously pushed for this call.
5280 :
5281 : REG_PARM_STACK_SPACE is nonzero if functions require stack space
5282 : for arguments passed in registers. If nonzero, it will be the number
5283 : of bytes required. */
5284 :
5285 : bool
5286 2168397 : emit_push_insn (rtx x, machine_mode mode, tree type, rtx size,
5287 : unsigned int align, int partial, rtx reg, poly_int64 extra,
5288 : rtx args_addr, rtx args_so_far, int reg_parm_stack_space,
5289 : rtx alignment_pad, bool sibcall_p)
5290 : {
5291 2168397 : rtx xinner;
5292 2168397 : pad_direction stack_direction
5293 : = STACK_GROWS_DOWNWARD ? PAD_DOWNWARD : PAD_UPWARD;
5294 :
5295 : /* Decide where to pad the argument: PAD_DOWNWARD for below,
5296 : PAD_UPWARD for above, or PAD_NONE for don't pad it.
5297 : Default is below for small data on big-endian machines; else above. */
5298 2168397 : pad_direction where_pad = targetm.calls.function_arg_padding (mode, type);
5299 :
5300 : /* Invert direction if stack is post-decrement.
5301 : FIXME: why? */
5302 2168397 : if (STACK_PUSH_CODE == POST_DEC)
5303 : if (where_pad != PAD_NONE)
5304 : where_pad = (where_pad == PAD_DOWNWARD ? PAD_UPWARD : PAD_DOWNWARD);
5305 :
5306 2168397 : xinner = x;
5307 :
5308 2168397 : int nregs = partial / UNITS_PER_WORD;
5309 2168397 : rtx *tmp_regs = NULL;
5310 2168397 : int overlapping = 0;
5311 :
5312 2168397 : if (mode == BLKmode
5313 : || (STRICT_ALIGNMENT && align < GET_MODE_ALIGNMENT (mode)))
5314 : {
5315 : /* Copy a block into the stack, entirely or partially. */
5316 :
5317 268967 : rtx temp;
5318 268967 : int used;
5319 268967 : int offset;
5320 268967 : int skip;
5321 :
5322 268967 : offset = partial % (PARM_BOUNDARY / BITS_PER_UNIT);
5323 268967 : used = partial - offset;
5324 :
5325 268967 : if (mode != BLKmode)
5326 : {
5327 : /* A value is to be stored in an insufficiently aligned
5328 : stack slot; copy via a suitably aligned slot if
5329 : necessary. */
5330 : size = gen_int_mode (GET_MODE_SIZE (mode), Pmode);
5331 : if (!MEM_P (xinner))
5332 : {
5333 : temp = assign_temp (type, 1, 1);
5334 : emit_move_insn (temp, xinner);
5335 : xinner = temp;
5336 : }
5337 : }
5338 :
5339 268967 : gcc_assert (size);
5340 :
5341 : /* USED is now the # of bytes we need not copy to the stack
5342 : because registers will take care of them. */
5343 :
5344 268967 : if (partial != 0)
5345 0 : xinner = adjust_address (xinner, BLKmode, used);
5346 :
5347 : /* If the partial register-part of the arg counts in its stack size,
5348 : skip the part of stack space corresponding to the registers.
5349 : Otherwise, start copying to the beginning of the stack space,
5350 : by setting SKIP to 0. */
5351 268967 : skip = (reg_parm_stack_space == 0) ? 0 : used;
5352 :
5353 : #ifdef PUSH_ROUNDING
5354 : /* NB: Let the backend known the number of bytes to push and
5355 : decide if push insns should be generated. */
5356 268967 : unsigned int push_size;
5357 268967 : if (CONST_INT_P (size))
5358 268967 : push_size = INTVAL (size);
5359 : else
5360 : push_size = 0;
5361 :
5362 : /* Do it with several push insns if that doesn't take lots of insns
5363 : and if there is no difficulty with push insns that skip bytes
5364 : on the stack for alignment purposes. */
5365 268967 : if (args_addr == 0
5366 268913 : && targetm.calls.push_argument (push_size)
5367 3780 : && CONST_INT_P (size)
5368 3780 : && skip == 0
5369 3780 : && MEM_ALIGN (xinner) >= align
5370 2938 : && can_move_by_pieces ((unsigned) INTVAL (size) - used, align)
5371 : /* Here we avoid the case of a structure whose weak alignment
5372 : forces many pushes of a small amount of data,
5373 : and such small pushes do rounding that causes trouble. */
5374 2938 : && ((!targetm.slow_unaligned_access (word_mode, align))
5375 0 : || align >= BIGGEST_ALIGNMENT
5376 0 : || known_eq (PUSH_ROUNDING (align / BITS_PER_UNIT),
5377 : align / BITS_PER_UNIT))
5378 271905 : && known_eq (PUSH_ROUNDING (INTVAL (size)), INTVAL (size)))
5379 : {
5380 : /* Push padding now if padding above and stack grows down,
5381 : or if padding below and stack grows up.
5382 : But if space already allocated, this has already been done. */
5383 45 : if (maybe_ne (extra, 0)
5384 : && args_addr == 0
5385 0 : && where_pad != PAD_NONE
5386 45 : && where_pad != stack_direction)
5387 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5388 :
5389 45 : move_by_pieces (NULL, xinner, INTVAL (size) - used, align,
5390 : RETURN_BEGIN);
5391 : }
5392 : else
5393 : #endif /* PUSH_ROUNDING */
5394 : {
5395 268922 : rtx target;
5396 :
5397 : /* Otherwise make space on the stack and copy the data
5398 : to the address of that space. */
5399 :
5400 : /* Deduct words put into registers from the size we must copy. */
5401 268922 : if (partial != 0)
5402 : {
5403 0 : if (CONST_INT_P (size))
5404 0 : size = GEN_INT (INTVAL (size) - used);
5405 : else
5406 0 : size = expand_binop (GET_MODE (size), sub_optab, size,
5407 0 : gen_int_mode (used, GET_MODE (size)),
5408 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
5409 : }
5410 :
5411 : /* Get the address of the stack space.
5412 : In this case, we do not deal with EXTRA separately.
5413 : A single stack adjust will do. */
5414 268922 : poly_int64 const_args_so_far;
5415 268922 : if (! args_addr)
5416 : {
5417 268868 : temp = push_block (size, extra, where_pad == PAD_DOWNWARD);
5418 268868 : extra = 0;
5419 : }
5420 54 : else if (poly_int_rtx_p (args_so_far, &const_args_so_far))
5421 72 : temp = memory_address (BLKmode,
5422 : plus_constant (Pmode, args_addr,
5423 : skip + const_args_so_far));
5424 : else
5425 0 : temp = memory_address (BLKmode,
5426 : plus_constant (Pmode,
5427 : gen_rtx_PLUS (Pmode,
5428 : args_addr,
5429 : args_so_far),
5430 : skip));
5431 :
5432 268922 : if (!ACCUMULATE_OUTGOING_ARGS)
5433 : {
5434 : /* If the source is referenced relative to the stack pointer,
5435 : copy it to another register to stabilize it. We do not need
5436 : to do this if we know that we won't be changing sp. */
5437 :
5438 268922 : if (reg_mentioned_p (virtual_stack_dynamic_rtx, temp)
5439 268922 : || reg_mentioned_p (virtual_outgoing_args_rtx, temp))
5440 268868 : temp = copy_to_reg (temp);
5441 : }
5442 :
5443 268922 : target = gen_rtx_MEM (BLKmode, temp);
5444 :
5445 : /* We do *not* set_mem_attributes here, because incoming arguments
5446 : may overlap with sibling call outgoing arguments and we cannot
5447 : allow reordering of reads from function arguments with stores
5448 : to outgoing arguments of sibling calls. We do, however, want
5449 : to record the alignment of the stack slot. */
5450 : /* ALIGN may well be better aligned than TYPE, e.g. due to
5451 : PARM_BOUNDARY. Assume the caller isn't lying. */
5452 268922 : set_mem_align (target, align);
5453 :
5454 : /* If part should go in registers and pushing to that part would
5455 : overwrite some of the values that need to go into regs, load the
5456 : overlapping values into temporary pseudos to be moved into the hard
5457 : regs at the end after the stack pushing has completed.
5458 : We cannot load them directly into the hard regs here because
5459 : they can be clobbered by the block move expansions.
5460 : See PR 65358. */
5461 :
5462 268922 : if (partial > 0 && reg != 0 && mode == BLKmode
5463 0 : && GET_CODE (reg) != PARALLEL)
5464 : {
5465 0 : overlapping = memory_load_overlap (XEXP (x, 0), temp, partial);
5466 0 : if (overlapping > 0)
5467 : {
5468 0 : gcc_assert (overlapping % UNITS_PER_WORD == 0);
5469 0 : overlapping /= UNITS_PER_WORD;
5470 :
5471 0 : tmp_regs = XALLOCAVEC (rtx, overlapping);
5472 :
5473 0 : for (int i = 0; i < overlapping; i++)
5474 0 : tmp_regs[i] = gen_reg_rtx (word_mode);
5475 :
5476 0 : for (int i = 0; i < overlapping; i++)
5477 0 : emit_move_insn (tmp_regs[i],
5478 0 : operand_subword_force (target, i, mode));
5479 : }
5480 0 : else if (overlapping == -1)
5481 : overlapping = 0;
5482 : /* Could not determine whether there is overlap.
5483 : Fail the sibcall. */
5484 : else
5485 : {
5486 0 : overlapping = 0;
5487 0 : if (sibcall_p)
5488 0 : return false;
5489 : }
5490 : }
5491 :
5492 : /* If source is a constant VAR_DECL with a simple constructor,
5493 : store the constructor to the stack instead of moving it. */
5494 268922 : const_tree decl;
5495 268922 : HOST_WIDE_INT sz;
5496 268922 : if (partial == 0
5497 268922 : && MEM_P (xinner)
5498 268922 : && SYMBOL_REF_P (XEXP (xinner, 0))
5499 82544 : && (decl = SYMBOL_REF_DECL (XEXP (xinner, 0))) != NULL_TREE
5500 82544 : && VAR_P (decl)
5501 82544 : && TREE_READONLY (decl)
5502 5174 : && !TREE_SIDE_EFFECTS (decl)
5503 5174 : && immediate_const_ctor_p (DECL_INITIAL (decl), 2)
5504 9 : && (sz = int_expr_size (DECL_INITIAL (decl))) > 0
5505 9 : && CONST_INT_P (size)
5506 268931 : && INTVAL (size) == sz)
5507 3 : store_constructor (DECL_INITIAL (decl), target, 0, sz, false);
5508 : else
5509 268919 : emit_block_move (target, xinner, size, BLOCK_OP_CALL_PARM);
5510 : }
5511 : }
5512 1899430 : else if (partial > 0)
5513 : {
5514 : /* Scalar partly in registers. This case is only supported
5515 : for fixed-wdth modes. */
5516 0 : int num_words = GET_MODE_SIZE (mode).to_constant ();
5517 0 : num_words /= UNITS_PER_WORD;
5518 0 : int i;
5519 0 : int not_stack;
5520 : /* # bytes of start of argument
5521 : that we must make space for but need not store. */
5522 0 : int offset = partial % (PARM_BOUNDARY / BITS_PER_UNIT);
5523 0 : int args_offset = INTVAL (args_so_far);
5524 0 : int skip;
5525 :
5526 : /* Push padding now if padding above and stack grows down,
5527 : or if padding below and stack grows up.
5528 : But if space already allocated, this has already been done. */
5529 0 : if (maybe_ne (extra, 0)
5530 0 : && args_addr == 0
5531 0 : && where_pad != PAD_NONE
5532 0 : && where_pad != stack_direction)
5533 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5534 :
5535 : /* If we make space by pushing it, we might as well push
5536 : the real data. Otherwise, we can leave OFFSET nonzero
5537 : and leave the space uninitialized. */
5538 0 : if (args_addr == 0)
5539 0 : offset = 0;
5540 :
5541 : /* Now NOT_STACK gets the number of words that we don't need to
5542 : allocate on the stack. Convert OFFSET to words too. */
5543 0 : not_stack = (partial - offset) / UNITS_PER_WORD;
5544 0 : offset /= UNITS_PER_WORD;
5545 :
5546 : /* If the partial register-part of the arg counts in its stack size,
5547 : skip the part of stack space corresponding to the registers.
5548 : Otherwise, start copying to the beginning of the stack space,
5549 : by setting SKIP to 0. */
5550 0 : skip = (reg_parm_stack_space == 0) ? 0 : not_stack;
5551 :
5552 0 : if (CONSTANT_P (x) && !targetm.legitimate_constant_p (mode, x))
5553 0 : x = validize_mem (force_const_mem (mode, x));
5554 :
5555 : /* If X is a hard register in a non-integer mode, copy it into a pseudo;
5556 : SUBREGs of such registers are not allowed. */
5557 0 : if ((REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER
5558 0 : && GET_MODE_CLASS (GET_MODE (x)) != MODE_INT))
5559 0 : x = copy_to_reg (x);
5560 :
5561 : /* Loop over all the words allocated on the stack for this arg. */
5562 : /* We can do it by words, because any scalar bigger than a word
5563 : has a size a multiple of a word. */
5564 0 : tree word_mode_type = lang_hooks.types.type_for_mode (word_mode, 1);
5565 0 : for (i = num_words - 1; i >= not_stack; i--)
5566 0 : if (i >= not_stack + offset)
5567 0 : if (!emit_push_insn (operand_subword_force (x, i, mode),
5568 : word_mode, word_mode_type, NULL_RTX, align, 0,
5569 : NULL_RTX, 0, args_addr,
5570 0 : GEN_INT (args_offset + ((i - not_stack + skip)
5571 : * UNITS_PER_WORD)),
5572 : reg_parm_stack_space, alignment_pad, sibcall_p))
5573 0 : return false;
5574 : }
5575 : else
5576 : {
5577 1899430 : rtx addr;
5578 1899430 : rtx dest;
5579 :
5580 : /* Push padding now if padding above and stack grows down,
5581 : or if padding below and stack grows up.
5582 : But if space already allocated, this has already been done. */
5583 1899430 : if (maybe_ne (extra, 0)
5584 0 : && args_addr == 0
5585 0 : && where_pad != PAD_NONE
5586 1899430 : && where_pad != stack_direction)
5587 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5588 :
5589 : #ifdef PUSH_ROUNDING
5590 1899430 : if (args_addr == 0 && targetm.calls.push_argument (0))
5591 1844028 : emit_single_push_insn (mode, x, type);
5592 : else
5593 : #endif
5594 : {
5595 63684 : addr = simplify_gen_binary (PLUS, Pmode, args_addr, args_so_far);
5596 55402 : dest = gen_rtx_MEM (mode, memory_address (mode, addr));
5597 :
5598 : /* We do *not* set_mem_attributes here, because incoming arguments
5599 : may overlap with sibling call outgoing arguments and we cannot
5600 : allow reordering of reads from function arguments with stores
5601 : to outgoing arguments of sibling calls. We do, however, want
5602 : to record the alignment of the stack slot. */
5603 : /* ALIGN may well be better aligned than TYPE, e.g. due to
5604 : PARM_BOUNDARY. Assume the caller isn't lying. */
5605 55402 : set_mem_align (dest, align);
5606 :
5607 55402 : emit_move_insn (dest, x);
5608 : }
5609 : }
5610 :
5611 : /* Move the partial arguments into the registers and any overlapping
5612 : values that we moved into the pseudos in tmp_regs. */
5613 2168397 : if (partial > 0 && reg != 0)
5614 : {
5615 : /* Handle calls that pass values in multiple non-contiguous locations.
5616 : The Irix 6 ABI has examples of this. */
5617 0 : if (GET_CODE (reg) == PARALLEL)
5618 0 : emit_group_load (reg, x, type, -1);
5619 : else
5620 : {
5621 0 : gcc_assert (partial % UNITS_PER_WORD == 0);
5622 0 : move_block_to_reg (REGNO (reg), x, nregs - overlapping, mode);
5623 :
5624 0 : for (int i = 0; i < overlapping; i++)
5625 0 : emit_move_insn (gen_rtx_REG (word_mode, REGNO (reg)
5626 0 : + nregs - overlapping + i),
5627 0 : tmp_regs[i]);
5628 :
5629 : }
5630 : }
5631 :
5632 2168397 : if (maybe_ne (extra, 0) && args_addr == 0 && where_pad == stack_direction)
5633 0 : anti_adjust_stack (gen_int_mode (extra, Pmode));
5634 :
5635 2168397 : if (alignment_pad && args_addr == 0)
5636 2112941 : anti_adjust_stack (alignment_pad);
5637 :
5638 : return true;
5639 : }
5640 :
5641 : /* Return X if X can be used as a subtarget in a sequence of arithmetic
5642 : operations. */
5643 :
5644 : static rtx
5645 199411437 : get_subtarget (rtx x)
5646 : {
5647 199411437 : return (optimize
5648 52125612 : || x == 0
5649 : /* Only registers can be subtargets. */
5650 17452885 : || !REG_P (x)
5651 : /* Don't use hard regs to avoid extending their life. */
5652 11849353 : || REGNO (x) < FIRST_PSEUDO_REGISTER
5653 199411437 : ? 0 : x);
5654 : }
5655 :
5656 : /* A subroutine of expand_assignment. Optimize FIELD op= VAL, where
5657 : FIELD is a bitfield. Returns true if the optimization was successful,
5658 : and there's nothing else to do. */
5659 :
5660 : static bool
5661 4683273 : optimize_bitfield_assignment_op (poly_uint64 pbitsize,
5662 : poly_uint64 pbitpos,
5663 : poly_uint64 pbitregion_start,
5664 : poly_uint64 pbitregion_end,
5665 : machine_mode mode1, rtx str_rtx,
5666 : tree to, tree src, bool reverse)
5667 : {
5668 : /* str_mode is not guaranteed to be a scalar type. */
5669 4683273 : machine_mode str_mode = GET_MODE (str_rtx);
5670 4683273 : unsigned int str_bitsize;
5671 4683273 : tree op0, op1;
5672 4683273 : rtx value, result;
5673 4683273 : optab binop;
5674 4683273 : gimple *srcstmt;
5675 4683273 : enum tree_code code;
5676 :
5677 4683273 : unsigned HOST_WIDE_INT bitsize, bitpos, bitregion_start, bitregion_end;
5678 4683273 : if (mode1 != VOIDmode
5679 66788 : || !pbitsize.is_constant (&bitsize)
5680 66788 : || !pbitpos.is_constant (&bitpos)
5681 66788 : || !pbitregion_start.is_constant (&bitregion_start)
5682 66788 : || !pbitregion_end.is_constant (&bitregion_end)
5683 67441 : || bitsize >= BITS_PER_WORD
5684 66517 : || !GET_MODE_BITSIZE (str_mode).is_constant (&str_bitsize)
5685 67170 : || str_bitsize > BITS_PER_WORD
5686 61119 : || TREE_SIDE_EFFECTS (to)
5687 4744263 : || TREE_THIS_VOLATILE (to))
5688 : return false;
5689 :
5690 60990 : STRIP_NOPS (src);
5691 60990 : if (TREE_CODE (src) != SSA_NAME)
5692 : return false;
5693 19843 : if (TREE_CODE (TREE_TYPE (src)) != INTEGER_TYPE)
5694 : return false;
5695 :
5696 18269 : srcstmt = get_gimple_for_ssa_name (src);
5697 18269 : if (!srcstmt
5698 4549 : || !is_gimple_assign (srcstmt)
5699 22818 : || TREE_CODE_CLASS (gimple_assign_rhs_code (srcstmt)) != tcc_binary)
5700 : return false;
5701 :
5702 840 : code = gimple_assign_rhs_code (srcstmt);
5703 :
5704 840 : op0 = gimple_assign_rhs1 (srcstmt);
5705 :
5706 : /* If OP0 is an SSA_NAME, then we want to walk the use-def chain
5707 : to find its initialization. Hopefully the initialization will
5708 : be from a bitfield load. */
5709 840 : if (TREE_CODE (op0) == SSA_NAME)
5710 : {
5711 840 : gimple *op0stmt = get_gimple_for_ssa_name (op0);
5712 :
5713 : /* We want to eventually have OP0 be the same as TO, which
5714 : should be a bitfield. */
5715 840 : if (!op0stmt
5716 774 : || !is_gimple_assign (op0stmt)
5717 1614 : || gimple_assign_rhs_code (op0stmt) != TREE_CODE (to))
5718 : return false;
5719 542 : op0 = gimple_assign_rhs1 (op0stmt);
5720 : }
5721 :
5722 542 : op1 = gimple_assign_rhs2 (srcstmt);
5723 :
5724 542 : if (!operand_equal_p (to, op0, 0))
5725 : return false;
5726 :
5727 462 : if (MEM_P (str_rtx))
5728 : {
5729 421 : unsigned HOST_WIDE_INT offset1;
5730 :
5731 421 : if (str_bitsize == 0 || str_bitsize > BITS_PER_WORD)
5732 80 : str_bitsize = BITS_PER_WORD;
5733 :
5734 421 : scalar_int_mode best_mode;
5735 421 : if (!get_best_mode (bitsize, bitpos, bitregion_start, bitregion_end,
5736 421 : MEM_ALIGN (str_rtx), str_bitsize, false, &best_mode))
5737 0 : return false;
5738 421 : str_mode = best_mode;
5739 421 : str_bitsize = GET_MODE_BITSIZE (best_mode);
5740 :
5741 421 : offset1 = bitpos;
5742 421 : bitpos %= str_bitsize;
5743 421 : offset1 = (offset1 - bitpos) / BITS_PER_UNIT;
5744 421 : str_rtx = adjust_address (str_rtx, str_mode, offset1);
5745 : }
5746 41 : else if (!REG_P (str_rtx) && GET_CODE (str_rtx) != SUBREG)
5747 : return false;
5748 :
5749 : /* If the bit field covers the whole REG/MEM, store_field
5750 : will likely generate better code. */
5751 462 : if (bitsize >= str_bitsize)
5752 : return false;
5753 :
5754 : /* We can't handle fields split across multiple entities. */
5755 462 : if (bitpos + bitsize > str_bitsize)
5756 : return false;
5757 :
5758 462 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
5759 0 : bitpos = str_bitsize - bitpos - bitsize;
5760 :
5761 462 : switch (code)
5762 : {
5763 141 : case PLUS_EXPR:
5764 141 : case MINUS_EXPR:
5765 : /* For now, just optimize the case of the topmost bitfield
5766 : where we don't need to do any masking and also
5767 : 1 bit bitfields where xor can be used.
5768 : We might win by one instruction for the other bitfields
5769 : too if insv/extv instructions aren't used, so that
5770 : can be added later. */
5771 141 : if ((reverse || bitpos + bitsize != str_bitsize)
5772 97 : && (bitsize != 1 || TREE_CODE (op1) != INTEGER_CST))
5773 : break;
5774 :
5775 70 : value = expand_expr (op1, NULL_RTX, str_mode, EXPAND_NORMAL);
5776 70 : value = convert_modes (str_mode,
5777 70 : TYPE_MODE (TREE_TYPE (op1)), value,
5778 70 : TYPE_UNSIGNED (TREE_TYPE (op1)));
5779 :
5780 : /* We may be accessing data outside the field, which means
5781 : we can alias adjacent data. */
5782 70 : if (MEM_P (str_rtx))
5783 : {
5784 70 : str_rtx = shallow_copy_rtx (str_rtx);
5785 70 : set_mem_alias_set (str_rtx, 0);
5786 70 : set_mem_expr (str_rtx, 0);
5787 : }
5788 :
5789 70 : if (bitsize == 1 && (reverse || bitpos + bitsize != str_bitsize))
5790 : {
5791 26 : value = expand_and (str_mode, value, const1_rtx, NULL);
5792 26 : binop = xor_optab;
5793 : }
5794 : else
5795 44 : binop = code == PLUS_EXPR ? add_optab : sub_optab;
5796 :
5797 70 : value = expand_shift (LSHIFT_EXPR, str_mode, value, bitpos, NULL_RTX, 1);
5798 70 : if (reverse)
5799 0 : value = flip_storage_order (str_mode, value);
5800 70 : result = expand_binop (str_mode, binop, str_rtx,
5801 : value, str_rtx, 1, OPTAB_WIDEN);
5802 70 : if (result != str_rtx)
5803 0 : emit_move_insn (str_rtx, result);
5804 : return true;
5805 :
5806 253 : case BIT_IOR_EXPR:
5807 253 : case BIT_XOR_EXPR:
5808 253 : if (TREE_CODE (op1) != INTEGER_CST)
5809 : break;
5810 97 : value = expand_expr (op1, NULL_RTX, str_mode, EXPAND_NORMAL);
5811 97 : value = convert_modes (str_mode,
5812 97 : TYPE_MODE (TREE_TYPE (op1)), value,
5813 97 : TYPE_UNSIGNED (TREE_TYPE (op1)));
5814 :
5815 : /* We may be accessing data outside the field, which means
5816 : we can alias adjacent data. */
5817 97 : if (MEM_P (str_rtx))
5818 : {
5819 61 : str_rtx = shallow_copy_rtx (str_rtx);
5820 61 : set_mem_alias_set (str_rtx, 0);
5821 61 : set_mem_expr (str_rtx, 0);
5822 : }
5823 :
5824 97 : binop = code == BIT_IOR_EXPR ? ior_optab : xor_optab;
5825 97 : if (bitpos + bitsize != str_bitsize)
5826 : {
5827 55 : rtx mask = gen_int_mode ((HOST_WIDE_INT_1U << bitsize) - 1,
5828 : str_mode);
5829 55 : value = expand_and (str_mode, value, mask, NULL_RTX);
5830 : }
5831 97 : value = expand_shift (LSHIFT_EXPR, str_mode, value, bitpos, NULL_RTX, 1);
5832 97 : if (reverse)
5833 0 : value = flip_storage_order (str_mode, value);
5834 97 : result = expand_binop (str_mode, binop, str_rtx,
5835 : value, str_rtx, 1, OPTAB_WIDEN);
5836 97 : if (result != str_rtx)
5837 0 : emit_move_insn (str_rtx, result);
5838 : return true;
5839 :
5840 : default:
5841 : break;
5842 : }
5843 :
5844 : return false;
5845 : }
5846 :
5847 : /* In the C++ memory model, consecutive bit fields in a structure are
5848 : considered one memory location.
5849 :
5850 : Given a COMPONENT_REF EXP at position (BITPOS, OFFSET), this function
5851 : returns the bit range of consecutive bits in which this COMPONENT_REF
5852 : belongs. The values are returned in *BITSTART and *BITEND. *BITPOS
5853 : and *OFFSET may be adjusted in the process.
5854 :
5855 : If the access does not need to be restricted, 0 is returned in both
5856 : *BITSTART and *BITEND. */
5857 :
5858 : void
5859 823312 : get_bit_range (poly_uint64 *bitstart, poly_uint64 *bitend, tree exp,
5860 : poly_int64 *bitpos, tree *offset)
5861 : {
5862 823312 : poly_int64 bitoffset;
5863 823312 : tree field, repr;
5864 :
5865 823312 : gcc_assert (TREE_CODE (exp) == COMPONENT_REF);
5866 :
5867 823312 : field = TREE_OPERAND (exp, 1);
5868 823312 : repr = DECL_BIT_FIELD_REPRESENTATIVE (field);
5869 : /* If we do not have a DECL_BIT_FIELD_REPRESENTATIVE there is no
5870 : need to limit the range we can access. */
5871 823312 : if (!repr)
5872 : {
5873 65 : *bitstart = *bitend = 0;
5874 65 : return;
5875 : }
5876 :
5877 : /* If we have a DECL_BIT_FIELD_REPRESENTATIVE but the enclosing record is
5878 : part of a larger bit field, then the representative does not serve any
5879 : useful purpose. This can occur in Ada. */
5880 823247 : if (handled_component_p (TREE_OPERAND (exp, 0)))
5881 : {
5882 419752 : machine_mode rmode;
5883 419752 : poly_int64 rbitsize, rbitpos;
5884 419752 : tree roffset;
5885 419752 : int unsignedp, reversep, volatilep = 0;
5886 419752 : get_inner_reference (TREE_OPERAND (exp, 0), &rbitsize, &rbitpos,
5887 : &roffset, &rmode, &unsignedp, &reversep,
5888 : &volatilep);
5889 839504 : if (!multiple_p (rbitpos, BITS_PER_UNIT))
5890 : {
5891 0 : *bitstart = *bitend = 0;
5892 0 : return;
5893 : }
5894 : }
5895 :
5896 : /* Compute the adjustment to bitpos from the offset of the field
5897 : relative to the representative. DECL_FIELD_OFFSET of field and
5898 : repr are the same by construction if they are not constants,
5899 : see finish_bitfield_layout. */
5900 823247 : poly_uint64 field_offset, repr_offset;
5901 823247 : if (poly_int_tree_p (DECL_FIELD_OFFSET (field), &field_offset)
5902 1646489 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
5903 823242 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
5904 : else
5905 : bitoffset = 0;
5906 823247 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (field))
5907 823247 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
5908 :
5909 : /* If the adjustment is larger than bitpos, we would have a negative bit
5910 : position for the lower bound and this may wreak havoc later. Adjust
5911 : offset and bitpos to make the lower bound non-negative in that case. */
5912 823247 : if (maybe_gt (bitoffset, *bitpos))
5913 : {
5914 9 : poly_int64 adjust_bits = upper_bound (bitoffset, *bitpos) - *bitpos;
5915 9 : poly_int64 adjust_bytes = exact_div (adjust_bits, BITS_PER_UNIT);
5916 :
5917 9 : *bitpos += adjust_bits;
5918 9 : if (*offset == NULL_TREE)
5919 5 : *offset = size_int (-adjust_bytes);
5920 : else
5921 4 : *offset = size_binop (MINUS_EXPR, *offset, size_int (adjust_bytes));
5922 9 : *bitstart = 0;
5923 : }
5924 : else
5925 823238 : *bitstart = *bitpos - bitoffset;
5926 :
5927 823247 : *bitend = *bitstart + tree_to_poly_uint64 (DECL_SIZE (repr)) - 1;
5928 : }
5929 :
5930 : /* Returns true if BASE is a DECL that does not reside in memory and
5931 : has non-BLKmode. DECL_RTL must not be a MEM; if
5932 : DECL_RTL was not set yet, return false. */
5933 :
5934 : bool
5935 5398945 : non_mem_decl_p (tree base)
5936 : {
5937 5398945 : if (!DECL_P (base)
5938 5398578 : || TREE_ADDRESSABLE (base)
5939 7827119 : || DECL_MODE (base) == BLKmode)
5940 : return false;
5941 :
5942 1570646 : if (!DECL_RTL_SET_P (base))
5943 : return false;
5944 :
5945 1452308 : return (!MEM_P (DECL_RTL (base)));
5946 : }
5947 :
5948 : /* Returns true if REF refers to an object that does not
5949 : reside in memory and has non-BLKmode. */
5950 :
5951 : bool
5952 11741957 : mem_ref_refers_to_non_mem_p (tree ref)
5953 : {
5954 11741957 : tree base;
5955 :
5956 11741957 : if (TREE_CODE (ref) == MEM_REF
5957 11741957 : || TREE_CODE (ref) == TARGET_MEM_REF)
5958 : {
5959 10240155 : tree addr = TREE_OPERAND (ref, 0);
5960 :
5961 10240155 : if (TREE_CODE (addr) != ADDR_EXPR)
5962 : return false;
5963 :
5964 3878402 : base = TREE_OPERAND (addr, 0);
5965 : }
5966 : else
5967 : base = ref;
5968 :
5969 5380204 : return non_mem_decl_p (base);
5970 : }
5971 :
5972 : /* Helper function of expand_assignment. Check if storing field of
5973 : size BITSIZE at position BITPOS overlaps with the most significant
5974 : bit of TO_RTX, known to be SUBREG_PROMOTED_VAR_P.
5975 : Updating this field requires an explicit extension. */
5976 : static bool
5977 0 : store_field_updates_msb_p (poly_int64 bitpos, poly_int64 bitsize, rtx to_rtx)
5978 : {
5979 0 : return (BYTES_BIG_ENDIAN
5980 : ? maybe_le (bitpos, 0)
5981 0 : : maybe_ge (bitpos + bitsize, GET_MODE_BITSIZE (GET_MODE (to_rtx))));
5982 : }
5983 :
5984 : /* Expand an assignment that stores the value of FROM into TO. If NONTEMPORAL
5985 : is true, try generating a nontemporal store. */
5986 :
5987 : void
5988 18303991 : expand_assignment (tree to, tree from, bool nontemporal)
5989 : {
5990 18303991 : rtx to_rtx = 0;
5991 18303991 : rtx result;
5992 18303991 : machine_mode mode;
5993 18303991 : unsigned int align;
5994 18303991 : enum insn_code icode;
5995 :
5996 : /* Don't crash if the lhs of the assignment was erroneous. */
5997 18303991 : if (TREE_CODE (to) == ERROR_MARK)
5998 : {
5999 0 : expand_normal (from);
6000 0 : return;
6001 : }
6002 :
6003 : /* Optimize away no-op moves without side-effects. */
6004 18303991 : if (operand_equal_p (to, from, 0))
6005 : return;
6006 :
6007 : /* Handle misaligned stores. */
6008 18303837 : mode = TYPE_MODE (TREE_TYPE (to));
6009 18303837 : if ((TREE_CODE (to) == MEM_REF
6010 18303837 : || TREE_CODE (to) == TARGET_MEM_REF
6011 16126090 : || DECL_P (to))
6012 4012454 : && mode != BLKmode
6013 3549870 : && !mem_ref_refers_to_non_mem_p (to)
6014 6366674 : && ((align = get_object_alignment (to))
6015 3183337 : < GET_MODE_ALIGNMENT (mode))
6016 18732931 : && (((icode = optab_handler (movmisalign_optab, mode))
6017 : != CODE_FOR_nothing)
6018 96481 : || targetm.slow_unaligned_access (mode, align)))
6019 : {
6020 332613 : rtx reg, mem;
6021 :
6022 332613 : reg = expand_expr (from, NULL_RTX, VOIDmode, EXPAND_NORMAL);
6023 : /* Handle PARALLEL. */
6024 332613 : reg = maybe_emit_group_store (reg, TREE_TYPE (from));
6025 332613 : reg = force_not_mem (reg);
6026 332613 : mem = expand_expr (to, NULL_RTX, VOIDmode, EXPAND_WRITE);
6027 332613 : if (TREE_CODE (to) == MEM_REF && REF_REVERSE_STORAGE_ORDER (to))
6028 0 : reg = flip_storage_order (mode, reg);
6029 :
6030 332613 : if (icode != CODE_FOR_nothing)
6031 : {
6032 332613 : class expand_operand ops[2];
6033 :
6034 332613 : create_fixed_operand (&ops[0], mem);
6035 332613 : create_input_operand (&ops[1], reg, mode);
6036 : /* The movmisalign<mode> pattern cannot fail, else the assignment
6037 : would silently be omitted. */
6038 332613 : expand_insn (icode, 2, ops);
6039 : }
6040 : else
6041 0 : store_bit_field (mem, GET_MODE_BITSIZE (mode), 0, 0, 0, mode, reg,
6042 : false, false);
6043 332613 : return;
6044 : }
6045 :
6046 : /* Assignment of a structure component needs special treatment
6047 : if the structure component's rtx is not simply a MEM.
6048 : Assignment of an array element at a constant index, and assignment of
6049 : an array element in an unaligned packed structure field, has the same
6050 : problem. Same for (partially) storing into a non-memory object. */
6051 17971224 : if (handled_component_p (to)
6052 13515511 : || (TREE_CODE (to) == MEM_REF
6053 1610574 : && (REF_REVERSE_STORAGE_ORDER (to)
6054 1610494 : || mem_ref_refers_to_non_mem_p (to)))
6055 13407884 : || TREE_CODE (TREE_TYPE (to)) == ARRAY_TYPE)
6056 : {
6057 4683402 : machine_mode mode1;
6058 4683402 : poly_int64 bitsize, bitpos;
6059 4683402 : poly_uint64 bitregion_start = 0;
6060 4683402 : poly_uint64 bitregion_end = 0;
6061 4683402 : tree offset;
6062 4683402 : int unsignedp, reversep, volatilep = 0;
6063 4683402 : tree tem;
6064 :
6065 4683402 : push_temp_slots ();
6066 4683402 : tem = get_inner_reference (to, &bitsize, &bitpos, &offset, &mode1,
6067 : &unsignedp, &reversep, &volatilep);
6068 :
6069 : /* Make sure bitpos is not negative, it can wreak havoc later. */
6070 4683402 : if (maybe_lt (bitpos, 0))
6071 : {
6072 159 : gcc_assert (offset == NULL_TREE);
6073 159 : offset = size_int (bits_to_bytes_round_down (bitpos));
6074 159 : bitpos = num_trailing_bits (bitpos);
6075 : }
6076 :
6077 4683402 : if (TREE_CODE (to) == COMPONENT_REF
6078 4683402 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (to, 1)))
6079 85333 : get_bit_range (&bitregion_start, &bitregion_end, to, &bitpos, &offset);
6080 : /* The C++ memory model naturally applies to byte-aligned fields.
6081 : However, if we do not have a DECL_BIT_FIELD_TYPE but BITPOS or
6082 : BITSIZE are not byte-aligned, there is no need to limit the range
6083 : we can access. This can occur with packed structures in Ada. */
6084 4598069 : else if (maybe_gt (bitsize, 0)
6085 4598054 : && multiple_p (bitsize, BITS_PER_UNIT)
6086 9195856 : && multiple_p (bitpos, BITS_PER_UNIT))
6087 : {
6088 4597787 : bitregion_start = bitpos;
6089 4597787 : bitregion_end = bitpos + bitsize - 1;
6090 : }
6091 :
6092 4683402 : to_rtx = expand_expr (tem, NULL_RTX, VOIDmode, EXPAND_WRITE);
6093 :
6094 : /* If the field has a mode, we want to access it in the
6095 : field's mode, not the computed mode.
6096 : If a MEM has VOIDmode (external with incomplete type),
6097 : use BLKmode for it instead. */
6098 4683402 : if (MEM_P (to_rtx))
6099 : {
6100 4011707 : if (mode1 != VOIDmode)
6101 3945656 : to_rtx = adjust_address (to_rtx, mode1, 0);
6102 66051 : else if (GET_MODE (to_rtx) == VOIDmode)
6103 0 : to_rtx = adjust_address (to_rtx, BLKmode, 0);
6104 : }
6105 :
6106 4683402 : rtx stemp = NULL_RTX, old_to_rtx = NULL_RTX;
6107 4683402 : if (offset != 0)
6108 : {
6109 177234 : machine_mode address_mode;
6110 177234 : rtx offset_rtx;
6111 :
6112 177234 : if (!MEM_P (to_rtx))
6113 : {
6114 : /* We can get constant negative offsets into arrays with broken
6115 : user code. Translate this to a trap instead of ICEing. */
6116 4 : if (TREE_CODE (offset) == INTEGER_CST)
6117 : {
6118 1 : expand_builtin_trap ();
6119 1 : to_rtx = gen_rtx_MEM (BLKmode, const0_rtx);
6120 : }
6121 : /* Else spill for variable offset to the destination. We expect
6122 : to run into this only for hard registers. */
6123 : else
6124 : {
6125 3 : gcc_assert (VAR_P (tem) && DECL_HARD_REGISTER (tem));
6126 3 : stemp = assign_stack_temp (GET_MODE (to_rtx),
6127 6 : GET_MODE_SIZE (GET_MODE (to_rtx)));
6128 3 : emit_move_insn (stemp, to_rtx);
6129 3 : old_to_rtx = to_rtx;
6130 3 : to_rtx = stemp;
6131 : }
6132 : }
6133 :
6134 177234 : offset_rtx = expand_expr (offset, NULL_RTX, VOIDmode, EXPAND_SUM);
6135 177234 : address_mode = get_address_mode (to_rtx);
6136 177234 : if (GET_MODE (offset_rtx) != address_mode)
6137 : {
6138 : /* We cannot be sure that the RTL in offset_rtx is valid outside
6139 : of a memory address context, so force it into a register
6140 : before attempting to convert it to the desired mode. */
6141 183 : offset_rtx = force_operand (offset_rtx, NULL_RTX);
6142 183 : offset_rtx = convert_to_mode (address_mode, offset_rtx, 0);
6143 : }
6144 :
6145 : /* If we have an expression in OFFSET_RTX and a non-zero
6146 : byte offset in BITPOS, adding the byte offset before the
6147 : OFFSET_RTX results in better intermediate code, which makes
6148 : later rtl optimization passes perform better.
6149 :
6150 : We prefer intermediate code like this:
6151 :
6152 : r124:DI=r123:DI+0x18
6153 : [r124:DI]=r121:DI
6154 :
6155 : ... instead of ...
6156 :
6157 : r124:DI=r123:DI+0x10
6158 : [r124:DI+0x8]=r121:DI
6159 :
6160 : This is only done for aligned data values, as these can
6161 : be expected to result in single move instructions. */
6162 177234 : poly_int64 bytepos;
6163 177234 : if (mode1 != VOIDmode
6164 177126 : && maybe_ne (bitpos, 0)
6165 47734 : && maybe_gt (bitsize, 0)
6166 224968 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
6167 224844 : && multiple_p (bitpos, bitsize)
6168 95220 : && multiple_p (bitsize, GET_MODE_ALIGNMENT (mode1))
6169 224844 : && MEM_ALIGN (to_rtx) >= GET_MODE_ALIGNMENT (mode1))
6170 : {
6171 47363 : to_rtx = adjust_address (to_rtx, mode1, bytepos);
6172 47363 : bitregion_start = 0;
6173 47363 : if (known_ge (bitregion_end, poly_uint64 (bitpos)))
6174 47363 : bitregion_end -= bitpos;
6175 47363 : bitpos = 0;
6176 : }
6177 :
6178 177234 : to_rtx = offset_address (to_rtx, offset_rtx,
6179 : highest_pow2_factor_for_target (to,
6180 : offset));
6181 : }
6182 :
6183 : /* No action is needed if the target is not a memory and the field
6184 : lies completely outside that target. This can occur if the source
6185 : code contains an out-of-bounds access to a small array. */
6186 4683402 : if (!MEM_P (to_rtx)
6187 671691 : && GET_MODE (to_rtx) != BLKmode
6188 5355093 : && known_ge (bitpos, GET_MODE_PRECISION (GET_MODE (to_rtx))))
6189 : {
6190 3 : expand_normal (from);
6191 3 : result = NULL;
6192 : }
6193 : /* Handle expand_expr of a complex value returning a CONCAT. */
6194 4683399 : else if (GET_CODE (to_rtx) == CONCAT)
6195 : {
6196 120 : machine_mode to_mode = GET_MODE (to_rtx);
6197 120 : gcc_checking_assert (COMPLEX_MODE_P (to_mode));
6198 240 : poly_int64 mode_bitsize = GET_MODE_BITSIZE (to_mode);
6199 120 : unsigned short inner_bitsize = GET_MODE_UNIT_BITSIZE (to_mode);
6200 120 : if (TYPE_MODE (TREE_TYPE (from)) == to_mode
6201 0 : && known_eq (bitpos, 0)
6202 120 : && known_eq (bitsize, mode_bitsize))
6203 0 : result = store_expr (from, to_rtx, false, nontemporal, reversep);
6204 120 : else if (TYPE_MODE (TREE_TYPE (from)) == GET_MODE_INNER (to_mode)
6205 70 : && known_eq (bitsize, inner_bitsize)
6206 190 : && (known_eq (bitpos, 0)
6207 31 : || known_eq (bitpos, inner_bitsize)))
6208 70 : result = store_expr (from, XEXP (to_rtx, maybe_ne (bitpos, 0)),
6209 : false, nontemporal, reversep);
6210 50 : else if (known_le (bitpos + bitsize, inner_bitsize))
6211 5 : result = store_field (XEXP (to_rtx, 0), bitsize, bitpos,
6212 : bitregion_start, bitregion_end,
6213 : mode1, from, get_alias_set (to),
6214 : nontemporal, reversep);
6215 45 : else if (known_ge (bitpos, inner_bitsize))
6216 3 : result = store_field (XEXP (to_rtx, 1), bitsize,
6217 : bitpos - inner_bitsize,
6218 : bitregion_start, bitregion_end,
6219 : mode1, from, get_alias_set (to),
6220 : nontemporal, reversep);
6221 42 : else if (known_eq (bitpos, 0) && known_eq (bitsize, mode_bitsize))
6222 : {
6223 35 : result = expand_normal (from);
6224 35 : if (GET_CODE (result) == CONCAT)
6225 : {
6226 0 : to_mode = GET_MODE_INNER (to_mode);
6227 0 : machine_mode from_mode = GET_MODE_INNER (GET_MODE (result));
6228 0 : rtx from_real
6229 0 : = force_subreg (to_mode, XEXP (result, 0), from_mode, 0);
6230 0 : rtx from_imag
6231 0 : = force_subreg (to_mode, XEXP (result, 1), from_mode, 0);
6232 0 : if (!from_real || !from_imag)
6233 0 : goto concat_store_slow;
6234 0 : emit_move_insn (XEXP (to_rtx, 0), from_real);
6235 0 : emit_move_insn (XEXP (to_rtx, 1), from_imag);
6236 : }
6237 : else
6238 : {
6239 35 : machine_mode from_mode
6240 35 : = GET_MODE (result) == VOIDmode
6241 35 : ? TYPE_MODE (TREE_TYPE (from))
6242 35 : : GET_MODE (result);
6243 35 : rtx from_rtx;
6244 35 : if (MEM_P (result))
6245 1 : from_rtx = change_address (result, to_mode, NULL_RTX);
6246 : else
6247 34 : from_rtx = force_subreg (to_mode, result, from_mode, 0);
6248 35 : if (from_rtx)
6249 : {
6250 35 : emit_move_insn (XEXP (to_rtx, 0),
6251 : read_complex_part (from_rtx, false));
6252 35 : emit_move_insn (XEXP (to_rtx, 1),
6253 : read_complex_part (from_rtx, true));
6254 : }
6255 : else
6256 : {
6257 0 : to_mode = GET_MODE_INNER (to_mode);
6258 0 : rtx from_real
6259 0 : = force_subreg (to_mode, result, from_mode, 0);
6260 0 : rtx from_imag
6261 0 : = force_subreg (to_mode, result, from_mode,
6262 0 : GET_MODE_SIZE (to_mode));
6263 0 : if (!from_real || !from_imag)
6264 0 : goto concat_store_slow;
6265 0 : emit_move_insn (XEXP (to_rtx, 0), from_real);
6266 0 : emit_move_insn (XEXP (to_rtx, 1), from_imag);
6267 : }
6268 : }
6269 : }
6270 : else
6271 : {
6272 7 : concat_store_slow:;
6273 7 : rtx temp = assign_stack_temp (GET_MODE (to_rtx),
6274 14 : GET_MODE_SIZE (GET_MODE (to_rtx)));
6275 7 : write_complex_part (temp, XEXP (to_rtx, 0), false, true);
6276 7 : write_complex_part (temp, XEXP (to_rtx, 1), true, false);
6277 7 : result = store_field (temp, bitsize, bitpos,
6278 : bitregion_start, bitregion_end,
6279 : mode1, from, get_alias_set (to),
6280 : nontemporal, reversep);
6281 7 : emit_move_insn (XEXP (to_rtx, 0), read_complex_part (temp, false));
6282 7 : emit_move_insn (XEXP (to_rtx, 1), read_complex_part (temp, true));
6283 : }
6284 : }
6285 : /* For calls to functions returning variable length structures, if TO_RTX
6286 : is not a MEM, go through a MEM because we must not create temporaries
6287 : of the VLA type. */
6288 4683279 : else if (!MEM_P (to_rtx)
6289 671568 : && TREE_CODE (from) == CALL_EXPR
6290 235 : && COMPLETE_TYPE_P (TREE_TYPE (from))
6291 4683514 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (from))) != INTEGER_CST)
6292 : {
6293 6 : rtx temp = assign_stack_temp (GET_MODE (to_rtx),
6294 12 : GET_MODE_SIZE (GET_MODE (to_rtx)));
6295 6 : result = store_field (temp, bitsize, bitpos, bitregion_start,
6296 : bitregion_end, mode1, from, get_alias_set (to),
6297 : nontemporal, reversep);
6298 6 : emit_move_insn (to_rtx, temp);
6299 : }
6300 : else
6301 : {
6302 4683273 : if (MEM_P (to_rtx))
6303 : {
6304 : /* If the field is at offset zero, we could have been given the
6305 : DECL_RTX of the parent struct. Don't munge it. */
6306 4011711 : to_rtx = shallow_copy_rtx (to_rtx);
6307 4011711 : set_mem_attributes_minus_bitpos (to_rtx, to, 0, bitpos);
6308 4011711 : if (volatilep)
6309 8689 : MEM_VOLATILE_P (to_rtx) = 1;
6310 : }
6311 :
6312 4683273 : gcc_checking_assert (known_ge (bitpos, 0));
6313 4683273 : if (optimize_bitfield_assignment_op (bitsize, bitpos,
6314 : bitregion_start, bitregion_end,
6315 : mode1, to_rtx, to, from,
6316 : reversep))
6317 : result = NULL;
6318 4683106 : else if (SUBREG_P (to_rtx)
6319 4683106 : && SUBREG_PROMOTED_VAR_P (to_rtx))
6320 : {
6321 : /* If to_rtx is a promoted subreg, we need to zero or sign
6322 : extend the value afterwards. */
6323 0 : if (TREE_CODE (to) == MEM_REF
6324 0 : && TYPE_MODE (TREE_TYPE (from)) != BLKmode
6325 0 : && !REF_REVERSE_STORAGE_ORDER (to)
6326 0 : && known_eq (bitpos, 0)
6327 0 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (to_rtx))))
6328 0 : result = store_expr (from, to_rtx, 0, nontemporal, false);
6329 : /* Check if the field overlaps the MSB, requiring extension. */
6330 0 : else if (store_field_updates_msb_p (bitpos, bitsize, to_rtx))
6331 : {
6332 0 : scalar_int_mode imode = subreg_unpromoted_mode (to_rtx);
6333 0 : scalar_int_mode omode = subreg_promoted_mode (to_rtx);
6334 0 : rtx to_rtx1 = lowpart_subreg (imode, SUBREG_REG (to_rtx),
6335 : omode);
6336 0 : result = store_field (to_rtx1, bitsize, bitpos,
6337 : bitregion_start, bitregion_end,
6338 : mode1, from, get_alias_set (to),
6339 : nontemporal, reversep);
6340 : /* If the target usually keeps IMODE appropriately
6341 : extended in OMODE it's unsafe to refer to it using
6342 : a SUBREG whilst this invariant doesn't hold. */
6343 0 : if (targetm.mode_rep_extended (imode, omode) != UNKNOWN)
6344 0 : to_rtx1 = simplify_gen_unary (TRUNCATE, imode,
6345 : SUBREG_REG (to_rtx), omode);
6346 0 : convert_move (SUBREG_REG (to_rtx), to_rtx1,
6347 0 : SUBREG_PROMOTED_SIGN (to_rtx));
6348 : }
6349 : else
6350 0 : result = store_field (to_rtx, bitsize, bitpos,
6351 : bitregion_start, bitregion_end,
6352 : mode1, from, get_alias_set (to),
6353 : nontemporal, reversep);
6354 : }
6355 : else
6356 4683106 : result = store_field (to_rtx, bitsize, bitpos,
6357 : bitregion_start, bitregion_end,
6358 : mode1, from, get_alias_set (to),
6359 : nontemporal, reversep);
6360 : /* Move the temporary storage back to the non-MEM_P. */
6361 4683273 : if (stemp)
6362 3 : emit_move_insn (old_to_rtx, stemp);
6363 : }
6364 :
6365 4683402 : if (result)
6366 811652 : preserve_temp_slots (result);
6367 4683402 : pop_temp_slots ();
6368 4683402 : return;
6369 : }
6370 :
6371 : /* If the rhs is a function call and its value is not an aggregate,
6372 : call the function before we start to compute the lhs.
6373 : This is needed for correct code for cases such as
6374 : val = setjmp (buf) on machines where reference to val
6375 : requires loading up part of an address in a separate insn.
6376 :
6377 : Don't do this if TO is a VAR_DECL or PARM_DECL whose DECL_RTL is REG
6378 : since it might be a promoted variable where the zero- or sign- extension
6379 : needs to be done. Handling this in the normal way is safe because no
6380 : computation is done before the call. The same is true for SSA names. */
6381 2321630 : if (TREE_CODE (from) == CALL_EXPR && ! aggregate_value_p (from, from)
6382 2123732 : && COMPLETE_TYPE_P (TREE_TYPE (from))
6383 2123732 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (from))) == INTEGER_CST
6384 15411554 : && ! (((VAR_P (to)
6385 : || TREE_CODE (to) == PARM_DECL
6386 : || TREE_CODE (to) == RESULT_DECL)
6387 116567 : && REG_P (DECL_RTL (to)))
6388 2016665 : || TREE_CODE (to) == SSA_NAME))
6389 : {
6390 9790 : rtx value;
6391 :
6392 9790 : push_temp_slots ();
6393 9790 : value = expand_normal (from);
6394 :
6395 9790 : if (to_rtx == 0)
6396 9790 : to_rtx = expand_expr (to, NULL_RTX, VOIDmode, EXPAND_WRITE);
6397 :
6398 : /* Handle calls that return values in multiple non-contiguous locations.
6399 : The Irix 6 ABI has examples of this. */
6400 9790 : if (GET_CODE (to_rtx) == PARALLEL)
6401 : {
6402 0 : if (GET_CODE (value) == PARALLEL)
6403 0 : emit_group_move (to_rtx, value);
6404 : else
6405 0 : emit_group_load (to_rtx, value, TREE_TYPE (from),
6406 0 : int_size_in_bytes (TREE_TYPE (from)));
6407 : }
6408 9790 : else if (GET_CODE (value) == PARALLEL)
6409 1945 : emit_group_store (to_rtx, value, TREE_TYPE (from),
6410 1945 : int_size_in_bytes (TREE_TYPE (from)));
6411 7845 : else if (GET_MODE (to_rtx) == BLKmode)
6412 : {
6413 : /* Handle calls that return BLKmode values in registers. */
6414 254 : if (REG_P (value))
6415 254 : copy_blkmode_from_reg (to_rtx, value, TREE_TYPE (from));
6416 : else
6417 0 : emit_block_move (to_rtx, value, expr_size (from), BLOCK_OP_NORMAL);
6418 : }
6419 : else
6420 : {
6421 7591 : if (POINTER_TYPE_P (TREE_TYPE (to)))
6422 0 : value = convert_memory_address_addr_space
6423 0 : (as_a <scalar_int_mode> (GET_MODE (to_rtx)), value,
6424 0 : TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (to))));
6425 :
6426 7591 : emit_move_insn (to_rtx, value);
6427 : }
6428 :
6429 9790 : preserve_temp_slots (to_rtx);
6430 9790 : pop_temp_slots ();
6431 9790 : return;
6432 : }
6433 :
6434 : /* Ordinary treatment. Expand TO to get a REG or MEM rtx. */
6435 13278032 : to_rtx = expand_expr (to, NULL_RTX, VOIDmode, EXPAND_WRITE);
6436 :
6437 : /* Don't move directly into a return register. */
6438 13278032 : if (TREE_CODE (to) == RESULT_DECL
6439 29040 : && (REG_P (to_rtx) || GET_CODE (to_rtx) == PARALLEL))
6440 : {
6441 0 : rtx temp;
6442 :
6443 0 : push_temp_slots ();
6444 :
6445 : /* If the source is itself a return value, it still is in a pseudo at
6446 : this point so we can move it back to the return register directly. */
6447 0 : if (REG_P (to_rtx)
6448 0 : && TYPE_MODE (TREE_TYPE (from)) == BLKmode
6449 0 : && TREE_CODE (from) != CALL_EXPR)
6450 0 : temp = copy_blkmode_to_reg (GET_MODE (to_rtx), from);
6451 : else
6452 0 : temp = expand_expr (from, NULL_RTX, GET_MODE (to_rtx), EXPAND_NORMAL);
6453 :
6454 : /* Handle calls that return values in multiple non-contiguous locations.
6455 : The Irix 6 ABI has examples of this. */
6456 0 : if (GET_CODE (to_rtx) == PARALLEL)
6457 : {
6458 0 : if (GET_CODE (temp) == PARALLEL)
6459 0 : emit_group_move (to_rtx, temp);
6460 : else
6461 0 : emit_group_load (to_rtx, temp, TREE_TYPE (from),
6462 0 : int_size_in_bytes (TREE_TYPE (from)));
6463 : }
6464 0 : else if (temp)
6465 0 : emit_move_insn (to_rtx, temp);
6466 :
6467 0 : preserve_temp_slots (to_rtx);
6468 0 : pop_temp_slots ();
6469 0 : return;
6470 : }
6471 :
6472 : /* In case we are returning the contents of an object which overlaps
6473 : the place the value is being stored, use a safe function when copying
6474 : a value through a pointer into a structure value return block. */
6475 13278032 : if (TREE_CODE (to) == RESULT_DECL
6476 29040 : && TREE_CODE (from) == INDIRECT_REF
6477 0 : && ADDR_SPACE_GENERIC_P
6478 : (TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (from, 0)))))
6479 0 : && refs_may_alias_p (to, from)
6480 0 : && cfun->returns_struct
6481 13278032 : && !cfun->returns_pcc_struct)
6482 : {
6483 0 : rtx from_rtx, size;
6484 :
6485 0 : push_temp_slots ();
6486 0 : size = expr_size (from);
6487 0 : from_rtx = expand_normal (from);
6488 :
6489 0 : emit_block_move_via_libcall (XEXP (to_rtx, 0), XEXP (from_rtx, 0), size);
6490 :
6491 0 : preserve_temp_slots (to_rtx);
6492 0 : pop_temp_slots ();
6493 0 : return;
6494 : }
6495 :
6496 : /* Compute FROM and store the value in the rtx we got. */
6497 :
6498 13278032 : push_temp_slots ();
6499 13278032 : result = store_expr (from, to_rtx, 0, nontemporal, false);
6500 13278031 : preserve_temp_slots (result);
6501 13278031 : pop_temp_slots ();
6502 13278031 : return;
6503 : }
6504 :
6505 : /* Emits nontemporal store insn that moves FROM to TO. Returns true if this
6506 : succeeded, false otherwise. */
6507 :
6508 : bool
6509 17 : emit_storent_insn (rtx to, rtx from)
6510 : {
6511 17 : class expand_operand ops[2];
6512 17 : machine_mode mode = GET_MODE (to);
6513 17 : enum insn_code code = optab_handler (storent_optab, mode);
6514 :
6515 17 : if (code == CODE_FOR_nothing)
6516 : return false;
6517 :
6518 17 : create_fixed_operand (&ops[0], to);
6519 17 : create_input_operand (&ops[1], from, mode);
6520 17 : return maybe_expand_insn (code, 2, ops);
6521 : }
6522 :
6523 : /* Helper function for store_expr storing of STRING_CST. */
6524 :
6525 : static rtx
6526 76997 : string_cst_read_str (void *data, void *, HOST_WIDE_INT offset,
6527 : fixed_size_mode mode)
6528 : {
6529 76997 : tree str = (tree) data;
6530 :
6531 76997 : gcc_assert (offset >= 0);
6532 76997 : if (offset >= TREE_STRING_LENGTH (str))
6533 3316 : return const0_rtx;
6534 :
6535 73681 : if ((unsigned HOST_WIDE_INT) offset + GET_MODE_SIZE (mode)
6536 73681 : > (unsigned HOST_WIDE_INT) TREE_STRING_LENGTH (str))
6537 : {
6538 2440 : char *p = XALLOCAVEC (char, GET_MODE_SIZE (mode));
6539 2440 : size_t l = TREE_STRING_LENGTH (str) - offset;
6540 2440 : memcpy (p, TREE_STRING_POINTER (str) + offset, l);
6541 2440 : memset (p + l, '\0', GET_MODE_SIZE (mode) - l);
6542 2440 : return c_readstr (p, mode, false);
6543 : }
6544 :
6545 71241 : return c_readstr (TREE_STRING_POINTER (str) + offset, mode, false);
6546 : }
6547 :
6548 : /* Helper function for store_expr storing of RAW_DATA_CST. */
6549 :
6550 : static rtx
6551 36 : raw_data_cst_read_str (void *data, void *, HOST_WIDE_INT offset,
6552 : fixed_size_mode mode)
6553 : {
6554 36 : tree cst = (tree) data;
6555 :
6556 36 : gcc_assert (offset >= 0);
6557 36 : if (offset >= RAW_DATA_LENGTH (cst))
6558 0 : return const0_rtx;
6559 :
6560 36 : if ((unsigned HOST_WIDE_INT) offset + GET_MODE_SIZE (mode)
6561 36 : > (unsigned HOST_WIDE_INT) RAW_DATA_LENGTH (cst))
6562 : {
6563 0 : char *p = XALLOCAVEC (char, GET_MODE_SIZE (mode));
6564 0 : size_t l = RAW_DATA_LENGTH (cst) - offset;
6565 0 : memcpy (p, RAW_DATA_POINTER (cst) + offset, l);
6566 0 : memset (p + l, '\0', GET_MODE_SIZE (mode) - l);
6567 0 : return c_readstr (p, mode, false);
6568 : }
6569 :
6570 36 : return c_readstr (RAW_DATA_POINTER (cst) + offset, mode, false);
6571 : }
6572 :
6573 : /* Generate code for computing expression EXP,
6574 : and storing the value into TARGET.
6575 :
6576 : If the mode is BLKmode then we may return TARGET itself.
6577 : It turns out that in BLKmode it doesn't cause a problem.
6578 : because C has no operators that could combine two different
6579 : assignments into the same BLKmode object with different values
6580 : with no sequence point. Will other languages need this to
6581 : be more thorough?
6582 :
6583 : If CALL_PARAM_P is nonzero, this is a store into a call param on the
6584 : stack, and block moves may need to be treated specially.
6585 :
6586 : If NONTEMPORAL is true, try using a nontemporal store instruction.
6587 :
6588 : If REVERSE is true, the store is to be done in reverse order. */
6589 :
6590 : rtx
6591 17191201 : store_expr (tree exp, rtx target, int call_param_p,
6592 : bool nontemporal, bool reverse)
6593 : {
6594 17191201 : rtx temp;
6595 17191201 : rtx alt_rtl = NULL_RTX;
6596 17191201 : location_t loc = curr_insn_location ();
6597 17191201 : bool shortened_string_cst = false;
6598 :
6599 17191201 : if (VOID_TYPE_P (TREE_TYPE (exp)))
6600 : {
6601 : /* C++ can generate ?: expressions with a throw expression in one
6602 : branch and an rvalue in the other. Here, we resolve attempts to
6603 : store the throw expression's nonexistent result. */
6604 0 : gcc_assert (!call_param_p);
6605 0 : expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
6606 0 : return NULL_RTX;
6607 : }
6608 17191201 : if (TREE_CODE (exp) == COMPOUND_EXPR)
6609 : {
6610 : /* Perform first part of compound expression, then assign from second
6611 : part. */
6612 0 : expand_expr (TREE_OPERAND (exp, 0), const0_rtx, VOIDmode,
6613 : call_param_p ? EXPAND_STACK_PARM : EXPAND_NORMAL);
6614 0 : return store_expr (TREE_OPERAND (exp, 1), target,
6615 0 : call_param_p, nontemporal, reverse);
6616 : }
6617 17191201 : else if (TREE_CODE (exp) == COND_EXPR && GET_MODE (target) == BLKmode)
6618 : {
6619 : /* For conditional expression, get safe form of the target. Then
6620 : test the condition, doing the appropriate assignment on either
6621 : side. This avoids the creation of unnecessary temporaries.
6622 : For non-BLKmode, it is more efficient not to do this. */
6623 :
6624 0 : rtx_code_label *lab1 = gen_label_rtx (), *lab2 = gen_label_rtx ();
6625 :
6626 0 : do_pending_stack_adjust ();
6627 0 : NO_DEFER_POP;
6628 0 : jumpifnot (TREE_OPERAND (exp, 0), lab1,
6629 : profile_probability::uninitialized ());
6630 0 : store_expr (TREE_OPERAND (exp, 1), target, call_param_p,
6631 : nontemporal, reverse);
6632 0 : emit_jump_insn (targetm.gen_jump (lab2));
6633 0 : emit_barrier ();
6634 0 : emit_label (lab1);
6635 0 : store_expr (TREE_OPERAND (exp, 2), target, call_param_p,
6636 : nontemporal, reverse);
6637 0 : emit_label (lab2);
6638 0 : OK_DEFER_POP;
6639 :
6640 0 : return NULL_RTX;
6641 : }
6642 17191201 : else if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
6643 : /* If this is a scalar in a register that is stored in a wider mode
6644 : than the declared mode, compute the result into its declared mode
6645 : and then convert to the wider mode. Our value is the computed
6646 : expression. */
6647 : {
6648 7 : rtx inner_target = 0;
6649 7 : scalar_int_mode outer_mode = subreg_unpromoted_mode (target);
6650 7 : scalar_int_mode inner_mode = subreg_promoted_mode (target);
6651 :
6652 : /* We can do the conversion inside EXP, which will often result
6653 : in some optimizations. Do the conversion in two steps: first
6654 : change the signedness, if needed, then the extend. But don't
6655 : do this if the type of EXP is a subtype of something else
6656 : since then the conversion might involve more than just
6657 : converting modes. */
6658 14 : if (INTEGRAL_TYPE_P (TREE_TYPE (exp))
6659 0 : && TREE_TYPE (TREE_TYPE (exp)) == 0
6660 7 : && GET_MODE_PRECISION (outer_mode)
6661 0 : == TYPE_PRECISION (TREE_TYPE (exp)))
6662 : {
6663 0 : if (!SUBREG_CHECK_PROMOTED_SIGN (target,
6664 : TYPE_UNSIGNED (TREE_TYPE (exp))))
6665 : {
6666 : /* Some types, e.g. Fortran's logical*4, won't have a signed
6667 : version, so use the mode instead. */
6668 0 : tree ntype
6669 : = (signed_or_unsigned_type_for
6670 0 : (SUBREG_PROMOTED_SIGN (target), TREE_TYPE (exp)));
6671 0 : if (ntype == NULL)
6672 0 : ntype = lang_hooks.types.type_for_mode
6673 0 : (TYPE_MODE (TREE_TYPE (exp)),
6674 0 : SUBREG_PROMOTED_SIGN (target));
6675 :
6676 0 : exp = fold_convert_loc (loc, ntype, exp);
6677 : }
6678 :
6679 0 : exp = fold_convert_loc (loc, lang_hooks.types.type_for_mode
6680 0 : (inner_mode, SUBREG_PROMOTED_SIGN (target)),
6681 : exp);
6682 :
6683 0 : inner_target = SUBREG_REG (target);
6684 : }
6685 :
6686 7 : temp = expand_expr (exp, inner_target, VOIDmode,
6687 : call_param_p ? EXPAND_STACK_PARM : EXPAND_NORMAL);
6688 :
6689 :
6690 : /* If TEMP is a VOIDmode constant, use convert_modes to make
6691 : sure that we properly convert it. */
6692 7 : if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode)
6693 : {
6694 0 : temp = convert_modes (outer_mode, TYPE_MODE (TREE_TYPE (exp)),
6695 0 : temp, SUBREG_PROMOTED_SIGN (target));
6696 0 : temp = convert_modes (inner_mode, outer_mode, temp,
6697 0 : SUBREG_PROMOTED_SIGN (target));
6698 0 : }
6699 7 : else if (!SCALAR_INT_MODE_P (GET_MODE (temp)))
6700 0 : temp = convert_modes (outer_mode, TYPE_MODE (TREE_TYPE (exp)),
6701 0 : temp, SUBREG_PROMOTED_SIGN (target));
6702 :
6703 21 : convert_move (SUBREG_REG (target), temp,
6704 7 : SUBREG_PROMOTED_SIGN (target));
6705 :
6706 7 : return NULL_RTX;
6707 : }
6708 17191194 : else if ((TREE_CODE (exp) == STRING_CST
6709 17182259 : || (TREE_CODE (exp) == MEM_REF
6710 1329215 : && TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR
6711 430606 : && TREE_CODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
6712 : == STRING_CST
6713 4363 : && integer_zerop (TREE_OPERAND (exp, 1))))
6714 13298 : && !nontemporal && !call_param_p
6715 17204492 : && MEM_P (target))
6716 : {
6717 : /* Optimize initialization of an array with a STRING_CST. */
6718 13280 : HOST_WIDE_INT exp_len, str_copy_len;
6719 13280 : rtx dest_mem;
6720 13280 : tree str = TREE_CODE (exp) == STRING_CST
6721 13280 : ? exp : TREE_OPERAND (TREE_OPERAND (exp, 0), 0);
6722 :
6723 13280 : exp_len = int_expr_size (exp);
6724 13280 : if (exp_len <= 0)
6725 0 : goto normal_expr;
6726 :
6727 13280 : if (TREE_STRING_LENGTH (str) <= 0)
6728 0 : goto normal_expr;
6729 :
6730 26560 : if (can_store_by_pieces (exp_len, string_cst_read_str, (void *) str,
6731 13280 : MEM_ALIGN (target), false))
6732 : {
6733 13100 : store_by_pieces (target, exp_len, string_cst_read_str, (void *) str,
6734 13100 : MEM_ALIGN (target), false, RETURN_BEGIN);
6735 13100 : return NULL_RTX;
6736 : }
6737 :
6738 180 : str_copy_len = TREE_STRING_LENGTH (str);
6739 :
6740 : /* Trailing NUL bytes in EXP will be handled by the call to
6741 : clear_storage, which is more efficient than copying them from
6742 : the STRING_CST, so trim those from STR_COPY_LEN. */
6743 303 : while (str_copy_len)
6744 : {
6745 260 : if (TREE_STRING_POINTER (str)[str_copy_len - 1])
6746 : break;
6747 123 : str_copy_len--;
6748 : }
6749 :
6750 180 : if ((STORE_MAX_PIECES & (STORE_MAX_PIECES - 1)) == 0)
6751 : {
6752 180 : str_copy_len += STORE_MAX_PIECES - 1;
6753 180 : str_copy_len &= ~(STORE_MAX_PIECES - 1);
6754 : }
6755 180 : if (str_copy_len >= exp_len)
6756 129 : goto normal_expr;
6757 :
6758 102 : if (!can_store_by_pieces (str_copy_len, string_cst_read_str,
6759 51 : (void *) str, MEM_ALIGN (target), false))
6760 2 : goto normal_expr;
6761 :
6762 49 : dest_mem = store_by_pieces (target, str_copy_len, string_cst_read_str,
6763 49 : (void *) str, MEM_ALIGN (target), false,
6764 : RETURN_END);
6765 49 : clear_storage (adjust_address_1 (dest_mem, BLKmode, 0, 1, 1, 0,
6766 49 : exp_len - str_copy_len),
6767 : GEN_INT (exp_len - str_copy_len), BLOCK_OP_NORMAL);
6768 49 : return NULL_RTX;
6769 : }
6770 : else
6771 : {
6772 17178045 : rtx tmp_target;
6773 :
6774 17178045 : normal_expr:
6775 : /* If we want to use a nontemporal or a reverse order store, force the
6776 : value into a register first. */
6777 17178045 : tmp_target = nontemporal || reverse ? NULL_RTX : target;
6778 17178045 : tree rexp = exp;
6779 17178045 : if (TREE_CODE (exp) == STRING_CST
6780 42 : && tmp_target == target
6781 42 : && GET_MODE (target) == BLKmode
6782 17178087 : && TYPE_MODE (TREE_TYPE (exp)) == BLKmode)
6783 : {
6784 42 : rtx size = expr_size (exp);
6785 42 : if (CONST_INT_P (size)
6786 42 : && size != const0_rtx
6787 84 : && (UINTVAL (size)
6788 42 : > ((unsigned HOST_WIDE_INT) TREE_STRING_LENGTH (exp) + 32)))
6789 : {
6790 : /* If the STRING_CST has much larger array type than
6791 : TREE_STRING_LENGTH, only emit the TREE_STRING_LENGTH part of
6792 : it into the rodata section as the code later on will use
6793 : memset zero for the remainder anyway. See PR95052. */
6794 2 : tmp_target = NULL_RTX;
6795 2 : rexp = copy_node (exp);
6796 2 : tree index
6797 2 : = build_index_type (size_int (TREE_STRING_LENGTH (exp) - 1));
6798 2 : TREE_TYPE (rexp) = build_array_type (TREE_TYPE (TREE_TYPE (exp)),
6799 : index);
6800 2 : shortened_string_cst = true;
6801 : }
6802 : }
6803 34356090 : temp = expand_expr_real (rexp, tmp_target, GET_MODE (target),
6804 : (call_param_p
6805 : ? EXPAND_STACK_PARM : EXPAND_NORMAL),
6806 : &alt_rtl, false);
6807 17178044 : if (shortened_string_cst)
6808 : {
6809 2 : gcc_assert (MEM_P (temp));
6810 2 : temp = change_address (temp, BLKmode, NULL_RTX);
6811 : }
6812 : }
6813 :
6814 : /* If TEMP is a VOIDmode constant and the mode of the type of EXP is not
6815 : the same as that of TARGET, adjust the constant. This is needed, for
6816 : example, in case it is a CONST_DOUBLE or CONST_WIDE_INT and we want
6817 : only a word-sized value. */
6818 3545018 : if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode
6819 2559562 : && TREE_CODE (exp) != ERROR_MARK
6820 19737606 : && GET_MODE (target) != TYPE_MODE (TREE_TYPE (exp)))
6821 : {
6822 0 : gcc_assert (!shortened_string_cst);
6823 0 : if (GET_MODE_CLASS (GET_MODE (target))
6824 0 : != GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (exp)))
6825 0 : && known_eq (GET_MODE_BITSIZE (GET_MODE (target)),
6826 : GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (exp)))))
6827 : {
6828 0 : rtx t = simplify_gen_subreg (GET_MODE (target), temp,
6829 0 : TYPE_MODE (TREE_TYPE (exp)), 0);
6830 0 : if (t)
6831 0 : temp = t;
6832 : }
6833 0 : if (GET_MODE (temp) == VOIDmode)
6834 0 : temp = convert_modes (GET_MODE (target), TYPE_MODE (TREE_TYPE (exp)),
6835 0 : temp, TYPE_UNSIGNED (TREE_TYPE (exp)));
6836 : }
6837 :
6838 : /* If value was not generated in the target, store it there.
6839 : Convert the value to TARGET's type first if necessary and emit the
6840 : pending incrementations that have been queued when expanding EXP.
6841 : Note that we cannot emit the whole queue blindly because this will
6842 : effectively disable the POST_INC optimization later.
6843 :
6844 : If TEMP and TARGET compare equal according to rtx_equal_p, but
6845 : one or both of them are volatile memory refs, we have to distinguish
6846 : two cases:
6847 : - expand_expr has used TARGET. In this case, we must not generate
6848 : another copy. This can be detected by TARGET being equal according
6849 : to == .
6850 : - expand_expr has not used TARGET - that means that the source just
6851 : happens to have the same RTX form. Since temp will have been created
6852 : by expand_expr, it will compare unequal according to == .
6853 : We must generate a copy in this case, to reach the correct number
6854 : of volatile memory references. */
6855 :
6856 17178044 : if ((! rtx_equal_p (temp, target)
6857 3055900 : || (temp != target && (side_effects_p (temp)
6858 46758 : || side_effects_p (target)
6859 46758 : || (MEM_P (temp)
6860 46280 : && !mems_same_for_tbaa_p (temp, target)))))
6861 14122148 : && TREE_CODE (exp) != ERROR_MARK
6862 : /* If store_expr stores a DECL whose DECL_RTL(exp) == TARGET,
6863 : but TARGET is not valid memory reference, TEMP will differ
6864 : from TARGET although it is really the same location. */
6865 14122148 : && !(alt_rtl
6866 1967138 : && rtx_equal_p (alt_rtl, target)
6867 1 : && !side_effects_p (alt_rtl)
6868 0 : && !side_effects_p (target))
6869 : /* If there's nothing to copy, don't bother. Don't call
6870 : expr_size unless necessary, because some front-ends (C++)
6871 : expr_size-hook must not be given objects that are not
6872 : supposed to be bit-copied or bit-initialized. */
6873 31300192 : && expr_size (exp) != const0_rtx)
6874 : {
6875 14122148 : if (GET_MODE (temp) != GET_MODE (target) && GET_MODE (temp) != VOIDmode)
6876 : {
6877 1513 : gcc_assert (!shortened_string_cst);
6878 1513 : if (GET_MODE (target) == BLKmode)
6879 : {
6880 : /* Handle calls that return BLKmode values in registers. */
6881 2 : if (REG_P (temp) && TREE_CODE (exp) == CALL_EXPR)
6882 2 : copy_blkmode_from_reg (target, temp, TREE_TYPE (exp));
6883 : else
6884 0 : store_bit_field (target,
6885 0 : rtx_to_poly_int64 (expr_size (exp))
6886 0 : * BITS_PER_UNIT,
6887 : 0, 0, 0, GET_MODE (temp), temp, reverse,
6888 : false);
6889 : }
6890 : else
6891 1511 : convert_move (target, temp, TYPE_UNSIGNED (TREE_TYPE (exp)));
6892 : }
6893 :
6894 14120635 : else if (GET_MODE (temp) == BLKmode && TREE_CODE (exp) == STRING_CST)
6895 : {
6896 : /* Handle copying a string constant into an array. The string
6897 : constant may be shorter than the array. So copy just the string's
6898 : actual length, and clear the rest. First get the size of the data
6899 : type of the string, which is actually the size of the target. */
6900 42 : rtx size = expr_size (exp);
6901 :
6902 42 : if (CONST_INT_P (size)
6903 42 : && INTVAL (size) < TREE_STRING_LENGTH (exp))
6904 0 : emit_block_move (target, temp, size,
6905 : (call_param_p
6906 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
6907 : else
6908 : {
6909 42 : machine_mode pointer_mode
6910 42 : = targetm.addr_space.pointer_mode (MEM_ADDR_SPACE (target));
6911 42 : machine_mode address_mode = get_address_mode (target);
6912 :
6913 : /* Compute the size of the data to copy from the string. */
6914 42 : tree copy_size
6915 42 : = size_binop_loc (loc, MIN_EXPR,
6916 : make_tree (sizetype, size),
6917 42 : size_int (TREE_STRING_LENGTH (exp)));
6918 42 : rtx copy_size_rtx
6919 42 : = expand_expr (copy_size, NULL_RTX, VOIDmode,
6920 : (call_param_p
6921 : ? EXPAND_STACK_PARM : EXPAND_NORMAL));
6922 42 : rtx_code_label *label = 0;
6923 :
6924 : /* Copy that much. */
6925 126 : copy_size_rtx = convert_to_mode (pointer_mode, copy_size_rtx,
6926 42 : TYPE_UNSIGNED (sizetype));
6927 84 : emit_block_move (target, temp, copy_size_rtx,
6928 : (call_param_p
6929 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
6930 :
6931 : /* Figure out how much is left in TARGET that we have to clear.
6932 : Do all calculations in pointer_mode. */
6933 42 : poly_int64 const_copy_size;
6934 42 : if (poly_int_rtx_p (copy_size_rtx, &const_copy_size))
6935 : {
6936 42 : size = plus_constant (address_mode, size, -const_copy_size);
6937 42 : target = adjust_address (target, BLKmode, const_copy_size);
6938 : }
6939 : else
6940 : {
6941 0 : size = expand_binop (TYPE_MODE (sizetype), sub_optab, size,
6942 : copy_size_rtx, NULL_RTX, 0,
6943 : OPTAB_LIB_WIDEN);
6944 :
6945 0 : if (GET_MODE (copy_size_rtx) != address_mode)
6946 0 : copy_size_rtx = convert_to_mode (address_mode,
6947 : copy_size_rtx,
6948 0 : TYPE_UNSIGNED (sizetype));
6949 :
6950 0 : target = offset_address (target, copy_size_rtx,
6951 : highest_pow2_factor (copy_size));
6952 0 : label = gen_label_rtx ();
6953 0 : emit_cmp_and_jump_insns (size, const0_rtx, LT, NULL_RTX,
6954 0 : GET_MODE (size), 0, label);
6955 : }
6956 :
6957 42 : if (size != const0_rtx)
6958 2 : clear_storage (target, size, BLOCK_OP_NORMAL);
6959 :
6960 42 : if (label)
6961 0 : emit_label (label);
6962 : }
6963 : }
6964 14120593 : else if (shortened_string_cst)
6965 0 : gcc_unreachable ();
6966 : /* Handle calls that return values in multiple non-contiguous locations.
6967 : The Irix 6 ABI has examples of this. */
6968 14120593 : else if (GET_CODE (target) == PARALLEL)
6969 : {
6970 0 : if (GET_CODE (temp) == PARALLEL)
6971 0 : emit_group_move (target, temp);
6972 : else
6973 0 : emit_group_load (target, temp, TREE_TYPE (exp),
6974 0 : int_size_in_bytes (TREE_TYPE (exp)));
6975 : }
6976 14120593 : else if (GET_CODE (temp) == PARALLEL)
6977 0 : emit_group_store (target, temp, TREE_TYPE (exp),
6978 0 : int_size_in_bytes (TREE_TYPE (exp)));
6979 14120593 : else if (GET_MODE (temp) == BLKmode)
6980 598754 : emit_block_move (target, temp, expr_size (exp),
6981 : (call_param_p
6982 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
6983 : /* If we emit a nontemporal store, there is nothing else to do. */
6984 13821216 : else if (nontemporal && emit_storent_insn (target, temp))
6985 : ;
6986 : else
6987 : {
6988 13821199 : if (reverse)
6989 722 : temp = flip_storage_order (GET_MODE (target), temp);
6990 13821199 : temp = force_operand (temp, target);
6991 13821199 : if (temp != target)
6992 13819707 : emit_move_insn (target, temp);
6993 : }
6994 : }
6995 : else
6996 3055896 : gcc_assert (!shortened_string_cst);
6997 :
6998 : return NULL_RTX;
6999 : }
7000 :
7001 : /* Return true if field F of structure TYPE is a flexible array. */
7002 :
7003 : static bool
7004 3752342 : flexible_array_member_p (const_tree f, const_tree type)
7005 : {
7006 3752342 : const_tree tf;
7007 :
7008 3752342 : tf = TREE_TYPE (f);
7009 3752342 : return (DECL_CHAIN (f) == NULL
7010 1078906 : && TREE_CODE (tf) == ARRAY_TYPE
7011 2648 : && TYPE_DOMAIN (tf)
7012 2648 : && TYPE_MIN_VALUE (TYPE_DOMAIN (tf))
7013 2648 : && integer_zerop (TYPE_MIN_VALUE (TYPE_DOMAIN (tf)))
7014 2648 : && !TYPE_MAX_VALUE (TYPE_DOMAIN (tf))
7015 3752381 : && int_size_in_bytes (type) >= 0);
7016 : }
7017 :
7018 : /* If FOR_CTOR_P, return the number of top-level elements that a constructor
7019 : must have in order for it to completely initialize a value of type TYPE.
7020 : Return -1 if the number isn't known.
7021 :
7022 : If !FOR_CTOR_P, return an estimate of the number of scalars in TYPE. */
7023 :
7024 : static HOST_WIDE_INT
7025 3624038 : count_type_elements (const_tree type, bool for_ctor_p)
7026 : {
7027 3624038 : switch (TREE_CODE (type))
7028 : {
7029 149404 : case ARRAY_TYPE:
7030 149404 : {
7031 149404 : tree nelts_minus_one;
7032 :
7033 149404 : nelts_minus_one = array_type_nelts_minus_one (type);
7034 149404 : if (nelts_minus_one && tree_fits_uhwi_p (nelts_minus_one))
7035 : {
7036 149393 : unsigned HOST_WIDE_INT n;
7037 :
7038 149393 : n = tree_to_uhwi (nelts_minus_one) + 1;
7039 149393 : if (n == 0 || for_ctor_p)
7040 148307 : return n;
7041 : else
7042 1086 : return n * count_type_elements (TREE_TYPE (type), false);
7043 : }
7044 11 : return for_ctor_p ? -1 : 1;
7045 : }
7046 :
7047 1742738 : case RECORD_TYPE:
7048 1742738 : {
7049 1742738 : unsigned HOST_WIDE_INT n;
7050 1742738 : tree f;
7051 :
7052 1742738 : n = 0;
7053 15088057 : for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
7054 13345319 : if (TREE_CODE (f) == FIELD_DECL)
7055 : {
7056 4213602 : if (!for_ctor_p)
7057 461260 : n += count_type_elements (TREE_TYPE (f), false);
7058 3752342 : else if (!flexible_array_member_p (f, type))
7059 : /* Don't count flexible arrays, which are not supposed
7060 : to be initialized. */
7061 3752303 : n += 1;
7062 : }
7063 :
7064 1742738 : return n;
7065 : }
7066 :
7067 3282 : case UNION_TYPE:
7068 3282 : case QUAL_UNION_TYPE:
7069 3282 : {
7070 3282 : tree f;
7071 3282 : HOST_WIDE_INT n, m;
7072 :
7073 3282 : gcc_assert (!for_ctor_p);
7074 : /* Estimate the number of scalars in each field and pick the
7075 : maximum. Other estimates would do instead; the idea is simply
7076 : to make sure that the estimate is not sensitive to the ordering
7077 : of the fields. */
7078 3282 : n = 1;
7079 74579 : for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
7080 71297 : if (TREE_CODE (f) == FIELD_DECL)
7081 : {
7082 66888 : m = count_type_elements (TREE_TYPE (f), false);
7083 : /* If the field doesn't span the whole union, add an extra
7084 : scalar for the rest. */
7085 66888 : if (simple_cst_equal (TYPE_SIZE (TREE_TYPE (f)),
7086 66888 : TYPE_SIZE (type)) != 1)
7087 47670 : m++;
7088 66888 : if (n < m)
7089 71297 : n = m;
7090 : }
7091 : return n;
7092 : }
7093 :
7094 : case COMPLEX_TYPE:
7095 : return 2;
7096 :
7097 357 : case VECTOR_TYPE:
7098 357 : {
7099 357 : unsigned HOST_WIDE_INT nelts;
7100 357 : if (TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts))
7101 357 : return nelts;
7102 : else
7103 : return -1;
7104 : }
7105 :
7106 : case INTEGER_TYPE:
7107 : case REAL_TYPE:
7108 : case FIXED_POINT_TYPE:
7109 : case ENUMERAL_TYPE:
7110 : case BOOLEAN_TYPE:
7111 : case POINTER_TYPE:
7112 : case OFFSET_TYPE:
7113 : case REFERENCE_TYPE:
7114 : case NULLPTR_TYPE:
7115 : case OPAQUE_TYPE:
7116 : case BITINT_TYPE:
7117 : return 1;
7118 :
7119 0 : case ERROR_MARK:
7120 0 : return 0;
7121 :
7122 0 : case VOID_TYPE:
7123 0 : case METHOD_TYPE:
7124 0 : case FUNCTION_TYPE:
7125 0 : case LANG_TYPE:
7126 0 : default:
7127 0 : gcc_unreachable ();
7128 : }
7129 : }
7130 :
7131 : /* Helper for categorize_ctor_elements. Identical interface. */
7132 :
7133 : static bool
7134 1502072 : categorize_ctor_elements_1 (const_tree ctor, HOST_WIDE_INT *p_nz_elts,
7135 : HOST_WIDE_INT *p_unique_nz_elts,
7136 : HOST_WIDE_INT *p_init_elts,
7137 : ctor_completeness *p_complete)
7138 : {
7139 1502072 : unsigned HOST_WIDE_INT idx;
7140 1502072 : HOST_WIDE_INT nz_elts, unique_nz_elts, init_elts, num_fields;
7141 1502072 : tree value, purpose, elt_type;
7142 :
7143 : /* Whether CTOR is a valid constant initializer, in accordance with what
7144 : initializer_constant_valid_p does. If inferred from the constructor
7145 : elements, true until proven otherwise. */
7146 1502072 : bool const_from_elts_p = constructor_static_from_elts_p (ctor);
7147 1502072 : bool const_p = const_from_elts_p ? true : TREE_STATIC (ctor);
7148 :
7149 1502072 : nz_elts = 0;
7150 1502072 : unique_nz_elts = 0;
7151 1502072 : init_elts = 0;
7152 1502072 : num_fields = 0;
7153 1502072 : elt_type = NULL_TREE;
7154 :
7155 5744340 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (ctor), idx, purpose, value)
7156 : {
7157 4242268 : HOST_WIDE_INT mult = 1;
7158 :
7159 4242268 : if (purpose && TREE_CODE (purpose) == RANGE_EXPR)
7160 : {
7161 669 : tree lo_index = TREE_OPERAND (purpose, 0);
7162 669 : tree hi_index = TREE_OPERAND (purpose, 1);
7163 :
7164 669 : if (tree_fits_uhwi_p (lo_index) && tree_fits_uhwi_p (hi_index))
7165 669 : mult = (tree_to_uhwi (hi_index)
7166 669 : - tree_to_uhwi (lo_index) + 1);
7167 : }
7168 4242268 : num_fields += mult;
7169 4242268 : elt_type = TREE_TYPE (value);
7170 :
7171 4242268 : switch (TREE_CODE (value))
7172 : {
7173 528324 : case CONSTRUCTOR:
7174 528324 : {
7175 528324 : HOST_WIDE_INT nz = 0, unz = 0, ic = 0;
7176 :
7177 528324 : bool const_elt_p = categorize_ctor_elements_1 (value, &nz, &unz,
7178 : &ic, p_complete);
7179 :
7180 528324 : nz_elts += mult * nz;
7181 528324 : unique_nz_elts += unz;
7182 528324 : init_elts += mult * ic;
7183 :
7184 528324 : if (const_from_elts_p && const_p)
7185 315190 : const_p = const_elt_p;
7186 : }
7187 528324 : break;
7188 :
7189 2265195 : case INTEGER_CST:
7190 2265195 : case REAL_CST:
7191 2265195 : case FIXED_CST:
7192 2265195 : if (!initializer_zerop (value))
7193 : {
7194 1663041 : nz_elts += mult;
7195 1663041 : unique_nz_elts++;
7196 : }
7197 2265195 : init_elts += mult;
7198 2265195 : break;
7199 :
7200 6296 : case STRING_CST:
7201 6296 : nz_elts += mult * TREE_STRING_LENGTH (value);
7202 6296 : unique_nz_elts += TREE_STRING_LENGTH (value);
7203 6296 : init_elts += mult * TREE_STRING_LENGTH (value);
7204 6296 : break;
7205 :
7206 166 : case RAW_DATA_CST:
7207 166 : nz_elts += mult * RAW_DATA_LENGTH (value);
7208 166 : unique_nz_elts += RAW_DATA_LENGTH (value);
7209 166 : init_elts += mult * RAW_DATA_LENGTH (value);
7210 166 : num_fields += mult * (RAW_DATA_LENGTH (value) - 1);
7211 166 : break;
7212 :
7213 2635 : case COMPLEX_CST:
7214 2635 : if (!initializer_zerop (TREE_REALPART (value)))
7215 : {
7216 2209 : nz_elts += mult;
7217 2209 : unique_nz_elts++;
7218 : }
7219 2635 : if (!initializer_zerop (TREE_IMAGPART (value)))
7220 : {
7221 2123 : nz_elts += mult;
7222 2123 : unique_nz_elts++;
7223 : }
7224 2635 : init_elts += 2 * mult;
7225 2635 : break;
7226 :
7227 3195 : case VECTOR_CST:
7228 3195 : {
7229 3195 : unsigned int nunits
7230 : = constant_lower_bound
7231 3195 : (TYPE_VECTOR_SUBPARTS (TREE_TYPE (value)));
7232 20830 : for (unsigned int i = 0; i < nunits; ++i)
7233 : {
7234 17635 : tree v = VECTOR_CST_ELT (value, i);
7235 17635 : if (!initializer_zerop (v))
7236 : {
7237 9688 : nz_elts += mult;
7238 9688 : unique_nz_elts++;
7239 : }
7240 17635 : init_elts += mult;
7241 : }
7242 : }
7243 : break;
7244 :
7245 1436457 : default:
7246 1436457 : {
7247 1436457 : HOST_WIDE_INT tc = count_type_elements (elt_type, false);
7248 1436457 : nz_elts += mult * tc;
7249 1436457 : unique_nz_elts += tc;
7250 1436457 : init_elts += mult * tc;
7251 :
7252 1436457 : if (const_from_elts_p && const_p)
7253 589279 : const_p
7254 589279 : = initializer_constant_valid_p (value,
7255 : elt_type,
7256 589279 : TYPE_REVERSE_STORAGE_ORDER
7257 : (TREE_TYPE (ctor)))
7258 : != NULL_TREE;
7259 : }
7260 : break;
7261 : }
7262 : }
7263 :
7264 1502072 : if (!p_complete->sparse && !complete_ctor_at_level_p (TREE_TYPE (ctor),
7265 : num_fields, elt_type))
7266 178241 : p_complete->sparse = true;
7267 1323831 : else if (TREE_CODE (TREE_TYPE (ctor)) == UNION_TYPE
7268 1323831 : || TREE_CODE (TREE_TYPE (ctor)) == QUAL_UNION_TYPE)
7269 : {
7270 11903 : if (!p_complete->sparse
7271 7742 : && CONSTRUCTOR_ZERO_PADDING_BITS (ctor)
7272 21790 : && (num_fields
7273 4945 : ? simple_cst_equal (TYPE_SIZE (TREE_TYPE (ctor)),
7274 4942 : TYPE_SIZE (elt_type)) != 1
7275 3 : : type_has_padding_at_level_p (TREE_TYPE (ctor))))
7276 3360 : p_complete->sparse = true;
7277 4382 : else if (!p_complete->sparse && !p_complete->padded_union
7278 17299 : && (num_fields
7279 4378 : ? simple_cst_equal (TYPE_SIZE (TREE_TYPE (ctor)),
7280 4378 : TYPE_SIZE (elt_type)) != 1
7281 0 : : type_has_padding_at_level_p (TREE_TYPE (ctor))))
7282 373 : p_complete->padded_union = true;
7283 : }
7284 1311928 : else if (!p_complete->sparse
7285 1205680 : && (CONSTRUCTOR_ZERO_PADDING_BITS (ctor)
7286 1186740 : || flag_zero_init_padding_bits == ZERO_INIT_PADDING_BITS_ALL)
7287 1330897 : && type_has_padding_at_level_p (TREE_TYPE (ctor)))
7288 2648 : p_complete->sparse = true;
7289 1203032 : else if (!p_complete->sparse && !p_complete->padded_non_union
7290 2464708 : && type_has_padding_at_level_p (TREE_TYPE (ctor)))
7291 14474 : p_complete->padded_non_union = true;
7292 :
7293 1502072 : *p_nz_elts += nz_elts;
7294 1502072 : *p_unique_nz_elts += unique_nz_elts;
7295 1502072 : *p_init_elts += init_elts;
7296 :
7297 1502072 : return const_p;
7298 : }
7299 :
7300 : /* Examine CTOR to discover:
7301 : * how many scalar fields are set to nonzero values,
7302 : and place it in *P_NZ_ELTS;
7303 : * the same, but counting RANGE_EXPRs as multiplier of 1 instead of
7304 : high - low + 1 (this can be useful for callers to determine ctors
7305 : that could be cheaply initialized with - perhaps nested - loops
7306 : compared to copied from huge read-only data),
7307 : and place it in *P_UNIQUE_NZ_ELTS;
7308 : * how many scalar fields in total are in CTOR,
7309 : and place it in *P_ELT_COUNT.
7310 : * whether the constructor is complete -- in the sense that every
7311 : meaningful byte is explicitly given a value --
7312 : and place it in *P_COMPLETE:
7313 :
7314 : Return whether or not CTOR is a valid static constant initializer, the same
7315 : as "initializer_constant_valid_p (CTOR, TREE_TYPE (CTOR)) != 0". */
7316 :
7317 : bool
7318 973748 : categorize_ctor_elements (const_tree ctor, HOST_WIDE_INT *p_nz_elts,
7319 : HOST_WIDE_INT *p_unique_nz_elts,
7320 : HOST_WIDE_INT *p_init_elts,
7321 : ctor_completeness *p_complete)
7322 : {
7323 973748 : *p_nz_elts = 0;
7324 973748 : *p_unique_nz_elts = 0;
7325 973748 : *p_init_elts = 0;
7326 973748 : *p_complete = {};
7327 :
7328 973748 : return categorize_ctor_elements_1 (ctor, p_nz_elts, p_unique_nz_elts,
7329 973748 : p_init_elts, p_complete);
7330 : }
7331 :
7332 : /* Return true if constructor CTOR is simple enough to be materialized
7333 : in an integer mode register. Limit the size to WORDS words, which
7334 : is 1 by default. */
7335 :
7336 : bool
7337 22845 : immediate_const_ctor_p (const_tree ctor, unsigned int words)
7338 : {
7339 : /* Allow function to be called with a VAR_DECL's DECL_INITIAL. */
7340 22845 : if (!ctor || TREE_CODE (ctor) != CONSTRUCTOR)
7341 : return false;
7342 :
7343 2834 : return TREE_CONSTANT (ctor)
7344 2834 : && !TREE_ADDRESSABLE (ctor)
7345 2728 : && CONSTRUCTOR_NELTS (ctor)
7346 2705 : && TREE_CODE (TREE_TYPE (ctor)) != ARRAY_TYPE
7347 630 : && int_expr_size (ctor) <= words * UNITS_PER_WORD
7348 3023 : && initializer_constant_valid_for_bitfield_p (ctor);
7349 : }
7350 :
7351 : /* TYPE is initialized by a constructor with NUM_ELTS elements, the last
7352 : of which had type LAST_TYPE. Each element was itself a complete
7353 : initializer, in the sense that every meaningful byte was explicitly
7354 : given a value. Return true if the same is true for the constructor
7355 : as a whole. */
7356 :
7357 : bool
7358 1667321 : complete_ctor_at_level_p (const_tree type, HOST_WIDE_INT num_elts,
7359 : const_tree last_type)
7360 : {
7361 1667321 : if (TREE_CODE (type) == UNION_TYPE || TREE_CODE (type) == QUAL_UNION_TYPE)
7362 : {
7363 8974 : if (num_elts == 0)
7364 : {
7365 24 : if (flag_zero_init_padding_bits >= ZERO_INIT_PADDING_BITS_UNIONS)
7366 : return false;
7367 :
7368 : /* If the CONSTRUCTOR doesn't have any elts, it is
7369 : incomplete if the union has at least one field. */
7370 327 : for (tree f = TYPE_FIELDS (type); f; f = DECL_CHAIN (f))
7371 324 : if (TREE_CODE (f) == FIELD_DECL)
7372 : return false;
7373 :
7374 : return true;
7375 : }
7376 :
7377 8950 : gcc_assert (num_elts == 1 && last_type);
7378 :
7379 8950 : if (flag_zero_init_padding_bits >= ZERO_INIT_PADDING_BITS_UNIONS)
7380 : /* ??? We could look at each element of the union, and find the
7381 : largest element. Which would avoid comparing the size of the
7382 : initialized element against any tail padding in the union.
7383 : Doesn't seem worth the effort... */
7384 102 : return simple_cst_equal (TYPE_SIZE (type), TYPE_SIZE (last_type)) == 1;
7385 :
7386 : return true;
7387 : }
7388 :
7389 1658347 : return count_type_elements (type, true) == num_elts;
7390 : }
7391 :
7392 : /* Return true if EXP contains mostly (3/4) zeros. */
7393 :
7394 : static bool
7395 352990 : mostly_zeros_p (const_tree exp)
7396 : {
7397 352990 : if (TREE_CODE (exp) == CONSTRUCTOR)
7398 : {
7399 137 : HOST_WIDE_INT nz_elts, unz_elts, init_elts;
7400 137 : ctor_completeness complete_p;
7401 :
7402 137 : categorize_ctor_elements (exp, &nz_elts, &unz_elts, &init_elts,
7403 : &complete_p);
7404 137 : return complete_p.sparse || nz_elts < init_elts / 4;
7405 : }
7406 :
7407 352853 : return initializer_zerop (exp);
7408 : }
7409 :
7410 : /* Return true if EXP contains all zeros. */
7411 :
7412 : static bool
7413 1015 : all_zeros_p (const_tree exp)
7414 : {
7415 1015 : if (TREE_CODE (exp) == CONSTRUCTOR)
7416 : {
7417 1015 : HOST_WIDE_INT nz_elts, unz_elts, init_elts;
7418 1015 : ctor_completeness complete_p;
7419 :
7420 1015 : categorize_ctor_elements (exp, &nz_elts, &unz_elts, &init_elts,
7421 : &complete_p);
7422 1015 : return nz_elts == 0;
7423 : }
7424 :
7425 0 : return initializer_zerop (exp);
7426 : }
7427 :
7428 : /* Helper function for store_constructor.
7429 : TARGET, BITSIZE, BITPOS, MODE, EXP are as for store_field.
7430 : CLEARED is as for store_constructor.
7431 : ALIAS_SET is the alias set to use for any stores.
7432 : If REVERSE is true, the store is to be done in reverse order.
7433 :
7434 : This provides a recursive shortcut back to store_constructor when it isn't
7435 : necessary to go through store_field. This is so that we can pass through
7436 : the cleared field to let store_constructor know that we may not have to
7437 : clear a substructure if the outer structure has already been cleared. */
7438 :
7439 : static void
7440 35937 : store_constructor_field (rtx target, poly_uint64 bitsize, poly_int64 bitpos,
7441 : poly_uint64 bitregion_start,
7442 : poly_uint64 bitregion_end,
7443 : machine_mode mode,
7444 : tree exp, int cleared,
7445 : alias_set_type alias_set, bool reverse)
7446 : {
7447 35937 : poly_int64 bytepos;
7448 35937 : poly_uint64 bytesize;
7449 35937 : if (TREE_CODE (exp) == CONSTRUCTOR
7450 : /* We can only call store_constructor recursively if the size and
7451 : bit position are on a byte boundary. */
7452 64 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
7453 64 : && maybe_ne (bitsize, 0U)
7454 35937 : && multiple_p (bitsize, BITS_PER_UNIT, &bytesize)
7455 : /* If we have a nonzero bitpos for a register target, then we just
7456 : let store_field do the bitfield handling. This is unlikely to
7457 : generate unnecessary clear instructions anyways. */
7458 36001 : && (known_eq (bitpos, 0) || MEM_P (target)))
7459 : {
7460 64 : if (MEM_P (target))
7461 : {
7462 22 : machine_mode target_mode = GET_MODE (target);
7463 22 : if (target_mode != BLKmode
7464 22 : && !multiple_p (bitpos, GET_MODE_ALIGNMENT (target_mode)))
7465 : target_mode = BLKmode;
7466 22 : target = adjust_address (target, target_mode, bytepos);
7467 : }
7468 :
7469 :
7470 : /* Update the alias set, if required. */
7471 22 : if (MEM_P (target) && ! MEM_KEEP_ALIAS_SET_P (target)
7472 86 : && MEM_ALIAS_SET (target) != 0)
7473 : {
7474 18 : target = copy_rtx (target);
7475 18 : set_mem_alias_set (target, alias_set);
7476 : }
7477 :
7478 64 : store_constructor (exp, target, cleared, bytesize, reverse);
7479 : }
7480 : else
7481 35873 : store_field (target, bitsize, bitpos, bitregion_start, bitregion_end, mode,
7482 : exp, alias_set, false, reverse);
7483 35937 : }
7484 :
7485 :
7486 : /* Returns the number of FIELD_DECLs in TYPE. */
7487 :
7488 : static int
7489 66331 : fields_length (const_tree type)
7490 : {
7491 66331 : tree t = TYPE_FIELDS (type);
7492 66331 : int count = 0;
7493 :
7494 773188 : for (; t; t = DECL_CHAIN (t))
7495 706857 : if (TREE_CODE (t) == FIELD_DECL)
7496 293000 : ++count;
7497 :
7498 66331 : return count;
7499 : }
7500 :
7501 :
7502 : /* Store the value of constructor EXP into the rtx TARGET.
7503 : TARGET is either a REG or a MEM; we know it cannot conflict, since
7504 : safe_from_p has been called.
7505 : CLEARED is true if TARGET is known to have been zero'd.
7506 : SIZE is the number of bytes of TARGET we are allowed to modify: this
7507 : may not be the same as the size of EXP if we are assigning to a field
7508 : which has been packed to exclude padding bits.
7509 : If REVERSE is true, the store is to be done in reverse order. */
7510 :
7511 : void
7512 241144 : store_constructor (tree exp, rtx target, int cleared, poly_int64 size,
7513 : bool reverse)
7514 : {
7515 241144 : tree type = TREE_TYPE (exp);
7516 241144 : HOST_WIDE_INT exp_size = int_size_in_bytes (type);
7517 241144 : poly_int64 bitregion_end = known_gt (size, 0) ? size * BITS_PER_UNIT - 1 : 0;
7518 :
7519 241144 : switch (TREE_CODE (type))
7520 : {
7521 66787 : case RECORD_TYPE:
7522 66787 : case UNION_TYPE:
7523 66787 : case QUAL_UNION_TYPE:
7524 66787 : {
7525 66787 : unsigned HOST_WIDE_INT idx;
7526 66787 : tree field, value;
7527 :
7528 : /* The storage order is specified for every aggregate type. */
7529 66787 : reverse = TYPE_REVERSE_STORAGE_ORDER (type);
7530 :
7531 : /* If size is zero or the target is already cleared, do nothing. */
7532 66787 : if (known_eq (size, 0) || cleared)
7533 : cleared = 1;
7534 : /* We either clear the aggregate or indicate the value is dead. */
7535 66787 : else if ((TREE_CODE (type) == UNION_TYPE
7536 66787 : || TREE_CODE (type) == QUAL_UNION_TYPE)
7537 67090 : && ! CONSTRUCTOR_ELTS (exp))
7538 : /* If the constructor is empty, clear the union. */
7539 : {
7540 254 : clear_storage (target, expr_size (exp), BLOCK_OP_NORMAL);
7541 254 : cleared = 1;
7542 : }
7543 :
7544 : /* If we are building a static constructor into a register,
7545 : set the initial value as zero so we can fold the value into
7546 : a constant. But if more than one register is involved,
7547 : this probably loses. */
7548 4141 : else if (REG_P (target) && TREE_STATIC (exp)
7549 67059 : && known_le (GET_MODE_SIZE (GET_MODE (target)),
7550 : REGMODE_NATURAL_SIZE (GET_MODE (target))))
7551 : {
7552 202 : emit_move_insn (target, CONST0_RTX (GET_MODE (target)));
7553 202 : cleared = 1;
7554 : }
7555 :
7556 : /* If the constructor has fewer fields than the structure or
7557 : if we are initializing the structure to mostly zeros, clear
7558 : the whole structure first. Don't do this if TARGET is a
7559 : register whose mode size isn't equal to SIZE since
7560 : clear_storage can't handle this case. */
7561 66331 : else if (known_size_p (size)
7562 66440 : && (((int) CONSTRUCTOR_NELTS (exp) != fields_length (type))
7563 96 : || mostly_zeros_p (exp))
7564 132570 : && (!REG_P (target)
7565 7868 : || known_eq (GET_MODE_SIZE (GET_MODE (target)), size)))
7566 : {
7567 76948 : clear_storage (target, gen_int_mode (size, Pmode),
7568 : BLOCK_OP_NORMAL);
7569 66239 : cleared = 1;
7570 : }
7571 :
7572 66787 : if (REG_P (target) && !cleared)
7573 5 : emit_clobber (target);
7574 :
7575 : /* Store each element of the constructor into the
7576 : corresponding field of TARGET. */
7577 67039 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (exp), idx, field, value)
7578 : {
7579 252 : machine_mode mode;
7580 252 : HOST_WIDE_INT bitsize;
7581 252 : HOST_WIDE_INT bitpos = 0;
7582 252 : tree offset;
7583 252 : rtx to_rtx = target;
7584 :
7585 : /* Just ignore missing fields. We cleared the whole
7586 : structure, above, if any fields are missing. */
7587 252 : if (field == 0)
7588 252 : continue;
7589 :
7590 252 : if (cleared && initializer_zerop (value))
7591 5 : continue;
7592 :
7593 : /* Variable sized arrays are ignored. */
7594 247 : tree decl_size = DECL_SIZE (field);
7595 247 : if (!decl_size)
7596 4 : continue;
7597 :
7598 243 : if (tree_fits_uhwi_p (decl_size))
7599 243 : bitsize = tree_to_uhwi (decl_size);
7600 : else
7601 0 : gcc_unreachable ();
7602 :
7603 243 : mode = DECL_MODE (field);
7604 243 : if (DECL_BIT_FIELD (field))
7605 24 : mode = VOIDmode;
7606 :
7607 243 : offset = DECL_FIELD_OFFSET (field);
7608 243 : if (tree_fits_shwi_p (offset)
7609 243 : && tree_fits_shwi_p (bit_position (field)))
7610 : {
7611 243 : bitpos = int_bit_position (field);
7612 243 : offset = NULL_TREE;
7613 : }
7614 : else
7615 0 : gcc_unreachable ();
7616 :
7617 : /* If this initializes a field that is smaller than a
7618 : word, at the start of a word, try to widen it to a full
7619 : word. This special case allows us to output C++ member
7620 : function initializations in a form that the optimizers
7621 : can understand. */
7622 243 : if (WORD_REGISTER_OPERATIONS
7623 : && REG_P (target)
7624 : && bitsize < BITS_PER_WORD
7625 : && bitpos % BITS_PER_WORD == 0
7626 : && GET_MODE_CLASS (mode) == MODE_INT
7627 : && TREE_CODE (value) == INTEGER_CST
7628 : && exp_size >= 0
7629 : && bitpos + BITS_PER_WORD <= exp_size * BITS_PER_UNIT)
7630 : {
7631 : type = TREE_TYPE (value);
7632 :
7633 : if (TYPE_PRECISION (type) < BITS_PER_WORD)
7634 : {
7635 : type = lang_hooks.types.type_for_mode
7636 : (word_mode, TYPE_UNSIGNED (type));
7637 : value = fold_convert (type, value);
7638 : /* Make sure the bits beyond the original bitsize are zero
7639 : so that we can correctly avoid extra zeroing stores in
7640 : later constructor elements. */
7641 : tree bitsize_mask
7642 : = wide_int_to_tree (type, wi::mask (bitsize, false,
7643 : BITS_PER_WORD));
7644 : value = fold_build2 (BIT_AND_EXPR, type, value, bitsize_mask);
7645 : }
7646 :
7647 : if (BYTES_BIG_ENDIAN)
7648 : value
7649 : = fold_build2 (LSHIFT_EXPR, type, value,
7650 : build_int_cst (type,
7651 : BITS_PER_WORD - bitsize));
7652 : bitsize = BITS_PER_WORD;
7653 : mode = word_mode;
7654 : }
7655 :
7656 125 : if (MEM_P (to_rtx) && !MEM_KEEP_ALIAS_SET_P (to_rtx)
7657 368 : && DECL_NONADDRESSABLE_P (field))
7658 : {
7659 0 : to_rtx = copy_rtx (to_rtx);
7660 0 : MEM_KEEP_ALIAS_SET_P (to_rtx) = 1;
7661 : }
7662 :
7663 243 : store_constructor_field (to_rtx, bitsize, bitpos,
7664 243 : 0, bitregion_end, mode,
7665 : value, cleared,
7666 243 : get_alias_set (TREE_TYPE (field)),
7667 : reverse);
7668 : }
7669 : break;
7670 : }
7671 39080 : case ARRAY_TYPE:
7672 39080 : {
7673 39080 : tree value, index;
7674 39080 : unsigned HOST_WIDE_INT i, j = 0;
7675 39080 : bool need_to_clear;
7676 39080 : tree domain;
7677 39080 : tree elttype = TREE_TYPE (type);
7678 39080 : bool const_bounds_p;
7679 39080 : unsigned HOST_WIDE_INT minelt = 0;
7680 39080 : unsigned HOST_WIDE_INT maxelt = 0;
7681 :
7682 : /* The storage order is specified for every aggregate type. */
7683 39080 : reverse = TYPE_REVERSE_STORAGE_ORDER (type);
7684 :
7685 39080 : domain = TYPE_DOMAIN (type);
7686 39080 : const_bounds_p = (TYPE_MIN_VALUE (domain)
7687 39080 : && TYPE_MAX_VALUE (domain)
7688 39080 : && tree_fits_uhwi_p (TYPE_MIN_VALUE (domain))
7689 78160 : && tree_fits_uhwi_p (TYPE_MAX_VALUE (domain)));
7690 :
7691 : /* If we have constant bounds for the range of the type, get them. */
7692 39080 : if (const_bounds_p)
7693 : {
7694 39080 : minelt = tree_to_uhwi (TYPE_MIN_VALUE (domain));
7695 39080 : maxelt = tree_to_uhwi (TYPE_MAX_VALUE (domain));
7696 : }
7697 :
7698 : /* If the constructor has fewer elements than the array, clear
7699 : the whole array first. Similarly if this is static
7700 : constructor of a non-BLKmode object. */
7701 39080 : if (cleared)
7702 : need_to_clear = false;
7703 39034 : else if (REG_P (target) && TREE_STATIC (exp))
7704 : need_to_clear = true;
7705 : else
7706 : {
7707 38929 : unsigned HOST_WIDE_INT idx;
7708 38929 : unsigned HOST_WIDE_INT count = 0, zero_count = 0;
7709 38929 : need_to_clear = ! const_bounds_p;
7710 :
7711 : /* This loop is a more accurate version of the loop in
7712 : mostly_zeros_p (it handles RANGE_EXPR in an index). It
7713 : is also needed to check for missing elements. */
7714 38977 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (exp), idx, index, value)
7715 : {
7716 48 : unsigned HOST_WIDE_INT this_node_count;
7717 :
7718 48 : if (need_to_clear)
7719 : break;
7720 :
7721 48 : if (index != NULL_TREE && TREE_CODE (index) == RANGE_EXPR)
7722 : {
7723 0 : tree lo_index = TREE_OPERAND (index, 0);
7724 0 : tree hi_index = TREE_OPERAND (index, 1);
7725 :
7726 0 : if (! tree_fits_uhwi_p (lo_index)
7727 0 : || ! tree_fits_uhwi_p (hi_index))
7728 : {
7729 : need_to_clear = true;
7730 : break;
7731 : }
7732 :
7733 0 : this_node_count = (tree_to_uhwi (hi_index)
7734 0 : - tree_to_uhwi (lo_index) + 1);
7735 0 : }
7736 48 : else if (TREE_CODE (value) == RAW_DATA_CST)
7737 0 : this_node_count = RAW_DATA_LENGTH (value);
7738 : else
7739 : this_node_count = 1;
7740 :
7741 48 : count += this_node_count;
7742 48 : if (mostly_zeros_p (value))
7743 0 : zero_count += this_node_count;
7744 : }
7745 :
7746 : /* Clear the entire array first if there are any missing
7747 : elements, or if the incidence of zero elements is >=
7748 : 75%. */
7749 38929 : if (! need_to_clear
7750 38929 : && (count < maxelt - minelt + 1
7751 6 : || 4 * zero_count >= 3 * count))
7752 : need_to_clear = true;
7753 : }
7754 :
7755 39028 : if (need_to_clear && maybe_gt (size, 0))
7756 : {
7757 39028 : if (REG_P (target))
7758 1580 : emit_move_insn (target, CONST0_RTX (GET_MODE (target)));
7759 : else
7760 42207 : clear_storage (target, gen_int_mode (size, Pmode),
7761 : BLOCK_OP_NORMAL);
7762 : cleared = 1;
7763 : }
7764 :
7765 52 : if (!cleared && REG_P (target))
7766 : /* Inform later passes that the old value is dead. */
7767 0 : emit_clobber (target);
7768 :
7769 : /* Store each element of the constructor into the
7770 : corresponding element of TARGET, determined by counting the
7771 : elements. */
7772 40236 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (exp), i, index, value)
7773 : {
7774 1156 : machine_mode mode;
7775 1156 : poly_int64 bitsize;
7776 1156 : HOST_WIDE_INT bitpos;
7777 1156 : rtx xtarget = target;
7778 :
7779 1156 : if (cleared && initializer_zerop (value))
7780 : {
7781 672 : if (TREE_CODE (value) == RAW_DATA_CST)
7782 0 : j += RAW_DATA_LENGTH (value) - 1;
7783 674 : continue;
7784 : }
7785 :
7786 484 : mode = TYPE_MODE (elttype);
7787 484 : if (mode != BLKmode)
7788 968 : bitsize = GET_MODE_BITSIZE (mode);
7789 0 : else if (!poly_int_tree_p (TYPE_SIZE (elttype), &bitsize))
7790 0 : bitsize = -1;
7791 :
7792 484 : if (index != NULL_TREE && TREE_CODE (index) == RANGE_EXPR)
7793 : {
7794 0 : tree lo_index = TREE_OPERAND (index, 0);
7795 0 : tree hi_index = TREE_OPERAND (index, 1);
7796 0 : rtx index_r;
7797 0 : unsigned HOST_WIDE_INT lo, hi, count;
7798 0 : tree offset;
7799 :
7800 0 : gcc_assert (TREE_CODE (value) != RAW_DATA_CST);
7801 :
7802 : /* If the range is constant and "small", unroll the loop. */
7803 0 : if (const_bounds_p
7804 0 : && tree_fits_uhwi_p (lo_index)
7805 0 : && tree_fits_uhwi_p (hi_index)
7806 0 : && (lo = tree_to_uhwi (lo_index),
7807 0 : hi = tree_to_uhwi (hi_index),
7808 0 : count = hi - lo + 1,
7809 0 : (!MEM_P (target)
7810 0 : || count <= 2
7811 0 : || (tree_fits_uhwi_p (TYPE_SIZE (elttype))
7812 0 : && (tree_to_uhwi (TYPE_SIZE (elttype)) * count
7813 : <= 40 * 8)))))
7814 : {
7815 0 : lo -= minelt; hi -= minelt;
7816 0 : for (; lo <= hi; lo++)
7817 : {
7818 0 : bitpos = lo * tree_to_uhwi (TYPE_SIZE (elttype));
7819 :
7820 0 : if (MEM_P (target)
7821 0 : && !MEM_KEEP_ALIAS_SET_P (target)
7822 0 : && TREE_CODE (type) == ARRAY_TYPE
7823 0 : && TYPE_NONALIASED_COMPONENT (type))
7824 : {
7825 0 : target = copy_rtx (target);
7826 0 : MEM_KEEP_ALIAS_SET_P (target) = 1;
7827 : }
7828 :
7829 0 : store_constructor_field
7830 0 : (target, bitsize, bitpos, 0, bitregion_end,
7831 : mode, value, cleared,
7832 : get_alias_set (elttype), reverse);
7833 : }
7834 : }
7835 : else
7836 : {
7837 0 : rtx_code_label *loop_start = gen_label_rtx ();
7838 0 : rtx_code_label *loop_end = gen_label_rtx ();
7839 0 : tree exit_cond;
7840 :
7841 0 : expand_normal (hi_index);
7842 :
7843 0 : index = build_decl (EXPR_LOCATION (exp),
7844 : VAR_DECL, NULL_TREE, domain);
7845 0 : index_r = gen_reg_rtx (promote_decl_mode (index, NULL));
7846 0 : SET_DECL_RTL (index, index_r);
7847 0 : store_expr (lo_index, index_r, 0, false, reverse);
7848 :
7849 : /* Build the head of the loop. */
7850 0 : do_pending_stack_adjust ();
7851 0 : emit_label (loop_start);
7852 :
7853 : /* Assign value to element index. */
7854 0 : offset = fold_build2 (MINUS_EXPR,
7855 : TREE_TYPE (index),
7856 : index,
7857 : TYPE_MIN_VALUE (domain));
7858 :
7859 0 : offset = size_binop (MULT_EXPR,
7860 : fold_convert (sizetype, offset),
7861 : TYPE_SIZE_UNIT (elttype));
7862 :
7863 0 : xtarget = offset_address (target,
7864 : expand_normal (offset),
7865 : highest_pow2_factor (offset));
7866 0 : xtarget = adjust_address (xtarget, mode, 0);
7867 0 : if (TREE_CODE (value) == CONSTRUCTOR)
7868 0 : store_constructor (value, xtarget, cleared,
7869 : exact_div (bitsize, BITS_PER_UNIT),
7870 : reverse);
7871 : else
7872 0 : store_expr (value, xtarget, 0, false, reverse);
7873 :
7874 : /* Generate a conditional jump to exit the loop. */
7875 0 : exit_cond = build2 (LT_EXPR, integer_type_node,
7876 : index, hi_index);
7877 0 : jumpif (exit_cond, loop_end,
7878 : profile_probability::uninitialized ());
7879 :
7880 : /* Update the loop counter, and jump to the head of
7881 : the loop. */
7882 0 : expand_assignment (index,
7883 0 : build2 (PLUS_EXPR, TREE_TYPE (index),
7884 : index, integer_one_node),
7885 : false);
7886 :
7887 0 : emit_jump (loop_start);
7888 :
7889 : /* Build the end of the loop. */
7890 0 : emit_label (loop_end);
7891 : }
7892 : }
7893 484 : else if ((index && !tree_fits_uhwi_p (index))
7894 484 : || !tree_fits_uhwi_p (TYPE_SIZE (elttype)))
7895 : {
7896 0 : tree offset;
7897 :
7898 0 : gcc_assert (TREE_CODE (value) != RAW_DATA_CST);
7899 0 : if (index)
7900 0 : offset = fold_build2 (MINUS_EXPR,
7901 : TREE_TYPE (index),
7902 : index,
7903 : TYPE_MIN_VALUE (domain));
7904 : else
7905 0 : offset = size_int (i + j);
7906 :
7907 0 : offset = size_binop (MULT_EXPR,
7908 : fold_convert (sizetype, offset),
7909 : TYPE_SIZE_UNIT (elttype));
7910 0 : xtarget = offset_address (target,
7911 : expand_normal (offset),
7912 : highest_pow2_factor (offset));
7913 0 : xtarget = adjust_address (xtarget, mode, 0);
7914 0 : store_expr (value, xtarget, 0, false, reverse);
7915 : }
7916 : else
7917 : {
7918 484 : if (index)
7919 968 : bitpos = ((tree_to_uhwi (index) - minelt)
7920 484 : * tree_to_uhwi (TYPE_SIZE (elttype)));
7921 : else
7922 0 : bitpos = ((i + j) * tree_to_uhwi (TYPE_SIZE (elttype)));
7923 :
7924 52 : if (MEM_P (target) && !MEM_KEEP_ALIAS_SET_P (target)
7925 52 : && TREE_CODE (type) == ARRAY_TYPE
7926 536 : && TYPE_NONALIASED_COMPONENT (type))
7927 : {
7928 0 : target = copy_rtx (target);
7929 0 : MEM_KEEP_ALIAS_SET_P (target) = 1;
7930 : }
7931 484 : if (TREE_CODE (value) != RAW_DATA_CST)
7932 480 : store_constructor_field (target, bitsize, bitpos, 0,
7933 480 : bitregion_end, mode, value,
7934 : cleared, get_alias_set (elttype),
7935 : reverse);
7936 : else
7937 : {
7938 4 : j += RAW_DATA_LENGTH (value) - 1;
7939 4 : gcc_assert (known_eq (bitsize, BITS_PER_UNIT));
7940 4 : rtx to_rtx = adjust_address (target, mode,
7941 : bitpos / BITS_PER_UNIT);
7942 :
7943 4 : if (to_rtx == target)
7944 0 : to_rtx = copy_rtx (to_rtx);
7945 :
7946 4 : if (!MEM_KEEP_ALIAS_SET_P (to_rtx)
7947 8 : && MEM_ALIAS_SET (to_rtx) != 0)
7948 0 : set_mem_alias_set (to_rtx, get_alias_set (elttype));
7949 :
7950 8 : if (can_store_by_pieces (RAW_DATA_LENGTH (value),
7951 : raw_data_cst_read_str,
7952 : (void *) value,
7953 4 : MEM_ALIGN (target), false))
7954 : {
7955 4 : store_by_pieces (target, RAW_DATA_LENGTH (value),
7956 : raw_data_cst_read_str, (void *) value,
7957 2 : MEM_ALIGN (target), false,
7958 : RETURN_BEGIN);
7959 2 : continue;
7960 : }
7961 :
7962 2 : elttype
7963 2 : = build_array_type_nelts (TREE_TYPE (value),
7964 2 : RAW_DATA_LENGTH (value));
7965 2 : tree ctor = build_constructor_single (elttype, NULL_TREE,
7966 : value);
7967 2 : ctor = tree_output_constant_def (ctor);
7968 2 : mode = TYPE_MODE (type);
7969 2 : store_constructor_field (target,
7970 2 : bitsize * RAW_DATA_LENGTH (value),
7971 2 : bitpos, 0, bitregion_end, mode,
7972 : ctor, cleared,
7973 : get_alias_set (elttype), reverse);
7974 : }
7975 : }
7976 : }
7977 : break;
7978 : }
7979 :
7980 135277 : case VECTOR_TYPE:
7981 135277 : {
7982 135277 : unsigned HOST_WIDE_INT idx;
7983 135277 : constructor_elt *ce;
7984 135277 : bool need_to_clear;
7985 135277 : insn_code icode = CODE_FOR_nothing;
7986 135277 : tree elt;
7987 135277 : tree elttype = TREE_TYPE (type);
7988 135277 : int elt_size = vector_element_bits (type);
7989 135277 : machine_mode eltmode = TYPE_MODE (elttype);
7990 135277 : poly_int64 bitsize;
7991 135277 : poly_int64 bitpos;
7992 135277 : rtvec vector = NULL;
7993 135277 : poly_uint64 n_elts;
7994 135277 : unsigned HOST_WIDE_INT const_n_elts;
7995 135277 : alias_set_type alias;
7996 135277 : bool vec_vec_init_p = false;
7997 135277 : machine_mode mode = GET_MODE (target);
7998 :
7999 135277 : gcc_assert (eltmode != BLKmode);
8000 :
8001 : /* Try using vec_duplicate_optab for uniform vectors. */
8002 135277 : icode = optab_handler (vec_duplicate_optab, mode);
8003 135277 : if (!TREE_SIDE_EFFECTS (exp)
8004 135277 : && VECTOR_MODE_P (mode)
8005 131885 : && icode != CODE_FOR_nothing
8006 : /* If the vec_duplicate target pattern does not specify an element
8007 : mode check that eltmode is the normal inner mode of the
8008 : requested vector mode. But if the target allows eltmode
8009 : explicitly go ahead and use it. */
8010 90745 : && (eltmode == GET_MODE_INNER (mode)
8011 0 : || insn_data[icode].operand[1].mode == eltmode)
8012 90745 : && (elt = uniform_vector_p (exp))
8013 152665 : && !VECTOR_TYPE_P (TREE_TYPE (elt)))
8014 : {
8015 17388 : class expand_operand ops[2];
8016 17388 : create_output_operand (&ops[0], target, mode);
8017 17388 : create_input_operand (&ops[1], expand_normal (elt), eltmode);
8018 17388 : expand_insn (icode, 2, ops);
8019 17388 : if (!rtx_equal_p (target, ops[0].value))
8020 0 : emit_move_insn (target, ops[0].value);
8021 17388 : break;
8022 : }
8023 : /* Use sign-extension for uniform boolean vectors with
8024 : integer modes and single-bit mask entries.
8025 : Effectively "vec_duplicate" for bitmasks. */
8026 117889 : if (elt_size == 1
8027 146 : && !TREE_SIDE_EFFECTS (exp)
8028 146 : && VECTOR_BOOLEAN_TYPE_P (type)
8029 146 : && SCALAR_INT_MODE_P (TYPE_MODE (type))
8030 146 : && (elt = uniform_vector_p (exp))
8031 143 : && !VECTOR_TYPE_P (TREE_TYPE (elt))
8032 117889 : && !BYTES_BIG_ENDIAN)
8033 : {
8034 143 : rtx op0 = force_reg (TYPE_MODE (TREE_TYPE (elt)),
8035 : expand_normal (elt));
8036 143 : rtx tmp = gen_reg_rtx (mode);
8037 143 : convert_move (tmp, op0, 0);
8038 :
8039 : /* Ensure no excess bits are set.
8040 : GCN needs this for nunits < 64.
8041 : x86 needs this for nunits < 8. */
8042 143 : unsigned int nunits = TYPE_VECTOR_SUBPARTS (type).to_constant ();
8043 143 : if (maybe_ne (GET_MODE_PRECISION (mode), nunits))
8044 2 : tmp = expand_binop (mode, and_optab, tmp,
8045 2 : GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
8046 : target, true, OPTAB_WIDEN);
8047 143 : if (tmp != target)
8048 141 : emit_move_insn (target, tmp);
8049 : break;
8050 : }
8051 :
8052 117746 : n_elts = TYPE_VECTOR_SUBPARTS (type);
8053 117746 : if (REG_P (target)
8054 114656 : && VECTOR_MODE_P (mode))
8055 : {
8056 114497 : const_n_elts = 0;
8057 114497 : if (CONSTRUCTOR_NELTS (exp)
8058 114497 : && (TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (exp, 0)->value))
8059 : == VECTOR_TYPE))
8060 : {
8061 1151 : tree etype = TREE_TYPE (CONSTRUCTOR_ELT (exp, 0)->value);
8062 1151 : gcc_assert (known_eq (CONSTRUCTOR_NELTS (exp)
8063 : * TYPE_VECTOR_SUBPARTS (etype),
8064 : n_elts));
8065 :
8066 3453 : icode = convert_optab_handler (vec_init_optab, mode,
8067 1151 : TYPE_MODE (etype));
8068 1151 : const_n_elts = CONSTRUCTOR_NELTS (exp);
8069 1151 : vec_vec_init_p = icode != CODE_FOR_nothing;
8070 : }
8071 226692 : else if (exact_div (n_elts, GET_MODE_NUNITS (eltmode))
8072 113346 : .is_constant (&const_n_elts))
8073 : {
8074 : /* For a non-const type vector, we check it is made up of
8075 : similarly non-const type vectors. */
8076 113346 : icode = convert_optab_handler (vec_init_optab, mode, eltmode);
8077 : }
8078 :
8079 114497 : if (const_n_elts && icode != CODE_FOR_nothing)
8080 : {
8081 113829 : vector = rtvec_alloc (const_n_elts);
8082 424785 : for (unsigned int k = 0; k < const_n_elts; k++)
8083 310956 : RTVEC_ELT (vector, k) = CONST0_RTX (eltmode);
8084 : }
8085 : }
8086 :
8087 : /* Compute the size of the elements in the CTOR. It differs
8088 : from the size of the vector type elements only when the
8089 : CTOR elements are vectors themselves. */
8090 117746 : tree val_type = (CONSTRUCTOR_NELTS (exp) != 0
8091 117746 : ? TREE_TYPE (CONSTRUCTOR_ELT (exp, 0)->value)
8092 117746 : : elttype);
8093 117746 : if (VECTOR_TYPE_P (val_type))
8094 1471 : bitsize = tree_to_poly_uint64 (TYPE_SIZE (val_type));
8095 : else
8096 116275 : bitsize = elt_size;
8097 :
8098 : /* If the constructor has fewer elements than the vector,
8099 : clear the whole array first. Similarly if this is static
8100 : constructor of a non-BLKmode object. */
8101 117746 : if (cleared)
8102 : need_to_clear = false;
8103 117746 : else if (REG_P (target) && TREE_STATIC (exp))
8104 : need_to_clear = true;
8105 : else
8106 : {
8107 : poly_uint64 count = 0, zero_count = 0;
8108 : tree value;
8109 :
8110 470551 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), idx, value)
8111 : {
8112 352805 : poly_int64 n_elts_here = exact_div (bitsize, elt_size);
8113 352805 : count += n_elts_here;
8114 352805 : if (mostly_zeros_p (value))
8115 352805 : zero_count += n_elts_here;
8116 : }
8117 :
8118 : /* Clear the entire vector first if there are any missing elements,
8119 : or if the incidence of zero elements is >= 75%. */
8120 117746 : need_to_clear = (maybe_lt (count, n_elts)
8121 117746 : || maybe_gt (4 * zero_count, 3 * count));
8122 : }
8123 :
8124 1100 : if (need_to_clear && maybe_gt (size, 0) && !vector)
8125 : {
8126 683 : if (REG_P (target))
8127 1 : emit_move_insn (target, CONST0_RTX (mode));
8128 : else
8129 682 : clear_storage (target, gen_int_mode (size, Pmode),
8130 : BLOCK_OP_NORMAL);
8131 : cleared = 1;
8132 : }
8133 :
8134 : /* Inform later passes that the old value is dead. */
8135 117746 : if (!cleared && !vector && REG_P (target) && maybe_gt (n_elts, 1u))
8136 : {
8137 513 : emit_move_insn (target, CONST0_RTX (mode));
8138 513 : cleared = 1;
8139 : }
8140 :
8141 117746 : if (MEM_P (target))
8142 3090 : alias = MEM_ALIAS_SET (target);
8143 : else
8144 114656 : alias = get_alias_set (elttype);
8145 :
8146 : /* Store each element of the constructor into the corresponding
8147 : element of TARGET, determined by counting the elements. */
8148 117746 : HOST_WIDE_INT chunk_size = 0;
8149 117746 : bool chunk_multiple_p = constant_multiple_p (bitsize, elt_size,
8150 : &chunk_size);
8151 117746 : gcc_assert (chunk_multiple_p || vec_vec_init_p);
8152 :
8153 470551 : for (idx = 0; vec_safe_iterate (CONSTRUCTOR_ELTS (exp), idx, &ce);
8154 : idx++)
8155 : {
8156 352805 : HOST_WIDE_INT eltpos;
8157 352805 : tree value = ce->value;
8158 :
8159 352805 : if (cleared && initializer_zerop (value))
8160 7654 : continue;
8161 :
8162 345151 : if (ce->index)
8163 45764 : eltpos = tree_to_uhwi (ce->index);
8164 : else
8165 299387 : eltpos = idx * chunk_size;
8166 :
8167 345151 : if (vector)
8168 : {
8169 309939 : if (vec_vec_init_p)
8170 : {
8171 2242 : gcc_assert (ce->index == NULL_TREE);
8172 2242 : gcc_assert (TREE_CODE (TREE_TYPE (value)) == VECTOR_TYPE);
8173 2242 : eltpos = idx;
8174 : }
8175 : else
8176 307697 : gcc_assert (TREE_CODE (TREE_TYPE (value)) != VECTOR_TYPE);
8177 309939 : RTVEC_ELT (vector, eltpos) = expand_normal (value);
8178 : }
8179 : else
8180 : {
8181 35212 : machine_mode value_mode
8182 35212 : = (TREE_CODE (TREE_TYPE (value)) == VECTOR_TYPE
8183 35212 : ? TYPE_MODE (TREE_TYPE (value)) : eltmode);
8184 35212 : bitpos = eltpos * elt_size;
8185 35212 : store_constructor_field (target, bitsize, bitpos, 0,
8186 35212 : bitregion_end, value_mode,
8187 : value, cleared, alias, reverse);
8188 : }
8189 : }
8190 :
8191 117746 : if (vector)
8192 113829 : emit_insn (GEN_FCN (icode) (target,
8193 : gen_rtx_PARALLEL (mode, vector)));
8194 : break;
8195 : }
8196 :
8197 0 : default:
8198 0 : gcc_unreachable ();
8199 : }
8200 241144 : }
8201 :
8202 : /* Store the value of EXP (an expression tree)
8203 : into a subfield of TARGET which has mode MODE and occupies
8204 : BITSIZE bits, starting BITPOS bits from the start of TARGET.
8205 : If MODE is VOIDmode, it means that we are storing into a bit-field.
8206 :
8207 : BITREGION_START is bitpos of the first bitfield in this region.
8208 : BITREGION_END is the bitpos of the ending bitfield in this region.
8209 : These two fields are 0, if the C++ memory model does not apply,
8210 : or we are not interested in keeping track of bitfield regions.
8211 :
8212 : Always return const0_rtx unless we have something particular to
8213 : return.
8214 :
8215 : ALIAS_SET is the alias set for the destination. This value will
8216 : (in general) be different from that for TARGET, since TARGET is a
8217 : reference to the containing structure.
8218 :
8219 : If NONTEMPORAL is true, try generating a nontemporal store.
8220 :
8221 : If REVERSE is true, the store is to be done in reverse order. */
8222 :
8223 : static rtx
8224 4719000 : store_field (rtx target, poly_int64 bitsize, poly_int64 bitpos,
8225 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
8226 : machine_mode mode, tree exp,
8227 : alias_set_type alias_set, bool nontemporal, bool reverse)
8228 : {
8229 4719000 : if (TREE_CODE (exp) == ERROR_MARK)
8230 0 : return const0_rtx;
8231 :
8232 : /* If we have nothing to store, do nothing unless the expression has
8233 : side-effects. Don't do that for zero sized addressable lhs of
8234 : calls. */
8235 4719000 : if (known_eq (bitsize, 0)
8236 4719000 : && (!TREE_ADDRESSABLE (TREE_TYPE (exp))
8237 3 : || TREE_CODE (exp) != CALL_EXPR))
8238 0 : return expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
8239 :
8240 4719000 : if (GET_CODE (target) == CONCAT)
8241 : {
8242 : /* We're storing into a struct containing a single __complex. */
8243 :
8244 0 : gcc_assert (known_eq (bitpos, 0));
8245 0 : return store_expr (exp, target, 0, nontemporal, reverse);
8246 : }
8247 :
8248 : /* If the structure is in a register or if the component
8249 : is a bit field, we cannot use addressing to access it.
8250 : Use bit-field techniques or SUBREG to store in it. */
8251 :
8252 4719000 : poly_int64 decl_bitsize;
8253 4719000 : if (mode == VOIDmode
8254 4652355 : || (mode != BLKmode && ! direct_store[(int) mode]
8255 5338 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_INT
8256 5336 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_FLOAT)
8257 4647048 : || REG_P (target)
8258 3974212 : || GET_CODE (target) == SUBREG
8259 : /* If the field isn't aligned enough to store as an ordinary memref,
8260 : store it as a bit field. */
8261 3974212 : || (mode != BLKmode
8262 3859721 : && ((((MEM_ALIGN (target) < GET_MODE_ALIGNMENT (mode))
8263 7631714 : || !multiple_p (bitpos, GET_MODE_ALIGNMENT (mode)))
8264 46631 : && targetm.slow_unaligned_access (mode, MEM_ALIGN (target)))
8265 3859721 : || !multiple_p (bitpos, BITS_PER_UNIT)))
8266 3974212 : || (known_size_p (bitsize)
8267 3974200 : && mode != BLKmode
8268 3859721 : && maybe_gt (GET_MODE_BITSIZE (mode), bitsize))
8269 : /* If the RHS and field are a constant size and the size of the
8270 : RHS isn't the same size as the bitfield, we must use bitfield
8271 : operations. */
8272 3974203 : || (known_size_p (bitsize)
8273 3974191 : && poly_int_tree_p (TYPE_SIZE (TREE_TYPE (exp)))
8274 3974191 : && maybe_ne (wi::to_poly_offset (TYPE_SIZE (TREE_TYPE (exp))),
8275 : bitsize)
8276 : /* Except for initialization of full bytes from a CONSTRUCTOR, which
8277 : we will handle specially below. */
8278 34 : && !(TREE_CODE (exp) == CONSTRUCTOR
8279 11 : && multiple_p (bitsize, BITS_PER_UNIT))
8280 : /* And except for bitwise copying of TREE_ADDRESSABLE types,
8281 : where the FIELD_DECL has the right bitsize, but TREE_TYPE (exp)
8282 : includes some extra padding. store_expr / expand_expr will in
8283 : that case call get_inner_reference that will have the bitsize
8284 : we check here and thus the block move will not clobber the
8285 : padding that shouldn't be clobbered. In the future we could
8286 : replace the TREE_ADDRESSABLE check with a check that
8287 : get_base_address needs to live in memory. */
8288 23 : && (!TREE_ADDRESSABLE (TREE_TYPE (exp))
8289 9 : || TREE_CODE (exp) != COMPONENT_REF
8290 6 : || !multiple_p (bitsize, BITS_PER_UNIT)
8291 6 : || !multiple_p (bitpos, BITS_PER_UNIT)
8292 6 : || !poly_int_tree_p (DECL_SIZE (TREE_OPERAND (exp, 1)),
8293 : &decl_bitsize)
8294 6 : || maybe_ne (decl_bitsize, bitsize))
8295 : /* A call with an addressable return type and return-slot
8296 : optimization must not need bitfield operations but we must
8297 : pass down the original target. */
8298 17 : && (TREE_CODE (exp) != CALL_EXPR
8299 6 : || !TREE_ADDRESSABLE (TREE_TYPE (exp))
8300 0 : || !CALL_EXPR_RETURN_SLOT_OPT (exp)))
8301 : /* If we are expanding a MEM_REF of a non-BLKmode non-addressable
8302 : decl we must use bitfield operations. */
8303 8693186 : || (known_size_p (bitsize)
8304 3974174 : && TREE_CODE (exp) == MEM_REF
8305 30287 : && TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR
8306 24384 : && DECL_P (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
8307 20048 : && !TREE_ADDRESSABLE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
8308 5489 : && DECL_MODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0)) != BLKmode))
8309 : {
8310 745068 : rtx temp;
8311 745068 : gimple *nop_def;
8312 :
8313 : /* If EXP is a NOP_EXPR of precision less than its mode, then that
8314 : implies a mask operation. If the precision is the same size as
8315 : the field we're storing into, that mask is redundant. This is
8316 : particularly common with bit field assignments generated by the
8317 : C front end. */
8318 745068 : nop_def = get_def_for_expr (exp, NOP_EXPR);
8319 745068 : if (nop_def)
8320 : {
8321 5532 : tree type = TREE_TYPE (exp);
8322 5532 : if (INTEGRAL_TYPE_P (type)
8323 5325 : && maybe_ne (TYPE_PRECISION (type),
8324 10650 : GET_MODE_BITSIZE (TYPE_MODE (type)))
8325 8924 : && known_eq (bitsize, TYPE_PRECISION (type)))
8326 : {
8327 3287 : tree op = gimple_assign_rhs1 (nop_def);
8328 3287 : type = TREE_TYPE (op);
8329 3287 : if (INTEGRAL_TYPE_P (type)
8330 3287 : && known_ge (TYPE_PRECISION (type), bitsize))
8331 : exp = op;
8332 : }
8333 : }
8334 :
8335 745068 : temp = expand_normal (exp);
8336 :
8337 : /* We don't support variable-sized BLKmode bitfields, since our
8338 : handling of BLKmode is bound up with the ability to break
8339 : things into words. */
8340 745068 : gcc_assert (mode != BLKmode || bitsize.is_constant ());
8341 :
8342 : /* Handle calls that return values in multiple non-contiguous locations.
8343 : The Irix 6 ABI has examples of this. */
8344 745068 : if (GET_CODE (temp) == PARALLEL)
8345 : {
8346 6 : HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));
8347 6 : machine_mode temp_mode = GET_MODE (temp);
8348 6 : if (temp_mode == BLKmode || temp_mode == VOIDmode)
8349 6 : temp_mode
8350 6 : = smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
8351 6 : rtx temp_target = gen_reg_rtx (temp_mode);
8352 6 : emit_group_store (temp_target, temp, TREE_TYPE (exp), size);
8353 6 : temp = temp_target;
8354 : }
8355 :
8356 : /* Handle calls that return BLKmode values in registers. */
8357 745062 : else if (mode == BLKmode && REG_P (temp) && TREE_CODE (exp) == CALL_EXPR)
8358 : {
8359 0 : rtx temp_target = gen_reg_rtx (GET_MODE (temp));
8360 0 : copy_blkmode_from_reg (temp_target, temp, TREE_TYPE (exp));
8361 0 : temp = temp_target;
8362 : }
8363 :
8364 : /* If the value has aggregate type and an integral mode then, if BITSIZE
8365 : is narrower than this mode and this is for big-endian data, we first
8366 : need to put the value into the low-order bits for store_bit_field,
8367 : except when MODE is BLKmode and BITSIZE larger than the word size
8368 : (see the handling of fields larger than a word in store_bit_field).
8369 : Moreover, the field may be not aligned on a byte boundary; in this
8370 : case, if it has reverse storage order, it needs to be accessed as a
8371 : scalar field with reverse storage order and we must first put the
8372 : value into target order. */
8373 745068 : scalar_int_mode temp_mode;
8374 1488135 : if (AGGREGATE_TYPE_P (TREE_TYPE (exp))
8375 745785 : && is_int_mode (GET_MODE (temp), &temp_mode))
8376 : {
8377 2417 : HOST_WIDE_INT size = GET_MODE_BITSIZE (temp_mode);
8378 :
8379 2417 : reverse = TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (exp));
8380 :
8381 2417 : if (reverse)
8382 0 : temp = flip_storage_order (temp_mode, temp);
8383 :
8384 2417 : gcc_checking_assert (known_le (bitsize, size));
8385 2417 : if (maybe_lt (bitsize, size)
8386 2417 : && reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN
8387 : /* Use of to_constant for BLKmode was checked above. */
8388 : && !(mode == BLKmode && bitsize.to_constant () > BITS_PER_WORD))
8389 0 : temp = expand_shift (RSHIFT_EXPR, temp_mode, temp,
8390 : size - bitsize, NULL_RTX, 1);
8391 : }
8392 :
8393 : /* Unless MODE is VOIDmode or BLKmode, convert TEMP to MODE. */
8394 678423 : if (mode != VOIDmode && mode != BLKmode
8395 1423214 : && mode != TYPE_MODE (TREE_TYPE (exp)))
8396 4 : temp = convert_modes (mode, TYPE_MODE (TREE_TYPE (exp)), temp, 1);
8397 :
8398 : /* If the mode of TEMP and TARGET is BLKmode, both must be in memory
8399 : and BITPOS must be aligned on a byte boundary. If so, we simply do
8400 : a block copy. Likewise for a BLKmode-like TARGET. */
8401 745068 : if (GET_MODE (temp) == BLKmode
8402 745068 : && (GET_MODE (target) == BLKmode
8403 182 : || (MEM_P (target)
8404 0 : && GET_MODE_CLASS (GET_MODE (target)) == MODE_INT
8405 0 : && multiple_p (bitpos, BITS_PER_UNIT)
8406 0 : && multiple_p (bitsize, BITS_PER_UNIT))))
8407 : {
8408 89 : gcc_assert (MEM_P (target) && MEM_P (temp));
8409 89 : poly_int64 bytepos = exact_div (bitpos, BITS_PER_UNIT);
8410 89 : poly_int64 bytesize = bits_to_bytes_round_up (bitsize);
8411 :
8412 89 : target = adjust_address (target, VOIDmode, bytepos);
8413 89 : emit_block_move (target, temp,
8414 89 : gen_int_mode (bytesize, Pmode),
8415 : BLOCK_OP_NORMAL);
8416 :
8417 89 : return const0_rtx;
8418 : }
8419 :
8420 : /* If the mode of TEMP is still BLKmode and BITSIZE not larger than the
8421 : word size, we need to load the value (see again store_bit_field). */
8422 744997 : if (GET_MODE (temp) == BLKmode && known_le (bitsize, BITS_PER_WORD))
8423 : {
8424 61 : temp_mode = smallest_int_mode_for_size (bitsize).require ();
8425 61 : temp = extract_bit_field (temp, bitsize, 0, 1, NULL_RTX, temp_mode,
8426 : temp_mode, false, NULL);
8427 : }
8428 :
8429 : /* Store the value in the bitfield. */
8430 744979 : gcc_checking_assert (known_ge (bitpos, 0));
8431 744979 : store_bit_field (target, bitsize, bitpos,
8432 : bitregion_start, bitregion_end,
8433 : mode, temp, reverse, false);
8434 :
8435 744979 : return const0_rtx;
8436 : }
8437 : else
8438 : {
8439 : /* Now build a reference to just the desired component. */
8440 3973932 : rtx to_rtx = adjust_address (target, mode,
8441 : exact_div (bitpos, BITS_PER_UNIT));
8442 :
8443 3973932 : if (to_rtx == target)
8444 1271936 : to_rtx = copy_rtx (to_rtx);
8445 :
8446 7673255 : if (!MEM_KEEP_ALIAS_SET_P (to_rtx) && MEM_ALIAS_SET (to_rtx) != 0)
8447 3443617 : set_mem_alias_set (to_rtx, alias_set);
8448 :
8449 : /* Above we avoided using bitfield operations for storing a CONSTRUCTOR
8450 : into a target smaller than its type; handle that case now. */
8451 3973932 : if (TREE_CODE (exp) == CONSTRUCTOR && known_size_p (bitsize))
8452 : {
8453 68895 : poly_int64 bytesize = exact_div (bitsize, BITS_PER_UNIT);
8454 68895 : store_constructor (exp, to_rtx, 0, bytesize, reverse);
8455 68895 : return to_rtx;
8456 : }
8457 :
8458 3905037 : return store_expr (exp, to_rtx, 0, nontemporal, reverse);
8459 : }
8460 : }
8461 :
8462 : /* Given an expression EXP that may be a COMPONENT_REF, a BIT_FIELD_REF,
8463 : an ARRAY_REF, or an ARRAY_RANGE_REF, look for nested operations of these
8464 : codes and find the ultimate containing object, which we return.
8465 :
8466 : We set *PBITSIZE to the size in bits that we want, *PBITPOS to the
8467 : bit position, *PUNSIGNEDP to the signedness and *PREVERSEP to the
8468 : storage order of the field.
8469 : If the position of the field is variable, we store a tree
8470 : giving the variable offset (in units) in *POFFSET.
8471 : This offset is in addition to the bit position.
8472 : If the position is not variable, we store 0 in *POFFSET.
8473 :
8474 : If any of the extraction expressions is volatile,
8475 : we store 1 in *PVOLATILEP. Otherwise we don't change that.
8476 :
8477 : If the field is a non-BLKmode bit-field, *PMODE is set to VOIDmode.
8478 : Otherwise, it is a mode that can be used to access the field.
8479 :
8480 : If the field describes a variable-sized object, *PMODE is set to
8481 : BLKmode and *PBITSIZE is set to -1. An access cannot be made in
8482 : this case, but the address of the object can be found. */
8483 :
8484 : tree
8485 248599711 : get_inner_reference (tree exp, poly_int64 *pbitsize,
8486 : poly_int64 *pbitpos, tree *poffset,
8487 : machine_mode *pmode, int *punsignedp,
8488 : int *preversep, int *pvolatilep)
8489 : {
8490 248599711 : tree size_tree = 0;
8491 248599711 : machine_mode mode = VOIDmode;
8492 248599711 : bool blkmode_bitfield = false;
8493 248599711 : tree offset = size_zero_node;
8494 248599711 : poly_offset_int bit_offset = 0;
8495 :
8496 : /* First get the mode, signedness, storage order and size. We do this from
8497 : just the outermost expression. */
8498 248599711 : *pbitsize = -1;
8499 248599711 : if (TREE_CODE (exp) == COMPONENT_REF)
8500 : {
8501 105144804 : tree field = TREE_OPERAND (exp, 1);
8502 105144804 : size_tree = DECL_SIZE (field);
8503 105144804 : if (flag_strict_volatile_bitfields > 0
8504 58 : && TREE_THIS_VOLATILE (exp)
8505 40 : && DECL_BIT_FIELD_TYPE (field)
8506 105144826 : && DECL_MODE (field) != BLKmode)
8507 : /* Volatile bitfields should be accessed in the mode of the
8508 : field's type, not the mode computed based on the bit
8509 : size. */
8510 22 : mode = TYPE_MODE (DECL_BIT_FIELD_TYPE (field));
8511 105144782 : else if (!DECL_BIT_FIELD (field))
8512 : {
8513 103930214 : mode = DECL_MODE (field);
8514 : /* For vector fields re-check the target flags, as DECL_MODE
8515 : could have been set with different target flags than
8516 : the current function has. */
8517 103930214 : if (VECTOR_TYPE_P (TREE_TYPE (field))
8518 103930214 : && VECTOR_MODE_P (TYPE_MODE_RAW (TREE_TYPE (field))))
8519 345783 : mode = TYPE_MODE (TREE_TYPE (field));
8520 : }
8521 1214568 : else if (DECL_MODE (field) == BLKmode)
8522 624 : blkmode_bitfield = true;
8523 :
8524 105144804 : *punsignedp = DECL_UNSIGNED (field);
8525 : }
8526 143454907 : else if (TREE_CODE (exp) == BIT_FIELD_REF)
8527 : {
8528 579092 : size_tree = TREE_OPERAND (exp, 1);
8529 1157765 : *punsignedp = (! INTEGRAL_TYPE_P (TREE_TYPE (exp))
8530 1013229 : || TYPE_UNSIGNED (TREE_TYPE (exp)));
8531 :
8532 : /* For vector element types with the correct size of access or for
8533 : vector typed accesses use the mode of the access type. */
8534 579092 : if ((TREE_CODE (TREE_TYPE (TREE_OPERAND (exp, 0))) == VECTOR_TYPE
8535 440793 : && TREE_TYPE (exp) == TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0)))
8536 400983 : && tree_int_cst_equal (size_tree, TYPE_SIZE (TREE_TYPE (exp))))
8537 618937 : || VECTOR_TYPE_P (TREE_TYPE (exp)))
8538 423020 : mode = TYPE_MODE (TREE_TYPE (exp));
8539 : }
8540 : else
8541 : {
8542 142875815 : mode = TYPE_MODE (TREE_TYPE (exp));
8543 142875815 : *punsignedp = TYPE_UNSIGNED (TREE_TYPE (exp));
8544 :
8545 142875815 : if (mode == BLKmode)
8546 57310855 : size_tree = TYPE_SIZE (TREE_TYPE (exp));
8547 : else
8548 171129920 : *pbitsize = GET_MODE_BITSIZE (mode);
8549 : }
8550 :
8551 248599711 : if (size_tree != 0)
8552 : {
8553 162335064 : if (!poly_int_tree_p (size_tree, pbitsize))
8554 369675 : mode = BLKmode, *pbitsize = -1;
8555 : }
8556 :
8557 248599711 : *preversep = reverse_storage_order_for_component_p (exp);
8558 :
8559 : /* Compute cumulative bit-offset for nested component-refs and array-refs,
8560 : and find the ultimate containing object. */
8561 620137555 : while (1)
8562 : {
8563 434368633 : switch (TREE_CODE (exp))
8564 : {
8565 579092 : case BIT_FIELD_REF:
8566 579092 : bit_offset += wi::to_poly_offset (TREE_OPERAND (exp, 2));
8567 579092 : break;
8568 :
8569 152684441 : case COMPONENT_REF:
8570 152684441 : {
8571 152684441 : tree field = TREE_OPERAND (exp, 1);
8572 152684441 : tree this_offset = component_ref_field_offset (exp);
8573 :
8574 : /* If this field hasn't been filled in yet, don't go past it.
8575 : This should only happen when folding expressions made during
8576 : type construction. */
8577 152684441 : if (this_offset == 0)
8578 : break;
8579 :
8580 152684441 : offset = size_binop (PLUS_EXPR, offset, this_offset);
8581 152684441 : bit_offset += wi::to_poly_offset (DECL_FIELD_BIT_OFFSET (field));
8582 :
8583 : /* ??? Right now we don't do anything with DECL_OFFSET_ALIGN. */
8584 : }
8585 152684441 : break;
8586 :
8587 29337466 : case ARRAY_REF:
8588 29337466 : case ARRAY_RANGE_REF:
8589 29337466 : {
8590 29337466 : tree index = TREE_OPERAND (exp, 1);
8591 29337466 : tree low_bound = array_ref_low_bound (exp);
8592 29337466 : tree unit_size = array_ref_element_size (exp);
8593 :
8594 : /* We assume all arrays have sizes that are a multiple of a byte.
8595 : First subtract the lower bound, if any, in the type of the
8596 : index, then convert to sizetype and multiply by the size of
8597 : the array element. */
8598 29337466 : if (! integer_zerop (low_bound))
8599 815730 : index = fold_build2 (MINUS_EXPR, TREE_TYPE (index),
8600 : index, low_bound);
8601 :
8602 29337466 : offset = size_binop (PLUS_EXPR, offset,
8603 : size_binop (MULT_EXPR,
8604 : fold_convert (sizetype, index),
8605 : unit_size));
8606 : }
8607 29337466 : break;
8608 :
8609 : case REALPART_EXPR:
8610 : break;
8611 :
8612 : case IMAGPART_EXPR:
8613 185768922 : bit_offset += *pbitsize;
8614 : break;
8615 :
8616 : case VIEW_CONVERT_EXPR:
8617 : break;
8618 :
8619 88042092 : case MEM_REF:
8620 : /* Hand back the decl for MEM[&decl, off]. */
8621 88042092 : if (TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR)
8622 : {
8623 19379621 : tree off = TREE_OPERAND (exp, 1);
8624 19379621 : if (!integer_zerop (off))
8625 : {
8626 10675672 : poly_offset_int boff = mem_ref_offset (exp);
8627 10675672 : boff <<= LOG2_BITS_PER_UNIT;
8628 10675672 : bit_offset += boff;
8629 : }
8630 19379621 : exp = TREE_OPERAND (TREE_OPERAND (exp, 0), 0);
8631 : }
8632 88042092 : goto done;
8633 :
8634 160557619 : default:
8635 160557619 : goto done;
8636 : }
8637 :
8638 : /* If any reference in the chain is volatile, the effect is volatile. */
8639 185768922 : if (TREE_THIS_VOLATILE (exp))
8640 553231 : *pvolatilep = 1;
8641 :
8642 185768922 : exp = TREE_OPERAND (exp, 0);
8643 185768922 : }
8644 248599711 : done:
8645 :
8646 : /* If OFFSET is constant, see if we can return the whole thing as a
8647 : constant bit position. Make sure to handle overflow during
8648 : this conversion. */
8649 248599711 : if (poly_int_tree_p (offset))
8650 : {
8651 234925487 : poly_offset_int tem = wi::sext (wi::to_poly_offset (offset),
8652 234925487 : TYPE_PRECISION (sizetype));
8653 234925487 : tem <<= LOG2_BITS_PER_UNIT;
8654 234925487 : tem += bit_offset;
8655 234925487 : if (tem.to_shwi (pbitpos))
8656 234924084 : *poffset = offset = NULL_TREE;
8657 : }
8658 :
8659 : /* Otherwise, split it up. */
8660 234925487 : if (offset)
8661 : {
8662 : /* Avoid returning a negative bitpos as this may wreak havoc later. */
8663 13675627 : if (!bit_offset.to_shwi (pbitpos) || maybe_lt (*pbitpos, 0))
8664 : {
8665 283 : *pbitpos = num_trailing_bits (bit_offset.force_shwi ());
8666 283 : poly_offset_int bytes = bits_to_bytes_round_down (bit_offset);
8667 283 : offset = size_binop (PLUS_EXPR, offset,
8668 : build_int_cst (sizetype, bytes.force_shwi ()));
8669 : }
8670 :
8671 13675627 : *poffset = offset;
8672 : }
8673 :
8674 : /* We can use BLKmode for a byte-aligned BLKmode bitfield. */
8675 248599711 : if (mode == VOIDmode
8676 2926927 : && blkmode_bitfield
8677 624 : && multiple_p (*pbitpos, BITS_PER_UNIT)
8678 248600199 : && multiple_p (*pbitsize, BITS_PER_UNIT))
8679 5 : *pmode = BLKmode;
8680 : else
8681 248599706 : *pmode = mode;
8682 :
8683 248599711 : return exp;
8684 : }
8685 :
8686 : /* Alignment in bits the TARGET of an assignment may be assumed to have. */
8687 :
8688 : static unsigned HOST_WIDE_INT
8689 510072 : target_align (const_tree target)
8690 : {
8691 : /* We might have a chain of nested references with intermediate misaligning
8692 : bitfields components, so need to recurse to find out. */
8693 :
8694 510072 : unsigned HOST_WIDE_INT this_align, outer_align;
8695 :
8696 510072 : switch (TREE_CODE (target))
8697 : {
8698 : case BIT_FIELD_REF:
8699 : return 1;
8700 :
8701 139711 : case COMPONENT_REF:
8702 139711 : this_align = DECL_ALIGN (TREE_OPERAND (target, 1));
8703 139711 : outer_align = target_align (TREE_OPERAND (target, 0));
8704 139711 : return MIN (this_align, outer_align);
8705 :
8706 188471 : case ARRAY_REF:
8707 188471 : case ARRAY_RANGE_REF:
8708 188471 : this_align = TYPE_ALIGN (TREE_TYPE (target));
8709 188471 : outer_align = target_align (TREE_OPERAND (target, 0));
8710 188471 : return MIN (this_align, outer_align);
8711 :
8712 4656 : CASE_CONVERT:
8713 4656 : case NON_LVALUE_EXPR:
8714 4656 : case VIEW_CONVERT_EXPR:
8715 4656 : this_align = TYPE_ALIGN (TREE_TYPE (target));
8716 4656 : outer_align = target_align (TREE_OPERAND (target, 0));
8717 4656 : return MAX (this_align, outer_align);
8718 :
8719 177229 : default:
8720 177229 : return TYPE_ALIGN (TREE_TYPE (target));
8721 : }
8722 : }
8723 :
8724 :
8725 : /* Given an rtx VALUE that may contain additions and multiplications, return
8726 : an equivalent value that just refers to a register, memory, or constant.
8727 : This is done by generating instructions to perform the arithmetic and
8728 : returning a pseudo-register containing the value.
8729 :
8730 : The returned value may be a REG, SUBREG, MEM or constant. */
8731 :
8732 : rtx
8733 30872590 : force_operand (rtx value, rtx target)
8734 : {
8735 30872590 : rtx op1, op2;
8736 : /* Use subtarget as the target for operand 0 of a binary operation. */
8737 30872590 : rtx subtarget = get_subtarget (target);
8738 30872590 : enum rtx_code code = GET_CODE (value);
8739 :
8740 : /* Check for subreg applied to an expression produced by loop optimizer. */
8741 30872590 : if (code == SUBREG
8742 240948 : && !REG_P (SUBREG_REG (value))
8743 174 : && !MEM_P (SUBREG_REG (value)))
8744 : {
8745 174 : value
8746 174 : = simplify_gen_subreg (GET_MODE (value),
8747 174 : force_reg (GET_MODE (SUBREG_REG (value)),
8748 : force_operand (SUBREG_REG (value),
8749 : NULL_RTX)),
8750 174 : GET_MODE (SUBREG_REG (value)),
8751 174 : SUBREG_BYTE (value));
8752 174 : code = GET_CODE (value);
8753 : }
8754 :
8755 : /* Check for a PIC address load. */
8756 30872590 : if ((code == PLUS || code == MINUS)
8757 3666445 : && XEXP (value, 0) == pic_offset_table_rtx
8758 1947 : && (GET_CODE (XEXP (value, 1)) == SYMBOL_REF
8759 1947 : || GET_CODE (XEXP (value, 1)) == LABEL_REF
8760 1947 : || GET_CODE (XEXP (value, 1)) == CONST))
8761 : {
8762 200 : if (!subtarget)
8763 200 : subtarget = gen_reg_rtx (GET_MODE (value));
8764 200 : emit_move_insn (subtarget, value);
8765 200 : return subtarget;
8766 : }
8767 :
8768 30872390 : if (ARITHMETIC_P (value))
8769 : {
8770 3780323 : op2 = XEXP (value, 1);
8771 3780323 : if (!CONSTANT_P (op2) && !(REG_P (op2) && op2 != subtarget))
8772 3780323 : subtarget = 0;
8773 3780323 : if (code == MINUS && CONST_INT_P (op2))
8774 : {
8775 0 : code = PLUS;
8776 0 : op2 = negate_rtx (GET_MODE (value), op2);
8777 : }
8778 :
8779 : /* Check for an addition with OP2 a constant integer and our first
8780 : operand a PLUS of a virtual register and something else. In that
8781 : case, we want to emit the sum of the virtual register and the
8782 : constant first and then add the other value. This allows virtual
8783 : register instantiation to simply modify the constant rather than
8784 : creating another one around this addition. */
8785 3606360 : if (code == PLUS && CONST_INT_P (op2)
8786 3222852 : && GET_CODE (XEXP (value, 0)) == PLUS
8787 113695 : && REG_P (XEXP (XEXP (value, 0), 0))
8788 3837221 : && VIRTUAL_REGISTER_P (XEXP (XEXP (value, 0), 0)))
8789 : {
8790 1024 : rtx temp = expand_simple_binop (GET_MODE (value), code,
8791 : XEXP (XEXP (value, 0), 0), op2,
8792 : subtarget, 0, OPTAB_LIB_WIDEN);
8793 1024 : return expand_simple_binop (GET_MODE (value), code, temp,
8794 1024 : force_operand (XEXP (XEXP (value,
8795 : 0), 1), 0),
8796 1024 : target, 0, OPTAB_LIB_WIDEN);
8797 : }
8798 :
8799 3779299 : op1 = force_operand (XEXP (value, 0), subtarget);
8800 3779299 : op2 = force_operand (op2, NULL_RTX);
8801 3779299 : switch (code)
8802 : {
8803 96911 : case MULT:
8804 96911 : return expand_mult (GET_MODE (value), op1, op2, target, 1);
8805 222 : case DIV:
8806 222 : if (!INTEGRAL_MODE_P (GET_MODE (value)))
8807 222 : return expand_simple_binop (GET_MODE (value), code, op1, op2,
8808 222 : target, 1, OPTAB_LIB_WIDEN);
8809 : else
8810 0 : return expand_divmod (0,
8811 : FLOAT_MODE_P (GET_MODE (value))
8812 : ? RDIV_EXPR : TRUNC_DIV_EXPR,
8813 0 : GET_MODE (value), op1, op2, target, 0);
8814 0 : case MOD:
8815 0 : return expand_divmod (1, TRUNC_MOD_EXPR, GET_MODE (value), op1, op2,
8816 0 : target, 0);
8817 316 : case UDIV:
8818 316 : return expand_divmod (0, TRUNC_DIV_EXPR, GET_MODE (value), op1, op2,
8819 316 : target, 1);
8820 0 : case UMOD:
8821 0 : return expand_divmod (1, TRUNC_MOD_EXPR, GET_MODE (value), op1, op2,
8822 0 : target, 1);
8823 72 : case ASHIFTRT:
8824 72 : return expand_simple_binop (GET_MODE (value), code, op1, op2,
8825 72 : target, 0, OPTAB_LIB_WIDEN);
8826 3681778 : default:
8827 3681778 : return expand_simple_binop (GET_MODE (value), code, op1, op2,
8828 3681778 : target, 1, OPTAB_LIB_WIDEN);
8829 : }
8830 : }
8831 27092067 : if (UNARY_P (value))
8832 : {
8833 16509 : if (!target)
8834 6194 : target = gen_reg_rtx (GET_MODE (value));
8835 : /* FIX or UNSIGNED_FIX with integral mode has unspecified rounding,
8836 : while FIX with floating point mode rounds toward zero. So, some
8837 : targets use expressions like (fix:SI (fix:DF (reg:DF ...)))
8838 : to express rounding toward zero during the conversion to int.
8839 : expand_fix isn't able to handle that, it can only handle
8840 : FIX/UNSIGNED_FIX from floating point mode to integral one. */
8841 16509 : if ((code == FIX || code == UNSIGNED_FIX)
8842 4 : && GET_CODE (XEXP (value, 0)) == FIX
8843 0 : && (GET_MODE (XEXP (value, 0))
8844 0 : == GET_MODE (XEXP (XEXP (value, 0), 0))))
8845 0 : op1 = force_operand (XEXP (XEXP (value, 0), 0), NULL_RTX);
8846 : else
8847 16509 : op1 = force_operand (XEXP (value, 0), NULL_RTX);
8848 16509 : switch (code)
8849 : {
8850 4747 : case ZERO_EXTEND:
8851 4747 : case SIGN_EXTEND:
8852 4747 : case TRUNCATE:
8853 4747 : case FLOAT_EXTEND:
8854 4747 : case FLOAT_TRUNCATE:
8855 4747 : convert_move (target, op1, code == ZERO_EXTEND);
8856 4747 : return target;
8857 :
8858 4 : case FIX:
8859 4 : case UNSIGNED_FIX:
8860 4 : expand_fix (target, op1, code == UNSIGNED_FIX);
8861 4 : return target;
8862 :
8863 120 : case FLOAT:
8864 120 : case UNSIGNED_FLOAT:
8865 120 : expand_float (target, op1, code == UNSIGNED_FLOAT);
8866 120 : return target;
8867 :
8868 11638 : default:
8869 11638 : return expand_simple_unop (GET_MODE (value), code, op1, target, 0);
8870 : }
8871 : }
8872 :
8873 : #ifdef INSN_SCHEDULING
8874 : /* On machines that have insn scheduling, we want all memory reference to be
8875 : explicit, so we need to deal with such paradoxical SUBREGs. */
8876 27075558 : if (paradoxical_subreg_p (value) && MEM_P (SUBREG_REG (value)))
8877 0 : value
8878 0 : = simplify_gen_subreg (GET_MODE (value),
8879 0 : force_reg (GET_MODE (SUBREG_REG (value)),
8880 : force_operand (SUBREG_REG (value),
8881 : NULL_RTX)),
8882 : GET_MODE (SUBREG_REG (value)),
8883 0 : SUBREG_BYTE (value));
8884 : #endif
8885 :
8886 : return value;
8887 : }
8888 :
8889 : /* Subroutine of expand_expr: return true iff there is no way that
8890 : EXP can reference X, which is being modified. TOP_P is nonzero if this
8891 : call is going to be used to determine whether we need a temporary
8892 : for EXP, as opposed to a recursive call to this function.
8893 :
8894 : It is always safe for this routine to return false since it merely
8895 : searches for optimization opportunities. */
8896 :
8897 : bool
8898 8611497 : safe_from_p (const_rtx x, tree exp, int top_p)
8899 : {
8900 8611513 : rtx exp_rtl = 0;
8901 8611513 : int i, nops;
8902 :
8903 8611513 : if (x == 0
8904 : /* If EXP has varying size, we MUST use a target since we currently
8905 : have no way of allocating temporaries of variable size
8906 : (except for arrays that have TYPE_ARRAY_MAX_SIZE set).
8907 : So we assume here that something at a higher level has prevented a
8908 : clash. This is somewhat bogus, but the best we can do. Only
8909 : do this when X is BLKmode and when we are at the top level. */
8910 1979037 : || (top_p && TREE_TYPE (exp) != 0 && COMPLETE_TYPE_P (TREE_TYPE (exp))
8911 1848892 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (exp))) != INTEGER_CST
8912 0 : && (TREE_CODE (TREE_TYPE (exp)) != ARRAY_TYPE
8913 0 : || TYPE_ARRAY_MAX_SIZE (TREE_TYPE (exp)) == NULL_TREE
8914 0 : || TREE_CODE (TYPE_ARRAY_MAX_SIZE (TREE_TYPE (exp)))
8915 : != INTEGER_CST)
8916 0 : && GET_MODE (x) == BLKmode)
8917 : /* If X is in the outgoing argument area, it is always safe. */
8918 10590550 : || (MEM_P (x)
8919 176448 : && (XEXP (x, 0) == virtual_outgoing_args_rtx
8920 176448 : || (GET_CODE (XEXP (x, 0)) == PLUS
8921 128807 : && XEXP (XEXP (x, 0), 0) == virtual_outgoing_args_rtx))))
8922 : return true;
8923 :
8924 : /* If this is a subreg of a hard register, declare it unsafe, otherwise,
8925 : find the underlying pseudo. */
8926 1979037 : if (GET_CODE (x) == SUBREG)
8927 : {
8928 0 : x = SUBREG_REG (x);
8929 0 : if (REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER)
8930 : return false;
8931 : }
8932 :
8933 : /* Now look at our tree code and possibly recurse. */
8934 1979037 : switch (TREE_CODE_CLASS (TREE_CODE (exp)))
8935 : {
8936 486 : case tcc_declaration:
8937 486 : exp_rtl = DECL_RTL_IF_SET (exp);
8938 : break;
8939 :
8940 : case tcc_constant:
8941 : return true;
8942 :
8943 862177 : case tcc_exceptional:
8944 862177 : if (TREE_CODE (exp) == TREE_LIST)
8945 : {
8946 0 : while (1)
8947 : {
8948 0 : if (TREE_VALUE (exp) && !safe_from_p (x, TREE_VALUE (exp), 0))
8949 : return false;
8950 0 : exp = TREE_CHAIN (exp);
8951 0 : if (!exp)
8952 : return true;
8953 0 : if (TREE_CODE (exp) != TREE_LIST)
8954 : return safe_from_p (x, exp, 0);
8955 : }
8956 : }
8957 862177 : else if (TREE_CODE (exp) == CONSTRUCTOR)
8958 : {
8959 : constructor_elt *ce;
8960 : unsigned HOST_WIDE_INT idx;
8961 :
8962 7912353 : FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (exp), idx, ce)
8963 4240 : if ((ce->index != NULL_TREE && !safe_from_p (x, ce->index, 0))
8964 129848 : || !safe_from_p (x, ce->value, 0))
8965 103478 : return false;
8966 : return true;
8967 : }
8968 722457 : else if (TREE_CODE (exp) == ERROR_MARK)
8969 : return true; /* An already-visited SAVE_EXPR? */
8970 : else
8971 : return false;
8972 :
8973 0 : case tcc_statement:
8974 : /* The only case we look at here is the DECL_INITIAL inside a
8975 : DECL_EXPR. */
8976 0 : return (TREE_CODE (exp) != DECL_EXPR
8977 0 : || TREE_CODE (DECL_EXPR_DECL (exp)) != VAR_DECL
8978 0 : || !DECL_INITIAL (DECL_EXPR_DECL (exp))
8979 0 : || safe_from_p (x, DECL_INITIAL (DECL_EXPR_DECL (exp)), 0));
8980 :
8981 0 : case tcc_binary:
8982 0 : case tcc_comparison:
8983 0 : if (!safe_from_p (x, TREE_OPERAND (exp, 1), 0))
8984 : return false;
8985 : /* Fall through. */
8986 :
8987 16 : case tcc_unary:
8988 16 : return safe_from_p (x, TREE_OPERAND (exp, 0), 0);
8989 :
8990 357 : case tcc_expression:
8991 357 : case tcc_reference:
8992 357 : case tcc_vl_exp:
8993 : /* Now do code-specific tests. EXP_RTL is set to any rtx we find in
8994 : the expression. If it is set, we conflict iff we are that rtx or
8995 : both are in memory. Otherwise, we check all operands of the
8996 : expression recursively. */
8997 :
8998 357 : switch (TREE_CODE (exp))
8999 : {
9000 301 : case ADDR_EXPR:
9001 : /* If the operand is static or we are static, we can't conflict.
9002 : Likewise if we don't conflict with the operand at all. */
9003 301 : if (staticp (TREE_OPERAND (exp, 0))
9004 169 : || TREE_STATIC (exp)
9005 470 : || safe_from_p (x, TREE_OPERAND (exp, 0), 0))
9006 301 : return true;
9007 :
9008 : /* Otherwise, the only way this can conflict is if we are taking
9009 : the address of a DECL a that address if part of X, which is
9010 : very rare. */
9011 0 : exp = TREE_OPERAND (exp, 0);
9012 0 : if (DECL_P (exp))
9013 : {
9014 0 : if (!DECL_RTL_SET_P (exp)
9015 0 : || !MEM_P (DECL_RTL (exp)))
9016 0 : return false;
9017 : else
9018 0 : exp_rtl = XEXP (DECL_RTL (exp), 0);
9019 : }
9020 : break;
9021 :
9022 41 : case MEM_REF:
9023 41 : if (MEM_P (x)
9024 41 : && alias_sets_conflict_p (MEM_ALIAS_SET (x),
9025 : get_alias_set (exp)))
9026 : return false;
9027 : break;
9028 :
9029 0 : case CALL_EXPR:
9030 : /* Assume that the call will clobber all hard registers and
9031 : all of memory. */
9032 0 : if ((REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER)
9033 0 : || MEM_P (x))
9034 : return false;
9035 : break;
9036 :
9037 0 : case WITH_CLEANUP_EXPR:
9038 0 : case CLEANUP_POINT_EXPR:
9039 : /* Lowered by gimplify.cc. */
9040 0 : gcc_unreachable ();
9041 :
9042 0 : case SAVE_EXPR:
9043 0 : return safe_from_p (x, TREE_OPERAND (exp, 0), 0);
9044 :
9045 : default:
9046 : break;
9047 : }
9048 :
9049 : /* If we have an rtx, we do not need to scan our operands. */
9050 0 : if (exp_rtl)
9051 : break;
9052 :
9053 56 : nops = TREE_OPERAND_LENGTH (exp);
9054 243 : for (i = 0; i < nops; i++)
9055 131 : if (TREE_OPERAND (exp, i) != 0
9056 131 : && ! safe_from_p (x, TREE_OPERAND (exp, i), 0))
9057 : return false;
9058 :
9059 : break;
9060 :
9061 0 : case tcc_type:
9062 : /* Should never get a type here. */
9063 0 : gcc_unreachable ();
9064 : }
9065 :
9066 : /* If we have an rtl, find any enclosed object. Then see if we conflict
9067 : with it. */
9068 328 : if (exp_rtl)
9069 : {
9070 272 : if (GET_CODE (exp_rtl) == SUBREG)
9071 : {
9072 0 : exp_rtl = SUBREG_REG (exp_rtl);
9073 0 : if (REG_P (exp_rtl)
9074 0 : && REGNO (exp_rtl) < FIRST_PSEUDO_REGISTER)
9075 : return false;
9076 : }
9077 :
9078 : /* If the rtl is X, then it is not safe. Otherwise, it is unless both
9079 : are memory and they conflict. */
9080 272 : return ! (rtx_equal_p (x, exp_rtl)
9081 272 : || (MEM_P (x) && MEM_P (exp_rtl)
9082 4 : && true_dependence (exp_rtl, VOIDmode, x)));
9083 : }
9084 :
9085 : /* If we reach here, it is safe. */
9086 : return true;
9087 : }
9088 :
9089 :
9090 : /* Return the highest power of two that EXP is known to be a multiple of.
9091 : This is used in updating alignment of MEMs in array references. */
9092 :
9093 : unsigned HOST_WIDE_INT
9094 32509404 : highest_pow2_factor (const_tree exp)
9095 : {
9096 32509404 : unsigned HOST_WIDE_INT ret;
9097 32509404 : int trailing_zeros = tree_ctz (exp);
9098 32509404 : if (trailing_zeros >= HOST_BITS_PER_WIDE_INT)
9099 44497379 : return BIGGEST_ALIGNMENT;
9100 9917240 : ret = HOST_WIDE_INT_1U << trailing_zeros;
9101 19369352 : if (ret > BIGGEST_ALIGNMENT)
9102 13109894 : return BIGGEST_ALIGNMENT;
9103 : return ret;
9104 : }
9105 :
9106 : /* Similar, except that the alignment requirements of TARGET are
9107 : taken into account. Assume it is at least as aligned as its
9108 : type, unless it is a COMPONENT_REF in which case the layout of
9109 : the structure gives the alignment. */
9110 :
9111 : static unsigned HOST_WIDE_INT
9112 177234 : highest_pow2_factor_for_target (const_tree target, const_tree exp)
9113 : {
9114 177234 : unsigned HOST_WIDE_INT talign = target_align (target) / BITS_PER_UNIT;
9115 177234 : unsigned HOST_WIDE_INT factor = highest_pow2_factor (exp);
9116 :
9117 177234 : return MAX (factor, talign);
9118 : }
9119 :
9120 : /* Convert the tree comparison code TCODE to the rtl one where the
9121 : signedness is UNSIGNEDP. */
9122 :
9123 : enum rtx_code
9124 21075 : convert_tree_comp_to_rtx (enum tree_code tcode, int unsignedp)
9125 : {
9126 21075 : enum rtx_code code;
9127 21075 : switch (tcode)
9128 : {
9129 : case EQ_EXPR:
9130 : code = EQ;
9131 : break;
9132 2372 : case NE_EXPR:
9133 2372 : code = NE;
9134 2372 : break;
9135 4065 : case LT_EXPR:
9136 4065 : code = unsignedp ? LTU : LT;
9137 : break;
9138 2704 : case LE_EXPR:
9139 2704 : code = unsignedp ? LEU : LE;
9140 : break;
9141 3184 : case GT_EXPR:
9142 3184 : code = unsignedp ? GTU : GT;
9143 : break;
9144 4385 : case GE_EXPR:
9145 4385 : code = unsignedp ? GEU : GE;
9146 : break;
9147 31 : case UNORDERED_EXPR:
9148 31 : code = UNORDERED;
9149 31 : break;
9150 31 : case ORDERED_EXPR:
9151 31 : code = ORDERED;
9152 31 : break;
9153 7 : case UNLT_EXPR:
9154 7 : code = UNLT;
9155 7 : break;
9156 19 : case UNLE_EXPR:
9157 19 : code = UNLE;
9158 19 : break;
9159 10 : case UNGT_EXPR:
9160 10 : code = UNGT;
9161 10 : break;
9162 11 : case UNGE_EXPR:
9163 11 : code = UNGE;
9164 11 : break;
9165 4 : case UNEQ_EXPR:
9166 4 : code = UNEQ;
9167 4 : break;
9168 4 : case LTGT_EXPR:
9169 4 : code = LTGT;
9170 4 : break;
9171 :
9172 0 : default:
9173 0 : gcc_unreachable ();
9174 : }
9175 21075 : return code;
9176 : }
9177 :
9178 : /* Subroutine of expand_expr. Expand the two operands of a binary
9179 : expression EXP0 and EXP1 placing the results in OP0 and OP1.
9180 : The value may be stored in TARGET if TARGET is nonzero. The
9181 : MODIFIER argument is as documented by expand_expr. */
9182 :
9183 : void
9184 8030240 : expand_operands (tree exp0, tree exp1, rtx target, rtx *op0, rtx *op1,
9185 : enum expand_modifier modifier)
9186 : {
9187 8030240 : if (! safe_from_p (target, exp1, 1))
9188 602128 : target = 0;
9189 8030240 : if (operand_equal_p (exp0, exp1, 0))
9190 : {
9191 46697 : *op0 = expand_expr (exp0, target, VOIDmode, modifier);
9192 46697 : *op1 = copy_rtx (*op0);
9193 : }
9194 : else
9195 : {
9196 7983543 : *op0 = expand_expr (exp0, target, VOIDmode, modifier);
9197 7983543 : *op1 = expand_expr (exp1, NULL_RTX, VOIDmode, modifier);
9198 : }
9199 8030240 : }
9200 :
9201 :
9202 : /* Return a MEM that contains constant EXP. DEFER is as for
9203 : output_constant_def and MODIFIER is as for expand_expr. */
9204 :
9205 : static rtx
9206 2904106 : expand_expr_constant (tree exp, int defer, enum expand_modifier modifier)
9207 : {
9208 2904106 : rtx mem;
9209 :
9210 2904106 : mem = output_constant_def (exp, defer);
9211 2904106 : if (modifier != EXPAND_INITIALIZER)
9212 1852623 : mem = use_anchored_address (mem);
9213 2904106 : return mem;
9214 : }
9215 :
9216 : /* A subroutine of expand_expr_addr_expr. Evaluate the address of EXP.
9217 : The TARGET, TMODE and MODIFIER arguments are as for expand_expr. */
9218 :
9219 : static rtx
9220 14223122 : expand_expr_addr_expr_1 (tree exp, rtx target, scalar_int_mode tmode,
9221 : enum expand_modifier modifier, addr_space_t as)
9222 : {
9223 14223122 : rtx result, subtarget;
9224 14223122 : tree inner, offset;
9225 14223122 : poly_int64 bitsize, bitpos;
9226 14223122 : int unsignedp, reversep, volatilep = 0;
9227 14223122 : machine_mode mode1;
9228 :
9229 : /* If we are taking the address of a constant and are at the top level,
9230 : we have to use output_constant_def since we can't call force_const_mem
9231 : at top level. */
9232 : /* ??? This should be considered a front-end bug. We should not be
9233 : generating ADDR_EXPR of something that isn't an LVALUE. The only
9234 : exception here is STRING_CST. */
9235 14223122 : if (CONSTANT_CLASS_P (exp))
9236 : {
9237 2657499 : result = XEXP (expand_expr_constant (exp, 0, modifier), 0);
9238 2657499 : if (modifier < EXPAND_SUM)
9239 1762705 : result = force_operand (result, target);
9240 2657499 : return result;
9241 : }
9242 :
9243 : /* Everything must be something allowed by is_gimple_addressable. */
9244 11565623 : switch (TREE_CODE (exp))
9245 : {
9246 36 : case INDIRECT_REF:
9247 : /* This case will happen via recursion for &a->b. */
9248 36 : return expand_expr (TREE_OPERAND (exp, 0), target, tmode, modifier);
9249 :
9250 546460 : case MEM_REF:
9251 546460 : {
9252 546460 : tree tem = TREE_OPERAND (exp, 0);
9253 546460 : if (!integer_zerop (TREE_OPERAND (exp, 1)))
9254 279551 : tem = fold_build_pointer_plus (tem, TREE_OPERAND (exp, 1));
9255 546460 : return expand_expr (tem, target, tmode, modifier);
9256 : }
9257 :
9258 1151 : case TARGET_MEM_REF:
9259 1151 : return addr_for_mem_ref (exp, as, true);
9260 :
9261 59440 : case CONST_DECL:
9262 : /* Expand the initializer like constants above. */
9263 59440 : result = XEXP (expand_expr_constant (DECL_INITIAL (exp),
9264 : 0, modifier), 0);
9265 59440 : if (modifier < EXPAND_SUM)
9266 59434 : result = force_operand (result, target);
9267 : return result;
9268 :
9269 167 : case REALPART_EXPR:
9270 : /* The real part of the complex number is always first, therefore
9271 : the address is the same as the address of the parent object. */
9272 167 : offset = 0;
9273 167 : bitpos = 0;
9274 167 : inner = TREE_OPERAND (exp, 0);
9275 167 : break;
9276 :
9277 84 : case IMAGPART_EXPR:
9278 : /* The imaginary part of the complex number is always second.
9279 : The expression is therefore always offset by the size of the
9280 : scalar type. */
9281 84 : offset = 0;
9282 168 : bitpos = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (exp)));
9283 84 : inner = TREE_OPERAND (exp, 0);
9284 84 : break;
9285 :
9286 13 : case COMPOUND_LITERAL_EXPR:
9287 : /* Allow COMPOUND_LITERAL_EXPR in initializers or coming from
9288 : initializers, if e.g. rtl_for_decl_init is called on DECL_INITIAL
9289 : with COMPOUND_LITERAL_EXPRs in it, or ARRAY_REF on a const static
9290 : array with address of COMPOUND_LITERAL_EXPR in DECL_INITIAL;
9291 : the initializers aren't gimplified. */
9292 13 : if (COMPOUND_LITERAL_EXPR_DECL (exp)
9293 13 : && is_global_var (COMPOUND_LITERAL_EXPR_DECL (exp)))
9294 13 : return expand_expr_addr_expr_1 (COMPOUND_LITERAL_EXPR_DECL (exp),
9295 13 : target, tmode, modifier, as);
9296 : /* FALLTHRU */
9297 10958272 : default:
9298 : /* If the object is a DECL, then expand it for its rtl. Don't bypass
9299 : expand_expr, as that can have various side effects; LABEL_DECLs for
9300 : example, may not have their DECL_RTL set yet. Expand the rtl of
9301 : CONSTRUCTORs too, which should yield a memory reference for the
9302 : constructor's contents. Assume language specific tree nodes can
9303 : be expanded in some interesting way. */
9304 10958272 : gcc_assert (TREE_CODE (exp) < LAST_AND_UNUSED_TREE_CODE);
9305 10958272 : if (DECL_P (exp)
9306 1285817 : || TREE_CODE (exp) == CONSTRUCTOR
9307 1285817 : || TREE_CODE (exp) == COMPOUND_LITERAL_EXPR)
9308 : {
9309 15840464 : result = expand_expr (exp, target, tmode,
9310 : modifier == EXPAND_INITIALIZER
9311 : ? EXPAND_INITIALIZER : EXPAND_CONST_ADDRESS);
9312 :
9313 : /* If the DECL isn't in memory, then the DECL wasn't properly
9314 : marked TREE_ADDRESSABLE, which will be either a front-end
9315 : or a tree optimizer bug. */
9316 :
9317 9672455 : gcc_assert (MEM_P (result));
9318 9672454 : result = XEXP (result, 0);
9319 :
9320 : /* ??? Is this needed anymore? */
9321 9672454 : if (DECL_P (exp))
9322 9672454 : TREE_USED (exp) = 1;
9323 :
9324 9672454 : if (modifier != EXPAND_INITIALIZER
9325 : && modifier != EXPAND_CONST_ADDRESS
9326 9672454 : && modifier != EXPAND_SUM)
9327 4168883 : result = force_operand (result, target);
9328 9672454 : return result;
9329 : }
9330 :
9331 : /* Pass FALSE as the last argument to get_inner_reference although
9332 : we are expanding to RTL. The rationale is that we know how to
9333 : handle "aligning nodes" here: we can just bypass them because
9334 : they won't change the final object whose address will be returned
9335 : (they actually exist only for that purpose). */
9336 1285817 : inner = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
9337 : &unsignedp, &reversep, &volatilep);
9338 1285817 : break;
9339 : }
9340 :
9341 : /* We must have made progress. */
9342 1286068 : gcc_assert (inner != exp);
9343 :
9344 1286068 : subtarget = offset || maybe_ne (bitpos, 0) ? NULL_RTX : target;
9345 : /* For VIEW_CONVERT_EXPR, where the outer alignment is bigger than
9346 : inner alignment, force the inner to be sufficiently aligned. */
9347 1286068 : if (CONSTANT_CLASS_P (inner)
9348 1286068 : && TYPE_ALIGN (TREE_TYPE (inner)) < TYPE_ALIGN (TREE_TYPE (exp)))
9349 : {
9350 0 : inner = copy_node (inner);
9351 0 : TREE_TYPE (inner) = copy_node (TREE_TYPE (inner));
9352 0 : SET_TYPE_ALIGN (TREE_TYPE (inner), TYPE_ALIGN (TREE_TYPE (exp)));
9353 0 : TYPE_USER_ALIGN (TREE_TYPE (inner)) = 1;
9354 : }
9355 1286068 : result = expand_expr_addr_expr_1 (inner, subtarget, tmode, modifier, as);
9356 :
9357 1286068 : if (offset)
9358 : {
9359 52719 : rtx tmp;
9360 :
9361 52719 : if (modifier != EXPAND_NORMAL)
9362 1629 : result = force_operand (result, NULL);
9363 54348 : tmp = expand_expr (offset, NULL_RTX, tmode,
9364 : modifier == EXPAND_INITIALIZER
9365 : ? EXPAND_INITIALIZER : EXPAND_NORMAL);
9366 :
9367 : /* expand_expr is allowed to return an object in a mode other
9368 : than TMODE. If it did, we need to convert. */
9369 52719 : if (GET_MODE (tmp) != VOIDmode && tmode != GET_MODE (tmp))
9370 0 : tmp = convert_modes (tmode, GET_MODE (tmp),
9371 0 : tmp, TYPE_UNSIGNED (TREE_TYPE (offset)));
9372 52719 : result = convert_memory_address_addr_space (tmode, result, as);
9373 52719 : tmp = convert_memory_address_addr_space (tmode, tmp, as);
9374 :
9375 52719 : if (modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER)
9376 1629 : result = simplify_gen_binary (PLUS, tmode, result, tmp);
9377 : else
9378 : {
9379 51090 : subtarget = maybe_ne (bitpos, 0) ? NULL_RTX : target;
9380 51090 : result = expand_simple_binop (tmode, PLUS, result, tmp, subtarget,
9381 : 1, OPTAB_LIB_WIDEN);
9382 : }
9383 : }
9384 :
9385 1286068 : if (maybe_ne (bitpos, 0))
9386 : {
9387 : /* Someone beforehand should have rejected taking the address
9388 : of an object that isn't byte-aligned. */
9389 611216 : poly_int64 bytepos = exact_div (bitpos, BITS_PER_UNIT);
9390 611216 : result = convert_memory_address_addr_space (tmode, result, as);
9391 611216 : result = plus_constant (tmode, result, bytepos);
9392 611216 : if (modifier < EXPAND_SUM)
9393 576697 : result = force_operand (result, target);
9394 : }
9395 :
9396 : return result;
9397 : }
9398 :
9399 : /* A subroutine of expand_expr. Evaluate EXP, which is an ADDR_EXPR.
9400 : The TARGET, TMODE and MODIFIER arguments are as for expand_expr. */
9401 :
9402 : static rtx
9403 12937039 : expand_expr_addr_expr (tree exp, rtx target, machine_mode tmode,
9404 : enum expand_modifier modifier)
9405 : {
9406 12937039 : addr_space_t as = ADDR_SPACE_GENERIC;
9407 12937039 : scalar_int_mode address_mode = Pmode;
9408 12937039 : scalar_int_mode pointer_mode = ptr_mode;
9409 12937039 : machine_mode rmode;
9410 12937039 : rtx result;
9411 :
9412 : /* Target mode of VOIDmode says "whatever's natural". */
9413 12937039 : if (tmode == VOIDmode)
9414 11021191 : tmode = TYPE_MODE (TREE_TYPE (exp));
9415 :
9416 12937039 : if (POINTER_TYPE_P (TREE_TYPE (exp)))
9417 : {
9418 12937039 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)));
9419 12937039 : address_mode = targetm.addr_space.address_mode (as);
9420 12937039 : pointer_mode = targetm.addr_space.pointer_mode (as);
9421 : }
9422 :
9423 : /* We can get called with some Weird Things if the user does silliness
9424 : like "(short) &a". In that case, convert_memory_address won't do
9425 : the right thing, so ignore the given target mode. */
9426 12937039 : scalar_int_mode new_tmode = (tmode == pointer_mode
9427 12937039 : ? pointer_mode
9428 12937039 : : address_mode);
9429 :
9430 12937039 : result = expand_expr_addr_expr_1 (TREE_OPERAND (exp, 0), target,
9431 : new_tmode, modifier, as);
9432 :
9433 : /* Despite expand_expr claims concerning ignoring TMODE when not
9434 : strictly convenient, stuff breaks if we don't honor it. Note
9435 : that combined with the above, we only do this for pointer modes. */
9436 12937038 : rmode = GET_MODE (result);
9437 12937038 : if (rmode == VOIDmode)
9438 6 : rmode = new_tmode;
9439 12937038 : if (rmode != new_tmode)
9440 74 : result = convert_memory_address_addr_space (new_tmode, result, as);
9441 :
9442 12937038 : return result;
9443 : }
9444 :
9445 : /* Generate code for computing CONSTRUCTOR EXP.
9446 : An rtx for the computed value is returned. If AVOID_TEMP_MEM
9447 : is TRUE, instead of creating a temporary variable in memory
9448 : NULL is returned and the caller needs to handle it differently. */
9449 :
9450 : static rtx
9451 173164 : expand_constructor (tree exp, rtx target, enum expand_modifier modifier,
9452 : bool avoid_temp_mem)
9453 : {
9454 173164 : tree type = TREE_TYPE (exp);
9455 173164 : machine_mode mode = TYPE_MODE (type);
9456 :
9457 : /* Try to avoid creating a temporary at all. This is possible
9458 : if all of the initializer is zero.
9459 : FIXME: try to handle all [0..255] initializers we can handle
9460 : with memset. */
9461 173164 : if (TREE_STATIC (exp)
9462 2051 : && !TREE_ADDRESSABLE (exp)
9463 2051 : && target != 0 && mode == BLKmode
9464 174179 : && all_zeros_p (exp))
9465 : {
9466 1006 : clear_storage (target, expr_size (exp), BLOCK_OP_NORMAL);
9467 1006 : return target;
9468 : }
9469 :
9470 : /* All elts simple constants => refer to a constant in memory. But
9471 : if this is a non-BLKmode mode, let it store a field at a time
9472 : since that should make a CONST_INT, CONST_WIDE_INT or
9473 : CONST_DOUBLE when we fold. Likewise, if we have a target we can
9474 : use, it is best to store directly into the target unless the type
9475 : is large enough that memcpy will be used. If we are making an
9476 : initializer and all operands are constant, put it in memory as
9477 : well.
9478 :
9479 : FIXME: Avoid trying to fill vector constructors piece-meal.
9480 : Output them with output_constant_def below unless we're sure
9481 : they're zeros. This should go away when vector initializers
9482 : are treated like VECTOR_CST instead of arrays. */
9483 172158 : if ((TREE_STATIC (exp)
9484 1045 : && ((mode == BLKmode
9485 46 : && ! (target != 0 && safe_from_p (target, exp, 1)))
9486 1008 : || TREE_ADDRESSABLE (exp)
9487 1008 : || (tree_fits_uhwi_p (TYPE_SIZE_UNIT (type))
9488 1008 : && (! can_move_by_pieces
9489 1008 : (tree_to_uhwi (TYPE_SIZE_UNIT (type)),
9490 1008 : TYPE_ALIGN (type)))
9491 4 : && ! mostly_zeros_p (exp))))
9492 173164 : || ((modifier == EXPAND_INITIALIZER || modifier == EXPAND_CONST_ADDRESS)
9493 0 : && TREE_CONSTANT (exp)))
9494 : {
9495 39 : rtx constructor;
9496 :
9497 39 : if (avoid_temp_mem)
9498 : return NULL_RTX;
9499 :
9500 37 : constructor = expand_expr_constant (exp, 1, modifier);
9501 :
9502 37 : if (modifier != EXPAND_CONST_ADDRESS
9503 : && modifier != EXPAND_INITIALIZER
9504 37 : && modifier != EXPAND_SUM)
9505 37 : constructor = validize_mem (constructor);
9506 :
9507 37 : return constructor;
9508 : }
9509 :
9510 : /* If the CTOR is available in static storage and not mostly
9511 : zeros and we can move it by pieces prefer to do so since
9512 : that's usually more efficient than performing a series of
9513 : stores from immediates. */
9514 172119 : if (avoid_temp_mem
9515 76 : && TREE_STATIC (exp)
9516 38 : && TREE_CONSTANT (exp)
9517 38 : && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type))
9518 38 : && can_move_by_pieces (tree_to_uhwi (TYPE_SIZE_UNIT (type)),
9519 38 : TYPE_ALIGN (type))
9520 172156 : && ! mostly_zeros_p (exp))
9521 : return NULL_RTX;
9522 :
9523 : /* Handle calls that pass values in multiple non-contiguous
9524 : locations. The Irix 6 ABI has examples of this. */
9525 139685 : if (target == 0 || ! safe_from_p (target, exp, 1)
9526 36207 : || GET_CODE (target) == PARALLEL || modifier == EXPAND_STACK_PARM
9527 : /* Also make a temporary if the store is to volatile memory, to
9528 : avoid individual accesses to aggregate members. */
9529 208284 : || (GET_CODE (target) == MEM
9530 32162 : && MEM_VOLATILE_P (target)
9531 134 : && !TREE_ADDRESSABLE (TREE_TYPE (exp))))
9532 : {
9533 136022 : if (avoid_temp_mem)
9534 : return NULL_RTX;
9535 :
9536 136011 : target = assign_temp (type, TREE_ADDRESSABLE (exp), 1);
9537 : }
9538 :
9539 172078 : store_constructor (exp, target, 0, int_expr_size (exp), false);
9540 172078 : return target;
9541 : }
9542 :
9543 :
9544 : /* expand_expr: generate code for computing expression EXP.
9545 : An rtx for the computed value is returned. The value is never null.
9546 : In the case of a void EXP, const0_rtx is returned.
9547 :
9548 : The value may be stored in TARGET if TARGET is nonzero.
9549 : TARGET is just a suggestion; callers must assume that
9550 : the rtx returned may not be the same as TARGET.
9551 :
9552 : If TARGET is CONST0_RTX, it means that the value will be ignored.
9553 :
9554 : If TMODE is not VOIDmode, it suggests generating the
9555 : result in mode TMODE. But this is done only when convenient.
9556 : Otherwise, TMODE is ignored and the value generated in its natural mode.
9557 : TMODE is just a suggestion; callers must assume that
9558 : the rtx returned may not have mode TMODE.
9559 :
9560 : Note that TARGET may have neither TMODE nor MODE. In that case, it
9561 : probably will not be used.
9562 :
9563 : If MODIFIER is EXPAND_SUM then when EXP is an addition
9564 : we can return an rtx of the form (MULT (REG ...) (CONST_INT ...))
9565 : or a nest of (PLUS ...) and (MINUS ...) where the terms are
9566 : products as above, or REG or MEM, or constant.
9567 : Ordinarily in such cases we would output mul or add instructions
9568 : and then return a pseudo reg containing the sum.
9569 :
9570 : EXPAND_INITIALIZER is much like EXPAND_SUM except that
9571 : it also marks a label as absolutely required (it can't be dead).
9572 : It also makes a ZERO_EXTEND or SIGN_EXTEND instead of emitting extend insns.
9573 : This is used for outputting expressions used in initializers.
9574 :
9575 : EXPAND_CONST_ADDRESS says that it is okay to return a MEM
9576 : with a constant address even if that address is not normally legitimate.
9577 : EXPAND_INITIALIZER and EXPAND_SUM also have this effect.
9578 :
9579 : EXPAND_STACK_PARM is used when expanding to a TARGET on the stack for
9580 : a call parameter. Such targets require special care as we haven't yet
9581 : marked TARGET so that it's safe from being trashed by libcalls. We
9582 : don't want to use TARGET for anything but the final result;
9583 : Intermediate values must go elsewhere. Additionally, calls to
9584 : emit_block_move will be flagged with BLOCK_OP_CALL_PARM.
9585 :
9586 : If EXP is a VAR_DECL whose DECL_RTL was a MEM with an invalid
9587 : address, and ALT_RTL is non-NULL, then *ALT_RTL is set to the
9588 : DECL_RTL of the VAR_DECL. *ALT_RTL is also set if EXP is a
9589 : COMPOUND_EXPR whose second argument is such a VAR_DECL, and so on
9590 : recursively.
9591 : If the result can be stored at TARGET, and ALT_RTL is non-NULL,
9592 : then *ALT_RTL is set to TARGET (before legitimziation).
9593 :
9594 : If INNER_REFERENCE_P is true, we are expanding an inner reference.
9595 : In this case, we don't adjust a returned MEM rtx that wouldn't be
9596 : sufficiently aligned for its mode; instead, it's up to the caller
9597 : to deal with it afterwards. This is used to make sure that unaligned
9598 : base objects for which out-of-bounds accesses are supported, for
9599 : example record types with trailing arrays, aren't realigned behind
9600 : the back of the caller.
9601 : The normal operating mode is to pass FALSE for this parameter. */
9602 :
9603 : rtx
9604 154781960 : expand_expr_real (tree exp, rtx target, machine_mode tmode,
9605 : enum expand_modifier modifier, rtx *alt_rtl,
9606 : bool inner_reference_p)
9607 : {
9608 154781960 : rtx ret;
9609 :
9610 : /* Handle ERROR_MARK before anybody tries to access its type. */
9611 154781960 : if (TREE_CODE (exp) == ERROR_MARK
9612 154781960 : || (TREE_CODE (TREE_TYPE (exp)) == ERROR_MARK))
9613 : {
9614 0 : ret = CONST0_RTX (tmode);
9615 0 : return ret ? ret : const0_rtx;
9616 : }
9617 :
9618 154781960 : ret = expand_expr_real_1 (exp, target, tmode, modifier, alt_rtl,
9619 : inner_reference_p);
9620 154781960 : return ret;
9621 : }
9622 :
9623 : /* Try to expand the conditional expression which is represented by
9624 : TREEOP0 ? TREEOP1 : TREEOP2 using conditonal moves. If it succeeds
9625 : return the rtl reg which represents the result. Otherwise return
9626 : NULL_RTX. */
9627 :
9628 : static rtx
9629 17962 : expand_cond_expr_using_cmove (tree treeop0 ATTRIBUTE_UNUSED,
9630 : tree treeop1 ATTRIBUTE_UNUSED,
9631 : tree treeop2 ATTRIBUTE_UNUSED)
9632 : {
9633 17962 : rtx insn;
9634 17962 : rtx op00, op01, op1, op2;
9635 17962 : enum rtx_code comparison_code;
9636 17962 : machine_mode comparison_mode;
9637 17962 : gimple *srcstmt;
9638 17962 : rtx temp;
9639 17962 : tree type = TREE_TYPE (treeop1);
9640 17962 : int unsignedp = TYPE_UNSIGNED (type);
9641 17962 : machine_mode mode = TYPE_MODE (type);
9642 17962 : machine_mode orig_mode = mode;
9643 17962 : static bool expanding_cond_expr_using_cmove = false;
9644 :
9645 : /* Conditional move expansion can end up TERing two operands which,
9646 : when recursively hitting conditional expressions can result in
9647 : exponential behavior if the cmove expansion ultimatively fails.
9648 : It's hardly profitable to TER a cmove into a cmove so avoid doing
9649 : that by failing early if we end up recursing. */
9650 17962 : if (expanding_cond_expr_using_cmove)
9651 : return NULL_RTX;
9652 :
9653 : /* If we cannot do a conditional move on the mode, try doing it
9654 : with the promoted mode. */
9655 16996 : if (!can_conditionally_move_p (mode))
9656 : {
9657 241 : mode = promote_mode (type, mode, &unsignedp);
9658 241 : if (!can_conditionally_move_p (mode))
9659 : return NULL_RTX;
9660 0 : temp = assign_temp (type, 0, 0); /* Use promoted mode for temp. */
9661 : }
9662 : else
9663 16755 : temp = assign_temp (type, 0, 1);
9664 :
9665 16755 : expanding_cond_expr_using_cmove = true;
9666 16755 : start_sequence ();
9667 16755 : expand_operands (treeop1, treeop2,
9668 : mode == orig_mode ? temp : NULL_RTX, &op1, &op2,
9669 : EXPAND_NORMAL);
9670 :
9671 16755 : if (TREE_CODE (treeop0) == SSA_NAME
9672 16606 : && (srcstmt = get_def_for_expr_class (treeop0, tcc_comparison))
9673 31142 : && !VECTOR_TYPE_P (TREE_TYPE (gimple_assign_rhs1 (srcstmt))))
9674 : {
9675 14384 : type = TREE_TYPE (gimple_assign_rhs1 (srcstmt));
9676 14384 : enum tree_code cmpcode = gimple_assign_rhs_code (srcstmt);
9677 14384 : op00 = expand_normal (gimple_assign_rhs1 (srcstmt));
9678 14384 : op01 = expand_normal (gimple_assign_rhs2 (srcstmt));
9679 14384 : comparison_mode = TYPE_MODE (type);
9680 14384 : unsignedp = TYPE_UNSIGNED (type);
9681 14384 : comparison_code = convert_tree_comp_to_rtx (cmpcode, unsignedp);
9682 : }
9683 2371 : else if (COMPARISON_CLASS_P (treeop0)
9684 2371 : && !VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (treeop0, 0))))
9685 : {
9686 149 : type = TREE_TYPE (TREE_OPERAND (treeop0, 0));
9687 149 : enum tree_code cmpcode = TREE_CODE (treeop0);
9688 149 : op00 = expand_normal (TREE_OPERAND (treeop0, 0));
9689 149 : op01 = expand_normal (TREE_OPERAND (treeop0, 1));
9690 149 : unsignedp = TYPE_UNSIGNED (type);
9691 149 : comparison_mode = TYPE_MODE (type);
9692 149 : comparison_code = convert_tree_comp_to_rtx (cmpcode, unsignedp);
9693 : }
9694 : else
9695 : {
9696 2222 : op00 = expand_normal (treeop0);
9697 2222 : op01 = const0_rtx;
9698 2222 : comparison_code = NE;
9699 2222 : comparison_mode = GET_MODE (op00);
9700 2222 : if (comparison_mode == VOIDmode)
9701 0 : comparison_mode = TYPE_MODE (TREE_TYPE (treeop0));
9702 : }
9703 16755 : expanding_cond_expr_using_cmove = false;
9704 :
9705 16755 : if (GET_MODE (op1) != mode)
9706 2341 : op1 = gen_lowpart (mode, op1);
9707 :
9708 16755 : if (GET_MODE (op2) != mode)
9709 9065 : op2 = gen_lowpart (mode, op2);
9710 :
9711 : /* Try to emit the conditional move. */
9712 16755 : insn = emit_conditional_move (temp,
9713 : { comparison_code, op00, op01,
9714 : comparison_mode },
9715 : op1, op2, mode,
9716 : unsignedp);
9717 :
9718 : /* If we could do the conditional move, emit the sequence,
9719 : and return. */
9720 16755 : if (insn)
9721 : {
9722 14224 : rtx_insn *seq = end_sequence ();
9723 14224 : emit_insn (seq);
9724 14224 : return convert_modes (orig_mode, mode, temp, 0);
9725 : }
9726 :
9727 : /* Otherwise discard the sequence and fall back to code with
9728 : branches. */
9729 2531 : end_sequence ();
9730 2531 : return NULL_RTX;
9731 : }
9732 :
9733 : /* A helper function for expand_expr_real_2 to be used with a
9734 : misaligned mem_ref TEMP. Assume an unsigned type if UNSIGNEDP
9735 : is nonzero, with alignment ALIGN in bits.
9736 : Store the value at TARGET if possible (if TARGET is nonzero).
9737 : Regardless of TARGET, we return the rtx for where the value is placed.
9738 : If the result can be stored at TARGET, and ALT_RTL is non-NULL,
9739 : then *ALT_RTL is set to TARGET (before legitimziation). */
9740 :
9741 : static rtx
9742 207447 : expand_misaligned_mem_ref (rtx temp, machine_mode mode, int unsignedp,
9743 : unsigned int align, rtx target, rtx *alt_rtl)
9744 : {
9745 207447 : enum insn_code icode;
9746 :
9747 207447 : if ((icode = optab_handler (movmisalign_optab, mode))
9748 : != CODE_FOR_nothing)
9749 : {
9750 132549 : class expand_operand ops[2];
9751 :
9752 : /* We've already validated the memory, and we're creating a
9753 : new pseudo destination. The predicates really can't fail,
9754 : nor can the generator. */
9755 132549 : create_output_operand (&ops[0], NULL_RTX, mode);
9756 132549 : create_fixed_operand (&ops[1], temp);
9757 132549 : expand_insn (icode, 2, ops);
9758 132549 : temp = ops[0].value;
9759 : }
9760 74898 : else if (targetm.slow_unaligned_access (mode, align))
9761 0 : temp = extract_bit_field (temp, GET_MODE_BITSIZE (mode),
9762 : 0, unsignedp, target,
9763 : mode, mode, false, alt_rtl);
9764 207447 : return temp;
9765 : }
9766 :
9767 : /* Helper function of expand_expr_2, expand a division or modulo.
9768 : op0 and op1 should be already expanded treeop0 and treeop1, using
9769 : expand_operands. */
9770 :
9771 : static rtx
9772 155564 : expand_expr_divmod (tree_code code, machine_mode mode, tree treeop0,
9773 : tree treeop1, rtx op0, rtx op1, rtx target, int unsignedp)
9774 : {
9775 311128 : bool mod_p = (code == TRUNC_MOD_EXPR || code == FLOOR_MOD_EXPR
9776 155564 : || code == CEIL_MOD_EXPR || code == ROUND_MOD_EXPR);
9777 155564 : if (SCALAR_INT_MODE_P (mode)
9778 155564 : && optimize >= 2
9779 129047 : && get_range_pos_neg (treeop0, currently_expanding_gimple_stmt) == 1
9780 187635 : && get_range_pos_neg (treeop1, currently_expanding_gimple_stmt) == 1)
9781 : {
9782 : /* If both arguments are known to be positive when interpreted
9783 : as signed, we can expand it as both signed and unsigned
9784 : division or modulo. Choose the cheaper sequence in that case. */
9785 18619 : bool speed_p = optimize_insn_for_speed_p ();
9786 18619 : do_pending_stack_adjust ();
9787 18619 : start_sequence ();
9788 18619 : rtx uns_ret = expand_divmod (mod_p, code, mode, op0, op1, target, 1);
9789 18619 : rtx_insn *uns_insns = end_sequence ();
9790 18619 : start_sequence ();
9791 18619 : rtx sgn_ret = expand_divmod (mod_p, code, mode, op0, op1, target, 0);
9792 18619 : rtx_insn *sgn_insns = end_sequence ();
9793 18619 : unsigned uns_cost = seq_cost (uns_insns, speed_p);
9794 18619 : unsigned sgn_cost = seq_cost (sgn_insns, speed_p);
9795 18619 : bool was_tie = false;
9796 :
9797 : /* If costs are the same then use as tie breaker the other other
9798 : factor. */
9799 18619 : if (uns_cost == sgn_cost)
9800 : {
9801 8302 : uns_cost = seq_cost (uns_insns, !speed_p);
9802 8302 : sgn_cost = seq_cost (sgn_insns, !speed_p);
9803 8302 : was_tie = true;
9804 : }
9805 :
9806 18619 : if (dump_file && (dump_flags & TDF_DETAILS))
9807 0 : fprintf (dump_file, ";; positive division:%s unsigned cost: %u; "
9808 : "signed cost: %u\n",
9809 : was_tie ? " (needed tie breaker)" : "", uns_cost, sgn_cost);
9810 :
9811 18619 : if (uns_cost < sgn_cost || (uns_cost == sgn_cost && unsignedp))
9812 : {
9813 11128 : emit_insn (uns_insns);
9814 11128 : return uns_ret;
9815 : }
9816 7491 : emit_insn (sgn_insns);
9817 7491 : return sgn_ret;
9818 : }
9819 136945 : return expand_divmod (mod_p, code, mode, op0, op1, target, unsignedp);
9820 : }
9821 :
9822 : /* Return true if EXP has a range of values [0..1], false
9823 : otherwise. This works for constants and ssa names, calling back into the ranger. */
9824 : static bool
9825 1465054 : expr_has_boolean_range (tree exp, gimple *stmt)
9826 : {
9827 : /* An integral type with a single bit of precision. */
9828 2930095 : if (INTEGRAL_TYPE_P (TREE_TYPE (exp))
9829 1465054 : && TYPE_UNSIGNED (TREE_TYPE (exp))
9830 2464502 : && TYPE_PRECISION (TREE_TYPE (exp)) == 1)
9831 : return true;
9832 :
9833 : /* Signed 1 bit integers are not boolean ranges. */
9834 2930095 : if (!INTEGRAL_TYPE_P (TREE_TYPE (exp))
9835 2930095 : || TYPE_PRECISION (TREE_TYPE (exp)) <= 1)
9836 : return false;
9837 :
9838 1465010 : if (TREE_CODE (exp) == SSA_NAME)
9839 854365 : return ssa_name_has_boolean_range (exp, stmt);
9840 610645 : if (TREE_CODE (exp) == INTEGER_CST)
9841 538572 : return wi::leu_p (wi::to_wide (exp), 1);
9842 : return false;
9843 : }
9844 :
9845 : rtx
9846 13197464 : expand_expr_real_2 (const_sepops ops, rtx target, machine_mode tmode,
9847 : enum expand_modifier modifier)
9848 : {
9849 13197464 : rtx op0, op1, op2, temp;
9850 13197464 : rtx_code_label *lab;
9851 13197464 : tree type;
9852 13197464 : int unsignedp;
9853 13197464 : machine_mode mode;
9854 13197464 : scalar_int_mode int_mode;
9855 13197464 : enum tree_code code = ops->code;
9856 13197464 : optab this_optab;
9857 13197464 : rtx subtarget, original_target;
9858 13197464 : int ignore;
9859 13197464 : bool reduce_bit_field;
9860 13197464 : location_t loc = ops->location;
9861 13197464 : tree treeop0, treeop1, treeop2;
9862 : #define REDUCE_BIT_FIELD(expr) (reduce_bit_field \
9863 : ? reduce_to_bit_field_precision ((expr), \
9864 : target, \
9865 : type) \
9866 : : (expr))
9867 :
9868 13197464 : type = ops->type;
9869 13197464 : mode = TYPE_MODE (type);
9870 13197464 : unsignedp = TYPE_UNSIGNED (type);
9871 :
9872 13197464 : treeop0 = ops->op0;
9873 13197464 : treeop1 = ops->op1;
9874 13197464 : treeop2 = ops->op2;
9875 :
9876 : /* We should be called only on simple (binary or unary) expressions,
9877 : exactly those that are valid in gimple expressions that aren't
9878 : GIMPLE_SINGLE_RHS (or invalid). */
9879 13197464 : gcc_assert (get_gimple_rhs_class (code) == GIMPLE_UNARY_RHS
9880 : || get_gimple_rhs_class (code) == GIMPLE_BINARY_RHS
9881 : || get_gimple_rhs_class (code) == GIMPLE_TERNARY_RHS);
9882 :
9883 26394928 : ignore = (target == const0_rtx
9884 13197464 : || ((CONVERT_EXPR_CODE_P (code)
9885 9476279 : || code == COND_EXPR || code == VIEW_CONVERT_EXPR)
9886 3739147 : && TREE_CODE (type) == VOID_TYPE));
9887 :
9888 : /* We should be called only if we need the result. */
9889 0 : gcc_assert (!ignore);
9890 :
9891 : /* An operation in what may be a bit-field type needs the
9892 : result to be reduced to the precision of the bit-field type,
9893 : which is narrower than that of the type's mode. */
9894 27165315 : reduce_bit_field = (INTEGRAL_TYPE_P (type)
9895 13197464 : && !type_has_mode_precision_p (type));
9896 :
9897 770387 : if (reduce_bit_field
9898 770387 : && (modifier == EXPAND_STACK_PARM
9899 769863 : || (target && GET_MODE (target) != mode)))
9900 459664 : target = 0;
9901 :
9902 : /* Use subtarget as the target for operand 0 of a binary operation. */
9903 13197464 : subtarget = get_subtarget (target);
9904 13197464 : original_target = target;
9905 :
9906 13197464 : switch (code)
9907 : {
9908 3724387 : case NON_LVALUE_EXPR:
9909 3724387 : case PAREN_EXPR:
9910 3724387 : CASE_CONVERT:
9911 3724387 : if (treeop0 == error_mark_node)
9912 0 : return const0_rtx;
9913 :
9914 3724387 : if (TREE_CODE (type) == UNION_TYPE)
9915 : {
9916 0 : tree valtype = TREE_TYPE (treeop0);
9917 :
9918 : /* If both input and output are BLKmode, this conversion isn't doing
9919 : anything except possibly changing memory attribute. */
9920 0 : if (mode == BLKmode && TYPE_MODE (valtype) == BLKmode)
9921 : {
9922 0 : rtx result = expand_expr (treeop0, target, tmode,
9923 : modifier);
9924 :
9925 0 : result = copy_rtx (result);
9926 0 : set_mem_attributes (result, type, 0);
9927 0 : return result;
9928 : }
9929 :
9930 0 : if (target == 0)
9931 : {
9932 0 : if (TYPE_MODE (type) != BLKmode)
9933 0 : target = gen_reg_rtx (TYPE_MODE (type));
9934 : else
9935 0 : target = assign_temp (type, 1, 1);
9936 : }
9937 :
9938 0 : if (MEM_P (target))
9939 : /* Store data into beginning of memory target. */
9940 0 : store_expr (treeop0,
9941 0 : adjust_address (target, TYPE_MODE (valtype), 0),
9942 : modifier == EXPAND_STACK_PARM,
9943 0 : false, TYPE_REVERSE_STORAGE_ORDER (type));
9944 :
9945 : else
9946 : {
9947 0 : gcc_assert (REG_P (target)
9948 : && !TYPE_REVERSE_STORAGE_ORDER (type));
9949 :
9950 : /* Store this field into a union of the proper type. */
9951 0 : poly_uint64 op0_size
9952 0 : = tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (treeop0)));
9953 0 : poly_uint64 union_size = GET_MODE_BITSIZE (mode);
9954 0 : store_field (target,
9955 : /* The conversion must be constructed so that
9956 : we know at compile time how many bits
9957 : to preserve. */
9958 0 : ordered_min (op0_size, union_size),
9959 0 : 0, 0, 0, TYPE_MODE (valtype), treeop0, 0,
9960 : false, false);
9961 : }
9962 :
9963 : /* Return the entire union. */
9964 0 : return target;
9965 : }
9966 :
9967 3724387 : if (mode == TYPE_MODE (TREE_TYPE (treeop0)))
9968 : {
9969 2210022 : op0 = expand_expr (treeop0, target, VOIDmode,
9970 : modifier);
9971 :
9972 2210022 : return REDUCE_BIT_FIELD (op0);
9973 : }
9974 :
9975 1867924 : op0 = expand_expr (treeop0, NULL_RTX, mode,
9976 : modifier == EXPAND_SUM ? EXPAND_NORMAL : modifier);
9977 1514365 : if (GET_MODE (op0) == mode)
9978 : ;
9979 :
9980 : /* If OP0 is a constant, just convert it into the proper mode. */
9981 1474525 : else if (CONSTANT_P (op0))
9982 : {
9983 907 : tree inner_type = TREE_TYPE (treeop0);
9984 907 : machine_mode inner_mode = GET_MODE (op0);
9985 :
9986 907 : if (inner_mode == VOIDmode)
9987 15 : inner_mode = TYPE_MODE (inner_type);
9988 :
9989 907 : if (modifier == EXPAND_INITIALIZER)
9990 0 : op0 = force_lowpart_subreg (mode, op0, inner_mode);
9991 : else
9992 907 : op0 = convert_modes (mode, inner_mode, op0,
9993 907 : TYPE_UNSIGNED (inner_type));
9994 : }
9995 :
9996 1473618 : else if (modifier == EXPAND_INITIALIZER)
9997 24 : op0 = gen_rtx_fmt_e (TYPE_UNSIGNED (TREE_TYPE (treeop0))
9998 : ? ZERO_EXTEND : SIGN_EXTEND, mode, op0);
9999 :
10000 1473606 : else if (SCALAR_INT_MODE_P (GET_MODE (op0))
10001 1300938 : && optimize >= 2
10002 764153 : && SCALAR_INT_MODE_P (mode)
10003 764153 : && INTEGRAL_TYPE_P (TREE_TYPE (treeop0))
10004 763944 : && (GET_MODE_SIZE (as_a <scalar_int_mode> (mode))
10005 1527888 : > GET_MODE_SIZE (as_a <scalar_int_mode> (GET_MODE (op0))))
10006 2009997 : && get_range_pos_neg (treeop0,
10007 : currently_expanding_gimple_stmt) == 1)
10008 : {
10009 : /* If argument is known to be positive when interpreted
10010 : as signed, we can expand it as both sign and zero
10011 : extension. Choose the cheaper sequence in that case. */
10012 138492 : bool speed_p = optimize_insn_for_speed_p ();
10013 138492 : rtx uns_ret = NULL_RTX, sgn_ret = NULL_RTX;
10014 138492 : do_pending_stack_adjust ();
10015 138492 : start_sequence ();
10016 138492 : if (target == NULL_RTX)
10017 114159 : uns_ret = convert_to_mode (mode, op0, 1);
10018 : else
10019 24333 : convert_move (target, op0, 1);
10020 138492 : rtx_insn *uns_insns = end_sequence ();
10021 138492 : start_sequence ();
10022 138492 : if (target == NULL_RTX)
10023 114159 : sgn_ret = convert_to_mode (mode, op0, 0);
10024 : else
10025 24333 : convert_move (target, op0, 0);
10026 138492 : rtx_insn *sgn_insns = end_sequence ();
10027 138492 : unsigned uns_cost = seq_cost (uns_insns, speed_p);
10028 138492 : unsigned sgn_cost = seq_cost (sgn_insns, speed_p);
10029 138492 : bool was_tie = false;
10030 :
10031 : /* If costs are the same then use as tie breaker the other other
10032 : factor. */
10033 138492 : if (uns_cost == sgn_cost)
10034 : {
10035 38726 : uns_cost = seq_cost (uns_insns, !speed_p);
10036 38726 : sgn_cost = seq_cost (sgn_insns, !speed_p);
10037 38726 : was_tie = true;
10038 : }
10039 :
10040 138492 : if (dump_file && (dump_flags & TDF_DETAILS))
10041 0 : fprintf (dump_file, ";; positive extension:%s unsigned cost: %u; "
10042 : "signed cost: %u\n",
10043 : was_tie ? " (needed tie breaker)" : "",
10044 : uns_cost, sgn_cost);
10045 138492 : if (uns_cost < sgn_cost
10046 138492 : || (uns_cost == sgn_cost && TYPE_UNSIGNED (TREE_TYPE (treeop0))))
10047 : {
10048 123620 : emit_insn (uns_insns);
10049 123620 : sgn_ret = uns_ret;
10050 : }
10051 : else
10052 14872 : emit_insn (sgn_insns);
10053 138492 : if (target == NULL_RTX)
10054 114159 : op0 = sgn_ret;
10055 : else
10056 24333 : op0 = target;
10057 : }
10058 1335114 : else if (target == 0)
10059 844169 : op0 = convert_to_mode (mode, op0, TYPE_UNSIGNED (TREE_TYPE (treeop0)));
10060 : else
10061 : {
10062 490945 : convert_move (target, op0, TYPE_UNSIGNED (TREE_TYPE (treeop0)));
10063 490945 : op0 = target;
10064 : }
10065 :
10066 1514365 : return REDUCE_BIT_FIELD (op0);
10067 :
10068 0 : case ADDR_SPACE_CONVERT_EXPR:
10069 0 : {
10070 0 : tree treeop0_type = TREE_TYPE (treeop0);
10071 :
10072 0 : gcc_assert (POINTER_TYPE_P (type));
10073 0 : gcc_assert (POINTER_TYPE_P (treeop0_type));
10074 :
10075 0 : addr_space_t as_to = TYPE_ADDR_SPACE (TREE_TYPE (type));
10076 0 : addr_space_t as_from = TYPE_ADDR_SPACE (TREE_TYPE (treeop0_type));
10077 :
10078 : /* Conversions between pointers to the same address space should
10079 : have been implemented via CONVERT_EXPR / NOP_EXPR. */
10080 0 : gcc_assert (as_to != as_from);
10081 :
10082 0 : op0 = expand_expr (treeop0, NULL_RTX, VOIDmode, modifier);
10083 :
10084 : /* Ask target code to handle conversion between pointers
10085 : to overlapping address spaces. */
10086 0 : if (targetm.addr_space.subset_p (as_to, as_from)
10087 0 : || targetm.addr_space.subset_p (as_from, as_to))
10088 : {
10089 0 : op0 = targetm.addr_space.convert (op0, treeop0_type, type);
10090 : }
10091 : else
10092 : {
10093 : /* For disjoint address spaces, converting anything but a null
10094 : pointer invokes undefined behavior. We truncate or extend the
10095 : value as if we'd converted via integers, which handles 0 as
10096 : required, and all others as the programmer likely expects. */
10097 : #ifndef POINTERS_EXTEND_UNSIGNED
10098 : const int POINTERS_EXTEND_UNSIGNED = 1;
10099 : #endif
10100 0 : op0 = convert_modes (mode, TYPE_MODE (treeop0_type),
10101 : op0, POINTERS_EXTEND_UNSIGNED);
10102 : }
10103 0 : gcc_assert (op0);
10104 : return op0;
10105 : }
10106 :
10107 1325771 : case POINTER_PLUS_EXPR:
10108 : /* Even though the sizetype mode and the pointer's mode can be different
10109 : expand is able to handle this correctly and get the correct result out
10110 : of the PLUS_EXPR code. */
10111 : /* Make sure to sign-extend the sizetype offset in a POINTER_PLUS_EXPR
10112 : if sizetype precision is smaller than pointer precision. */
10113 1325771 : if (TYPE_PRECISION (sizetype) < TYPE_PRECISION (type))
10114 0 : treeop1 = fold_convert_loc (loc, type,
10115 : fold_convert_loc (loc, ssizetype,
10116 : treeop1));
10117 : /* If sizetype precision is larger than pointer precision, truncate the
10118 : offset to have matching modes. */
10119 1325771 : else if (TYPE_PRECISION (sizetype) > TYPE_PRECISION (type))
10120 0 : treeop1 = fold_convert_loc (loc, type, treeop1);
10121 : /* FALLTHRU */
10122 :
10123 5266610 : case PLUS_EXPR:
10124 : /* If we are adding a constant, a VAR_DECL that is sp, fp, or ap, and
10125 : something else, make sure we add the register to the constant and
10126 : then to the other thing. This case can occur during strength
10127 : reduction and doing it this way will produce better code if the
10128 : frame pointer or argument pointer is eliminated.
10129 :
10130 : fold-const.cc will ensure that the constant is always in the inner
10131 : PLUS_EXPR, so the only case we need to do anything about is if
10132 : sp, ap, or fp is our second argument, in which case we must swap
10133 : the innermost first argument and our second argument. */
10134 :
10135 5266610 : if (TREE_CODE (treeop0) == PLUS_EXPR
10136 6911 : && TREE_CODE (TREE_OPERAND (treeop0, 1)) == INTEGER_CST
10137 0 : && VAR_P (treeop1)
10138 5266610 : && (DECL_RTL (treeop1) == frame_pointer_rtx
10139 0 : || DECL_RTL (treeop1) == stack_pointer_rtx
10140 0 : || DECL_RTL (treeop1) == arg_pointer_rtx))
10141 : {
10142 0 : gcc_unreachable ();
10143 : }
10144 :
10145 : /* If the result is to be ptr_mode and we are adding an integer to
10146 : something, we might be forming a constant. So try to use
10147 : plus_constant. If it produces a sum and we can't accept it,
10148 : use force_operand. This allows P = &ARR[const] to generate
10149 : efficient code on machines where a SYMBOL_REF is not a valid
10150 : address.
10151 :
10152 : If this is an EXPAND_SUM call, always return the sum. */
10153 5266610 : if (modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER
10154 5266610 : || (mode == ptr_mode && (unsignedp || ! flag_trapv)))
10155 : {
10156 3421389 : if (modifier == EXPAND_STACK_PARM)
10157 18868 : target = 0;
10158 3421389 : if (TREE_CODE (treeop0) == INTEGER_CST
10159 3422396 : && HWI_COMPUTABLE_MODE_P (mode)
10160 3422400 : && TREE_CONSTANT (treeop1))
10161 : {
10162 4 : rtx constant_part;
10163 4 : HOST_WIDE_INT wc;
10164 4 : machine_mode wmode = TYPE_MODE (TREE_TYPE (treeop1));
10165 :
10166 4 : op1 = expand_expr (treeop1, subtarget, VOIDmode,
10167 : EXPAND_SUM);
10168 : /* Use wi::shwi to ensure that the constant is
10169 : truncated according to the mode of OP1, then sign extended
10170 : to a HOST_WIDE_INT. Using the constant directly can result
10171 : in non-canonical RTL in a 64x32 cross compile. */
10172 4 : wc = TREE_INT_CST_LOW (treeop0);
10173 4 : constant_part =
10174 4 : immed_wide_int_const (wi::shwi (wc, wmode), wmode);
10175 4 : op1 = plus_constant (mode, op1, INTVAL (constant_part));
10176 4 : if (modifier != EXPAND_SUM && modifier != EXPAND_INITIALIZER)
10177 1 : op1 = force_operand (op1, target);
10178 4 : return REDUCE_BIT_FIELD (op1);
10179 : }
10180 :
10181 3421385 : else if (TREE_CODE (treeop1) == INTEGER_CST
10182 7300884 : && HWI_COMPUTABLE_MODE_P (mode)
10183 5727875 : && TREE_CONSTANT (treeop0))
10184 : {
10185 272212 : rtx constant_part;
10186 272212 : HOST_WIDE_INT wc;
10187 272212 : machine_mode wmode = TYPE_MODE (TREE_TYPE (treeop0));
10188 :
10189 474664 : op0 = expand_expr (treeop0, subtarget, VOIDmode,
10190 : (modifier == EXPAND_INITIALIZER
10191 : ? EXPAND_INITIALIZER : EXPAND_SUM));
10192 272212 : if (! CONSTANT_P (op0))
10193 : {
10194 6 : op1 = expand_expr (treeop1, NULL_RTX,
10195 : VOIDmode, modifier);
10196 : /* Return a PLUS if modifier says it's OK. */
10197 6 : if (modifier == EXPAND_SUM
10198 : || modifier == EXPAND_INITIALIZER)
10199 6 : return simplify_gen_binary (PLUS, mode, op0, op1);
10200 0 : goto binop2;
10201 : }
10202 : /* Use wi::shwi to ensure that the constant is
10203 : truncated according to the mode of OP1, then sign extended
10204 : to a HOST_WIDE_INT. Using the constant directly can result
10205 : in non-canonical RTL in a 64x32 cross compile. */
10206 272206 : wc = TREE_INT_CST_LOW (treeop1);
10207 272206 : constant_part
10208 272206 : = immed_wide_int_const (wi::shwi (wc, wmode), wmode);
10209 272206 : op0 = plus_constant (mode, op0, INTVAL (constant_part));
10210 272206 : if (modifier != EXPAND_SUM && modifier != EXPAND_INITIALIZER)
10211 201654 : op0 = force_operand (op0, target);
10212 272206 : return REDUCE_BIT_FIELD (op0);
10213 : }
10214 : }
10215 :
10216 : /* Use TER to expand pointer addition of a negated value
10217 : as pointer subtraction. */
10218 8952613 : if ((POINTER_TYPE_P (TREE_TYPE (treeop0))
10219 3940826 : || (TREE_CODE (TREE_TYPE (treeop0)) == VECTOR_TYPE
10220 80441 : && POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (treeop0)))))
10221 1053568 : && TREE_CODE (treeop1) == SSA_NAME
10222 6107990 : && TYPE_MODE (TREE_TYPE (treeop0))
10223 556798 : == TYPE_MODE (TREE_TYPE (treeop1)))
10224 : {
10225 556798 : gimple *def = get_def_for_expr (treeop1, NEGATE_EXPR);
10226 556798 : if (def)
10227 : {
10228 2566 : treeop1 = gimple_assign_rhs1 (def);
10229 2566 : code = MINUS_EXPR;
10230 2566 : goto do_minus;
10231 : }
10232 : }
10233 :
10234 : /* No sense saving up arithmetic to be done
10235 : if it's all in the wrong mode to form part of an address.
10236 : And force_operand won't know whether to sign-extend or
10237 : zero-extend. */
10238 4991828 : if (modifier != EXPAND_INITIALIZER
10239 4991828 : && (modifier != EXPAND_SUM || mode != ptr_mode))
10240 : {
10241 4427035 : expand_operands (treeop0, treeop1,
10242 : subtarget, &op0, &op1, modifier);
10243 4427035 : if (op0 == const0_rtx)
10244 8670 : return op1;
10245 4418365 : if (op1 == const0_rtx)
10246 : return op0;
10247 4418113 : goto binop2;
10248 : }
10249 :
10250 564793 : expand_operands (treeop0, treeop1,
10251 : subtarget, &op0, &op1, modifier);
10252 564793 : return REDUCE_BIT_FIELD (simplify_gen_binary (PLUS, mode, op0, op1));
10253 :
10254 515399 : case MINUS_EXPR:
10255 515399 : case POINTER_DIFF_EXPR:
10256 515399 : do_minus:
10257 : /* For initializers, we are allowed to return a MINUS of two
10258 : symbolic constants. Here we handle all cases when both operands
10259 : are constant. */
10260 : /* Handle difference of two symbolic constants,
10261 : for the sake of an initializer. */
10262 515399 : if ((modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER)
10263 3217 : && really_constant_p (treeop0)
10264 515904 : && really_constant_p (treeop1))
10265 : {
10266 68 : expand_operands (treeop0, treeop1,
10267 : NULL_RTX, &op0, &op1, modifier);
10268 68 : return simplify_gen_binary (MINUS, mode, op0, op1);
10269 : }
10270 :
10271 : /* No sense saving up arithmetic to be done
10272 : if it's all in the wrong mode to form part of an address.
10273 : And force_operand won't know whether to sign-extend or
10274 : zero-extend. */
10275 515331 : if (modifier != EXPAND_INITIALIZER
10276 515331 : && (modifier != EXPAND_SUM || mode != ptr_mode))
10277 512182 : goto binop;
10278 :
10279 3149 : expand_operands (treeop0, treeop1,
10280 : subtarget, &op0, &op1, modifier);
10281 :
10282 : /* Convert A - const to A + (-const). */
10283 3149 : if (CONST_INT_P (op1))
10284 : {
10285 0 : op1 = negate_rtx (mode, op1);
10286 0 : return REDUCE_BIT_FIELD (simplify_gen_binary (PLUS, mode, op0, op1));
10287 : }
10288 :
10289 3149 : goto binop2;
10290 :
10291 0 : case WIDEN_MULT_PLUS_EXPR:
10292 0 : case WIDEN_MULT_MINUS_EXPR:
10293 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
10294 0 : op2 = expand_normal (treeop2);
10295 0 : target = expand_widen_pattern_expr (ops, op0, op1, op2,
10296 : target, unsignedp);
10297 0 : return target;
10298 :
10299 17988 : case WIDEN_MULT_EXPR:
10300 : /* If first operand is constant, swap them.
10301 : Thus the following special case checks need only
10302 : check the second operand. */
10303 17988 : if (TREE_CODE (treeop0) == INTEGER_CST)
10304 368 : std::swap (treeop0, treeop1);
10305 :
10306 : /* First, check if we have a multiplication of one signed and one
10307 : unsigned operand. */
10308 17988 : if (TREE_CODE (treeop1) != INTEGER_CST
10309 17988 : && (TYPE_UNSIGNED (TREE_TYPE (treeop0))
10310 13761 : != TYPE_UNSIGNED (TREE_TYPE (treeop1))))
10311 : {
10312 0 : machine_mode innermode = TYPE_MODE (TREE_TYPE (treeop0));
10313 0 : this_optab = usmul_widen_optab;
10314 0 : if (find_widening_optab_handler (this_optab, mode, innermode)
10315 : != CODE_FOR_nothing)
10316 : {
10317 0 : if (TYPE_UNSIGNED (TREE_TYPE (treeop0)))
10318 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1,
10319 : EXPAND_NORMAL);
10320 : else
10321 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op1, &op0,
10322 : EXPAND_NORMAL);
10323 : /* op0 and op1 might still be constant, despite the above
10324 : != INTEGER_CST check. Handle it. */
10325 0 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
10326 : {
10327 0 : op0 = convert_modes (mode, innermode, op0, true);
10328 0 : op1 = convert_modes (mode, innermode, op1, false);
10329 0 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1,
10330 : target, unsignedp));
10331 : }
10332 0 : goto binop3;
10333 : }
10334 : }
10335 : /* Check for a multiplication with matching signedness. */
10336 17988 : else if ((TREE_CODE (treeop1) == INTEGER_CST
10337 4227 : && int_fits_type_p (treeop1, TREE_TYPE (treeop0)))
10338 17992 : || (TYPE_UNSIGNED (TREE_TYPE (treeop1))
10339 13765 : == TYPE_UNSIGNED (TREE_TYPE (treeop0))))
10340 : {
10341 17988 : tree op0type = TREE_TYPE (treeop0);
10342 17988 : machine_mode innermode = TYPE_MODE (op0type);
10343 17988 : bool zextend_p = TYPE_UNSIGNED (op0type);
10344 17988 : optab other_optab = zextend_p ? smul_widen_optab : umul_widen_optab;
10345 2555 : this_optab = zextend_p ? umul_widen_optab : smul_widen_optab;
10346 :
10347 17988 : if (TREE_CODE (treeop0) != INTEGER_CST)
10348 : {
10349 17988 : if (find_widening_optab_handler (this_optab, mode, innermode)
10350 : != CODE_FOR_nothing)
10351 : {
10352 17988 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1,
10353 : EXPAND_NORMAL);
10354 : /* op0 and op1 might still be constant, despite the above
10355 : != INTEGER_CST check. Handle it. */
10356 17988 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
10357 : {
10358 0 : widen_mult_const:
10359 0 : op0 = convert_modes (mode, innermode, op0, zextend_p);
10360 0 : op1
10361 0 : = convert_modes (mode, innermode, op1,
10362 0 : TYPE_UNSIGNED (TREE_TYPE (treeop1)));
10363 0 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1,
10364 : target,
10365 : unsignedp));
10366 : }
10367 17988 : temp = expand_widening_mult (mode, op0, op1, target,
10368 : unsignedp, this_optab);
10369 17988 : return REDUCE_BIT_FIELD (temp);
10370 : }
10371 0 : if (find_widening_optab_handler (other_optab, mode, innermode)
10372 : != CODE_FOR_nothing
10373 0 : && innermode == word_mode)
10374 : {
10375 0 : rtx htem, hipart;
10376 0 : op0 = expand_normal (treeop0);
10377 0 : op1 = expand_normal (treeop1);
10378 : /* op0 and op1 might be constants, despite the above
10379 : != INTEGER_CST check. Handle it. */
10380 0 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
10381 0 : goto widen_mult_const;
10382 0 : temp = expand_binop (mode, other_optab, op0, op1, target,
10383 : unsignedp, OPTAB_LIB_WIDEN);
10384 0 : hipart = gen_highpart (word_mode, temp);
10385 0 : htem = expand_mult_highpart_adjust (word_mode, hipart,
10386 : op0, op1, hipart,
10387 : zextend_p);
10388 0 : if (htem != hipart)
10389 0 : emit_move_insn (hipart, htem);
10390 0 : return REDUCE_BIT_FIELD (temp);
10391 : }
10392 : }
10393 : }
10394 0 : treeop0 = fold_build1 (CONVERT_EXPR, type, treeop0);
10395 0 : treeop1 = fold_build1 (CONVERT_EXPR, type, treeop1);
10396 0 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10397 0 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1, target, unsignedp));
10398 :
10399 1390758 : case MULT_EXPR:
10400 : /* If this is a fixed-point operation, then we cannot use the code
10401 : below because "expand_mult" doesn't support sat/no-sat fixed-point
10402 : multiplications. */
10403 1390758 : if (ALL_FIXED_POINT_MODE_P (mode))
10404 0 : goto binop;
10405 :
10406 : /* If first operand is constant, swap them.
10407 : Thus the following special case checks need only
10408 : check the second operand. */
10409 1390758 : if (TREE_CODE (treeop0) == INTEGER_CST)
10410 686 : std::swap (treeop0, treeop1);
10411 :
10412 : /* Attempt to return something suitable for generating an
10413 : indexed address, for machines that support that. */
10414 :
10415 533812 : if (modifier == EXPAND_SUM && mode == ptr_mode
10416 1924570 : && tree_fits_shwi_p (treeop1))
10417 : {
10418 526866 : tree exp1 = treeop1;
10419 :
10420 526866 : op0 = expand_expr (treeop0, subtarget, VOIDmode,
10421 : EXPAND_SUM);
10422 :
10423 526866 : if (!REG_P (op0))
10424 246086 : op0 = force_operand (op0, NULL_RTX);
10425 526866 : if (!REG_P (op0))
10426 2256 : op0 = copy_to_mode_reg (mode, op0);
10427 :
10428 526866 : op1 = gen_int_mode (tree_to_shwi (exp1),
10429 526866 : TYPE_MODE (TREE_TYPE (exp1)));
10430 526866 : return REDUCE_BIT_FIELD (gen_rtx_MULT (mode, op0, op1));
10431 : }
10432 :
10433 863892 : if (modifier == EXPAND_STACK_PARM)
10434 2953 : target = 0;
10435 :
10436 863892 : if (SCALAR_INT_MODE_P (mode) && optimize >= 2)
10437 : {
10438 515710 : gimple *def_stmt0 = get_def_for_expr (treeop0, TRUNC_DIV_EXPR);
10439 515710 : gimple *def_stmt1 = get_def_for_expr (treeop1, TRUNC_DIV_EXPR);
10440 515710 : if (def_stmt0
10441 515710 : && !operand_equal_p (treeop1, gimple_assign_rhs2 (def_stmt0), 0))
10442 : def_stmt0 = NULL;
10443 515710 : if (def_stmt1
10444 515710 : && !operand_equal_p (treeop0, gimple_assign_rhs2 (def_stmt1), 0))
10445 : def_stmt1 = NULL;
10446 :
10447 515710 : if (def_stmt0 || def_stmt1)
10448 : {
10449 : /* X / Y * Y can be expanded as X - X % Y too.
10450 : Choose the cheaper sequence of those two. */
10451 260 : if (def_stmt0)
10452 260 : treeop0 = gimple_assign_rhs1 (def_stmt0);
10453 : else
10454 : {
10455 0 : treeop1 = treeop0;
10456 0 : treeop0 = gimple_assign_rhs1 (def_stmt1);
10457 : }
10458 260 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1,
10459 : EXPAND_NORMAL);
10460 260 : bool speed_p = optimize_insn_for_speed_p ();
10461 260 : do_pending_stack_adjust ();
10462 260 : start_sequence ();
10463 260 : rtx divmul_ret
10464 260 : = expand_expr_divmod (TRUNC_DIV_EXPR, mode, treeop0, treeop1,
10465 : op0, op1, NULL_RTX, unsignedp);
10466 260 : divmul_ret = expand_mult (mode, divmul_ret, op1, target,
10467 : unsignedp);
10468 260 : rtx_insn *divmul_insns = end_sequence ();
10469 260 : start_sequence ();
10470 260 : rtx modsub_ret
10471 260 : = expand_expr_divmod (TRUNC_MOD_EXPR, mode, treeop0, treeop1,
10472 : op0, op1, NULL_RTX, unsignedp);
10473 260 : this_optab = optab_for_tree_code (MINUS_EXPR, type,
10474 : optab_default);
10475 260 : modsub_ret = expand_binop (mode, this_optab, op0, modsub_ret,
10476 : target, unsignedp, OPTAB_LIB_WIDEN);
10477 260 : rtx_insn *modsub_insns = end_sequence ();
10478 260 : unsigned divmul_cost = seq_cost (divmul_insns, speed_p);
10479 260 : unsigned modsub_cost = seq_cost (modsub_insns, speed_p);
10480 : /* If costs are the same then use as tie breaker the other other
10481 : factor. */
10482 260 : if (divmul_cost == modsub_cost)
10483 : {
10484 22 : divmul_cost = seq_cost (divmul_insns, !speed_p);
10485 22 : modsub_cost = seq_cost (modsub_insns, !speed_p);
10486 : }
10487 :
10488 260 : if (divmul_cost <= modsub_cost)
10489 : {
10490 186 : emit_insn (divmul_insns);
10491 186 : return REDUCE_BIT_FIELD (divmul_ret);
10492 : }
10493 74 : emit_insn (modsub_insns);
10494 74 : return REDUCE_BIT_FIELD (modsub_ret);
10495 : }
10496 : }
10497 :
10498 863632 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10499 :
10500 : /* Expand X*Y as X&-Y when Y must be zero or one. */
10501 863632 : if (SCALAR_INT_MODE_P (mode))
10502 : {
10503 732527 : bool bit0_p = expr_has_boolean_range (treeop0, currently_expanding_gimple_stmt);
10504 732527 : bool bit1_p = expr_has_boolean_range (treeop1, currently_expanding_gimple_stmt);
10505 :
10506 : /* Expand X*Y as X&Y when both X and Y must be zero or one. */
10507 732527 : if (bit0_p && bit1_p)
10508 4 : return REDUCE_BIT_FIELD (expand_and (mode, op0, op1, target));
10509 :
10510 732523 : if (bit0_p || bit1_p)
10511 : {
10512 4748 : bool speed = optimize_insn_for_speed_p ();
10513 4748 : int cost = add_cost (speed, mode) + neg_cost (speed, mode);
10514 4748 : struct algorithm algorithm;
10515 4748 : enum mult_variant variant;
10516 4748 : if (CONST_INT_P (op1)
10517 4748 : ? !choose_mult_variant (mode, INTVAL (op1),
10518 : &algorithm, &variant, cost)
10519 675 : : cost < mul_cost (speed, mode))
10520 : {
10521 2048 : temp = bit0_p ? expand_and (mode, negate_rtx (mode, op0),
10522 : op1, target)
10523 262 : : expand_and (mode, op0,
10524 : negate_rtx (mode, op1),
10525 : target);
10526 2048 : return REDUCE_BIT_FIELD (temp);
10527 : }
10528 : }
10529 : }
10530 :
10531 861580 : return REDUCE_BIT_FIELD (expand_mult (mode, op0, op1, target, unsignedp));
10532 :
10533 155044 : case TRUNC_MOD_EXPR:
10534 155044 : case FLOOR_MOD_EXPR:
10535 155044 : case CEIL_MOD_EXPR:
10536 155044 : case ROUND_MOD_EXPR:
10537 :
10538 155044 : case TRUNC_DIV_EXPR:
10539 155044 : case FLOOR_DIV_EXPR:
10540 155044 : case CEIL_DIV_EXPR:
10541 155044 : case ROUND_DIV_EXPR:
10542 155044 : case EXACT_DIV_EXPR:
10543 : /* If this is a fixed-point operation, then we cannot use the code
10544 : below because "expand_divmod" doesn't support sat/no-sat fixed-point
10545 : divisions. */
10546 155044 : if (ALL_FIXED_POINT_MODE_P (mode))
10547 0 : goto binop;
10548 :
10549 155044 : if (modifier == EXPAND_STACK_PARM)
10550 331 : target = 0;
10551 : /* Possible optimization: compute the dividend with EXPAND_SUM
10552 : then if the divisor is constant can optimize the case
10553 : where some terms of the dividend have coeffs divisible by it. */
10554 155044 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10555 155044 : return expand_expr_divmod (code, mode, treeop0, treeop1, op0, op1,
10556 155044 : target, unsignedp);
10557 :
10558 30695 : case RDIV_EXPR:
10559 30695 : goto binop;
10560 :
10561 1736 : case MULT_HIGHPART_EXPR:
10562 1736 : expand_operands (treeop0, treeop1, subtarget, &op0, &op1, EXPAND_NORMAL);
10563 1736 : temp = expand_mult_highpart (mode, op0, op1, target, unsignedp);
10564 1736 : gcc_assert (temp);
10565 : return temp;
10566 :
10567 0 : case FIXED_CONVERT_EXPR:
10568 0 : op0 = expand_normal (treeop0);
10569 0 : if (target == 0 || modifier == EXPAND_STACK_PARM)
10570 0 : target = gen_reg_rtx (mode);
10571 :
10572 0 : if ((TREE_CODE (TREE_TYPE (treeop0)) == INTEGER_TYPE
10573 0 : && TYPE_UNSIGNED (TREE_TYPE (treeop0)))
10574 0 : || (TREE_CODE (type) == INTEGER_TYPE && TYPE_UNSIGNED (type)))
10575 0 : expand_fixed_convert (target, op0, 1, TYPE_SATURATING (type));
10576 : else
10577 0 : expand_fixed_convert (target, op0, 0, TYPE_SATURATING (type));
10578 : return target;
10579 :
10580 47978 : case FIX_TRUNC_EXPR:
10581 47978 : op0 = expand_normal (treeop0);
10582 47978 : if (target == 0 || modifier == EXPAND_STACK_PARM)
10583 3180 : target = gen_reg_rtx (mode);
10584 47978 : expand_fix (target, op0, unsignedp);
10585 47978 : return target;
10586 :
10587 132862 : case FLOAT_EXPR:
10588 132862 : op0 = expand_normal (treeop0);
10589 132862 : if (target == 0 || modifier == EXPAND_STACK_PARM)
10590 62743 : target = gen_reg_rtx (mode);
10591 : /* expand_float can't figure out what to do if FROM has VOIDmode.
10592 : So give it the correct mode. With -O, cse will optimize this. */
10593 132862 : if (GET_MODE (op0) == VOIDmode)
10594 109 : op0 = copy_to_mode_reg (TYPE_MODE (TREE_TYPE (treeop0)),
10595 : op0);
10596 265724 : expand_float (target, op0,
10597 132862 : TYPE_UNSIGNED (TREE_TYPE (treeop0)));
10598 132862 : return target;
10599 :
10600 49989 : case NEGATE_EXPR:
10601 49989 : op0 = expand_expr (treeop0, subtarget,
10602 : VOIDmode, EXPAND_NORMAL);
10603 49989 : if (modifier == EXPAND_STACK_PARM)
10604 243 : target = 0;
10605 49989 : temp = expand_unop (mode,
10606 : optab_for_tree_code (NEGATE_EXPR, type,
10607 : optab_default),
10608 : op0, target, 0);
10609 49989 : gcc_assert (temp);
10610 49989 : return REDUCE_BIT_FIELD (temp);
10611 :
10612 25005 : case ABS_EXPR:
10613 25005 : case ABSU_EXPR:
10614 25005 : op0 = expand_expr (treeop0, subtarget,
10615 : VOIDmode, EXPAND_NORMAL);
10616 25005 : if (modifier == EXPAND_STACK_PARM)
10617 63 : target = 0;
10618 :
10619 : /* ABS_EXPR is not valid for complex arguments. */
10620 25005 : gcc_assert (GET_MODE_CLASS (mode) != MODE_COMPLEX_INT
10621 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_FLOAT);
10622 :
10623 : /* Unsigned abs is simply the operand. Testing here means we don't
10624 : risk generating incorrect code below. */
10625 25005 : if (TYPE_UNSIGNED (TREE_TYPE (treeop0)))
10626 : return op0;
10627 :
10628 25005 : return expand_abs (mode, op0, target, unsignedp,
10629 50010 : safe_from_p (target, treeop0, 1));
10630 :
10631 120962 : case MAX_EXPR:
10632 120962 : case MIN_EXPR:
10633 120962 : target = original_target;
10634 120962 : if (target == 0
10635 120962 : || modifier == EXPAND_STACK_PARM
10636 66888 : || (MEM_P (target) && MEM_VOLATILE_P (target))
10637 66888 : || GET_MODE (target) != mode
10638 187850 : || (REG_P (target)
10639 60347 : && REGNO (target) < FIRST_PSEUDO_REGISTER))
10640 54074 : target = gen_reg_rtx (mode);
10641 120962 : expand_operands (treeop0, treeop1,
10642 : target, &op0, &op1, EXPAND_NORMAL);
10643 :
10644 : /* First try to do it with a special MIN or MAX instruction.
10645 : If that does not win, use a conditional jump to select the proper
10646 : value. */
10647 120962 : this_optab = optab_for_tree_code (code, type, optab_default);
10648 120962 : temp = expand_binop (mode, this_optab, op0, op1, target, unsignedp,
10649 : OPTAB_WIDEN);
10650 120962 : if (temp != 0)
10651 : return temp;
10652 :
10653 122 : if (VECTOR_TYPE_P (type))
10654 0 : gcc_unreachable ();
10655 :
10656 : /* At this point, a MEM target is no longer useful; we will get better
10657 : code without it. */
10658 :
10659 122 : if (! REG_P (target))
10660 1 : target = gen_reg_rtx (mode);
10661 :
10662 : /* If op1 was placed in target, swap op0 and op1. */
10663 122 : if (target != op0 && target == op1)
10664 0 : std::swap (op0, op1);
10665 :
10666 : /* We generate better code and avoid problems with op1 mentioning
10667 : target by forcing op1 into a pseudo if it isn't a constant. */
10668 122 : if (! CONSTANT_P (op1))
10669 42 : op1 = force_reg (mode, op1);
10670 :
10671 122 : {
10672 122 : enum rtx_code comparison_code;
10673 122 : rtx cmpop1 = op1;
10674 :
10675 122 : if (code == MAX_EXPR)
10676 62 : comparison_code = unsignedp ? GEU : GE;
10677 : else
10678 60 : comparison_code = unsignedp ? LEU : LE;
10679 :
10680 : /* Canonicalize to comparisons against 0. */
10681 122 : if (op1 == const1_rtx)
10682 : {
10683 : /* Converting (a >= 1 ? a : 1) into (a > 0 ? a : 1)
10684 : or (a != 0 ? a : 1) for unsigned.
10685 : For MIN we are safe converting (a <= 1 ? a : 1)
10686 : into (a <= 0 ? a : 1) */
10687 0 : cmpop1 = const0_rtx;
10688 0 : if (code == MAX_EXPR)
10689 0 : comparison_code = unsignedp ? NE : GT;
10690 : }
10691 122 : if (op1 == constm1_rtx && !unsignedp)
10692 : {
10693 : /* Converting (a >= -1 ? a : -1) into (a >= 0 ? a : -1)
10694 : and (a <= -1 ? a : -1) into (a < 0 ? a : -1) */
10695 0 : cmpop1 = const0_rtx;
10696 0 : if (code == MIN_EXPR)
10697 0 : comparison_code = LT;
10698 : }
10699 :
10700 : /* Use a conditional move if possible. */
10701 122 : if (can_conditionally_move_p (mode))
10702 : {
10703 75 : rtx insn;
10704 :
10705 75 : start_sequence ();
10706 :
10707 : /* Try to emit the conditional move. */
10708 75 : insn = emit_conditional_move (target,
10709 : { comparison_code,
10710 : op0, cmpop1, mode },
10711 : op0, op1, mode,
10712 : unsignedp);
10713 :
10714 : /* If we could do the conditional move, emit the sequence,
10715 : and return. */
10716 75 : if (insn)
10717 : {
10718 41 : rtx_insn *seq = end_sequence ();
10719 41 : emit_insn (seq);
10720 41 : return target;
10721 : }
10722 :
10723 : /* Otherwise discard the sequence and fall back to code with
10724 : branches. */
10725 34 : end_sequence ();
10726 : }
10727 :
10728 81 : if (target != op0)
10729 52 : emit_move_insn (target, op0);
10730 :
10731 81 : lab = gen_label_rtx ();
10732 81 : do_compare_rtx_and_jump (target, cmpop1, comparison_code,
10733 : unsignedp, mode, NULL_RTX, NULL, lab,
10734 : profile_probability::uninitialized ());
10735 : }
10736 81 : emit_move_insn (target, op1);
10737 81 : emit_label (lab);
10738 81 : return target;
10739 :
10740 58557 : case BIT_NOT_EXPR:
10741 58557 : op0 = expand_expr (treeop0, subtarget,
10742 : VOIDmode, EXPAND_NORMAL);
10743 58557 : if (modifier == EXPAND_STACK_PARM)
10744 88 : target = 0;
10745 : /* In case we have to reduce the result to bitfield precision
10746 : for unsigned bitfield expand this as XOR with a proper constant
10747 : instead. */
10748 58557 : if (reduce_bit_field && TYPE_UNSIGNED (type))
10749 : {
10750 21248 : int_mode = SCALAR_INT_TYPE_MODE (type);
10751 21248 : wide_int mask = wi::mask (TYPE_PRECISION (type),
10752 42496 : false, GET_MODE_PRECISION (int_mode));
10753 :
10754 42496 : temp = expand_binop (int_mode, xor_optab, op0,
10755 42496 : immed_wide_int_const (mask, int_mode),
10756 : target, 1, OPTAB_LIB_WIDEN);
10757 21248 : }
10758 : else
10759 37309 : temp = expand_unop (mode, one_cmpl_optab, op0, target, 1);
10760 58557 : gcc_assert (temp);
10761 : return temp;
10762 :
10763 : /* ??? Can optimize bitwise operations with one arg constant.
10764 : Can optimize (a bitwise1 n) bitwise2 (a bitwise3 b)
10765 : and (a bitwise1 b) bitwise2 b (etc)
10766 : but that is probably not worth while. */
10767 :
10768 604847 : case BIT_AND_EXPR:
10769 604847 : case BIT_IOR_EXPR:
10770 604847 : case BIT_XOR_EXPR:
10771 604847 : goto binop;
10772 :
10773 7938 : case LROTATE_EXPR:
10774 7938 : case RROTATE_EXPR:
10775 7938 : gcc_assert (VECTOR_MODE_P (TYPE_MODE (type))
10776 : || type_has_mode_precision_p (type));
10777 : /* fall through */
10778 :
10779 284254 : case LSHIFT_EXPR:
10780 284254 : case RSHIFT_EXPR:
10781 284254 : {
10782 : /* If this is a fixed-point operation, then we cannot use the code
10783 : below because "expand_shift" doesn't support sat/no-sat fixed-point
10784 : shifts. */
10785 284254 : if (ALL_FIXED_POINT_MODE_P (mode))
10786 0 : goto binop;
10787 :
10788 284254 : if (! safe_from_p (subtarget, treeop1, 1))
10789 4880 : subtarget = 0;
10790 284254 : if (modifier == EXPAND_STACK_PARM)
10791 2576 : target = 0;
10792 284254 : op0 = expand_expr (treeop0, subtarget,
10793 : VOIDmode, EXPAND_NORMAL);
10794 :
10795 : /* Left shift optimization when shifting across word_size boundary.
10796 :
10797 : If mode == GET_MODE_WIDER_MODE (word_mode), then normally
10798 : there isn't native instruction to support this wide mode
10799 : left shift. Given below scenario:
10800 :
10801 : Type A = (Type) B << C
10802 :
10803 : |< T >|
10804 : | dest_high | dest_low |
10805 :
10806 : | word_size |
10807 :
10808 : If the shift amount C caused we shift B to across the word
10809 : size boundary, i.e part of B shifted into high half of
10810 : destination register, and part of B remains in the low
10811 : half, then GCC will use the following left shift expand
10812 : logic:
10813 :
10814 : 1. Initialize dest_low to B.
10815 : 2. Initialize every bit of dest_high to the sign bit of B.
10816 : 3. Logic left shift dest_low by C bit to finalize dest_low.
10817 : The value of dest_low before this shift is kept in a temp D.
10818 : 4. Logic left shift dest_high by C.
10819 : 5. Logic right shift D by (word_size - C).
10820 : 6. Or the result of 4 and 5 to finalize dest_high.
10821 :
10822 : While, by checking gimple statements, if operand B is
10823 : coming from signed extension, then we can simplify above
10824 : expand logic into:
10825 :
10826 : 1. dest_high = src_low >> (word_size - C).
10827 : 2. dest_low = src_low << C.
10828 :
10829 : We can use one arithmetic right shift to finish all the
10830 : purpose of steps 2, 4, 5, 6, thus we reduce the steps
10831 : needed from 6 into 2.
10832 :
10833 : The case is similar for zero extension, except that we
10834 : initialize dest_high to zero rather than copies of the sign
10835 : bit from B. Furthermore, we need to use a logical right shift
10836 : in this case.
10837 :
10838 : The choice of sign-extension versus zero-extension is
10839 : determined entirely by whether or not B is signed and is
10840 : independent of the current setting of unsignedp. */
10841 :
10842 284254 : temp = NULL_RTX;
10843 284254 : if (code == LSHIFT_EXPR
10844 284254 : && target
10845 32858 : && REG_P (target)
10846 313990 : && GET_MODE_2XWIDER_MODE (word_mode).exists (&int_mode)
10847 29803 : && mode == int_mode
10848 1502 : && TREE_CONSTANT (treeop1)
10849 285089 : && TREE_CODE (treeop0) == SSA_NAME)
10850 : {
10851 835 : gimple *def = SSA_NAME_DEF_STMT (treeop0);
10852 835 : if (is_gimple_assign (def)
10853 835 : && gimple_assign_rhs_code (def) == NOP_EXPR)
10854 : {
10855 325 : scalar_int_mode rmode = SCALAR_INT_TYPE_MODE
10856 : (TREE_TYPE (gimple_assign_rhs1 (def)));
10857 :
10858 650 : if (GET_MODE_SIZE (rmode) < GET_MODE_SIZE (int_mode)
10859 604 : && TREE_INT_CST_LOW (treeop1) < GET_MODE_BITSIZE (word_mode)
10860 467 : && ((TREE_INT_CST_LOW (treeop1) + GET_MODE_BITSIZE (rmode))
10861 71 : >= GET_MODE_BITSIZE (word_mode)))
10862 : {
10863 67 : rtx_insn *seq, *seq_old;
10864 67 : poly_uint64 high_off = subreg_highpart_offset (word_mode,
10865 : int_mode);
10866 67 : bool extend_unsigned
10867 67 : = TYPE_UNSIGNED (TREE_TYPE (gimple_assign_rhs1 (def)));
10868 67 : rtx low = lowpart_subreg (word_mode, op0, int_mode);
10869 67 : rtx dest_low = lowpart_subreg (word_mode, target, int_mode);
10870 67 : rtx dest_high = simplify_gen_subreg (word_mode, target,
10871 : int_mode, high_off);
10872 67 : HOST_WIDE_INT ramount = (BITS_PER_WORD
10873 67 : - TREE_INT_CST_LOW (treeop1));
10874 67 : tree rshift = build_int_cst (TREE_TYPE (treeop1), ramount);
10875 :
10876 67 : start_sequence ();
10877 : /* dest_high = src_low >> (word_size - C). */
10878 67 : temp = expand_variable_shift (RSHIFT_EXPR, word_mode, low,
10879 : rshift, dest_high,
10880 : extend_unsigned);
10881 67 : if (temp != dest_high)
10882 0 : emit_move_insn (dest_high, temp);
10883 :
10884 : /* dest_low = src_low << C. */
10885 67 : temp = expand_variable_shift (LSHIFT_EXPR, word_mode, low,
10886 : treeop1, dest_low, unsignedp);
10887 67 : if (temp != dest_low)
10888 2 : emit_move_insn (dest_low, temp);
10889 :
10890 67 : seq = end_sequence ();
10891 67 : temp = target ;
10892 :
10893 67 : if (have_insn_for (ASHIFT, int_mode))
10894 : {
10895 67 : bool speed_p = optimize_insn_for_speed_p ();
10896 67 : start_sequence ();
10897 67 : rtx ret_old = expand_variable_shift (code, int_mode,
10898 : op0, treeop1,
10899 : target,
10900 : unsignedp);
10901 :
10902 67 : seq_old = end_sequence ();
10903 134 : if (seq_cost (seq, speed_p)
10904 67 : >= seq_cost (seq_old, speed_p))
10905 : {
10906 67 : seq = seq_old;
10907 67 : temp = ret_old;
10908 : }
10909 : }
10910 67 : emit_insn (seq);
10911 : }
10912 : }
10913 : }
10914 :
10915 67 : if (temp == NULL_RTX)
10916 284187 : temp = expand_variable_shift (code, mode, op0, treeop1, target,
10917 : unsignedp);
10918 284254 : if (code == LSHIFT_EXPR)
10919 77304 : temp = REDUCE_BIT_FIELD (temp);
10920 : return temp;
10921 : }
10922 :
10923 : /* Could determine the answer when only additive constants differ. Also,
10924 : the addition of one can be handled by changing the condition. */
10925 592382 : case LT_EXPR:
10926 592382 : case LE_EXPR:
10927 592382 : case GT_EXPR:
10928 592382 : case GE_EXPR:
10929 592382 : case EQ_EXPR:
10930 592382 : case NE_EXPR:
10931 592382 : case UNORDERED_EXPR:
10932 592382 : case ORDERED_EXPR:
10933 592382 : case UNLT_EXPR:
10934 592382 : case UNLE_EXPR:
10935 592382 : case UNGT_EXPR:
10936 592382 : case UNGE_EXPR:
10937 592382 : case UNEQ_EXPR:
10938 592382 : case LTGT_EXPR:
10939 592382 : {
10940 960650 : temp = do_store_flag (ops,
10941 : modifier != EXPAND_STACK_PARM ? target : NULL_RTX,
10942 : tmode != VOIDmode ? tmode : mode);
10943 592382 : if (temp)
10944 : return temp;
10945 :
10946 : /* Use a compare and a jump for BLKmode comparisons, or for function
10947 : type comparisons is have_canonicalize_funcptr_for_compare. */
10948 :
10949 0 : if ((target == 0
10950 0 : || modifier == EXPAND_STACK_PARM
10951 0 : || ! safe_from_p (target, treeop0, 1)
10952 0 : || ! safe_from_p (target, treeop1, 1)
10953 : /* Make sure we don't have a hard reg (such as function's return
10954 : value) live across basic blocks, if not optimizing. */
10955 0 : || (!optimize && REG_P (target)
10956 0 : && REGNO (target) < FIRST_PSEUDO_REGISTER)))
10957 0 : target = gen_reg_rtx (tmode != VOIDmode ? tmode : mode);
10958 :
10959 0 : emit_move_insn (target, const0_rtx);
10960 :
10961 0 : rtx_code_label *lab1 = gen_label_rtx ();
10962 0 : jumpifnot_1 (code, treeop0, treeop1, lab1,
10963 : profile_probability::uninitialized ());
10964 :
10965 0 : if (TYPE_PRECISION (type) == 1 && !TYPE_UNSIGNED (type))
10966 0 : emit_move_insn (target, constm1_rtx);
10967 : else
10968 0 : emit_move_insn (target, const1_rtx);
10969 :
10970 0 : emit_label (lab1);
10971 0 : return target;
10972 : }
10973 54901 : case COMPLEX_EXPR:
10974 : /* Get the rtx code of the operands. */
10975 54901 : op0 = expand_normal (treeop0);
10976 54901 : op1 = expand_normal (treeop1);
10977 :
10978 54901 : if (!target)
10979 3103 : target = gen_reg_rtx (TYPE_MODE (type));
10980 : else
10981 : /* If target overlaps with op1, then either we need to force
10982 : op1 into a pseudo (if target also overlaps with op0),
10983 : or write the complex parts in reverse order. */
10984 51798 : switch (GET_CODE (target))
10985 : {
10986 49044 : case CONCAT:
10987 49044 : if (reg_overlap_mentioned_p (XEXP (target, 0), op1))
10988 : {
10989 0 : if (reg_overlap_mentioned_p (XEXP (target, 1), op0))
10990 : {
10991 0 : complex_expr_force_op1:
10992 1724 : temp = gen_reg_rtx (GET_MODE_INNER (GET_MODE (target)));
10993 862 : emit_move_insn (temp, op1);
10994 862 : op1 = temp;
10995 862 : break;
10996 : }
10997 0 : complex_expr_swap_order:
10998 : /* Move the imaginary (op1) and real (op0) parts to their
10999 : location. */
11000 1 : write_complex_part (target, op1, true, true);
11001 1 : write_complex_part (target, op0, false, false);
11002 :
11003 1 : return target;
11004 : }
11005 : break;
11006 2754 : case MEM:
11007 5508 : temp = adjust_address_nv (target,
11008 : GET_MODE_INNER (GET_MODE (target)), 0);
11009 2754 : if (reg_overlap_mentioned_p (temp, op1))
11010 : {
11011 863 : scalar_mode imode = GET_MODE_INNER (GET_MODE (target));
11012 1726 : temp = adjust_address_nv (target, imode,
11013 : GET_MODE_SIZE (imode));
11014 863 : if (reg_overlap_mentioned_p (temp, op0))
11015 862 : goto complex_expr_force_op1;
11016 1 : goto complex_expr_swap_order;
11017 : }
11018 : break;
11019 0 : default:
11020 0 : if (reg_overlap_mentioned_p (target, op1))
11021 : {
11022 0 : if (reg_overlap_mentioned_p (target, op0))
11023 0 : goto complex_expr_force_op1;
11024 0 : goto complex_expr_swap_order;
11025 : }
11026 : break;
11027 : }
11028 :
11029 : /* Move the real (op0) and imaginary (op1) parts to their location. */
11030 54900 : write_complex_part (target, op0, false, true);
11031 54900 : write_complex_part (target, op1, true, false);
11032 :
11033 54900 : return target;
11034 :
11035 0 : case WIDEN_SUM_EXPR:
11036 0 : {
11037 0 : tree oprnd0 = treeop0;
11038 0 : tree oprnd1 = treeop1;
11039 :
11040 0 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11041 0 : target = expand_widen_pattern_expr (ops, op0, NULL_RTX, op1,
11042 : target, unsignedp);
11043 0 : return target;
11044 : }
11045 :
11046 17702 : case VEC_UNPACK_HI_EXPR:
11047 17702 : case VEC_UNPACK_LO_EXPR:
11048 17702 : case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
11049 17702 : case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
11050 17702 : {
11051 17702 : op0 = expand_normal (treeop0);
11052 17702 : temp = expand_widen_pattern_expr (ops, op0, NULL_RTX, NULL_RTX,
11053 : target, unsignedp);
11054 17702 : gcc_assert (temp);
11055 : return temp;
11056 : }
11057 :
11058 1728 : case VEC_UNPACK_FLOAT_HI_EXPR:
11059 1728 : case VEC_UNPACK_FLOAT_LO_EXPR:
11060 1728 : {
11061 1728 : op0 = expand_normal (treeop0);
11062 : /* The signedness is determined from input operand. */
11063 1728 : temp = expand_widen_pattern_expr
11064 3456 : (ops, op0, NULL_RTX, NULL_RTX,
11065 1728 : target, TYPE_UNSIGNED (TREE_TYPE (treeop0)));
11066 :
11067 1728 : gcc_assert (temp);
11068 : return temp;
11069 : }
11070 :
11071 993 : case VEC_WIDEN_MULT_HI_EXPR:
11072 993 : case VEC_WIDEN_MULT_LO_EXPR:
11073 993 : case VEC_WIDEN_MULT_EVEN_EXPR:
11074 993 : case VEC_WIDEN_MULT_ODD_EXPR:
11075 993 : case VEC_WIDEN_LSHIFT_HI_EXPR:
11076 993 : case VEC_WIDEN_LSHIFT_LO_EXPR:
11077 993 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11078 993 : target = expand_widen_pattern_expr (ops, op0, op1, NULL_RTX,
11079 : target, unsignedp);
11080 993 : gcc_assert (target);
11081 : return target;
11082 :
11083 351 : case VEC_PACK_SAT_EXPR:
11084 351 : case VEC_PACK_FIX_TRUNC_EXPR:
11085 351 : mode = TYPE_MODE (TREE_TYPE (treeop0));
11086 351 : subtarget = NULL_RTX;
11087 351 : goto binop;
11088 :
11089 10961 : case VEC_PACK_TRUNC_EXPR:
11090 10961 : if (VECTOR_BOOLEAN_TYPE_P (type)
11091 2467 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (treeop0))
11092 2467 : && mode == TYPE_MODE (TREE_TYPE (treeop0))
11093 11665 : && SCALAR_INT_MODE_P (mode))
11094 : {
11095 704 : class expand_operand eops[4];
11096 704 : machine_mode imode = TYPE_MODE (TREE_TYPE (treeop0));
11097 704 : expand_operands (treeop0, treeop1,
11098 : subtarget, &op0, &op1, EXPAND_NORMAL);
11099 704 : this_optab = vec_pack_sbool_trunc_optab;
11100 704 : enum insn_code icode = optab_handler (this_optab, imode);
11101 704 : create_output_operand (&eops[0], target, mode);
11102 704 : create_convert_operand_from (&eops[1], op0, imode, false);
11103 704 : create_convert_operand_from (&eops[2], op1, imode, false);
11104 704 : temp = GEN_INT (TYPE_VECTOR_SUBPARTS (type).to_constant ());
11105 704 : create_input_operand (&eops[3], temp, imode);
11106 704 : expand_insn (icode, 4, eops);
11107 704 : return eops[0].value;
11108 : }
11109 10257 : mode = TYPE_MODE (TREE_TYPE (treeop0));
11110 10257 : subtarget = NULL_RTX;
11111 10257 : goto binop;
11112 :
11113 27 : case VEC_PACK_FLOAT_EXPR:
11114 27 : mode = TYPE_MODE (TREE_TYPE (treeop0));
11115 27 : expand_operands (treeop0, treeop1,
11116 : subtarget, &op0, &op1, EXPAND_NORMAL);
11117 27 : this_optab = optab_for_tree_code (code, TREE_TYPE (treeop0),
11118 : optab_default);
11119 81 : target = expand_binop (mode, this_optab, op0, op1, target,
11120 27 : TYPE_UNSIGNED (TREE_TYPE (treeop0)),
11121 : OPTAB_LIB_WIDEN);
11122 27 : gcc_assert (target);
11123 : return target;
11124 :
11125 74465 : case VEC_PERM_EXPR:
11126 74465 : {
11127 74465 : expand_operands (treeop0, treeop1, target, &op0, &op1, EXPAND_NORMAL);
11128 74465 : vec_perm_builder sel;
11129 74465 : if (TREE_CODE (treeop2) == VECTOR_CST
11130 74465 : && tree_to_vec_perm_builder (&sel, treeop2))
11131 : {
11132 74447 : machine_mode sel_mode = TYPE_MODE (TREE_TYPE (treeop2));
11133 74447 : temp = expand_vec_perm_const (mode, op0, op1, sel,
11134 : sel_mode, target);
11135 : }
11136 : else
11137 : {
11138 18 : op2 = expand_normal (treeop2);
11139 18 : temp = expand_vec_perm_var (mode, op0, op1, op2, target);
11140 : }
11141 74465 : gcc_assert (temp);
11142 74465 : return temp;
11143 74465 : }
11144 :
11145 419 : case DOT_PROD_EXPR:
11146 419 : {
11147 419 : tree oprnd0 = treeop0;
11148 419 : tree oprnd1 = treeop1;
11149 419 : tree oprnd2 = treeop2;
11150 :
11151 419 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11152 419 : op2 = expand_normal (oprnd2);
11153 419 : target = expand_widen_pattern_expr (ops, op0, op1, op2,
11154 : target, unsignedp);
11155 419 : return target;
11156 : }
11157 :
11158 113 : case SAD_EXPR:
11159 113 : {
11160 113 : tree oprnd0 = treeop0;
11161 113 : tree oprnd1 = treeop1;
11162 113 : tree oprnd2 = treeop2;
11163 :
11164 113 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11165 113 : op2 = expand_normal (oprnd2);
11166 113 : target = expand_widen_pattern_expr (ops, op0, op1, op2,
11167 : target, unsignedp);
11168 113 : return target;
11169 : }
11170 :
11171 0 : case REALIGN_LOAD_EXPR:
11172 0 : {
11173 0 : tree oprnd0 = treeop0;
11174 0 : tree oprnd1 = treeop1;
11175 0 : tree oprnd2 = treeop2;
11176 :
11177 0 : this_optab = optab_for_tree_code (code, type, optab_default);
11178 0 : expand_operands (oprnd0, oprnd1, NULL_RTX, &op0, &op1, EXPAND_NORMAL);
11179 0 : op2 = expand_normal (oprnd2);
11180 0 : temp = expand_ternary_op (mode, this_optab, op0, op1, op2,
11181 : target, unsignedp);
11182 0 : gcc_assert (temp);
11183 : return temp;
11184 : }
11185 :
11186 17962 : case COND_EXPR:
11187 17962 : {
11188 : /* A COND_EXPR with its type being VOID_TYPE represents a
11189 : conditional jump and is handled in
11190 : expand_gimple_cond_expr. */
11191 17962 : gcc_assert (!VOID_TYPE_P (type));
11192 :
11193 : /* Note that COND_EXPRs whose type is a structure or union
11194 : are required to be constructed to contain assignments of
11195 : a temporary variable, so that we can evaluate them here
11196 : for side effect only. If type is void, we must do likewise. */
11197 :
11198 17962 : gcc_assert (!TREE_ADDRESSABLE (type)
11199 : && !ignore
11200 : && TREE_TYPE (treeop1) != void_type_node
11201 : && TREE_TYPE (treeop2) != void_type_node);
11202 :
11203 17962 : temp = expand_cond_expr_using_cmove (treeop0, treeop1, treeop2);
11204 17962 : if (temp)
11205 : return temp;
11206 :
11207 : /* If we are not to produce a result, we have no target. Otherwise,
11208 : if a target was specified use it; it will not be used as an
11209 : intermediate target unless it is safe. If no target, use a
11210 : temporary. */
11211 :
11212 3738 : if (modifier != EXPAND_STACK_PARM
11213 3738 : && original_target
11214 2175 : && safe_from_p (original_target, treeop0, 1)
11215 0 : && GET_MODE (original_target) == mode
11216 3738 : && !MEM_P (original_target))
11217 : temp = original_target;
11218 : else
11219 3738 : temp = assign_temp (type, 0, 1);
11220 :
11221 3738 : do_pending_stack_adjust ();
11222 3738 : NO_DEFER_POP;
11223 3738 : rtx_code_label *lab0 = gen_label_rtx ();
11224 3738 : rtx_code_label *lab1 = gen_label_rtx ();
11225 3738 : jumpifnot (treeop0, lab0,
11226 : profile_probability::uninitialized ());
11227 3738 : store_expr (treeop1, temp,
11228 : modifier == EXPAND_STACK_PARM,
11229 : false, false);
11230 :
11231 3738 : emit_jump_insn (targetm.gen_jump (lab1));
11232 3738 : emit_barrier ();
11233 3738 : emit_label (lab0);
11234 3738 : store_expr (treeop2, temp,
11235 : modifier == EXPAND_STACK_PARM,
11236 : false, false);
11237 :
11238 3738 : emit_label (lab1);
11239 3738 : OK_DEFER_POP;
11240 3738 : return temp;
11241 : }
11242 :
11243 0 : case VEC_DUPLICATE_EXPR:
11244 0 : op0 = expand_expr (treeop0, NULL_RTX, VOIDmode, modifier);
11245 0 : target = expand_vector_broadcast (mode, op0);
11246 0 : gcc_assert (target);
11247 : return target;
11248 :
11249 0 : case VEC_SERIES_EXPR:
11250 0 : expand_operands (treeop0, treeop1, NULL_RTX, &op0, &op1, modifier);
11251 0 : return expand_vec_series_expr (mode, op0, op1, target);
11252 :
11253 955 : case BIT_INSERT_EXPR:
11254 955 : {
11255 955 : unsigned bitpos = tree_to_uhwi (treeop2);
11256 955 : unsigned bitsize;
11257 955 : if (INTEGRAL_TYPE_P (TREE_TYPE (treeop1)))
11258 636 : bitsize = TYPE_PRECISION (TREE_TYPE (treeop1));
11259 : else
11260 319 : bitsize = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (treeop1)));
11261 955 : op0 = expand_normal (treeop0);
11262 955 : op1 = expand_normal (treeop1);
11263 955 : rtx dst = gen_reg_rtx (mode);
11264 955 : emit_move_insn (dst, op0);
11265 955 : store_bit_field (dst, bitsize, bitpos, 0, 0,
11266 955 : TYPE_MODE (TREE_TYPE (treeop1)), op1, false, false);
11267 955 : return dst;
11268 : }
11269 :
11270 0 : default:
11271 0 : gcc_unreachable ();
11272 : }
11273 :
11274 : /* Here to do an ordinary binary operator. */
11275 1158332 : binop:
11276 1158332 : expand_operands (treeop0, treeop1,
11277 : subtarget, &op0, &op1, EXPAND_NORMAL);
11278 5579594 : binop2:
11279 5579594 : this_optab = optab_for_tree_code (code, type, optab_default);
11280 5579594 : binop3:
11281 5579594 : if (modifier == EXPAND_STACK_PARM)
11282 27138 : target = 0;
11283 5579594 : temp = expand_binop (mode, this_optab, op0, op1, target,
11284 : unsignedp, OPTAB_LIB_WIDEN);
11285 5579594 : gcc_assert (temp);
11286 : /* Bitwise operations do not need bitfield reduction as we expect their
11287 : operands being properly truncated. */
11288 5579594 : if (code == BIT_XOR_EXPR
11289 : || code == BIT_AND_EXPR
11290 5579594 : || code == BIT_IOR_EXPR)
11291 : return temp;
11292 4974747 : return REDUCE_BIT_FIELD (temp);
11293 : }
11294 : #undef REDUCE_BIT_FIELD
11295 :
11296 :
11297 : /* Return TRUE if expression STMT is suitable for replacement.
11298 : Never consider memory loads as replaceable, because those don't ever lead
11299 : into constant expressions. */
11300 :
11301 : static bool
11302 8 : stmt_is_replaceable_p (gimple *stmt)
11303 : {
11304 8 : if (ssa_is_replaceable_p (stmt))
11305 : {
11306 : /* Don't move around loads. */
11307 7 : if (!gimple_assign_single_p (stmt)
11308 7 : || is_gimple_val (gimple_assign_rhs1 (stmt)))
11309 6 : return true;
11310 : }
11311 : return false;
11312 : }
11313 :
11314 : /* A subroutine of expand_expr_real_1. Expand gimple assignment G,
11315 : which is known to set an SSA_NAME result. The other arguments are
11316 : as for expand_expr_real_1. */
11317 :
11318 : rtx
11319 14680822 : expand_expr_real_gassign (gassign *g, rtx target, machine_mode tmode,
11320 : enum expand_modifier modifier, rtx *alt_rtl,
11321 : bool inner_reference_p)
11322 : {
11323 14680822 : separate_ops ops;
11324 14680822 : rtx r;
11325 14680822 : location_t saved_loc = curr_insn_location ();
11326 14680822 : auto loc = gimple_location (g);
11327 14680822 : if (loc != UNKNOWN_LOCATION)
11328 11736780 : set_curr_insn_location (loc);
11329 14680822 : tree lhs = gimple_assign_lhs (g);
11330 14680822 : ops.code = gimple_assign_rhs_code (g);
11331 14680822 : ops.type = TREE_TYPE (lhs);
11332 14680822 : switch (get_gimple_rhs_class (ops.code))
11333 : {
11334 93765 : case GIMPLE_TERNARY_RHS:
11335 187530 : ops.op2 = gimple_assign_rhs3 (g);
11336 : /* Fallthru */
11337 7890015 : case GIMPLE_BINARY_RHS:
11338 7890015 : ops.op1 = gimple_assign_rhs2 (g);
11339 :
11340 : /* Try to expand conditonal compare. */
11341 7890015 : if (targetm.have_ccmp ())
11342 : {
11343 164955 : gcc_checking_assert (targetm.gen_ccmp_next != NULL);
11344 164955 : r = expand_ccmp_expr (g, TYPE_MODE (ops.type));
11345 164955 : if (r)
11346 : break;
11347 : }
11348 : /* Fallthru */
11349 11345255 : case GIMPLE_UNARY_RHS:
11350 11345255 : ops.op0 = gimple_assign_rhs1 (g);
11351 11345255 : ops.location = loc;
11352 11345255 : r = expand_expr_real_2 (&ops, target, tmode, modifier);
11353 11345255 : break;
11354 3335549 : case GIMPLE_SINGLE_RHS:
11355 3335549 : {
11356 3335549 : r = expand_expr_real (gimple_assign_rhs1 (g), target,
11357 : tmode, modifier, alt_rtl,
11358 : inner_reference_p);
11359 3335549 : break;
11360 : }
11361 0 : default:
11362 0 : gcc_unreachable ();
11363 : }
11364 14680822 : set_curr_insn_location (saved_loc);
11365 14680822 : if (REG_P (r) && !REG_EXPR (r))
11366 3819415 : set_reg_attrs_for_decl_rtl (lhs, r);
11367 14680822 : return r;
11368 : }
11369 :
11370 : rtx
11371 154783462 : expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
11372 : enum expand_modifier modifier, rtx *alt_rtl,
11373 : bool inner_reference_p)
11374 : {
11375 154783462 : rtx op0, op1, temp, decl_rtl;
11376 154783462 : tree type;
11377 154783462 : int unsignedp;
11378 154783462 : machine_mode mode, dmode;
11379 154783462 : enum tree_code code = TREE_CODE (exp);
11380 154783462 : rtx subtarget, original_target;
11381 154783462 : int ignore;
11382 154783462 : bool reduce_bit_field;
11383 154783462 : location_t loc = EXPR_LOCATION (exp);
11384 154783462 : struct separate_ops ops;
11385 154783462 : tree treeop0, treeop1, treeop2;
11386 154783462 : tree ssa_name = NULL_TREE;
11387 154783462 : gimple *g;
11388 :
11389 : /* Some ABIs define padding bits in _BitInt uninitialized. Normally, RTL
11390 : expansion sign/zero extends integral types with less than mode precision
11391 : when reading from bit-fields and after arithmetic operations (see
11392 : REDUCE_BIT_FIELD in expand_expr_real_2) and on subsequent loads relies
11393 : on those extensions to have been already performed, but because of the
11394 : above for _BitInt they need to be sign/zero extended when reading from
11395 : locations that could be exposed to ABI boundaries (when loading from
11396 : objects in memory, or function arguments, return value). Because we
11397 : internally extend after arithmetic operations, we can avoid doing that
11398 : when reading from SSA_NAMEs of vars. */
11399 : #define EXTEND_BITINT(expr) \
11400 : ((BITINT_TYPE_P (type) \
11401 : && !bitint_extended \
11402 : && reduce_bit_field \
11403 : && mode != BLKmode \
11404 : && modifier != EXPAND_MEMORY \
11405 : && modifier != EXPAND_WRITE \
11406 : && modifier != EXPAND_INITIALIZER \
11407 : && modifier != EXPAND_CONST_ADDRESS) \
11408 : ? reduce_to_bit_field_precision ((expr), NULL_RTX, type) : (expr))
11409 :
11410 154783462 : type = TREE_TYPE (exp);
11411 154783462 : mode = TYPE_MODE (type);
11412 154783462 : unsignedp = TYPE_UNSIGNED (type);
11413 154783462 : if (BITINT_TYPE_P (type) && bitint_extended == -1)
11414 : {
11415 8769 : struct bitint_info info;
11416 8769 : bool ok = targetm.c.bitint_type_info (TYPE_PRECISION (type), &info);
11417 8769 : gcc_assert (ok);
11418 8769 : bitint_extended = info.extended;
11419 : }
11420 :
11421 154783462 : treeop0 = treeop1 = treeop2 = NULL_TREE;
11422 154783462 : if (!VL_EXP_CLASS_P (exp))
11423 148137035 : switch (TREE_CODE_LENGTH (code))
11424 : {
11425 5429338 : default:
11426 5429338 : case 3: treeop2 = TREE_OPERAND (exp, 2); /* FALLTHRU */
11427 13265229 : case 2: treeop1 = TREE_OPERAND (exp, 1); /* FALLTHRU */
11428 27478365 : case 1: treeop0 = TREE_OPERAND (exp, 0); /* FALLTHRU */
11429 : case 0: break;
11430 : }
11431 154783462 : ops.code = code;
11432 154783462 : ops.type = type;
11433 154783462 : ops.op0 = treeop0;
11434 154783462 : ops.op1 = treeop1;
11435 154783462 : ops.op2 = treeop2;
11436 154783462 : ops.location = loc;
11437 :
11438 309566924 : ignore = (target == const0_rtx
11439 154783462 : || ((CONVERT_EXPR_CODE_P (code)
11440 149808766 : || code == COND_EXPR || code == VIEW_CONVERT_EXPR)
11441 941647 : && TREE_CODE (type) == VOID_TYPE));
11442 :
11443 : /* An operation in what may be a bit-field type needs the
11444 : result to be reduced to the precision of the bit-field type,
11445 : which is narrower than that of the type's mode. */
11446 309566688 : reduce_bit_field = (!ignore
11447 150410853 : && INTEGRAL_TYPE_P (type)
11448 79020680 : && !type_has_mode_precision_p (type));
11449 :
11450 : /* If we are going to ignore this result, we need only do something
11451 : if there is a side-effect somewhere in the expression. If there
11452 : is, short-circuit the most common cases here. Note that we must
11453 : not call expand_expr with anything but const0_rtx in case this
11454 : is an initial expansion of a size that contains a PLACEHOLDER_EXPR. */
11455 :
11456 4372609 : if (ignore)
11457 : {
11458 4372609 : if (! TREE_SIDE_EFFECTS (exp))
11459 : return const0_rtx;
11460 :
11461 : /* Ensure we reference a volatile object even if value is ignored, but
11462 : don't do this if all we are doing is taking its address. */
11463 4372373 : if (TREE_THIS_VOLATILE (exp)
11464 0 : && TREE_CODE (exp) != FUNCTION_DECL
11465 0 : && mode != VOIDmode && mode != BLKmode
11466 0 : && modifier != EXPAND_CONST_ADDRESS)
11467 : {
11468 0 : temp = expand_expr (exp, NULL_RTX, VOIDmode, modifier);
11469 0 : if (MEM_P (temp))
11470 0 : copy_to_reg (temp);
11471 0 : return const0_rtx;
11472 : }
11473 :
11474 4372373 : if (TREE_CODE_CLASS (code) == tcc_unary
11475 : || code == BIT_FIELD_REF
11476 4372373 : || code == COMPONENT_REF
11477 4372373 : || code == INDIRECT_REF)
11478 0 : return expand_expr (treeop0, const0_rtx, VOIDmode,
11479 0 : modifier);
11480 :
11481 4372373 : else if (TREE_CODE_CLASS (code) == tcc_binary
11482 4372373 : || TREE_CODE_CLASS (code) == tcc_comparison
11483 4372373 : || code == ARRAY_REF || code == ARRAY_RANGE_REF)
11484 : {
11485 0 : expand_expr (treeop0, const0_rtx, VOIDmode, modifier);
11486 0 : expand_expr (treeop1, const0_rtx, VOIDmode, modifier);
11487 0 : return const0_rtx;
11488 : }
11489 :
11490 : target = 0;
11491 : }
11492 :
11493 154783226 : if (reduce_bit_field && modifier == EXPAND_STACK_PARM)
11494 39429 : target = 0;
11495 :
11496 : /* Use subtarget as the target for operand 0 of a binary operation. */
11497 154783226 : subtarget = get_subtarget (target);
11498 154783226 : original_target = target;
11499 :
11500 154783226 : switch (code)
11501 : {
11502 10817 : case LABEL_DECL:
11503 10817 : {
11504 10817 : tree function = decl_function_context (exp);
11505 :
11506 10817 : temp = label_rtx (exp);
11507 14761 : temp = gen_rtx_LABEL_REF (Pmode, temp);
11508 :
11509 10817 : if (function != current_function_decl
11510 1270 : && function != 0)
11511 1270 : LABEL_REF_NONLOCAL_P (temp) = 1;
11512 :
11513 10817 : temp = gen_rtx_MEM (FUNCTION_MODE, temp);
11514 10817 : return temp;
11515 : }
11516 :
11517 55908650 : case SSA_NAME:
11518 : /* ??? ivopts calls expander, without any preparation from
11519 : out-of-ssa. So fake instructions as if this was an access to the
11520 : base variable. This unnecessarily allocates a pseudo, see how we can
11521 : reuse it, if partition base vars have it set already. */
11522 55908650 : if (!currently_expanding_to_rtl)
11523 : {
11524 0 : tree var = SSA_NAME_VAR (exp);
11525 0 : if (var && DECL_RTL_SET_P (var))
11526 0 : return DECL_RTL (var);
11527 0 : return gen_raw_REG (TYPE_MODE (TREE_TYPE (exp)),
11528 0 : LAST_VIRTUAL_REGISTER + 1);
11529 : }
11530 :
11531 55908650 : g = get_gimple_for_ssa_name (exp);
11532 : /* For EXPAND_INITIALIZER try harder to get something simpler. */
11533 55908650 : if (g == NULL
11534 55908650 : && modifier == EXPAND_INITIALIZER
11535 26 : && !SSA_NAME_IS_DEFAULT_DEF (exp)
11536 11 : && (optimize || !SSA_NAME_VAR (exp)
11537 1 : || DECL_IGNORED_P (SSA_NAME_VAR (exp)))
11538 10 : && is_gimple_assign (SSA_NAME_DEF_STMT (exp))
11539 55908658 : && stmt_is_replaceable_p (SSA_NAME_DEF_STMT (exp)))
11540 6 : g = SSA_NAME_DEF_STMT (exp);
11541 55908650 : if (safe_is_a <gassign *> (g))
11542 8803411 : return expand_expr_real_gassign (as_a<gassign *> (g), target, tmode,
11543 8803411 : modifier, alt_rtl, inner_reference_p);
11544 47105239 : else if (safe_is_a <gcall *> (g))
11545 : {
11546 : /* ??? internal call expansion doesn't follow the usual API
11547 : of returning the destination RTX and being passed a desired
11548 : target. */
11549 72554 : if (modifier == EXPAND_WRITE)
11550 36281 : return DECL_RTL (SSA_NAME_VAR (exp));
11551 36273 : rtx dest = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
11552 36273 : tree tmplhs = make_tree (TREE_TYPE (exp), dest);
11553 36273 : tree var_or_id = SSA_NAME_VAR (exp);
11554 : if (!var_or_id)
11555 26327 : var_or_id = SSA_NAME_IDENTIFIER (exp);
11556 36273 : SET_SSA_NAME_VAR_OR_IDENTIFIER (exp, tmplhs);
11557 36273 : expand_internal_call (as_a <gcall *> (g));
11558 36273 : SET_SSA_NAME_VAR_OR_IDENTIFIER (exp, var_or_id);
11559 36273 : return dest;
11560 : }
11561 :
11562 47032685 : ssa_name = exp;
11563 47032685 : decl_rtl = get_rtx_for_ssa_name (ssa_name);
11564 47032685 : exp = SSA_NAME_VAR (ssa_name);
11565 : /* Optimize and avoid to EXTEND_BITINIT doing anything if it is an
11566 : SSA_NAME computed within the current function. In such case the
11567 : value have been already extended before. While if it is a function
11568 : parameter, result or some memory location, we need to be prepared
11569 : for some other compiler leaving the bits uninitialized. */
11570 18490485 : if (!exp || VAR_P (exp))
11571 : reduce_bit_field = false;
11572 47032685 : goto expand_decl_rtl;
11573 :
11574 19478282 : case VAR_DECL:
11575 : /* Allow accel compiler to handle variables that require special
11576 : treatment, e.g. if they have been modified in some way earlier in
11577 : compilation by the adjust_private_decl OpenACC hook. */
11578 19478282 : if (flag_openacc && targetm.goacc.expand_var_decl)
11579 : {
11580 0 : temp = targetm.goacc.expand_var_decl (exp);
11581 0 : if (temp)
11582 : return temp;
11583 : }
11584 : /* Expand const VAR_DECLs with CONSTRUCTOR initializers that
11585 : have scalar integer modes to a reg via store_constructor. */
11586 19478282 : if (TREE_READONLY (exp)
11587 3362066 : && !TREE_SIDE_EFFECTS (exp)
11588 3339658 : && (modifier == EXPAND_NORMAL || modifier == EXPAND_STACK_PARM)
11589 17655 : && immediate_const_ctor_p (DECL_INITIAL (exp))
11590 163 : && SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (exp)))
11591 127 : && crtl->emit.regno_pointer_align_length
11592 19478409 : && !target)
11593 : {
11594 88 : target = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
11595 88 : store_constructor (DECL_INITIAL (exp), target, 0,
11596 88 : int_expr_size (DECL_INITIAL (exp)), false);
11597 88 : return target;
11598 : }
11599 : /* ... fall through ... */
11600 :
11601 20017614 : case PARM_DECL:
11602 : /* If a static var's type was incomplete when the decl was written,
11603 : but the type is complete now, lay out the decl now. */
11604 20017614 : if (DECL_SIZE (exp) == 0
11605 44185 : && COMPLETE_OR_UNBOUND_ARRAY_TYPE_P (TREE_TYPE (exp))
11606 20061687 : && (TREE_STATIC (exp) || DECL_EXTERNAL (exp)))
11607 44073 : layout_decl (exp, 0);
11608 :
11609 : /* fall through */
11610 :
11611 21474488 : case FUNCTION_DECL:
11612 21474488 : case RESULT_DECL:
11613 21474488 : decl_rtl = DECL_RTL (exp);
11614 68507173 : expand_decl_rtl:
11615 68507173 : gcc_assert (decl_rtl);
11616 :
11617 : /* DECL_MODE might change when TYPE_MODE depends on attribute target
11618 : settings for VECTOR_TYPE_P that might switch for the function. */
11619 68507173 : if (currently_expanding_to_rtl
11620 64838580 : && code == VAR_DECL && MEM_P (decl_rtl)
11621 83029210 : && VECTOR_TYPE_P (type) && exp && DECL_MODE (exp) != mode)
11622 57 : decl_rtl = change_address (decl_rtl, TYPE_MODE (type), 0);
11623 : else
11624 68507116 : decl_rtl = copy_rtx (decl_rtl);
11625 :
11626 : /* Record writes to register variables. */
11627 68507173 : if (modifier == EXPAND_WRITE
11628 21565425 : && REG_P (decl_rtl)
11629 84263192 : && HARD_REGISTER_P (decl_rtl))
11630 2027 : add_to_hard_reg_set (&crtl->asm_clobbers,
11631 2027 : GET_MODE (decl_rtl), REGNO (decl_rtl));
11632 :
11633 : /* Ensure variable marked as used even if it doesn't go through
11634 : a parser. If it hasn't be used yet, write out an external
11635 : definition. */
11636 68507173 : if (exp)
11637 39964973 : TREE_USED (exp) = 1;
11638 :
11639 : /* Show we haven't gotten RTL for this yet. */
11640 108472146 : temp = 0;
11641 :
11642 : /* Variables inherited from containing functions should have
11643 : been lowered by this point. */
11644 39964973 : if (exp)
11645 : {
11646 39964973 : tree context = decl_function_context (exp);
11647 39964973 : gcc_assert (SCOPE_FILE_SCOPE_P (context)
11648 : || context == current_function_decl
11649 : || TREE_STATIC (exp)
11650 : || DECL_EXTERNAL (exp)
11651 : /* ??? C++ creates functions that are not
11652 : TREE_STATIC. */
11653 : || TREE_CODE (exp) == FUNCTION_DECL);
11654 : }
11655 :
11656 : /* This is the case of an array whose size is to be determined
11657 : from its initializer, while the initializer is still being parsed.
11658 : ??? We aren't parsing while expanding anymore. */
11659 :
11660 68507173 : if (MEM_P (decl_rtl) && REG_P (XEXP (decl_rtl, 0)))
11661 392222 : temp = validize_mem (decl_rtl);
11662 :
11663 : /* If DECL_RTL is memory, we are in the normal case and the
11664 : address is not valid, get the address into a register. */
11665 :
11666 68114951 : else if (MEM_P (decl_rtl) && modifier != EXPAND_INITIALIZER)
11667 : {
11668 18512028 : if (alt_rtl)
11669 2013347 : *alt_rtl = decl_rtl;
11670 18512028 : decl_rtl = use_anchored_address (decl_rtl);
11671 18512028 : if (modifier != EXPAND_CONST_ADDRESS
11672 18512028 : && modifier != EXPAND_SUM
11673 30708256 : && !memory_address_addr_space_p (exp ? DECL_MODE (exp)
11674 17279 : : GET_MODE (decl_rtl),
11675 : XEXP (decl_rtl, 0),
11676 12196228 : MEM_ADDR_SPACE (decl_rtl)))
11677 136953 : temp = replace_equiv_address (decl_rtl,
11678 : copy_rtx (XEXP (decl_rtl, 0)));
11679 : }
11680 :
11681 : /* If we got something, return it. But first, set the alignment
11682 : if the address is a register. */
11683 18904250 : if (temp != 0)
11684 : {
11685 529175 : if (exp && MEM_P (temp) && REG_P (XEXP (temp, 0)))
11686 497300 : mark_reg_pointer (XEXP (temp, 0), DECL_ALIGN (exp));
11687 : }
11688 67977998 : else if (MEM_P (decl_rtl))
11689 : temp = decl_rtl;
11690 :
11691 46596526 : if (temp != 0)
11692 : {
11693 22407957 : if (MEM_P (temp)
11694 : && modifier != EXPAND_WRITE
11695 22407957 : && modifier != EXPAND_MEMORY
11696 : && modifier != EXPAND_INITIALIZER
11697 16520412 : && modifier != EXPAND_CONST_ADDRESS
11698 6858768 : && modifier != EXPAND_SUM
11699 6858768 : && !inner_reference_p
11700 4505331 : && mode != BLKmode
11701 26547430 : && MEM_ALIGN (temp) < GET_MODE_ALIGNMENT (mode))
11702 19837 : temp = expand_misaligned_mem_ref (temp, mode, unsignedp,
11703 19837 : MEM_ALIGN (temp), NULL_RTX, NULL);
11704 :
11705 22407957 : return EXTEND_BITINT (temp);
11706 : }
11707 :
11708 46099216 : if (exp)
11709 17574305 : dmode = DECL_MODE (exp);
11710 : else
11711 28524911 : dmode = TYPE_MODE (TREE_TYPE (ssa_name));
11712 :
11713 : /* If the mode of DECL_RTL does not match that of the decl,
11714 : there are two cases: we are dealing with a BLKmode value
11715 : that is returned in a register, or we are dealing with
11716 : a promoted value. In the latter case, return a SUBREG
11717 : of the wanted mode, but mark it so that we know that it
11718 : was already extended. */
11719 46099216 : if (REG_P (decl_rtl)
11720 45618533 : && dmode != BLKmode
11721 45618533 : && GET_MODE (decl_rtl) != dmode)
11722 : {
11723 82 : machine_mode pmode;
11724 :
11725 : /* Get the signedness to be used for this variable. Ensure we get
11726 : the same mode we got when the variable was declared. */
11727 82 : if (code != SSA_NAME)
11728 0 : pmode = promote_decl_mode (exp, &unsignedp);
11729 82 : else if ((g = SSA_NAME_DEF_STMT (ssa_name))
11730 82 : && gimple_code (g) == GIMPLE_CALL
11731 84 : && !gimple_call_internal_p (g))
11732 2 : pmode = promote_function_mode (type, mode, &unsignedp,
11733 2 : gimple_call_fntype (g),
11734 : 2);
11735 : else
11736 80 : pmode = promote_ssa_mode (ssa_name, &unsignedp);
11737 82 : gcc_assert (GET_MODE (decl_rtl) == pmode);
11738 :
11739 : /* Some ABIs require scalar floating point modes to be passed
11740 : in a wider scalar integer mode. We need to explicitly
11741 : truncate to an integer mode of the correct precision before
11742 : using a SUBREG to reinterpret as a floating point value. */
11743 82 : if (SCALAR_FLOAT_MODE_P (mode)
11744 0 : && SCALAR_INT_MODE_P (pmode)
11745 82 : && known_lt (GET_MODE_SIZE (mode), GET_MODE_SIZE (pmode)))
11746 0 : return convert_wider_int_to_float (mode, pmode, decl_rtl);
11747 :
11748 82 : temp = gen_lowpart_SUBREG (mode, decl_rtl);
11749 82 : SUBREG_PROMOTED_VAR_P (temp) = 1;
11750 82 : SUBREG_PROMOTED_SET (temp, unsignedp);
11751 82 : return EXTEND_BITINT (temp);
11752 : }
11753 :
11754 46099134 : return EXTEND_BITINT (decl_rtl);
11755 :
11756 41493727 : case INTEGER_CST:
11757 41493727 : {
11758 41493727 : if (BITINT_TYPE_P (type))
11759 : {
11760 11561 : unsigned int prec = TYPE_PRECISION (type);
11761 11561 : struct bitint_info info;
11762 11561 : bool ok = targetm.c.bitint_type_info (prec, &info);
11763 11561 : gcc_assert (ok);
11764 11561 : scalar_int_mode limb_mode
11765 11561 : = as_a <scalar_int_mode> (info.limb_mode);
11766 11561 : unsigned int limb_prec = GET_MODE_PRECISION (limb_mode);
11767 19124 : if (prec > limb_prec && prec > MAX_FIXED_MODE_SIZE)
11768 : {
11769 : /* Emit large/huge _BitInt INTEGER_CSTs into memory. */
11770 5104 : exp = tree_output_constant_def (exp);
11771 5104 : return expand_expr (exp, target, VOIDmode, modifier);
11772 : }
11773 : }
11774 :
11775 : /* Given that TYPE_PRECISION (type) is not always equal to
11776 : GET_MODE_PRECISION (TYPE_MODE (type)), we need to extend from
11777 : the former to the latter according to the signedness of the
11778 : type. */
11779 41488623 : scalar_int_mode int_mode = SCALAR_INT_TYPE_MODE (type);
11780 41488623 : temp = immed_wide_int_const
11781 41488623 : (wi::to_wide (exp, GET_MODE_PRECISION (int_mode)), int_mode);
11782 41488623 : return temp;
11783 : }
11784 :
11785 560362 : case VECTOR_CST:
11786 560362 : {
11787 560362 : tree tmp = NULL_TREE;
11788 560362 : if (VECTOR_MODE_P (mode))
11789 557579 : return const_vector_from_tree (exp);
11790 2783 : scalar_int_mode int_mode;
11791 2783 : if (is_int_mode (mode, &int_mode))
11792 : {
11793 896 : tree type_for_mode = lang_hooks.types.type_for_mode (int_mode, 1);
11794 896 : if (type_for_mode)
11795 896 : tmp = fold_unary_loc (loc, VIEW_CONVERT_EXPR,
11796 : type_for_mode, exp);
11797 : }
11798 896 : if (!tmp)
11799 : {
11800 1887 : vec<constructor_elt, va_gc> *v;
11801 : /* Constructors need to be fixed-length. FIXME. */
11802 1887 : unsigned int nunits = VECTOR_CST_NELTS (exp).to_constant ();
11803 1887 : vec_alloc (v, nunits);
11804 28722 : for (unsigned int i = 0; i < nunits; ++i)
11805 26835 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, VECTOR_CST_ELT (exp, i));
11806 1887 : tmp = build_constructor (type, v);
11807 : }
11808 2783 : return expand_expr (tmp, ignore ? const0_rtx : target,
11809 2783 : tmode, modifier);
11810 : }
11811 :
11812 141 : case CONST_DECL:
11813 141 : if (modifier == EXPAND_WRITE)
11814 : {
11815 : /* Writing into CONST_DECL is always invalid, but handle it
11816 : gracefully. */
11817 2 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (exp));
11818 2 : scalar_int_mode address_mode = targetm.addr_space.address_mode (as);
11819 2 : op0 = expand_expr_addr_expr_1 (exp, NULL_RTX, address_mode,
11820 : EXPAND_NORMAL, as);
11821 2 : op0 = memory_address_addr_space (mode, op0, as);
11822 2 : temp = gen_rtx_MEM (mode, op0);
11823 2 : set_mem_addr_space (temp, as);
11824 2 : return temp;
11825 : }
11826 139 : return expand_expr (DECL_INITIAL (exp), target, VOIDmode, modifier);
11827 :
11828 849955 : case REAL_CST:
11829 : /* If optimized, generate immediate CONST_DOUBLE
11830 : which will be turned into memory by reload if necessary.
11831 :
11832 : We used to force a register so that loop.c could see it. But
11833 : this does not allow gen_* patterns to perform optimizations with
11834 : the constants. It also produces two insns in cases like "x = 1.0;".
11835 : On most machines, floating-point constants are not permitted in
11836 : many insns, so we'd end up copying it to a register in any case.
11837 :
11838 : Now, we do the copying in expand_binop, if appropriate. */
11839 849955 : return const_double_from_real_value (TREE_REAL_CST (exp),
11840 1699910 : TYPE_MODE (TREE_TYPE (exp)));
11841 :
11842 0 : case FIXED_CST:
11843 0 : return CONST_FIXED_FROM_FIXED_VALUE (TREE_FIXED_CST (exp),
11844 : TYPE_MODE (TREE_TYPE (exp)));
11845 :
11846 18395 : case COMPLEX_CST:
11847 : /* Handle evaluating a complex constant in a CONCAT target. */
11848 18395 : if (original_target && GET_CODE (original_target) == CONCAT)
11849 : {
11850 219 : rtx rtarg, itarg;
11851 :
11852 219 : mode = TYPE_MODE (TREE_TYPE (TREE_TYPE (exp)));
11853 219 : rtarg = XEXP (original_target, 0);
11854 219 : itarg = XEXP (original_target, 1);
11855 :
11856 : /* Move the real and imaginary parts separately. */
11857 219 : op0 = expand_expr (TREE_REALPART (exp), rtarg, mode, EXPAND_NORMAL);
11858 219 : op1 = expand_expr (TREE_IMAGPART (exp), itarg, mode, EXPAND_NORMAL);
11859 :
11860 219 : if (op0 != rtarg)
11861 219 : emit_move_insn (rtarg, op0);
11862 219 : if (op1 != itarg)
11863 219 : emit_move_insn (itarg, op1);
11864 :
11865 219 : return original_target;
11866 : }
11867 :
11868 : /* fall through */
11869 :
11870 187130 : case STRING_CST:
11871 187130 : temp = expand_expr_constant (exp, 1, modifier);
11872 :
11873 : /* temp contains a constant address.
11874 : On RISC machines where a constant address isn't valid,
11875 : make some insns to get that address into a register. */
11876 187130 : if (modifier != EXPAND_CONST_ADDRESS
11877 : && modifier != EXPAND_INITIALIZER
11878 187130 : && modifier != EXPAND_SUM
11879 205702 : && ! memory_address_addr_space_p (mode, XEXP (temp, 0),
11880 18572 : MEM_ADDR_SPACE (temp)))
11881 12 : return replace_equiv_address (temp,
11882 12 : copy_rtx (XEXP (temp, 0)));
11883 : return temp;
11884 :
11885 0 : case POLY_INT_CST:
11886 0 : return immed_wide_int_const (poly_int_cst_value (exp), mode);
11887 :
11888 1502 : case SAVE_EXPR:
11889 1502 : {
11890 1502 : tree val = treeop0;
11891 1502 : rtx ret = expand_expr_real_1 (val, target, tmode, modifier, alt_rtl,
11892 : inner_reference_p);
11893 :
11894 1502 : if (!SAVE_EXPR_RESOLVED_P (exp))
11895 : {
11896 : /* We can indeed still hit this case, typically via builtin
11897 : expanders calling save_expr immediately before expanding
11898 : something. Assume this means that we only have to deal
11899 : with non-BLKmode values. */
11900 1455 : gcc_assert (GET_MODE (ret) != BLKmode);
11901 :
11902 1455 : val = build_decl (curr_insn_location (),
11903 1455 : VAR_DECL, NULL, TREE_TYPE (exp));
11904 1455 : DECL_ARTIFICIAL (val) = 1;
11905 1455 : DECL_IGNORED_P (val) = 1;
11906 1455 : treeop0 = val;
11907 1455 : TREE_OPERAND (exp, 0) = treeop0;
11908 1455 : SAVE_EXPR_RESOLVED_P (exp) = 1;
11909 :
11910 1455 : if (!CONSTANT_P (ret))
11911 1455 : ret = copy_to_reg (ret);
11912 1455 : SET_DECL_RTL (val, ret);
11913 : }
11914 :
11915 : return ret;
11916 : }
11917 :
11918 :
11919 173085 : case CONSTRUCTOR:
11920 : /* If we don't need the result, just ensure we evaluate any
11921 : subexpressions. */
11922 173085 : if (ignore)
11923 : {
11924 : unsigned HOST_WIDE_INT idx;
11925 : tree value;
11926 :
11927 0 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), idx, value)
11928 0 : expand_expr (value, const0_rtx, VOIDmode, EXPAND_NORMAL);
11929 :
11930 0 : return const0_rtx;
11931 : }
11932 :
11933 173085 : return expand_constructor (exp, target, modifier, false);
11934 :
11935 852262 : case TARGET_MEM_REF:
11936 852262 : {
11937 852262 : addr_space_t as
11938 852262 : = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
11939 852262 : unsigned int align;
11940 :
11941 852262 : op0 = addr_for_mem_ref (exp, as, true);
11942 852262 : op0 = memory_address_addr_space (mode, op0, as);
11943 852262 : temp = gen_rtx_MEM (mode, op0);
11944 852262 : set_mem_attributes (temp, exp, 0);
11945 852262 : set_mem_addr_space (temp, as);
11946 852262 : align = get_object_alignment (exp);
11947 852262 : if (modifier != EXPAND_WRITE
11948 852262 : && modifier != EXPAND_MEMORY
11949 586619 : && mode != BLKmode
11950 1432943 : && align < GET_MODE_ALIGNMENT (mode))
11951 53632 : temp = expand_misaligned_mem_ref (temp, mode, unsignedp,
11952 : align, NULL_RTX, NULL);
11953 852262 : return EXTEND_BITINT (temp);
11954 : }
11955 :
11956 6561153 : case MEM_REF:
11957 6561153 : {
11958 6561153 : const bool reverse = REF_REVERSE_STORAGE_ORDER (exp);
11959 6561153 : addr_space_t as
11960 6561153 : = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
11961 6561153 : machine_mode address_mode;
11962 6561153 : tree base = TREE_OPERAND (exp, 0);
11963 6561153 : gimple *def_stmt;
11964 6561153 : unsigned align;
11965 : /* Handle expansion of non-aliased memory with non-BLKmode. That
11966 : might end up in a register. */
11967 6561153 : if (mem_ref_refers_to_non_mem_p (exp))
11968 : {
11969 77312 : poly_int64 offset = mem_ref_offset (exp).force_shwi ();
11970 77312 : base = TREE_OPERAND (base, 0);
11971 77312 : poly_uint64 type_size;
11972 77312 : if (known_eq (offset, 0)
11973 43924 : && !reverse
11974 43924 : && poly_int_tree_p (TYPE_SIZE (type), &type_size)
11975 165160 : && known_eq (GET_MODE_BITSIZE (DECL_MODE (base)), type_size))
11976 17920 : return expand_expr (build1 (VIEW_CONVERT_EXPR, type, base),
11977 17920 : target, tmode, modifier);
11978 59392 : unsigned align;
11979 59392 : if (TYPE_MODE (type) == BLKmode || maybe_lt (offset, 0))
11980 : {
11981 218 : temp = assign_stack_temp (DECL_MODE (base),
11982 436 : GET_MODE_SIZE (DECL_MODE (base)));
11983 218 : store_expr (base, temp, 0, false, false);
11984 218 : temp = adjust_address (temp, TYPE_MODE (type), offset);
11985 218 : if (TYPE_MODE (type) == BLKmode)
11986 197 : set_mem_size (temp, int_size_in_bytes (type));
11987 : /* When the original ref was misaligned so will be the
11988 : access to the stack temporary. Not all targets handle
11989 : this correctly, some will ICE in sanity checking.
11990 : Handle this by doing bitfield extraction when necessary. */
11991 42 : else if ((align = get_object_alignment (exp))
11992 21 : < GET_MODE_ALIGNMENT (TYPE_MODE (type)))
11993 3 : temp
11994 3 : = expand_misaligned_mem_ref (temp, TYPE_MODE (type),
11995 : unsignedp, align,
11996 : modifier == EXPAND_STACK_PARM
11997 : ? NULL_RTX : target, NULL);
11998 218 : return temp;
11999 : }
12000 : /* When the access is fully outside of the underlying object
12001 : expand the offset as zero. This avoids out-of-bound
12002 : BIT_FIELD_REFs and generates smaller code for these cases
12003 : with UB. */
12004 59174 : type_size = tree_to_poly_uint64 (TYPE_SIZE_UNIT (type));
12005 118348 : if (!ranges_maybe_overlap_p (offset, type_size, 0,
12006 118348 : GET_MODE_SIZE (DECL_MODE (base))))
12007 37 : offset = 0;
12008 59174 : exp = build3 (BIT_FIELD_REF, type, base, TYPE_SIZE (type),
12009 59174 : bitsize_int (offset * BITS_PER_UNIT));
12010 59174 : REF_REVERSE_STORAGE_ORDER (exp) = reverse;
12011 59174 : return expand_expr (exp, target, tmode, modifier);
12012 : }
12013 6483841 : address_mode = targetm.addr_space.address_mode (as);
12014 6483841 : if ((def_stmt = get_def_for_expr (base, BIT_AND_EXPR)))
12015 : {
12016 39 : tree mask = gimple_assign_rhs2 (def_stmt);
12017 39 : base = build2 (BIT_AND_EXPR, TREE_TYPE (base),
12018 : gimple_assign_rhs1 (def_stmt), mask);
12019 39 : TREE_OPERAND (exp, 0) = base;
12020 : }
12021 6483841 : align = get_object_alignment (exp);
12022 6483841 : op0 = expand_expr (base, NULL_RTX, VOIDmode, EXPAND_SUM);
12023 6483841 : op0 = memory_address_addr_space (mode, op0, as);
12024 6483841 : if (!integer_zerop (TREE_OPERAND (exp, 1)))
12025 : {
12026 1986980 : rtx off = immed_wide_int_const (mem_ref_offset (exp), address_mode);
12027 1986980 : op0 = simplify_gen_binary (PLUS, address_mode, op0, off);
12028 1986980 : op0 = memory_address_addr_space (mode, op0, as);
12029 : }
12030 6483841 : temp = gen_rtx_MEM (mode, op0);
12031 6483841 : set_mem_attributes (temp, exp, 0);
12032 6483841 : set_mem_addr_space (temp, as);
12033 6483841 : if (TREE_THIS_VOLATILE (exp))
12034 12335 : MEM_VOLATILE_P (temp) = 1;
12035 6483841 : if (modifier == EXPAND_WRITE || modifier == EXPAND_MEMORY)
12036 : return temp;
12037 3842915 : if (!inner_reference_p
12038 2015739 : && mode != BLKmode
12039 5797922 : && align < GET_MODE_ALIGNMENT (mode))
12040 139289 : temp = expand_misaligned_mem_ref (temp, mode, unsignedp, align,
12041 : modifier == EXPAND_STACK_PARM
12042 : ? NULL_RTX : target, alt_rtl);
12043 3842915 : if (reverse)
12044 7 : temp = flip_storage_order (mode, temp);
12045 3842915 : return EXTEND_BITINT (temp);
12046 : }
12047 :
12048 604499 : case ARRAY_REF:
12049 :
12050 604499 : {
12051 604499 : tree array = treeop0;
12052 604499 : tree index = treeop1;
12053 604499 : tree init;
12054 :
12055 : /* Fold an expression like: "foo"[2].
12056 : This is not done in fold so it won't happen inside &.
12057 : Don't fold if this is for wide characters since it's too
12058 : difficult to do correctly and this is a very rare case. */
12059 :
12060 604499 : if (modifier != EXPAND_CONST_ADDRESS
12061 604499 : && modifier != EXPAND_INITIALIZER
12062 604499 : && modifier != EXPAND_MEMORY)
12063 : {
12064 604390 : tree t = fold_read_from_constant_string (exp);
12065 :
12066 604390 : if (t)
12067 0 : return expand_expr (t, target, tmode, modifier);
12068 : }
12069 :
12070 : /* If this is a constant index into a constant array,
12071 : just get the value from the array. Handle both the cases when
12072 : we have an explicit constructor and when our operand is a variable
12073 : that was declared const. */
12074 :
12075 604499 : if (modifier != EXPAND_CONST_ADDRESS
12076 : && modifier != EXPAND_INITIALIZER
12077 : && modifier != EXPAND_MEMORY
12078 604390 : && TREE_CODE (array) == CONSTRUCTOR
12079 0 : && ! TREE_SIDE_EFFECTS (array)
12080 604499 : && TREE_CODE (index) == INTEGER_CST)
12081 : {
12082 : unsigned HOST_WIDE_INT ix;
12083 : tree field, value;
12084 :
12085 0 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (array), ix,
12086 : field, value)
12087 0 : if (tree_int_cst_equal (field, index))
12088 : {
12089 0 : if (!TREE_SIDE_EFFECTS (value)
12090 0 : && TREE_CODE (value) != RAW_DATA_CST)
12091 0 : return expand_expr (fold (value), target, tmode, modifier);
12092 : break;
12093 : }
12094 : }
12095 :
12096 604499 : else if (optimize >= 1
12097 : && modifier != EXPAND_CONST_ADDRESS
12098 337348 : && modifier != EXPAND_INITIALIZER
12099 337348 : && modifier != EXPAND_MEMORY
12100 337243 : && TREE_READONLY (array) && ! TREE_SIDE_EFFECTS (array)
12101 8921 : && TREE_CODE (index) == INTEGER_CST
12102 1924 : && (VAR_P (array) || TREE_CODE (array) == CONST_DECL)
12103 604648 : && (init = ctor_for_folding (array)) != error_mark_node)
12104 : {
12105 108 : if (init == NULL_TREE)
12106 : {
12107 5 : tree value = build_zero_cst (type);
12108 5 : if (TREE_CODE (value) == CONSTRUCTOR)
12109 : {
12110 : /* If VALUE is a CONSTRUCTOR, this optimization is only
12111 : useful if this doesn't store the CONSTRUCTOR into
12112 : memory. If it does, it is more efficient to just
12113 : load the data from the array directly. */
12114 5 : rtx ret = expand_constructor (value, target,
12115 : modifier, true);
12116 5 : if (ret == NULL_RTX)
12117 : value = NULL_TREE;
12118 : }
12119 :
12120 : if (value)
12121 0 : return expand_expr (value, target, tmode, modifier);
12122 : }
12123 103 : else if (TREE_CODE (init) == CONSTRUCTOR)
12124 : {
12125 : unsigned HOST_WIDE_INT ix;
12126 : tree field, value;
12127 :
12128 212 : FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (init), ix,
12129 : field, value)
12130 189 : if (tree_int_cst_equal (field, index))
12131 : {
12132 79 : if (TREE_SIDE_EFFECTS (value)
12133 79 : || TREE_CODE (value) == RAW_DATA_CST)
12134 : break;
12135 :
12136 79 : if (TREE_CODE (value) == CONSTRUCTOR)
12137 : {
12138 : /* If VALUE is a CONSTRUCTOR, this
12139 : optimization is only useful if
12140 : this doesn't store the CONSTRUCTOR
12141 : into memory. If it does, it is more
12142 : efficient to just load the data from
12143 : the array directly. */
12144 74 : rtx ret = expand_constructor (value, target,
12145 : modifier, true);
12146 74 : if (ret == NULL_RTX)
12147 : break;
12148 : }
12149 :
12150 41 : return expand_expr (fold (value), target, tmode,
12151 41 : modifier);
12152 : }
12153 : }
12154 1 : else if (TREE_CODE (init) == STRING_CST)
12155 : {
12156 1 : tree low_bound = array_ref_low_bound (exp);
12157 1 : tree index1 = fold_convert_loc (loc, sizetype, treeop1);
12158 :
12159 : /* Optimize the special case of a zero lower bound.
12160 :
12161 : We convert the lower bound to sizetype to avoid problems
12162 : with constant folding. E.g. suppose the lower bound is
12163 : 1 and its mode is QI. Without the conversion
12164 : (ARRAY + (INDEX - (unsigned char)1))
12165 : becomes
12166 : (ARRAY + (-(unsigned char)1) + INDEX)
12167 : which becomes
12168 : (ARRAY + 255 + INDEX). Oops! */
12169 1 : if (!integer_zerop (low_bound))
12170 0 : index1 = size_diffop_loc (loc, index1,
12171 : fold_convert_loc (loc, sizetype,
12172 : low_bound));
12173 :
12174 1 : if (tree_fits_uhwi_p (index1)
12175 2 : && compare_tree_int (index1, TREE_STRING_LENGTH (init)) < 0)
12176 : {
12177 0 : tree char_type = TREE_TYPE (TREE_TYPE (init));
12178 0 : scalar_int_mode char_mode;
12179 :
12180 604458 : if (is_int_mode (TYPE_MODE (char_type), &char_mode)
12181 0 : && GET_MODE_SIZE (char_mode) == 1)
12182 0 : return gen_int_mode (TREE_STRING_POINTER (init)
12183 0 : [TREE_INT_CST_LOW (index1)],
12184 : char_mode);
12185 : }
12186 : }
12187 : }
12188 : }
12189 604458 : goto normal_inner_ref;
12190 :
12191 3640969 : case COMPONENT_REF:
12192 3640969 : gcc_assert (TREE_CODE (treeop0) != CONSTRUCTOR);
12193 : /* Fall through. */
12194 4495501 : case BIT_FIELD_REF:
12195 4495501 : case ARRAY_RANGE_REF:
12196 3640969 : normal_inner_ref:
12197 4495501 : {
12198 4495501 : machine_mode mode1, mode2;
12199 4495501 : poly_int64 bitsize, bitpos, bytepos;
12200 4495501 : tree offset;
12201 4495501 : int reversep, volatilep = 0;
12202 4495501 : tree tem
12203 4495501 : = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
12204 : &unsignedp, &reversep, &volatilep);
12205 4495501 : rtx orig_op0, memloc;
12206 4495501 : bool clear_mem_expr = false;
12207 4495501 : bool must_force_mem;
12208 :
12209 : /* If we got back the original object, something is wrong. Perhaps
12210 : we are evaluating an expression too early. In any event, don't
12211 : infinitely recurse. */
12212 4495501 : gcc_assert (tem != exp);
12213 :
12214 : /* Make sure bitpos is not negative, this can wreak havoc later. */
12215 4495501 : if (maybe_lt (bitpos, 0))
12216 : {
12217 249 : gcc_checking_assert (offset == NULL_TREE);
12218 249 : offset = size_int (bits_to_bytes_round_down (bitpos));
12219 249 : bitpos = num_trailing_bits (bitpos);
12220 : }
12221 :
12222 : /* If we have either an offset, a BLKmode result, or a reference
12223 : outside the underlying object, we must force it to memory.
12224 : Such a case can occur in Ada if we have unchecked conversion
12225 : of an expression from a scalar type to an aggregate type or
12226 : for an ARRAY_RANGE_REF whose type is BLKmode, or if we were
12227 : passed a partially uninitialized object or a view-conversion
12228 : to a larger size. */
12229 8991002 : must_force_mem = offset != NULL_TREE
12230 4218675 : || mode1 == BLKmode
12231 8642026 : || (mode == BLKmode
12232 0 : && !int_mode_for_size (bitsize, 1).exists ());
12233 :
12234 516810 : const enum expand_modifier tem_modifier
12235 : = must_force_mem
12236 : ? EXPAND_MEMORY
12237 4146525 : : modifier == EXPAND_SUM ? EXPAND_NORMAL : modifier;
12238 :
12239 : /* If TEM's type is a union of variable size, pass TARGET to the inner
12240 : computation, since it will need a temporary and TARGET is known
12241 : to have to do. This occurs in unchecked conversion in Ada. */
12242 4495501 : const rtx tem_target
12243 4495501 : = TREE_CODE (TREE_TYPE (tem)) == UNION_TYPE
12244 36673 : && COMPLETE_TYPE_P (TREE_TYPE (tem))
12245 36668 : && TREE_CODE (TYPE_SIZE (TREE_TYPE (tem))) != INTEGER_CST
12246 0 : && modifier != EXPAND_STACK_PARM
12247 4495501 : ? target
12248 : : NULL_RTX;
12249 :
12250 8991002 : orig_op0 = op0
12251 4495501 : = expand_expr_real (tem, tem_target, VOIDmode, tem_modifier, NULL,
12252 : true);
12253 :
12254 : /* If the field has a mode, we want to access it in the
12255 : field's mode, not the computed mode.
12256 : If a MEM has VOIDmode (external with incomplete type),
12257 : use BLKmode for it instead. */
12258 4495501 : if (MEM_P (op0))
12259 : {
12260 4163635 : if (mode1 != VOIDmode)
12261 4010638 : op0 = adjust_address (op0, mode1, 0);
12262 152997 : else if (GET_MODE (op0) == VOIDmode)
12263 0 : op0 = adjust_address (op0, BLKmode, 0);
12264 : }
12265 :
12266 4495501 : mode2
12267 4495501 : = CONSTANT_P (op0) ? TYPE_MODE (TREE_TYPE (tem)) : GET_MODE (op0);
12268 :
12269 : /* See above for the rationale. */
12270 8991002 : if (maybe_gt (bitpos + bitsize, GET_MODE_BITSIZE (mode2)))
12271 2830444 : must_force_mem = true;
12272 :
12273 : /* Handle CONCAT first. */
12274 4495501 : if (GET_CODE (op0) == CONCAT && !must_force_mem)
12275 : {
12276 114 : if (known_eq (bitpos, 0)
12277 206 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0)))
12278 98 : && COMPLEX_MODE_P (mode1)
12279 93 : && COMPLEX_MODE_P (GET_MODE (op0))
12280 393 : && (GET_MODE_PRECISION (GET_MODE_INNER (mode1))
12281 186 : == GET_MODE_PRECISION (GET_MODE_INNER (GET_MODE (op0)))))
12282 : {
12283 93 : if (reversep)
12284 0 : op0 = flip_storage_order (GET_MODE (op0), op0);
12285 93 : if (mode1 != GET_MODE (op0))
12286 : {
12287 : rtx parts[2];
12288 0 : for (int i = 0; i < 2; i++)
12289 : {
12290 0 : rtx op = read_complex_part (op0, i != 0);
12291 0 : if (GET_CODE (op) == SUBREG)
12292 0 : op = force_reg (GET_MODE (op), op);
12293 0 : temp = gen_lowpart_common (GET_MODE_INNER (mode1), op);
12294 0 : if (temp)
12295 : op = temp;
12296 : else
12297 : {
12298 0 : if (!REG_P (op) && !MEM_P (op))
12299 0 : op = force_reg (GET_MODE (op), op);
12300 0 : op = gen_lowpart (GET_MODE_INNER (mode1), op);
12301 : }
12302 0 : parts[i] = op;
12303 : }
12304 0 : op0 = gen_rtx_CONCAT (mode1, parts[0], parts[1]);
12305 : }
12306 93 : return op0;
12307 : }
12308 21 : if (known_eq (bitpos, 0)
12309 20 : && known_eq (bitsize,
12310 : GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 0))))
12311 23 : && maybe_ne (bitsize, 0))
12312 : {
12313 : op0 = XEXP (op0, 0);
12314 : mode2 = GET_MODE (op0);
12315 : }
12316 19 : else if (known_eq (bitpos,
12317 : GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 0))))
12318 4 : && known_eq (bitsize,
12319 : GET_MODE_BITSIZE (GET_MODE (XEXP (op0, 1))))
12320 0 : && maybe_ne (bitpos, 0)
12321 19 : && maybe_ne (bitsize, 0))
12322 : {
12323 0 : op0 = XEXP (op0, 1);
12324 0 : bitpos = 0;
12325 0 : mode2 = GET_MODE (op0);
12326 : }
12327 : else
12328 : /* Otherwise force into memory. */
12329 : must_force_mem = true;
12330 : }
12331 :
12332 : /* If this is a constant, put it in a register if it is a legitimate
12333 : constant and we don't need a memory reference. */
12334 4495408 : if (CONSTANT_P (op0)
12335 25 : && mode2 != BLKmode
12336 25 : && targetm.legitimate_constant_p (mode2, op0)
12337 4495421 : && !must_force_mem)
12338 13 : op0 = force_reg (mode2, op0);
12339 :
12340 : /* Otherwise, if this is a constant, try to force it to the constant
12341 : pool. Note that back-ends, e.g. MIPS, may refuse to do so if it
12342 : is a legitimate constant. */
12343 4495395 : else if (CONSTANT_P (op0) && (memloc = force_const_mem (mode2, op0)))
12344 12 : op0 = validize_mem (memloc);
12345 :
12346 : /* Otherwise, if this is a constant or the object is not in memory
12347 : and need be, put it there. */
12348 4495383 : else if (CONSTANT_P (op0) || (!MEM_P (op0) && must_force_mem))
12349 : {
12350 1095 : machine_mode tem_mode = TYPE_MODE (TREE_TYPE (tem));
12351 1095 : poly_int64 size;
12352 1095 : if (!poly_int_tree_p (TYPE_SIZE_UNIT (TREE_TYPE (tem)), &size))
12353 0 : size = max_int_size_in_bytes (TREE_TYPE (tem));
12354 1095 : unsigned int align = TREE_CODE (tem) == SSA_NAME
12355 1095 : ? TYPE_ALIGN (TREE_TYPE (tem))
12356 1095 : : get_object_alignment (tem);
12357 1095 : if (STRICT_ALIGNMENT)
12358 : {
12359 : /* For STRICT_ALIGNMENT targets, when we force the operand to
12360 : memory, we may need to increase the alignment to meet the
12361 : expectation in later RTL lowering passes. The increased
12362 : alignment is capped by MAX_SUPPORTED_STACK_ALIGNMENT. */
12363 : if (tem_mode != BLKmode)
12364 : align = MAX (align, GET_MODE_ALIGNMENT (tem_mode));
12365 : else
12366 : align = MAX (align, TYPE_ALIGN (TREE_TYPE (tem)));
12367 : align = MIN (align, (unsigned) MAX_SUPPORTED_STACK_ALIGNMENT);
12368 : }
12369 1095 : memloc = assign_stack_local (tem_mode, size, align);
12370 1095 : emit_move_insn (memloc, op0);
12371 1095 : op0 = memloc;
12372 1095 : clear_mem_expr = true;
12373 : }
12374 :
12375 4495408 : if (offset)
12376 : {
12377 276826 : machine_mode address_mode;
12378 276826 : rtx offset_rtx = expand_expr (offset, NULL_RTX, VOIDmode,
12379 : EXPAND_SUM);
12380 :
12381 276826 : gcc_assert (MEM_P (op0));
12382 :
12383 276826 : address_mode = get_address_mode (op0);
12384 276826 : if (GET_MODE (offset_rtx) != address_mode)
12385 : {
12386 : /* We cannot be sure that the RTL in offset_rtx is valid outside
12387 : of a memory address context, so force it into a register
12388 : before attempting to convert it to the desired mode. */
12389 422 : offset_rtx = force_operand (offset_rtx, NULL_RTX);
12390 422 : offset_rtx = convert_to_mode (address_mode, offset_rtx, 0);
12391 : }
12392 :
12393 : /* See the comment in expand_assignment for the rationale. */
12394 276826 : if (mode1 != VOIDmode
12395 276610 : && maybe_ne (bitpos, 0)
12396 74268 : && maybe_gt (bitsize, 0)
12397 351094 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
12398 350555 : && multiple_p (bitpos, bitsize)
12399 147458 : && multiple_p (bitsize, GET_MODE_ALIGNMENT (mode1))
12400 350555 : && MEM_ALIGN (op0) >= GET_MODE_ALIGNMENT (mode1))
12401 : {
12402 73374 : op0 = adjust_address (op0, mode1, bytepos);
12403 73374 : bitpos = 0;
12404 : }
12405 :
12406 276826 : op0 = offset_address (op0, offset_rtx,
12407 : highest_pow2_factor (offset));
12408 : }
12409 :
12410 : /* If OFFSET is making OP0 more aligned than BIGGEST_ALIGNMENT,
12411 : record its alignment as BIGGEST_ALIGNMENT. */
12412 4495408 : if (MEM_P (op0)
12413 4164742 : && known_eq (bitpos, 0)
12414 1419159 : && offset != 0
12415 4771190 : && is_aligning_offset (offset, tem))
12416 0 : set_mem_align (op0, BIGGEST_ALIGNMENT);
12417 :
12418 : /* Don't forget about volatility even if this is a bitfield. */
12419 4495408 : if (MEM_P (op0) && volatilep && ! MEM_VOLATILE_P (op0))
12420 : {
12421 528 : if (op0 == orig_op0)
12422 158 : op0 = copy_rtx (op0);
12423 :
12424 528 : MEM_VOLATILE_P (op0) = 1;
12425 : }
12426 :
12427 4495408 : if (MEM_P (op0) && TREE_CODE (tem) == FUNCTION_DECL)
12428 : {
12429 6 : if (op0 == orig_op0)
12430 0 : op0 = copy_rtx (op0);
12431 :
12432 6 : set_mem_align (op0, BITS_PER_UNIT);
12433 : }
12434 :
12435 : /* In cases where an aligned union has an unaligned object
12436 : as a field, we might be extracting a BLKmode value from
12437 : an integer-mode (e.g., SImode) object. Handle this case
12438 : by doing the extract into an object as wide as the field
12439 : (which we know to be the width of a basic mode), then
12440 : storing into memory, and changing the mode to BLKmode. */
12441 4495408 : if (mode1 == VOIDmode
12442 4271438 : || REG_P (op0) || GET_CODE (op0) == SUBREG
12443 4011711 : || (mode1 != BLKmode && ! direct_load[(int) mode1]
12444 26619 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_INT
12445 22460 : && GET_MODE_CLASS (mode) != MODE_COMPLEX_FLOAT
12446 : && modifier != EXPAND_CONST_ADDRESS
12447 15397 : && modifier != EXPAND_INITIALIZER
12448 15397 : && modifier != EXPAND_MEMORY)
12449 : /* If the bitfield is volatile and the bitsize
12450 : is narrower than the access size of the bitfield,
12451 : we need to extract bitfields from the access. */
12452 3996314 : || (volatilep && TREE_CODE (exp) == COMPONENT_REF
12453 1390 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (exp, 1))
12454 10 : && mode1 != BLKmode
12455 20 : && maybe_lt (bitsize, GET_MODE_SIZE (mode1) * BITS_PER_UNIT))
12456 : /* If the field isn't aligned enough to fetch as a memref,
12457 : fetch it as a bit field. */
12458 3996305 : || (mode1 != BLKmode
12459 3923305 : && (((MEM_P (op0)
12460 3923305 : ? MEM_ALIGN (op0) < GET_MODE_ALIGNMENT (mode1)
12461 7702417 : || !multiple_p (bitpos, GET_MODE_ALIGNMENT (mode1))
12462 0 : : TYPE_ALIGN (TREE_TYPE (tem)) < GET_MODE_ALIGNMENT (mode)
12463 0 : || !multiple_p (bitpos, GET_MODE_ALIGNMENT (mode)))
12464 75417 : && modifier != EXPAND_MEMORY
12465 75417 : && ((modifier == EXPAND_CONST_ADDRESS
12466 75417 : || modifier == EXPAND_INITIALIZER)
12467 75417 : ? STRICT_ALIGNMENT
12468 75417 : : targetm.slow_unaligned_access (mode1,
12469 75417 : MEM_ALIGN (op0))))
12470 3923305 : || !multiple_p (bitpos, BITS_PER_UNIT)))
12471 : /* If the type and the field are a constant size and the
12472 : size of the type isn't the same size as the bitfield,
12473 : we must use bitfield operations. */
12474 12415018 : || (known_size_p (bitsize)
12475 3996305 : && TYPE_SIZE (TREE_TYPE (exp))
12476 3996305 : && poly_int_tree_p (TYPE_SIZE (TREE_TYPE (exp)))
12477 3996305 : && maybe_ne (wi::to_poly_offset (TYPE_SIZE (TREE_TYPE (exp))),
12478 : bitsize)))
12479 : {
12480 499132 : machine_mode ext_mode = mode;
12481 :
12482 499132 : if (ext_mode == BLKmode
12483 499132 : && ! (target != 0 && MEM_P (op0)
12484 22 : && MEM_P (target)
12485 22 : && multiple_p (bitpos, BITS_PER_UNIT)))
12486 3 : ext_mode = int_mode_for_size (bitsize, 1).else_blk ();
12487 :
12488 499132 : if (ext_mode == BLKmode)
12489 : {
12490 25 : if (target == 0)
12491 3 : target = assign_temp (type, 1, 1);
12492 :
12493 : /* ??? Unlike the similar test a few lines below, this one is
12494 : very likely obsolete. */
12495 25 : if (known_eq (bitsize, 0))
12496 : return target;
12497 :
12498 : /* In this case, BITPOS must start at a byte boundary and
12499 : TARGET, if specified, must be a MEM. */
12500 25 : gcc_assert (MEM_P (op0)
12501 : && (!target || MEM_P (target)));
12502 :
12503 25 : bytepos = exact_div (bitpos, BITS_PER_UNIT);
12504 25 : poly_int64 bytesize = bits_to_bytes_round_up (bitsize);
12505 50 : emit_block_move (target,
12506 : adjust_address (op0, VOIDmode, bytepos),
12507 25 : gen_int_mode (bytesize, Pmode),
12508 : (modifier == EXPAND_STACK_PARM
12509 : ? BLOCK_OP_CALL_PARM : BLOCK_OP_NORMAL));
12510 :
12511 25 : return target;
12512 : }
12513 :
12514 : /* If we have nothing to extract, the result will be 0 for targets
12515 : with SHIFT_COUNT_TRUNCATED == 0 and garbage otherwise. Always
12516 : return 0 for the sake of consistency, as reading a zero-sized
12517 : bitfield is valid in Ada and the value is fully specified. */
12518 499107 : if (known_eq (bitsize, 0))
12519 0 : return const0_rtx;
12520 :
12521 499107 : op0 = validize_mem (op0);
12522 :
12523 499107 : if (MEM_P (op0) && REG_P (XEXP (op0, 0)))
12524 45804 : mark_reg_pointer (XEXP (op0, 0), MEM_ALIGN (op0));
12525 :
12526 : /* If the result has aggregate type and the extraction is done in
12527 : an integral mode, then the field may be not aligned on a byte
12528 : boundary; in this case, if it has reverse storage order, it
12529 : needs to be extracted as a scalar field with reverse storage
12530 : order and put back into memory order afterwards. */
12531 499107 : if (AGGREGATE_TYPE_P (type)
12532 4794 : && GET_MODE_CLASS (ext_mode) == MODE_INT)
12533 4712 : reversep = TYPE_REVERSE_STORAGE_ORDER (type);
12534 :
12535 499107 : gcc_checking_assert (known_ge (bitpos, 0));
12536 511725 : op0 = extract_bit_field (op0, bitsize, bitpos, unsignedp,
12537 : (modifier == EXPAND_STACK_PARM
12538 : ? NULL_RTX : target),
12539 : ext_mode, ext_mode, reversep, alt_rtl);
12540 :
12541 : /* If the result has aggregate type and the mode of OP0 is an
12542 : integral mode then, if BITSIZE is narrower than this mode
12543 : and this is for big-endian data, we must put the field
12544 : into the high-order bits. And we must also put it back
12545 : into memory order if it has been previously reversed. */
12546 499107 : scalar_int_mode op0_mode;
12547 499107 : if (AGGREGATE_TYPE_P (type)
12548 499107 : && is_int_mode (GET_MODE (op0), &op0_mode))
12549 : {
12550 4712 : HOST_WIDE_INT size = GET_MODE_BITSIZE (op0_mode);
12551 :
12552 4712 : gcc_checking_assert (known_le (bitsize, size));
12553 4712 : if (maybe_lt (bitsize, size)
12554 4712 : && reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
12555 0 : op0 = expand_shift (LSHIFT_EXPR, op0_mode, op0,
12556 : size - bitsize, op0, 1);
12557 :
12558 4712 : if (reversep)
12559 0 : op0 = flip_storage_order (op0_mode, op0);
12560 : }
12561 :
12562 : /* If the result type is BLKmode, store the data into a temporary
12563 : of the appropriate type, but with the mode corresponding to the
12564 : mode for the data we have (op0's mode). */
12565 499107 : if (mode == BLKmode)
12566 : {
12567 0 : rtx new_rtx
12568 0 : = assign_stack_temp_for_type (ext_mode,
12569 0 : GET_MODE_BITSIZE (ext_mode),
12570 : type);
12571 0 : emit_move_insn (new_rtx, op0);
12572 0 : op0 = copy_rtx (new_rtx);
12573 0 : PUT_MODE (op0, BLKmode);
12574 : }
12575 :
12576 499107 : return op0;
12577 : }
12578 :
12579 : /* If the result is BLKmode, use that to access the object
12580 : now as well. */
12581 3996276 : if (mode == BLKmode)
12582 72975 : mode1 = BLKmode;
12583 :
12584 : /* Get a reference to just this component. */
12585 3996276 : bytepos = bits_to_bytes_round_down (bitpos);
12586 3996276 : if (modifier == EXPAND_CONST_ADDRESS
12587 3996276 : || modifier == EXPAND_SUM || modifier == EXPAND_INITIALIZER)
12588 158487 : op0 = adjust_address_nv (op0, mode1, bytepos);
12589 : else
12590 3837789 : op0 = adjust_address (op0, mode1, bytepos);
12591 :
12592 3996276 : if (op0 == orig_op0)
12593 122990 : op0 = copy_rtx (op0);
12594 :
12595 : /* Don't set memory attributes if the base expression is
12596 : SSA_NAME that got expanded as a MEM or a CONSTANT. In that case,
12597 : we should just honor its original memory attributes. */
12598 3996276 : if (!(TREE_CODE (tem) == SSA_NAME
12599 8778 : && (MEM_P (orig_op0) || CONSTANT_P (orig_op0))))
12600 3987498 : set_mem_attributes (op0, exp, 0);
12601 :
12602 3996276 : if (REG_P (XEXP (op0, 0)))
12603 647710 : mark_reg_pointer (XEXP (op0, 0), MEM_ALIGN (op0));
12604 :
12605 : /* If op0 is a temporary because the original expressions was forced
12606 : to memory, clear MEM_EXPR so that the original expression cannot
12607 : be marked as addressable through MEM_EXPR of the temporary. */
12608 3996276 : if (clear_mem_expr)
12609 1061 : set_mem_expr (op0, NULL_TREE);
12610 :
12611 3996276 : MEM_VOLATILE_P (op0) |= volatilep;
12612 :
12613 3996276 : if (reversep
12614 : && modifier != EXPAND_MEMORY
12615 491 : && modifier != EXPAND_WRITE)
12616 491 : op0 = flip_storage_order (mode1, op0);
12617 :
12618 3996276 : op0 = EXTEND_BITINT (op0);
12619 :
12620 3996276 : if (mode == mode1 || mode1 == BLKmode || mode1 == tmode
12621 : || modifier == EXPAND_CONST_ADDRESS
12622 0 : || modifier == EXPAND_INITIALIZER)
12623 : return op0;
12624 :
12625 0 : if (target == 0)
12626 0 : target = gen_reg_rtx (tmode != VOIDmode ? tmode : mode);
12627 :
12628 0 : convert_move (target, op0, unsignedp);
12629 0 : return target;
12630 : }
12631 :
12632 81385 : case OBJ_TYPE_REF:
12633 81385 : return expand_expr (OBJ_TYPE_REF_EXPR (exp), target, tmode, modifier);
12634 :
12635 6646199 : case CALL_EXPR:
12636 : /* All valid uses of __builtin_va_arg_pack () are removed during
12637 : inlining. */
12638 6646199 : if (CALL_EXPR_VA_ARG_PACK (exp))
12639 6 : error ("invalid use of %<__builtin_va_arg_pack ()%>");
12640 6646199 : {
12641 6646199 : tree fndecl = get_callee_fndecl (exp), attr;
12642 :
12643 6646199 : if (fndecl
12644 : /* Don't diagnose the error attribute in thunks, those are
12645 : artificially created. */
12646 6462457 : && !CALL_FROM_THUNK_P (exp)
12647 13105063 : && (attr = lookup_attribute ("error",
12648 6458864 : DECL_ATTRIBUTES (fndecl))) != NULL)
12649 : {
12650 5 : const char *ident = lang_hooks.decl_printable_name (fndecl, 1);
12651 10 : error ("call to %qs declared with attribute error: %s",
12652 : identifier_to_locale (ident),
12653 5 : TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr))));
12654 : }
12655 6646199 : if (fndecl
12656 : /* Don't diagnose the warning attribute in thunks, those are
12657 : artificially created. */
12658 6462457 : && !CALL_FROM_THUNK_P (exp)
12659 13105063 : && (attr = lookup_attribute ("warning",
12660 6458864 : DECL_ATTRIBUTES (fndecl))) != NULL)
12661 : {
12662 17 : const char *ident = lang_hooks.decl_printable_name (fndecl, 1);
12663 34 : warning_at (EXPR_LOCATION (exp),
12664 17 : OPT_Wattribute_warning,
12665 : "call to %qs declared with attribute warning: %s",
12666 : identifier_to_locale (ident),
12667 17 : TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr))));
12668 : }
12669 :
12670 : /* Check for a built-in function. */
12671 6646199 : if (fndecl && fndecl_built_in_p (fndecl))
12672 : {
12673 1983709 : gcc_assert (DECL_BUILT_IN_CLASS (fndecl) != BUILT_IN_FRONTEND);
12674 1983709 : return expand_builtin (exp, target, subtarget, tmode, ignore);
12675 : }
12676 : }
12677 4662490 : temp = expand_call (exp, target, ignore);
12678 4662489 : return EXTEND_BITINT (temp);
12679 :
12680 339411 : case VIEW_CONVERT_EXPR:
12681 339411 : op0 = NULL_RTX;
12682 :
12683 : /* If we are converting to BLKmode, try to avoid an intermediate
12684 : temporary by fetching an inner memory reference. */
12685 339411 : if (mode == BLKmode
12686 74198 : && poly_int_tree_p (TYPE_SIZE (type))
12687 74198 : && TYPE_MODE (TREE_TYPE (treeop0)) != BLKmode
12688 339411 : && handled_component_p (treeop0))
12689 : {
12690 0 : machine_mode mode1;
12691 0 : poly_int64 bitsize, bitpos, bytepos;
12692 0 : tree offset;
12693 0 : int reversep, volatilep = 0;
12694 0 : tree tem
12695 0 : = get_inner_reference (treeop0, &bitsize, &bitpos, &offset, &mode1,
12696 : &unsignedp, &reversep, &volatilep);
12697 :
12698 : /* ??? We should work harder and deal with non-zero offsets. */
12699 0 : if (!offset
12700 0 : && multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
12701 0 : && !reversep
12702 0 : && known_size_p (bitsize)
12703 0 : && known_eq (wi::to_poly_offset (TYPE_SIZE (type)), bitsize))
12704 : {
12705 : /* See the normal_inner_ref case for the rationale. */
12706 0 : rtx orig_op0
12707 0 : = expand_expr_real (tem,
12708 0 : (TREE_CODE (TREE_TYPE (tem)) == UNION_TYPE
12709 0 : && (TREE_CODE (TYPE_SIZE (TREE_TYPE (tem)))
12710 : != INTEGER_CST)
12711 0 : && modifier != EXPAND_STACK_PARM
12712 : ? target : NULL_RTX),
12713 : VOIDmode,
12714 : modifier == EXPAND_SUM ? EXPAND_NORMAL : modifier,
12715 : NULL, true);
12716 :
12717 0 : if (MEM_P (orig_op0))
12718 : {
12719 0 : op0 = orig_op0;
12720 :
12721 : /* Get a reference to just this component. */
12722 0 : if (modifier == EXPAND_CONST_ADDRESS
12723 : || modifier == EXPAND_SUM
12724 0 : || modifier == EXPAND_INITIALIZER)
12725 0 : op0 = adjust_address_nv (op0, mode, bytepos);
12726 : else
12727 0 : op0 = adjust_address (op0, mode, bytepos);
12728 :
12729 0 : if (op0 == orig_op0)
12730 0 : op0 = copy_rtx (op0);
12731 :
12732 0 : set_mem_attributes (op0, treeop0, 0);
12733 0 : if (REG_P (XEXP (op0, 0)))
12734 0 : mark_reg_pointer (XEXP (op0, 0), MEM_ALIGN (op0));
12735 :
12736 0 : MEM_VOLATILE_P (op0) |= volatilep;
12737 : }
12738 : }
12739 : }
12740 :
12741 0 : if (!op0)
12742 339411 : op0 = expand_expr_real (treeop0, NULL_RTX, VOIDmode, modifier,
12743 339411 : NULL, inner_reference_p || mode == BLKmode);
12744 :
12745 : /* If the input and output modes are both the same, we are done. */
12746 339411 : if (mode == GET_MODE (op0))
12747 : ;
12748 : /* Similarly if the output mode is BLKmode and input is a MEM,
12749 : adjust_address done below is all we need. */
12750 146373 : else if (mode == BLKmode && MEM_P (op0))
12751 : ;
12752 : /* If neither mode is BLKmode, and both modes are the same size
12753 : then we can use gen_lowpart. */
12754 : else if (mode != BLKmode
12755 146339 : && GET_MODE (op0) != BLKmode
12756 144992 : && known_eq (GET_MODE_PRECISION (mode),
12757 : GET_MODE_PRECISION (GET_MODE (op0)))
12758 141095 : && !COMPLEX_MODE_P (GET_MODE (op0)))
12759 : {
12760 139682 : if (GET_CODE (op0) == SUBREG)
12761 103 : op0 = force_reg (GET_MODE (op0), op0);
12762 139682 : temp = gen_lowpart_common (mode, op0);
12763 139682 : if (temp)
12764 : op0 = temp;
12765 : else
12766 : {
12767 27380 : if (!REG_P (op0) && !MEM_P (op0))
12768 18 : op0 = force_reg (GET_MODE (op0), op0);
12769 27380 : op0 = gen_lowpart (mode, op0);
12770 : }
12771 : }
12772 : /* If both types are integral, convert from one mode to the other. */
12773 6689 : else if (INTEGRAL_TYPE_P (type)
12774 3141 : && INTEGRAL_TYPE_P (TREE_TYPE (treeop0))
12775 0 : && mode != BLKmode
12776 6689 : && GET_MODE (op0) != BLKmode)
12777 0 : op0 = convert_modes (mode, GET_MODE (op0), op0,
12778 0 : TYPE_UNSIGNED (TREE_TYPE (treeop0)));
12779 : /* If the output type is a bit-field type, do an extraction. */
12780 6689 : else if (reduce_bit_field
12781 2 : && mode != BLKmode
12782 2 : && (MEM_P (op0) || !COMPLEX_MODE_P (GET_MODE (op0))))
12783 0 : return extract_bit_field (op0, TYPE_PRECISION (type), 0,
12784 0 : TYPE_UNSIGNED (type), NULL_RTX,
12785 : mode, mode, false, NULL);
12786 : /* As a last resort, spill op0 to memory, and reload it in a
12787 : different mode. */
12788 6689 : else if (!MEM_P (op0))
12789 : {
12790 : /* If the operand is not a MEM, force it into memory. Since we
12791 : are going to be changing the mode of the MEM, don't call
12792 : force_const_mem for constants because we don't allow pool
12793 : constants to change mode. */
12794 5332 : tree inner_type = TREE_TYPE (treeop0);
12795 :
12796 5332 : gcc_assert (!TREE_ADDRESSABLE (exp));
12797 :
12798 5332 : if (target == 0 || GET_MODE (target) != TYPE_MODE (inner_type))
12799 5328 : target
12800 : = assign_stack_temp_for_type
12801 5328 : (TYPE_MODE (inner_type),
12802 10656 : GET_MODE_SIZE (TYPE_MODE (inner_type)), inner_type);
12803 :
12804 5332 : emit_move_insn (target, op0);
12805 5332 : op0 = target;
12806 :
12807 5332 : if (reduce_bit_field && mode != BLKmode)
12808 2 : return extract_bit_field (op0, TYPE_PRECISION (type), 0,
12809 2 : TYPE_UNSIGNED (type), NULL_RTX,
12810 : mode, mode, false, NULL);
12811 : }
12812 :
12813 : /* If OP0 is (now) a MEM, we need to deal with alignment issues. If the
12814 : output type is such that the operand is known to be aligned, indicate
12815 : that it is. Otherwise, we need only be concerned about alignment for
12816 : non-BLKmode results. */
12817 339409 : if (MEM_P (op0))
12818 : {
12819 127524 : enum insn_code icode;
12820 :
12821 127524 : if (modifier != EXPAND_WRITE
12822 127524 : && modifier != EXPAND_MEMORY
12823 127524 : && !inner_reference_p
12824 127523 : && mode != BLKmode
12825 180849 : && MEM_ALIGN (op0) < GET_MODE_ALIGNMENT (mode))
12826 : {
12827 : /* If the target does have special handling for unaligned
12828 : loads of mode then use them. */
12829 15696 : if ((icode = optab_handler (movmisalign_optab, mode))
12830 : != CODE_FOR_nothing)
12831 : {
12832 1804 : rtx reg;
12833 :
12834 1804 : op0 = adjust_address (op0, mode, 0);
12835 : /* We've already validated the memory, and we're creating a
12836 : new pseudo destination. The predicates really can't
12837 : fail. */
12838 1804 : reg = gen_reg_rtx (mode);
12839 :
12840 : /* Nor can the insn generator. */
12841 1804 : rtx_insn *insn = GEN_FCN (icode) (reg, op0);
12842 1804 : emit_insn (insn);
12843 1804 : return reg;
12844 : }
12845 : else if (STRICT_ALIGNMENT)
12846 : {
12847 : poly_uint64 mode_size = GET_MODE_SIZE (mode);
12848 : poly_uint64 temp_size = mode_size;
12849 : if (GET_MODE (op0) != BLKmode)
12850 : temp_size = upper_bound (temp_size,
12851 : GET_MODE_SIZE (GET_MODE (op0)));
12852 : rtx new_rtx
12853 : = assign_stack_temp_for_type (mode, temp_size, type);
12854 : rtx new_with_op0_mode
12855 : = adjust_address (new_rtx, GET_MODE (op0), 0);
12856 :
12857 : gcc_assert (!TREE_ADDRESSABLE (exp));
12858 :
12859 : if (GET_MODE (op0) == BLKmode)
12860 : {
12861 : rtx size_rtx = gen_int_mode (mode_size, Pmode);
12862 : emit_block_move (new_with_op0_mode, op0, size_rtx,
12863 : (modifier == EXPAND_STACK_PARM
12864 : ? BLOCK_OP_CALL_PARM
12865 : : BLOCK_OP_NORMAL));
12866 : }
12867 : else
12868 : emit_move_insn (new_with_op0_mode, op0);
12869 :
12870 : op0 = new_rtx;
12871 : }
12872 : }
12873 :
12874 125720 : op0 = adjust_address (op0, mode, 0);
12875 : }
12876 :
12877 : return op0;
12878 :
12879 63161 : case MODIFY_EXPR:
12880 63161 : {
12881 63161 : tree lhs = treeop0;
12882 63161 : tree rhs = treeop1;
12883 63161 : gcc_assert (ignore);
12884 :
12885 : /* Check for |= or &= of a bitfield of size one into another bitfield
12886 : of size 1. In this case, (unless we need the result of the
12887 : assignment) we can do this more efficiently with a
12888 : test followed by an assignment, if necessary.
12889 :
12890 : ??? At this point, we can't get a BIT_FIELD_REF here. But if
12891 : things change so we do, this code should be enhanced to
12892 : support it. */
12893 63161 : if (TREE_CODE (lhs) == COMPONENT_REF
12894 60156 : && (TREE_CODE (rhs) == BIT_IOR_EXPR
12895 60156 : || TREE_CODE (rhs) == BIT_AND_EXPR)
12896 0 : && TREE_OPERAND (rhs, 0) == lhs
12897 0 : && TREE_CODE (TREE_OPERAND (rhs, 1)) == COMPONENT_REF
12898 0 : && integer_onep (DECL_SIZE (TREE_OPERAND (lhs, 1)))
12899 63161 : && integer_onep (DECL_SIZE (TREE_OPERAND (TREE_OPERAND (rhs, 1), 1))))
12900 : {
12901 0 : rtx_code_label *label = gen_label_rtx ();
12902 0 : int value = TREE_CODE (rhs) == BIT_IOR_EXPR;
12903 0 : profile_probability prob = profile_probability::uninitialized ();
12904 0 : if (value)
12905 0 : jumpifnot (TREE_OPERAND (rhs, 1), label, prob);
12906 : else
12907 0 : jumpif (TREE_OPERAND (rhs, 1), label, prob);
12908 0 : expand_assignment (lhs, build_int_cst (TREE_TYPE (rhs), value),
12909 : false);
12910 0 : do_pending_stack_adjust ();
12911 0 : emit_label (label);
12912 0 : return const0_rtx;
12913 : }
12914 :
12915 63161 : expand_assignment (lhs, rhs, false);
12916 63161 : return const0_rtx;
12917 : }
12918 :
12919 12937039 : case ADDR_EXPR:
12920 12937039 : return expand_expr_addr_expr (exp, target, tmode, modifier);
12921 :
12922 154258 : case REALPART_EXPR:
12923 154258 : op0 = expand_normal (treeop0);
12924 154258 : return read_complex_part (op0, false);
12925 :
12926 177976 : case IMAGPART_EXPR:
12927 177976 : op0 = expand_normal (treeop0);
12928 177976 : return read_complex_part (op0, true);
12929 :
12930 0 : case RETURN_EXPR:
12931 0 : case LABEL_EXPR:
12932 0 : case GOTO_EXPR:
12933 0 : case SWITCH_EXPR:
12934 0 : case ASM_EXPR:
12935 : /* Expanded in cfgexpand.cc. */
12936 0 : gcc_unreachable ();
12937 :
12938 0 : case TRY_CATCH_EXPR:
12939 0 : case CATCH_EXPR:
12940 0 : case EH_FILTER_EXPR:
12941 0 : case TRY_FINALLY_EXPR:
12942 0 : case EH_ELSE_EXPR:
12943 : /* Lowered by tree-eh.cc. */
12944 0 : gcc_unreachable ();
12945 :
12946 0 : case WITH_CLEANUP_EXPR:
12947 0 : case CLEANUP_POINT_EXPR:
12948 0 : case TARGET_EXPR:
12949 0 : case CASE_LABEL_EXPR:
12950 0 : case VA_ARG_EXPR:
12951 0 : case BIND_EXPR:
12952 0 : case INIT_EXPR:
12953 0 : case CONJ_EXPR:
12954 0 : case COMPOUND_EXPR:
12955 0 : case PREINCREMENT_EXPR:
12956 0 : case PREDECREMENT_EXPR:
12957 0 : case POSTINCREMENT_EXPR:
12958 0 : case POSTDECREMENT_EXPR:
12959 0 : case LOOP_EXPR:
12960 0 : case EXIT_EXPR:
12961 0 : case COMPOUND_LITERAL_EXPR:
12962 : /* Lowered by gimplify.cc. */
12963 0 : gcc_unreachable ();
12964 :
12965 0 : case FDESC_EXPR:
12966 : /* Function descriptors are not valid except for as
12967 : initialization constants, and should not be expanded. */
12968 0 : gcc_unreachable ();
12969 :
12970 258 : case WITH_SIZE_EXPR:
12971 : /* WITH_SIZE_EXPR expands to its first argument. The caller should
12972 : have pulled out the size to use in whatever context it needed. */
12973 258 : return expand_expr_real (treeop0, original_target, tmode,
12974 258 : modifier, alt_rtl, inner_reference_p);
12975 :
12976 1814418 : default:
12977 1814418 : return expand_expr_real_2 (&ops, target, tmode, modifier);
12978 : }
12979 : }
12980 : #undef EXTEND_BITINT
12981 :
12982 : /* Subroutine of above: reduce EXP to the precision of TYPE (in the
12983 : signedness of TYPE), possibly returning the result in TARGET.
12984 : TYPE is known to be a partial integer type. */
12985 : static rtx
12986 85692 : reduce_to_bit_field_precision (rtx exp, rtx target, tree type)
12987 : {
12988 85692 : scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
12989 85692 : HOST_WIDE_INT prec = TYPE_PRECISION (type);
12990 85692 : gcc_assert ((GET_MODE (exp) == VOIDmode || GET_MODE (exp) == mode)
12991 : && (!target || GET_MODE (target) == mode));
12992 :
12993 : /* For constant values, reduce using wide_int_to_tree. */
12994 85692 : if (poly_int_rtx_p (exp))
12995 : {
12996 0 : auto value = wi::to_poly_wide (exp, mode);
12997 0 : tree t = wide_int_to_tree (type, value);
12998 0 : return expand_expr (t, target, VOIDmode, EXPAND_NORMAL);
12999 : }
13000 85692 : else if (TYPE_UNSIGNED (type))
13001 : {
13002 66754 : rtx mask = immed_wide_int_const
13003 66754 : (wi::mask (prec, false, GET_MODE_PRECISION (mode)), mode);
13004 66754 : return expand_and (mode, exp, mask, target);
13005 : }
13006 : else
13007 : {
13008 18938 : int count = GET_MODE_PRECISION (mode) - prec;
13009 18938 : exp = expand_shift (LSHIFT_EXPR, mode, exp, count, target, 0);
13010 18938 : return expand_shift (RSHIFT_EXPR, mode, exp, count, target, 0);
13011 : }
13012 : }
13013 :
13014 : /* Subroutine of above: returns true if OFFSET corresponds to an offset that
13015 : when applied to the address of EXP produces an address known to be
13016 : aligned more than BIGGEST_ALIGNMENT. */
13017 :
13018 : static bool
13019 275782 : is_aligning_offset (const_tree offset, const_tree exp)
13020 : {
13021 : /* Strip off any conversions. */
13022 291952 : while (CONVERT_EXPR_P (offset))
13023 16170 : offset = TREE_OPERAND (offset, 0);
13024 :
13025 : /* We must now have a BIT_AND_EXPR with a constant that is one less than
13026 : power of 2 and which is larger than BIGGEST_ALIGNMENT. */
13027 275782 : if (TREE_CODE (offset) != BIT_AND_EXPR
13028 0 : || !tree_fits_uhwi_p (TREE_OPERAND (offset, 1))
13029 0 : || compare_tree_int (TREE_OPERAND (offset, 1),
13030 0 : BIGGEST_ALIGNMENT / BITS_PER_UNIT) <= 0
13031 275782 : || !pow2p_hwi (tree_to_uhwi (TREE_OPERAND (offset, 1)) + 1))
13032 275782 : return false;
13033 :
13034 : /* Look at the first operand of BIT_AND_EXPR and strip any conversion.
13035 : It must be NEGATE_EXPR. Then strip any more conversions. */
13036 0 : offset = TREE_OPERAND (offset, 0);
13037 0 : while (CONVERT_EXPR_P (offset))
13038 0 : offset = TREE_OPERAND (offset, 0);
13039 :
13040 0 : if (TREE_CODE (offset) != NEGATE_EXPR)
13041 : return false;
13042 :
13043 0 : offset = TREE_OPERAND (offset, 0);
13044 0 : while (CONVERT_EXPR_P (offset))
13045 0 : offset = TREE_OPERAND (offset, 0);
13046 :
13047 : /* This must now be the address of EXP. */
13048 0 : return TREE_CODE (offset) == ADDR_EXPR && TREE_OPERAND (offset, 0) == exp;
13049 : }
13050 :
13051 : /* Return a STRING_CST corresponding to ARG's constant initializer either
13052 : if it's a string constant, or, when VALREP is set, any other constant,
13053 : or null otherwise.
13054 : On success, set *PTR_OFFSET to the (possibly non-constant) byte offset
13055 : within the byte string that ARG is references. If nonnull set *MEM_SIZE
13056 : to the size of the byte string. If nonnull, set *DECL to the constant
13057 : declaration ARG refers to. */
13058 :
13059 : static tree
13060 15369546 : constant_byte_string (tree arg, tree *ptr_offset, tree *mem_size, tree *decl,
13061 : bool valrep = false)
13062 : {
13063 15369546 : tree dummy = NULL_TREE;
13064 15369546 : if (!mem_size)
13065 10893 : mem_size = &dummy;
13066 :
13067 : /* Store the type of the original expression before conversions
13068 : via NOP_EXPR or POINTER_PLUS_EXPR to other types have been
13069 : removed. */
13070 15369546 : tree argtype = TREE_TYPE (arg);
13071 :
13072 15369546 : tree array;
13073 15369546 : STRIP_NOPS (arg);
13074 :
13075 : /* Non-constant index into the character array in an ARRAY_REF
13076 : expression or null. */
13077 15369546 : tree varidx = NULL_TREE;
13078 :
13079 15369546 : poly_int64 base_off = 0;
13080 :
13081 15369546 : if (TREE_CODE (arg) == ADDR_EXPR)
13082 : {
13083 6330952 : arg = TREE_OPERAND (arg, 0);
13084 6330952 : tree ref = arg;
13085 6330952 : if (TREE_CODE (arg) == ARRAY_REF)
13086 : {
13087 560821 : tree idx = TREE_OPERAND (arg, 1);
13088 560821 : if (TREE_CODE (idx) != INTEGER_CST)
13089 : {
13090 : /* From a pointer (but not array) argument extract the variable
13091 : index to prevent get_addr_base_and_unit_offset() from failing
13092 : due to it. Use it later to compute the non-constant offset
13093 : into the string and return it to the caller. */
13094 166893 : varidx = idx;
13095 166893 : ref = TREE_OPERAND (arg, 0);
13096 :
13097 166893 : if (TREE_CODE (TREE_TYPE (arg)) == ARRAY_TYPE)
13098 : return NULL_TREE;
13099 :
13100 28087 : if (!integer_zerop (array_ref_low_bound (arg)))
13101 : return NULL_TREE;
13102 :
13103 27325 : if (!integer_onep (array_ref_element_size (arg)))
13104 : return NULL_TREE;
13105 : }
13106 : }
13107 6190309 : array = get_addr_base_and_unit_offset (ref, &base_off);
13108 6190309 : if (!array
13109 6142893 : || (TREE_CODE (array) != VAR_DECL
13110 6142893 : && TREE_CODE (array) != CONST_DECL
13111 3852423 : && TREE_CODE (array) != STRING_CST))
13112 : return NULL_TREE;
13113 : }
13114 9038594 : else if (TREE_CODE (arg) == PLUS_EXPR || TREE_CODE (arg) == POINTER_PLUS_EXPR)
13115 : {
13116 26447 : tree arg0 = TREE_OPERAND (arg, 0);
13117 26447 : tree arg1 = TREE_OPERAND (arg, 1);
13118 :
13119 26447 : tree offset;
13120 26447 : tree str = string_constant (arg0, &offset, mem_size, decl);
13121 26447 : if (!str)
13122 : {
13123 17143 : str = string_constant (arg1, &offset, mem_size, decl);
13124 17143 : arg1 = arg0;
13125 : }
13126 :
13127 17143 : if (str)
13128 : {
13129 : /* Avoid pointers to arrays (see bug 86622). */
13130 9304 : if (POINTER_TYPE_P (TREE_TYPE (arg))
13131 9304 : && TREE_CODE (TREE_TYPE (TREE_TYPE (arg))) == ARRAY_TYPE
13132 1890 : && !(decl && !*decl)
13133 11824 : && !(decl && tree_fits_uhwi_p (DECL_SIZE_UNIT (*decl))
13134 1260 : && tree_fits_uhwi_p (*mem_size)
13135 1260 : && tree_int_cst_equal (*mem_size, DECL_SIZE_UNIT (*decl))))
13136 1890 : return NULL_TREE;
13137 :
13138 7414 : tree type = TREE_TYPE (offset);
13139 7414 : arg1 = fold_convert (type, arg1);
13140 7414 : *ptr_offset = fold_build2 (PLUS_EXPR, type, offset, arg1);
13141 7414 : return str;
13142 : }
13143 : return NULL_TREE;
13144 : }
13145 9012147 : else if (TREE_CODE (arg) == SSA_NAME)
13146 : {
13147 3865358 : gimple *stmt = SSA_NAME_DEF_STMT (arg);
13148 3865358 : if (!is_gimple_assign (stmt))
13149 : return NULL_TREE;
13150 :
13151 1021008 : tree rhs1 = gimple_assign_rhs1 (stmt);
13152 1021008 : tree_code code = gimple_assign_rhs_code (stmt);
13153 1021008 : if (code == ADDR_EXPR)
13154 232437 : return string_constant (rhs1, ptr_offset, mem_size, decl);
13155 788571 : else if (code != POINTER_PLUS_EXPR)
13156 : return NULL_TREE;
13157 :
13158 189122 : tree offset;
13159 189122 : if (tree str = string_constant (rhs1, &offset, mem_size, decl))
13160 : {
13161 : /* Avoid pointers to arrays (see bug 86622). */
13162 21608 : if (POINTER_TYPE_P (TREE_TYPE (rhs1))
13163 21608 : && TREE_CODE (TREE_TYPE (TREE_TYPE (rhs1))) == ARRAY_TYPE
13164 18911 : && !(decl && !*decl)
13165 37904 : && !(decl && tree_fits_uhwi_p (DECL_SIZE_UNIT (*decl))
13166 8148 : && tree_fits_uhwi_p (*mem_size)
13167 8148 : && tree_int_cst_equal (*mem_size, DECL_SIZE_UNIT (*decl))))
13168 15979 : return NULL_TREE;
13169 :
13170 5629 : tree rhs2 = gimple_assign_rhs2 (stmt);
13171 5629 : tree type = TREE_TYPE (offset);
13172 5629 : rhs2 = fold_convert (type, rhs2);
13173 5629 : *ptr_offset = fold_build2 (PLUS_EXPR, type, offset, rhs2);
13174 5629 : return str;
13175 : }
13176 : return NULL_TREE;
13177 : }
13178 5146789 : else if (DECL_P (arg))
13179 : array = arg;
13180 : else
13181 : return NULL_TREE;
13182 :
13183 8511402 : tree offset = wide_int_to_tree (sizetype, base_off);
13184 8511402 : if (varidx)
13185 : {
13186 2256 : if (TREE_CODE (TREE_TYPE (array)) != ARRAY_TYPE)
13187 : return NULL_TREE;
13188 :
13189 1745 : gcc_assert (TREE_CODE (arg) == ARRAY_REF);
13190 1745 : tree chartype = TREE_TYPE (TREE_TYPE (TREE_OPERAND (arg, 0)));
13191 1745 : if (TREE_CODE (chartype) != INTEGER_TYPE)
13192 : return NULL;
13193 :
13194 1589 : offset = fold_convert (sizetype, varidx);
13195 : }
13196 :
13197 8510735 : if (TREE_CODE (array) == STRING_CST)
13198 : {
13199 3470085 : *ptr_offset = fold_convert (sizetype, offset);
13200 3470085 : *mem_size = TYPE_SIZE_UNIT (TREE_TYPE (array));
13201 3470085 : if (decl)
13202 765291 : *decl = NULL_TREE;
13203 3470085 : gcc_checking_assert (tree_to_shwi (TYPE_SIZE_UNIT (TREE_TYPE (array)))
13204 : >= TREE_STRING_LENGTH (array));
13205 : return array;
13206 : }
13207 :
13208 5040650 : tree init = ctor_for_folding (array);
13209 5040650 : if (!init || init == error_mark_node)
13210 : return NULL_TREE;
13211 :
13212 157663 : if (valrep)
13213 : {
13214 57836 : HOST_WIDE_INT cstoff;
13215 57836 : if (!base_off.is_constant (&cstoff))
13216 : return NULL_TREE;
13217 :
13218 : /* Check that the host and target are sane. */
13219 57836 : if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
13220 : return NULL_TREE;
13221 :
13222 57836 : HOST_WIDE_INT typesz = int_size_in_bytes (TREE_TYPE (init));
13223 57836 : if (typesz <= 0 || (int) typesz != typesz)
13224 : return NULL_TREE;
13225 :
13226 57683 : HOST_WIDE_INT size = typesz;
13227 57683 : if (VAR_P (array)
13228 57671 : && DECL_SIZE_UNIT (array)
13229 115354 : && tree_fits_shwi_p (DECL_SIZE_UNIT (array)))
13230 : {
13231 57671 : size = tree_to_shwi (DECL_SIZE_UNIT (array));
13232 57671 : gcc_checking_assert (size >= typesz);
13233 : }
13234 :
13235 : /* If value representation was requested convert the initializer
13236 : for the whole array or object into a string of bytes forming
13237 : its value representation and return it. */
13238 57683 : unsigned char *bytes = XNEWVEC (unsigned char, size);
13239 57683 : int r = native_encode_initializer (init, bytes, size);
13240 57683 : if (r < typesz)
13241 : {
13242 58 : XDELETEVEC (bytes);
13243 58 : return NULL_TREE;
13244 : }
13245 :
13246 57625 : if (r < size)
13247 0 : memset (bytes + r, '\0', size - r);
13248 :
13249 57625 : const char *p = reinterpret_cast<const char *>(bytes);
13250 57625 : init = build_string_literal (size, p, char_type_node);
13251 57625 : init = TREE_OPERAND (init, 0);
13252 57625 : init = TREE_OPERAND (init, 0);
13253 57625 : XDELETE (bytes);
13254 :
13255 57625 : *mem_size = size_int (TREE_STRING_LENGTH (init));
13256 57625 : *ptr_offset = wide_int_to_tree (ssizetype, base_off);
13257 :
13258 57625 : if (decl)
13259 0 : *decl = array;
13260 :
13261 57625 : return init;
13262 : }
13263 :
13264 99827 : if (TREE_CODE (init) == CONSTRUCTOR)
13265 : {
13266 : /* Convert the 64-bit constant offset to a wider type to avoid
13267 : overflow and use it to obtain the initializer for the subobject
13268 : it points into. */
13269 80266 : offset_int wioff;
13270 80266 : if (!base_off.is_constant (&wioff))
13271 155 : return NULL_TREE;
13272 :
13273 80266 : wioff *= BITS_PER_UNIT;
13274 80266 : if (!wi::fits_uhwi_p (wioff))
13275 : return NULL_TREE;
13276 :
13277 80158 : base_off = wioff.to_uhwi ();
13278 80158 : unsigned HOST_WIDE_INT fieldoff = 0;
13279 80158 : init = fold_ctor_reference (TREE_TYPE (arg), init, base_off, 0, array,
13280 : &fieldoff);
13281 80158 : if (!init || init == error_mark_node)
13282 : return NULL_TREE;
13283 :
13284 80111 : HOST_WIDE_INT cstoff;
13285 80111 : if (!base_off.is_constant (&cstoff))
13286 : return NULL_TREE;
13287 :
13288 80111 : cstoff = (cstoff - fieldoff) / BITS_PER_UNIT;
13289 80111 : tree off = build_int_cst (sizetype, cstoff);
13290 80111 : if (varidx)
13291 944 : offset = fold_build2 (PLUS_EXPR, TREE_TYPE (offset), offset, off);
13292 : else
13293 : offset = off;
13294 : }
13295 :
13296 99672 : *ptr_offset = offset;
13297 :
13298 99672 : tree inittype = TREE_TYPE (init);
13299 :
13300 99672 : if (TREE_CODE (init) == INTEGER_CST
13301 99672 : && (TREE_CODE (TREE_TYPE (array)) == INTEGER_TYPE
13302 1103 : || TYPE_MAIN_VARIANT (inittype) == char_type_node))
13303 : {
13304 : /* Check that the host and target are sane. */
13305 905 : if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
13306 : return NULL_TREE;
13307 :
13308 : /* For a reference to (address of) a single constant character,
13309 : store the native representation of the character in CHARBUF.
13310 : If the reference is to an element of an array or a member
13311 : of a struct, only consider narrow characters until ctors
13312 : for wide character arrays are transformed to STRING_CSTs
13313 : like those for narrow arrays. */
13314 905 : unsigned char charbuf[MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT];
13315 905 : int len = native_encode_expr (init, charbuf, sizeof charbuf, 0);
13316 905 : if (len > 0)
13317 : {
13318 : /* Construct a string literal with elements of INITTYPE and
13319 : the representation above. Then strip
13320 : the ADDR_EXPR (ARRAY_REF (...)) around the STRING_CST. */
13321 905 : init = build_string_literal (len, (char *)charbuf, inittype);
13322 905 : init = TREE_OPERAND (TREE_OPERAND (init, 0), 0);
13323 : }
13324 : }
13325 :
13326 99672 : tree initsize = TYPE_SIZE_UNIT (inittype);
13327 :
13328 99672 : if (TREE_CODE (init) == CONSTRUCTOR && initializer_zerop (init))
13329 : {
13330 : /* Fold an empty/zero constructor for an implicitly initialized
13331 : object or subobject into the empty string. */
13332 :
13333 : /* Determine the character type from that of the original
13334 : expression. */
13335 9330 : tree chartype = argtype;
13336 9330 : if (POINTER_TYPE_P (chartype))
13337 9323 : chartype = TREE_TYPE (chartype);
13338 14944 : while (TREE_CODE (chartype) == ARRAY_TYPE)
13339 5614 : chartype = TREE_TYPE (chartype);
13340 :
13341 9330 : if (INTEGRAL_TYPE_P (chartype)
13342 9330 : && TYPE_PRECISION (chartype) == TYPE_PRECISION (char_type_node))
13343 : {
13344 : /* Convert a char array to an empty STRING_CST having an array
13345 : of the expected type and size. */
13346 9133 : if (!initsize)
13347 3090 : initsize = integer_zero_node;
13348 6043 : else if (!tree_fits_uhwi_p (initsize))
13349 : return NULL_TREE;
13350 :
13351 9127 : unsigned HOST_WIDE_INT size = tree_to_uhwi (initsize);
13352 9127 : if (size > (unsigned HOST_WIDE_INT) INT_MAX)
13353 : return NULL_TREE;
13354 :
13355 9127 : init = build_string_literal (size, NULL, chartype, size);
13356 9127 : init = TREE_OPERAND (init, 0);
13357 9127 : init = TREE_OPERAND (init, 0);
13358 :
13359 9127 : *ptr_offset = integer_zero_node;
13360 : }
13361 : }
13362 :
13363 99666 : if (decl)
13364 54191 : *decl = array;
13365 :
13366 99666 : if (TREE_CODE (init) != STRING_CST)
13367 : return NULL_TREE;
13368 :
13369 68357 : *mem_size = initsize;
13370 :
13371 68357 : gcc_checking_assert (tree_to_shwi (initsize) >= TREE_STRING_LENGTH (init));
13372 :
13373 : return init;
13374 : }
13375 :
13376 : /* Return STRING_CST if an ARG corresponds to a string constant or zero
13377 : if it doesn't. If we return nonzero, set *PTR_OFFSET to the (possibly
13378 : non-constant) offset in bytes within the string that ARG is accessing.
13379 : If MEM_SIZE is non-zero the storage size of the memory is returned.
13380 : If DECL is non-zero the constant declaration is returned if available. */
13381 :
13382 : tree
13383 10720014 : string_constant (tree arg, tree *ptr_offset, tree *mem_size, tree *decl)
13384 : {
13385 10720014 : return constant_byte_string (arg, ptr_offset, mem_size, decl, false);
13386 : }
13387 :
13388 : /* Similar to string_constant, return a STRING_CST corresponding
13389 : to the value representation of the first argument if it's
13390 : a constant. */
13391 :
13392 : tree
13393 4649532 : byte_representation (tree arg, tree *ptr_offset, tree *mem_size, tree *decl)
13394 : {
13395 4649532 : return constant_byte_string (arg, ptr_offset, mem_size, decl, true);
13396 : }
13397 :
13398 : /* Optimize x % C1 == C2 for signed modulo if C1 is a power of two and C2
13399 : is non-zero and C3 ((1<<(prec-1)) | (C1 - 1)):
13400 : for C2 > 0 to x & C3 == C2
13401 : for C2 < 0 to x & C3 == (C2 & C3). */
13402 : enum tree_code
13403 35 : maybe_optimize_pow2p_mod_cmp (enum tree_code code, tree *arg0, tree *arg1)
13404 : {
13405 35 : gimple *stmt = get_def_for_expr (*arg0, TRUNC_MOD_EXPR);
13406 35 : tree treeop0 = gimple_assign_rhs1 (stmt);
13407 35 : tree treeop1 = gimple_assign_rhs2 (stmt);
13408 35 : tree type = TREE_TYPE (*arg0);
13409 35 : scalar_int_mode mode;
13410 35 : if (!is_a <scalar_int_mode> (TYPE_MODE (type), &mode))
13411 : return code;
13412 70 : if (GET_MODE_BITSIZE (mode) != TYPE_PRECISION (type)
13413 35 : || TYPE_PRECISION (type) <= 1
13414 35 : || TYPE_UNSIGNED (type)
13415 : /* Signed x % c == 0 should have been optimized into unsigned modulo
13416 : earlier. */
13417 35 : || integer_zerop (*arg1)
13418 : /* If c is known to be non-negative, modulo will be expanded as unsigned
13419 : modulo. */
13420 70 : || get_range_pos_neg (treeop0, currently_expanding_gimple_stmt) == 1)
13421 0 : return code;
13422 :
13423 : /* x % c == d where d < 0 && d <= -c should be always false. */
13424 35 : if (tree_int_cst_sgn (*arg1) == -1
13425 51 : && -wi::to_widest (treeop1) >= wi::to_widest (*arg1))
13426 : return code;
13427 :
13428 35 : int prec = TYPE_PRECISION (type);
13429 35 : wide_int w = wi::to_wide (treeop1) - 1;
13430 35 : w |= wi::shifted_mask (0, prec - 1, true, prec);
13431 35 : tree c3 = wide_int_to_tree (type, w);
13432 35 : tree c4 = *arg1;
13433 35 : if (tree_int_cst_sgn (*arg1) == -1)
13434 16 : c4 = wide_int_to_tree (type, w & wi::to_wide (*arg1));
13435 :
13436 35 : rtx op0 = expand_normal (treeop0);
13437 35 : treeop0 = make_tree (TREE_TYPE (treeop0), op0);
13438 :
13439 35 : bool speed_p = optimize_insn_for_speed_p ();
13440 :
13441 35 : do_pending_stack_adjust ();
13442 :
13443 35 : location_t loc = gimple_location (stmt);
13444 35 : struct separate_ops ops;
13445 35 : ops.code = TRUNC_MOD_EXPR;
13446 35 : ops.location = loc;
13447 35 : ops.type = TREE_TYPE (treeop0);
13448 35 : ops.op0 = treeop0;
13449 35 : ops.op1 = treeop1;
13450 35 : ops.op2 = NULL_TREE;
13451 35 : start_sequence ();
13452 35 : rtx mor = expand_expr_real_2 (&ops, NULL_RTX, TYPE_MODE (ops.type),
13453 : EXPAND_NORMAL);
13454 35 : rtx_insn *moinsns = end_sequence ();
13455 :
13456 35 : unsigned mocost = seq_cost (moinsns, speed_p);
13457 35 : mocost += rtx_cost (mor, mode, EQ, 0, speed_p);
13458 35 : mocost += rtx_cost (expand_normal (*arg1), mode, EQ, 1, speed_p);
13459 :
13460 35 : ops.code = BIT_AND_EXPR;
13461 35 : ops.location = loc;
13462 35 : ops.type = TREE_TYPE (treeop0);
13463 35 : ops.op0 = treeop0;
13464 35 : ops.op1 = c3;
13465 35 : ops.op2 = NULL_TREE;
13466 35 : start_sequence ();
13467 35 : rtx mur = expand_expr_real_2 (&ops, NULL_RTX, TYPE_MODE (ops.type),
13468 : EXPAND_NORMAL);
13469 35 : rtx_insn *muinsns = end_sequence ();
13470 :
13471 35 : unsigned mucost = seq_cost (muinsns, speed_p);
13472 35 : mucost += rtx_cost (mur, mode, EQ, 0, speed_p);
13473 35 : mucost += rtx_cost (expand_normal (c4), mode, EQ, 1, speed_p);
13474 :
13475 35 : if (mocost <= mucost)
13476 : {
13477 0 : emit_insn (moinsns);
13478 0 : *arg0 = make_tree (TREE_TYPE (*arg0), mor);
13479 0 : return code;
13480 : }
13481 :
13482 35 : emit_insn (muinsns);
13483 35 : *arg0 = make_tree (TREE_TYPE (*arg0), mur);
13484 35 : *arg1 = c4;
13485 35 : return code;
13486 35 : }
13487 :
13488 : /* Attempt to optimize unsigned (X % C1) == C2 (or (X % C1) != C2).
13489 : If C1 is odd to:
13490 : (X - C2) * C3 <= C4 (or >), where
13491 : C3 is modular multiplicative inverse of C1 and 1<<prec and
13492 : C4 is ((1<<prec) - 1) / C1 or ((1<<prec) - 1) / C1 - 1 (the latter
13493 : if C2 > ((1<<prec) - 1) % C1).
13494 : If C1 is even, S = ctz (C1) and C2 is 0, use
13495 : ((X * C3) r>> S) <= C4, where C3 is modular multiplicative
13496 : inverse of C1>>S and 1<<prec and C4 is (((1<<prec) - 1) / (C1>>S)) >> S.
13497 :
13498 : For signed (X % C1) == 0 if C1 is odd to (all operations in it
13499 : unsigned):
13500 : (X * C3) + C4 <= 2 * C4, where
13501 : C3 is modular multiplicative inverse of (unsigned) C1 and 1<<prec and
13502 : C4 is ((1<<(prec - 1) - 1) / C1).
13503 : If C1 is even, S = ctz(C1), use
13504 : ((X * C3) + C4) r>> S <= (C4 >> (S - 1))
13505 : where C3 is modular multiplicative inverse of (unsigned)(C1>>S) and 1<<prec
13506 : and C4 is ((1<<(prec - 1) - 1) / (C1>>S)) & (-1<<S).
13507 :
13508 : See the Hacker's Delight book, section 10-17. */
13509 : enum tree_code
13510 3266213 : maybe_optimize_mod_cmp (enum tree_code code, tree *arg0, tree *arg1)
13511 : {
13512 3266213 : gcc_checking_assert (code == EQ_EXPR || code == NE_EXPR);
13513 3266213 : gcc_checking_assert (TREE_CODE (*arg1) == INTEGER_CST);
13514 :
13515 3266213 : if (optimize < 2)
13516 : return code;
13517 :
13518 2464738 : gimple *stmt = get_def_for_expr (*arg0, TRUNC_MOD_EXPR);
13519 2464738 : if (stmt == NULL)
13520 : return code;
13521 :
13522 2411 : tree treeop0 = gimple_assign_rhs1 (stmt);
13523 2411 : tree treeop1 = gimple_assign_rhs2 (stmt);
13524 2411 : if (TREE_CODE (treeop0) != SSA_NAME
13525 2313 : || TREE_CODE (treeop1) != INTEGER_CST
13526 : /* Don't optimize the undefined behavior case x % 0;
13527 : x % 1 should have been optimized into zero, punt if
13528 : it makes it here for whatever reason;
13529 : x % -c should have been optimized into x % c. */
13530 1958 : || compare_tree_int (treeop1, 2) <= 0
13531 : /* Likewise x % c == d where d >= c should be always false. */
13532 4293 : || tree_int_cst_le (treeop1, *arg1))
13533 529 : return code;
13534 :
13535 : /* Unsigned x % pow2 is handled right already, for signed
13536 : modulo handle it in maybe_optimize_pow2p_mod_cmp. */
13537 1882 : if (integer_pow2p (treeop1))
13538 35 : return maybe_optimize_pow2p_mod_cmp (code, arg0, arg1);
13539 :
13540 1847 : tree type = TREE_TYPE (*arg0);
13541 1847 : scalar_int_mode mode;
13542 3266185 : if (!is_a <scalar_int_mode> (TYPE_MODE (type), &mode))
13543 : return code;
13544 3694 : if (GET_MODE_BITSIZE (mode) != TYPE_PRECISION (type)
13545 1847 : || TYPE_PRECISION (type) <= 1)
13546 : return code;
13547 :
13548 1847 : signop sgn = UNSIGNED;
13549 : /* If both operands are known to have the sign bit clear, handle
13550 : even the signed modulo case as unsigned. treeop1 is always
13551 : positive >= 2, checked above. */
13552 1847 : if (!TYPE_UNSIGNED (type)
13553 1847 : && get_range_pos_neg (treeop0, currently_expanding_gimple_stmt) != 1)
13554 : sgn = SIGNED;
13555 :
13556 1847 : if (!TYPE_UNSIGNED (type))
13557 : {
13558 1619 : if (tree_int_cst_sgn (*arg1) == -1)
13559 : return code;
13560 1612 : type = unsigned_type_for (type);
13561 1612 : if (!type || TYPE_MODE (type) != TYPE_MODE (TREE_TYPE (*arg0)))
13562 0 : return code;
13563 : }
13564 :
13565 1840 : int prec = TYPE_PRECISION (type);
13566 1840 : wide_int w = wi::to_wide (treeop1);
13567 1840 : int shift = wi::ctz (w);
13568 : /* Unsigned (X % C1) == C2 is equivalent to (X - C2) % C1 == 0 if
13569 : C2 <= -1U % C1, because for any Z >= 0U - C2 in that case (Z % C1) != 0.
13570 : If C1 is odd, we can handle all cases by subtracting
13571 : C4 below. We could handle even the even C1 and C2 > -1U % C1 cases
13572 : e.g. by testing for overflow on the subtraction, punt on that for now
13573 : though. */
13574 1840 : if ((sgn == SIGNED || shift) && !integer_zerop (*arg1))
13575 : {
13576 189 : if (sgn == SIGNED)
13577 174 : return code;
13578 26 : wide_int x = wi::umod_trunc (wi::mask (prec, false, prec), w);
13579 26 : if (wi::gtu_p (wi::to_wide (*arg1), x))
13580 11 : return code;
13581 26 : }
13582 :
13583 1666 : imm_use_iterator imm_iter;
13584 1666 : use_operand_p use_p;
13585 12371 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, treeop0)
13586 : {
13587 9049 : gimple *use_stmt = USE_STMT (use_p);
13588 : /* Punt if treeop0 is used in the same bb in a division
13589 : or another modulo with the same divisor. We should expect
13590 : the division and modulo combined together. */
13591 17682 : if (use_stmt == stmt
13592 9049 : || gimple_bb (use_stmt) != gimple_bb (stmt))
13593 8633 : continue;
13594 416 : if (!is_gimple_assign (use_stmt)
13595 416 : || (gimple_assign_rhs_code (use_stmt) != TRUNC_DIV_EXPR
13596 400 : && gimple_assign_rhs_code (use_stmt) != TRUNC_MOD_EXPR))
13597 397 : continue;
13598 19 : if (gimple_assign_rhs1 (use_stmt) != treeop0
13599 19 : || !operand_equal_p (gimple_assign_rhs2 (use_stmt), treeop1, 0))
13600 9 : continue;
13601 10 : return code;
13602 10 : }
13603 :
13604 1656 : w = wi::lrshift (w, shift);
13605 1656 : wide_int a = wide_int::from (w, prec + 1, UNSIGNED);
13606 1656 : wide_int b = wi::shifted_mask (prec, 1, false, prec + 1);
13607 1656 : wide_int m = wide_int::from (wi::mod_inv (a, b), prec, UNSIGNED);
13608 1656 : tree c3 = wide_int_to_tree (type, m);
13609 1656 : tree c5 = NULL_TREE;
13610 1656 : wide_int d, e;
13611 1656 : if (sgn == UNSIGNED)
13612 : {
13613 1063 : d = wi::divmod_trunc (wi::mask (prec, false, prec), w, UNSIGNED, &e);
13614 : /* Use <= floor ((1<<prec) - 1) / C1 only if C2 <= ((1<<prec) - 1) % C1,
13615 : otherwise use < or subtract one from C4. E.g. for
13616 : x % 3U == 0 we transform this into x * 0xaaaaaaab <= 0x55555555, but
13617 : x % 3U == 1 already needs to be
13618 : (x - 1) * 0xaaaaaaabU <= 0x55555554. */
13619 1063 : if (!shift && wi::gtu_p (wi::to_wide (*arg1), e))
13620 15 : d -= 1;
13621 1063 : if (shift)
13622 442 : d = wi::lrshift (d, shift);
13623 : }
13624 : else
13625 : {
13626 593 : e = wi::udiv_trunc (wi::mask (prec - 1, false, prec), w);
13627 593 : if (!shift)
13628 402 : d = wi::lshift (e, 1);
13629 : else
13630 : {
13631 191 : e = wi::bit_and (e, wi::mask (shift, true, prec));
13632 191 : d = wi::lrshift (e, shift - 1);
13633 : }
13634 593 : c5 = wide_int_to_tree (type, e);
13635 : }
13636 1656 : tree c4 = wide_int_to_tree (type, d);
13637 :
13638 1656 : rtx op0 = expand_normal (treeop0);
13639 1656 : treeop0 = make_tree (TREE_TYPE (treeop0), op0);
13640 :
13641 1656 : bool speed_p = optimize_insn_for_speed_p ();
13642 :
13643 1656 : do_pending_stack_adjust ();
13644 :
13645 1656 : location_t loc = gimple_location (stmt);
13646 1656 : struct separate_ops ops;
13647 1656 : ops.code = TRUNC_MOD_EXPR;
13648 1656 : ops.location = loc;
13649 1656 : ops.type = TREE_TYPE (treeop0);
13650 1656 : ops.op0 = treeop0;
13651 1656 : ops.op1 = treeop1;
13652 1656 : ops.op2 = NULL_TREE;
13653 1656 : start_sequence ();
13654 1656 : rtx mor = expand_expr_real_2 (&ops, NULL_RTX, TYPE_MODE (ops.type),
13655 : EXPAND_NORMAL);
13656 1656 : rtx_insn *moinsns = end_sequence ();
13657 :
13658 1656 : unsigned mocost = seq_cost (moinsns, speed_p);
13659 1656 : mocost += rtx_cost (mor, mode, EQ, 0, speed_p);
13660 1656 : mocost += rtx_cost (expand_normal (*arg1), mode, EQ, 1, speed_p);
13661 :
13662 1656 : tree t = fold_convert_loc (loc, type, treeop0);
13663 1656 : if (!integer_zerop (*arg1))
13664 36 : t = fold_build2_loc (loc, MINUS_EXPR, type, t, fold_convert (type, *arg1));
13665 1656 : t = fold_build2_loc (loc, MULT_EXPR, type, t, c3);
13666 1656 : if (sgn == SIGNED)
13667 593 : t = fold_build2_loc (loc, PLUS_EXPR, type, t, c5);
13668 1656 : if (shift)
13669 : {
13670 633 : tree s = build_int_cst (NULL_TREE, shift);
13671 633 : t = fold_build2_loc (loc, RROTATE_EXPR, type, t, s);
13672 : }
13673 :
13674 1656 : start_sequence ();
13675 1656 : rtx mur = expand_normal (t);
13676 1656 : rtx_insn *muinsns = end_sequence ();
13677 :
13678 1656 : unsigned mucost = seq_cost (muinsns, speed_p);
13679 1656 : mucost += rtx_cost (mur, mode, LE, 0, speed_p);
13680 1656 : mucost += rtx_cost (expand_normal (c4), mode, LE, 1, speed_p);
13681 :
13682 1656 : if (mocost <= mucost)
13683 : {
13684 266 : emit_insn (moinsns);
13685 266 : *arg0 = make_tree (TREE_TYPE (*arg0), mor);
13686 266 : return code;
13687 : }
13688 :
13689 1390 : emit_insn (muinsns);
13690 1390 : *arg0 = make_tree (type, mur);
13691 1390 : *arg1 = c4;
13692 1390 : return code == EQ_EXPR ? LE_EXPR : GT_EXPR;
13693 3496 : }
13694 :
13695 : /* Optimize x - y < 0 into x < 0 if x - y has undefined overflow. */
13696 :
13697 : void
13698 239689 : maybe_optimize_sub_cmp_0 (enum tree_code code, tree *arg0, tree *arg1)
13699 : {
13700 239689 : gcc_checking_assert (code == GT_EXPR || code == GE_EXPR
13701 : || code == LT_EXPR || code == LE_EXPR);
13702 239689 : gcc_checking_assert (integer_zerop (*arg1));
13703 :
13704 239689 : if (!optimize)
13705 : return;
13706 :
13707 206318 : gimple *stmt = get_def_for_expr (*arg0, MINUS_EXPR);
13708 206318 : if (stmt == NULL)
13709 : return;
13710 :
13711 1370 : tree treeop0 = gimple_assign_rhs1 (stmt);
13712 1370 : tree treeop1 = gimple_assign_rhs2 (stmt);
13713 1370 : if (!TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (treeop0)))
13714 : return;
13715 :
13716 1273 : if (issue_strict_overflow_warning (WARN_STRICT_OVERFLOW_COMPARISON))
13717 1 : warning_at (gimple_location (stmt), OPT_Wstrict_overflow,
13718 : "assuming signed overflow does not occur when "
13719 : "simplifying %<X - Y %s 0%> to %<X %s Y%>",
13720 : op_symbol_code (code), op_symbol_code (code));
13721 :
13722 1273 : *arg0 = treeop0;
13723 1273 : *arg1 = treeop1;
13724 : }
13725 :
13726 :
13727 : /* Expand CODE with arguments INNER & (1<<BITNUM) and 0 that represents
13728 : a single bit equality/inequality test, returns where the result is located. */
13729 :
13730 : static rtx
13731 10120 : expand_single_bit_test (location_t loc, enum tree_code code,
13732 : tree inner, int bitnum,
13733 : tree result_type, rtx target,
13734 : machine_mode mode)
13735 : {
13736 10120 : gcc_assert (code == NE_EXPR || code == EQ_EXPR);
13737 :
13738 10120 : tree type = TREE_TYPE (inner);
13739 10120 : scalar_int_mode operand_mode = SCALAR_INT_TYPE_MODE (type);
13740 10120 : int ops_unsigned;
13741 10120 : tree signed_type, unsigned_type, intermediate_type;
13742 10120 : gimple *inner_def;
13743 :
13744 : /* First, see if we can fold the single bit test into a sign-bit
13745 : test. */
13746 10120 : if (bitnum == TYPE_PRECISION (type) - 1
13747 10120 : && type_has_mode_precision_p (type))
13748 : {
13749 243 : tree stype = signed_type_for (type);
13750 448 : tree tmp = fold_build2_loc (loc, code == EQ_EXPR ? GE_EXPR : LT_EXPR,
13751 : result_type,
13752 : fold_convert_loc (loc, stype, inner),
13753 : build_int_cst (stype, 0));
13754 243 : return expand_expr (tmp, target, VOIDmode, EXPAND_NORMAL);
13755 : }
13756 :
13757 : /* Otherwise we have (A & C) != 0 where C is a single bit,
13758 : convert that into ((A >> C2) & 1). Where C2 = log2(C).
13759 : Similarly for (A & C) == 0. */
13760 :
13761 : /* If INNER is a right shift of a constant and it plus BITNUM does
13762 : not overflow, adjust BITNUM and INNER. */
13763 9877 : if ((inner_def = get_def_for_expr (inner, RSHIFT_EXPR))
13764 93 : && TREE_CODE (gimple_assign_rhs2 (inner_def)) == INTEGER_CST
13765 16 : && bitnum < TYPE_PRECISION (type)
13766 9893 : && wi::ltu_p (wi::to_wide (gimple_assign_rhs2 (inner_def)),
13767 9877 : TYPE_PRECISION (type) - bitnum))
13768 : {
13769 16 : bitnum += tree_to_uhwi (gimple_assign_rhs2 (inner_def));
13770 16 : inner = gimple_assign_rhs1 (inner_def);
13771 : }
13772 :
13773 : /* If we are going to be able to omit the AND below, we must do our
13774 : operations as unsigned. If we must use the AND, we have a choice.
13775 : Normally unsigned is faster, but for some machines signed is. */
13776 9877 : ops_unsigned = (load_extend_op (operand_mode) == SIGN_EXTEND
13777 : && !flag_syntax_only) ? 0 : 1;
13778 :
13779 9877 : signed_type = lang_hooks.types.type_for_mode (operand_mode, 0);
13780 9877 : unsigned_type = lang_hooks.types.type_for_mode (operand_mode, 1);
13781 9877 : intermediate_type = ops_unsigned ? unsigned_type : signed_type;
13782 9877 : inner = fold_convert_loc (loc, intermediate_type, inner);
13783 :
13784 9877 : rtx inner0 = expand_expr (inner, NULL_RTX, VOIDmode, EXPAND_NORMAL);
13785 :
13786 9877 : if (CONST_SCALAR_INT_P (inner0))
13787 : {
13788 0 : wide_int t = rtx_mode_t (inner0, operand_mode);
13789 0 : bool setp = (wi::lrshift (t, bitnum) & 1) != 0;
13790 0 : return (setp ^ (code == EQ_EXPR)) ? const1_rtx : const0_rtx;
13791 0 : }
13792 9877 : int bitpos = bitnum;
13793 :
13794 9877 : if (BYTES_BIG_ENDIAN)
13795 : bitpos = GET_MODE_BITSIZE (operand_mode) - 1 - bitpos;
13796 :
13797 9877 : inner0 = extract_bit_field (inner0, 1, bitpos, 1, target,
13798 : operand_mode, mode, 0, NULL);
13799 :
13800 9877 : if (code == EQ_EXPR)
13801 3074 : inner0 = expand_binop (GET_MODE (inner0), xor_optab, inner0, const1_rtx,
13802 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
13803 9877 : if (GET_MODE (inner0) != mode)
13804 : {
13805 0 : rtx t = gen_reg_rtx (mode);
13806 0 : convert_move (t, inner0, 0);
13807 0 : return t;
13808 : }
13809 : return inner0;
13810 : }
13811 :
13812 : /* Generate code to calculate OPS, and exploded expression
13813 : using a store-flag instruction and return an rtx for the result.
13814 : OPS reflects a comparison.
13815 :
13816 : If TARGET is nonzero, store the result there if convenient.
13817 :
13818 : Return zero if there is no suitable set-flag instruction
13819 : available on this machine.
13820 :
13821 : Once expand_expr has been called on the arguments of the comparison,
13822 : we are committed to doing the store flag, since it is not safe to
13823 : re-evaluate the expression. We emit the store-flag insn by calling
13824 : emit_store_flag, but only expand the arguments if we have a reason
13825 : to believe that emit_store_flag will be successful. If we think that
13826 : it will, but it isn't, we have to simulate the store-flag with a
13827 : set/jump/set sequence. */
13828 :
13829 : static rtx
13830 592474 : do_store_flag (const_sepops ops, rtx target, machine_mode mode)
13831 : {
13832 592474 : enum rtx_code code;
13833 592474 : tree arg0, arg1, type;
13834 592474 : machine_mode operand_mode;
13835 592474 : int unsignedp;
13836 592474 : rtx op0, op1;
13837 592474 : rtx subtarget = target;
13838 592474 : location_t loc = ops->location;
13839 592474 : unsigned HOST_WIDE_INT nunits;
13840 :
13841 592474 : arg0 = ops->op0;
13842 592474 : arg1 = ops->op1;
13843 :
13844 : /* Don't crash if the comparison was erroneous. */
13845 592474 : if (arg0 == error_mark_node || arg1 == error_mark_node)
13846 0 : return const0_rtx;
13847 :
13848 592474 : type = TREE_TYPE (arg0);
13849 592474 : operand_mode = TYPE_MODE (type);
13850 592474 : unsignedp = TYPE_UNSIGNED (type);
13851 :
13852 : /* We won't bother with BLKmode store-flag operations because it would mean
13853 : passing a lot of information to emit_store_flag. */
13854 592474 : if (operand_mode == BLKmode)
13855 : return 0;
13856 :
13857 : /* We won't bother with store-flag operations involving function pointers
13858 : when function pointers must be canonicalized before comparisons. */
13859 592474 : if (targetm.have_canonicalize_funcptr_for_compare ()
13860 592474 : && ((POINTER_TYPE_P (TREE_TYPE (arg0))
13861 0 : && FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (arg0))))
13862 0 : || (POINTER_TYPE_P (TREE_TYPE (arg1))
13863 0 : && FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (arg1))))))
13864 : return 0;
13865 :
13866 592474 : STRIP_NOPS (arg0);
13867 592474 : STRIP_NOPS (arg1);
13868 :
13869 : /* For vector typed comparisons emit code to generate the desired
13870 : all-ones or all-zeros mask. */
13871 592474 : if (VECTOR_TYPE_P (ops->type))
13872 : {
13873 24105 : tree ifexp = build2 (ops->code, ops->type, arg0, arg1);
13874 24105 : if (VECTOR_BOOLEAN_TYPE_P (ops->type)
13875 48210 : && expand_vec_cmp_expr_p (TREE_TYPE (arg0), ops->type, ops->code))
13876 24105 : return expand_vec_cmp_expr (ops->type, ifexp, target);
13877 : else
13878 0 : gcc_unreachable ();
13879 : }
13880 :
13881 : /* Optimize (x % C1) == C2 or (x % C1) != C2 if it is beneficial
13882 : into (x - C2) * C3 < C4. */
13883 568369 : if ((ops->code == EQ_EXPR || ops->code == NE_EXPR)
13884 376927 : && TREE_CODE (arg0) == SSA_NAME
13885 376384 : && TREE_CODE (arg1) == INTEGER_CST)
13886 : {
13887 218321 : enum tree_code new_code = maybe_optimize_mod_cmp (ops->code,
13888 : &arg0, &arg1);
13889 218321 : if (new_code != ops->code)
13890 : {
13891 92 : struct separate_ops nops = *ops;
13892 92 : nops.code = new_code;
13893 92 : nops.op0 = arg0;
13894 92 : nops.op1 = arg1;
13895 92 : nops.type = TREE_TYPE (arg0);
13896 92 : return do_store_flag (&nops, target, mode);
13897 : }
13898 : }
13899 :
13900 : /* Optimize (x - y) < 0 into x < y if x - y has undefined overflow. */
13901 568277 : if (!unsignedp
13902 406160 : && (ops->code == LT_EXPR || ops->code == LE_EXPR
13903 406160 : || ops->code == GT_EXPR || ops->code == GE_EXPR)
13904 122243 : && integer_zerop (arg1)
13905 606336 : && TREE_CODE (arg0) == SSA_NAME)
13906 38055 : maybe_optimize_sub_cmp_0 (ops->code, &arg0, &arg1);
13907 :
13908 : /* Get the rtx comparison code to use. We know that EXP is a comparison
13909 : operation of some type. Some comparisons against 1 and -1 can be
13910 : converted to comparisons with zero. Do so here so that the tests
13911 : below will be aware that we have a comparison with zero. These
13912 : tests will not catch constants in the first operand, but constants
13913 : are rarely passed as the first operand. */
13914 :
13915 568277 : switch (ops->code)
13916 : {
13917 : case EQ_EXPR:
13918 : code = EQ;
13919 : break;
13920 225205 : case NE_EXPR:
13921 225205 : code = NE;
13922 225205 : break;
13923 38428 : case LT_EXPR:
13924 38428 : if (integer_onep (arg1))
13925 0 : arg1 = integer_zero_node, code = unsignedp ? LEU : LE;
13926 : else
13927 38428 : code = unsignedp ? LTU : LT;
13928 : break;
13929 32704 : case LE_EXPR:
13930 32704 : if (! unsignedp && integer_all_onesp (arg1))
13931 0 : arg1 = integer_zero_node, code = LT;
13932 : else
13933 32704 : code = unsignedp ? LEU : LE;
13934 : break;
13935 52336 : case GT_EXPR:
13936 52336 : if (! unsignedp && integer_all_onesp (arg1))
13937 0 : arg1 = integer_zero_node, code = GE;
13938 : else
13939 52336 : code = unsignedp ? GTU : GT;
13940 : break;
13941 44506 : case GE_EXPR:
13942 44506 : if (integer_onep (arg1))
13943 28679 : arg1 = integer_zero_node, code = unsignedp ? GTU : GT;
13944 : else
13945 44506 : code = unsignedp ? GEU : GE;
13946 : break;
13947 :
13948 693 : case UNORDERED_EXPR:
13949 693 : code = UNORDERED;
13950 693 : break;
13951 669 : case ORDERED_EXPR:
13952 669 : code = ORDERED;
13953 669 : break;
13954 477 : case UNLT_EXPR:
13955 477 : code = UNLT;
13956 477 : break;
13957 10778 : case UNLE_EXPR:
13958 10778 : code = UNLE;
13959 10778 : break;
13960 828 : case UNGT_EXPR:
13961 828 : code = UNGT;
13962 828 : break;
13963 9813 : case UNGE_EXPR:
13964 9813 : code = UNGE;
13965 9813 : break;
13966 136 : case UNEQ_EXPR:
13967 136 : code = UNEQ;
13968 136 : break;
13969 74 : case LTGT_EXPR:
13970 74 : code = LTGT;
13971 74 : break;
13972 :
13973 0 : default:
13974 0 : gcc_unreachable ();
13975 : }
13976 :
13977 : /* Put a constant second. */
13978 568277 : if (TREE_CODE (arg0) == REAL_CST || TREE_CODE (arg0) == INTEGER_CST
13979 566742 : || TREE_CODE (arg0) == FIXED_CST)
13980 : {
13981 1535 : std::swap (arg0, arg1);
13982 1535 : code = swap_condition (code);
13983 : }
13984 :
13985 : /* If this is an equality or inequality test of a single bit, we can
13986 : do this by shifting the bit being tested to the low-order bit and
13987 : masking the result with the constant 1. If the condition was EQ,
13988 : we xor it with 1. This does not require an scc insn and is faster
13989 : than an scc insn even if we have it. */
13990 :
13991 568277 : if ((code == NE || code == EQ)
13992 376835 : && (integer_zerop (arg1)
13993 228192 : || integer_pow2p (arg1))
13994 : /* vector types are not handled here. */
13995 177706 : && TREE_CODE (TREE_TYPE (arg1)) != VECTOR_TYPE
13996 745941 : && (TYPE_PRECISION (ops->type) != 1 || TYPE_UNSIGNED (ops->type)))
13997 : {
13998 177664 : tree narg0 = arg0;
13999 177664 : wide_int nz = tree_nonzero_bits (narg0);
14000 177664 : gimple *srcstmt = get_def_for_expr (narg0, BIT_AND_EXPR);
14001 : /* If the defining statement was (x & POW2), then use that instead of
14002 : the non-zero bits. */
14003 177664 : if (srcstmt && integer_pow2p (gimple_assign_rhs2 (srcstmt)))
14004 : {
14005 5754 : nz = wi::to_wide (gimple_assign_rhs2 (srcstmt));
14006 5754 : narg0 = gimple_assign_rhs1 (srcstmt);
14007 : }
14008 :
14009 177664 : if (wi::popcount (nz) == 1
14010 177664 : && (integer_zerop (arg1)
14011 167576 : || wi::to_wide (arg1) == nz))
14012 : {
14013 10120 : int bitnum = wi::exact_log2 (nz);
14014 10120 : enum tree_code tcode = EQ_EXPR;
14015 10120 : if ((code == NE) ^ !integer_zerop (arg1))
14016 7008 : tcode = NE_EXPR;
14017 :
14018 10120 : type = lang_hooks.types.type_for_mode (mode, unsignedp);
14019 10120 : return expand_single_bit_test (loc, tcode,
14020 : narg0,
14021 : bitnum, type, target, mode);
14022 : }
14023 177664 : }
14024 :
14025 :
14026 558157 : if (! get_subtarget (target)
14027 558157 : || GET_MODE (subtarget) != operand_mode)
14028 : subtarget = 0;
14029 :
14030 558157 : expand_operands (arg0, arg1, subtarget, &op0, &op1, EXPAND_NORMAL);
14031 :
14032 : /* For boolean vectors with less than mode precision
14033 : make sure to fill padding with consistent values. */
14034 42 : if (VECTOR_BOOLEAN_TYPE_P (type)
14035 29 : && SCALAR_INT_MODE_P (operand_mode)
14036 558157 : && TYPE_VECTOR_SUBPARTS (type).is_constant (&nunits)
14037 558157 : && maybe_ne (GET_MODE_PRECISION (operand_mode), nunits))
14038 : {
14039 0 : gcc_assert (code == EQ || code == NE);
14040 0 : op0 = expand_binop (mode, and_optab, op0,
14041 0 : GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
14042 : NULL_RTX, true, OPTAB_WIDEN);
14043 0 : op1 = expand_binop (mode, and_optab, op1,
14044 : GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
14045 : NULL_RTX, true, OPTAB_WIDEN);
14046 : }
14047 :
14048 558157 : if (target == 0)
14049 369346 : target = gen_reg_rtx (mode);
14050 :
14051 : /* Try a cstore if possible. */
14052 558157 : return emit_store_flag_force (target, code, op0, op1,
14053 : operand_mode, unsignedp,
14054 558157 : (TYPE_PRECISION (ops->type) == 1
14055 1115660 : && !TYPE_UNSIGNED (ops->type)) ? -1 : 1);
14056 : }
14057 :
14058 : /* Attempt to generate a casesi instruction. Returns true if successful,
14059 : false otherwise (i.e. if there is no casesi instruction).
14060 :
14061 : DEFAULT_PROBABILITY is the probability of jumping to the default
14062 : label. */
14063 : bool
14064 6318 : try_casesi (tree index_type, tree index_expr, tree minval, tree range,
14065 : rtx table_label, rtx default_label, rtx fallback_label,
14066 : profile_probability default_probability)
14067 : {
14068 6318 : class expand_operand ops[5];
14069 6318 : scalar_int_mode index_mode = SImode;
14070 6318 : rtx op1, op2, index;
14071 :
14072 6318 : if (! targetm.have_casesi ())
14073 : return false;
14074 :
14075 : /* The index must be some form of integer. Convert it to SImode. */
14076 0 : scalar_int_mode omode = SCALAR_INT_TYPE_MODE (index_type);
14077 0 : if (GET_MODE_BITSIZE (omode) > GET_MODE_BITSIZE (index_mode))
14078 : {
14079 0 : rtx rangertx = expand_normal (range);
14080 :
14081 : /* We must handle the endpoints in the original mode. */
14082 0 : index_expr = build2 (MINUS_EXPR, index_type,
14083 : index_expr, minval);
14084 0 : minval = integer_zero_node;
14085 0 : index = expand_normal (index_expr);
14086 0 : if (default_label)
14087 0 : emit_cmp_and_jump_insns (rangertx, index, LTU, NULL_RTX,
14088 : omode, 1, default_label,
14089 : default_probability);
14090 : /* Now we can safely truncate. */
14091 0 : index = convert_to_mode (index_mode, index, 0);
14092 : }
14093 : else
14094 : {
14095 0 : if (omode != index_mode)
14096 : {
14097 0 : index_type = lang_hooks.types.type_for_mode (index_mode, 0);
14098 0 : index_expr = fold_convert (index_type, index_expr);
14099 : }
14100 :
14101 0 : index = expand_normal (index_expr);
14102 : }
14103 :
14104 0 : do_pending_stack_adjust ();
14105 :
14106 0 : op1 = expand_normal (minval);
14107 0 : op2 = expand_normal (range);
14108 :
14109 0 : create_input_operand (&ops[0], index, index_mode);
14110 0 : create_convert_operand_from_type (&ops[1], op1, TREE_TYPE (minval));
14111 0 : create_convert_operand_from_type (&ops[2], op2, TREE_TYPE (range));
14112 0 : create_fixed_operand (&ops[3], table_label);
14113 0 : create_fixed_operand (&ops[4], (default_label
14114 : ? default_label
14115 : : fallback_label));
14116 0 : expand_jump_insn (targetm.code_for_casesi, 5, ops);
14117 0 : return true;
14118 : }
14119 :
14120 : /* Attempt to generate a tablejump instruction; same concept. */
14121 : /* Subroutine of the next function.
14122 :
14123 : INDEX is the value being switched on, with the lowest value
14124 : in the table already subtracted.
14125 : MODE is its expected mode (needed if INDEX is constant).
14126 : RANGE is the length of the jump table.
14127 : TABLE_LABEL is a CODE_LABEL rtx for the table itself.
14128 :
14129 : DEFAULT_LABEL is a CODE_LABEL rtx to jump to if the
14130 : index value is out of range.
14131 : DEFAULT_PROBABILITY is the probability of jumping to
14132 : the default label. */
14133 :
14134 : static void
14135 6318 : do_tablejump (rtx index, machine_mode mode, rtx range, rtx table_label,
14136 : rtx default_label, profile_probability default_probability)
14137 : {
14138 6318 : rtx temp, vector;
14139 :
14140 6318 : if (INTVAL (range) > cfun->cfg->max_jumptable_ents)
14141 5251 : cfun->cfg->max_jumptable_ents = INTVAL (range);
14142 :
14143 : /* Do an unsigned comparison (in the proper mode) between the index
14144 : expression and the value which represents the length of the range.
14145 : Since we just finished subtracting the lower bound of the range
14146 : from the index expression, this comparison allows us to simultaneously
14147 : check that the original index expression value is both greater than
14148 : or equal to the minimum value of the range and less than or equal to
14149 : the maximum value of the range. */
14150 :
14151 6318 : if (default_label)
14152 4504 : emit_cmp_and_jump_insns (index, range, GTU, NULL_RTX, mode, 1,
14153 : default_label, default_probability);
14154 :
14155 : /* If index is in range, it must fit in Pmode.
14156 : Convert to Pmode so we can index with it. */
14157 7367 : if (mode != Pmode)
14158 : {
14159 5319 : unsigned int width;
14160 :
14161 : /* We know the value of INDEX is between 0 and RANGE. If we have a
14162 : sign-extended subreg, and RANGE does not have the sign bit set, then
14163 : we have a value that is valid for both sign and zero extension. In
14164 : this case, we get better code if we sign extend. */
14165 5319 : if (GET_CODE (index) == SUBREG
14166 9 : && SUBREG_PROMOTED_VAR_P (index)
14167 0 : && SUBREG_PROMOTED_SIGNED_P (index)
14168 0 : && ((width = GET_MODE_PRECISION (as_a <scalar_int_mode> (mode)))
14169 : <= HOST_BITS_PER_WIDE_INT)
14170 5319 : && ! (UINTVAL (range) & (HOST_WIDE_INT_1U << (width - 1))))
14171 0 : index = convert_to_mode (Pmode, index, 0);
14172 : else
14173 5319 : index = convert_to_mode (Pmode, index, 1);
14174 : }
14175 :
14176 : /* Don't let a MEM slip through, because then INDEX that comes
14177 : out of PIC_CASE_VECTOR_ADDRESS won't be a valid address,
14178 : and break_out_memory_refs will go to work on it and mess it up. */
14179 : #ifdef PIC_CASE_VECTOR_ADDRESS
14180 : if (flag_pic && !REG_P (index))
14181 : index = copy_to_mode_reg (Pmode, index);
14182 : #endif
14183 :
14184 : /* ??? The only correct use of CASE_VECTOR_MODE is the one inside the
14185 : GET_MODE_SIZE, because this indicates how large insns are. The other
14186 : uses should all be Pmode, because they are addresses. This code
14187 : could fail if addresses and insns are not the same size. */
14188 7367 : index = simplify_gen_binary (MULT, Pmode, index,
14189 12636 : gen_int_mode (GET_MODE_SIZE (CASE_VECTOR_MODE),
14190 6318 : Pmode));
14191 7367 : index = simplify_gen_binary (PLUS, Pmode, index,
14192 6318 : gen_rtx_LABEL_REF (Pmode, table_label));
14193 :
14194 : #ifdef PIC_CASE_VECTOR_ADDRESS
14195 : if (flag_pic)
14196 : index = PIC_CASE_VECTOR_ADDRESS (index);
14197 : else
14198 : #endif
14199 7967 : index = memory_address (CASE_VECTOR_MODE, index);
14200 7967 : temp = gen_reg_rtx (CASE_VECTOR_MODE);
14201 7967 : vector = gen_const_mem (CASE_VECTOR_MODE, index);
14202 6318 : convert_move (temp, vector, 0);
14203 :
14204 6318 : emit_jump_insn (targetm.gen_tablejump (temp, table_label));
14205 :
14206 : /* If we are generating PIC code or if the table is PC-relative, the
14207 : table and JUMP_INSN must be adjacent, so don't output a BARRIER. */
14208 6318 : if (! CASE_VECTOR_PC_RELATIVE && ! flag_pic)
14209 5168 : emit_barrier ();
14210 6318 : }
14211 :
14212 : bool
14213 6318 : try_tablejump (tree index_type, tree index_expr, tree minval, tree range,
14214 : rtx table_label, rtx default_label,
14215 : profile_probability default_probability)
14216 : {
14217 6318 : rtx index;
14218 :
14219 6318 : if (! targetm.have_tablejump ())
14220 : return false;
14221 :
14222 6318 : index_expr = fold_build2 (MINUS_EXPR, index_type,
14223 : fold_convert (index_type, index_expr),
14224 : fold_convert (index_type, minval));
14225 6318 : index = expand_normal (index_expr);
14226 6318 : do_pending_stack_adjust ();
14227 :
14228 6318 : do_tablejump (index, TYPE_MODE (index_type),
14229 6318 : convert_modes (TYPE_MODE (index_type),
14230 6318 : TYPE_MODE (TREE_TYPE (range)),
14231 : expand_normal (range),
14232 6318 : TYPE_UNSIGNED (TREE_TYPE (range))),
14233 : table_label, default_label, default_probability);
14234 6318 : return true;
14235 : }
14236 :
14237 : /* Return a CONST_VECTOR rtx representing vector mask for
14238 : a VECTOR_CST of booleans. */
14239 : static rtx
14240 654 : const_vector_mask_from_tree (tree exp)
14241 : {
14242 654 : machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
14243 654 : machine_mode inner = GET_MODE_INNER (mode);
14244 :
14245 654 : rtx_vector_builder builder (mode, VECTOR_CST_NPATTERNS (exp),
14246 654 : VECTOR_CST_NELTS_PER_PATTERN (exp));
14247 654 : unsigned int count = builder.encoded_nelts ();
14248 3562 : for (unsigned int i = 0; i < count; ++i)
14249 : {
14250 2908 : tree elt = VECTOR_CST_ELT (exp, i);
14251 2908 : gcc_assert (TREE_CODE (elt) == INTEGER_CST);
14252 2908 : if (integer_zerop (elt))
14253 1133 : builder.quick_push (CONST0_RTX (inner));
14254 1775 : else if (integer_onep (elt)
14255 1775 : || integer_minus_onep (elt))
14256 1775 : builder.quick_push (CONSTM1_RTX (inner));
14257 : else
14258 0 : gcc_unreachable ();
14259 : }
14260 654 : return builder.build ();
14261 654 : }
14262 :
14263 : /* Return a CONST_VECTOR rtx for a VECTOR_CST tree. */
14264 : static rtx
14265 557579 : const_vector_from_tree (tree exp)
14266 : {
14267 557579 : machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
14268 :
14269 557579 : if (initializer_zerop (exp))
14270 165094 : return CONST0_RTX (mode);
14271 :
14272 392485 : if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (exp)))
14273 654 : return const_vector_mask_from_tree (exp);
14274 :
14275 391831 : machine_mode inner = GET_MODE_INNER (mode);
14276 :
14277 391831 : rtx_vector_builder builder (mode, VECTOR_CST_NPATTERNS (exp),
14278 391831 : VECTOR_CST_NELTS_PER_PATTERN (exp));
14279 391831 : unsigned int count = builder.encoded_nelts ();
14280 1264546 : for (unsigned int i = 0; i < count; ++i)
14281 : {
14282 872715 : tree elt = VECTOR_CST_ELT (exp, i);
14283 872715 : if (TREE_CODE (elt) == REAL_CST)
14284 133000 : builder.quick_push (const_double_from_real_value (TREE_REAL_CST (elt),
14285 : inner));
14286 739715 : else if (TREE_CODE (elt) == FIXED_CST)
14287 0 : builder.quick_push (CONST_FIXED_FROM_FIXED_VALUE (TREE_FIXED_CST (elt),
14288 : inner));
14289 : else
14290 739715 : builder.quick_push (immed_wide_int_const (wi::to_poly_wide (elt),
14291 : inner));
14292 : }
14293 391831 : return builder.build ();
14294 391831 : }
14295 :
14296 : /* Build a decl for a personality function given a language prefix. */
14297 :
14298 : tree
14299 32396 : build_personality_function (const char *lang)
14300 : {
14301 32396 : const char *unwind_and_version;
14302 32396 : tree decl, type;
14303 32396 : char *name;
14304 :
14305 32396 : switch (targetm_common.except_unwind_info (&global_options))
14306 : {
14307 : case UI_NONE:
14308 : return NULL;
14309 : case UI_SJLJ:
14310 : unwind_and_version = "_sj0";
14311 : break;
14312 32396 : case UI_DWARF2:
14313 32396 : case UI_TARGET:
14314 32396 : unwind_and_version = "_v0";
14315 32396 : break;
14316 0 : case UI_SEH:
14317 0 : unwind_and_version = "_seh0";
14318 0 : break;
14319 0 : default:
14320 0 : gcc_unreachable ();
14321 : }
14322 :
14323 32396 : name = ACONCAT (("__", lang, "_personality", unwind_and_version, NULL));
14324 :
14325 32396 : type = build_function_type_list (unsigned_type_node,
14326 : integer_type_node, integer_type_node,
14327 : long_long_unsigned_type_node,
14328 : ptr_type_node, ptr_type_node, NULL_TREE);
14329 32396 : decl = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL,
14330 : get_identifier (name), type);
14331 32396 : DECL_ARTIFICIAL (decl) = 1;
14332 32396 : DECL_EXTERNAL (decl) = 1;
14333 32396 : TREE_PUBLIC (decl) = 1;
14334 :
14335 : /* Zap the nonsensical SYMBOL_REF_DECL for this. What we're left with
14336 : are the flags assigned by targetm.encode_section_info. */
14337 32396 : SET_SYMBOL_REF_DECL (XEXP (DECL_RTL (decl), 0), NULL);
14338 :
14339 32396 : return decl;
14340 : }
14341 :
14342 : /* Extracts the personality function of DECL and returns the corresponding
14343 : libfunc. */
14344 :
14345 : rtx
14346 1627851 : get_personality_function (tree decl)
14347 : {
14348 1627851 : tree personality = DECL_FUNCTION_PERSONALITY (decl);
14349 1627851 : enum eh_personality_kind pk;
14350 :
14351 1627851 : pk = function_needs_eh_personality (DECL_STRUCT_FUNCTION (decl));
14352 1627851 : if (pk == eh_personality_none)
14353 : return NULL;
14354 :
14355 152719 : if (!personality
14356 152719 : && pk == eh_personality_any)
14357 69843 : personality = lang_hooks.eh_personality ();
14358 :
14359 152719 : if (pk == eh_personality_lang)
14360 82876 : gcc_assert (personality != NULL_TREE);
14361 :
14362 152719 : return XEXP (DECL_RTL (personality), 0);
14363 : }
14364 :
14365 : /* Returns a tree for the size of EXP in bytes. */
14366 :
14367 : static tree
14368 14608457 : tree_expr_size (const_tree exp)
14369 : {
14370 14608457 : if (DECL_P (exp)
14371 14608457 : && DECL_SIZE_UNIT (exp) != 0)
14372 1937058 : return DECL_SIZE_UNIT (exp);
14373 : else
14374 12671399 : return size_in_bytes (TREE_TYPE (exp));
14375 : }
14376 :
14377 : /* Return an rtx for the size in bytes of the value of EXP. */
14378 :
14379 : rtx
14380 14423136 : expr_size (tree exp)
14381 : {
14382 14423136 : tree size;
14383 :
14384 14423136 : if (TREE_CODE (exp) == WITH_SIZE_EXPR)
14385 773 : size = TREE_OPERAND (exp, 1);
14386 : else
14387 : {
14388 14422363 : size = tree_expr_size (exp);
14389 14422363 : gcc_assert (size);
14390 14422363 : gcc_assert (size == SUBSTITUTE_PLACEHOLDER_IN_EXPR (size, exp));
14391 : }
14392 :
14393 14423136 : return expand_expr (size, NULL_RTX, TYPE_MODE (sizetype), EXPAND_NORMAL);
14394 : }
14395 :
14396 : /* Return a wide integer for the size in bytes of the value of EXP, or -1
14397 : if the size can vary or is larger than an integer. */
14398 :
14399 : HOST_WIDE_INT
14400 186094 : int_expr_size (const_tree exp)
14401 : {
14402 186094 : tree size;
14403 :
14404 186094 : if (TREE_CODE (exp) == WITH_SIZE_EXPR)
14405 0 : size = TREE_OPERAND (exp, 1);
14406 : else
14407 : {
14408 186094 : size = tree_expr_size (exp);
14409 186094 : gcc_assert (size);
14410 : }
14411 :
14412 186094 : if (size == 0 || !tree_fits_shwi_p (size))
14413 0 : return -1;
14414 :
14415 186094 : return tree_to_shwi (size);
14416 : }
14417 :
14418 : /* Return the quotient of polynomial long division of x^2N by POLYNOMIAL
14419 : in GF (2^N).
14420 : Author: Richard Sandiford <richard.sandiford@arm.com> */
14421 :
14422 : unsigned HOST_WIDE_INT
14423 0 : gf2n_poly_long_div_quotient (unsigned HOST_WIDE_INT polynomial,
14424 : unsigned short n)
14425 : {
14426 : /* The result has degree N, so needs N + 1 bits. */
14427 0 : gcc_assert (n < 64);
14428 :
14429 : /* Perform a division step for the x^2N coefficient. At this point the
14430 : quotient and remainder have N implicit trailing zeros. */
14431 : unsigned HOST_WIDE_INT quotient = 1;
14432 : unsigned HOST_WIDE_INT remainder = polynomial;
14433 :
14434 : /* Process the coefficients for x^(2N-1) down to x^N, with each step
14435 : reducing the number of implicit trailing zeros by one. */
14436 0 : for (unsigned int i = 0; i < n; ++i)
14437 : {
14438 0 : bool coeff = remainder & (HOST_WIDE_INT_1U << (n - 1));
14439 0 : quotient = (quotient << 1) | coeff;
14440 0 : remainder = (remainder << 1) ^ (coeff ? polynomial : 0);
14441 : }
14442 0 : return quotient;
14443 : }
14444 :
14445 : /* Calculate CRC for the initial CRC and given POLYNOMIAL.
14446 : CRC_BITS is CRC size. */
14447 :
14448 : static unsigned HOST_WIDE_INT
14449 34560 : calculate_crc (unsigned HOST_WIDE_INT crc,
14450 : unsigned HOST_WIDE_INT polynomial,
14451 : unsigned short crc_bits)
14452 : {
14453 34560 : unsigned HOST_WIDE_INT msb = HOST_WIDE_INT_1U << (crc_bits - 1);
14454 34560 : crc = crc << (crc_bits - 8);
14455 311040 : for (short i = 8; i > 0; --i)
14456 : {
14457 276480 : if (crc & msb)
14458 138240 : crc = (crc << 1) ^ polynomial;
14459 : else
14460 138240 : crc <<= 1;
14461 : }
14462 : /* Zero out bits in crc beyond the specified number of crc_bits. */
14463 34560 : if (crc_bits < sizeof (crc) * CHAR_BIT)
14464 31488 : crc &= (HOST_WIDE_INT_1U << crc_bits) - 1;
14465 34560 : return crc;
14466 : }
14467 :
14468 : /* Assemble CRC table with 256 elements for the given POLYNOM and CRC_BITS.
14469 : POLYNOM is the polynomial used to calculate the CRC table's elements.
14470 : CRC_BITS is the size of CRC, may be 8, 16, ... . */
14471 :
14472 : static rtx
14473 135 : assemble_crc_table (unsigned HOST_WIDE_INT polynom, unsigned short crc_bits)
14474 : {
14475 135 : unsigned table_el_n = 0x100;
14476 135 : tree ar = build_array_type (make_unsigned_type (crc_bits),
14477 135 : build_index_type (size_int (table_el_n - 1)));
14478 :
14479 : /* Initialize the table. */
14480 135 : vec<tree, va_gc> *initial_values;
14481 135 : vec_alloc (initial_values, table_el_n);
14482 34695 : for (size_t i = 0; i < table_el_n; ++i)
14483 : {
14484 34560 : unsigned HOST_WIDE_INT crc = calculate_crc (i, polynom, crc_bits);
14485 34560 : tree element = build_int_cstu (make_unsigned_type (crc_bits), crc);
14486 34560 : vec_safe_push (initial_values, element);
14487 : }
14488 135 : tree ctor = build_constructor_from_vec (ar, initial_values);
14489 135 : rtx mem = output_constant_def (ctor, 1);
14490 135 : gcc_assert (MEM_P (mem));
14491 135 : if (dump_file && (dump_flags & TDF_DETAILS))
14492 : {
14493 16 : fprintf (dump_file,
14494 : ";; emitting crc table crc_%u_polynomial_"
14495 : HOST_WIDE_INT_PRINT_HEX " ",
14496 : crc_bits, polynom);
14497 16 : print_rtl_single (dump_file, XEXP (mem, 0));
14498 16 : fprintf (dump_file, "\n");
14499 : }
14500 :
14501 135 : return XEXP (mem, 0);
14502 : }
14503 :
14504 : /* Generate CRC lookup table by calculating CRC for all possible
14505 : 8-bit data values. The table is stored with a specific name in the read-only
14506 : static data section.
14507 : POLYNOM is the polynomial used to calculate the CRC table's elements.
14508 : CRC_BITS is the size of CRC, may be 8, 16, ... . */
14509 :
14510 : static rtx
14511 135 : generate_crc_table (unsigned HOST_WIDE_INT polynom, unsigned short crc_bits)
14512 : {
14513 135 : gcc_assert (crc_bits <= 64);
14514 :
14515 135 : return assemble_crc_table (polynom, crc_bits);
14516 : }
14517 :
14518 : /* Calculate CRC for the initial CRC and given POLYNOMIAL.
14519 : CRC_BITS is CRC size. */
14520 :
14521 : static unsigned HOST_WIDE_INT
14522 35328 : calculate_reversed_crc (unsigned HOST_WIDE_INT crc,
14523 : unsigned HOST_WIDE_INT polynomial,
14524 : unsigned short crc_bits)
14525 : {
14526 35328 : unsigned HOST_WIDE_INT rev_polynom = reflect_hwi (polynomial, crc_bits);
14527 317952 : for (int j = 0; j < 8; j++)
14528 : {
14529 282624 : if (crc & 1)
14530 141312 : crc = (crc >> 1) ^ rev_polynom;
14531 : else
14532 141312 : crc >>= 1;
14533 : }
14534 : /* Zero out bits in crc beyond the specified number of crc_bits. */
14535 35328 : if (crc_bits < sizeof (crc) * CHAR_BIT)
14536 29952 : crc &= (HOST_WIDE_INT_1U << crc_bits) - 1;
14537 35328 : return crc;
14538 : }
14539 :
14540 : /* Assemble CRC table with 256 elements for the given POLYNOM and CRC_BITS.
14541 : POLYNOM is the polynomial used to calculate the CRC table's elements.
14542 : CRC_BITS is the size of CRC, may be 8, 16, ... . */
14543 :
14544 : static rtx
14545 138 : assemble_reversed_crc_table (unsigned HOST_WIDE_INT polynom, unsigned short crc_bits)
14546 : {
14547 138 : unsigned table_el_n = 0x100;
14548 138 : tree ar = build_array_type (make_unsigned_type (crc_bits),
14549 138 : build_index_type (size_int (table_el_n - 1)));
14550 :
14551 : /* Initialize the table. */
14552 138 : vec<tree, va_gc> *initial_values;
14553 138 : vec_alloc (initial_values, table_el_n);
14554 35466 : for (size_t i = 0; i < table_el_n; ++i)
14555 : {
14556 35328 : unsigned HOST_WIDE_INT crc = calculate_reversed_crc (i, polynom, crc_bits);
14557 35328 : tree element = build_int_cstu (make_unsigned_type (crc_bits), crc);
14558 35328 : vec_safe_push (initial_values, element);
14559 : }
14560 138 : tree ctor = build_constructor_from_vec (ar, initial_values);
14561 138 : rtx mem = output_constant_def (ctor, 1);
14562 138 : gcc_assert (MEM_P (mem));
14563 138 : if (dump_file && (dump_flags & TDF_DETAILS))
14564 : {
14565 16 : fprintf (dump_file,
14566 : ";; emitting reversed crc table crc_%u_polynomial_"
14567 : HOST_WIDE_INT_PRINT_HEX " ",
14568 : crc_bits, polynom);
14569 16 : print_rtl_single (dump_file, XEXP (mem, 0));
14570 16 : fprintf (dump_file, "\n");
14571 : }
14572 :
14573 138 : return XEXP (mem, 0);
14574 : }
14575 :
14576 : /* Generate reversed CRC table for the given POLYNOM and CRC_BITS. */
14577 :
14578 : static rtx
14579 138 : generate_reversed_crc_table (unsigned HOST_WIDE_INT polynom,
14580 : unsigned short crc_bits)
14581 : {
14582 138 : gcc_assert (crc_bits <= 64);
14583 :
14584 138 : return assemble_reversed_crc_table (polynom, crc_bits);
14585 : }
14586 :
14587 : /* Generate table-based CRC code for the given CRC, INPUT_DATA and the
14588 : POLYNOMIAL (without leading 1).
14589 :
14590 : First, using POLYNOMIAL's value generates CRC table of 256 elements,
14591 : then generates the assembly for the following code,
14592 : where crc_bit_size and data_bit_size may be 8, 16, 32, 64, depending on CRC:
14593 :
14594 : for (int i = 0; i < data_bit_size / 8; i++)
14595 : crc = (crc << 8) ^ crc_table[(crc >> (crc_bit_size - 8))
14596 : ^ (data >> (data_bit_size - (i + 1) * 8)
14597 : & 0xFF))];
14598 :
14599 : So to take values from the table, we need 8-bit data.
14600 : If input data size is not 8, then first we extract upper 8 bits,
14601 : then the other 8 bits, and so on. */
14602 :
14603 : static void
14604 135 : calculate_table_based_CRC (rtx *crc, const rtx &input_data,
14605 : const rtx &polynomial,
14606 : machine_mode data_mode)
14607 : {
14608 135 : machine_mode mode = GET_MODE (*crc);
14609 135 : unsigned short crc_bit_size = GET_MODE_BITSIZE (mode).to_constant ();
14610 135 : unsigned short data_size = GET_MODE_SIZE (data_mode).to_constant ();
14611 135 : rtx tab = generate_crc_table (UINTVAL (polynomial), crc_bit_size);
14612 :
14613 394 : for (unsigned short i = 0; i < data_size; i++)
14614 : {
14615 : /* crc >> (crc_bit_size - 8). */
14616 259 : *crc = force_reg (mode, *crc);
14617 259 : rtx op1 = expand_shift (RSHIFT_EXPR, mode, *crc, crc_bit_size - 8,
14618 : NULL_RTX, 1);
14619 :
14620 : /* data >> (8 * (GET_MODE_SIZE (data_mode).to_constant () - i - 1)). */
14621 259 : unsigned range_8 = 8 * (data_size - i - 1);
14622 : /* CRC's mode is always at least as wide as INPUT_DATA. Convert
14623 : INPUT_DATA into CRC's mode. */
14624 259 : rtx data = gen_reg_rtx (mode);
14625 259 : convert_move (data, input_data, 1);
14626 259 : data = expand_shift (RSHIFT_EXPR, mode, data, range_8, NULL_RTX, 1);
14627 :
14628 : /* data >> (8 * (GET_MODE_SIZE (mode)
14629 : .to_constant () - i - 1)) & 0xFF. */
14630 259 : rtx data_final = expand_and (mode, data,
14631 : gen_int_mode (255, mode), NULL_RTX);
14632 :
14633 : /* (crc >> (crc_bit_size - 8)) ^ data_8bit. */
14634 259 : rtx in = expand_binop (mode, xor_optab, op1, data_final,
14635 : NULL_RTX, 1, OPTAB_WIDEN);
14636 :
14637 : /* ((crc >> (crc_bit_size - 8)) ^ data_8bit) & 0xFF. */
14638 259 : rtx index = expand_and (mode, in, gen_int_mode (255, mode),
14639 : NULL_RTX);
14640 518 : int log_crc_size = exact_log2 (GET_MODE_SIZE (mode).to_constant ());
14641 259 : index = expand_shift (LSHIFT_EXPR, mode, index,
14642 259 : log_crc_size, NULL_RTX, 0);
14643 :
14644 264 : rtx addr = gen_reg_rtx (Pmode);
14645 259 : convert_move (addr, index, 1);
14646 264 : addr = expand_binop (Pmode, add_optab, addr, tab, NULL_RTX,
14647 : 0, OPTAB_DIRECT);
14648 :
14649 : /* crc_table[(crc >> (crc_bit_size - 8)) ^ data_8bit] */
14650 259 : rtx tab_el = validize_mem (gen_rtx_MEM (mode, addr));
14651 :
14652 : /* (crc << 8) if CRC is larger than 8, otherwise crc = 0. */
14653 259 : rtx high = NULL_RTX;
14654 259 : if (crc_bit_size != 8)
14655 229 : high = expand_shift (LSHIFT_EXPR, mode, *crc, 8, NULL_RTX, 0);
14656 : else
14657 30 : high = gen_int_mode (0, mode);
14658 :
14659 : /* crc = (crc << 8)
14660 : ^ crc_table[(crc >> (crc_bit_size - 8)) ^ data_8bit]; */
14661 259 : *crc = expand_binop (mode, xor_optab, tab_el, high, NULL_RTX, 1,
14662 : OPTAB_WIDEN);
14663 : }
14664 135 : }
14665 :
14666 : /* Generate table-based reversed CRC code for the given CRC, INPUT_DATA
14667 : and the POLYNOMIAL (without leading 1).
14668 :
14669 : This function generates code for reversed (bit-reflected) CRC calculation
14670 : using a pre-computed lookup table. Unlike the standard CRC calculation,
14671 : this processes data from LSB to MSB, eliminating the need for explicit
14672 : bit reflection before and after the CRC computation. */
14673 :
14674 : static void
14675 138 : calculate_table_based_reversed_CRC (rtx *crc, const rtx &input_data,
14676 : const rtx &polynomial,
14677 : machine_mode data_mode)
14678 : {
14679 138 : machine_mode mode = GET_MODE (*crc);
14680 138 : unsigned short crc_bit_size = GET_MODE_BITSIZE (mode).to_constant ();
14681 138 : unsigned short data_size = GET_MODE_SIZE (data_mode).to_constant ();
14682 138 : rtx tab = generate_reversed_crc_table (UINTVAL (polynomial), crc_bit_size);
14683 :
14684 : /* CRC's mode is always at least as wide as INPUT_DATA. Convert
14685 : INPUT_DATA into CRC's mode once outside the loop since INPUT_DATA
14686 : is loop-invariant. */
14687 138 : rtx data_in_crc_mode = gen_reg_rtx (mode);
14688 138 : convert_move (data_in_crc_mode, input_data, 1);
14689 :
14690 364 : for (unsigned short i = 0; i < data_size; i++)
14691 : {
14692 226 : *crc = force_reg (mode, *crc);
14693 :
14694 : /* data >> (8 * i). */
14695 226 : unsigned range_8 = 8 * i;
14696 226 : rtx data = expand_shift (RSHIFT_EXPR, mode, data_in_crc_mode,
14697 226 : range_8, NULL_RTX, 1);
14698 :
14699 : /* crc ^ (data >> (8 * i)). */
14700 226 : rtx in = expand_binop (mode, xor_optab, *crc, data,
14701 : NULL_RTX, 1, OPTAB_WIDEN);
14702 :
14703 : /* (crc ^ data) & 0xFF. */
14704 226 : rtx index = expand_and (mode, in, gen_int_mode (255, mode),
14705 : NULL_RTX);
14706 452 : int log_crc_size = exact_log2 (GET_MODE_SIZE (mode).to_constant ());
14707 226 : index = expand_shift (LSHIFT_EXPR, mode, index,
14708 226 : log_crc_size, NULL_RTX, 0);
14709 :
14710 239 : rtx addr = gen_reg_rtx (Pmode);
14711 226 : convert_move (addr, index, 1);
14712 239 : addr = expand_binop (Pmode, add_optab, addr, tab, NULL_RTX,
14713 : 0, OPTAB_DIRECT);
14714 :
14715 : /* crc_table[(crc ^ data) & 0xFF]. */
14716 226 : rtx tab_el = validize_mem (gen_rtx_MEM (mode, addr));
14717 :
14718 : /* (crc >> 8) if CRC is larger than 8, otherwise 0. */
14719 226 : rtx high = NULL_RTX;
14720 226 : if (crc_bit_size != 8)
14721 217 : high = expand_shift (RSHIFT_EXPR, mode, *crc, 8, NULL_RTX, 1);
14722 : else
14723 9 : high = gen_int_mode (0, mode);
14724 :
14725 : /* crc = (crc >> 8) ^ crc_table[(crc ^ data) & 0xFF]. */
14726 226 : *crc = expand_binop (mode, xor_optab, tab_el, high, NULL_RTX, 1,
14727 : OPTAB_WIDEN);
14728 : }
14729 138 : }
14730 :
14731 : /* Generate table-based CRC code for the given CRC, INPUT_DATA and the
14732 : POLYNOMIAL (without leading 1).
14733 :
14734 : CRC is OP1, data is OP2 and the polynomial is OP3.
14735 : This must generate a CRC table and an assembly for the following code,
14736 : where crc_bit_size and data_bit_size may be 8, 16, 32, 64:
14737 : uint_crc_bit_size_t
14738 : crc_crc_bit_size (uint_crc_bit_size_t crc_init,
14739 : uint_data_bit_size_t data, size_t size)
14740 : {
14741 : uint_crc_bit_size_t crc = crc_init;
14742 : for (int i = 0; i < data_bit_size / 8; i++)
14743 : crc = (crc << 8) ^ crc_table[(crc >> (crc_bit_size - 8))
14744 : ^ (data >> (data_bit_size - (i + 1) * 8)
14745 : & 0xFF))];
14746 : return crc;
14747 : } */
14748 :
14749 : void
14750 135 : expand_crc_table_based (rtx op0, rtx op1, rtx op2, rtx op3,
14751 : machine_mode data_mode)
14752 : {
14753 135 : gcc_assert (!CONST_INT_P (op0));
14754 135 : gcc_assert (CONST_INT_P (op3));
14755 135 : machine_mode crc_mode = GET_MODE (op0);
14756 135 : rtx crc = gen_reg_rtx (crc_mode);
14757 135 : convert_move (crc, op1, 0);
14758 135 : calculate_table_based_CRC (&crc, op2, op3, data_mode);
14759 135 : convert_move (op0, crc, 0);
14760 135 : }
14761 :
14762 : /* Generate the common operation for reflecting values:
14763 : *OP = (*OP & AND1_VALUE) << SHIFT_VAL | (*OP & AND2_VALUE) >> SHIFT_VAL; */
14764 :
14765 : void
14766 0 : gen_common_operation_to_reflect (rtx *op,
14767 : unsigned HOST_WIDE_INT and1_value,
14768 : unsigned HOST_WIDE_INT and2_value,
14769 : unsigned shift_val)
14770 : {
14771 0 : rtx op1 = expand_and (GET_MODE (*op), *op,
14772 0 : gen_int_mode (and1_value, GET_MODE (*op)), NULL_RTX);
14773 0 : op1 = expand_shift (LSHIFT_EXPR, GET_MODE (*op), op1, shift_val, op1, 0);
14774 0 : rtx op2 = expand_and (GET_MODE (*op), *op,
14775 0 : gen_int_mode (and2_value, GET_MODE (*op)), NULL_RTX);
14776 0 : op2 = expand_shift (RSHIFT_EXPR, GET_MODE (*op), op2, shift_val, op2, 1);
14777 0 : *op = expand_binop (GET_MODE (*op), ior_optab, op1,
14778 : op2, *op, 0, OPTAB_LIB_WIDEN);
14779 0 : }
14780 :
14781 : /* Reflect 64-bit value for the 64-bit target. */
14782 :
14783 : void
14784 0 : reflect_64_bit_value (rtx *op)
14785 : {
14786 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x00000000FFFFFFFF),
14787 : HOST_WIDE_INT_C (0xFFFFFFFF00000000), 32);
14788 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0000FFFF0000FFFF),
14789 : HOST_WIDE_INT_C (0xFFFF0000FFFF0000), 16);
14790 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x00FF00FF00FF00FF),
14791 : HOST_WIDE_INT_C (0xFF00FF00FF00FF00), 8);
14792 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0F0F0F0F0F0F0F0F),
14793 : HOST_WIDE_INT_C (0xF0F0F0F0F0F0F0F0), 4);
14794 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x3333333333333333),
14795 : HOST_WIDE_INT_C (0xCCCCCCCCCCCCCCCC), 2);
14796 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x5555555555555555),
14797 : HOST_WIDE_INT_C (0xAAAAAAAAAAAAAAAA), 1);
14798 0 : }
14799 :
14800 : /* Reflect 32-bit value for the 32-bit target. */
14801 :
14802 : void
14803 0 : reflect_32_bit_value (rtx *op)
14804 : {
14805 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0000FFFF),
14806 : HOST_WIDE_INT_C (0xFFFF0000), 16);
14807 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x00FF00FF),
14808 : HOST_WIDE_INT_C (0xFF00FF00), 8);
14809 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0F0F0F0F),
14810 : HOST_WIDE_INT_C (0xF0F0F0F0), 4);
14811 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x33333333),
14812 : HOST_WIDE_INT_C (0xCCCCCCCC), 2);
14813 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x55555555),
14814 : HOST_WIDE_INT_C (0xAAAAAAAA), 1);
14815 0 : }
14816 :
14817 : /* Reflect 16-bit value for the 16-bit target. */
14818 :
14819 : void
14820 0 : reflect_16_bit_value (rtx *op)
14821 : {
14822 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x00FF),
14823 : HOST_WIDE_INT_C (0xFF00), 8);
14824 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0F0F),
14825 : HOST_WIDE_INT_C (0xF0F0), 4);
14826 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x3333),
14827 : HOST_WIDE_INT_C (0xCCCC), 2);
14828 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x5555),
14829 : HOST_WIDE_INT_C (0xAAAA), 1);
14830 0 : }
14831 :
14832 : /* Reflect 8-bit value for the 8-bit target. */
14833 :
14834 : void
14835 0 : reflect_8_bit_value (rtx *op)
14836 : {
14837 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x0F),
14838 : HOST_WIDE_INT_C (0xF0), 4);
14839 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x33),
14840 : HOST_WIDE_INT_C (0xCC), 2);
14841 0 : gen_common_operation_to_reflect (op, HOST_WIDE_INT_C (0x55),
14842 : HOST_WIDE_INT_C (0xAA), 1);
14843 0 : }
14844 :
14845 : /* Generate instruction sequence which reflects the value of the OP
14846 : using shift, and, or operations. OP's mode may be less than word_mode. */
14847 :
14848 : void
14849 0 : generate_reflecting_code_standard (rtx *op)
14850 : {
14851 0 : gcc_assert (GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () >= 8
14852 : && GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () <= 64);
14853 :
14854 0 : if (GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () == 64)
14855 0 : reflect_64_bit_value (op);
14856 0 : else if (GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () == 32)
14857 0 : reflect_32_bit_value (op);
14858 0 : else if (GET_MODE_BITSIZE (GET_MODE (*op)).to_constant () == 16)
14859 0 : reflect_16_bit_value (op);
14860 : else
14861 0 : reflect_8_bit_value (op);
14862 0 : }
14863 :
14864 : /* Generate table-based reversed CRC code for the given CRC, INPUT_DATA and
14865 : the POLYNOMIAL (without leading 1).
14866 :
14867 : CRC is OP1, data is OP2 and the polynomial is OP3.
14868 : This generates a reversed CRC table and assembly for the following code,
14869 : where crc_bit_size and data_bit_size may be 8, 16, 32, 64:
14870 : uint_crc_bit_size_t
14871 : crc_crc_bit_size (uint_crc_bit_size_t crc_init,
14872 : uint_data_bit_size_t data)
14873 : {
14874 : uint_crc_bit_size_t crc = crc_init;
14875 : for (int i = 0; i < data_bit_size / 8; i++)
14876 : crc = (crc >> 8) ^ crc_table[(crc ^ (data >> (i * 8))) & 0xFF];
14877 : return crc;
14878 : }
14879 :
14880 : This approach uses a pre-computed reversed polynomial table, eliminating
14881 : the need for explicit bit reflection before and after the CRC computation. */
14882 :
14883 : void
14884 138 : expand_reversed_crc_table_based (rtx op0, rtx op1, rtx op2, rtx op3,
14885 : machine_mode data_mode)
14886 : {
14887 138 : gcc_assert (!CONST_INT_P (op0));
14888 138 : gcc_assert (CONST_INT_P (op3));
14889 138 : machine_mode crc_mode = GET_MODE (op0);
14890 138 : rtx crc = gen_reg_rtx (crc_mode);
14891 138 : convert_move (crc, op1, 0);
14892 138 : calculate_table_based_reversed_CRC (&crc, op2, op3, data_mode);
14893 138 : convert_move (op0, crc, 0);
14894 138 : }
|
