Line data Source code
1 : /* Medium-level subroutines: convert bit-field store and extract
2 : and shifts, multiplies and divides to rtl instructions.
3 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* Work around tree-optimization/91825. */
22 : #pragma GCC diagnostic warning "-Wmaybe-uninitialized"
23 :
24 : #include "config.h"
25 : #include "system.h"
26 : #include "coretypes.h"
27 : #include "backend.h"
28 : #include "target.h"
29 : #include "rtl.h"
30 : #include "tree.h"
31 : #include "predict.h"
32 : #include "memmodel.h"
33 : #include "tm_p.h"
34 : #include "optabs.h"
35 : #include "expmed.h"
36 : #include "regs.h"
37 : #include "emit-rtl.h"
38 : #include "diagnostic-core.h"
39 : #include "fold-const.h"
40 : #include "stor-layout.h"
41 : #include "dojump.h"
42 : #include "explow.h"
43 : #include "expr.h"
44 : #include "langhooks.h"
45 : #include "tree-vector-builder.h"
46 : #include "recog.h"
47 :
48 : struct target_expmed default_target_expmed;
49 : #if SWITCHABLE_TARGET
50 : struct target_expmed *this_target_expmed = &default_target_expmed;
51 : #endif
52 :
53 : static bool store_integral_bit_field (rtx, opt_scalar_int_mode,
54 : unsigned HOST_WIDE_INT,
55 : unsigned HOST_WIDE_INT,
56 : poly_uint64, poly_uint64,
57 : machine_mode, rtx, bool, bool);
58 : /* The constant divisor of the last division expanded, so that a modulo by
59 : the same constant can reuse the quotient. Reset for each function by
60 : prepare_function_start: carrying it across functions would make a
61 : function's code depend on what was expanded before it in the translation
62 : unit. */
63 : HOST_WIDE_INT last_div_const = 0;
64 :
65 : static void store_fixed_bit_field (rtx, opt_scalar_int_mode,
66 : unsigned HOST_WIDE_INT,
67 : unsigned HOST_WIDE_INT,
68 : poly_uint64, poly_uint64,
69 : rtx, scalar_int_mode, bool);
70 : static void store_fixed_bit_field_1 (rtx, scalar_int_mode,
71 : unsigned HOST_WIDE_INT,
72 : unsigned HOST_WIDE_INT,
73 : rtx, scalar_int_mode, bool);
74 : static void store_split_bit_field (rtx, opt_scalar_int_mode,
75 : unsigned HOST_WIDE_INT,
76 : unsigned HOST_WIDE_INT,
77 : poly_uint64, poly_uint64,
78 : rtx, scalar_int_mode, bool);
79 : static rtx extract_integral_bit_field (rtx, opt_scalar_int_mode,
80 : unsigned HOST_WIDE_INT,
81 : unsigned HOST_WIDE_INT, int, rtx,
82 : machine_mode, machine_mode, bool, bool);
83 : static rtx extract_fixed_bit_field (machine_mode, rtx, opt_scalar_int_mode,
84 : unsigned HOST_WIDE_INT,
85 : unsigned HOST_WIDE_INT, rtx, int, bool);
86 : static rtx extract_fixed_bit_field_1 (machine_mode, rtx, scalar_int_mode,
87 : unsigned HOST_WIDE_INT,
88 : unsigned HOST_WIDE_INT, rtx, int, bool);
89 : static rtx lshift_value (machine_mode, unsigned HOST_WIDE_INT, int);
90 : static rtx extract_split_bit_field (rtx, opt_scalar_int_mode,
91 : unsigned HOST_WIDE_INT,
92 : unsigned HOST_WIDE_INT, int, bool);
93 : static void do_cmp_and_jump (rtx, rtx, enum rtx_code, machine_mode, rtx_code_label *);
94 : static rtx expand_smod_pow2 (scalar_int_mode, rtx, HOST_WIDE_INT);
95 : static rtx expand_sdiv_pow2 (scalar_int_mode, rtx, HOST_WIDE_INT);
96 :
97 : /* Return a constant integer mask value of mode MODE with BITSIZE ones
98 : followed by BITPOS zeros, or the complement of that if COMPLEMENT.
99 : The mask is truncated if necessary to the width of mode MODE. The
100 : mask is zero-extended if BITSIZE+BITPOS is too small for MODE. */
101 :
102 : static inline rtx
103 215650 : mask_rtx (scalar_int_mode mode, int bitpos, int bitsize, bool complement)
104 : {
105 215650 : return immed_wide_int_const
106 215650 : (wi::shifted_mask (bitpos, bitsize, complement,
107 215650 : GET_MODE_PRECISION (mode)), mode);
108 : }
109 :
110 : /* Test whether a value is zero of a power of two. */
111 : #define EXACT_POWER_OF_2_OR_ZERO_P(x) \
112 : (((x) & ((x) - HOST_WIDE_INT_1U)) == 0)
113 :
114 : struct init_expmed_rtl
115 : {
116 : rtx reg;
117 : rtx plus;
118 : rtx neg;
119 : rtx mult;
120 : rtx sdiv;
121 : rtx udiv;
122 : rtx sdiv_32;
123 : rtx smod_32;
124 : rtx wide_mult;
125 : rtx wide_lshr;
126 : rtx wide_trunc;
127 : rtx shift;
128 : rtx shift_mult;
129 : rtx shift_add;
130 : rtx shift_sub0;
131 : rtx shift_sub1;
132 : rtx zext;
133 : rtx trunc;
134 :
135 : rtx pow2[MAX_BITS_PER_WORD];
136 : rtx cint[MAX_BITS_PER_WORD];
137 : };
138 :
139 : static void
140 30840320 : init_expmed_one_conv (struct init_expmed_rtl *all, scalar_int_mode to_mode,
141 : scalar_int_mode from_mode, bool speed)
142 : {
143 30840320 : int to_size, from_size;
144 30840320 : rtx which;
145 :
146 30840320 : to_size = GET_MODE_PRECISION (to_mode);
147 30840320 : from_size = GET_MODE_PRECISION (from_mode);
148 :
149 : /* Most partial integers have a precision less than the "full"
150 : integer it requires for storage. In case one doesn't, for
151 : comparison purposes here, reduce the bit size by one in that
152 : case. */
153 30840320 : if (GET_MODE_CLASS (to_mode) == MODE_PARTIAL_INT
154 30840320 : && pow2p_hwi (to_size))
155 6168064 : to_size --;
156 30840320 : if (GET_MODE_CLASS (from_mode) == MODE_PARTIAL_INT
157 30840320 : && pow2p_hwi (from_size))
158 0 : from_size --;
159 :
160 : /* Assume cost of zero-extend and sign-extend is the same. */
161 30840320 : which = (to_size < from_size ? all->trunc : all->zext);
162 :
163 30840320 : PUT_MODE (all->reg, from_mode);
164 30840320 : set_convert_cost (to_mode, from_mode, speed,
165 : set_src_cost (which, to_mode, speed));
166 : /* Restore all->reg's mode. */
167 30840320 : PUT_MODE (all->reg, to_mode);
168 30840320 : }
169 :
170 : static void
171 18063616 : init_expmed_one_mode (struct init_expmed_rtl *all,
172 : machine_mode mode, int speed)
173 : {
174 18063616 : int m, n, mode_bitsize;
175 18063616 : machine_mode mode_from;
176 :
177 18063616 : mode_bitsize = GET_MODE_UNIT_BITSIZE (mode);
178 :
179 18063616 : PUT_MODE (all->reg, mode);
180 18063616 : PUT_MODE (all->plus, mode);
181 18063616 : PUT_MODE (all->neg, mode);
182 18063616 : PUT_MODE (all->mult, mode);
183 18063616 : PUT_MODE (all->sdiv, mode);
184 18063616 : PUT_MODE (all->udiv, mode);
185 18063616 : PUT_MODE (all->sdiv_32, mode);
186 18063616 : PUT_MODE (all->smod_32, mode);
187 18063616 : PUT_MODE (all->wide_trunc, mode);
188 18063616 : PUT_MODE (all->shift, mode);
189 18063616 : PUT_MODE (all->shift_mult, mode);
190 18063616 : PUT_MODE (all->shift_add, mode);
191 18063616 : PUT_MODE (all->shift_sub0, mode);
192 18063616 : PUT_MODE (all->shift_sub1, mode);
193 18063616 : PUT_MODE (all->zext, mode);
194 18063616 : PUT_MODE (all->trunc, mode);
195 :
196 18063616 : set_add_cost (speed, mode, set_src_cost (all->plus, mode, speed));
197 18063616 : set_neg_cost (speed, mode, set_src_cost (all->neg, mode, speed));
198 18063616 : set_mul_cost (speed, mode, set_src_cost (all->mult, mode, speed));
199 18063616 : set_sdiv_cost (speed, mode, set_src_cost (all->sdiv, mode, speed));
200 18063616 : set_udiv_cost (speed, mode, set_src_cost (all->udiv, mode, speed));
201 :
202 18063616 : set_sdiv_pow2_cheap (speed, mode, (set_src_cost (all->sdiv_32, mode, speed)
203 18063616 : <= 2 * add_cost (speed, mode)));
204 18063616 : set_smod_pow2_cheap (speed, mode, (set_src_cost (all->smod_32, mode, speed)
205 18063616 : <= 4 * add_cost (speed, mode)));
206 :
207 18063616 : set_shift_cost (speed, mode, 0, 0);
208 18063616 : {
209 18063616 : int cost = add_cost (speed, mode);
210 18063616 : set_shiftadd_cost (speed, mode, 0, cost);
211 18063616 : set_shiftsub0_cost (speed, mode, 0, cost);
212 18063616 : set_shiftsub1_cost (speed, mode, 0, cost);
213 : }
214 :
215 18063616 : n = MIN (MAX_BITS_PER_WORD, mode_bitsize);
216 606232576 : for (m = 1; m < n; m++)
217 : {
218 588168960 : XEXP (all->shift, 1) = all->cint[m];
219 588168960 : XEXP (all->shift_mult, 1) = all->pow2[m];
220 :
221 588168960 : set_shift_cost (speed, mode, m, set_src_cost (all->shift, mode, speed));
222 588168960 : set_shiftadd_cost (speed, mode, m, set_src_cost (all->shift_add, mode,
223 : speed));
224 588168960 : set_shiftsub0_cost (speed, mode, m, set_src_cost (all->shift_sub0, mode,
225 : speed));
226 588168960 : set_shiftsub1_cost (speed, mode, m, set_src_cost (all->shift_sub1, mode,
227 : speed));
228 : }
229 :
230 18063616 : scalar_int_mode int_mode_to;
231 18063616 : if (is_a <scalar_int_mode> (mode, &int_mode_to))
232 : {
233 35246080 : for (mode_from = MIN_MODE_INT; mode_from <= MAX_MODE_INT;
234 30840320 : mode_from = (machine_mode)(mode_from + 1))
235 30840320 : init_expmed_one_conv (all, int_mode_to,
236 : as_a <scalar_int_mode> (mode_from), speed);
237 :
238 4405760 : scalar_int_mode wider_mode;
239 4405760 : if (GET_MODE_CLASS (int_mode_to) == MODE_INT
240 4405760 : && GET_MODE_WIDER_MODE (int_mode_to).exists (&wider_mode))
241 : {
242 2643456 : PUT_MODE (all->reg, mode);
243 2643456 : PUT_MODE (all->zext, wider_mode);
244 2643456 : PUT_MODE (all->wide_mult, wider_mode);
245 2643456 : PUT_MODE (all->wide_lshr, wider_mode);
246 2643456 : XEXP (all->wide_lshr, 1)
247 2643456 : = gen_int_shift_amount (wider_mode, mode_bitsize);
248 :
249 2643456 : set_mul_widen_cost (speed, wider_mode,
250 : set_src_cost (all->wide_mult, wider_mode, speed));
251 2643456 : set_mul_highpart_cost (speed, int_mode_to,
252 : set_src_cost (all->wide_trunc,
253 : int_mode_to, speed));
254 : }
255 : }
256 18063616 : }
257 :
258 : void
259 220288 : init_expmed (void)
260 : {
261 220288 : struct init_expmed_rtl all;
262 220288 : machine_mode mode = QImode;
263 220288 : int m, speed;
264 :
265 220288 : memset (&all, 0, sizeof all);
266 14098432 : for (m = 1; m < MAX_BITS_PER_WORD; m++)
267 : {
268 13878144 : all.pow2[m] = GEN_INT (HOST_WIDE_INT_1 << m);
269 13878144 : all.cint[m] = GEN_INT (m);
270 : }
271 :
272 : /* Avoid using hard regs in ways which may be unsupported. */
273 220288 : all.reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
274 220288 : all.plus = gen_rtx_PLUS (mode, all.reg, all.reg);
275 220288 : all.neg = gen_rtx_NEG (mode, all.reg);
276 220288 : all.mult = gen_rtx_MULT (mode, all.reg, all.reg);
277 220288 : all.sdiv = gen_rtx_DIV (mode, all.reg, all.reg);
278 220288 : all.udiv = gen_rtx_UDIV (mode, all.reg, all.reg);
279 220288 : all.sdiv_32 = gen_rtx_DIV (mode, all.reg, all.pow2[5]);
280 220288 : all.smod_32 = gen_rtx_MOD (mode, all.reg, all.pow2[5]);
281 220288 : all.zext = gen_rtx_ZERO_EXTEND (mode, all.reg);
282 220288 : all.wide_mult = gen_rtx_MULT (mode, all.zext, all.zext);
283 220288 : all.wide_lshr = gen_rtx_LSHIFTRT (mode, all.wide_mult, all.reg);
284 220288 : all.wide_trunc = gen_rtx_TRUNCATE (mode, all.wide_lshr);
285 220288 : all.shift = gen_rtx_ASHIFT (mode, all.reg, all.reg);
286 220288 : all.shift_mult = gen_rtx_MULT (mode, all.reg, all.reg);
287 220288 : all.shift_add = gen_rtx_PLUS (mode, all.shift_mult, all.reg);
288 220288 : all.shift_sub0 = gen_rtx_MINUS (mode, all.shift_mult, all.reg);
289 220288 : all.shift_sub1 = gen_rtx_MINUS (mode, all.reg, all.shift_mult);
290 220288 : all.trunc = gen_rtx_TRUNCATE (mode, all.reg);
291 :
292 660864 : for (speed = 0; speed < 2; speed++)
293 : {
294 440576 : crtl->maybe_hot_insn_p = speed;
295 440576 : set_zero_cost (speed, set_src_cost (const0_rtx, QImode, speed));
296 :
297 3524608 : for (mode = MIN_MODE_INT; mode <= MAX_MODE_INT;
298 3084032 : mode = (machine_mode)(mode + 1))
299 3084032 : init_expmed_one_mode (&all, mode, speed);
300 :
301 : if (MIN_MODE_PARTIAL_INT != VOIDmode)
302 1762304 : for (mode = MIN_MODE_PARTIAL_INT; mode <= MAX_MODE_PARTIAL_INT;
303 1321728 : mode = (machine_mode)(mode + 1))
304 1321728 : init_expmed_one_mode (&all, mode, speed);
305 :
306 : if (MIN_MODE_VECTOR_INT != VOIDmode)
307 14098432 : for (mode = MIN_MODE_VECTOR_INT; mode <= MAX_MODE_VECTOR_INT;
308 13657856 : mode = (machine_mode)(mode + 1))
309 13657856 : init_expmed_one_mode (&all, mode, speed);
310 : }
311 :
312 220288 : if (alg_hash_used_p ())
313 : {
314 1057 : struct alg_hash_entry *p = alg_hash_entry_ptr (0);
315 1057 : memset (p, 0, sizeof (*p) * NUM_ALG_HASH_ENTRIES);
316 : }
317 : else
318 219231 : set_alg_hash_used_p (true);
319 220288 : default_rtl_profile ();
320 :
321 220288 : ggc_free (all.trunc);
322 220288 : ggc_free (all.shift_sub1);
323 220288 : ggc_free (all.shift_sub0);
324 220288 : ggc_free (all.shift_add);
325 220288 : ggc_free (all.shift_mult);
326 220288 : ggc_free (all.shift);
327 220288 : ggc_free (all.wide_trunc);
328 220288 : ggc_free (all.wide_lshr);
329 220288 : ggc_free (all.wide_mult);
330 220288 : ggc_free (all.zext);
331 220288 : ggc_free (all.smod_32);
332 220288 : ggc_free (all.sdiv_32);
333 220288 : ggc_free (all.udiv);
334 220288 : ggc_free (all.sdiv);
335 220288 : ggc_free (all.mult);
336 220288 : ggc_free (all.neg);
337 220288 : ggc_free (all.plus);
338 220288 : ggc_free (all.reg);
339 220288 : }
340 :
341 : /* Return an rtx representing minus the value of X.
342 : MODE is the intended mode of the result,
343 : useful if X is a CONST_INT. */
344 :
345 : rtx
346 1038494 : negate_rtx (machine_mode mode, rtx x)
347 : {
348 1038494 : rtx result = simplify_unary_operation (NEG, mode, x, mode);
349 :
350 1038494 : if (result == 0)
351 1321 : result = expand_unop (mode, neg_optab, x, NULL_RTX, 0);
352 :
353 1038494 : return result;
354 : }
355 :
356 : /* Whether reverse storage order is supported on the target. */
357 : static int reverse_storage_order_supported = -1;
358 :
359 : /* Check whether reverse storage order is supported on the target. */
360 :
361 : static void
362 293 : check_reverse_storage_order_support (void)
363 : {
364 293 : if (BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
365 : {
366 : reverse_storage_order_supported = 0;
367 : sorry ("reverse scalar storage order");
368 : }
369 : else
370 293 : reverse_storage_order_supported = 1;
371 0 : }
372 :
373 : /* Whether reverse FP storage order is supported on the target. */
374 : static int reverse_float_storage_order_supported = -1;
375 :
376 : /* Check whether reverse FP storage order is supported on the target. */
377 :
378 : static void
379 55 : check_reverse_float_storage_order_support (void)
380 : {
381 55 : if (FLOAT_WORDS_BIG_ENDIAN != WORDS_BIG_ENDIAN)
382 : {
383 : reverse_float_storage_order_supported = 0;
384 : sorry ("reverse floating-point scalar storage order");
385 : }
386 : else
387 55 : reverse_float_storage_order_supported = 1;
388 0 : }
389 :
390 : /* Return an rtx representing value of X with reverse storage order.
391 : MODE is the intended mode of the result,
392 : useful if X is a CONST_INT. */
393 :
394 : rtx
395 3319 : flip_storage_order (machine_mode mode, rtx x)
396 : {
397 3319 : scalar_int_mode int_mode;
398 3319 : rtx result;
399 :
400 3319 : if (mode == QImode)
401 : return x;
402 :
403 2441 : if (COMPLEX_MODE_P (mode))
404 : {
405 44 : rtx real = read_complex_part (x, false);
406 44 : rtx imag = read_complex_part (x, true);
407 :
408 88 : real = flip_storage_order (GET_MODE_INNER (mode), real);
409 88 : imag = flip_storage_order (GET_MODE_INNER (mode), imag);
410 :
411 44 : return gen_rtx_CONCAT (mode, real, imag);
412 : }
413 :
414 2397 : if (UNLIKELY (reverse_storage_order_supported < 0))
415 293 : check_reverse_storage_order_support ();
416 :
417 2397 : if (!is_a <scalar_int_mode> (mode, &int_mode))
418 : {
419 243 : if (FLOAT_MODE_P (mode)
420 243 : && UNLIKELY (reverse_float_storage_order_supported < 0))
421 55 : check_reverse_float_storage_order_support ();
422 :
423 243 : if (!int_mode_for_size (GET_MODE_PRECISION (mode), 0).exists (&int_mode)
424 243 : || !targetm.scalar_mode_supported_p (int_mode))
425 : {
426 0 : sorry ("reverse storage order for %smode", GET_MODE_NAME (mode));
427 0 : return x;
428 : }
429 243 : x = gen_lowpart (int_mode, x);
430 : }
431 :
432 2397 : result = simplify_unary_operation (BSWAP, int_mode, x, int_mode);
433 2397 : if (result == 0)
434 1073 : result = expand_unop (int_mode, bswap_optab, x, NULL_RTX, 1);
435 :
436 2397 : if (int_mode != mode)
437 243 : result = gen_lowpart (mode, result);
438 :
439 : return result;
440 : }
441 :
442 : /* If MODE is set, adjust bitfield memory MEM so that it points to the
443 : first unit of mode MODE that contains a bitfield of size BITSIZE at
444 : bit position BITNUM. If MODE is not set, return a BLKmode reference
445 : to every byte in the bitfield. Set *NEW_BITNUM to the bit position
446 : of the field within the new memory. */
447 :
448 : static rtx
449 425284 : narrow_bit_field_mem (rtx mem, opt_scalar_int_mode mode,
450 : unsigned HOST_WIDE_INT bitsize,
451 : unsigned HOST_WIDE_INT bitnum,
452 : unsigned HOST_WIDE_INT *new_bitnum)
453 : {
454 425284 : scalar_int_mode imode;
455 425284 : if (mode.exists (&imode))
456 : {
457 425284 : unsigned int unit = GET_MODE_BITSIZE (imode);
458 425284 : *new_bitnum = bitnum % unit;
459 425284 : HOST_WIDE_INT offset = (bitnum - *new_bitnum) / BITS_PER_UNIT;
460 425284 : return adjust_bitfield_address (mem, imode, offset);
461 : }
462 : else
463 : {
464 0 : *new_bitnum = bitnum % BITS_PER_UNIT;
465 0 : HOST_WIDE_INT offset = bitnum / BITS_PER_UNIT;
466 0 : HOST_WIDE_INT size = ((*new_bitnum + bitsize + BITS_PER_UNIT - 1)
467 0 : / BITS_PER_UNIT);
468 0 : return adjust_bitfield_address_size (mem, BLKmode, offset, size);
469 : }
470 : }
471 :
472 : /* The caller wants to perform insertion or extraction PATTERN on a
473 : bitfield of size BITSIZE at BITNUM bits into memory operand OP0.
474 : BITREGION_START and BITREGION_END are as for store_bit_field
475 : and FIELDMODE is the natural mode of the field.
476 :
477 : Search for a mode that is compatible with the memory access
478 : restrictions and (where applicable) with a register insertion or
479 : extraction. Return the new memory on success, storing the adjusted
480 : bit position in *NEW_BITNUM. Return null otherwise. */
481 :
482 : static rtx
483 205471 : adjust_bit_field_mem_for_reg (enum extraction_pattern pattern,
484 : rtx op0, HOST_WIDE_INT bitsize,
485 : HOST_WIDE_INT bitnum,
486 : poly_uint64 bitregion_start,
487 : poly_uint64 bitregion_end,
488 : machine_mode fieldmode,
489 : unsigned HOST_WIDE_INT *new_bitnum)
490 : {
491 410942 : bit_field_mode_iterator iter (bitsize, bitnum, bitregion_start,
492 410942 : bitregion_end, MEM_ALIGN (op0),
493 410942 : MEM_VOLATILE_P (op0));
494 205471 : scalar_int_mode best_mode;
495 205471 : if (iter.next_mode (&best_mode))
496 : {
497 : /* We can use a memory in BEST_MODE. See whether this is true for
498 : any wider modes. All other things being equal, we prefer to
499 : use the widest mode possible because it tends to expose more
500 : CSE opportunities. */
501 198971 : if (!iter.prefer_smaller_modes ())
502 : {
503 : /* Limit the search to the mode required by the corresponding
504 : register insertion or extraction instruction, if any. */
505 317 : scalar_int_mode limit_mode = word_mode;
506 317 : extraction_insn insn;
507 634 : if (get_best_reg_extraction_insn (&insn, pattern,
508 317 : GET_MODE_BITSIZE (best_mode),
509 : fieldmode))
510 317 : limit_mode = insn.field_mode;
511 :
512 317 : scalar_int_mode wider_mode;
513 317 : while (iter.next_mode (&wider_mode)
514 1052 : && GET_MODE_SIZE (wider_mode) <= GET_MODE_SIZE (limit_mode))
515 111 : best_mode = wider_mode;
516 : }
517 198971 : return narrow_bit_field_mem (op0, best_mode, bitsize, bitnum,
518 : new_bitnum);
519 : }
520 : return NULL_RTX;
521 : }
522 :
523 : /* Return true if a bitfield of size BITSIZE at bit number BITNUM within
524 : a structure of mode STRUCT_MODE represents a lowpart subreg. The subreg
525 : offset is then BITNUM / BITS_PER_UNIT. */
526 :
527 : static bool
528 832000 : lowpart_bit_field_p (poly_uint64 bitnum, poly_uint64 bitsize,
529 : machine_mode struct_mode)
530 : {
531 832000 : poly_uint64 regsize = REGMODE_NATURAL_SIZE (struct_mode);
532 832000 : if (BYTES_BIG_ENDIAN)
533 : return (multiple_p (bitnum, BITS_PER_UNIT)
534 : && (known_eq (bitnum + bitsize, GET_MODE_BITSIZE (struct_mode))
535 : || multiple_p (bitnum + bitsize,
536 : regsize * BITS_PER_UNIT)));
537 : else
538 832000 : return multiple_p (bitnum, regsize * BITS_PER_UNIT);
539 : }
540 :
541 : /* Return true if -fstrict-volatile-bitfields applies to an access of OP0
542 : containing BITSIZE bits starting at BITNUM, with field mode FIELDMODE.
543 : Return false if the access would touch memory outside the range
544 : BITREGION_START to BITREGION_END for conformance to the C++ memory
545 : model. */
546 :
547 : static bool
548 1604861 : strict_volatile_bitfield_p (rtx op0, unsigned HOST_WIDE_INT bitsize,
549 : unsigned HOST_WIDE_INT bitnum,
550 : scalar_int_mode fieldmode,
551 : poly_uint64 bitregion_start,
552 : poly_uint64 bitregion_end)
553 : {
554 1604861 : unsigned HOST_WIDE_INT modesize = GET_MODE_BITSIZE (fieldmode);
555 :
556 : /* -fstrict-volatile-bitfields must be enabled and we must have a
557 : volatile MEM. */
558 1604861 : if (!MEM_P (op0)
559 166380 : || !MEM_VOLATILE_P (op0)
560 1605073 : || flag_strict_volatile_bitfields <= 0)
561 : return false;
562 :
563 : /* The bit size must not be larger than the field mode, and
564 : the field mode must not be larger than a word. */
565 14 : if (bitsize > modesize || modesize > BITS_PER_WORD)
566 : return false;
567 :
568 : /* Check for cases of unaligned fields that must be split. */
569 14 : if (bitnum % modesize + bitsize > modesize)
570 : return false;
571 :
572 : /* The memory must be sufficiently aligned for a MODESIZE access.
573 : This condition guarantees, that the memory access will not
574 : touch anything after the end of the structure. */
575 11 : if (MEM_ALIGN (op0) < modesize)
576 : return false;
577 :
578 : /* Check for cases where the C++ memory model applies. */
579 11 : if (maybe_ne (bitregion_end, 0U)
580 11 : && (maybe_lt (bitnum - bitnum % modesize, bitregion_start)
581 4 : || maybe_gt (bitnum - bitnum % modesize + modesize - 1,
582 : bitregion_end)))
583 0 : return false;
584 :
585 : return true;
586 : }
587 :
588 : /* Return true if OP is a memory and if a bitfield of size BITSIZE at
589 : bit number BITNUM can be treated as a simple value of mode MODE.
590 : Store the byte offset in *BYTENUM if so. */
591 :
592 : static bool
593 556858 : simple_mem_bitfield_p (rtx op0, poly_uint64 bitsize, poly_uint64 bitnum,
594 : machine_mode mode, poly_uint64 *bytenum)
595 : {
596 556858 : return (MEM_P (op0)
597 260720 : && multiple_p (bitnum, BITS_PER_UNIT, bytenum)
598 208725 : && known_eq (bitsize, GET_MODE_BITSIZE (mode))
599 609220 : && (!targetm.slow_unaligned_access (mode, MEM_ALIGN (op0))
600 0 : || (multiple_p (bitnum, GET_MODE_ALIGNMENT (mode))
601 0 : && MEM_ALIGN (op0) >= GET_MODE_ALIGNMENT (mode))));
602 : }
603 :
604 : /* Try to use instruction INSV to store VALUE into a field of OP0.
605 : If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is a
606 : BLKmode MEM. VALUE_MODE is the mode of VALUE. BITSIZE and BITNUM
607 : are as for store_bit_field. */
608 :
609 : static bool
610 108993 : store_bit_field_using_insv (const extraction_insn *insv, rtx op0,
611 : opt_scalar_int_mode op0_mode,
612 : unsigned HOST_WIDE_INT bitsize,
613 : unsigned HOST_WIDE_INT bitnum,
614 : rtx value, scalar_int_mode value_mode)
615 : {
616 108993 : class expand_operand ops[4];
617 108993 : rtx value1;
618 108993 : rtx xop0 = op0;
619 108993 : rtx_insn *last = get_last_insn ();
620 108993 : bool copy_back = false;
621 :
622 108993 : scalar_int_mode op_mode = insv->field_mode;
623 108993 : unsigned int unit = GET_MODE_BITSIZE (op_mode);
624 108993 : if (bitsize == 0 || bitsize > unit)
625 : return false;
626 :
627 108980 : if (MEM_P (xop0))
628 : /* Get a reference to the first byte of the field. */
629 0 : xop0 = narrow_bit_field_mem (xop0, insv->struct_mode, bitsize, bitnum,
630 : &bitnum);
631 : else
632 : {
633 : /* Convert from counting within OP0 to counting in OP_MODE. */
634 108980 : if (BYTES_BIG_ENDIAN)
635 : bitnum += unit - GET_MODE_BITSIZE (op0_mode.require ());
636 :
637 : /* If xop0 is a register, we need it in OP_MODE
638 : to make it acceptable to the format of insv. */
639 108980 : if (GET_CODE (xop0) == SUBREG)
640 : {
641 : /* If such a SUBREG can't be created, give up. */
642 34940 : if (!validate_subreg (op_mode, GET_MODE (SUBREG_REG (xop0)),
643 34940 : SUBREG_REG (xop0), SUBREG_BYTE (xop0)))
644 : return false;
645 : /* We can't just change the mode, because this might clobber op0,
646 : and we will need the original value of op0 if insv fails. */
647 34940 : xop0 = gen_rtx_SUBREG (op_mode, SUBREG_REG (xop0),
648 34940 : SUBREG_BYTE (xop0));
649 : }
650 108980 : if (REG_P (xop0) && GET_MODE (xop0) != op_mode)
651 27088 : xop0 = gen_lowpart_SUBREG (op_mode, xop0);
652 : }
653 :
654 : /* If the destination is a paradoxical subreg such that we need a
655 : truncate to the inner mode, perform the insertion on a temporary and
656 : truncate the result to the original destination. Note that we can't
657 : just truncate the paradoxical subreg as (truncate:N (subreg:W (reg:N
658 : X) 0)) is (reg:N X). */
659 108980 : if (GET_CODE (xop0) == SUBREG
660 62028 : && REG_P (SUBREG_REG (xop0))
661 171008 : && !TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (SUBREG_REG (xop0)),
662 : op_mode))
663 : {
664 0 : rtx tem = gen_reg_rtx (op_mode);
665 0 : emit_move_insn (tem, xop0);
666 0 : xop0 = tem;
667 0 : copy_back = true;
668 : }
669 :
670 : /* There are similar overflow check at the start of store_bit_field_1,
671 : but that only check the situation where the field lies completely
672 : outside the register, while there do have situation where the field
673 : lies partially in the register, we need to adjust bitsize for this
674 : partial overflow situation. Without this fix, pr48335-2.c on big-endian
675 : will broken on those arch support bit insert instruction, like arm, aarch64
676 : etc. */
677 108980 : if (bitsize + bitnum > unit && bitnum < unit)
678 : {
679 2 : warning (OPT_Wextra, "write of %wu-bit data outside the bound of "
680 : "destination object, data truncated into %wu-bit",
681 : bitsize, unit - bitnum);
682 2 : bitsize = unit - bitnum;
683 : }
684 :
685 : /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
686 : "backwards" from the size of the unit we are inserting into.
687 : Otherwise, we count bits from the most significant on a
688 : BYTES/BITS_BIG_ENDIAN machine. */
689 :
690 108980 : if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
691 : bitnum = unit - bitsize - bitnum;
692 :
693 : /* Convert VALUE to op_mode (which insv insn wants) in VALUE1. */
694 108980 : value1 = value;
695 108980 : if (value_mode != op_mode)
696 : {
697 139952 : if (GET_MODE_BITSIZE (value_mode) >= bitsize)
698 : {
699 69976 : rtx tmp;
700 : /* Optimization: Don't bother really extending VALUE
701 : if it has all the bits we will actually use. However,
702 : if we must narrow it, be sure we do it correctly. */
703 :
704 209928 : if (GET_MODE_SIZE (value_mode) < GET_MODE_SIZE (op_mode))
705 : {
706 32541 : tmp = simplify_subreg (op_mode, value1, value_mode, 0);
707 32541 : if (! tmp)
708 31954 : tmp = simplify_gen_subreg (op_mode,
709 : force_reg (value_mode, value1),
710 : value_mode, 0);
711 : }
712 : else
713 : {
714 37435 : if (targetm.mode_rep_extended (op_mode, value_mode) != UNKNOWN)
715 0 : tmp = simplify_gen_unary (TRUNCATE, op_mode,
716 : value1, value_mode);
717 : else
718 : {
719 37435 : tmp = gen_lowpart_if_possible (op_mode, value1);
720 37435 : if (! tmp)
721 0 : tmp = gen_lowpart (op_mode, force_reg (value_mode, value1));
722 : }
723 : }
724 : value1 = tmp;
725 : }
726 0 : else if (CONST_INT_P (value))
727 0 : value1 = gen_int_mode (INTVAL (value), op_mode);
728 : else
729 : /* Parse phase is supposed to make VALUE's data type
730 : match that of the component reference, which is a type
731 : at least as wide as the field; so VALUE should have
732 : a mode that corresponds to that type. */
733 0 : gcc_assert (CONSTANT_P (value));
734 : }
735 :
736 108980 : create_fixed_operand (&ops[0], xop0);
737 108980 : create_integer_operand (&ops[1], bitsize);
738 108980 : create_integer_operand (&ops[2], bitnum);
739 108980 : create_input_operand (&ops[3], value1, op_mode);
740 108980 : if (maybe_expand_insn (insv->icode, 4, ops))
741 : {
742 2107 : if (copy_back)
743 0 : convert_move (op0, xop0, true);
744 : return true;
745 : }
746 106873 : delete_insns_since (last);
747 106873 : return false;
748 : }
749 :
750 : /* A subroutine of store_bit_field, with the same arguments. Return true
751 : if the operation could be implemented.
752 :
753 : If FALLBACK_P is true, fall back to store_fixed_bit_field if we have
754 : no other way of implementing the operation. If FALLBACK_P is false,
755 : return false instead.
756 :
757 : if UNDEFINED_P is true then STR_RTX is undefined and may be set using
758 : a subreg instead. */
759 :
760 : static bool
761 903448 : store_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
762 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
763 : machine_mode fieldmode,
764 : rtx value, bool reverse, bool fallback_p, bool undefined_p)
765 : {
766 903448 : rtx op0 = str_rtx;
767 :
768 903454 : while (GET_CODE (op0) == SUBREG)
769 : {
770 6 : bitnum += subreg_memory_offset (op0) * BITS_PER_UNIT;
771 6 : op0 = SUBREG_REG (op0);
772 : }
773 :
774 : /* No action is needed if the target is a register and if the field
775 : lies completely outside that register. This can occur if the source
776 : code contains an out-of-bounds access to a small array. */
777 1731588 : if (REG_P (op0) && known_ge (bitnum, GET_MODE_BITSIZE (GET_MODE (op0))))
778 : return true;
779 :
780 : /* Use vec_set patterns for inserting parts of vectors whenever
781 : available. */
782 903445 : machine_mode outermode = GET_MODE (op0);
783 903445 : scalar_mode innermode = GET_MODE_INNER (outermode);
784 903445 : poly_uint64 pos;
785 901390 : if (VECTOR_MODE_P (outermode)
786 2434 : && !MEM_P (op0)
787 2422 : && optab_handler (vec_set_optab, outermode) != CODE_FOR_nothing
788 1224 : && fieldmode == innermode
789 963 : && known_eq (bitsize, GET_MODE_PRECISION (innermode))
790 904408 : && multiple_p (bitnum, GET_MODE_PRECISION (innermode), &pos))
791 : {
792 963 : class expand_operand ops[3];
793 963 : enum insn_code icode = optab_handler (vec_set_optab, outermode);
794 :
795 963 : create_fixed_operand (&ops[0], op0);
796 963 : create_input_operand (&ops[1], value, innermode);
797 963 : create_integer_operand (&ops[2], pos);
798 963 : if (maybe_expand_insn (icode, 3, ops))
799 963 : return true;
800 : }
801 :
802 : /* If the target is a register, overwriting the entire object, or storing
803 : a full-word or multi-word field can be done with just a SUBREG. */
804 902482 : if (!MEM_P (op0)
805 1729656 : && known_eq (bitsize, GET_MODE_BITSIZE (fieldmode)))
806 : {
807 : /* Use the subreg machinery either to narrow OP0 to the required
808 : words or to cope with mode punning between equal-sized modes.
809 : In the latter case, use subreg on the rhs side, not lhs. */
810 760071 : rtx sub;
811 760071 : poly_uint64 bytenum;
812 760071 : poly_uint64 regsize = REGMODE_NATURAL_SIZE (GET_MODE (op0));
813 760071 : if (known_eq (bitnum, 0U)
814 1143600 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0))))
815 : {
816 60522 : sub = force_subreg (GET_MODE (op0), value, fieldmode, 0);
817 60522 : if (sub)
818 : {
819 60522 : if (reverse)
820 1 : sub = flip_storage_order (GET_MODE (op0), sub);
821 60522 : emit_move_insn (op0, sub);
822 60522 : return true;
823 : }
824 : }
825 889090 : else if (multiple_p (bitnum, BITS_PER_UNIT, &bytenum)
826 699542 : && (undefined_p
827 695888 : || (multiple_p (bitnum, regsize * BITS_PER_UNIT)
828 684438 : && multiple_p (bitsize, regsize * BITS_PER_UNIT)))
829 1304860 : && known_ge (GET_MODE_BITSIZE (GET_MODE (op0)), bitsize))
830 : {
831 652414 : sub = simplify_gen_subreg (fieldmode, op0, GET_MODE (op0), bytenum);
832 652414 : if (sub)
833 : {
834 652412 : if (reverse)
835 0 : value = flip_storage_order (fieldmode, value);
836 652412 : emit_move_insn (sub, value);
837 652412 : return true;
838 : }
839 : }
840 : }
841 :
842 : /* If the target is memory, storing any naturally aligned field can be
843 : done with a simple store. For targets that support fast unaligned
844 : memory, any naturally sized, unit aligned field can be done directly. */
845 189548 : poly_uint64 bytenum;
846 189548 : if (simple_mem_bitfield_p (op0, bitsize, bitnum, fieldmode, &bytenum))
847 : {
848 7035 : op0 = adjust_bitfield_address (op0, fieldmode, bytenum);
849 7035 : if (reverse)
850 0 : value = flip_storage_order (fieldmode, value);
851 7035 : emit_move_insn (op0, value);
852 7035 : return true;
853 : }
854 :
855 : /* It's possible we'll need to handle other cases here for
856 : polynomial bitnum and bitsize. */
857 :
858 : /* From here on we need to be looking at a fixed-size insertion. */
859 182513 : unsigned HOST_WIDE_INT ibitsize = bitsize.to_constant ();
860 182513 : unsigned HOST_WIDE_INT ibitnum = bitnum.to_constant ();
861 :
862 : /* Make sure we are playing with integral modes. Pun with subregs
863 : if we aren't. This must come after the entire register case above,
864 : since that case is valid for any mode. The following cases are only
865 : valid for integral modes. */
866 182513 : opt_scalar_int_mode op0_mode = int_mode_for_mode (GET_MODE (op0));
867 182513 : scalar_int_mode imode;
868 182513 : bool need_stack_p = false;
869 182513 : if (!op0_mode.exists (&imode) || imode != GET_MODE (op0))
870 : {
871 19925 : if (MEM_P (op0))
872 15250 : op0 = adjust_bitfield_address_size (op0, op0_mode.else_blk (),
873 : 0, MEM_SIZE (op0));
874 4675 : else if (!op0_mode.exists ()
875 4675 : || maybe_lt
876 4815 : ((unsigned) UNITS_PER_WORD,
877 4675 : (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
878 : {
879 0 : if (ibitnum == 0
880 0 : && known_eq (ibitsize, GET_MODE_BITSIZE (GET_MODE (op0)))
881 0 : && MEM_P (value)
882 0 : && !reverse)
883 : {
884 0 : value = adjust_address (value, GET_MODE (op0), 0);
885 0 : emit_move_insn (op0, value);
886 0 : return true;
887 : }
888 0 : if (!fallback_p)
889 : return false;
890 : need_stack_p = true;
891 : }
892 : else
893 4675 : op0 = gen_lowpart (op0_mode.require (), op0);
894 : }
895 162588 : else if (!MEM_P (op0)
896 273496 : && maybe_lt ((unsigned) UNITS_PER_WORD,
897 109565 : (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
898 : need_stack_p = true;
899 :
900 : /* With or without punning we might be faced with a mode that we cannot
901 : split into words. If so, spill OP0 to the stack and recurse.
902 : This happens at most once. */
903 19925 : if (need_stack_p)
904 : {
905 0 : rtx temp = assign_stack_temp (GET_MODE (op0),
906 0 : GET_MODE_SIZE (GET_MODE (op0)));
907 0 : emit_move_insn (temp, op0);
908 0 : store_bit_field_1 (temp, bitsize, bitnum, 0, 0, fieldmode, value,
909 : reverse, fallback_p, undefined_p);
910 0 : emit_move_insn (op0, temp);
911 0 : return true;
912 : }
913 :
914 182513 : return store_integral_bit_field (op0, op0_mode, ibitsize, ibitnum,
915 : bitregion_start, bitregion_end,
916 182513 : fieldmode, value, reverse, fallback_p);
917 : }
918 :
919 : /* Subroutine of store_bit_field_1, with the same arguments, except
920 : that BITSIZE and BITNUM are constant. Handle cases specific to
921 : integral modes. If OP0_MODE is defined, it is the mode of OP0,
922 : otherwise OP0 is a BLKmode MEM. */
923 :
924 : static bool
925 182513 : store_integral_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
926 : unsigned HOST_WIDE_INT bitsize,
927 : unsigned HOST_WIDE_INT bitnum,
928 : poly_uint64 bitregion_start,
929 : poly_uint64 bitregion_end,
930 : machine_mode fieldmode,
931 : rtx value, bool reverse, bool fallback_p)
932 : {
933 : /* Storing an lsb-aligned field in a register
934 : can be done with a movstrict instruction. */
935 :
936 182513 : if (!MEM_P (op0)
937 114240 : && !reverse
938 374602 : && lowpart_bit_field_p (bitnum, bitsize, op0_mode.require ())
939 82302 : && known_eq (bitsize, GET_MODE_BITSIZE (fieldmode))
940 218193 : && optab_handler (movstrict_optab, fieldmode) != CODE_FOR_nothing)
941 : {
942 4426 : class expand_operand ops[2];
943 4426 : enum insn_code icode = optab_handler (movstrict_optab, fieldmode);
944 4426 : rtx arg0 = op0;
945 4426 : unsigned HOST_WIDE_INT subreg_off;
946 :
947 4426 : if (GET_CODE (arg0) == SUBREG)
948 : {
949 : /* Else we've got some float mode source being extracted into
950 : a different float mode destination -- this combination of
951 : subregs results in Severe Tire Damage. */
952 467 : gcc_assert (GET_MODE (SUBREG_REG (arg0)) == fieldmode
953 : || GET_MODE_CLASS (fieldmode) == MODE_INT
954 : || GET_MODE_CLASS (fieldmode) == MODE_PARTIAL_INT);
955 : arg0 = SUBREG_REG (arg0);
956 : }
957 :
958 4426 : subreg_off = bitnum / BITS_PER_UNIT;
959 4460 : if (validate_subreg (fieldmode, GET_MODE (arg0), arg0, subreg_off)
960 : /* STRICT_LOW_PART must have a non-paradoxical subreg as
961 : operand. */
962 4426 : && !paradoxical_subreg_p (fieldmode, GET_MODE (arg0)))
963 : {
964 4392 : arg0 = gen_rtx_SUBREG (fieldmode, arg0, subreg_off);
965 :
966 4392 : create_fixed_operand (&ops[0], arg0);
967 : /* Shrink the source operand to FIELDMODE. */
968 4392 : create_convert_operand_to (&ops[1], value, fieldmode, false);
969 4392 : if (maybe_expand_insn (icode, 2, ops))
970 4391 : return true;
971 : }
972 : }
973 :
974 : /* Handle fields bigger than a word. */
975 :
976 179711 : if (bitsize > BITS_PER_WORD)
977 : {
978 : /* Here we transfer the words of the field
979 : in the order least significant first.
980 : This is because the most significant word is the one which may
981 : be less than full.
982 : However, only do that if the value is not BLKmode. */
983 :
984 915 : const bool backwards = WORDS_BIG_ENDIAN && fieldmode != BLKmode;
985 915 : const int nwords = (bitsize + (BITS_PER_WORD - 1)) / BITS_PER_WORD;
986 915 : rtx_insn *last;
987 :
988 : /* This is the mode we must force value to, so that there will be enough
989 : subwords to extract. Note that fieldmode will often (always?) be
990 : VOIDmode, because that is what store_field uses to indicate that this
991 : is a bit field, but passing VOIDmode to operand_subword_force
992 : is not allowed.
993 :
994 : The mode must be fixed-size, since insertions into variable-sized
995 : objects are meant to be handled before calling this function. */
996 915 : fixed_size_mode value_mode = as_a <fixed_size_mode> (GET_MODE (value));
997 915 : if (value_mode == VOIDmode)
998 24 : value_mode
999 24 : = smallest_int_mode_for_size (nwords * BITS_PER_WORD).require ();
1000 :
1001 915 : last = get_last_insn ();
1002 2740 : for (int i = 0; i < nwords; i++)
1003 : {
1004 : /* Number of bits to be stored in this iteration, i.e. BITS_PER_WORD
1005 : except maybe for the last iteration. */
1006 3660 : const unsigned HOST_WIDE_INT new_bitsize
1007 1866 : = MIN (BITS_PER_WORD, bitsize - i * BITS_PER_WORD);
1008 : /* Bit offset from the starting bit number in the target. */
1009 3672 : const unsigned int bit_offset
1010 : = backwards ^ reverse
1011 1830 : ? MAX ((int) bitsize - (i + 1) * BITS_PER_WORD, 0)
1012 : : i * BITS_PER_WORD;
1013 :
1014 : /* No further action is needed if the target is a register and if
1015 : this field lies completely outside that register. */
1016 2102 : if (REG_P (op0) && known_ge (bitnum + bit_offset,
1017 : GET_MODE_BITSIZE (GET_MODE (op0))))
1018 : {
1019 5 : if (backwards ^ reverse)
1020 0 : continue;
1021 : /* For forward operation we are finished. */
1022 182513 : return true;
1023 : }
1024 :
1025 : /* Starting word number in the value. */
1026 1825 : const unsigned int wordnum
1027 : = backwards
1028 1825 : ? GET_MODE_SIZE (value_mode) / UNITS_PER_WORD - (i + 1)
1029 : : i;
1030 : /* The chunk of the value in word_mode. We use bit-field extraction
1031 : in BLKmode to handle unaligned memory references and to shift the
1032 : last chunk right on big-endian machines if need be. */
1033 1825 : rtx value_word
1034 : = (fieldmode == BLKmode
1035 1576 : || maybe_lt ((unsigned) UNITS_PER_WORD,
1036 1576 : (poly_uint64) REGMODE_NATURAL_SIZE (value_mode)))
1037 1861 : ? extract_bit_field (value, new_bitsize, wordnum * BITS_PER_WORD,
1038 : 1, NULL_RTX, word_mode, word_mode, false,
1039 : NULL)
1040 1825 : : operand_subword_force (value, wordnum, value_mode);
1041 :
1042 1825 : if (!store_bit_field_1 (op0, new_bitsize,
1043 1825 : bitnum + bit_offset,
1044 : bitregion_start, bitregion_end,
1045 : word_mode,
1046 : value_word, reverse, fallback_p, false))
1047 : {
1048 0 : delete_insns_since (last);
1049 0 : return false;
1050 : }
1051 : }
1052 : return true;
1053 : }
1054 :
1055 : /* If VALUE has a floating-point or complex mode, access it as an
1056 : integer of the corresponding size. This can occur on a machine
1057 : with 64 bit registers that uses SFmode for float. It can also
1058 : occur for unaligned float or complex fields. */
1059 177207 : rtx orig_value = value;
1060 177207 : scalar_int_mode value_mode;
1061 177207 : if (GET_MODE (value) == VOIDmode)
1062 : /* By this point we've dealt with values that are bigger than a word,
1063 : so word_mode is a conservatively correct choice. */
1064 105863 : value_mode = word_mode;
1065 71344 : else if (!is_a <scalar_int_mode> (GET_MODE (value), &value_mode))
1066 : {
1067 1131 : value_mode = int_mode_for_mode (GET_MODE (value)).require ();
1068 1131 : value = gen_reg_rtx (value_mode);
1069 1131 : emit_move_insn (gen_lowpart (GET_MODE (orig_value), value), orig_value);
1070 : }
1071 :
1072 : /* If OP0 is a multi-word register, narrow it to the affected word.
1073 : If the region spans two words, defer to store_split_bit_field.
1074 : Don't do this if op0 is a single hard register wider than word
1075 : such as a float or vector register. */
1076 177207 : if (!MEM_P (op0)
1077 220327 : && GET_MODE_SIZE (op0_mode.require ()) > UNITS_PER_WORD
1078 212200 : && (!REG_P (op0)
1079 34972 : || !HARD_REGISTER_P (op0)
1080 177207 : || hard_regno_nregs (REGNO (op0), op0_mode.require ()) != 1))
1081 : {
1082 35066 : if (bitnum % BITS_PER_WORD + bitsize > BITS_PER_WORD)
1083 : {
1084 698 : if (!fallback_p)
1085 : return false;
1086 :
1087 71 : store_split_bit_field (op0, op0_mode, bitsize, bitnum,
1088 : bitregion_start, bitregion_end,
1089 : value, value_mode, reverse);
1090 71 : return true;
1091 : }
1092 34293 : rtx new_op0
1093 34293 : = simplify_gen_subreg (word_mode, op0, op0_mode.require (),
1094 34368 : bitnum / BITS_PER_WORD * UNITS_PER_WORD);
1095 34293 : if (!new_op0)
1096 : {
1097 : /* No valid word-mode SUBREG of op0 at this offset. Defer to
1098 : store_split_bit_field, which addresses op0 a word at a time. */
1099 0 : if (!fallback_p)
1100 : return false;
1101 0 : store_split_bit_field (op0, op0_mode, bitsize, bitnum,
1102 : bitregion_start, bitregion_end,
1103 : value, value_mode, reverse);
1104 0 : return true;
1105 : }
1106 34293 : op0 = new_op0;
1107 34293 : op0_mode = word_mode;
1108 34368 : bitnum %= BITS_PER_WORD;
1109 : }
1110 :
1111 : /* From here on we can assume that the field to be stored in fits
1112 : within a word. If the destination is a register, it too fits
1113 : in a word. */
1114 :
1115 176509 : extraction_insn insv;
1116 176509 : if (!MEM_P (op0)
1117 109012 : && !reverse
1118 108995 : && get_best_reg_extraction_insn (&insv, EP_insv,
1119 217990 : GET_MODE_BITSIZE (op0_mode.require ()),
1120 : fieldmode)
1121 285502 : && store_bit_field_using_insv (&insv, op0, op0_mode,
1122 : bitsize, bitnum, value, value_mode))
1123 2107 : return true;
1124 :
1125 : /* If OP0 is a memory, try copying it to a register and seeing if a
1126 : cheap register alternative is available. */
1127 174402 : if (MEM_P (op0) && !reverse)
1128 : {
1129 67098 : if (get_best_mem_extraction_insn (&insv, EP_insv, bitsize, bitnum,
1130 : fieldmode)
1131 67098 : && store_bit_field_using_insv (&insv, op0, op0_mode,
1132 : bitsize, bitnum, value, value_mode))
1133 0 : return true;
1134 :
1135 67098 : rtx_insn *last = get_last_insn ();
1136 :
1137 : /* Try loading part of OP0 into a register, inserting the bitfield
1138 : into that, and then copying the result back to OP0. */
1139 67098 : unsigned HOST_WIDE_INT bitpos;
1140 67098 : rtx xop0 = adjust_bit_field_mem_for_reg (EP_insv, op0, bitsize, bitnum,
1141 : bitregion_start, bitregion_end,
1142 : fieldmode, &bitpos);
1143 67098 : if (xop0)
1144 : {
1145 63263 : rtx tempreg = copy_to_reg (xop0);
1146 63263 : if (store_bit_field_1 (tempreg, bitsize, bitpos,
1147 : bitregion_start, bitregion_end,
1148 : fieldmode, orig_value, reverse, false, false))
1149 : {
1150 0 : emit_move_insn (xop0, tempreg);
1151 0 : return true;
1152 : }
1153 63263 : delete_insns_since (last);
1154 : }
1155 : }
1156 :
1157 174402 : if (!fallback_p)
1158 : return false;
1159 :
1160 111766 : store_fixed_bit_field (op0, op0_mode, bitsize, bitnum, bitregion_start,
1161 : bitregion_end, value, value_mode, reverse);
1162 111766 : return true;
1163 : }
1164 :
1165 : /* Generate code to store value from rtx VALUE
1166 : into a bit-field within structure STR_RTX
1167 : containing BITSIZE bits starting at bit BITNUM.
1168 :
1169 : BITREGION_START is bitpos of the first bitfield in this region.
1170 : BITREGION_END is the bitpos of the ending bitfield in this region.
1171 : These two fields are 0, if the C++ memory model does not apply,
1172 : or we are not interested in keeping track of bitfield regions.
1173 :
1174 : FIELDMODE is the machine-mode of the FIELD_DECL node for this field.
1175 :
1176 : If REVERSE is true, the store is to be done in reverse order.
1177 :
1178 : If UNDEFINED_P is true then STR_RTX is currently undefined. */
1179 :
1180 : void
1181 838360 : store_bit_field (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
1182 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
1183 : machine_mode fieldmode,
1184 : rtx value, bool reverse, bool undefined_p)
1185 : {
1186 : /* Handle -fstrict-volatile-bitfields in the cases where it applies. */
1187 838360 : unsigned HOST_WIDE_INT ibitsize = 0, ibitnum = 0;
1188 838360 : scalar_int_mode int_mode;
1189 838360 : if (bitsize.is_constant (&ibitsize)
1190 838360 : && bitnum.is_constant (&ibitnum)
1191 1582327 : && is_a <scalar_int_mode> (fieldmode, &int_mode)
1192 743971 : && strict_volatile_bitfield_p (str_rtx, ibitsize, ibitnum, int_mode,
1193 : bitregion_start, bitregion_end))
1194 : {
1195 : /* Storing of a full word can be done with a simple store.
1196 : We know here that the field can be accessed with one single
1197 : instruction. For targets that support unaligned memory,
1198 : an unaligned access may be necessary. */
1199 8 : if (ibitsize == GET_MODE_BITSIZE (int_mode))
1200 : {
1201 0 : str_rtx = adjust_bitfield_address (str_rtx, int_mode,
1202 : ibitnum / BITS_PER_UNIT);
1203 0 : if (reverse)
1204 0 : value = flip_storage_order (int_mode, value);
1205 0 : gcc_assert (ibitnum % BITS_PER_UNIT == 0);
1206 0 : emit_move_insn (str_rtx, value);
1207 : }
1208 : else
1209 : {
1210 4 : rtx temp;
1211 :
1212 4 : str_rtx = narrow_bit_field_mem (str_rtx, int_mode, ibitsize,
1213 : ibitnum, &ibitnum);
1214 8 : gcc_assert (ibitnum + ibitsize <= GET_MODE_BITSIZE (int_mode));
1215 4 : temp = copy_to_reg (str_rtx);
1216 4 : if (!store_bit_field_1 (temp, ibitsize, ibitnum, 0, 0,
1217 : int_mode, value, reverse, true, undefined_p))
1218 0 : gcc_unreachable ();
1219 :
1220 4 : emit_move_insn (str_rtx, temp);
1221 : }
1222 :
1223 4 : return;
1224 : }
1225 :
1226 : /* Under the C++0x memory model, we must not touch bits outside the
1227 : bit region. Adjust the address to start at the beginning of the
1228 : bit region. */
1229 838356 : if (MEM_P (str_rtx) && maybe_ne (bitregion_start, 0U))
1230 : {
1231 51203 : scalar_int_mode best_mode;
1232 51203 : machine_mode addr_mode = VOIDmode;
1233 :
1234 51203 : poly_uint64 offset = exact_div (bitregion_start, BITS_PER_UNIT);
1235 51203 : bitnum -= bitregion_start;
1236 51203 : poly_int64 size = bits_to_bytes_round_up (bitnum + bitsize);
1237 51203 : bitregion_end -= bitregion_start;
1238 51203 : bitregion_start = 0;
1239 51203 : if (bitsize.is_constant (&ibitsize)
1240 51203 : && bitnum.is_constant (&ibitnum)
1241 51203 : && get_best_mode (ibitsize, ibitnum,
1242 : bitregion_start, bitregion_end,
1243 51203 : MEM_ALIGN (str_rtx), INT_MAX,
1244 51203 : MEM_VOLATILE_P (str_rtx), &best_mode))
1245 47789 : addr_mode = best_mode;
1246 51203 : str_rtx = adjust_bitfield_address_size (str_rtx, addr_mode,
1247 : offset, size);
1248 : }
1249 :
1250 838356 : if (!store_bit_field_1 (str_rtx, bitsize, bitnum,
1251 : bitregion_start, bitregion_end,
1252 : fieldmode, value, reverse, true, undefined_p))
1253 0 : gcc_unreachable ();
1254 : }
1255 :
1256 : /* Use shifts and boolean operations to store VALUE into a bit field of
1257 : width BITSIZE in OP0, starting at bit BITNUM. If OP0_MODE is defined,
1258 : it is the mode of OP0, otherwise OP0 is a BLKmode MEM. VALUE_MODE is
1259 : the mode of VALUE.
1260 :
1261 : If REVERSE is true, the store is to be done in reverse order. */
1262 :
1263 : static void
1264 129261 : store_fixed_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
1265 : unsigned HOST_WIDE_INT bitsize,
1266 : unsigned HOST_WIDE_INT bitnum,
1267 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
1268 : rtx value, scalar_int_mode value_mode, bool reverse)
1269 : {
1270 : /* There is a case not handled here:
1271 : a structure with a known alignment of just a halfword
1272 : and a field split across two aligned halfwords within the structure.
1273 : Or likewise a structure with a known alignment of just a byte
1274 : and a field split across two bytes.
1275 : Such cases are not supposed to be able to occur. */
1276 :
1277 129261 : scalar_int_mode best_mode;
1278 129261 : if (MEM_P (op0))
1279 : {
1280 84850 : unsigned int max_bitsize = BITS_PER_WORD;
1281 84850 : scalar_int_mode imode;
1282 144879 : if (op0_mode.exists (&imode) && GET_MODE_BITSIZE (imode) < max_bitsize)
1283 91170 : max_bitsize = GET_MODE_BITSIZE (imode);
1284 :
1285 84850 : if (!get_best_mode (bitsize, bitnum, bitregion_start, bitregion_end,
1286 84850 : MEM_ALIGN (op0), max_bitsize, MEM_VOLATILE_P (op0),
1287 : &best_mode))
1288 : {
1289 : /* The only way this should occur is if the field spans word
1290 : boundaries. */
1291 6567 : store_split_bit_field (op0, op0_mode, bitsize, bitnum,
1292 : bitregion_start, bitregion_end,
1293 : value, value_mode, reverse);
1294 6567 : return;
1295 : }
1296 :
1297 78283 : op0 = narrow_bit_field_mem (op0, best_mode, bitsize, bitnum, &bitnum);
1298 : }
1299 : else
1300 44411 : best_mode = op0_mode.require ();
1301 :
1302 122694 : store_fixed_bit_field_1 (op0, best_mode, bitsize, bitnum,
1303 : value, value_mode, reverse);
1304 : }
1305 :
1306 : /* Helper function for store_fixed_bit_field, stores
1307 : the bit field always using MODE, which is the mode of OP0. The other
1308 : arguments are as for store_fixed_bit_field. */
1309 :
1310 : static void
1311 122694 : store_fixed_bit_field_1 (rtx op0, scalar_int_mode mode,
1312 : unsigned HOST_WIDE_INT bitsize,
1313 : unsigned HOST_WIDE_INT bitnum,
1314 : rtx value, scalar_int_mode value_mode, bool reverse)
1315 : {
1316 122694 : rtx temp;
1317 122694 : int all_zero = 0;
1318 122694 : int all_one = 0;
1319 :
1320 : /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
1321 : for invalid input, such as f5 from gcc.dg/pr48335-2.c. */
1322 :
1323 122694 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
1324 : /* BITNUM is the distance between our msb
1325 : and that of the containing datum.
1326 : Convert it to the distance from the lsb. */
1327 1056 : bitnum = GET_MODE_BITSIZE (mode) - bitsize - bitnum;
1328 :
1329 : /* Now BITNUM is always the distance between our lsb
1330 : and that of OP0. */
1331 :
1332 : /* Shift VALUE left by BITNUM bits. If VALUE is not constant,
1333 : we must first convert its mode to MODE. */
1334 :
1335 122694 : if (CONST_INT_P (value))
1336 : {
1337 73338 : unsigned HOST_WIDE_INT v = UINTVAL (value);
1338 :
1339 73338 : if (bitsize < HOST_BITS_PER_WIDE_INT)
1340 73321 : v &= (HOST_WIDE_INT_1U << bitsize) - 1;
1341 :
1342 73338 : if (v == 0)
1343 : all_zero = 1;
1344 59655 : else if ((bitsize < HOST_BITS_PER_WIDE_INT
1345 59647 : && v == (HOST_WIDE_INT_1U << bitsize) - 1)
1346 50988 : || (bitsize == HOST_BITS_PER_WIDE_INT
1347 50988 : && v == HOST_WIDE_INT_M1U))
1348 8667 : all_one = 1;
1349 :
1350 73338 : value = lshift_value (mode, v, bitnum);
1351 : }
1352 : else
1353 : {
1354 49356 : int must_and = (GET_MODE_BITSIZE (value_mode) != bitsize
1355 76842 : && bitnum + bitsize != GET_MODE_BITSIZE (mode));
1356 :
1357 49356 : if (value_mode != mode)
1358 26783 : value = convert_to_mode (mode, value, 1);
1359 :
1360 49356 : if (must_and)
1361 20689 : value = expand_binop (mode, and_optab, value,
1362 : mask_rtx (mode, 0, bitsize, 0),
1363 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
1364 49356 : if (bitnum > 0)
1365 14259 : value = expand_shift (LSHIFT_EXPR, mode, value,
1366 14259 : bitnum, NULL_RTX, 1);
1367 : }
1368 :
1369 122694 : if (reverse)
1370 528 : value = flip_storage_order (mode, value);
1371 :
1372 : /* A field filling the whole of a MEM has no surrounding bits to preserve,
1373 : so store it directly: the read of a volatile OP0 cannot be removed later
1374 : and would be a spurious access with side effects (PR71048). */
1375 182318 : if (MEM_P (op0) && bitnum == 0 && bitsize == GET_MODE_BITSIZE (mode))
1376 : {
1377 13637 : op0 = copy_rtx (op0);
1378 13637 : emit_move_insn (op0, value);
1379 13637 : return;
1380 : }
1381 :
1382 : /* Now clear the chosen bits in OP0,
1383 : except that if VALUE is -1 we need not bother. */
1384 : /* We keep the intermediates in registers to allow CSE to combine
1385 : consecutive bitfield assignments. */
1386 :
1387 109057 : temp = force_reg (mode, op0);
1388 :
1389 109057 : if (! all_one)
1390 : {
1391 101081 : rtx mask = mask_rtx (mode, bitnum, bitsize, 1);
1392 101081 : if (reverse)
1393 516 : mask = flip_storage_order (mode, mask);
1394 101081 : temp = expand_binop (mode, and_optab, temp, mask,
1395 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
1396 101081 : temp = force_reg (mode, temp);
1397 : }
1398 :
1399 : /* Now logical-or VALUE into OP0, unless it is zero. */
1400 :
1401 109057 : if (! all_zero)
1402 : {
1403 : /* Use aop_optab if the field has been cleared. */
1404 186340 : temp = expand_binop (mode, all_one ? ior_optab : aop_optab,
1405 : temp, value, NULL_RTX, 1, OPTAB_LIB_WIDEN);
1406 97158 : temp = force_reg (mode, temp);
1407 : }
1408 :
1409 109057 : if (op0 != temp)
1410 : {
1411 109057 : op0 = copy_rtx (op0);
1412 109057 : emit_move_insn (op0, temp);
1413 : }
1414 : }
1415 :
1416 : /* Store a bit field that is split across multiple accessible memory objects.
1417 :
1418 : OP0 is the REG, SUBREG or MEM rtx for the first of the objects.
1419 : BITSIZE is the field width; BITPOS the position of its first bit
1420 : (within the word).
1421 : VALUE is the value to store, which has mode VALUE_MODE.
1422 : If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is
1423 : a BLKmode MEM.
1424 :
1425 : If REVERSE is true, the store is to be done in reverse order.
1426 :
1427 : This does not yet handle fields wider than BITS_PER_WORD. */
1428 :
1429 : static void
1430 6638 : store_split_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
1431 : unsigned HOST_WIDE_INT bitsize,
1432 : unsigned HOST_WIDE_INT bitpos,
1433 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
1434 : rtx value, scalar_int_mode value_mode, bool reverse)
1435 : {
1436 6638 : unsigned int unit, total_bits, bitsdone = 0;
1437 :
1438 : /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
1439 : much at a time. */
1440 6638 : if (REG_P (op0) || GET_CODE (op0) == SUBREG)
1441 71 : unit = BITS_PER_WORD;
1442 : else
1443 6567 : unit = MIN (MEM_ALIGN (op0), BITS_PER_WORD);
1444 :
1445 : /* If OP0 is a memory with a mode, then UNIT must not be larger than
1446 : OP0's mode as well. Otherwise, store_fixed_bit_field will call us
1447 : again, and we will mutually recurse forever. */
1448 6638 : if (MEM_P (op0) && op0_mode.exists ())
1449 5111 : unit = MIN (unit, GET_MODE_BITSIZE (op0_mode.require ()));
1450 :
1451 : /* If VALUE is a constant other than a CONST_INT, get it into a register in
1452 : WORD_MODE. If we can do this using gen_lowpart_common, do so. Note
1453 : that VALUE might be a floating-point constant. */
1454 6638 : if (CONSTANT_P (value) && !CONST_INT_P (value))
1455 : {
1456 0 : rtx word = gen_lowpart_common (word_mode, value);
1457 :
1458 0 : if (word && (value != word))
1459 : value = word;
1460 : else
1461 0 : value = gen_lowpart_common (word_mode, force_reg (value_mode, value));
1462 0 : value_mode = word_mode;
1463 : }
1464 :
1465 6638 : total_bits = GET_MODE_BITSIZE (value_mode);
1466 :
1467 30931 : while (bitsdone < bitsize)
1468 : {
1469 24293 : unsigned HOST_WIDE_INT thissize;
1470 24293 : unsigned HOST_WIDE_INT thispos;
1471 24293 : unsigned HOST_WIDE_INT offset;
1472 24293 : rtx part;
1473 :
1474 24293 : offset = (bitpos + bitsdone) / unit;
1475 24293 : thispos = (bitpos + bitsdone) % unit;
1476 :
1477 : /* When region of bytes we can touch is restricted, decrease
1478 : UNIT close to the end of the region as needed. If op0 is a REG
1479 : or SUBREG of REG, don't do this, as there can't be data races
1480 : on a register and we can expand shorter code in some cases. */
1481 31091 : if (maybe_ne (bitregion_end, 0U)
1482 24293 : && unit > BITS_PER_UNIT
1483 14211 : && maybe_gt (bitpos + bitsdone - thispos + unit, bitregion_end + 1)
1484 6862 : && !REG_P (op0)
1485 31091 : && (GET_CODE (op0) != SUBREG || !REG_P (SUBREG_REG (op0))))
1486 : {
1487 6798 : unit = unit / 2;
1488 6798 : continue;
1489 : }
1490 :
1491 : /* THISSIZE must not overrun a word boundary. Otherwise,
1492 : store_fixed_bit_field will call us again, and we will mutually
1493 : recurse forever. */
1494 17495 : thissize = MIN (bitsize - bitsdone, BITS_PER_WORD);
1495 17495 : thissize = MIN (thissize, unit - thispos);
1496 :
1497 17495 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
1498 : {
1499 : /* Fetch successively less significant portions. */
1500 214 : if (CONST_INT_P (value))
1501 108 : part = GEN_INT (((unsigned HOST_WIDE_INT) (INTVAL (value))
1502 : >> (bitsize - bitsdone - thissize))
1503 : & ((HOST_WIDE_INT_1 << thissize) - 1));
1504 : /* Likewise, but the source is little-endian. */
1505 106 : else if (reverse)
1506 106 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1507 : thissize,
1508 : bitsize - bitsdone - thissize,
1509 : NULL_RTX, 1, false);
1510 : else
1511 : /* The args are chosen so that the last part includes the
1512 : lsb. Give extract_bit_field the value it needs (with
1513 : endianness compensation) to fetch the piece we want. */
1514 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1515 : thissize,
1516 : total_bits - bitsize + bitsdone,
1517 : NULL_RTX, 1, false);
1518 : }
1519 : else
1520 : {
1521 : /* Fetch successively more significant portions. */
1522 17281 : if (CONST_INT_P (value))
1523 12867 : part = GEN_INT (((unsigned HOST_WIDE_INT) (INTVAL (value))
1524 : >> bitsdone)
1525 : & ((HOST_WIDE_INT_1 << thissize) - 1));
1526 : /* Likewise, but the source is big-endian. */
1527 4414 : else if (reverse)
1528 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1529 : thissize,
1530 : total_bits - bitsdone - thissize,
1531 : NULL_RTX, 1, false);
1532 : else
1533 4414 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1534 : thissize, bitsdone, NULL_RTX,
1535 : 1, false);
1536 : }
1537 :
1538 : /* If OP0 is a register, then handle OFFSET here. */
1539 17495 : rtx op0_piece = op0;
1540 17495 : opt_scalar_int_mode op0_piece_mode = op0_mode;
1541 17495 : if (SUBREG_P (op0) || REG_P (op0))
1542 : {
1543 142 : scalar_int_mode imode;
1544 142 : if (op0_mode.exists (&imode)
1545 142 : && GET_MODE_SIZE (imode) < UNITS_PER_WORD)
1546 : {
1547 0 : if (offset)
1548 0 : op0_piece = const0_rtx;
1549 : }
1550 : else
1551 : {
1552 142 : op0_piece = operand_subword_force (op0,
1553 142 : offset * unit / BITS_PER_WORD,
1554 142 : GET_MODE (op0));
1555 142 : op0_piece_mode = word_mode;
1556 : }
1557 142 : offset &= BITS_PER_WORD / unit - 1;
1558 : }
1559 :
1560 : /* OFFSET is in UNITs, and UNIT is in bits. If WORD is const0_rtx,
1561 : it is just an out-of-bounds access. Ignore it. */
1562 17495 : if (op0_piece != const0_rtx)
1563 17495 : store_fixed_bit_field (op0_piece, op0_piece_mode, thissize,
1564 17495 : offset * unit + thispos, bitregion_start,
1565 : bitregion_end, part, word_mode, reverse);
1566 17495 : bitsdone += thissize;
1567 : }
1568 6638 : }
1569 :
1570 : /* A subroutine of extract_bit_field_1 that converts return value X
1571 : to either MODE or TMODE. MODE, TMODE and UNSIGNEDP are arguments
1572 : to extract_bit_field. */
1573 :
1574 : static rtx
1575 908626 : convert_extracted_bit_field (rtx x, machine_mode mode,
1576 : machine_mode tmode, bool unsignedp)
1577 : {
1578 908626 : if (GET_MODE (x) == tmode || GET_MODE (x) == mode)
1579 : return x;
1580 :
1581 : /* If the x mode is not a scalar integral, first convert to the
1582 : integer mode of that size and then access it as a floating-point
1583 : value via a SUBREG. */
1584 21975 : if (!SCALAR_INT_MODE_P (tmode))
1585 : {
1586 11910 : scalar_int_mode int_mode = int_mode_for_mode (tmode).require ();
1587 11910 : x = convert_to_mode (int_mode, x, unsignedp);
1588 11910 : x = force_reg (int_mode, x);
1589 11910 : return gen_lowpart (tmode, x);
1590 : }
1591 :
1592 10065 : return convert_to_mode (tmode, x, unsignedp);
1593 : }
1594 :
1595 : /* Try to use an ext(z)v pattern to extract a field from OP0.
1596 : Return the extracted value on success, otherwise return null.
1597 : EXTV describes the extraction instruction to use. If OP0_MODE
1598 : is defined, it is the mode of OP0, otherwise OP0 is a BLKmode MEM.
1599 : The other arguments are as for extract_bit_field. */
1600 :
1601 : static rtx
1602 166924 : extract_bit_field_using_extv (const extraction_insn *extv, rtx op0,
1603 : opt_scalar_int_mode op0_mode,
1604 : unsigned HOST_WIDE_INT bitsize,
1605 : unsigned HOST_WIDE_INT bitnum,
1606 : int unsignedp, rtx target,
1607 : machine_mode mode, machine_mode tmode)
1608 : {
1609 166924 : class expand_operand ops[4];
1610 166924 : rtx spec_target = target;
1611 166924 : rtx spec_target_subreg = 0;
1612 166924 : scalar_int_mode ext_mode = extv->field_mode;
1613 166924 : unsigned unit = GET_MODE_BITSIZE (ext_mode);
1614 :
1615 166924 : if (bitsize == 0 || unit < bitsize)
1616 : return NULL_RTX;
1617 :
1618 166924 : if (MEM_P (op0))
1619 : /* Get a reference to the first byte of the field. */
1620 0 : op0 = narrow_bit_field_mem (op0, extv->struct_mode, bitsize, bitnum,
1621 : &bitnum);
1622 : else
1623 : {
1624 : /* Convert from counting within OP0 to counting in EXT_MODE. */
1625 166924 : if (BYTES_BIG_ENDIAN)
1626 : bitnum += unit - GET_MODE_BITSIZE (op0_mode.require ());
1627 :
1628 : /* If op0 is a register, we need it in EXT_MODE to make it
1629 : acceptable to the format of ext(z)v. */
1630 166924 : if (GET_CODE (op0) == SUBREG && op0_mode.require () != ext_mode)
1631 0 : return NULL_RTX;
1632 166924 : if (REG_P (op0) && op0_mode.require () != ext_mode)
1633 50565 : op0 = gen_lowpart_SUBREG (ext_mode, op0);
1634 : }
1635 :
1636 : /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
1637 : "backwards" from the size of the unit we are extracting from.
1638 : Otherwise, we count bits from the most significant on a
1639 : BYTES/BITS_BIG_ENDIAN machine. */
1640 :
1641 166924 : if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
1642 : bitnum = unit - bitsize - bitnum;
1643 :
1644 166924 : if (target == 0)
1645 46928 : target = spec_target = gen_reg_rtx (tmode);
1646 :
1647 166924 : if (GET_MODE (target) != ext_mode)
1648 : {
1649 99625 : rtx temp;
1650 : /* Don't use LHS paradoxical subreg if explicit truncation is needed
1651 : between the mode of the extraction (word_mode) and the target
1652 : mode. Instead, create a temporary and use convert_move to set
1653 : the target. */
1654 99625 : if (REG_P (target)
1655 98362 : && TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (target), ext_mode)
1656 197987 : && (temp = gen_lowpart_if_possible (ext_mode, target)))
1657 : {
1658 97850 : target = temp;
1659 97850 : if (partial_subreg_p (GET_MODE (spec_target), ext_mode))
1660 96993 : spec_target_subreg = target;
1661 : }
1662 : else
1663 1775 : target = gen_reg_rtx (ext_mode);
1664 : }
1665 :
1666 166924 : create_output_operand (&ops[0], target, ext_mode);
1667 166924 : create_fixed_operand (&ops[1], op0);
1668 166924 : create_integer_operand (&ops[2], bitsize);
1669 166924 : create_integer_operand (&ops[3], bitnum);
1670 166924 : if (maybe_expand_insn (extv->icode, 4, ops))
1671 : {
1672 1711 : target = ops[0].value;
1673 1711 : if (target == spec_target)
1674 : return target;
1675 1711 : if (target == spec_target_subreg)
1676 : return spec_target;
1677 66 : return convert_extracted_bit_field (target, mode, tmode, unsignedp);
1678 : }
1679 : return NULL_RTX;
1680 : }
1681 :
1682 : /* See whether it would be valid to extract the part of OP0 with
1683 : mode OP0_MODE described by BITNUM and BITSIZE into a value of
1684 : mode MODE using a subreg operation.
1685 : Return the subreg if so, otherwise return null. */
1686 :
1687 : static rtx
1688 862113 : extract_bit_field_as_subreg (machine_mode mode, rtx op0,
1689 : machine_mode op0_mode,
1690 : poly_uint64 bitsize, poly_uint64 bitnum)
1691 : {
1692 862113 : poly_uint64 bytenum;
1693 862113 : if (multiple_p (bitnum, BITS_PER_UNIT, &bytenum)
1694 823345 : && known_eq (bitsize, GET_MODE_BITSIZE (mode))
1695 862113 : && lowpart_bit_field_p (bitnum, bitsize, op0_mode)
1696 1685458 : && TRULY_NOOP_TRUNCATION_MODES_P (mode, op0_mode))
1697 686556 : return force_subreg (mode, op0, op0_mode, bytenum);
1698 : return NULL_RTX;
1699 : }
1700 :
1701 : /* A subroutine of extract_bit_field, with the same arguments.
1702 : If UNSIGNEDP is -1, the result need not be sign or zero extended.
1703 : If FALLBACK_P is true, fall back to extract_fixed_bit_field
1704 : if we can find no other means of implementing the operation.
1705 : if FALLBACK_P is false, return NULL instead. */
1706 :
1707 : static rtx
1708 1134584 : extract_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
1709 : int unsignedp, rtx target, machine_mode mode,
1710 : machine_mode tmode, bool reverse, bool fallback_p,
1711 : rtx *alt_rtl)
1712 : {
1713 1134584 : rtx op0 = str_rtx;
1714 1134584 : machine_mode mode1;
1715 :
1716 1134584 : if (tmode == VOIDmode)
1717 0 : tmode = mode;
1718 :
1719 1145328 : while (GET_CODE (op0) == SUBREG)
1720 : {
1721 10744 : bitnum += SUBREG_BYTE (op0) * BITS_PER_UNIT;
1722 10744 : op0 = SUBREG_REG (op0);
1723 : }
1724 :
1725 : /* If we have an out-of-bounds access to a register, just return an
1726 : uninitialized register of the required mode. This can occur if the
1727 : source code contains an out-of-bounds access to a small array. */
1728 2083787 : if (REG_P (op0) && known_ge (bitnum, GET_MODE_BITSIZE (GET_MODE (op0))))
1729 0 : return gen_reg_rtx (tmode);
1730 :
1731 1134584 : if (REG_P (op0)
1732 949203 : && mode == GET_MODE (op0)
1733 151415 : && known_eq (bitnum, 0U)
1734 1375150 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0))))
1735 : {
1736 19816 : if (reverse)
1737 0 : op0 = flip_storage_order (mode, op0);
1738 : /* We're trying to extract a full register from itself. */
1739 : return op0;
1740 : }
1741 :
1742 : /* First try to check for vector from vector extractions. */
1743 1065730 : if (VECTOR_MODE_P (GET_MODE (op0))
1744 70602 : && !MEM_P (op0)
1745 69201 : && VECTOR_MODE_P (tmode)
1746 12821 : && known_eq (bitsize, GET_MODE_PRECISION (tmode))
1747 2255178 : && maybe_gt (GET_MODE_SIZE (GET_MODE (op0)), GET_MODE_SIZE (tmode)))
1748 : {
1749 12821 : machine_mode new_mode = GET_MODE (op0);
1750 38463 : if (GET_MODE_INNER (new_mode) != GET_MODE_INNER (tmode))
1751 : {
1752 214 : scalar_mode inner_mode = GET_MODE_INNER (tmode);
1753 214 : poly_uint64 nunits;
1754 428 : if (!multiple_p (GET_MODE_BITSIZE (GET_MODE (op0)),
1755 214 : GET_MODE_UNIT_BITSIZE (tmode), &nunits)
1756 428 : || !related_vector_mode (tmode, inner_mode,
1757 214 : nunits).exists (&new_mode)
1758 412 : || maybe_ne (GET_MODE_SIZE (new_mode),
1759 610 : GET_MODE_SIZE (GET_MODE (op0))))
1760 16 : new_mode = VOIDmode;
1761 : }
1762 12821 : poly_uint64 pos;
1763 12821 : if (new_mode != VOIDmode
1764 12805 : && (convert_optab_handler (vec_extract_optab, new_mode, tmode)
1765 : != CODE_FOR_nothing)
1766 25626 : && multiple_p (bitnum, GET_MODE_BITSIZE (tmode), &pos))
1767 : {
1768 8999 : class expand_operand ops[3];
1769 8999 : machine_mode outermode = new_mode;
1770 8999 : machine_mode innermode = tmode;
1771 8999 : enum insn_code icode
1772 8999 : = convert_optab_handler (vec_extract_optab, outermode, innermode);
1773 :
1774 8999 : if (new_mode != GET_MODE (op0))
1775 33 : op0 = gen_lowpart (new_mode, op0);
1776 8999 : create_output_operand (&ops[0], target, innermode);
1777 8999 : ops[0].target = 1;
1778 8999 : create_input_operand (&ops[1], op0, outermode);
1779 8999 : create_integer_operand (&ops[2], pos);
1780 8999 : if (maybe_expand_insn (icode, 3, ops))
1781 : {
1782 8999 : if (alt_rtl && ops[0].target)
1783 207 : *alt_rtl = target;
1784 8999 : target = ops[0].value;
1785 8999 : if (GET_MODE (target) != mode)
1786 8999 : return gen_lowpart (tmode, target);
1787 : return target;
1788 : }
1789 : }
1790 : }
1791 :
1792 : /* See if we can get a better vector mode before extracting. */
1793 1058331 : if (VECTOR_MODE_P (GET_MODE (op0))
1794 61603 : && !MEM_P (op0)
1795 1226173 : && GET_MODE_INNER (GET_MODE (op0)) != tmode)
1796 : {
1797 9503 : machine_mode new_mode;
1798 :
1799 9503 : if (GET_MODE_CLASS (tmode) == MODE_FLOAT)
1800 557 : new_mode = MIN_MODE_VECTOR_FLOAT;
1801 : else if (GET_MODE_CLASS (tmode) == MODE_FRACT)
1802 0 : new_mode = MIN_MODE_VECTOR_FRACT;
1803 : else if (GET_MODE_CLASS (tmode) == MODE_UFRACT)
1804 0 : new_mode = MIN_MODE_VECTOR_UFRACT;
1805 : else if (GET_MODE_CLASS (tmode) == MODE_ACCUM)
1806 0 : new_mode = MIN_MODE_VECTOR_ACCUM;
1807 : else if (GET_MODE_CLASS (tmode) == MODE_UACCUM)
1808 0 : new_mode = MIN_MODE_VECTOR_UACCUM;
1809 : else
1810 8946 : new_mode = MIN_MODE_VECTOR_INT;
1811 :
1812 150486 : FOR_EACH_MODE_FROM (new_mode, new_mode)
1813 300272 : if (known_eq (GET_MODE_SIZE (new_mode), GET_MODE_SIZE (GET_MODE (op0)))
1814 62562 : && known_eq (GET_MODE_UNIT_SIZE (new_mode), GET_MODE_SIZE (tmode))
1815 169142 : && known_eq (bitsize, GET_MODE_UNIT_PRECISION (new_mode))
1816 19006 : && multiple_p (bitnum, GET_MODE_UNIT_PRECISION (new_mode))
1817 9474 : && targetm.vector_mode_supported_p (new_mode)
1818 159592 : && targetm.modes_tieable_p (GET_MODE (op0), new_mode))
1819 : break;
1820 9503 : if (new_mode != VOIDmode)
1821 9153 : op0 = gen_lowpart (new_mode, op0);
1822 : }
1823 :
1824 : /* Use vec_extract patterns for extracting parts of vectors whenever
1825 : available. If that fails, see whether the current modes and bitregion
1826 : give a natural subreg. */
1827 1105769 : machine_mode outermode = GET_MODE (op0);
1828 1105769 : if (VECTOR_MODE_P (outermode) && !MEM_P (op0))
1829 : {
1830 60202 : scalar_mode innermode = GET_MODE_INNER (outermode);
1831 :
1832 60202 : enum insn_code icode
1833 60202 : = convert_optab_handler (vec_extract_optab, outermode, innermode);
1834 :
1835 60202 : poly_uint64 pos;
1836 60202 : if (icode != CODE_FOR_nothing
1837 60205 : && known_eq (bitsize, GET_MODE_PRECISION (innermode))
1838 118463 : && multiple_p (bitnum, GET_MODE_PRECISION (innermode), &pos))
1839 : {
1840 58258 : class expand_operand ops[3];
1841 :
1842 58258 : create_output_operand (&ops[0], target,
1843 58258 : insn_data[icode].operand[0].mode);
1844 58258 : ops[0].target = 1;
1845 58258 : create_input_operand (&ops[1], op0, outermode);
1846 58258 : create_integer_operand (&ops[2], pos);
1847 58258 : if (maybe_expand_insn (icode, 3, ops))
1848 : {
1849 58258 : if (alt_rtl && ops[0].target)
1850 14537 : *alt_rtl = target;
1851 58258 : target = ops[0].value;
1852 58258 : if (GET_MODE (target) != mode)
1853 58258 : return gen_lowpart (tmode, target);
1854 : return target;
1855 : }
1856 : }
1857 : /* Using subregs is useful if we're extracting one register vector
1858 : from a multi-register vector. extract_bit_field_as_subreg checks
1859 : for valid bitsize and bitnum, so we don't need to do that here. */
1860 1944 : if (VECTOR_MODE_P (mode))
1861 : {
1862 45 : rtx sub = extract_bit_field_as_subreg (mode, op0, outermode,
1863 : bitsize, bitnum);
1864 45 : if (sub)
1865 : return sub;
1866 : }
1867 : }
1868 :
1869 : /* Make sure we are playing with integral modes. Pun with subregs
1870 : if we aren't. */
1871 1047494 : opt_scalar_int_mode op0_mode = int_mode_for_mode (GET_MODE (op0));
1872 1047494 : scalar_int_mode imode;
1873 1047494 : bool need_stack_p = false;
1874 1047494 : if (!op0_mode.exists (&imode) || imode != GET_MODE (op0))
1875 : {
1876 198634 : if (MEM_P (op0))
1877 159883 : op0 = adjust_bitfield_address_size (op0, op0_mode.else_blk (),
1878 : 0, MEM_SIZE (op0));
1879 38751 : else if (op0_mode.exists (&imode)
1880 39718 : && known_ge
1881 : ((unsigned) UNITS_PER_WORD,
1882 : (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
1883 : {
1884 38720 : op0 = gen_lowpart (imode, op0);
1885 :
1886 : /* If we got a SUBREG, force it into a register since we
1887 : aren't going to be able to do another SUBREG on it. */
1888 38720 : if (GET_CODE (op0) == SUBREG)
1889 38441 : op0 = force_reg (imode, op0);
1890 : }
1891 : else
1892 : need_stack_p = true;
1893 : }
1894 848860 : else if (!MEM_P (op0)
1895 1752067 : && maybe_lt ((unsigned) UNITS_PER_WORD,
1896 823362 : (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
1897 0 : need_stack_p = true;
1898 :
1899 1047215 : if (need_stack_p)
1900 : {
1901 62 : poly_int64 size = GET_MODE_SIZE (GET_MODE (op0));
1902 31 : rtx mem = assign_stack_temp (GET_MODE (op0), size);
1903 31 : emit_move_insn (mem, op0);
1904 31 : op0 = adjust_bitfield_address_size (mem, BLKmode, 0, size);
1905 : }
1906 :
1907 : /* ??? We currently assume TARGET is at least as big as BITSIZE.
1908 : If that's wrong, the solution is to test for it and set TARGET to 0
1909 : if needed. */
1910 :
1911 : /* Get the mode of the field to use for atomic access or subreg
1912 : conversion. */
1913 1047494 : if (!SCALAR_INT_MODE_P (tmode)
1914 1047494 : || !mode_for_size (bitsize, GET_MODE_CLASS (tmode), 0).exists (&mode1))
1915 312442 : mode1 = mode;
1916 1047494 : gcc_assert (mode1 != BLKmode);
1917 :
1918 : /* Extraction of a full MODE1 value can be done with a subreg as long
1919 : as the least significant bit of the value is the least significant
1920 : bit of either OP0 or a word of OP0. */
1921 1047494 : if (!MEM_P (op0) && !reverse && op0_mode.exists (&imode))
1922 : {
1923 862068 : rtx sub = extract_bit_field_as_subreg (mode1, op0, imode,
1924 : bitsize, bitnum);
1925 862068 : if (sub)
1926 680184 : return convert_extracted_bit_field (sub, mode, tmode, unsignedp);
1927 : }
1928 :
1929 : /* Extraction of a full MODE1 value can be done with a load as long as
1930 : the field is on a byte boundary and is sufficiently aligned. */
1931 367310 : poly_uint64 bytenum;
1932 367310 : if (simple_mem_bitfield_p (op0, bitsize, bitnum, mode1, &bytenum))
1933 : {
1934 45310 : op0 = adjust_bitfield_address (op0, mode1, bytenum);
1935 45310 : if (reverse)
1936 52 : op0 = flip_storage_order (mode1, op0);
1937 45310 : return convert_extracted_bit_field (op0, mode, tmode, unsignedp);
1938 : }
1939 :
1940 : /* If we have a memory source and a non-constant bit offset, restrict
1941 : the memory to the referenced bytes. This is a worst-case fallback
1942 : but is useful for things like vector booleans. */
1943 322000 : if (MEM_P (op0) && !bitnum.is_constant ())
1944 : {
1945 : bytenum = bits_to_bytes_round_down (bitnum);
1946 : bitnum = num_trailing_bits (bitnum);
1947 : poly_uint64 bytesize = bits_to_bytes_round_up (bitnum + bitsize);
1948 : op0 = adjust_bitfield_address_size (op0, BLKmode, bytenum, bytesize);
1949 : op0_mode = opt_scalar_int_mode ();
1950 : }
1951 :
1952 : /* It's possible we'll need to handle other cases here for
1953 : polynomial bitnum and bitsize. */
1954 :
1955 : /* From here on we need to be looking at a fixed-size insertion. */
1956 322000 : return extract_integral_bit_field (op0, op0_mode, bitsize.to_constant (),
1957 : bitnum.to_constant (), unsignedp,
1958 322000 : target, mode, tmode, reverse, fallback_p);
1959 : }
1960 :
1961 : /* Subroutine of extract_bit_field_1, with the same arguments, except
1962 : that BITSIZE and BITNUM are constant. Handle cases specific to
1963 : integral modes. If OP0_MODE is defined, it is the mode of OP0,
1964 : otherwise OP0 is a BLKmode MEM. */
1965 :
1966 : static rtx
1967 322000 : extract_integral_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
1968 : unsigned HOST_WIDE_INT bitsize,
1969 : unsigned HOST_WIDE_INT bitnum, int unsignedp,
1970 : rtx target, machine_mode mode, machine_mode tmode,
1971 : bool reverse, bool fallback_p)
1972 : {
1973 : /* Handle fields bigger than a word. */
1974 :
1975 325273 : if (bitsize > BITS_PER_WORD)
1976 : {
1977 : /* Here we transfer the words of the field
1978 : in the order least significant first.
1979 : This is because the most significant word is the one which may
1980 : be less than full. */
1981 :
1982 1515 : const bool backwards = WORDS_BIG_ENDIAN;
1983 1515 : unsigned int nwords = (bitsize + (BITS_PER_WORD - 1)) / BITS_PER_WORD;
1984 1515 : unsigned int i;
1985 1515 : rtx_insn *last;
1986 :
1987 1515 : if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1988 1491 : target = gen_reg_rtx (mode);
1989 :
1990 : /* In case we're about to clobber a base register or something
1991 : (see gcc.c-torture/execute/20040625-1.c). */
1992 1515 : if (reg_mentioned_p (target, op0))
1993 0 : target = gen_reg_rtx (mode);
1994 :
1995 : /* Indicate for flow that the entire target reg is being set. */
1996 1515 : emit_clobber (target);
1997 :
1998 : /* The mode must be fixed-size, since extract_bit_field_1 handles
1999 : extractions from variable-sized objects before calling this
2000 : function. */
2001 1515 : unsigned int target_size
2002 1515 : = GET_MODE_SIZE (GET_MODE (target)).to_constant ();
2003 1515 : last = get_last_insn ();
2004 4545 : for (i = 0; i < nwords; i++)
2005 : {
2006 : /* If I is 0, use the low-order word in both field and target;
2007 : if I is 1, use the next to lowest word; and so on. */
2008 : /* Word number in TARGET to use. */
2009 3030 : unsigned int wordnum
2010 : = (backwards ? target_size / UNITS_PER_WORD - i - 1 : i);
2011 : /* Offset from start of field in OP0. */
2012 6060 : unsigned int bit_offset = (backwards ^ reverse
2013 3030 : ? MAX ((int) bitsize - ((int) i + 1)
2014 : * BITS_PER_WORD,
2015 : 0)
2016 3098 : : (int) i * BITS_PER_WORD);
2017 3030 : rtx target_part = operand_subword (target, wordnum, 1, VOIDmode);
2018 3030 : rtx result_part
2019 3206 : = extract_bit_field_1 (op0, MIN (BITS_PER_WORD,
2020 : bitsize - i * BITS_PER_WORD),
2021 3030 : bitnum + bit_offset,
2022 : (unsignedp ? 1 : -1), target_part,
2023 : mode, word_mode, reverse, fallback_p, NULL);
2024 :
2025 3030 : if (!result_part)
2026 : {
2027 0 : delete_insns_since (last);
2028 0 : return NULL;
2029 : }
2030 :
2031 3030 : if (!target_part)
2032 0 : store_bit_field (target, BITS_PER_WORD, i * BITS_PER_WORD,
2033 : 0, 0, word_mode, result_part, false, i == 0);
2034 3030 : else if (result_part != target_part)
2035 2914 : emit_move_insn (target_part, result_part);
2036 : }
2037 :
2038 1515 : if (unsignedp)
2039 : {
2040 : /* Unless we've filled TARGET, the upper regs in a multi-reg value
2041 : need to be zero'd out. */
2042 1529 : if (target_size > nwords * UNITS_PER_WORD)
2043 : {
2044 0 : unsigned int i, total_words;
2045 :
2046 0 : total_words = target_size / UNITS_PER_WORD;
2047 0 : for (i = nwords; i < total_words; i++)
2048 0 : emit_move_insn
2049 0 : (operand_subword (target,
2050 0 : backwards ? total_words - i - 1 : i,
2051 : 1, VOIDmode),
2052 : const0_rtx);
2053 : }
2054 : return target;
2055 : }
2056 :
2057 : /* Signed bit field: sign-extend with two arithmetic shifts. */
2058 40 : target = expand_shift (LSHIFT_EXPR, mode, target,
2059 20 : GET_MODE_BITSIZE (mode) - bitsize, NULL_RTX, 0);
2060 40 : return expand_shift (RSHIFT_EXPR, mode, target,
2061 20 : GET_MODE_BITSIZE (mode) - bitsize, NULL_RTX, 0);
2062 : }
2063 :
2064 : /* If OP0 is a multi-word register, narrow it to the affected word.
2065 : If the region spans two words, defer to extract_split_bit_field. */
2066 504913 : if (!MEM_P (op0) && GET_MODE_SIZE (op0_mode.require ()) > UNITS_PER_WORD)
2067 : {
2068 3913 : if (bitnum % BITS_PER_WORD + bitsize > BITS_PER_WORD)
2069 : {
2070 1142 : if (!fallback_p)
2071 : return NULL_RTX;
2072 60 : target = extract_split_bit_field (op0, op0_mode, bitsize, bitnum,
2073 : unsignedp, reverse);
2074 60 : return convert_extracted_bit_field (target, mode, tmode, unsignedp);
2075 : }
2076 : /* If OP0 is a hard register, copy it to a pseudo before calling
2077 : force_subreg. */
2078 2771 : if (REG_P (op0) && HARD_REGISTER_P (op0))
2079 1 : op0 = copy_to_reg (op0);
2080 2771 : op0 = force_subreg (word_mode, op0, op0_mode.require (),
2081 3235 : bitnum / BITS_PER_WORD * UNITS_PER_WORD);
2082 2771 : op0_mode = word_mode;
2083 3003 : bitnum %= BITS_PER_WORD;
2084 : }
2085 :
2086 : /* From here on we know the desired field is smaller than a word.
2087 : If OP0 is a register, it too fits within a word. */
2088 319343 : enum extraction_pattern pattern = unsignedp ? EP_extzv : EP_extv;
2089 319343 : extraction_insn extv;
2090 319343 : if (!MEM_P (op0)
2091 180747 : && !reverse
2092 : /* ??? We could limit the structure size to the part of OP0 that
2093 : contains the field, with appropriate checks for endianness
2094 : and TARGET_TRULY_NOOP_TRUNCATION. */
2095 500076 : && get_best_reg_extraction_insn (&extv, pattern,
2096 513885 : GET_MODE_BITSIZE (op0_mode.require ()),
2097 : tmode))
2098 : {
2099 166924 : rtx result = extract_bit_field_using_extv (&extv, op0, op0_mode,
2100 : bitsize, bitnum,
2101 : unsignedp, target, mode,
2102 : tmode);
2103 166924 : if (result)
2104 : return result;
2105 : }
2106 :
2107 : /* If OP0 is a memory, try copying it to a register and seeing if a
2108 : cheap register alternative is available. */
2109 317632 : if (MEM_P (op0) & !reverse)
2110 : {
2111 138373 : if (get_best_mem_extraction_insn (&extv, pattern, bitsize, bitnum,
2112 : tmode))
2113 : {
2114 0 : rtx result = extract_bit_field_using_extv (&extv, op0, op0_mode,
2115 : bitsize, bitnum,
2116 : unsignedp, target, mode,
2117 : tmode);
2118 0 : if (result)
2119 0 : return result;
2120 : }
2121 :
2122 138373 : rtx_insn *last = get_last_insn ();
2123 :
2124 : /* Try loading part of OP0 into a register and extracting the
2125 : bitfield from that. */
2126 138373 : unsigned HOST_WIDE_INT bitpos;
2127 138373 : rtx xop0 = adjust_bit_field_mem_for_reg (pattern, op0, bitsize, bitnum,
2128 : 0, 0, tmode, &bitpos);
2129 138373 : if (xop0)
2130 : {
2131 135708 : xop0 = copy_to_reg (xop0);
2132 135708 : rtx result = extract_bit_field_1 (xop0, bitsize, bitpos,
2133 : unsignedp, target,
2134 : mode, tmode, reverse, false, NULL);
2135 135708 : if (result)
2136 : return result;
2137 135708 : delete_insns_since (last);
2138 : }
2139 : }
2140 :
2141 317632 : if (!fallback_p)
2142 : return NULL;
2143 :
2144 : /* Find a correspondingly-sized integer field, so we can apply
2145 : shifts and masks to it. */
2146 183006 : scalar_int_mode int_mode;
2147 183006 : if (!int_mode_for_mode (tmode).exists (&int_mode))
2148 : /* If this fails, we should probably push op0 out to memory and then
2149 : do a load. */
2150 0 : int_mode = int_mode_for_mode (mode).require ();
2151 :
2152 183006 : target = extract_fixed_bit_field (int_mode, op0, op0_mode, bitsize,
2153 : bitnum, target, unsignedp, reverse);
2154 :
2155 : /* Complex values must be reversed piecewise, so we need to undo the global
2156 : reversal, convert to the complex mode and reverse again. */
2157 183006 : if (reverse && COMPLEX_MODE_P (tmode))
2158 : {
2159 0 : target = flip_storage_order (int_mode, target);
2160 0 : target = convert_extracted_bit_field (target, mode, tmode, unsignedp);
2161 0 : target = flip_storage_order (tmode, target);
2162 : }
2163 : else
2164 183006 : target = convert_extracted_bit_field (target, mode, tmode, unsignedp);
2165 :
2166 : return target;
2167 : }
2168 :
2169 : /* Generate code to extract a byte-field from STR_RTX
2170 : containing BITSIZE bits, starting at BITNUM,
2171 : and put it in TARGET if possible (if TARGET is nonzero).
2172 : Regardless of TARGET, we return the rtx for where the value is placed.
2173 :
2174 : STR_RTX is the structure containing the byte (a REG or MEM).
2175 : UNSIGNEDP is nonzero if this is an unsigned bit field.
2176 : MODE is the natural mode of the field value once extracted.
2177 : TMODE is the mode the caller would like the value to have;
2178 : but the value may be returned with type MODE instead.
2179 :
2180 : If REVERSE is true, the extraction is to be done in reverse order.
2181 :
2182 : If a TARGET is specified and we can store in it at no extra cost,
2183 : we do so, and return TARGET.
2184 : Otherwise, we return a REG of mode TMODE or MODE, with TMODE preferred
2185 : if they are equally easy.
2186 :
2187 : If the result can be stored at TARGET, and ALT_RTL is non-NULL,
2188 : then *ALT_RTL is set to TARGET (before legitimziation). */
2189 :
2190 : rtx
2191 995846 : extract_bit_field (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
2192 : int unsignedp, rtx target, machine_mode mode,
2193 : machine_mode tmode, bool reverse, rtx *alt_rtl)
2194 : {
2195 995846 : machine_mode mode1;
2196 :
2197 : /* Handle -fstrict-volatile-bitfields in the cases where it applies. */
2198 1991692 : if (maybe_ne (GET_MODE_BITSIZE (GET_MODE (str_rtx)), 0))
2199 : mode1 = GET_MODE (str_rtx);
2200 260966 : else if (target && maybe_ne (GET_MODE_BITSIZE (GET_MODE (target)), 0))
2201 : mode1 = GET_MODE (target);
2202 : else
2203 : mode1 = tmode;
2204 :
2205 995846 : unsigned HOST_WIDE_INT ibitsize, ibitnum;
2206 995846 : scalar_int_mode int_mode;
2207 995846 : if (bitsize.is_constant (&ibitsize)
2208 995846 : && bitnum.is_constant (&ibitnum)
2209 1856729 : && is_a <scalar_int_mode> (mode1, &int_mode)
2210 860890 : && strict_volatile_bitfield_p (str_rtx, ibitsize, ibitnum,
2211 : int_mode, 0, 0))
2212 : {
2213 : /* Extraction of a full INT_MODE value can be done with a simple load.
2214 : We know here that the field can be accessed with one single
2215 : instruction. For targets that support unaligned memory,
2216 : an unaligned access may be necessary. */
2217 14 : if (ibitsize == GET_MODE_BITSIZE (int_mode))
2218 : {
2219 0 : rtx result = adjust_bitfield_address (str_rtx, int_mode,
2220 : ibitnum / BITS_PER_UNIT);
2221 0 : if (reverse)
2222 0 : result = flip_storage_order (int_mode, result);
2223 0 : gcc_assert (ibitnum % BITS_PER_UNIT == 0);
2224 0 : return convert_extracted_bit_field (result, mode, tmode, unsignedp);
2225 : }
2226 :
2227 7 : str_rtx = narrow_bit_field_mem (str_rtx, int_mode, ibitsize, ibitnum,
2228 : &ibitnum);
2229 14 : gcc_assert (ibitnum + ibitsize <= GET_MODE_BITSIZE (int_mode));
2230 7 : str_rtx = copy_to_reg (str_rtx);
2231 7 : return extract_bit_field_1 (str_rtx, ibitsize, ibitnum, unsignedp,
2232 : target, mode, tmode, reverse, true, alt_rtl);
2233 : }
2234 :
2235 995839 : return extract_bit_field_1 (str_rtx, bitsize, bitnum, unsignedp,
2236 995839 : target, mode, tmode, reverse, true, alt_rtl);
2237 : }
2238 :
2239 : /* Use shifts and boolean operations to extract a field of BITSIZE bits
2240 : from bit BITNUM of OP0. If OP0_MODE is defined, it is the mode of OP0,
2241 : otherwise OP0 is a BLKmode MEM.
2242 :
2243 : UNSIGNEDP is nonzero for an unsigned bit field (don't sign-extend value).
2244 : If REVERSE is true, the extraction is to be done in reverse order.
2245 :
2246 : If TARGET is nonzero, attempts to store the value there
2247 : and return TARGET, but this is not guaranteed.
2248 : If TARGET is not used, create a pseudo-reg of mode TMODE for the value. */
2249 :
2250 : static rtx
2251 201346 : extract_fixed_bit_field (machine_mode tmode, rtx op0,
2252 : opt_scalar_int_mode op0_mode,
2253 : unsigned HOST_WIDE_INT bitsize,
2254 : unsigned HOST_WIDE_INT bitnum, rtx target,
2255 : int unsignedp, bool reverse)
2256 : {
2257 201346 : scalar_int_mode mode;
2258 201346 : if (MEM_P (op0))
2259 : {
2260 152320 : if (!get_best_mode (bitsize, bitnum, 0, 0, MEM_ALIGN (op0),
2261 152320 : BITS_PER_WORD, MEM_VOLATILE_P (op0), &mode))
2262 : /* The only way this should occur is if the field spans word
2263 : boundaries. */
2264 4301 : return extract_split_bit_field (op0, op0_mode, bitsize, bitnum,
2265 4301 : unsignedp, reverse);
2266 :
2267 148019 : op0 = narrow_bit_field_mem (op0, mode, bitsize, bitnum, &bitnum);
2268 : }
2269 : else
2270 49026 : mode = op0_mode.require ();
2271 :
2272 197045 : return extract_fixed_bit_field_1 (tmode, op0, mode, bitsize, bitnum,
2273 197045 : target, unsignedp, reverse);
2274 : }
2275 :
2276 : /* Helper function for extract_fixed_bit_field, extracts
2277 : the bit field always using MODE, which is the mode of OP0.
2278 : If UNSIGNEDP is -1, the result need not be sign or zero extended.
2279 : The other arguments are as for extract_fixed_bit_field. */
2280 :
2281 : static rtx
2282 197045 : extract_fixed_bit_field_1 (machine_mode tmode, rtx op0, scalar_int_mode mode,
2283 : unsigned HOST_WIDE_INT bitsize,
2284 : unsigned HOST_WIDE_INT bitnum, rtx target,
2285 : int unsignedp, bool reverse)
2286 : {
2287 : /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
2288 : for invalid input, such as extract equivalent of f5 from
2289 : gcc.dg/pr48335-2.c. */
2290 :
2291 197045 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
2292 : /* BITNUM is the distance between our msb and that of OP0.
2293 : Convert it to the distance from the lsb. */
2294 478 : bitnum = GET_MODE_BITSIZE (mode) - bitsize - bitnum;
2295 :
2296 : /* Now BITNUM is always the distance between the field's lsb and that of OP0.
2297 : We have reduced the big-endian case to the little-endian case. */
2298 197045 : if (reverse)
2299 239 : op0 = flip_storage_order (mode, op0);
2300 :
2301 197045 : if (unsignedp)
2302 : {
2303 137608 : if (bitnum)
2304 : {
2305 : /* If the field does not already start at the lsb,
2306 : shift it so it does. */
2307 : /* Maybe propagate the target for the shift. */
2308 61795 : rtx subtarget = (target != 0 && REG_P (target) ? target : 0);
2309 61795 : if (tmode != mode)
2310 41162 : subtarget = 0;
2311 61795 : op0 = expand_shift (RSHIFT_EXPR, mode, op0, bitnum, subtarget, 1);
2312 : }
2313 : /* Convert the value to the desired mode. TMODE must also be a
2314 : scalar integer for this conversion to make sense, since we
2315 : shouldn't reinterpret the bits. */
2316 137608 : scalar_int_mode new_mode = as_a <scalar_int_mode> (tmode);
2317 137608 : if (mode != new_mode)
2318 60611 : op0 = convert_to_mode (new_mode, op0, 1);
2319 :
2320 : /* Unless the msb of the field used to be the msb when we shifted,
2321 : mask out the upper bits. */
2322 :
2323 137608 : if (GET_MODE_BITSIZE (mode) != bitnum + bitsize
2324 137608 : && unsignedp != -1)
2325 93880 : return expand_binop (new_mode, and_optab, op0,
2326 : mask_rtx (new_mode, 0, bitsize, 0),
2327 93880 : target, 1, OPTAB_LIB_WIDEN);
2328 : return op0;
2329 : }
2330 :
2331 : /* To extract a signed bit-field, first shift its msb to the msb of the word,
2332 : then arithmetic-shift its lsb to the lsb of the word. */
2333 59437 : op0 = force_reg (mode, op0);
2334 :
2335 : /* Find the narrowest integer mode that contains the field. */
2336 :
2337 59437 : opt_scalar_int_mode mode_iter;
2338 147468 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
2339 294936 : if (GET_MODE_BITSIZE (mode_iter.require ()) >= bitsize + bitnum)
2340 : break;
2341 :
2342 59437 : mode = mode_iter.require ();
2343 59437 : op0 = convert_to_mode (mode, op0, 0);
2344 :
2345 59437 : if (mode != tmode)
2346 4855 : target = 0;
2347 :
2348 118874 : if (GET_MODE_BITSIZE (mode) != (bitsize + bitnum))
2349 : {
2350 54431 : int amount = GET_MODE_BITSIZE (mode) - (bitsize + bitnum);
2351 : /* Maybe propagate the target for the shift. */
2352 54431 : rtx subtarget = (target != 0 && REG_P (target) ? target : 0);
2353 54431 : op0 = expand_shift (LSHIFT_EXPR, mode, op0, amount, subtarget, 1);
2354 : }
2355 :
2356 118874 : return expand_shift (RSHIFT_EXPR, mode, op0,
2357 59437 : GET_MODE_BITSIZE (mode) - bitsize, target, 0);
2358 : }
2359 :
2360 : /* Return a constant integer (CONST_INT or CONST_DOUBLE) rtx with the value
2361 : VALUE << BITPOS. */
2362 :
2363 : static rtx
2364 73338 : lshift_value (machine_mode mode, unsigned HOST_WIDE_INT value,
2365 : int bitpos)
2366 : {
2367 73338 : return immed_wide_int_const (wi::lshift (value, bitpos), mode);
2368 : }
2369 :
2370 : /* Extract a bit field that is split across two words
2371 : and return an RTX for the result.
2372 :
2373 : OP0 is the REG, SUBREG or MEM rtx for the first of the two words.
2374 : BITSIZE is the field width; BITPOS, position of its first bit, in the word.
2375 : UNSIGNEDP is 1 if should zero-extend the contents; else sign-extend.
2376 : If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is
2377 : a BLKmode MEM.
2378 :
2379 : If REVERSE is true, the extraction is to be done in reverse order. */
2380 :
2381 : static rtx
2382 4361 : extract_split_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
2383 : unsigned HOST_WIDE_INT bitsize,
2384 : unsigned HOST_WIDE_INT bitpos, int unsignedp,
2385 : bool reverse)
2386 : {
2387 4361 : unsigned int unit;
2388 4361 : unsigned int bitsdone = 0;
2389 4361 : rtx result = NULL_RTX;
2390 4361 : int first = 1;
2391 :
2392 : /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
2393 : much at a time. */
2394 4361 : if (REG_P (op0) || GET_CODE (op0) == SUBREG)
2395 60 : unit = BITS_PER_WORD;
2396 : else
2397 6380 : unit = MIN (MEM_ALIGN (op0), BITS_PER_WORD);
2398 :
2399 18181 : while (bitsdone < bitsize)
2400 : {
2401 13820 : unsigned HOST_WIDE_INT thissize;
2402 13820 : rtx part;
2403 13820 : unsigned HOST_WIDE_INT thispos;
2404 13820 : unsigned HOST_WIDE_INT offset;
2405 :
2406 13820 : offset = (bitpos + bitsdone) / unit;
2407 13820 : thispos = (bitpos + bitsdone) % unit;
2408 :
2409 : /* THISSIZE must not overrun a word boundary. Otherwise,
2410 : extract_fixed_bit_field will call us again, and we will mutually
2411 : recurse forever. */
2412 13820 : thissize = MIN (bitsize - bitsdone, BITS_PER_WORD);
2413 13820 : thissize = MIN (thissize, unit - thispos);
2414 :
2415 : /* If OP0 is a register, then handle OFFSET here. */
2416 13820 : rtx op0_piece = op0;
2417 13820 : opt_scalar_int_mode op0_piece_mode = op0_mode;
2418 13820 : if (SUBREG_P (op0) || REG_P (op0))
2419 : {
2420 120 : op0_piece = operand_subword_force (op0, offset, op0_mode.require ());
2421 120 : op0_piece_mode = word_mode;
2422 120 : offset = 0;
2423 : }
2424 :
2425 : /* Extract the parts in bit-counting order,
2426 : whose meaning is determined by BYTES_PER_UNIT.
2427 : OFFSET is in UNITs, and UNIT is in bits. */
2428 27640 : part = extract_fixed_bit_field (word_mode, op0_piece, op0_piece_mode,
2429 13820 : thissize, offset * unit + thispos,
2430 : 0, 1, reverse);
2431 13820 : bitsdone += thissize;
2432 :
2433 : /* Shift this part into place for the result. */
2434 13820 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
2435 : {
2436 4 : if (bitsize != bitsdone)
2437 2 : part = expand_shift (LSHIFT_EXPR, word_mode, part,
2438 2 : bitsize - bitsdone, 0, 1);
2439 : }
2440 : else
2441 : {
2442 13816 : if (bitsdone != thissize)
2443 9457 : part = expand_shift (LSHIFT_EXPR, word_mode, part,
2444 9457 : bitsdone - thissize, 0, 1);
2445 : }
2446 :
2447 13820 : if (first)
2448 : result = part;
2449 : else
2450 : /* Combine the parts with bitwise or. This works
2451 : because we extracted each part as an unsigned bit field. */
2452 9459 : result = expand_binop (word_mode, ior_optab, part, result, NULL_RTX, 1,
2453 : OPTAB_LIB_WIDEN);
2454 :
2455 13820 : first = 0;
2456 : }
2457 :
2458 : /* Unsigned bit field: we are done. */
2459 4361 : if (unsignedp)
2460 : return result;
2461 : /* Signed bit field: sign-extend with two arithmetic shifts. */
2462 1464 : result = expand_shift (LSHIFT_EXPR, word_mode, result,
2463 1464 : BITS_PER_WORD - bitsize, NULL_RTX, 0);
2464 1464 : return expand_shift (RSHIFT_EXPR, word_mode, result,
2465 1464 : BITS_PER_WORD - bitsize, NULL_RTX, 0);
2466 : }
2467 :
2468 : /* Try to read the low bits of SRC as an rvalue of mode MODE, preserving
2469 : the bit pattern. SRC_MODE is the mode of SRC; if this is smaller than
2470 : MODE, fill the upper bits with zeros. Fail if the layout of either
2471 : mode is unknown (as for CC modes) or if the extraction would involve
2472 : unprofitable mode punning. Return the value on success, otherwise
2473 : return null.
2474 :
2475 : This is different from gen_lowpart* in these respects:
2476 :
2477 : - the returned value must always be considered an rvalue
2478 :
2479 : - when MODE is wider than SRC_MODE, the extraction involves
2480 : a zero extension
2481 :
2482 : - when MODE is smaller than SRC_MODE, the extraction involves
2483 : a truncation (and is thus subject to TARGET_TRULY_NOOP_TRUNCATION).
2484 :
2485 : In other words, this routine performs a computation, whereas the
2486 : gen_lowpart* routines are conceptually lvalue or rvalue subreg
2487 : operations. */
2488 :
2489 : rtx
2490 119613 : extract_low_bits (machine_mode mode, machine_mode src_mode, rtx src)
2491 : {
2492 119613 : scalar_int_mode int_mode, src_int_mode;
2493 :
2494 119613 : if (mode == src_mode)
2495 : return src;
2496 :
2497 81813 : if (CONSTANT_P (src))
2498 : {
2499 : /* simplify_gen_subreg can't be used here, as if simplify_subreg
2500 : fails, it will happily create (subreg (symbol_ref)) or similar
2501 : invalid SUBREGs. */
2502 19251 : poly_uint64 byte = subreg_lowpart_offset (mode, src_mode);
2503 19251 : rtx ret = simplify_subreg (mode, src, src_mode, byte);
2504 19251 : if (ret)
2505 : return ret;
2506 :
2507 22 : if (GET_MODE (src) == VOIDmode
2508 22 : || !validate_subreg (mode, src_mode, src, byte))
2509 : return NULL_RTX;
2510 :
2511 17 : src = force_reg (GET_MODE (src), src);
2512 17 : return gen_rtx_SUBREG (mode, src, byte);
2513 : }
2514 :
2515 62562 : if (GET_MODE_CLASS (mode) == MODE_CC || GET_MODE_CLASS (src_mode) == MODE_CC)
2516 : return NULL_RTX;
2517 :
2518 125124 : if (known_eq (GET_MODE_BITSIZE (mode), GET_MODE_BITSIZE (src_mode))
2519 62562 : && targetm.modes_tieable_p (mode, src_mode))
2520 : {
2521 3970 : rtx x = gen_lowpart_common (mode, src);
2522 3970 : if (x)
2523 : return x;
2524 : }
2525 :
2526 58603 : if (!int_mode_for_mode (src_mode).exists (&src_int_mode)
2527 58603 : || !int_mode_for_mode (mode).exists (&int_mode))
2528 : return NULL_RTX;
2529 :
2530 58590 : if (!targetm.modes_tieable_p (src_int_mode, src_mode))
2531 : return NULL_RTX;
2532 57504 : if (!targetm.modes_tieable_p (int_mode, mode))
2533 : return NULL_RTX;
2534 :
2535 55432 : src = gen_lowpart (src_int_mode, src);
2536 55432 : if (!validate_subreg (int_mode, src_int_mode, src,
2537 : subreg_lowpart_offset (int_mode, src_int_mode)))
2538 : return NULL_RTX;
2539 :
2540 55420 : src = convert_modes (int_mode, src_int_mode, src, true);
2541 55420 : src = gen_lowpart (mode, src);
2542 55420 : return src;
2543 : }
2544 :
2545 : /* Add INC into TARGET. */
2546 :
2547 : void
2548 1186 : expand_inc (rtx target, rtx inc)
2549 : {
2550 1186 : rtx value = expand_binop (GET_MODE (target), add_optab,
2551 : target, inc,
2552 : target, 0, OPTAB_LIB_WIDEN);
2553 1186 : if (value != target)
2554 61 : emit_move_insn (target, value);
2555 1186 : }
2556 :
2557 : /* Subtract DEC from TARGET. */
2558 :
2559 : void
2560 1221 : expand_dec (rtx target, rtx dec)
2561 : {
2562 1221 : rtx value = expand_binop (GET_MODE (target), sub_optab,
2563 : target, dec,
2564 : target, 0, OPTAB_LIB_WIDEN);
2565 1221 : if (value != target)
2566 0 : emit_move_insn (target, value);
2567 1221 : }
2568 :
2569 : /* Output a shift instruction for expression code CODE,
2570 : with SHIFTED being the rtx for the value to shift,
2571 : and AMOUNT the rtx for the amount to shift by.
2572 : Store the result in the rtx TARGET, if that is convenient.
2573 : If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2574 : Return the rtx for where the value is.
2575 : If that cannot be done, abort the compilation unless MAY_FAIL is true,
2576 : in which case 0 is returned. */
2577 :
2578 : static rtx
2579 1544531 : expand_shift_1 (enum tree_code code, machine_mode mode, rtx shifted,
2580 : rtx amount, rtx target, int unsignedp, bool may_fail = false)
2581 : {
2582 1544531 : rtx op1, temp = 0;
2583 1544531 : int left = (code == LSHIFT_EXPR || code == LROTATE_EXPR);
2584 1544531 : int rotate = (code == LROTATE_EXPR || code == RROTATE_EXPR);
2585 1544531 : optab lshift_optab = ashl_optab;
2586 1544531 : optab rshift_arith_optab = ashr_optab;
2587 1544531 : optab rshift_uns_optab = lshr_optab;
2588 1544531 : optab lrotate_optab = rotl_optab;
2589 1544531 : optab rrotate_optab = rotr_optab;
2590 1544531 : machine_mode op1_mode;
2591 1544531 : scalar_mode scalar_mode = GET_MODE_INNER (mode);
2592 1544531 : int attempt;
2593 1544531 : bool speed = optimize_insn_for_speed_p ();
2594 :
2595 1544531 : op1 = amount;
2596 1544531 : op1_mode = GET_MODE (op1);
2597 :
2598 : /* Determine whether the shift/rotate amount is a vector, or scalar. If the
2599 : shift amount is a vector, use the vector/vector shift patterns. */
2600 1544531 : if (VECTOR_MODE_P (mode) && VECTOR_MODE_P (op1_mode))
2601 : {
2602 1544531 : lshift_optab = vashl_optab;
2603 1544531 : rshift_arith_optab = vashr_optab;
2604 1544531 : rshift_uns_optab = vlshr_optab;
2605 1544531 : lrotate_optab = vrotl_optab;
2606 1544531 : rrotate_optab = vrotr_optab;
2607 : }
2608 :
2609 : /* Previously detected shift-counts computed by NEGATE_EXPR
2610 : and shifted in the other direction; but that does not work
2611 : on all machines. */
2612 :
2613 1544531 : if (SHIFT_COUNT_TRUNCATED)
2614 : {
2615 : if (CONST_INT_P (op1)
2616 : && ((unsigned HOST_WIDE_INT) INTVAL (op1) >=
2617 : (unsigned HOST_WIDE_INT) GET_MODE_BITSIZE (scalar_mode)))
2618 : op1 = gen_int_shift_amount (mode,
2619 : (unsigned HOST_WIDE_INT) INTVAL (op1)
2620 : % GET_MODE_BITSIZE (scalar_mode));
2621 : else if (GET_CODE (op1) == SUBREG
2622 : && subreg_lowpart_p (op1)
2623 : && SCALAR_INT_MODE_P (GET_MODE (SUBREG_REG (op1)))
2624 : && SCALAR_INT_MODE_P (GET_MODE (op1)))
2625 : op1 = SUBREG_REG (op1);
2626 : }
2627 :
2628 : /* Canonicalize rotates by constant amount. We may canonicalize
2629 : to reduce the immediate or if the ISA can rotate by constants
2630 : in only on direction. */
2631 1544531 : if (rotate && reverse_rotate_by_imm_p (scalar_mode, left, op1))
2632 : {
2633 3290 : op1 = gen_int_shift_amount (mode, (GET_MODE_BITSIZE (scalar_mode)
2634 3290 : - INTVAL (op1)));
2635 3290 : left = !left;
2636 3290 : code = left ? LROTATE_EXPR : RROTATE_EXPR;
2637 : }
2638 :
2639 : /* Rotation of 16bit values by 8 bits is effectively equivalent to a bswaphi.
2640 : Note that this is not the case for bigger values. For instance a rotation
2641 : of 0x01020304 by 16 bits gives 0x03040102 which is different from
2642 : 0x04030201 (bswapsi). */
2643 1544531 : if (rotate
2644 8655 : && CONST_INT_P (op1)
2645 5440 : && INTVAL (op1) == BITS_PER_UNIT
2646 1070 : && GET_MODE_SIZE (scalar_mode) == 2
2647 1545426 : && optab_handler (bswap_optab, mode) != CODE_FOR_nothing)
2648 894 : return expand_unop (mode, bswap_optab, shifted, NULL_RTX, unsignedp);
2649 :
2650 1543637 : if (op1 == const0_rtx)
2651 : return shifted;
2652 :
2653 : /* Check whether its cheaper to implement a left shift by a constant
2654 : bit count by a sequence of additions. */
2655 1494882 : if (code == LSHIFT_EXPR
2656 895354 : && CONST_INT_P (op1)
2657 865052 : && INTVAL (op1) > 0
2658 865027 : && INTVAL (op1) < GET_MODE_PRECISION (scalar_mode)
2659 865027 : && INTVAL (op1) < MAX_BITS_PER_WORD
2660 860269 : && (shift_cost (speed, mode, INTVAL (op1))
2661 860269 : > INTVAL (op1) * add_cost (speed, mode))
2662 1497864 : && shift_cost (speed, mode, INTVAL (op1)) != MAX_COST)
2663 : {
2664 : int i;
2665 6152 : for (i = 0; i < INTVAL (op1); i++)
2666 : {
2667 3170 : temp = force_reg (mode, shifted);
2668 3170 : shifted = expand_binop (mode, add_optab, temp, temp, NULL_RTX,
2669 : unsignedp, OPTAB_LIB_WIDEN);
2670 : }
2671 : return shifted;
2672 : }
2673 :
2674 2983838 : for (attempt = 0; temp == 0 && attempt < 3; attempt++)
2675 : {
2676 1491976 : enum optab_methods methods;
2677 :
2678 1491976 : if (attempt == 0)
2679 : methods = OPTAB_DIRECT;
2680 76 : else if (attempt == 1)
2681 : methods = OPTAB_WIDEN;
2682 : else
2683 38 : methods = OPTAB_LIB_WIDEN;
2684 :
2685 1491976 : if (rotate)
2686 : {
2687 : /* Widening does not work for rotation. */
2688 7837 : if (methods == OPTAB_WIDEN)
2689 38 : continue;
2690 7799 : else if (methods == OPTAB_LIB_WIDEN)
2691 : {
2692 : /* If we have been unable to open-code this by a rotation,
2693 : do it as the IOR or PLUS of two shifts. I.e., to rotate
2694 : A by N bits, compute
2695 : (A << N) | ((unsigned) A >> ((-N) & (C - 1)))
2696 : where C is the bitsize of A. If N cannot be zero,
2697 : use PLUS instead of IOR.
2698 :
2699 : It is theoretically possible that the target machine might
2700 : not be able to perform either shift and hence we would
2701 : be making two libcalls rather than just the one for the
2702 : shift (similarly if IOR could not be done). We will allow
2703 : this extremely unlikely lossage to avoid complicating the
2704 : code below. */
2705 :
2706 38 : rtx subtarget = target == shifted ? 0 : target;
2707 38 : rtx new_amount, other_amount;
2708 38 : rtx temp1;
2709 :
2710 38 : new_amount = op1;
2711 38 : if (op1 == const0_rtx)
2712 : return shifted;
2713 38 : else if (CONST_INT_P (op1))
2714 23 : other_amount = gen_int_shift_amount
2715 23 : (mode, GET_MODE_BITSIZE (scalar_mode) - INTVAL (op1));
2716 : else
2717 : {
2718 15 : other_amount
2719 30 : = simplify_gen_unary (NEG, GET_MODE (op1),
2720 15 : op1, GET_MODE (op1));
2721 15 : HOST_WIDE_INT mask = GET_MODE_PRECISION (scalar_mode) - 1;
2722 15 : other_amount
2723 15 : = simplify_gen_binary (AND, GET_MODE (op1), other_amount,
2724 15 : gen_int_mode (mask, GET_MODE (op1)));
2725 : }
2726 :
2727 38 : shifted = force_reg (mode, shifted);
2728 :
2729 51 : temp = expand_shift_1 (left ? LSHIFT_EXPR : RSHIFT_EXPR,
2730 : mode, shifted, new_amount, 0, 1);
2731 51 : temp1 = expand_shift_1 (left ? RSHIFT_EXPR : LSHIFT_EXPR,
2732 : mode, shifted, other_amount,
2733 : subtarget, 1);
2734 38 : return expand_binop (mode,
2735 38 : CONST_INT_P (op1) ? add_optab : ior_optab,
2736 38 : temp, temp1, target, unsignedp, methods);
2737 : }
2738 :
2739 11367 : temp = expand_binop (mode,
2740 : left ? lrotate_optab : rrotate_optab,
2741 : shifted, op1, target, unsignedp, methods);
2742 : }
2743 1484139 : else if (unsignedp)
2744 1234890 : temp = expand_binop (mode,
2745 : left ? lshift_optab : rshift_uns_optab,
2746 : shifted, op1, target, unsignedp, methods);
2747 :
2748 : /* Do arithmetic shifts.
2749 : Also, if we are going to widen the operand, we can just as well
2750 : use an arithmetic right-shift instead of a logical one. */
2751 1491900 : if (temp == 0 && ! rotate
2752 594614 : && (! unsignedp || (! left && methods == OPTAB_WIDEN)))
2753 : {
2754 : enum optab_methods methods1 = methods;
2755 :
2756 : /* If trying to widen a log shift to an arithmetic shift,
2757 : don't accept an arithmetic shift of the same size. */
2758 : if (unsignedp)
2759 : methods1 = OPTAB_MUST_WIDEN;
2760 :
2761 : /* Arithmetic shift */
2762 :
2763 841016 : temp = expand_binop (mode,
2764 : left ? lshift_optab : rshift_arith_optab,
2765 : shifted, op1, target, unsignedp, methods1);
2766 : }
2767 :
2768 : /* We used to try extzv here for logical right shifts, but that was
2769 : only useful for one machine, the VAX, and caused poor code
2770 : generation there for lshrdi3, so the code was deleted and a
2771 : define_expand for lshrsi3 was added to vax.md. */
2772 : }
2773 :
2774 1491862 : gcc_assert (temp != NULL_RTX || may_fail);
2775 : return temp;
2776 : }
2777 :
2778 : /* Output a shift instruction for expression code CODE,
2779 : with SHIFTED being the rtx for the value to shift,
2780 : and AMOUNT the amount to shift by.
2781 : Store the result in the rtx TARGET, if that is convenient.
2782 : If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2783 : Return the rtx for where the value is. */
2784 :
2785 : rtx
2786 1236814 : expand_shift (enum tree_code code, machine_mode mode, rtx shifted,
2787 : poly_int64 amount, rtx target, int unsignedp)
2788 : {
2789 1236814 : return expand_shift_1 (code, mode, shifted,
2790 : gen_int_shift_amount (mode, amount),
2791 1236814 : target, unsignedp);
2792 : }
2793 :
2794 : /* Likewise, but return 0 if that cannot be done. */
2795 :
2796 : rtx
2797 325 : maybe_expand_shift (enum tree_code code, machine_mode mode, rtx shifted,
2798 : int amount, rtx target, int unsignedp)
2799 : {
2800 325 : return expand_shift_1 (code, mode,
2801 325 : shifted, GEN_INT (amount), target, unsignedp, true);
2802 : }
2803 :
2804 : /* Output a shift instruction for expression code CODE,
2805 : with SHIFTED being the rtx for the value to shift,
2806 : and AMOUNT the tree for the amount to shift by.
2807 : Store the result in the rtx TARGET, if that is convenient.
2808 : If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2809 : Return the rtx for where the value is. */
2810 :
2811 : rtx
2812 307316 : expand_variable_shift (enum tree_code code, machine_mode mode, rtx shifted,
2813 : tree amount, rtx target, int unsignedp)
2814 : {
2815 307316 : return expand_shift_1 (code, mode,
2816 307316 : shifted, expand_normal (amount), target, unsignedp);
2817 : }
2818 :
2819 :
2820 : static void synth_mult (struct algorithm *, unsigned HOST_WIDE_INT,
2821 : const struct mult_cost *, machine_mode mode);
2822 : static rtx expand_mult_const (machine_mode, rtx, HOST_WIDE_INT, rtx,
2823 : const struct algorithm *, enum mult_variant);
2824 : static unsigned HOST_WIDE_INT invert_mod2n (unsigned HOST_WIDE_INT, int);
2825 : static rtx extract_high_half (scalar_int_mode, rtx);
2826 : static rtx expmed_mult_highpart (scalar_int_mode, rtx, rtx, rtx, int, int);
2827 :
2828 : /* Compute and return the best algorithm for multiplying by T.
2829 : The algorithm must cost less than cost_limit
2830 : If retval.cost >= COST_LIMIT, no algorithm was found and all
2831 : other field of the returned struct are undefined.
2832 : MODE is the machine mode of the multiplication. */
2833 :
2834 : static void
2835 38505376 : synth_mult (struct algorithm *alg_out, unsigned HOST_WIDE_INT t,
2836 : const struct mult_cost *cost_limit, machine_mode mode)
2837 : {
2838 38505376 : int m;
2839 38505376 : struct algorithm *alg_in, *best_alg;
2840 38505376 : struct mult_cost best_cost;
2841 38505376 : struct mult_cost new_limit;
2842 38505376 : int op_cost, op_latency;
2843 38505376 : unsigned HOST_WIDE_INT orig_t = t;
2844 38505376 : unsigned HOST_WIDE_INT q;
2845 38505376 : int maxm, hash_index;
2846 38505376 : bool cache_hit = false;
2847 38505376 : enum alg_code cache_alg = alg_zero;
2848 38505376 : bool speed = optimize_insn_for_speed_p ();
2849 38505376 : scalar_int_mode imode;
2850 38505376 : struct alg_hash_entry *entry_ptr;
2851 :
2852 : /* Indicate that no algorithm is yet found. If no algorithm
2853 : is found, this value will be returned and indicate failure. */
2854 38505376 : alg_out->cost.cost = cost_limit->cost + 1;
2855 38505376 : alg_out->cost.latency = cost_limit->latency + 1;
2856 :
2857 38505376 : if (cost_limit->cost < 0
2858 31419949 : || (cost_limit->cost == 0 && cost_limit->latency <= 0))
2859 31117050 : return;
2860 :
2861 : /* Be prepared for vector modes. */
2862 52048682 : imode = as_a <scalar_int_mode> (GET_MODE_INNER (mode));
2863 :
2864 76642965 : maxm = MIN (BITS_PER_WORD, GET_MODE_BITSIZE (imode));
2865 :
2866 : /* Restrict the bits of "t" to the multiplication's mode. */
2867 26024341 : t &= GET_MODE_MASK (imode);
2868 :
2869 : /* t == 1 can be done in zero cost. */
2870 26024341 : if (t == 1)
2871 : {
2872 6011927 : alg_out->ops = 1;
2873 6011927 : alg_out->cost.cost = 0;
2874 6011927 : alg_out->cost.latency = 0;
2875 6011927 : alg_out->op[0] = alg_m;
2876 6011927 : return;
2877 : }
2878 :
2879 : /* t == 0 sometimes has a cost. If it does and it exceeds our limit,
2880 : fail now. */
2881 20012414 : if (t == 0)
2882 : {
2883 531138 : if (MULT_COST_LESS (cost_limit, zero_cost (speed)))
2884 : return;
2885 : else
2886 : {
2887 531138 : alg_out->ops = 1;
2888 531138 : alg_out->cost.cost = zero_cost (speed);
2889 531138 : alg_out->cost.latency = zero_cost (speed);
2890 531138 : alg_out->op[0] = alg_zero;
2891 531138 : return;
2892 : }
2893 : }
2894 :
2895 : /* We'll be needing a couple extra algorithm structures now. */
2896 :
2897 19481276 : alg_in = XALLOCA (struct algorithm);
2898 19481276 : best_alg = XALLOCA (struct algorithm);
2899 19481276 : best_cost = *cost_limit;
2900 :
2901 : /* Compute the hash index. */
2902 19481276 : hash_index = (t ^ (unsigned int) mode ^ (speed * 256)) % NUM_ALG_HASH_ENTRIES;
2903 :
2904 : /* See if we already know what to do for T. */
2905 19481276 : entry_ptr = alg_hash_entry_ptr (hash_index);
2906 19481276 : if (entry_ptr->t == t
2907 15270055 : && entry_ptr->mode == mode
2908 15270055 : && entry_ptr->speed == speed
2909 15270055 : && entry_ptr->alg != alg_unknown)
2910 : {
2911 15270055 : cache_alg = entry_ptr->alg;
2912 :
2913 15270055 : if (cache_alg == alg_impossible)
2914 : {
2915 : /* The cache tells us that it's impossible to synthesize
2916 : multiplication by T within entry_ptr->cost. */
2917 7516452 : if (!CHEAPER_MULT_COST (&entry_ptr->cost, cost_limit))
2918 : /* COST_LIMIT is at least as restrictive as the one
2919 : recorded in the hash table, in which case we have no
2920 : hope of synthesizing a multiplication. Just
2921 : return. */
2922 : return;
2923 :
2924 : /* If we get here, COST_LIMIT is less restrictive than the
2925 : one recorded in the hash table, so we may be able to
2926 : synthesize a multiplication. Proceed as if we didn't
2927 : have the cache entry. */
2928 : }
2929 : else
2930 : {
2931 7753603 : if (CHEAPER_MULT_COST (cost_limit, &entry_ptr->cost))
2932 : /* The cached algorithm shows that this multiplication
2933 : requires more cost than COST_LIMIT. Just return. This
2934 : way, we don't clobber this cache entry with
2935 : alg_impossible but retain useful information. */
2936 : return;
2937 :
2938 6997646 : cache_hit = true;
2939 :
2940 6997646 : switch (cache_alg)
2941 : {
2942 4616668 : case alg_shift:
2943 4616668 : goto do_alg_shift;
2944 :
2945 942623 : case alg_add_t_m2:
2946 942623 : case alg_sub_t_m2:
2947 942623 : goto do_alg_addsub_t_m2;
2948 :
2949 118985 : case alg_add_factor:
2950 118985 : case alg_sub_factor:
2951 118985 : goto do_alg_addsub_factor;
2952 :
2953 1319359 : case alg_add_t2_m:
2954 1319359 : goto do_alg_add_t2_m;
2955 :
2956 11 : case alg_sub_t2_m:
2957 11 : goto do_alg_sub_t2_m;
2958 :
2959 0 : default:
2960 0 : gcc_unreachable ();
2961 : }
2962 : }
2963 : }
2964 :
2965 : /* If we have a group of zero bits at the low-order part of T, try
2966 : multiplying by the remaining bits and then doing a shift. */
2967 :
2968 5212463 : if ((t & 1) == 0)
2969 : {
2970 2677760 : do_alg_shift:
2971 7294428 : m = ctz_or_zero (t); /* m = number of low zero bits */
2972 7294428 : if (m < maxm)
2973 : {
2974 7292688 : q = t >> m;
2975 : /* The function expand_shift will choose between a shift and
2976 : a sequence of additions, so the observed cost is given as
2977 : MIN (m * add_cost(speed, mode), shift_cost(speed, mode, m)). */
2978 7292688 : op_cost = m * add_cost (speed, mode);
2979 7292688 : if (shift_cost (speed, mode, m) < op_cost)
2980 : op_cost = shift_cost (speed, mode, m);
2981 7292688 : new_limit.cost = best_cost.cost - op_cost;
2982 7292688 : new_limit.latency = best_cost.latency - op_cost;
2983 7292688 : synth_mult (alg_in, q, &new_limit, mode);
2984 :
2985 7292688 : alg_in->cost.cost += op_cost;
2986 7292688 : alg_in->cost.latency += op_cost;
2987 7292688 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
2988 : {
2989 4247872 : best_cost = alg_in->cost;
2990 4247872 : std::swap (alg_in, best_alg);
2991 4247872 : best_alg->log[best_alg->ops] = m;
2992 4247872 : best_alg->op[best_alg->ops] = alg_shift;
2993 : }
2994 :
2995 : /* See if treating ORIG_T as a signed number yields a better
2996 : sequence. Try this sequence only for a negative ORIG_T
2997 : as it would be useless for a non-negative ORIG_T. */
2998 7292688 : if ((HOST_WIDE_INT) orig_t < 0)
2999 : {
3000 : /* Shift ORIG_T as follows because a right shift of a
3001 : negative-valued signed type is implementation
3002 : defined. */
3003 628085 : q = ~(~orig_t >> m);
3004 : /* The function expand_shift will choose between a shift
3005 : and a sequence of additions, so the observed cost is
3006 : given as MIN (m * add_cost(speed, mode),
3007 : shift_cost(speed, mode, m)). */
3008 628085 : op_cost = m * add_cost (speed, mode);
3009 628085 : if (shift_cost (speed, mode, m) < op_cost)
3010 : op_cost = shift_cost (speed, mode, m);
3011 628085 : new_limit.cost = best_cost.cost - op_cost;
3012 628085 : new_limit.latency = best_cost.latency - op_cost;
3013 628085 : synth_mult (alg_in, q, &new_limit, mode);
3014 :
3015 628085 : alg_in->cost.cost += op_cost;
3016 628085 : alg_in->cost.latency += op_cost;
3017 628085 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3018 : {
3019 596050 : best_cost = alg_in->cost;
3020 596050 : std::swap (alg_in, best_alg);
3021 596050 : best_alg->log[best_alg->ops] = m;
3022 596050 : best_alg->op[best_alg->ops] = alg_shift;
3023 : }
3024 : }
3025 : }
3026 1740 : else if (2 * BITS_PER_WORD <= HOST_BITS_PER_WIDE_INT
3027 1740 : && GET_MODE_BITSIZE (imode) == 2 * BITS_PER_WORD
3028 1728 : && m >= BITS_PER_WORD
3029 3468 : && imode == mode)
3030 : {
3031 1728 : q = t >> m;
3032 1728 : int op1_cost = shift_cost (speed, mode, m - BITS_PER_WORD);
3033 1728 : int op2_cost = zero_cost (speed);
3034 1728 : op_latency = MAX (op1_cost, op2_cost);
3035 1728 : op_cost = op1_cost + op2_cost;
3036 :
3037 1728 : new_limit.cost = best_cost.cost - op_cost;
3038 1728 : new_limit.latency = best_cost.latency - op_latency;
3039 1728 : synth_mult (alg_in, q, &new_limit, mode);
3040 1728 : alg_in->cost.cost += op_cost;
3041 1728 : alg_in->cost.latency += op_latency;
3042 1728 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3043 : {
3044 1662 : best_cost = alg_in->cost;
3045 1662 : std::swap (alg_in, best_alg);
3046 1662 : best_alg->log[best_alg->ops] = m;
3047 1662 : best_alg->op[best_alg->ops] = alg_shift;
3048 : }
3049 : }
3050 7294428 : if (cache_hit)
3051 4616668 : goto done;
3052 : }
3053 :
3054 : /* If we have an odd number, add or subtract one. */
3055 2677760 : if ((t & 1) != 0)
3056 : {
3057 3477326 : unsigned HOST_WIDE_INT w;
3058 :
3059 0 : do_alg_addsub_t_m2:
3060 42053726 : for (w = 1; (w & t) != 0; w <<= 1)
3061 : ;
3062 : /* If T was -1, then W will be zero after the loop. This is another
3063 : case where T ends with ...111. Handling this with (T + 1) and
3064 : subtract 1 produces slightly better code and results in algorithm
3065 : selection much faster than treating it like the ...0111 case
3066 : below. */
3067 3477326 : if (w == 0
3068 3062794 : || (w > 2
3069 : /* Reject the case where t is 3.
3070 : Thus we prefer addition in that case. */
3071 3062794 : && t != 3))
3072 : {
3073 : /* T ends with ...111. Multiply by (T + 1) and subtract T. */
3074 :
3075 1909294 : op_cost = add_cost (speed, mode);
3076 1909294 : new_limit.cost = best_cost.cost - op_cost;
3077 1909294 : new_limit.latency = best_cost.latency - op_cost;
3078 1909294 : synth_mult (alg_in, t + 1, &new_limit, mode);
3079 :
3080 1909294 : alg_in->cost.cost += op_cost;
3081 1909294 : alg_in->cost.latency += op_cost;
3082 1909294 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3083 : {
3084 712665 : best_cost = alg_in->cost;
3085 712665 : std::swap (alg_in, best_alg);
3086 712665 : best_alg->log[best_alg->ops] = 0;
3087 712665 : best_alg->op[best_alg->ops] = alg_sub_t_m2;
3088 : }
3089 : }
3090 : else
3091 : {
3092 : /* T ends with ...01 or ...011. Multiply by (T - 1) and add T. */
3093 :
3094 1568032 : op_cost = add_cost (speed, mode);
3095 1568032 : new_limit.cost = best_cost.cost - op_cost;
3096 1568032 : new_limit.latency = best_cost.latency - op_cost;
3097 1568032 : synth_mult (alg_in, t - 1, &new_limit, mode);
3098 :
3099 1568032 : alg_in->cost.cost += op_cost;
3100 1568032 : alg_in->cost.latency += op_cost;
3101 1568032 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3102 : {
3103 193885 : best_cost = alg_in->cost;
3104 193885 : std::swap (alg_in, best_alg);
3105 193885 : best_alg->log[best_alg->ops] = 0;
3106 193885 : best_alg->op[best_alg->ops] = alg_add_t_m2;
3107 : }
3108 : }
3109 :
3110 : /* We may be able to calculate a * -7, a * -15, a * -31, etc
3111 : quickly with a - a * n for some appropriate constant n. */
3112 3477326 : m = exact_log2 (-orig_t + 1);
3113 3477326 : if (m >= 0 && m < maxm)
3114 : {
3115 737349 : op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
3116 : /* If the target has a cheap shift-and-subtract insn use
3117 : that in preference to a shift insn followed by a sub insn.
3118 : Assume that the shift-and-sub is "atomic" with a latency
3119 : equal to it's cost, otherwise assume that on superscalar
3120 : hardware the shift may be executed concurrently with the
3121 : earlier steps in the algorithm. */
3122 737349 : if (shiftsub1_cost (speed, mode, m) <= op_cost)
3123 : {
3124 : op_cost = shiftsub1_cost (speed, mode, m);
3125 : op_latency = op_cost;
3126 : }
3127 : else
3128 731587 : op_latency = add_cost (speed, mode);
3129 :
3130 737349 : new_limit.cost = best_cost.cost - op_cost;
3131 737349 : new_limit.latency = best_cost.latency - op_latency;
3132 737349 : synth_mult (alg_in, (unsigned HOST_WIDE_INT) (-orig_t + 1) >> m,
3133 : &new_limit, mode);
3134 :
3135 737349 : alg_in->cost.cost += op_cost;
3136 737349 : alg_in->cost.latency += op_latency;
3137 737349 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3138 : {
3139 190412 : best_cost = alg_in->cost;
3140 190412 : std::swap (alg_in, best_alg);
3141 190412 : best_alg->log[best_alg->ops] = m;
3142 190412 : best_alg->op[best_alg->ops] = alg_sub_t_m2;
3143 : }
3144 : }
3145 :
3146 3477326 : if (cache_hit)
3147 942623 : goto done;
3148 : }
3149 :
3150 : /* Look for factors of t of the form
3151 : t = q(2**m +- 1), 2 <= m <= floor(log2(t - 1)).
3152 : If we find such a factor, we can multiply by t using an algorithm that
3153 : multiplies by q, shift the result by m and add/subtract it to itself.
3154 :
3155 : We search for large factors first and loop down, even if large factors
3156 : are less probable than small; if we find a large factor we will find a
3157 : good sequence quickly, and therefore be able to prune (by decreasing
3158 : COST_LIMIT) the search. */
3159 :
3160 2677760 : do_alg_addsub_factor:
3161 113360341 : for (m = floor_log2 (t - 1); m >= 2; m--)
3162 : {
3163 110779498 : unsigned HOST_WIDE_INT d;
3164 :
3165 110779498 : d = (HOST_WIDE_INT_1U << m) + 1;
3166 110779498 : if (t % d == 0 && t > d && m < maxm
3167 1288105 : && (!cache_hit || cache_alg == alg_add_factor))
3168 : {
3169 1288105 : op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
3170 1288105 : if (shiftadd_cost (speed, mode, m) <= op_cost)
3171 : op_cost = shiftadd_cost (speed, mode, m);
3172 :
3173 1288105 : op_latency = op_cost;
3174 :
3175 :
3176 1288105 : new_limit.cost = best_cost.cost - op_cost;
3177 1288105 : new_limit.latency = best_cost.latency - op_latency;
3178 1288105 : synth_mult (alg_in, t / d, &new_limit, mode);
3179 :
3180 1288105 : alg_in->cost.cost += op_cost;
3181 1288105 : alg_in->cost.latency += op_latency;
3182 1288105 : if (alg_in->cost.latency < op_cost)
3183 260418 : alg_in->cost.latency = op_cost;
3184 1288105 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3185 : {
3186 117192 : best_cost = alg_in->cost;
3187 117192 : std::swap (alg_in, best_alg);
3188 117192 : best_alg->log[best_alg->ops] = m;
3189 117192 : best_alg->op[best_alg->ops] = alg_add_factor;
3190 : }
3191 : /* Other factors will have been taken care of in the recursion. */
3192 : break;
3193 : }
3194 :
3195 109491393 : d = (HOST_WIDE_INT_1U << m) - 1;
3196 109491393 : if (t % d == 0 && t > d && m < maxm
3197 1462500 : && (!cache_hit || cache_alg == alg_sub_factor))
3198 : {
3199 1462500 : op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
3200 1462500 : if (shiftsub0_cost (speed, mode, m) <= op_cost)
3201 : op_cost = shiftsub0_cost (speed, mode, m);
3202 :
3203 1462500 : op_latency = op_cost;
3204 :
3205 1462500 : new_limit.cost = best_cost.cost - op_cost;
3206 1462500 : new_limit.latency = best_cost.latency - op_latency;
3207 1462500 : synth_mult (alg_in, t / d, &new_limit, mode);
3208 :
3209 1462500 : alg_in->cost.cost += op_cost;
3210 1462500 : alg_in->cost.latency += op_latency;
3211 1462500 : if (alg_in->cost.latency < op_cost)
3212 355312 : alg_in->cost.latency = op_cost;
3213 1462500 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3214 : {
3215 32909 : best_cost = alg_in->cost;
3216 32909 : std::swap (alg_in, best_alg);
3217 32909 : best_alg->log[best_alg->ops] = m;
3218 32909 : best_alg->op[best_alg->ops] = alg_sub_factor;
3219 : }
3220 : break;
3221 : }
3222 : }
3223 5331448 : if (cache_hit)
3224 118985 : goto done;
3225 :
3226 : /* Try shift-and-add (load effective address) instructions,
3227 : i.e. do a*3, a*5, a*9. */
3228 5212463 : if ((t & 1) != 0)
3229 : {
3230 2534703 : do_alg_add_t2_m:
3231 3854062 : q = t - 1;
3232 3854062 : m = ctz_hwi (q);
3233 3854062 : if (q && m < maxm)
3234 : {
3235 3854040 : op_cost = shiftadd_cost (speed, mode, m);
3236 3854040 : new_limit.cost = best_cost.cost - op_cost;
3237 3854040 : new_limit.latency = best_cost.latency - op_cost;
3238 3854040 : synth_mult (alg_in, (t - 1) >> m, &new_limit, mode);
3239 :
3240 3854040 : alg_in->cost.cost += op_cost;
3241 3854040 : alg_in->cost.latency += op_cost;
3242 3854040 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3243 : {
3244 1373407 : best_cost = alg_in->cost;
3245 1373407 : std::swap (alg_in, best_alg);
3246 1373407 : best_alg->log[best_alg->ops] = m;
3247 1373407 : best_alg->op[best_alg->ops] = alg_add_t2_m;
3248 : }
3249 : }
3250 3854062 : if (cache_hit)
3251 1319359 : goto done;
3252 :
3253 2534703 : do_alg_sub_t2_m:
3254 2534714 : q = t + 1;
3255 2534714 : m = ctz_hwi (q);
3256 2534714 : if (q && m < maxm)
3257 : {
3258 2512413 : op_cost = shiftsub0_cost (speed, mode, m);
3259 2512413 : new_limit.cost = best_cost.cost - op_cost;
3260 2512413 : new_limit.latency = best_cost.latency - op_cost;
3261 2512413 : synth_mult (alg_in, (t + 1) >> m, &new_limit, mode);
3262 :
3263 2512413 : alg_in->cost.cost += op_cost;
3264 2512413 : alg_in->cost.latency += op_cost;
3265 2512413 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3266 : {
3267 95 : best_cost = alg_in->cost;
3268 95 : std::swap (alg_in, best_alg);
3269 95 : best_alg->log[best_alg->ops] = m;
3270 95 : best_alg->op[best_alg->ops] = alg_sub_t2_m;
3271 : }
3272 : }
3273 : if (cache_hit)
3274 : goto done;
3275 : }
3276 :
3277 2677760 : done:
3278 : /* If best_cost has not decreased, we have not found any algorithm. */
3279 12210109 : if (!CHEAPER_MULT_COST (&best_cost, cost_limit))
3280 : {
3281 : /* We failed to find an algorithm. Record alg_impossible for
3282 : this case (that is, <T, MODE, COST_LIMIT>) so that next time
3283 : we are asked to find an algorithm for T within the same or
3284 : lower COST_LIMIT, we can immediately return to the
3285 : caller. */
3286 4821783 : entry_ptr->t = t;
3287 4821783 : entry_ptr->mode = mode;
3288 4821783 : entry_ptr->speed = speed;
3289 4821783 : entry_ptr->alg = alg_impossible;
3290 4821783 : entry_ptr->cost = *cost_limit;
3291 4821783 : return;
3292 : }
3293 :
3294 : /* Cache the result. */
3295 7388326 : if (!cache_hit)
3296 : {
3297 711180 : entry_ptr->t = t;
3298 711180 : entry_ptr->mode = mode;
3299 711180 : entry_ptr->speed = speed;
3300 711180 : entry_ptr->alg = best_alg->op[best_alg->ops];
3301 711180 : entry_ptr->cost.cost = best_cost.cost;
3302 711180 : entry_ptr->cost.latency = best_cost.latency;
3303 : }
3304 :
3305 : /* If we are getting a too long sequence for `struct algorithm'
3306 : to record, make this search fail. */
3307 7388326 : if (best_alg->ops == MAX_BITS_PER_WORD)
3308 : return;
3309 :
3310 : /* Copy the algorithm from temporary space to the space at alg_out.
3311 : We avoid using structure assignment because the majority of
3312 : best_alg is normally undefined, and this is a critical function. */
3313 7388326 : alg_out->ops = best_alg->ops + 1;
3314 7388326 : alg_out->cost = best_cost;
3315 7388326 : memcpy (alg_out->op, best_alg->op,
3316 7388326 : alg_out->ops * sizeof *alg_out->op);
3317 7388326 : memcpy (alg_out->log, best_alg->log,
3318 : alg_out->ops * sizeof *alg_out->log);
3319 : }
3320 :
3321 : /* Find the cheapest way of multiplying a value of mode MODE by VAL.
3322 : Try three variations:
3323 :
3324 : - a shift/add sequence based on VAL itself
3325 : - a shift/add sequence based on -VAL, followed by a negation
3326 : - a shift/add sequence based on VAL - 1, followed by an addition.
3327 :
3328 : Return true if the cheapest of these cost less than MULT_COST,
3329 : describing the algorithm in *ALG and final fixup in *VARIANT. */
3330 :
3331 : bool
3332 7317019 : choose_mult_variant (machine_mode mode, HOST_WIDE_INT val,
3333 : struct algorithm *alg, enum mult_variant *variant,
3334 : int mult_cost)
3335 : {
3336 7317019 : struct algorithm alg2;
3337 7317019 : struct mult_cost limit;
3338 7317019 : int op_cost;
3339 7317019 : bool speed = optimize_insn_for_speed_p ();
3340 :
3341 : /* Fail quickly for impossible bounds. */
3342 7317019 : if (mult_cost < 0)
3343 : return false;
3344 :
3345 : /* Ensure that mult_cost provides a reasonable upper bound.
3346 : Any constant multiplication can be performed with less
3347 : than 2 * bits additions. */
3348 14630920 : op_cost = 2 * GET_MODE_UNIT_BITSIZE (mode) * add_cost (speed, mode);
3349 7315460 : if (mult_cost > op_cost)
3350 : mult_cost = op_cost;
3351 :
3352 7315460 : *variant = basic_variant;
3353 7315460 : limit.cost = mult_cost;
3354 7315460 : limit.latency = mult_cost;
3355 7315460 : synth_mult (alg, val, &limit, mode);
3356 :
3357 : /* This works only if the inverted value actually fits in an
3358 : `unsigned int' */
3359 14630920 : if (HOST_BITS_PER_INT >= GET_MODE_UNIT_BITSIZE (mode))
3360 : {
3361 2620222 : op_cost = neg_cost (speed, mode);
3362 2620222 : if (MULT_COST_LESS (&alg->cost, mult_cost))
3363 : {
3364 2513771 : limit.cost = alg->cost.cost - op_cost;
3365 2513771 : limit.latency = alg->cost.latency - op_cost;
3366 : }
3367 : else
3368 : {
3369 106451 : limit.cost = mult_cost - op_cost;
3370 106451 : limit.latency = mult_cost - op_cost;
3371 : }
3372 :
3373 2620222 : synth_mult (&alg2, -val, &limit, mode);
3374 2620222 : alg2.cost.cost += op_cost;
3375 2620222 : alg2.cost.latency += op_cost;
3376 2620222 : if (CHEAPER_MULT_COST (&alg2.cost, &alg->cost))
3377 14730 : *alg = alg2, *variant = negate_variant;
3378 : }
3379 :
3380 : /* This proves very useful for division-by-constant. */
3381 7315460 : op_cost = add_cost (speed, mode);
3382 7315460 : if (MULT_COST_LESS (&alg->cost, mult_cost))
3383 : {
3384 6461058 : limit.cost = alg->cost.cost - op_cost;
3385 6461058 : limit.latency = alg->cost.latency - op_cost;
3386 : }
3387 : else
3388 : {
3389 854402 : limit.cost = mult_cost - op_cost;
3390 854402 : limit.latency = mult_cost - op_cost;
3391 : }
3392 :
3393 7315460 : if (val != HOST_WIDE_INT_MIN
3394 7315470 : || GET_MODE_UNIT_PRECISION (mode) == HOST_BITS_PER_WIDE_INT)
3395 : {
3396 7315460 : synth_mult (&alg2, val - HOST_WIDE_INT_1U, &limit, mode);
3397 7315460 : alg2.cost.cost += op_cost;
3398 7315460 : alg2.cost.latency += op_cost;
3399 7315460 : if (CHEAPER_MULT_COST (&alg2.cost, &alg->cost))
3400 2743 : *alg = alg2, *variant = add_variant;
3401 : }
3402 :
3403 7315460 : return MULT_COST_LESS (&alg->cost, mult_cost);
3404 : }
3405 :
3406 : /* A subroutine of expand_mult, used for constant multiplications.
3407 : Multiply OP0 by VAL in mode MODE, storing the result in TARGET if
3408 : convenient. Use the shift/add sequence described by ALG and apply
3409 : the final fixup specified by VARIANT. */
3410 :
3411 : static rtx
3412 138723 : expand_mult_const (machine_mode mode, rtx op0, HOST_WIDE_INT val,
3413 : rtx target, const struct algorithm *alg,
3414 : enum mult_variant variant)
3415 : {
3416 138723 : unsigned HOST_WIDE_INT val_so_far;
3417 138723 : rtx_insn *insn;
3418 138723 : rtx accum, tem;
3419 138723 : int opno;
3420 138723 : machine_mode nmode;
3421 :
3422 : /* Avoid referencing memory over and over and invalid sharing
3423 : on SUBREGs. */
3424 138723 : op0 = force_reg (mode, op0);
3425 :
3426 : /* ACCUM starts out either as OP0 or as a zero, depending on
3427 : the first operation. */
3428 :
3429 138723 : if (alg->op[0] == alg_zero)
3430 : {
3431 5484 : accum = copy_to_mode_reg (mode, CONST0_RTX (mode));
3432 5484 : val_so_far = 0;
3433 : }
3434 133239 : else if (alg->op[0] == alg_m)
3435 : {
3436 133239 : accum = copy_to_mode_reg (mode, op0);
3437 133239 : val_so_far = 1;
3438 : }
3439 : else
3440 0 : gcc_unreachable ();
3441 :
3442 395589 : for (opno = 1; opno < alg->ops; opno++)
3443 : {
3444 256866 : int log = alg->log[opno];
3445 256866 : rtx shift_subtarget = optimize ? 0 : accum;
3446 240494 : rtx add_target
3447 138723 : = (opno == alg->ops - 1 && target != 0 && variant != add_variant
3448 41472 : && !optimize)
3449 256866 : ? target : 0;
3450 256866 : rtx accum_target = optimize ? 0 : accum;
3451 256866 : rtx accum_inner;
3452 :
3453 256866 : switch (alg->op[opno])
3454 : {
3455 112450 : case alg_shift:
3456 112450 : tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
3457 : /* REG_EQUAL note will be attached to the following insn. */
3458 112450 : emit_move_insn (accum, tem);
3459 112450 : val_so_far <<= log;
3460 112450 : break;
3461 :
3462 6698 : case alg_add_t_m2:
3463 6698 : tem = expand_shift (LSHIFT_EXPR, mode, op0, log, NULL_RTX, 0);
3464 13396 : accum = force_operand (gen_rtx_PLUS (mode, accum, tem),
3465 : add_target ? add_target : accum_target);
3466 6698 : val_so_far += HOST_WIDE_INT_1U << log;
3467 6698 : break;
3468 :
3469 20417 : case alg_sub_t_m2:
3470 20417 : tem = expand_shift (LSHIFT_EXPR, mode, op0, log, NULL_RTX, 0);
3471 40834 : accum = force_operand (gen_rtx_MINUS (mode, accum, tem),
3472 : add_target ? add_target : accum_target);
3473 20417 : val_so_far -= HOST_WIDE_INT_1U << log;
3474 20417 : break;
3475 :
3476 116062 : case alg_add_t2_m:
3477 116062 : accum = expand_shift (LSHIFT_EXPR, mode, accum,
3478 116062 : log, shift_subtarget, 0);
3479 232124 : accum = force_operand (gen_rtx_PLUS (mode, accum, op0),
3480 : add_target ? add_target : accum_target);
3481 116062 : val_so_far = (val_so_far << log) + 1;
3482 116062 : break;
3483 :
3484 0 : case alg_sub_t2_m:
3485 0 : accum = expand_shift (LSHIFT_EXPR, mode, accum,
3486 0 : log, shift_subtarget, 0);
3487 0 : accum = force_operand (gen_rtx_MINUS (mode, accum, op0),
3488 : add_target ? add_target : accum_target);
3489 0 : val_so_far = (val_so_far << log) - 1;
3490 0 : break;
3491 :
3492 1104 : case alg_add_factor:
3493 1104 : tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
3494 2208 : accum = force_operand (gen_rtx_PLUS (mode, accum, tem),
3495 : add_target ? add_target : accum_target);
3496 1104 : val_so_far += val_so_far << log;
3497 1104 : break;
3498 :
3499 135 : case alg_sub_factor:
3500 135 : tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
3501 270 : accum = force_operand (gen_rtx_MINUS (mode, tem, accum),
3502 : (add_target
3503 135 : ? add_target : (optimize ? 0 : tem)));
3504 135 : val_so_far = (val_so_far << log) - val_so_far;
3505 135 : break;
3506 :
3507 0 : default:
3508 0 : gcc_unreachable ();
3509 : }
3510 :
3511 256866 : if (SCALAR_INT_MODE_P (mode))
3512 : {
3513 : /* Write a REG_EQUAL note on the last insn so that we can cse
3514 : multiplication sequences. Note that if ACCUM is a SUBREG,
3515 : we've set the inner register and must properly indicate that. */
3516 245159 : tem = op0, nmode = mode;
3517 245159 : accum_inner = accum;
3518 245159 : if (GET_CODE (accum) == SUBREG)
3519 : {
3520 0 : accum_inner = SUBREG_REG (accum);
3521 0 : nmode = GET_MODE (accum_inner);
3522 0 : tem = gen_lowpart (nmode, op0);
3523 : }
3524 :
3525 : /* Don't add a REG_EQUAL note if tem is a paradoxical SUBREG.
3526 : In that case, only the low bits of accum would be guaranteed to
3527 : be equal to the content of the REG_EQUAL note, the upper bits
3528 : can be anything. */
3529 245159 : if (!paradoxical_subreg_p (tem))
3530 : {
3531 245159 : insn = get_last_insn ();
3532 245159 : wide_int wval_so_far
3533 245159 : = wi::uhwi (val_so_far,
3534 245159 : GET_MODE_PRECISION (as_a <scalar_mode> (nmode)));
3535 245159 : rtx c = immed_wide_int_const (wval_so_far, nmode);
3536 245159 : set_dst_reg_note (insn, REG_EQUAL, gen_rtx_MULT (nmode, tem, c),
3537 : accum_inner);
3538 245159 : }
3539 : }
3540 : }
3541 :
3542 138723 : if (variant == negate_variant)
3543 : {
3544 587 : val_so_far = -val_so_far;
3545 587 : accum = expand_unop (mode, neg_optab, accum, target, 0);
3546 : }
3547 138136 : else if (variant == add_variant)
3548 : {
3549 23 : val_so_far = val_so_far + 1;
3550 23 : accum = force_operand (gen_rtx_PLUS (mode, accum, op0), target);
3551 : }
3552 :
3553 : /* Compare only the bits of val and val_so_far that are significant
3554 : in the result mode, to avoid sign-/zero-extension confusion. */
3555 138723 : nmode = GET_MODE_INNER (mode);
3556 138723 : val &= GET_MODE_MASK (nmode);
3557 138723 : val_so_far &= GET_MODE_MASK (nmode);
3558 138723 : gcc_assert (val == (HOST_WIDE_INT) val_so_far);
3559 :
3560 138723 : return accum;
3561 : }
3562 :
3563 : /* Perform a multiplication and return an rtx for the result.
3564 : MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
3565 : TARGET is a suggestion for where to store the result (an rtx).
3566 :
3567 : We check specially for a constant integer as OP1.
3568 : If you want this check for OP0 as well, then before calling
3569 : you should swap the two operands if OP0 would be constant. */
3570 :
3571 : rtx
3572 1118120 : expand_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
3573 : int unsignedp, bool no_libcall)
3574 : {
3575 1118120 : enum mult_variant variant;
3576 1118120 : struct algorithm algorithm;
3577 1118120 : rtx scalar_op1;
3578 1118120 : int max_cost;
3579 1118120 : bool speed = optimize_insn_for_speed_p ();
3580 1118120 : bool do_trapv = flag_trapv && SCALAR_INT_MODE_P (mode) && !unsignedp;
3581 :
3582 1118120 : if (CONSTANT_P (op0))
3583 275 : std::swap (op0, op1);
3584 :
3585 : /* For vectors, there are several simplifications that can be made if
3586 : all elements of the vector constant are identical. */
3587 1118120 : scalar_op1 = unwrap_const_vec_duplicate (op1);
3588 :
3589 1118120 : if (INTEGRAL_MODE_P (mode))
3590 : {
3591 998863 : rtx fake_reg;
3592 998863 : HOST_WIDE_INT coeff;
3593 998863 : bool is_neg;
3594 998863 : int mode_bitsize;
3595 :
3596 998863 : if (op1 == CONST0_RTX (mode))
3597 : return op1;
3598 998863 : if (op1 == CONST1_RTX (mode))
3599 : return op0;
3600 959310 : if (op1 == CONSTM1_RTX (mode))
3601 2830 : return expand_unop (mode, do_trapv ? negv_optab : neg_optab,
3602 1415 : op0, target, 0);
3603 :
3604 957895 : if (do_trapv)
3605 34 : goto skip_synth;
3606 :
3607 : /* If mode is integer vector mode, check if the backend supports
3608 : vector lshift (by scalar or vector) at all. If not, we can't use
3609 : synthesized multiply. */
3610 957861 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
3611 15597 : && optab_handler (vashl_optab, mode) == CODE_FOR_nothing
3612 971069 : && optab_handler (ashl_optab, mode) == CODE_FOR_nothing)
3613 0 : goto skip_synth;
3614 :
3615 : /* These are the operations that are potentially turned into
3616 : a sequence of shifts and additions. */
3617 957861 : mode_bitsize = GET_MODE_UNIT_BITSIZE (mode);
3618 :
3619 : /* synth_mult does an `unsigned int' multiply. As long as the mode is
3620 : less than or equal in size to `unsigned int' this doesn't matter.
3621 : If the mode is larger than `unsigned int', then synth_mult works
3622 : only if the constant value exactly fits in an `unsigned int' without
3623 : any truncation. This means that multiplying by negative values does
3624 : not work; results are off by 2^32 on a 32 bit machine. */
3625 957861 : if (CONST_INT_P (scalar_op1))
3626 : {
3627 699508 : coeff = INTVAL (scalar_op1);
3628 699508 : is_neg = coeff < 0;
3629 : }
3630 : #if TARGET_SUPPORTS_WIDE_INT
3631 258353 : else if (CONST_WIDE_INT_P (scalar_op1))
3632 : #else
3633 : else if (CONST_DOUBLE_AS_INT_P (scalar_op1))
3634 : #endif
3635 : {
3636 1154 : machine_mode scalar_mode = VECTOR_MODE_P (mode)
3637 1154 : ? GET_MODE_INNER (mode) : mode;
3638 1154 : int shift = wi::exact_log2 (rtx_mode_t (scalar_op1, scalar_mode));
3639 : /* Perfect power of 2 (other than 1, which is handled above). */
3640 1154 : if (shift > 0)
3641 106 : return expand_shift (LSHIFT_EXPR, mode, op0,
3642 106 : shift, target, unsignedp);
3643 : else
3644 1048 : goto skip_synth;
3645 : }
3646 : else
3647 257199 : goto skip_synth;
3648 :
3649 : /* We used to test optimize here, on the grounds that it's better to
3650 : produce a smaller program when -O is not used. But this causes
3651 : such a terrible slowdown sometimes that it seems better to always
3652 : use synth_mult. */
3653 :
3654 : /* Special case powers of two. */
3655 699508 : if (EXACT_POWER_OF_2_OR_ZERO_P (coeff)
3656 472632 : && !(is_neg && mode_bitsize > HOST_BITS_PER_WIDE_INT))
3657 472624 : return expand_shift (LSHIFT_EXPR, mode, op0,
3658 945248 : floor_log2 (coeff), target, unsignedp);
3659 :
3660 226884 : fake_reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
3661 :
3662 : /* Attempt to handle multiplication of DImode values by negative
3663 : coefficients, by performing the multiplication by a positive
3664 : multiplier and then inverting the result. */
3665 226884 : if (is_neg && mode_bitsize > HOST_BITS_PER_WIDE_INT)
3666 : {
3667 : /* Its safe to use -coeff even for INT_MIN, as the
3668 : result is interpreted as an unsigned coefficient.
3669 : Exclude cost of op0 from max_cost to match the cost
3670 : calculation of the synth_mult. */
3671 218 : coeff = -(unsigned HOST_WIDE_INT) coeff;
3672 218 : max_cost = (set_src_cost (gen_rtx_MULT (mode, fake_reg, op1),
3673 : mode, speed)
3674 218 : - neg_cost (speed, mode));
3675 218 : if (max_cost <= 0)
3676 0 : goto skip_synth;
3677 :
3678 : /* Special case powers of two. */
3679 218 : if (EXACT_POWER_OF_2_OR_ZERO_P (coeff))
3680 : {
3681 340 : rtx temp = expand_shift (LSHIFT_EXPR, mode, op0,
3682 170 : floor_log2 (coeff), target, unsignedp);
3683 170 : return expand_unop (mode, neg_optab, temp, target, 0);
3684 : }
3685 :
3686 48 : if (choose_mult_variant (mode, coeff, &algorithm, &variant,
3687 : max_cost))
3688 : {
3689 47 : rtx temp = expand_mult_const (mode, op0, coeff, NULL_RTX,
3690 : &algorithm, variant);
3691 47 : return expand_unop (mode, neg_optab, temp, target, 0);
3692 : }
3693 1 : goto skip_synth;
3694 : }
3695 :
3696 : /* Exclude cost of op0 from max_cost to match the cost
3697 : calculation of the synth_mult. */
3698 226666 : max_cost = set_src_cost (gen_rtx_MULT (mode, fake_reg, op1), mode, speed);
3699 226666 : if (choose_mult_variant (mode, coeff, &algorithm, &variant, max_cost))
3700 137553 : return expand_mult_const (mode, op0, coeff, target,
3701 137553 : &algorithm, variant);
3702 : }
3703 89113 : skip_synth:
3704 :
3705 : /* Expand x*2.0 as x+x. */
3706 38320 : if (CONST_DOUBLE_AS_FLOAT_P (scalar_op1)
3707 504972 : && real_equal (CONST_DOUBLE_REAL_VALUE (scalar_op1), &dconst2))
3708 : {
3709 5944 : op0 = force_reg (GET_MODE (op0), op0);
3710 11888 : return expand_binop (mode, add_optab, op0, op0,
3711 : target, unsignedp,
3712 5944 : no_libcall ? OPTAB_WIDEN : OPTAB_LIB_WIDEN);
3713 : }
3714 :
3715 : /* This used to use umul_optab if unsigned, but for non-widening multiply
3716 : there is no difference between signed and unsigned. */
3717 1382090 : op0 = expand_binop (mode, do_trapv ? smulv_optab : smul_optab,
3718 : op0, op1, target, unsignedp,
3719 : no_libcall ? OPTAB_WIDEN : OPTAB_LIB_WIDEN);
3720 460708 : gcc_assert (op0 || no_libcall);
3721 : return op0;
3722 : }
3723 :
3724 : /* Return a cost estimate for multiplying a register by the given
3725 : COEFFicient in the given MODE and SPEED. */
3726 :
3727 : int
3728 6751041 : mult_by_coeff_cost (HOST_WIDE_INT coeff, machine_mode mode, bool speed)
3729 : {
3730 6751041 : int max_cost;
3731 6751041 : struct algorithm algorithm;
3732 6751041 : enum mult_variant variant;
3733 :
3734 6751041 : rtx fake_reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
3735 6751041 : max_cost = set_src_cost (gen_rtx_MULT (mode, fake_reg, fake_reg),
3736 : mode, speed);
3737 6751041 : if (choose_mult_variant (mode, coeff, &algorithm, &variant, max_cost))
3738 5989349 : return algorithm.cost.cost;
3739 : else
3740 : return max_cost;
3741 : }
3742 :
3743 : /* Perform a widening multiplication and return an rtx for the result.
3744 : MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
3745 : TARGET is a suggestion for where to store the result (an rtx).
3746 : THIS_OPTAB is the optab we should use, it must be either umul_widen_optab
3747 : or smul_widen_optab.
3748 :
3749 : We check specially for a constant integer as OP1, comparing the
3750 : cost of a widening multiply against the cost of a sequence of shifts
3751 : and adds. */
3752 :
3753 : rtx
3754 24107 : expand_widening_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
3755 : int unsignedp, optab this_optab)
3756 : {
3757 24107 : bool speed = optimize_insn_for_speed_p ();
3758 24107 : rtx cop1;
3759 :
3760 24107 : if (CONST_INT_P (op1)
3761 5473 : && GET_MODE (op0) != VOIDmode
3762 5473 : && (cop1 = convert_modes (mode, GET_MODE (op0), op1,
3763 : this_optab == umul_widen_optab))
3764 5473 : && CONST_INT_P (cop1)
3765 29052 : && (INTVAL (cop1) >= 0
3766 26756 : || HWI_COMPUTABLE_MODE_P (mode)))
3767 : {
3768 4706 : HOST_WIDE_INT coeff = INTVAL (cop1);
3769 4706 : int max_cost;
3770 4706 : enum mult_variant variant;
3771 4706 : struct algorithm algorithm;
3772 :
3773 4706 : if (coeff == 0)
3774 2057 : return CONST0_RTX (mode);
3775 :
3776 : /* Special case powers of two. */
3777 4330 : if (EXACT_POWER_OF_2_OR_ZERO_P (coeff))
3778 : {
3779 570 : op0 = convert_to_mode (mode, op0, this_optab == umul_widen_optab);
3780 570 : return expand_shift (LSHIFT_EXPR, mode, op0,
3781 570 : floor_log2 (coeff), target, unsignedp);
3782 : }
3783 :
3784 : /* Exclude cost of op0 from max_cost to match the cost
3785 : calculation of the synth_mult. */
3786 3760 : max_cost = mul_widen_cost (speed, mode);
3787 3760 : if (choose_mult_variant (mode, coeff, &algorithm, &variant,
3788 : max_cost))
3789 : {
3790 1111 : op0 = convert_to_mode (mode, op0, this_optab == umul_widen_optab);
3791 1111 : return expand_mult_const (mode, op0, coeff, target,
3792 1111 : &algorithm, variant);
3793 : }
3794 : }
3795 22050 : return expand_binop (mode, this_optab, op0, op1, target,
3796 22050 : unsignedp, OPTAB_LIB_WIDEN);
3797 : }
3798 :
3799 : /* Choose a minimal N + 1 bit approximation to 2**K / D that can be used to
3800 : replace division by D, put the least significant N bits of the result in
3801 : *MULTIPLIER_PTR, the value K - N in *POST_SHIFT_PTR, and return the most
3802 : significant bit.
3803 :
3804 : The width of operations is N (should be <= HOST_BITS_PER_WIDE_INT), the
3805 : needed precision is PRECISION (should be <= N).
3806 :
3807 : PRECISION should be as small as possible so this function can choose the
3808 : multiplier more freely. If PRECISION is <= N - 1, the most significant
3809 : bit returned by the function will be zero.
3810 :
3811 : Using this function, x / D is equal to (x*m) / 2**N >> (*POST_SHIFT_PTR),
3812 : where m is the full N + 1 bit multiplier. */
3813 :
3814 : unsigned HOST_WIDE_INT
3815 73158 : choose_multiplier (unsigned HOST_WIDE_INT d, int n, int precision,
3816 : unsigned HOST_WIDE_INT *multiplier_ptr,
3817 : int *post_shift_ptr)
3818 : {
3819 73158 : int lgup, post_shift;
3820 73158 : int pow1, pow2;
3821 :
3822 : /* lgup = ceil(log2(d)) */
3823 : /* Assuming d > 1, we have d >= 2^(lgup-1) + 1 */
3824 73158 : lgup = ceil_log2 (d);
3825 :
3826 73158 : gcc_assert (lgup <= n);
3827 73158 : gcc_assert (lgup <= precision);
3828 :
3829 73158 : pow1 = n + lgup;
3830 73158 : pow2 = n + lgup - precision;
3831 :
3832 : /* mlow = 2^(n + lgup)/d */
3833 : /* Trivially from above we have mlow < 2^(n+1) */
3834 73158 : wide_int val = wi::set_bit_in_zero (pow1, HOST_BITS_PER_DOUBLE_INT);
3835 73158 : wide_int mlow = wi::udiv_trunc (val, d);
3836 :
3837 : /* mhigh = (2^(n + lgup) + 2^(n + lgup - precision))/d */
3838 : /* From above we have mhigh < 2^(n+1) assuming lgup <= precision */
3839 : /* From precision <= n, the difference between the numerators of mhigh and
3840 : mlow is >= 2^lgup >= d. Therefore the difference of the quotients in
3841 : the Euclidean division by d is at least 1, so we have mlow < mhigh and
3842 : the exact value of 2^(n + lgup)/d lies in the interval [mlow; mhigh). */
3843 73158 : val |= wi::set_bit_in_zero (pow2, HOST_BITS_PER_DOUBLE_INT);
3844 73158 : wide_int mhigh = wi::udiv_trunc (val, d);
3845 :
3846 : /* Reduce to lowest terms. */
3847 : /* If precision <= n - 1, then the difference between the numerators of
3848 : mhigh and mlow is >= 2^(lgup + 1) >= 2 * 2^lgup >= 2 * d. Therefore
3849 : the difference of the quotients in the Euclidean division by d is at
3850 : least 2, which means that mhigh and mlow differ by at least one bit
3851 : not in the last place. The conclusion is that the first iteration of
3852 : the loop below completes and shifts mhigh and mlow by 1 bit, which in
3853 : particular means that mhigh < 2^n, that is to say, the most significant
3854 : bit in the n + 1 bit value is zero. */
3855 261379 : for (post_shift = lgup; post_shift > 0; post_shift--)
3856 : {
3857 182590 : unsigned HOST_WIDE_INT ml_lo = wi::extract_uhwi (mlow, 1,
3858 : HOST_BITS_PER_WIDE_INT);
3859 182590 : unsigned HOST_WIDE_INT mh_lo = wi::extract_uhwi (mhigh, 1,
3860 : HOST_BITS_PER_WIDE_INT);
3861 182590 : if (ml_lo >= mh_lo)
3862 : break;
3863 :
3864 115063 : mlow = wi::uhwi (ml_lo, HOST_BITS_PER_DOUBLE_INT);
3865 115063 : mhigh = wi::uhwi (mh_lo, HOST_BITS_PER_DOUBLE_INT);
3866 : }
3867 :
3868 73158 : *post_shift_ptr = post_shift;
3869 :
3870 73158 : if (n < HOST_BITS_PER_WIDE_INT)
3871 : {
3872 41300 : unsigned HOST_WIDE_INT mask = (HOST_WIDE_INT_1U << n) - 1;
3873 41300 : *multiplier_ptr = mhigh.to_uhwi () & mask;
3874 41300 : return mhigh.to_uhwi () > mask;
3875 : }
3876 : else
3877 : {
3878 31858 : *multiplier_ptr = mhigh.to_uhwi ();
3879 31858 : return wi::extract_uhwi (mhigh, HOST_BITS_PER_WIDE_INT, 1);
3880 : }
3881 73158 : }
3882 :
3883 : /* Compute the inverse of X mod 2**N, i.e., find Y such that X * Y is congruent
3884 : to 1 modulo 2**N, assuming that X is odd. Bézout's lemma guarantees that Y
3885 : exists for any given positive N. */
3886 :
3887 : static unsigned HOST_WIDE_INT
3888 46965 : invert_mod2n (unsigned HOST_WIDE_INT x, int n)
3889 : {
3890 46965 : gcc_assert ((x & 1) == 1);
3891 :
3892 : /* The algorithm notes that the choice Y = X satisfies X*Y == 1 mod 2^3,
3893 : since X is odd. Then each Newton-Raphson iteration doubles the number
3894 : of bits of significance in Y (Hensel's lemma). */
3895 :
3896 47987 : const unsigned HOST_WIDE_INT mask
3897 : = (n == HOST_BITS_PER_WIDE_INT
3898 46965 : ? HOST_WIDE_INT_M1U
3899 1022 : : (HOST_WIDE_INT_1U << n) - 1);
3900 46965 : unsigned HOST_WIDE_INT y = x;
3901 46965 : int nbit = 3;
3902 :
3903 280738 : while (nbit < n)
3904 : {
3905 233773 : y = y * (2 - x*y) & mask; /* Modulo 2^N */
3906 233773 : nbit *= 2;
3907 : }
3908 :
3909 46965 : return y;
3910 : }
3911 :
3912 : /* Emit code to adjust ADJ_OPERAND after multiplication of wrong signedness
3913 : flavor of OP0 and OP1. ADJ_OPERAND is already the high half of the
3914 : product OP0 x OP1. If UNSIGNEDP is nonzero, adjust the signed product
3915 : to become unsigned, if UNSIGNEDP is zero, adjust the unsigned product to
3916 : become signed.
3917 :
3918 : The result is put in TARGET if that is convenient.
3919 :
3920 : MODE is the mode of operation. */
3921 :
3922 : rtx
3923 0 : expand_mult_highpart_adjust (scalar_int_mode mode, rtx adj_operand, rtx op0,
3924 : rtx op1, rtx target, int unsignedp)
3925 : {
3926 0 : rtx tem;
3927 0 : enum rtx_code adj_code = unsignedp ? PLUS : MINUS;
3928 :
3929 0 : tem = expand_shift (RSHIFT_EXPR, mode, op0,
3930 0 : GET_MODE_BITSIZE (mode) - 1, NULL_RTX, 0);
3931 0 : tem = expand_and (mode, tem, op1, NULL_RTX);
3932 0 : adj_operand
3933 0 : = force_operand (gen_rtx_fmt_ee (adj_code, mode, adj_operand, tem),
3934 : adj_operand);
3935 :
3936 0 : tem = expand_shift (RSHIFT_EXPR, mode, op1,
3937 0 : GET_MODE_BITSIZE (mode) - 1, NULL_RTX, 0);
3938 0 : tem = expand_and (mode, tem, op0, NULL_RTX);
3939 0 : target = force_operand (gen_rtx_fmt_ee (adj_code, mode, adj_operand, tem),
3940 : target);
3941 :
3942 0 : return target;
3943 : }
3944 :
3945 : /* Subroutine of expmed_mult_highpart. Return the MODE high part of OP. */
3946 :
3947 : static rtx
3948 19523 : extract_high_half (scalar_int_mode mode, rtx op)
3949 : {
3950 19523 : if (mode == word_mode)
3951 0 : return gen_highpart (mode, op);
3952 :
3953 19523 : scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
3954 :
3955 39046 : op = expand_shift (RSHIFT_EXPR, wider_mode, op,
3956 19523 : GET_MODE_BITSIZE (mode), 0, 1);
3957 19523 : return convert_modes (mode, wider_mode, op, 0);
3958 : }
3959 :
3960 : /* Like expmed_mult_highpart, but only consider using multiplication optab. */
3961 :
3962 : rtx
3963 44893 : expmed_mult_highpart_optab (scalar_int_mode mode, rtx op0, rtx op1,
3964 : rtx target, int unsignedp, int max_cost)
3965 : {
3966 44893 : const scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
3967 44893 : const bool speed = optimize_insn_for_speed_p ();
3968 44893 : const int size = GET_MODE_BITSIZE (mode);
3969 44893 : optab moptab;
3970 44893 : rtx tem;
3971 :
3972 : /* Firstly, try using a multiplication insn that only generates the needed
3973 : high part of the product, and in the sign flavor of unsignedp. */
3974 44893 : if (mul_highpart_cost (speed, mode) < max_cost)
3975 : {
3976 42883 : moptab = unsignedp ? umul_highpart_optab : smul_highpart_optab;
3977 42883 : tem = expand_binop (mode, moptab, op0, op1, target, unsignedp,
3978 : OPTAB_DIRECT);
3979 42883 : if (tem)
3980 : return tem;
3981 : }
3982 :
3983 : /* Secondly, same as above, but use sign flavor opposite of unsignedp.
3984 : Need to adjust the result after the multiplication. */
3985 21699 : if (size - 1 < BITS_PER_WORD
3986 43142 : && (mul_highpart_cost (speed, mode)
3987 21443 : + 2 * shift_cost (speed, mode, size-1)
3988 21443 : + 4 * add_cost (speed, mode) < max_cost))
3989 : {
3990 5179 : moptab = unsignedp ? smul_highpart_optab : umul_highpart_optab;
3991 5179 : tem = expand_binop (mode, moptab, op0, op1, target, !unsignedp,
3992 : OPTAB_DIRECT);
3993 5179 : if (tem)
3994 : /* We used the wrong signedness. Adjust the result. */
3995 0 : return expand_mult_highpart_adjust (mode, tem, op0, op1, tem,
3996 0 : unsignedp);
3997 : }
3998 :
3999 : /* Try widening multiplication. */
4000 21699 : moptab = unsignedp ? umul_widen_optab : smul_widen_optab;
4001 21699 : if (convert_optab_handler (moptab, wider_mode, mode) != CODE_FOR_nothing
4002 21699 : && mul_widen_cost (speed, wider_mode) < max_cost)
4003 : {
4004 388 : tem = expand_binop (wider_mode, moptab, op0, op1, NULL_RTX, unsignedp,
4005 : OPTAB_WIDEN);
4006 388 : if (tem)
4007 388 : return extract_high_half (mode, tem);
4008 : }
4009 :
4010 : /* Try widening the mode and perform a non-widening multiplication. */
4011 21311 : if (optab_handler (smul_optab, wider_mode) != CODE_FOR_nothing
4012 20706 : && size - 1 < BITS_PER_WORD
4013 42014 : && (mul_cost (speed, wider_mode) + shift_cost (speed, mode, size-1)
4014 : < max_cost))
4015 : {
4016 19123 : rtx_insn *insns;
4017 19123 : rtx wop0, wop1;
4018 :
4019 : /* We need to widen the operands, for example to ensure the
4020 : constant multiplier is correctly sign or zero extended.
4021 : Use a sequence to clean-up any instructions emitted by
4022 : the conversions if things don't work out. */
4023 19123 : start_sequence ();
4024 19123 : wop0 = convert_modes (wider_mode, mode, op0, unsignedp);
4025 19123 : wop1 = convert_modes (wider_mode, mode, op1, unsignedp);
4026 19123 : tem = expand_binop (wider_mode, smul_optab, wop0, wop1, 0,
4027 : unsignedp, OPTAB_WIDEN);
4028 19123 : insns = end_sequence ();
4029 :
4030 19123 : if (tem)
4031 : {
4032 19123 : emit_insn (insns);
4033 19123 : return extract_high_half (mode, tem);
4034 : }
4035 : }
4036 :
4037 : /* Try widening multiplication of opposite signedness, and adjust. */
4038 2188 : moptab = unsignedp ? smul_widen_optab : umul_widen_optab;
4039 2188 : if (convert_optab_handler (moptab, wider_mode, mode) != CODE_FOR_nothing
4040 456 : && size - 1 < BITS_PER_WORD
4041 2982 : && (mul_widen_cost (speed, wider_mode)
4042 397 : + 2 * shift_cost (speed, mode, size-1)
4043 397 : + 4 * add_cost (speed, mode) < max_cost))
4044 : {
4045 0 : tem = expand_binop (wider_mode, moptab, op0, op1, NULL_RTX, !unsignedp,
4046 : OPTAB_WIDEN);
4047 0 : if (tem != 0)
4048 : {
4049 0 : tem = extract_high_half (mode, tem);
4050 : /* We used the wrong signedness. Adjust the result. */
4051 0 : return expand_mult_highpart_adjust (mode, tem, op0, op1, target,
4052 0 : unsignedp);
4053 : }
4054 : }
4055 :
4056 : return 0;
4057 : }
4058 :
4059 : /* Emit code to multiply OP0 and OP1 (where OP1 is an integer constant),
4060 : putting the high half of the result in TARGET if that is convenient,
4061 : and return where the result is. If the operation cannot be performed,
4062 : 0 is returned.
4063 :
4064 : MODE is the mode of operation and result.
4065 :
4066 : UNSIGNEDP nonzero means unsigned multiply.
4067 :
4068 : MAX_COST is the total allowed cost for the expanded RTL. */
4069 :
4070 : static rtx
4071 44893 : expmed_mult_highpart (scalar_int_mode mode, rtx op0, rtx op1,
4072 : rtx target, int unsignedp, int max_cost)
4073 : {
4074 44893 : const bool speed = optimize_insn_for_speed_p ();
4075 44893 : unsigned HOST_WIDE_INT cnst1;
4076 44893 : int extra_cost;
4077 44893 : bool sign_adjust = false;
4078 44893 : enum mult_variant variant;
4079 44893 : struct algorithm alg;
4080 44893 : rtx narrow_op1, tem;
4081 :
4082 : /* We can't support modes wider than HOST_BITS_PER_INT. */
4083 44893 : gcc_assert (HWI_COMPUTABLE_MODE_P (mode));
4084 :
4085 44893 : cnst1 = INTVAL (op1) & GET_MODE_MASK (mode);
4086 44893 : narrow_op1 = gen_int_mode (INTVAL (op1), mode);
4087 :
4088 : /* We can't optimize modes wider than BITS_PER_WORD.
4089 : ??? We might be able to perform double-word arithmetic if
4090 : mode == word_mode, however all the cost calculations in
4091 : synth_mult etc. assume single-word operations. */
4092 44893 : scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
4093 92903 : if (GET_MODE_BITSIZE (wider_mode) > BITS_PER_WORD)
4094 23802 : return expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
4095 23802 : unsignedp, max_cost);
4096 :
4097 42182 : extra_cost = shift_cost (speed, mode, GET_MODE_BITSIZE (mode) - 1);
4098 :
4099 : /* Check whether we try to multiply by a negative constant. */
4100 31268 : if (!unsignedp && ((cnst1 >> (GET_MODE_BITSIZE (mode) - 1)) & 1))
4101 : {
4102 2288 : sign_adjust = true;
4103 2288 : extra_cost += add_cost (speed, mode);
4104 : }
4105 :
4106 : /* See whether shift/add multiplication is cheap enough. */
4107 21091 : if (choose_mult_variant (wider_mode, cnst1, &alg, &variant,
4108 : max_cost - extra_cost))
4109 : {
4110 : /* See whether the specialized multiplication optabs are
4111 : cheaper than the shift/add version. */
4112 38902 : tem = expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
4113 : unsignedp,
4114 19451 : alg.cost.cost + extra_cost);
4115 19451 : if (tem)
4116 : return tem;
4117 :
4118 12 : tem = convert_to_mode (wider_mode, op0, unsignedp);
4119 12 : tem = expand_mult_const (wider_mode, tem, cnst1, 0, &alg, variant);
4120 12 : tem = extract_high_half (mode, tem);
4121 :
4122 : /* Adjust result for signedness. */
4123 12 : if (sign_adjust)
4124 0 : tem = force_operand (gen_rtx_MINUS (mode, tem, op0), tem);
4125 :
4126 : return tem;
4127 : }
4128 1640 : return expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
4129 1640 : unsignedp, max_cost);
4130 : }
4131 :
4132 :
4133 : /* Expand signed modulus of OP0 by a power of two D in mode MODE. */
4134 :
4135 : static rtx
4136 2462 : expand_smod_pow2 (scalar_int_mode mode, rtx op0, HOST_WIDE_INT d)
4137 : {
4138 2462 : rtx result, temp, shift;
4139 2462 : rtx_code_label *label;
4140 2462 : int logd;
4141 2462 : int prec = GET_MODE_PRECISION (mode);
4142 :
4143 2462 : logd = floor_log2 (d);
4144 2462 : result = gen_reg_rtx (mode);
4145 :
4146 : /* Avoid conditional branches when they're expensive. */
4147 2462 : if (BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2
4148 2462 : && optimize_insn_for_speed_p ())
4149 : {
4150 2461 : rtx signmask = emit_store_flag (result, LT, op0, const0_rtx,
4151 : mode, 0, -1);
4152 2461 : if (signmask)
4153 : {
4154 2461 : HOST_WIDE_INT masklow = (HOST_WIDE_INT_1 << logd) - 1;
4155 2461 : signmask = force_reg (mode, signmask);
4156 4922 : shift = gen_int_shift_amount (mode, GET_MODE_BITSIZE (mode) - logd);
4157 :
4158 : /* Use the rtx_cost of a LSHIFTRT instruction to determine
4159 : which instruction sequence to use. If logical right shifts
4160 : are expensive the use 2 XORs, 2 SUBs and an AND, otherwise
4161 : use a LSHIFTRT, 1 ADD, 1 SUB and an AND. */
4162 :
4163 2461 : temp = gen_rtx_LSHIFTRT (mode, result, shift);
4164 2461 : if (optab_handler (lshr_optab, mode) == CODE_FOR_nothing
4165 2461 : || (set_src_cost (temp, mode, optimize_insn_for_speed_p ())
4166 : > COSTS_N_INSNS (2)))
4167 : {
4168 109 : temp = expand_binop (mode, xor_optab, op0, signmask,
4169 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4170 109 : temp = expand_binop (mode, sub_optab, temp, signmask,
4171 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4172 109 : temp = expand_binop (mode, and_optab, temp,
4173 109 : gen_int_mode (masklow, mode),
4174 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4175 109 : temp = expand_binop (mode, xor_optab, temp, signmask,
4176 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4177 109 : temp = expand_binop (mode, sub_optab, temp, signmask,
4178 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4179 : }
4180 : else
4181 : {
4182 2352 : signmask = expand_binop (mode, lshr_optab, signmask, shift,
4183 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4184 2352 : signmask = force_reg (mode, signmask);
4185 :
4186 2352 : temp = expand_binop (mode, add_optab, op0, signmask,
4187 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4188 2352 : temp = expand_binop (mode, and_optab, temp,
4189 2352 : gen_int_mode (masklow, mode),
4190 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4191 2352 : temp = expand_binop (mode, sub_optab, temp, signmask,
4192 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4193 : }
4194 : return temp;
4195 : }
4196 : }
4197 :
4198 : /* Mask contains the mode's signbit and the significant bits of the
4199 : modulus. By including the signbit in the operation, many targets
4200 : can avoid an explicit compare operation in the following comparison
4201 : against zero. */
4202 1 : wide_int mask = wi::mask (logd, false, prec);
4203 1 : mask = wi::set_bit (mask, prec - 1);
4204 :
4205 2 : temp = expand_binop (mode, and_optab, op0,
4206 1 : immed_wide_int_const (mask, mode),
4207 : result, 1, OPTAB_LIB_WIDEN);
4208 1 : if (temp != result)
4209 0 : emit_move_insn (result, temp);
4210 :
4211 1 : label = gen_label_rtx ();
4212 1 : do_cmp_and_jump (result, const0_rtx, GE, mode, label);
4213 :
4214 1 : temp = expand_binop (mode, sub_optab, result, const1_rtx, result,
4215 : 0, OPTAB_LIB_WIDEN);
4216 :
4217 1 : mask = wi::mask (logd, true, prec);
4218 2 : temp = expand_binop (mode, ior_optab, temp,
4219 1 : immed_wide_int_const (mask, mode),
4220 : result, 1, OPTAB_LIB_WIDEN);
4221 1 : temp = expand_binop (mode, add_optab, temp, const1_rtx, result,
4222 : 0, OPTAB_LIB_WIDEN);
4223 1 : if (temp != result)
4224 0 : emit_move_insn (result, temp);
4225 1 : emit_label (label);
4226 1 : return result;
4227 1 : }
4228 :
4229 : /* Expand signed division of OP0 by a power of two D in mode MODE.
4230 : This routine is only called for positive values of D. */
4231 :
4232 : static rtx
4233 10521 : expand_sdiv_pow2 (scalar_int_mode mode, rtx op0, HOST_WIDE_INT d)
4234 : {
4235 10521 : rtx temp;
4236 10521 : rtx_code_label *label;
4237 10521 : int logd;
4238 :
4239 10521 : logd = floor_log2 (d);
4240 :
4241 10521 : if (d == 2
4242 10521 : && BRANCH_COST (optimize_insn_for_speed_p (),
4243 : false) >= 1)
4244 : {
4245 6921 : temp = gen_reg_rtx (mode);
4246 6921 : temp = emit_store_flag (temp, LT, op0, const0_rtx, mode, 0, 1);
4247 6921 : if (temp != NULL_RTX)
4248 : {
4249 6921 : temp = expand_binop (mode, add_optab, temp, op0, NULL_RTX,
4250 : 0, OPTAB_LIB_WIDEN);
4251 6921 : return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
4252 : }
4253 : }
4254 :
4255 7199 : if (HAVE_conditional_move
4256 3600 : && BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2)
4257 : {
4258 3600 : rtx temp2;
4259 :
4260 3600 : start_sequence ();
4261 3600 : temp2 = copy_to_mode_reg (mode, op0);
4262 3600 : temp = expand_binop (mode, add_optab, temp2, gen_int_mode (d - 1, mode),
4263 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
4264 3600 : temp = force_reg (mode, temp);
4265 :
4266 : /* Construct "temp2 = (temp2 < 0) ? temp : temp2". */
4267 3600 : temp2 = emit_conditional_move (temp2, { LT, temp2, const0_rtx, mode },
4268 : temp, temp2, mode, 0);
4269 3600 : if (temp2)
4270 : {
4271 3535 : rtx_insn *seq = end_sequence ();
4272 3535 : emit_insn (seq);
4273 3535 : return expand_shift (RSHIFT_EXPR, mode, temp2, logd, NULL_RTX, 0);
4274 : }
4275 65 : end_sequence ();
4276 : }
4277 :
4278 65 : if (BRANCH_COST (optimize_insn_for_speed_p (),
4279 : false) >= 2)
4280 : {
4281 65 : int ushift = GET_MODE_BITSIZE (mode) - logd;
4282 :
4283 65 : temp = gen_reg_rtx (mode);
4284 65 : temp = emit_store_flag (temp, LT, op0, const0_rtx, mode, 0, -1);
4285 65 : if (temp != NULL_RTX)
4286 : {
4287 130 : if (GET_MODE_BITSIZE (mode) >= BITS_PER_WORD
4288 65 : || shift_cost (optimize_insn_for_speed_p (), mode, ushift)
4289 : > COSTS_N_INSNS (1))
4290 65 : temp = expand_binop (mode, and_optab, temp,
4291 65 : gen_int_mode (d - 1, mode),
4292 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
4293 : else
4294 0 : temp = expand_shift (RSHIFT_EXPR, mode, temp,
4295 0 : ushift, NULL_RTX, 1);
4296 65 : temp = expand_binop (mode, add_optab, temp, op0, NULL_RTX,
4297 : 0, OPTAB_LIB_WIDEN);
4298 65 : return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
4299 : }
4300 : }
4301 :
4302 0 : label = gen_label_rtx ();
4303 0 : temp = copy_to_mode_reg (mode, op0);
4304 0 : do_cmp_and_jump (temp, const0_rtx, GE, mode, label);
4305 0 : expand_inc (temp, gen_int_mode (d - 1, mode));
4306 0 : emit_label (label);
4307 0 : return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
4308 : }
4309 :
4310 : /* Emit the code to divide OP0 by OP1, putting the result in TARGET
4311 : if that is convenient, and returning where the result is.
4312 : You may request either the quotient or the remainder as the result;
4313 : specify REM_FLAG nonzero to get the remainder.
4314 :
4315 : CODE is the expression code for which kind of division this is;
4316 : it controls how rounding is done. MODE is the machine mode to use.
4317 : UNSIGNEDP nonzero means do unsigned division. */
4318 :
4319 : /* ??? For CEIL_MOD_EXPR, can compute incorrect remainder with ANDI
4320 : and then correct it by or'ing in missing high bits
4321 : if result of ANDI is nonzero.
4322 : For ROUND_MOD_EXPR, can use ANDI and then sign-extend the result.
4323 : This could optimize to a bfexts instruction.
4324 : But C doesn't use these operations, so their optimizations are
4325 : left for later. */
4326 : /* ??? For modulo, we don't actually need the highpart of the first product,
4327 : the low part will do nicely. And for small divisors, the second multiply
4328 : can also be a low-part only multiply or even be completely left out.
4329 : E.g. to calculate the remainder of a division by 3 with a 32 bit
4330 : multiply, multiply with 0x55555556 and extract the upper two bits;
4331 : the result is exact for inputs up to 0x1fffffff.
4332 : The input range can be reduced by using cross-sum rules.
4333 : For odd divisors >= 3, the following table gives right shift counts
4334 : so that if a number is shifted by an integer multiple of the given
4335 : amount, the remainder stays the same:
4336 : 2, 4, 3, 6, 10, 12, 4, 8, 18, 6, 11, 20, 18, 0, 5, 10, 12, 0, 12, 20,
4337 : 14, 12, 23, 21, 8, 0, 20, 18, 0, 0, 6, 12, 0, 22, 0, 18, 20, 30, 0, 0,
4338 : 0, 8, 0, 11, 12, 10, 36, 0, 30, 0, 0, 12, 0, 0, 0, 0, 44, 12, 24, 0,
4339 : 20, 0, 7, 14, 0, 18, 36, 0, 0, 46, 60, 0, 42, 0, 15, 24, 20, 0, 0, 33,
4340 : 0, 20, 0, 0, 18, 0, 60, 0, 0, 0, 0, 0, 40, 18, 0, 0, 12
4341 :
4342 : Cross-sum rules for even numbers can be derived by leaving as many bits
4343 : to the right alone as the divisor has zeros to the right.
4344 : E.g. if x is an unsigned 32 bit number:
4345 : (x mod 12) == (((x & 1023) + ((x >> 8) & ~3)) * 0x15555558 >> 2 * 3) >> 28
4346 : */
4347 :
4348 : /* Helper for expand_divmod's unsigned constant division. For OP0 in
4349 : INT_MODE divided by a constant needing a (SIZE+1)-bit multiplier ML
4350 : with right shift POST_SHIFT (the mh != 0 case), try to obtain
4351 : the quotient from the high part of a single multiply in a mode twice
4352 : as wide as INT_MODE. Return the quotient in INT_MODE, having emitted
4353 : the insns, or NULL_RTX when the transformation is unavailable or not
4354 : cheaper than the classic sub/shift/add sequence. EXTRA_COST is the
4355 : cost of that sequence's follow-up ops, MAX_COST bounds the multiply
4356 : and SPEED selects the cost model.
4357 :
4358 : The magic constant occupies at most 2*SIZE bits and so must fit in a
4359 : HOST_WIDE_INT (always 64 bits today; checked below). A wider INT_MODE
4360 : such as DImode -- which would need a 128-bit magic and a single-word
4361 : high-part multiply in a 2x-wide mode that common targets lack -- is
4362 : therefore excluded. */
4363 :
4364 : static rtx
4365 2301 : expand_wide_mulh_udiv (scalar_int_mode int_mode, rtx op0,
4366 : unsigned HOST_WIDE_INT ml, int size, int post_shift,
4367 : int extra_cost, int max_cost, bool speed)
4368 : {
4369 2301 : scalar_int_mode wide_mode;
4370 :
4371 : /* We need POST_SHIFT >= 1, a wider integer mode that still fits in a
4372 : word, and the pre-shifted magic constant to fit in a HOST_WIDE_INT. */
4373 2301 : if (post_shift < 1
4374 2301 : || !GET_MODE_2XWIDER_MODE (int_mode).exists (&wide_mode)
4375 2394 : || GET_MODE_BITSIZE (wide_mode) > BITS_PER_WORD
4376 3335 : || GET_MODE_BITSIZE (wide_mode) > HOST_BITS_PER_WIDE_INT)
4377 : return NULL_RTX;
4378 :
4379 : /* The caller obtained ML and POST_SHIFT from choose_multiplier, which
4380 : guarantees POST_SHIFT <= ceil (log2 (d)) <= SIZE for a SIZE-bit
4381 : divisor d, so the shift count below is non-negative. */
4382 1034 : gcc_checking_assert (post_shift <= size);
4383 :
4384 : /* Pre-shift the (SIZE+1)-bit magic constant (2^SIZE + ML) by
4385 : (SIZE - POST_SHIFT) so that the quotient ends up in the high part
4386 : of the widened product. Since ML < 2^SIZE and POST_SHIFT >= 1, the
4387 : result is below 2^(2*SIZE) and thus fits in both WIDE_MODE and an
4388 : unsigned HOST_WIDE_INT (2*SIZE <= HOST_BITS_PER_WIDE_INT was
4389 : checked above). */
4390 1034 : unsigned HOST_WIDE_INT magic
4391 1034 : = ((HOST_WIDE_INT_1U << size) + ml) << (size - post_shift);
4392 :
4393 1034 : start_sequence ();
4394 1034 : rtx x_wide = convert_to_mode (wide_mode, op0, 1);
4395 1034 : rtx hi = expmed_mult_highpart (wide_mode, x_wide,
4396 1034 : gen_int_mode (magic, wide_mode),
4397 : NULL_RTX, 1, max_cost);
4398 1034 : rtx quotient = hi ? convert_to_mode (int_mode, hi, 1) : NULL_RTX;
4399 1034 : rtx_insn *insns = end_sequence ();
4400 :
4401 : /* Use the widened multiply only when it is no more expensive than
4402 : the classic sub/shift/add sequence. */
4403 1034 : unsigned classic_cost = mul_highpart_cost (speed, int_mode) + extra_cost;
4404 1034 : if (quotient == NULL_RTX || seq_cost (insns, speed) > classic_cost)
4405 : return NULL_RTX;
4406 :
4407 957 : emit_insn (insns);
4408 957 : return quotient;
4409 : }
4410 :
4411 : rtx
4412 239939 : expand_divmod (int rem_flag, enum tree_code code, machine_mode mode,
4413 : rtx op0, rtx op1, rtx target, int unsignedp,
4414 : enum optab_methods methods)
4415 : {
4416 239939 : machine_mode compute_mode;
4417 239939 : rtx tquotient;
4418 239939 : rtx quotient = 0, remainder = 0;
4419 239939 : rtx_insn *last;
4420 239939 : rtx_insn *insn;
4421 239939 : optab optab1, optab2;
4422 239939 : int op1_is_constant, op1_is_pow2 = 0;
4423 239939 : int max_cost, extra_cost;
4424 239939 : bool speed = optimize_insn_for_speed_p ();
4425 :
4426 239939 : op1_is_constant = CONST_INT_P (op1);
4427 239939 : if (op1_is_constant)
4428 : {
4429 137409 : wide_int ext_op1 = rtx_mode_t (op1, mode);
4430 167198 : op1_is_pow2 = (wi::popcount (ext_op1) == 1
4431 274818 : || (! unsignedp
4432 167198 : && wi::popcount (wi::neg (ext_op1)) == 1));
4433 137409 : }
4434 :
4435 : /*
4436 : This is the structure of expand_divmod:
4437 :
4438 : First comes code to fix up the operands so we can perform the operations
4439 : correctly and efficiently.
4440 :
4441 : Second comes a switch statement with code specific for each rounding mode.
4442 : For some special operands this code emits all RTL for the desired
4443 : operation, for other cases, it generates only a quotient and stores it in
4444 : QUOTIENT. The case for trunc division/remainder might leave quotient = 0,
4445 : to indicate that it has not done anything.
4446 :
4447 : Last comes code that finishes the operation. If QUOTIENT is set and
4448 : REM_FLAG is set, the remainder is computed as OP0 - QUOTIENT * OP1. If
4449 : QUOTIENT is not set, it is computed using trunc rounding.
4450 :
4451 : We try to generate special code for division and remainder when OP1 is a
4452 : constant. If |OP1| = 2**n we can use shifts and some other fast
4453 : operations. For other values of OP1, we compute a carefully selected
4454 : fixed-point approximation m = 1/OP1, and generate code that multiplies OP0
4455 : by m.
4456 :
4457 : In all cases but EXACT_DIV_EXPR, this multiplication requires the upper
4458 : half of the product. Different strategies for generating the product are
4459 : implemented in expmed_mult_highpart.
4460 :
4461 : If what we actually want is the remainder, we generate that by another
4462 : by-constant multiplication and a subtraction. */
4463 :
4464 : /* We shouldn't be called with OP1 == const1_rtx, but some of the
4465 : code below will malfunction if we are, so check here and handle
4466 : the special case if so. */
4467 239939 : if (op1 == const1_rtx)
4468 15 : return rem_flag ? const0_rtx : op0;
4469 :
4470 : /* When dividing by -1, we could get an overflow.
4471 : negv_optab can handle overflows. */
4472 239924 : if (! unsignedp && op1 == constm1_rtx)
4473 : {
4474 0 : if (rem_flag)
4475 0 : return const0_rtx;
4476 0 : return expand_unop (mode, flag_trapv && GET_MODE_CLASS (mode) == MODE_INT
4477 0 : ? negv_optab : neg_optab, op0, target, 0);
4478 : }
4479 :
4480 239924 : if (target
4481 : /* Don't use the function value register as a target
4482 : since we have to read it as well as write it,
4483 : and function-inlining gets confused by this. */
4484 239924 : && ((REG_P (target) && REG_FUNCTION_VALUE_P (target))
4485 : /* Don't clobber an operand while doing a multi-step calculation. */
4486 95517 : || ((rem_flag || op1_is_constant)
4487 75129 : && (reg_mentioned_p (target, op0)
4488 72692 : || (MEM_P (op0) && MEM_P (target))))
4489 92352 : || reg_mentioned_p (target, op1)
4490 92268 : || (MEM_P (op1) && MEM_P (target))))
4491 : target = 0;
4492 :
4493 : /* Get the mode in which to perform this computation. Normally it will
4494 : be MODE, but sometimes we can't do the desired operation in MODE.
4495 : If so, pick a wider mode in which we can do the operation. Convert
4496 : to that mode at the start to avoid repeated conversions.
4497 :
4498 : First see what operations we need. These depend on the expression
4499 : we are evaluating. (We assume that divxx3 insns exist under the
4500 : same conditions that modxx3 insns and that these insns don't normally
4501 : fail. If these assumptions are not correct, we may generate less
4502 : efficient code in some cases.)
4503 :
4504 : Then see if we find a mode in which we can open-code that operation
4505 : (either a division, modulus, or shift). Finally, check for the smallest
4506 : mode for which we can do the operation with a library call. */
4507 :
4508 : /* We might want to refine this now that we have division-by-constant
4509 : optimization. Since expmed_mult_highpart tries so many variants, it is
4510 : not straightforward to generalize this. Maybe we should make an array
4511 : of possible modes in init_expmed? Save this for GCC 2.7. */
4512 :
4513 134688 : optab1 = (op1_is_pow2
4514 239924 : ? (unsignedp ? lshr_optab : ashr_optab)
4515 158778 : : (unsignedp ? udiv_optab : sdiv_optab));
4516 320775 : optab2 = (op1_is_pow2 ? optab1
4517 158778 : : (unsignedp ? udivmod_optab : sdivmod_optab));
4518 :
4519 239924 : if (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN)
4520 : {
4521 251697 : FOR_EACH_MODE_FROM (compute_mode, mode)
4522 247462 : if (optab_handler (optab1, compute_mode) != CODE_FOR_nothing
4523 247462 : || optab_handler (optab2, compute_mode) != CODE_FOR_nothing)
4524 : break;
4525 :
4526 238131 : if (compute_mode == VOIDmode && methods == OPTAB_LIB_WIDEN)
4527 4235 : FOR_EACH_MODE_FROM (compute_mode, mode)
4528 4235 : if (optab_libfunc (optab1, compute_mode)
4529 4235 : || optab_libfunc (optab2, compute_mode))
4530 : break;
4531 : }
4532 : else
4533 : compute_mode = mode;
4534 :
4535 : /* If we still couldn't find a mode, use MODE, but expand_binop will
4536 : probably die. */
4537 6028 : if (compute_mode == VOIDmode)
4538 0 : compute_mode = mode;
4539 :
4540 239924 : if (target && GET_MODE (target) == compute_mode)
4541 : tquotient = target;
4542 : else
4543 147826 : tquotient = gen_reg_rtx (compute_mode);
4544 :
4545 : #if 0
4546 : /* It should be possible to restrict the precision to GET_MODE_BITSIZE
4547 : (mode), and thereby get better code when OP1 is a constant. Do that
4548 : later. It will require going over all usages of SIZE below. */
4549 : size = GET_MODE_BITSIZE (mode);
4550 : #endif
4551 :
4552 : /* Only deduct something for a REM if the last divide done was
4553 : for a different constant. Then set the constant of the last
4554 : divide. */
4555 239924 : max_cost = (unsignedp
4556 239924 : ? udiv_cost (speed, compute_mode)
4557 134688 : : sdiv_cost (speed, compute_mode));
4558 239924 : if (rem_flag && ! (last_div_const != 0 && op1_is_constant
4559 6303 : && INTVAL (op1) == last_div_const))
4560 61042 : max_cost -= (mul_cost (speed, compute_mode)
4561 61042 : + add_cost (speed, compute_mode));
4562 :
4563 239924 : last_div_const = ! rem_flag && op1_is_constant ? INTVAL (op1) : 0;
4564 :
4565 : /* Now convert to the best mode to use. */
4566 239924 : if (compute_mode != mode)
4567 : {
4568 0 : op0 = convert_modes (compute_mode, mode, op0, unsignedp);
4569 0 : op1 = convert_modes (compute_mode, mode, op1, unsignedp);
4570 :
4571 : /* convert_modes may have placed op1 into a register, so we
4572 : must recompute the following. */
4573 0 : op1_is_constant = CONST_INT_P (op1);
4574 0 : if (op1_is_constant)
4575 : {
4576 0 : wide_int ext_op1 = rtx_mode_t (op1, compute_mode);
4577 0 : op1_is_pow2 = (wi::popcount (ext_op1) == 1
4578 0 : || (! unsignedp
4579 0 : && wi::popcount (wi::neg (ext_op1)) == 1));
4580 0 : }
4581 : else
4582 : op1_is_pow2 = 0;
4583 : }
4584 :
4585 : /* If one of the operands is a volatile MEM, copy it into a register. */
4586 :
4587 239924 : if (MEM_P (op0) && MEM_VOLATILE_P (op0))
4588 0 : op0 = force_reg (compute_mode, op0);
4589 239924 : if (MEM_P (op1) && MEM_VOLATILE_P (op1))
4590 0 : op1 = force_reg (compute_mode, op1);
4591 :
4592 : /* If we need the remainder or if OP1 is constant, we need to
4593 : put OP0 in a register in case it has any queued subexpressions. */
4594 239924 : if (rem_flag || op1_is_constant)
4595 176788 : op0 = force_reg (compute_mode, op0);
4596 :
4597 239924 : last = get_last_insn ();
4598 :
4599 : /* Promote floor rounding to trunc rounding for unsigned operations. */
4600 239924 : if (unsignedp)
4601 : {
4602 105236 : if (code == FLOOR_DIV_EXPR)
4603 : code = TRUNC_DIV_EXPR;
4604 105183 : if (code == FLOOR_MOD_EXPR)
4605 148 : code = TRUNC_MOD_EXPR;
4606 105236 : if (code == EXACT_DIV_EXPR && op1_is_pow2)
4607 84 : code = TRUNC_DIV_EXPR;
4608 : }
4609 :
4610 239924 : if (op1 != const0_rtx)
4611 239557 : switch (code)
4612 : {
4613 190433 : case TRUNC_MOD_EXPR:
4614 190433 : case TRUNC_DIV_EXPR:
4615 190433 : if (op1_is_constant)
4616 : {
4617 89017 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
4618 89017 : int size = GET_MODE_BITSIZE (int_mode);
4619 89017 : if (unsignedp)
4620 : {
4621 52353 : unsigned HOST_WIDE_INT mh, ml;
4622 52353 : int pre_shift, post_shift;
4623 52353 : wide_int wd = rtx_mode_t (op1, int_mode);
4624 52353 : unsigned HOST_WIDE_INT d = wd.to_uhwi ();
4625 :
4626 52353 : if (wi::popcount (wd) == 1)
4627 : {
4628 27309 : pre_shift = floor_log2 (d);
4629 27309 : if (rem_flag)
4630 : {
4631 277 : unsigned HOST_WIDE_INT mask
4632 277 : = (HOST_WIDE_INT_1U << pre_shift) - 1;
4633 277 : remainder
4634 277 : = expand_binop (int_mode, and_optab, op0,
4635 277 : gen_int_mode (mask, int_mode),
4636 : remainder, 1, methods);
4637 277 : if (remainder)
4638 277 : return gen_lowpart (mode, remainder);
4639 : }
4640 27032 : quotient = expand_shift (RSHIFT_EXPR, int_mode, op0,
4641 27032 : pre_shift, tquotient, 1);
4642 : }
4643 25044 : else if (size <= HOST_BITS_PER_WIDE_INT)
4644 : {
4645 23432 : if (d >= (HOST_WIDE_INT_1U << (size - 1)))
4646 : {
4647 : /* Most significant bit of divisor is set; emit an scc
4648 : insn. */
4649 161 : quotient = emit_store_flag_force (tquotient, GEU, op0, op1,
4650 : int_mode, 1, 1);
4651 : }
4652 : else
4653 : {
4654 : /* Find a suitable multiplier and right shift count
4655 : instead of directly dividing by D. */
4656 23271 : mh = choose_multiplier (d, size, size,
4657 : &ml, &post_shift);
4658 :
4659 : /* If the suggested multiplier is more than SIZE bits,
4660 : we can do better for even divisors, using an
4661 : initial right shift. */
4662 23271 : if (mh != 0 && (d & 1) == 0)
4663 : {
4664 2360 : pre_shift = ctz_or_zero (d);
4665 2360 : mh = choose_multiplier (d >> pre_shift, size,
4666 : size - pre_shift,
4667 : &ml, &post_shift);
4668 2360 : gcc_assert (!mh);
4669 : }
4670 : else
4671 : pre_shift = 0;
4672 :
4673 2301 : if (mh != 0)
4674 : {
4675 2301 : rtx t1, t2, t3, t4;
4676 :
4677 2394 : if (post_shift - 1 >= BITS_PER_WORD)
4678 0 : goto fail1;
4679 :
4680 2301 : extra_cost
4681 2301 : = (shift_cost (speed, int_mode, post_shift - 1)
4682 2301 : + shift_cost (speed, int_mode, 1)
4683 2301 : + 2 * add_cost (speed, int_mode));
4684 :
4685 : /* Try a single widened multiply first; use it when
4686 : it is no more expensive. */
4687 2301 : quotient
4688 2301 : = expand_wide_mulh_udiv (int_mode, op0, ml, size,
4689 : post_shift, extra_cost,
4690 : max_cost, speed);
4691 2301 : if (quotient == NULL_RTX)
4692 : {
4693 1344 : t1 = expmed_mult_highpart
4694 1344 : (int_mode, op0, gen_int_mode (ml, int_mode),
4695 : NULL_RTX, 1, max_cost - extra_cost);
4696 1344 : if (t1 == 0)
4697 107 : goto fail1;
4698 1237 : t2 = force_operand (gen_rtx_MINUS (int_mode,
4699 : op0, t1),
4700 : NULL_RTX);
4701 1237 : t3 = expand_shift (RSHIFT_EXPR, int_mode,
4702 : t2, 1, NULL_RTX, 1);
4703 1237 : t4 = force_operand (gen_rtx_PLUS (int_mode,
4704 : t1, t3),
4705 : NULL_RTX);
4706 1237 : quotient = expand_shift
4707 1237 : (RSHIFT_EXPR, int_mode, t4,
4708 1237 : post_shift - 1, tquotient, 1);
4709 : }
4710 : }
4711 : else
4712 : {
4713 20970 : rtx t1, t2;
4714 :
4715 22793 : if (pre_shift >= BITS_PER_WORD
4716 20970 : || post_shift >= BITS_PER_WORD)
4717 3 : goto fail1;
4718 :
4719 20967 : t1 = expand_shift
4720 41934 : (RSHIFT_EXPR, int_mode, op0,
4721 20967 : pre_shift, NULL_RTX, 1);
4722 20967 : extra_cost
4723 20967 : = (shift_cost (speed, int_mode, pre_shift)
4724 20967 : + shift_cost (speed, int_mode, post_shift));
4725 20967 : t2 = expmed_mult_highpart
4726 20967 : (int_mode, t1,
4727 20967 : gen_int_mode (ml, int_mode),
4728 : NULL_RTX, 1, max_cost - extra_cost);
4729 20967 : if (t2 == 0)
4730 963 : goto fail1;
4731 20004 : quotient = expand_shift
4732 20004 : (RSHIFT_EXPR, int_mode, t2,
4733 20004 : post_shift, tquotient, 1);
4734 : }
4735 : }
4736 : }
4737 : else /* Too wide mode to use tricky code */
4738 : break;
4739 :
4740 49391 : insn = get_last_insn ();
4741 49391 : if (insn != last)
4742 49391 : set_dst_reg_note (insn, REG_EQUAL,
4743 : gen_rtx_UDIV (int_mode, op0, op1),
4744 : quotient);
4745 50741 : }
4746 : else /* TRUNC_DIV, signed */
4747 : {
4748 36664 : unsigned HOST_WIDE_INT ml;
4749 36664 : int post_shift;
4750 36664 : rtx mlr;
4751 36664 : HOST_WIDE_INT d = INTVAL (op1);
4752 36664 : unsigned HOST_WIDE_INT abs_d;
4753 :
4754 : /* Not prepared to handle division/remainder by
4755 : 0xffffffffffffffff8000000000000000 etc. */
4756 36664 : if (d == HOST_WIDE_INT_MIN && size > HOST_BITS_PER_WIDE_INT)
4757 : break;
4758 :
4759 : /* Since d might be INT_MIN, we have to cast to
4760 : unsigned HOST_WIDE_INT before negating to avoid
4761 : undefined signed overflow. */
4762 36664 : abs_d = (d >= 0
4763 36664 : ? (unsigned HOST_WIDE_INT) d
4764 : : - (unsigned HOST_WIDE_INT) d);
4765 :
4766 : /* n rem d = n rem -d */
4767 36664 : if (rem_flag && d < 0)
4768 : {
4769 141 : d = abs_d;
4770 141 : op1 = gen_int_mode (abs_d, int_mode);
4771 : }
4772 :
4773 36664 : if (d == 1)
4774 : quotient = op0;
4775 36664 : else if (d == -1)
4776 0 : quotient = expand_unop (int_mode, neg_optab, op0,
4777 : tquotient, 0);
4778 36664 : else if (size <= HOST_BITS_PER_WIDE_INT
4779 35263 : && abs_d == HOST_WIDE_INT_1U << (size - 1))
4780 : {
4781 : /* This case is not handled correctly below. */
4782 133 : quotient = emit_store_flag (tquotient, EQ, op0, op1,
4783 : int_mode, 1, 1);
4784 133 : if (quotient == 0)
4785 1291 : goto fail1;
4786 : }
4787 36531 : else if (EXACT_POWER_OF_2_OR_ZERO_P (d)
4788 13142 : && (size <= HOST_BITS_PER_WIDE_INT || d >= 0)
4789 2594 : && (rem_flag
4790 2594 : ? smod_pow2_cheap (speed, int_mode)
4791 10548 : : sdiv_pow2_cheap (speed, int_mode))
4792 : /* We assume that cheap metric is true if the
4793 : optab has an expander for this mode. */
4794 50868 : && ((optab_handler ((rem_flag ? smod_optab
4795 : : sdiv_optab),
4796 : int_mode)
4797 : != CODE_FOR_nothing)
4798 664 : || (optab_handler (sdivmod_optab, int_mode)
4799 : != CODE_FOR_nothing)))
4800 : ;
4801 35870 : else if (EXACT_POWER_OF_2_OR_ZERO_P (abs_d))
4802 : {
4803 12992 : if (rem_flag)
4804 : {
4805 2462 : remainder = expand_smod_pow2 (int_mode, op0, d);
4806 2462 : if (remainder)
4807 2462 : return gen_lowpart (mode, remainder);
4808 : }
4809 :
4810 10530 : if (sdiv_pow2_cheap (speed, int_mode)
4811 10530 : && ((optab_handler (sdiv_optab, int_mode)
4812 : != CODE_FOR_nothing)
4813 12 : || (optab_handler (sdivmod_optab, int_mode)
4814 : != CODE_FOR_nothing)))
4815 9 : quotient = expand_divmod (0, TRUNC_DIV_EXPR,
4816 : int_mode, op0,
4817 9 : gen_int_mode (abs_d,
4818 : int_mode),
4819 : NULL_RTX, 0);
4820 : else
4821 10521 : quotient = expand_sdiv_pow2 (int_mode, op0, abs_d);
4822 :
4823 : /* We have computed OP0 / abs(OP1). If OP1 is negative,
4824 : negate the quotient. */
4825 10530 : if (d < 0)
4826 : {
4827 511 : insn = get_last_insn ();
4828 511 : if (insn != last
4829 511 : && abs_d < (HOST_WIDE_INT_1U
4830 : << (HOST_BITS_PER_WIDE_INT - 1)))
4831 511 : set_dst_reg_note (insn, REG_EQUAL,
4832 511 : gen_rtx_DIV (int_mode, op0,
4833 : gen_int_mode
4834 : (abs_d,
4835 : int_mode)),
4836 : quotient);
4837 :
4838 511 : quotient = expand_unop (int_mode, neg_optab,
4839 : quotient, quotient, 0);
4840 : }
4841 : }
4842 22878 : else if (size <= HOST_BITS_PER_WIDE_INT)
4843 : {
4844 21539 : choose_multiplier (abs_d, size, size - 1,
4845 : &ml, &post_shift);
4846 21539 : if (ml < HOST_WIDE_INT_1U << (size - 1))
4847 : {
4848 16110 : rtx t1, t2, t3;
4849 :
4850 17225 : if (post_shift >= BITS_PER_WORD
4851 16110 : || size - 1 >= BITS_PER_WORD)
4852 250 : goto fail1;
4853 :
4854 15860 : extra_cost = (shift_cost (speed, int_mode, post_shift)
4855 15860 : + shift_cost (speed, int_mode, size - 1)
4856 15860 : + add_cost (speed, int_mode));
4857 15860 : t1 = expmed_mult_highpart
4858 15860 : (int_mode, op0, gen_int_mode (ml, int_mode),
4859 : NULL_RTX, 0, max_cost - extra_cost);
4860 15860 : if (t1 == 0)
4861 780 : goto fail1;
4862 15080 : t2 = expand_shift
4863 30160 : (RSHIFT_EXPR, int_mode, t1,
4864 15080 : post_shift, NULL_RTX, 0);
4865 15080 : t3 = expand_shift
4866 15080 : (RSHIFT_EXPR, int_mode, op0,
4867 15080 : size - 1, NULL_RTX, 0);
4868 15080 : if (d < 0)
4869 198 : quotient
4870 198 : = force_operand (gen_rtx_MINUS (int_mode, t3, t2),
4871 : tquotient);
4872 : else
4873 14882 : quotient
4874 14882 : = force_operand (gen_rtx_MINUS (int_mode, t2, t3),
4875 : tquotient);
4876 : }
4877 : else
4878 : {
4879 5429 : rtx t1, t2, t3, t4;
4880 :
4881 5789 : if (post_shift >= BITS_PER_WORD
4882 5424 : || size - 1 >= BITS_PER_WORD)
4883 43 : goto fail1;
4884 :
4885 5386 : ml |= HOST_WIDE_INT_M1U << (size - 1);
4886 5386 : mlr = gen_int_mode (ml, int_mode);
4887 5386 : extra_cost = (shift_cost (speed, int_mode, post_shift)
4888 5386 : + shift_cost (speed, int_mode, size - 1)
4889 5386 : + 2 * add_cost (speed, int_mode));
4890 5386 : t1 = expmed_mult_highpart (int_mode, op0, mlr,
4891 : NULL_RTX, 0,
4892 : max_cost - extra_cost);
4893 5386 : if (t1 == 0)
4894 218 : goto fail1;
4895 5168 : t2 = force_operand (gen_rtx_PLUS (int_mode, t1, op0),
4896 : NULL_RTX);
4897 5168 : t3 = expand_shift
4898 10336 : (RSHIFT_EXPR, int_mode, t2,
4899 5168 : post_shift, NULL_RTX, 0);
4900 5168 : t4 = expand_shift
4901 5168 : (RSHIFT_EXPR, int_mode, op0,
4902 5168 : size - 1, NULL_RTX, 0);
4903 5168 : if (d < 0)
4904 52 : quotient
4905 52 : = force_operand (gen_rtx_MINUS (int_mode, t4, t3),
4906 : tquotient);
4907 : else
4908 5116 : quotient
4909 5116 : = force_operand (gen_rtx_MINUS (int_mode, t3, t4),
4910 : tquotient);
4911 : }
4912 : }
4913 : else /* Too wide mode to use tricky code */
4914 : break;
4915 :
4916 31572 : insn = get_last_insn ();
4917 31572 : if (insn != last)
4918 30911 : set_dst_reg_note (insn, REG_EQUAL,
4919 : gen_rtx_DIV (int_mode, op0, op1),
4920 : quotient);
4921 : }
4922 : break;
4923 : }
4924 101416 : fail1:
4925 103780 : delete_insns_since (last);
4926 103780 : break;
4927 :
4928 1765 : case FLOOR_DIV_EXPR:
4929 1765 : case FLOOR_MOD_EXPR:
4930 : /* We will come here only for signed operations. */
4931 1765 : if (op1_is_constant && HWI_COMPUTABLE_MODE_P (compute_mode))
4932 : {
4933 971 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
4934 971 : int size = GET_MODE_BITSIZE (int_mode);
4935 971 : unsigned HOST_WIDE_INT mh, ml;
4936 971 : int pre_shift, post_shift;
4937 971 : HOST_WIDE_INT d = INTVAL (op1);
4938 :
4939 971 : if (d > 0)
4940 : {
4941 : /* We could just as easily deal with negative constants here,
4942 : but it does not seem worth the trouble for GCC 2.6. */
4943 945 : if (EXACT_POWER_OF_2_OR_ZERO_P (d))
4944 : {
4945 641 : pre_shift = floor_log2 (d);
4946 641 : if (rem_flag)
4947 : {
4948 64 : unsigned HOST_WIDE_INT mask
4949 64 : = (HOST_WIDE_INT_1U << pre_shift) - 1;
4950 64 : remainder = expand_binop
4951 64 : (int_mode, and_optab, op0,
4952 64 : gen_int_mode (mask, int_mode),
4953 : remainder, 0, methods);
4954 64 : if (remainder)
4955 64 : return gen_lowpart (mode, remainder);
4956 : }
4957 577 : quotient = expand_shift
4958 577 : (RSHIFT_EXPR, int_mode, op0,
4959 577 : pre_shift, tquotient, 0);
4960 : }
4961 : else
4962 : {
4963 304 : rtx t1, t2, t3, t4;
4964 :
4965 304 : mh = choose_multiplier (d, size, size - 1,
4966 : &ml, &post_shift);
4967 304 : gcc_assert (!mh);
4968 :
4969 328 : if (post_shift < BITS_PER_WORD
4970 304 : && size - 1 < BITS_PER_WORD)
4971 : {
4972 302 : t1 = expand_shift
4973 302 : (RSHIFT_EXPR, int_mode, op0,
4974 302 : size - 1, NULL_RTX, 0);
4975 302 : t2 = expand_binop (int_mode, xor_optab, op0, t1,
4976 : NULL_RTX, 0, OPTAB_WIDEN);
4977 302 : extra_cost = (shift_cost (speed, int_mode, post_shift)
4978 302 : + shift_cost (speed, int_mode, size - 1)
4979 302 : + 2 * add_cost (speed, int_mode));
4980 302 : t3 = expmed_mult_highpart
4981 302 : (int_mode, t2, gen_int_mode (ml, int_mode),
4982 : NULL_RTX, 1, max_cost - extra_cost);
4983 302 : if (t3 != 0)
4984 : {
4985 271 : t4 = expand_shift
4986 542 : (RSHIFT_EXPR, int_mode, t3,
4987 271 : post_shift, NULL_RTX, 1);
4988 271 : quotient = expand_binop (int_mode, xor_optab,
4989 : t4, t1, tquotient, 0,
4990 : OPTAB_WIDEN);
4991 : }
4992 : }
4993 : }
4994 : }
4995 : else
4996 : {
4997 26 : rtx nsign, t1, t2, t3, t4;
4998 26 : t1 = force_operand (gen_rtx_PLUS (int_mode,
4999 : op0, constm1_rtx), NULL_RTX);
5000 26 : t2 = expand_binop (int_mode, ior_optab, op0, t1, NULL_RTX,
5001 : 0, OPTAB_WIDEN);
5002 52 : nsign = expand_shift (RSHIFT_EXPR, int_mode, t2,
5003 26 : size - 1, NULL_RTX, 0);
5004 26 : t3 = force_operand (gen_rtx_MINUS (int_mode, t1, nsign),
5005 : NULL_RTX);
5006 26 : t4 = expand_divmod (0, TRUNC_DIV_EXPR, int_mode, t3, op1,
5007 : NULL_RTX, 0);
5008 26 : if (t4)
5009 : {
5010 26 : rtx t5;
5011 26 : t5 = expand_unop (int_mode, one_cmpl_optab, nsign,
5012 : NULL_RTX, 0);
5013 26 : quotient = force_operand (gen_rtx_PLUS (int_mode, t4, t5),
5014 : tquotient);
5015 : }
5016 : }
5017 : }
5018 :
5019 907 : if (quotient != 0)
5020 : break;
5021 827 : delete_insns_since (last);
5022 :
5023 : /* Try using an instruction that produces both the quotient and
5024 : remainder, using truncation. We can easily compensate the quotient
5025 : or remainder to get floor rounding, once we have the remainder.
5026 : Notice that we compute also the final remainder value here,
5027 : and return the result right away. */
5028 827 : if (target == 0 || GET_MODE (target) != compute_mode)
5029 126 : target = gen_reg_rtx (compute_mode);
5030 :
5031 827 : if (rem_flag)
5032 : {
5033 329 : remainder
5034 329 : = REG_P (target) ? target : gen_reg_rtx (compute_mode);
5035 329 : quotient = gen_reg_rtx (compute_mode);
5036 : }
5037 : else
5038 : {
5039 498 : quotient
5040 498 : = REG_P (target) ? target : gen_reg_rtx (compute_mode);
5041 498 : remainder = gen_reg_rtx (compute_mode);
5042 : }
5043 :
5044 827 : if (expand_twoval_binop (sdivmod_optab, op0, op1,
5045 : quotient, remainder, 0))
5046 : {
5047 : /* This could be computed with a branch-less sequence.
5048 : Save that for later. */
5049 792 : rtx tem;
5050 792 : rtx_code_label *label = gen_label_rtx ();
5051 792 : do_cmp_and_jump (remainder, const0_rtx, EQ, compute_mode, label);
5052 792 : tem = expand_binop (compute_mode, xor_optab, op0, op1,
5053 : NULL_RTX, 0, OPTAB_WIDEN);
5054 792 : do_cmp_and_jump (tem, const0_rtx, GE, compute_mode, label);
5055 792 : expand_dec (quotient, const1_rtx);
5056 792 : expand_inc (remainder, op1);
5057 792 : emit_label (label);
5058 1281 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5059 : }
5060 :
5061 : /* No luck with division elimination or divmod. Have to do it
5062 : by conditionally adjusting op0 *and* the result. */
5063 35 : {
5064 35 : rtx_code_label *label1, *label2, *label3, *label4, *label5;
5065 35 : rtx adjusted_op0;
5066 35 : rtx tem;
5067 :
5068 35 : quotient = gen_reg_rtx (compute_mode);
5069 35 : adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
5070 35 : label1 = gen_label_rtx ();
5071 35 : label2 = gen_label_rtx ();
5072 35 : label3 = gen_label_rtx ();
5073 35 : label4 = gen_label_rtx ();
5074 35 : label5 = gen_label_rtx ();
5075 35 : do_cmp_and_jump (op1, const0_rtx, LT, compute_mode, label2);
5076 35 : do_cmp_and_jump (adjusted_op0, const0_rtx, LT, compute_mode, label1);
5077 35 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5078 : quotient, 0, methods);
5079 35 : if (tem != quotient)
5080 35 : emit_move_insn (quotient, tem);
5081 35 : emit_jump_insn (targetm.gen_jump (label5));
5082 35 : emit_barrier ();
5083 35 : emit_label (label1);
5084 35 : expand_inc (adjusted_op0, const1_rtx);
5085 35 : emit_jump_insn (targetm.gen_jump (label4));
5086 35 : emit_barrier ();
5087 35 : emit_label (label2);
5088 35 : do_cmp_and_jump (adjusted_op0, const0_rtx, GT, compute_mode, label3);
5089 35 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5090 : quotient, 0, methods);
5091 35 : if (tem != quotient)
5092 35 : emit_move_insn (quotient, tem);
5093 35 : emit_jump_insn (targetm.gen_jump (label5));
5094 35 : emit_barrier ();
5095 35 : emit_label (label3);
5096 35 : expand_dec (adjusted_op0, const1_rtx);
5097 35 : emit_label (label4);
5098 35 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5099 : quotient, 0, methods);
5100 35 : if (tem != quotient)
5101 35 : emit_move_insn (quotient, tem);
5102 35 : expand_dec (quotient, const1_rtx);
5103 35 : emit_label (label5);
5104 : }
5105 35 : break;
5106 :
5107 384 : case CEIL_DIV_EXPR:
5108 384 : case CEIL_MOD_EXPR:
5109 384 : if (unsignedp)
5110 : {
5111 0 : if (op1_is_constant
5112 0 : && EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1))
5113 0 : && (HWI_COMPUTABLE_MODE_P (compute_mode)
5114 0 : || INTVAL (op1) >= 0))
5115 : {
5116 0 : scalar_int_mode int_mode
5117 0 : = as_a <scalar_int_mode> (compute_mode);
5118 0 : rtx t1, t2, t3;
5119 0 : unsigned HOST_WIDE_INT d = INTVAL (op1);
5120 0 : t1 = expand_shift (RSHIFT_EXPR, int_mode, op0,
5121 0 : floor_log2 (d), tquotient, 1);
5122 0 : t2 = expand_binop (int_mode, and_optab, op0,
5123 0 : gen_int_mode (d - 1, int_mode),
5124 : NULL_RTX, 1, methods);
5125 0 : t3 = gen_reg_rtx (int_mode);
5126 0 : t3 = emit_store_flag (t3, NE, t2, const0_rtx, int_mode, 1, 1);
5127 0 : if (t3 == 0)
5128 : {
5129 0 : rtx_code_label *lab;
5130 0 : lab = gen_label_rtx ();
5131 0 : do_cmp_and_jump (t2, const0_rtx, EQ, int_mode, lab);
5132 0 : expand_inc (t1, const1_rtx);
5133 0 : emit_label (lab);
5134 0 : quotient = t1;
5135 : }
5136 : else
5137 0 : quotient = force_operand (gen_rtx_PLUS (int_mode, t1, t3),
5138 : tquotient);
5139 : break;
5140 : }
5141 :
5142 : /* Try using an instruction that produces both the quotient and
5143 : remainder, using truncation. We can easily compensate the
5144 : quotient or remainder to get ceiling rounding, once we have the
5145 : remainder. Notice that we compute also the final remainder
5146 : value here, and return the result right away. */
5147 0 : if (target == 0 || GET_MODE (target) != compute_mode)
5148 0 : target = gen_reg_rtx (compute_mode);
5149 :
5150 0 : if (rem_flag)
5151 : {
5152 0 : remainder = (REG_P (target)
5153 0 : ? target : gen_reg_rtx (compute_mode));
5154 0 : quotient = gen_reg_rtx (compute_mode);
5155 : }
5156 : else
5157 : {
5158 0 : quotient = (REG_P (target)
5159 0 : ? target : gen_reg_rtx (compute_mode));
5160 0 : remainder = gen_reg_rtx (compute_mode);
5161 : }
5162 :
5163 0 : if (expand_twoval_binop (udivmod_optab, op0, op1, quotient,
5164 : remainder, 1))
5165 : {
5166 : /* This could be computed with a branch-less sequence.
5167 : Save that for later. */
5168 0 : rtx_code_label *label = gen_label_rtx ();
5169 0 : do_cmp_and_jump (remainder, const0_rtx, EQ,
5170 : compute_mode, label);
5171 0 : expand_inc (quotient, const1_rtx);
5172 0 : expand_dec (remainder, op1);
5173 0 : emit_label (label);
5174 0 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5175 : }
5176 :
5177 : /* No luck with division elimination or divmod. Have to do it
5178 : by conditionally adjusting op0 *and* the result. */
5179 0 : {
5180 0 : rtx_code_label *label1, *label2;
5181 0 : rtx adjusted_op0, tem;
5182 :
5183 0 : quotient = gen_reg_rtx (compute_mode);
5184 0 : adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
5185 0 : label1 = gen_label_rtx ();
5186 0 : label2 = gen_label_rtx ();
5187 0 : do_cmp_and_jump (adjusted_op0, const0_rtx, NE,
5188 : compute_mode, label1);
5189 0 : emit_move_insn (quotient, const0_rtx);
5190 0 : emit_jump_insn (targetm.gen_jump (label2));
5191 0 : emit_barrier ();
5192 0 : emit_label (label1);
5193 0 : expand_dec (adjusted_op0, const1_rtx);
5194 0 : tem = expand_binop (compute_mode, udiv_optab, adjusted_op0, op1,
5195 : quotient, 1, methods);
5196 0 : if (tem != quotient)
5197 0 : emit_move_insn (quotient, tem);
5198 0 : expand_inc (quotient, const1_rtx);
5199 0 : emit_label (label2);
5200 : }
5201 : }
5202 : else /* signed */
5203 : {
5204 384 : if (op1_is_constant && EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1))
5205 27 : && INTVAL (op1) >= 0)
5206 : {
5207 : /* This is extremely similar to the code for the unsigned case
5208 : above. For 2.7 we should merge these variants, but for
5209 : 2.6.1 I don't want to touch the code for unsigned since that
5210 : get used in C. The signed case will only be used by other
5211 : languages (Ada). */
5212 :
5213 27 : rtx t1, t2, t3;
5214 27 : unsigned HOST_WIDE_INT d = INTVAL (op1);
5215 54 : t1 = expand_shift (RSHIFT_EXPR, compute_mode, op0,
5216 27 : floor_log2 (d), tquotient, 0);
5217 27 : t2 = expand_binop (compute_mode, and_optab, op0,
5218 27 : gen_int_mode (d - 1, compute_mode),
5219 : NULL_RTX, 1, methods);
5220 27 : t3 = gen_reg_rtx (compute_mode);
5221 27 : t3 = emit_store_flag (t3, NE, t2, const0_rtx,
5222 : compute_mode, 1, 1);
5223 27 : if (t3 == 0)
5224 : {
5225 0 : rtx_code_label *lab;
5226 0 : lab = gen_label_rtx ();
5227 0 : do_cmp_and_jump (t2, const0_rtx, EQ, compute_mode, lab);
5228 0 : expand_inc (t1, const1_rtx);
5229 0 : emit_label (lab);
5230 0 : quotient = t1;
5231 : }
5232 : else
5233 27 : quotient = force_operand (gen_rtx_PLUS (compute_mode,
5234 : t1, t3),
5235 : tquotient);
5236 : break;
5237 : }
5238 :
5239 : /* Try using an instruction that produces both the quotient and
5240 : remainder, using truncation. We can easily compensate the
5241 : quotient or remainder to get ceiling rounding, once we have the
5242 : remainder. Notice that we compute also the final remainder
5243 : value here, and return the result right away. */
5244 357 : if (target == 0 || GET_MODE (target) != compute_mode)
5245 15 : target = gen_reg_rtx (compute_mode);
5246 357 : if (rem_flag)
5247 : {
5248 149 : remainder= (REG_P (target)
5249 149 : ? target : gen_reg_rtx (compute_mode));
5250 149 : quotient = gen_reg_rtx (compute_mode);
5251 : }
5252 : else
5253 : {
5254 208 : quotient = (REG_P (target)
5255 208 : ? target : gen_reg_rtx (compute_mode));
5256 208 : remainder = gen_reg_rtx (compute_mode);
5257 : }
5258 :
5259 357 : if (expand_twoval_binop (sdivmod_optab, op0, op1, quotient,
5260 : remainder, 0))
5261 : {
5262 : /* This could be computed with a branch-less sequence.
5263 : Save that for later. */
5264 357 : rtx tem;
5265 357 : rtx_code_label *label = gen_label_rtx ();
5266 357 : do_cmp_and_jump (remainder, const0_rtx, EQ,
5267 : compute_mode, label);
5268 357 : tem = expand_binop (compute_mode, xor_optab, op0, op1,
5269 : NULL_RTX, 0, OPTAB_WIDEN);
5270 357 : do_cmp_and_jump (tem, const0_rtx, LT, compute_mode, label);
5271 357 : expand_inc (quotient, const1_rtx);
5272 357 : expand_dec (remainder, op1);
5273 357 : emit_label (label);
5274 565 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5275 : }
5276 :
5277 : /* No luck with division elimination or divmod. Have to do it
5278 : by conditionally adjusting op0 *and* the result. */
5279 0 : {
5280 0 : rtx_code_label *label1, *label2, *label3, *label4, *label5;
5281 0 : rtx adjusted_op0;
5282 0 : rtx tem;
5283 :
5284 0 : quotient = gen_reg_rtx (compute_mode);
5285 0 : adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
5286 0 : label1 = gen_label_rtx ();
5287 0 : label2 = gen_label_rtx ();
5288 0 : label3 = gen_label_rtx ();
5289 0 : label4 = gen_label_rtx ();
5290 0 : label5 = gen_label_rtx ();
5291 0 : do_cmp_and_jump (op1, const0_rtx, LT, compute_mode, label2);
5292 0 : do_cmp_and_jump (adjusted_op0, const0_rtx, GT,
5293 : compute_mode, label1);
5294 0 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5295 : quotient, 0, methods);
5296 0 : if (tem != quotient)
5297 0 : emit_move_insn (quotient, tem);
5298 0 : emit_jump_insn (targetm.gen_jump (label5));
5299 0 : emit_barrier ();
5300 0 : emit_label (label1);
5301 0 : expand_dec (adjusted_op0, const1_rtx);
5302 0 : emit_jump_insn (targetm.gen_jump (label4));
5303 0 : emit_barrier ();
5304 0 : emit_label (label2);
5305 0 : do_cmp_and_jump (adjusted_op0, const0_rtx, LT,
5306 : compute_mode, label3);
5307 0 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5308 : quotient, 0, methods);
5309 0 : if (tem != quotient)
5310 0 : emit_move_insn (quotient, tem);
5311 0 : emit_jump_insn (targetm.gen_jump (label5));
5312 0 : emit_barrier ();
5313 0 : emit_label (label3);
5314 0 : expand_inc (adjusted_op0, const1_rtx);
5315 0 : emit_label (label4);
5316 0 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5317 : quotient, 0, methods);
5318 0 : if (tem != quotient)
5319 0 : emit_move_insn (quotient, tem);
5320 0 : expand_inc (quotient, const1_rtx);
5321 0 : emit_label (label5);
5322 : }
5323 : }
5324 : break;
5325 :
5326 46973 : case EXACT_DIV_EXPR:
5327 46973 : if (op1_is_constant && HWI_COMPUTABLE_MODE_P (compute_mode))
5328 : {
5329 46965 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
5330 46965 : int size = GET_MODE_BITSIZE (int_mode);
5331 46965 : HOST_WIDE_INT d = INTVAL (op1);
5332 46965 : unsigned HOST_WIDE_INT ml;
5333 46965 : int pre_shift;
5334 46965 : rtx t1;
5335 :
5336 46965 : pre_shift = ctz_or_zero (d);
5337 46965 : ml = invert_mod2n (d >> pre_shift, size);
5338 46965 : t1 = expand_shift (RSHIFT_EXPR, int_mode, op0,
5339 46965 : pre_shift, NULL_RTX, unsignedp);
5340 46965 : quotient = expand_mult (int_mode, t1, gen_int_mode (ml, int_mode),
5341 : NULL_RTX, 1);
5342 :
5343 46965 : insn = get_last_insn ();
5344 93930 : set_dst_reg_note (insn, REG_EQUAL,
5345 : gen_rtx_fmt_ee (unsignedp ? UDIV : DIV,
5346 : int_mode, op0, op1),
5347 : quotient);
5348 : }
5349 : break;
5350 :
5351 2 : case ROUND_DIV_EXPR:
5352 2 : case ROUND_MOD_EXPR:
5353 2 : if (unsignedp)
5354 : {
5355 0 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
5356 0 : rtx tem;
5357 0 : rtx_code_label *label;
5358 0 : label = gen_label_rtx ();
5359 0 : quotient = gen_reg_rtx (int_mode);
5360 0 : remainder = gen_reg_rtx (int_mode);
5361 0 : if (expand_twoval_binop (udivmod_optab, op0, op1, quotient, remainder, 1) == 0)
5362 : {
5363 0 : rtx tem;
5364 0 : quotient = expand_binop (int_mode, udiv_optab, op0, op1,
5365 : quotient, 1, methods);
5366 0 : tem = expand_mult (int_mode, quotient, op1, NULL_RTX, 1);
5367 0 : remainder = expand_binop (int_mode, sub_optab, op0, tem,
5368 : remainder, 1, methods);
5369 : }
5370 0 : tem = plus_constant (int_mode, op1, -1);
5371 0 : tem = expand_shift (RSHIFT_EXPR, int_mode, tem, 1, NULL_RTX, 1);
5372 0 : do_cmp_and_jump (remainder, tem, LEU, int_mode, label);
5373 0 : expand_inc (quotient, const1_rtx);
5374 0 : expand_dec (remainder, op1);
5375 0 : emit_label (label);
5376 : }
5377 : else
5378 : {
5379 2 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
5380 2 : int size = GET_MODE_BITSIZE (int_mode);
5381 2 : rtx abs_rem, abs_op1, tem, mask;
5382 2 : rtx_code_label *label;
5383 2 : label = gen_label_rtx ();
5384 2 : quotient = gen_reg_rtx (int_mode);
5385 2 : remainder = gen_reg_rtx (int_mode);
5386 2 : if (expand_twoval_binop (sdivmod_optab, op0, op1, quotient, remainder, 0) == 0)
5387 : {
5388 0 : rtx tem;
5389 0 : quotient = expand_binop (int_mode, sdiv_optab, op0, op1,
5390 : quotient, 0, methods);
5391 0 : tem = expand_mult (int_mode, quotient, op1, NULL_RTX, 0);
5392 0 : remainder = expand_binop (int_mode, sub_optab, op0, tem,
5393 : remainder, 0, methods);
5394 : }
5395 2 : abs_rem = expand_abs (int_mode, remainder, NULL_RTX, 1, 0);
5396 2 : abs_op1 = expand_abs (int_mode, op1, NULL_RTX, 1, 0);
5397 2 : tem = expand_shift (LSHIFT_EXPR, int_mode, abs_rem,
5398 : 1, NULL_RTX, 1);
5399 2 : do_cmp_and_jump (tem, abs_op1, LTU, int_mode, label);
5400 2 : tem = expand_binop (int_mode, xor_optab, op0, op1,
5401 : NULL_RTX, 0, OPTAB_WIDEN);
5402 4 : mask = expand_shift (RSHIFT_EXPR, int_mode, tem,
5403 2 : size - 1, NULL_RTX, 0);
5404 2 : tem = expand_binop (int_mode, xor_optab, mask, const1_rtx,
5405 : NULL_RTX, 0, OPTAB_WIDEN);
5406 2 : tem = expand_binop (int_mode, sub_optab, tem, mask,
5407 : NULL_RTX, 0, OPTAB_WIDEN);
5408 2 : expand_inc (quotient, tem);
5409 2 : tem = expand_binop (int_mode, xor_optab, mask, op1,
5410 : NULL_RTX, 0, OPTAB_WIDEN);
5411 2 : tem = expand_binop (int_mode, sub_optab, tem, mask,
5412 : NULL_RTX, 0, OPTAB_WIDEN);
5413 2 : expand_dec (remainder, tem);
5414 2 : emit_label (label);
5415 : }
5416 3 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5417 :
5418 0 : default:
5419 0 : gcc_unreachable ();
5420 : }
5421 :
5422 234721 : if (quotient == 0)
5423 : {
5424 107767 : if (target && GET_MODE (target) != compute_mode)
5425 64524 : target = 0;
5426 :
5427 107767 : if (rem_flag)
5428 : {
5429 : /* Try to produce the remainder without producing the quotient.
5430 : If we seem to have a divmod pattern that does not require widening,
5431 : don't try widening here. We should really have a WIDEN argument
5432 : to expand_twoval_binop, since what we'd really like to do here is
5433 : 1) try a mod insn in compute_mode
5434 : 2) try a divmod insn in compute_mode
5435 : 3) try a div insn in compute_mode and multiply-subtract to get
5436 : remainder
5437 : 4) try the same things with widening allowed. */
5438 41475 : remainder
5439 42969 : = sign_expand_binop (compute_mode, umod_optab, smod_optab,
5440 : op0, op1, target,
5441 : unsignedp,
5442 41475 : ((optab_handler (optab2, compute_mode)
5443 : != CODE_FOR_nothing)
5444 : ? OPTAB_DIRECT : OPTAB_WIDEN));
5445 41475 : if (remainder == 0)
5446 : {
5447 : /* No luck there. Can we do remainder and divide at once
5448 : without a library call? */
5449 41268 : remainder = gen_reg_rtx (compute_mode);
5450 65064 : if (! expand_twoval_binop ((unsignedp
5451 : ? udivmod_optab
5452 : : sdivmod_optab),
5453 : op0, op1,
5454 : NULL_RTX, remainder, unsignedp))
5455 : remainder = 0;
5456 : }
5457 :
5458 39981 : if (remainder)
5459 40188 : return gen_lowpart (mode, remainder);
5460 : }
5461 :
5462 : /* Produce the quotient. Try a quotient insn, but not a library call.
5463 : If we have a divmod in this mode, use it in preference to widening
5464 : the div (for this test we assume it will not fail). Note that optab2
5465 : is set to the one of the two optabs that the call below will use. */
5466 67579 : quotient
5467 72861 : = sign_expand_binop (compute_mode, udiv_optab, sdiv_optab,
5468 : op0, op1, rem_flag ? NULL_RTX : target,
5469 : unsignedp,
5470 67579 : ((optab_handler (optab2, compute_mode)
5471 : != CODE_FOR_nothing)
5472 : ? OPTAB_DIRECT : OPTAB_WIDEN));
5473 :
5474 67579 : if (quotient == 0)
5475 : {
5476 : /* No luck there. Try a quotient-and-remainder insn,
5477 : keeping the quotient alone. */
5478 67144 : quotient = gen_reg_rtx (compute_mode);
5479 97772 : if (! expand_twoval_binop (unsignedp ? udivmod_optab : sdivmod_optab,
5480 : op0, op1,
5481 : quotient, NULL_RTX, unsignedp))
5482 : {
5483 3561 : quotient = 0;
5484 3561 : if (! rem_flag)
5485 : /* Still no luck. If we are not computing the remainder,
5486 : use a library call for the quotient. */
5487 2312 : quotient = sign_expand_binop (compute_mode,
5488 : udiv_optab, sdiv_optab,
5489 : op0, op1, target,
5490 : unsignedp, methods);
5491 : }
5492 : }
5493 : }
5494 :
5495 194533 : if (rem_flag)
5496 : {
5497 20715 : if (target && GET_MODE (target) != compute_mode)
5498 13653 : target = 0;
5499 :
5500 20715 : if (quotient == 0)
5501 : {
5502 : /* No divide instruction either. Use library for remainder. */
5503 1249 : remainder = sign_expand_binop (compute_mode, umod_optab, smod_optab,
5504 : op0, op1, target,
5505 : unsignedp, methods);
5506 : /* No remainder function. Try a quotient-and-remainder
5507 : function, keeping the remainder. */
5508 1249 : if (!remainder
5509 0 : && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
5510 : {
5511 0 : remainder = gen_reg_rtx (compute_mode);
5512 0 : if (!expand_twoval_binop_libfunc
5513 0 : (unsignedp ? udivmod_optab : sdivmod_optab,
5514 : op0, op1,
5515 : NULL_RTX, remainder,
5516 : unsignedp ? UMOD : MOD))
5517 0 : remainder = NULL_RTX;
5518 : }
5519 : }
5520 : else
5521 : {
5522 : /* We divided. Now finish doing X - Y * (X / Y). */
5523 19466 : remainder = expand_mult (compute_mode, quotient, op1,
5524 : NULL_RTX, unsignedp);
5525 19466 : remainder = expand_binop (compute_mode, sub_optab, op0,
5526 : remainder, target, unsignedp,
5527 : methods);
5528 : }
5529 : }
5530 :
5531 195782 : if (methods != OPTAB_LIB_WIDEN
5532 1793 : && (rem_flag ? remainder : quotient) == NULL_RTX)
5533 : return NULL_RTX;
5534 :
5535 370849 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5536 : }
5537 :
5538 : /* Return a tree node with data type TYPE, describing the value of X.
5539 : Usually this is an VAR_DECL, if there is no obvious better choice.
5540 : X may be an expression, however we only support those expressions
5541 : generated by loop.c. */
5542 :
5543 : tree
5544 691764 : make_tree (tree type, rtx x)
5545 : {
5546 691764 : tree t;
5547 :
5548 691764 : switch (GET_CODE (x))
5549 : {
5550 23019 : case CONST_INT:
5551 23019 : case CONST_WIDE_INT:
5552 23019 : t = wide_int_to_tree (type, rtx_mode_t (x, TYPE_MODE (type)));
5553 23019 : return t;
5554 :
5555 0 : case CONST_POLY_INT:
5556 0 : return wide_int_to_tree (type, const_poly_int_value (x));
5557 :
5558 0 : case CONST_DOUBLE:
5559 0 : STATIC_ASSERT (HOST_BITS_PER_WIDE_INT * 2 <= MAX_BITSIZE_MODE_ANY_INT);
5560 0 : if (TARGET_SUPPORTS_WIDE_INT == 0 && GET_MODE (x) == VOIDmode)
5561 : t = wide_int_to_tree (type,
5562 : wide_int::from_array (&CONST_DOUBLE_LOW (x), 2,
5563 : HOST_BITS_PER_WIDE_INT * 2));
5564 : else
5565 0 : t = build_real (type, *CONST_DOUBLE_REAL_VALUE (x));
5566 :
5567 0 : return t;
5568 :
5569 0 : case CONST_VECTOR:
5570 0 : {
5571 0 : unsigned int npatterns = CONST_VECTOR_NPATTERNS (x);
5572 0 : unsigned int nelts_per_pattern = CONST_VECTOR_NELTS_PER_PATTERN (x);
5573 0 : tree itype = TREE_TYPE (type);
5574 :
5575 : /* Build a tree with vector elements. */
5576 0 : tree_vector_builder elts (type, npatterns, nelts_per_pattern);
5577 0 : unsigned int count = elts.encoded_nelts ();
5578 0 : for (unsigned int i = 0; i < count; ++i)
5579 : {
5580 0 : rtx elt = CONST_VECTOR_ELT (x, i);
5581 0 : elts.quick_push (make_tree (itype, elt));
5582 : }
5583 :
5584 0 : return elts.build ();
5585 0 : }
5586 :
5587 0 : case PLUS:
5588 0 : return fold_build2 (PLUS_EXPR, type, make_tree (type, XEXP (x, 0)),
5589 : make_tree (type, XEXP (x, 1)));
5590 :
5591 0 : case MINUS:
5592 0 : return fold_build2 (MINUS_EXPR, type, make_tree (type, XEXP (x, 0)),
5593 : make_tree (type, XEXP (x, 1)));
5594 :
5595 0 : case NEG:
5596 0 : return fold_build1 (NEGATE_EXPR, type, make_tree (type, XEXP (x, 0)));
5597 :
5598 0 : case MULT:
5599 0 : return fold_build2 (MULT_EXPR, type, make_tree (type, XEXP (x, 0)),
5600 : make_tree (type, XEXP (x, 1)));
5601 :
5602 0 : case ASHIFT:
5603 0 : return fold_build2 (LSHIFT_EXPR, type, make_tree (type, XEXP (x, 0)),
5604 : make_tree (type, XEXP (x, 1)));
5605 :
5606 0 : case LSHIFTRT:
5607 0 : t = unsigned_type_for (type);
5608 0 : return fold_convert (type, build2 (RSHIFT_EXPR, t,
5609 : make_tree (t, XEXP (x, 0)),
5610 : make_tree (type, XEXP (x, 1))));
5611 :
5612 0 : case ASHIFTRT:
5613 0 : t = signed_type_for (type);
5614 0 : return fold_convert (type, build2 (RSHIFT_EXPR, t,
5615 : make_tree (t, XEXP (x, 0)),
5616 : make_tree (type, XEXP (x, 1))));
5617 :
5618 0 : case DIV:
5619 0 : if (TREE_CODE (type) != REAL_TYPE)
5620 0 : t = signed_type_for (type);
5621 : else
5622 : t = type;
5623 :
5624 0 : return fold_convert (type, build2 (TRUNC_DIV_EXPR, t,
5625 : make_tree (t, XEXP (x, 0)),
5626 : make_tree (t, XEXP (x, 1))));
5627 0 : case UDIV:
5628 0 : t = unsigned_type_for (type);
5629 0 : return fold_convert (type, build2 (TRUNC_DIV_EXPR, t,
5630 : make_tree (t, XEXP (x, 0)),
5631 : make_tree (t, XEXP (x, 1))));
5632 :
5633 0 : case SIGN_EXTEND:
5634 0 : case ZERO_EXTEND:
5635 0 : t = lang_hooks.types.type_for_mode (GET_MODE (XEXP (x, 0)),
5636 : GET_CODE (x) == ZERO_EXTEND);
5637 0 : return fold_convert (type, make_tree (t, XEXP (x, 0)));
5638 :
5639 0 : case CONST:
5640 0 : return make_tree (type, XEXP (x, 0));
5641 :
5642 0 : case SYMBOL_REF:
5643 0 : t = SYMBOL_REF_DECL (x);
5644 0 : if (t)
5645 0 : return fold_convert (type, build_fold_addr_expr (t));
5646 : /* fall through. */
5647 :
5648 668745 : default:
5649 668745 : t = build_decl (RTL_LOCATION (x), VAR_DECL, NULL_TREE, type);
5650 :
5651 : /* If TYPE is a POINTER_TYPE, we might need to convert X from
5652 : address mode to pointer mode. */
5653 668745 : if (POINTER_TYPE_P (type))
5654 835586 : x = convert_memory_address_addr_space
5655 417793 : (SCALAR_INT_TYPE_MODE (type), x, TYPE_ADDR_SPACE (TREE_TYPE (type)));
5656 :
5657 : /* Note that we do *not* use SET_DECL_RTL here, because we do not
5658 : want set_decl_rtl to go adjusting REG_ATTRS for this temporary. */
5659 668745 : t->decl_with_rtl.rtl = x;
5660 :
5661 668745 : return t;
5662 : }
5663 : }
5664 :
5665 : /* Compute the logical-and of OP0 and OP1, storing it in TARGET
5666 : and returning TARGET.
5667 :
5668 : If TARGET is 0, a pseudo-register or constant is returned. */
5669 :
5670 : rtx
5671 73152 : expand_and (machine_mode mode, rtx op0, rtx op1, rtx target)
5672 : {
5673 73152 : rtx tem = 0;
5674 :
5675 73152 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
5676 80 : tem = simplify_binary_operation (AND, mode, op0, op1);
5677 80 : if (tem == 0)
5678 73072 : tem = expand_binop (mode, and_optab, op0, op1, target, 0, OPTAB_LIB_WIDEN);
5679 :
5680 73152 : if (target == 0)
5681 : target = tem;
5682 45017 : else if (tem != target)
5683 13 : emit_move_insn (target, tem);
5684 73152 : return target;
5685 : }
5686 :
5687 : /* Helper function for emit_store_flag. */
5688 : rtx
5689 751615 : emit_cstore (rtx target, enum insn_code icode, enum rtx_code code,
5690 : machine_mode mode, machine_mode compare_mode,
5691 : int unsignedp, rtx x, rtx y, int normalizep,
5692 : machine_mode target_mode)
5693 : {
5694 751615 : class expand_operand ops[4];
5695 751615 : rtx op0, comparison, subtarget;
5696 751615 : rtx_insn *last;
5697 751615 : scalar_int_mode result_mode = targetm.cstore_mode (icode);
5698 751615 : scalar_int_mode int_target_mode;
5699 :
5700 751615 : last = get_last_insn ();
5701 751615 : x = prepare_operand (icode, x, 2, mode, compare_mode, unsignedp);
5702 751615 : y = prepare_operand (icode, y, 3, mode, compare_mode, unsignedp);
5703 751615 : if (!x || !y)
5704 : {
5705 284 : delete_insns_since (last);
5706 284 : return NULL_RTX;
5707 : }
5708 :
5709 751331 : if (target_mode == VOIDmode)
5710 : int_target_mode = result_mode;
5711 : else
5712 751315 : int_target_mode = as_a <scalar_int_mode> (target_mode);
5713 751331 : if (!target)
5714 69519 : target = gen_reg_rtx (int_target_mode);
5715 :
5716 751331 : comparison = gen_rtx_fmt_ee (code, result_mode, x, y);
5717 :
5718 751331 : create_output_operand (&ops[0], optimize ? NULL_RTX : target, result_mode);
5719 751331 : create_fixed_operand (&ops[1], comparison);
5720 751331 : create_fixed_operand (&ops[2], x);
5721 751331 : create_fixed_operand (&ops[3], y);
5722 751331 : if (!maybe_expand_insn (icode, 4, ops))
5723 : {
5724 154825 : delete_insns_since (last);
5725 154825 : return NULL_RTX;
5726 : }
5727 596506 : subtarget = ops[0].value;
5728 :
5729 : /* If we are converting to a wider mode, first convert to
5730 : INT_TARGET_MODE, then normalize. This produces better combining
5731 : opportunities on machines that have a SIGN_EXTRACT when we are
5732 : testing a single bit. This mostly benefits the 68k.
5733 :
5734 : If STORE_FLAG_VALUE does not have the sign bit set when
5735 : interpreted in MODE, we can do this conversion as unsigned, which
5736 : is usually more efficient. */
5737 596506 : if (GET_MODE_PRECISION (int_target_mode) > GET_MODE_PRECISION (result_mode))
5738 : {
5739 115379 : gcc_assert (GET_MODE_PRECISION (result_mode) != 1
5740 : || STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1);
5741 :
5742 115379 : bool unsignedp = (STORE_FLAG_VALUE >= 0);
5743 115379 : convert_move (target, subtarget, unsignedp);
5744 :
5745 115379 : op0 = target;
5746 115379 : result_mode = int_target_mode;
5747 : }
5748 : else
5749 : op0 = subtarget;
5750 :
5751 : /* If we want to keep subexpressions around, don't reuse our last
5752 : target. */
5753 596506 : if (optimize)
5754 478459 : subtarget = 0;
5755 :
5756 : /* Now normalize to the proper value in MODE. Sometimes we don't
5757 : have to do anything. */
5758 596506 : if (normalizep == 0 || normalizep == STORE_FLAG_VALUE)
5759 : ;
5760 : /* STORE_FLAG_VALUE might be the most negative number, so write
5761 : the comparison this way to avoid a compiler-time warning. */
5762 354 : else if (- normalizep == STORE_FLAG_VALUE)
5763 354 : op0 = expand_unop (result_mode, neg_optab, op0, subtarget, 0);
5764 :
5765 : /* We don't want to use STORE_FLAG_VALUE < 0 below since this makes
5766 : it hard to use a value of just the sign bit due to ANSI integer
5767 : constant typing rules. */
5768 0 : else if (val_signbit_known_set_p (result_mode, STORE_FLAG_VALUE))
5769 0 : op0 = expand_shift (RSHIFT_EXPR, result_mode, op0,
5770 0 : GET_MODE_BITSIZE (result_mode) - 1, subtarget,
5771 : normalizep == 1);
5772 : else
5773 : {
5774 0 : gcc_assert (STORE_FLAG_VALUE & 1);
5775 :
5776 0 : op0 = expand_and (result_mode, op0, const1_rtx, subtarget);
5777 0 : if (normalizep == -1)
5778 : op0 = expand_unop (result_mode, neg_optab, op0, op0, 0);
5779 : }
5780 :
5781 : /* If we were converting to a smaller mode, do the conversion now. */
5782 596506 : if (int_target_mode != result_mode)
5783 : {
5784 0 : convert_move (target, op0, 0);
5785 0 : return target;
5786 : }
5787 : else
5788 : return op0;
5789 : }
5790 :
5791 :
5792 : /* A subroutine of emit_store_flag only including "tricks" that do not
5793 : need a recursive call. These are kept separate to avoid infinite
5794 : loops. */
5795 :
5796 : static rtx
5797 719224 : emit_store_flag_1 (rtx target, enum rtx_code code, rtx op0, rtx op1,
5798 : machine_mode mode, int unsignedp, int normalizep,
5799 : machine_mode target_mode)
5800 : {
5801 719224 : rtx subtarget;
5802 719224 : enum insn_code icode;
5803 719224 : machine_mode compare_mode;
5804 719224 : enum mode_class mclass;
5805 :
5806 719224 : if (unsignedp)
5807 167526 : code = unsigned_condition (code);
5808 :
5809 : /* If one operand is constant, make it the second one. Only do this
5810 : if the other operand is not constant as well. */
5811 :
5812 719224 : if (swap_commutative_operands_p (op0, op1))
5813 : {
5814 5178 : std::swap (op0, op1);
5815 5178 : code = swap_condition (code);
5816 : }
5817 :
5818 719224 : if (mode == VOIDmode)
5819 42445 : mode = GET_MODE (op0);
5820 :
5821 719224 : if (CONST_SCALAR_INT_P (op1))
5822 333229 : canonicalize_comparison (mode, &code, &op1);
5823 :
5824 : /* For some comparisons with 1 and -1, we can convert this to
5825 : comparisons with zero. This will often produce more opportunities for
5826 : store-flag insns. */
5827 :
5828 719224 : switch (code)
5829 : {
5830 37407 : case LT:
5831 37407 : if (op1 == const1_rtx)
5832 59 : op1 = const0_rtx, code = LE;
5833 : break;
5834 26715 : case LE:
5835 26715 : if (op1 == constm1_rtx)
5836 0 : op1 = const0_rtx, code = LT;
5837 : break;
5838 39650 : case GE:
5839 39650 : if (op1 == const1_rtx)
5840 0 : op1 = const0_rtx, code = GT;
5841 : break;
5842 31103 : case GT:
5843 31103 : if (op1 == constm1_rtx)
5844 135 : op1 = const0_rtx, code = GE;
5845 : break;
5846 6285 : case GEU:
5847 6285 : if (op1 == const1_rtx)
5848 0 : op1 = const0_rtx, code = NE;
5849 : break;
5850 10716 : case LTU:
5851 10716 : if (op1 == const1_rtx)
5852 10 : op1 = const0_rtx, code = EQ;
5853 : break;
5854 : default:
5855 : break;
5856 : }
5857 :
5858 : /* If this is A < 0 or A >= 0, we can do this by taking the ones
5859 : complement of A (for GE) and shifting the sign bit to the low bit. */
5860 719224 : scalar_int_mode int_mode;
5861 200264 : if (op1 == const0_rtx && (code == LT || code == GE)
5862 719224 : && is_int_mode (mode, &int_mode)
5863 719224 : && (normalizep || STORE_FLAG_VALUE == 1
5864 : || val_signbit_p (int_mode, STORE_FLAG_VALUE)))
5865 : {
5866 40967 : scalar_int_mode int_target_mode;
5867 40967 : subtarget = target;
5868 :
5869 40967 : if (!target)
5870 : int_target_mode = int_mode;
5871 : else
5872 : {
5873 : /* If the result is to be wider than OP0, it is best to convert it
5874 : first. If it is to be narrower, it is *incorrect* to convert it
5875 : first. */
5876 40967 : int_target_mode = as_a <scalar_int_mode> (target_mode);
5877 122901 : if (GET_MODE_SIZE (int_target_mode) > GET_MODE_SIZE (int_mode))
5878 : {
5879 461 : op0 = convert_modes (int_target_mode, int_mode, op0, 0);
5880 461 : int_mode = int_target_mode;
5881 : }
5882 : }
5883 :
5884 40967 : if (int_target_mode != int_mode)
5885 27030 : subtarget = 0;
5886 :
5887 40967 : if (code == GE)
5888 20766 : op0 = expand_unop (int_mode, one_cmpl_optab, op0,
5889 : ((STORE_FLAG_VALUE == 1 || normalizep)
5890 : ? 0 : subtarget), 0);
5891 :
5892 40967 : if (STORE_FLAG_VALUE == 1 || normalizep)
5893 : /* If we are supposed to produce a 0/1 value, we want to do
5894 : a logical shift from the sign bit to the low-order bit; for
5895 : a -1/0 value, we do an arithmetic shift. */
5896 81934 : op0 = expand_shift (RSHIFT_EXPR, int_mode, op0,
5897 40967 : GET_MODE_BITSIZE (int_mode) - 1,
5898 : subtarget, normalizep != -1);
5899 :
5900 40967 : if (int_mode != int_target_mode)
5901 27030 : op0 = convert_modes (int_target_mode, int_mode, op0, 0);
5902 :
5903 : return op0;
5904 : }
5905 :
5906 : /* Next try expanding this via the backend's cstore<mode>4. */
5907 678257 : mclass = GET_MODE_CLASS (mode);
5908 690257 : FOR_EACH_WIDER_MODE_FROM (compare_mode, mode)
5909 : {
5910 682971 : machine_mode optab_mode = mclass == MODE_CC ? CCmode : compare_mode;
5911 682971 : icode = optab_handler (cstore_optab, optab_mode);
5912 682971 : if (icode != CODE_FOR_nothing)
5913 : {
5914 670971 : do_pending_stack_adjust ();
5915 670971 : rtx tem = emit_cstore (target, icode, code, mode, compare_mode,
5916 : unsignedp, op0, op1, normalizep, target_mode);
5917 670971 : if (tem)
5918 : return tem;
5919 :
5920 89820 : if (GET_MODE_CLASS (mode) == MODE_FLOAT)
5921 : {
5922 80624 : enum rtx_code scode = swap_condition (code);
5923 :
5924 80624 : tem = emit_cstore (target, icode, scode, mode, compare_mode,
5925 : unsignedp, op1, op0, normalizep, target_mode);
5926 80624 : if (tem)
5927 : return tem;
5928 : }
5929 : break;
5930 : }
5931 : }
5932 :
5933 : /* If we are comparing a double-word integer with zero or -1, we can
5934 : convert the comparison into one involving a single word. */
5935 81771 : if (is_int_mode (mode, &int_mode)
5936 16066 : && GET_MODE_BITSIZE (int_mode) == BITS_PER_WORD * 2
5937 8912 : && (!MEM_P (op0) || ! MEM_VOLATILE_P (op0)))
5938 : {
5939 8912 : rtx tem;
5940 8912 : if ((code == EQ || code == NE)
5941 0 : && (op1 == const0_rtx || op1 == constm1_rtx))
5942 : {
5943 0 : rtx op00, op01;
5944 :
5945 : /* Do a logical OR or AND of the two words and compare the
5946 : result. */
5947 0 : op00 = force_subreg (word_mode, op0, int_mode, 0);
5948 0 : op01 = force_subreg (word_mode, op0, int_mode, UNITS_PER_WORD);
5949 0 : tem = expand_binop (word_mode,
5950 0 : op1 == const0_rtx ? ior_optab : and_optab,
5951 : op00, op01, NULL_RTX, unsignedp,
5952 : OPTAB_DIRECT);
5953 :
5954 0 : if (tem != 0)
5955 0 : tem = emit_store_flag (NULL_RTX, code, tem, op1, word_mode,
5956 : unsignedp, normalizep);
5957 : }
5958 8912 : else if ((code == LT || code == GE) && op1 == const0_rtx)
5959 : {
5960 0 : rtx op0h;
5961 :
5962 : /* If testing the sign bit, can just test on high word. */
5963 0 : op0h = force_highpart_subreg (word_mode, op0, int_mode);
5964 0 : tem = emit_store_flag (NULL_RTX, code, op0h, op1, word_mode,
5965 : unsignedp, normalizep);
5966 0 : }
5967 : else
5968 : tem = NULL_RTX;
5969 :
5970 0 : if (tem)
5971 : {
5972 0 : if (target_mode == VOIDmode || GET_MODE (tem) == target_mode)
5973 : return tem;
5974 0 : if (!target)
5975 0 : target = gen_reg_rtx (target_mode);
5976 :
5977 0 : convert_move (target, tem,
5978 0 : !val_signbit_known_set_p (word_mode,
5979 : (normalizep ? normalizep
5980 : : STORE_FLAG_VALUE)));
5981 0 : return target;
5982 : }
5983 : }
5984 :
5985 : return 0;
5986 : }
5987 :
5988 : /* Subroutine of emit_store_flag that handles cases in which the operands
5989 : are scalar integers. SUBTARGET is the target to use for temporary
5990 : operations and TRUEVAL is the value to store when the condition is
5991 : true. All other arguments are as for emit_store_flag. */
5992 :
5993 : rtx
5994 4598 : emit_store_flag_int (rtx target, rtx subtarget, enum rtx_code code, rtx op0,
5995 : rtx op1, scalar_int_mode mode, int unsignedp,
5996 : int normalizep, rtx trueval)
5997 : {
5998 4598 : machine_mode target_mode = target ? GET_MODE (target) : VOIDmode;
5999 4598 : rtx_insn *last = get_last_insn ();
6000 :
6001 : /* If this is an equality comparison of integers, we can try to exclusive-or
6002 : (or subtract) the two operands and use a recursive call to try the
6003 : comparison with zero. Don't do any of these cases if branches are
6004 : very cheap. */
6005 :
6006 4598 : if ((code == EQ || code == NE) && op1 != const0_rtx)
6007 : {
6008 0 : rtx tem = expand_binop (mode, xor_optab, op0, op1, subtarget, 1,
6009 : OPTAB_WIDEN);
6010 :
6011 0 : if (tem == 0)
6012 0 : tem = expand_binop (mode, sub_optab, op0, op1, subtarget, 1,
6013 : OPTAB_WIDEN);
6014 0 : if (tem != 0)
6015 0 : tem = emit_store_flag (target, code, tem, const0_rtx,
6016 : mode, unsignedp, normalizep);
6017 0 : if (tem != 0)
6018 : return tem;
6019 :
6020 0 : delete_insns_since (last);
6021 : }
6022 :
6023 : /* For integer comparisons, try the reverse comparison. However, for
6024 : small X and if we'd have anyway to extend, implementing "X != 0"
6025 : as "-(int)X >> 31" is still cheaper than inverting "(int)X == 0". */
6026 4598 : rtx_code rcode = reverse_condition (code);
6027 4598 : if (can_compare_p (rcode, mode, ccp_store_flag)
6028 4598 : && ! (optab_handler (cstore_optab, mode) == CODE_FOR_nothing
6029 0 : && code == NE
6030 0 : && GET_MODE_SIZE (mode) < UNITS_PER_WORD
6031 0 : && op1 == const0_rtx))
6032 : {
6033 4598 : int want_add = ((STORE_FLAG_VALUE == 1 && normalizep == -1)
6034 : || (STORE_FLAG_VALUE == -1 && normalizep == 1));
6035 :
6036 : /* Again, for the reverse comparison, use either an addition or a XOR. */
6037 4598 : if (want_add
6038 4598 : && rtx_cost (GEN_INT (normalizep), mode, PLUS, 1,
6039 : optimize_insn_for_speed_p ()) == 0)
6040 : {
6041 0 : rtx tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6042 : STORE_FLAG_VALUE, target_mode);
6043 0 : if (tem != 0)
6044 0 : tem = expand_binop (target_mode, add_optab, tem,
6045 0 : gen_int_mode (normalizep, target_mode),
6046 : target, 0, OPTAB_WIDEN);
6047 0 : if (tem != 0)
6048 : return tem;
6049 : }
6050 4598 : else if (!want_add
6051 4598 : && rtx_cost (trueval, mode, XOR, 1,
6052 : optimize_insn_for_speed_p ()) == 0)
6053 : {
6054 4598 : rtx tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6055 : normalizep, target_mode);
6056 4598 : if (tem != 0)
6057 0 : tem = expand_binop (target_mode, xor_optab, tem, trueval, target,
6058 0 : INTVAL (trueval) >= 0, OPTAB_WIDEN);
6059 0 : if (tem != 0)
6060 : return tem;
6061 : }
6062 :
6063 4598 : delete_insns_since (last);
6064 : }
6065 :
6066 : /* Some other cases we can do are EQ, NE, LE, and GT comparisons with
6067 : the constant zero. Reject all other comparisons at this point. Only
6068 : do LE and GT if branches are expensive since they are expensive on
6069 : 2-operand machines. */
6070 :
6071 4598 : if (op1 != const0_rtx
6072 4598 : || (code != EQ && code != NE
6073 91 : && (BRANCH_COST (optimize_insn_for_speed_p (),
6074 91 : false) <= 1 || (code != LE && code != GT))))
6075 : return 0;
6076 :
6077 : /* Try to put the result of the comparison in the sign bit. Assume we can't
6078 : do the necessary operation below. */
6079 :
6080 233 : rtx tem = 0;
6081 :
6082 : /* To see if A <= 0, compute (A | (A - 1)). A <= 0 iff that result has
6083 : the sign bit set. */
6084 :
6085 233 : if (code == LE)
6086 : {
6087 : /* This is destructive, so SUBTARGET can't be OP0. */
6088 56 : if (rtx_equal_p (subtarget, op0))
6089 0 : subtarget = 0;
6090 :
6091 56 : tem = expand_binop (mode, sub_optab, op0, const1_rtx, subtarget, 0,
6092 : OPTAB_WIDEN);
6093 56 : if (tem)
6094 56 : tem = expand_binop (mode, ior_optab, op0, tem, subtarget, 0,
6095 : OPTAB_WIDEN);
6096 : }
6097 :
6098 : /* To see if A > 0, compute (((signed) A) << BITS) - A, where BITS is the
6099 : number of bits in the mode of OP0, minus one. */
6100 :
6101 233 : if (code == GT)
6102 : {
6103 35 : if (rtx_equal_p (subtarget, op0))
6104 0 : subtarget = 0;
6105 :
6106 35 : tem = maybe_expand_shift (RSHIFT_EXPR, mode, op0,
6107 35 : GET_MODE_BITSIZE (mode) - 1,
6108 : subtarget, 0);
6109 35 : if (tem)
6110 35 : tem = expand_binop (mode, sub_optab, tem, op0, subtarget, 0,
6111 : OPTAB_WIDEN);
6112 : }
6113 :
6114 233 : if (code == EQ || code == NE)
6115 : {
6116 : /* For EQ or NE, one way to do the comparison is to apply an operation
6117 : that converts the operand into a positive number if it is nonzero
6118 : or zero if it was originally zero. Then, for EQ, we subtract 1 and
6119 : for NE we negate. This puts the result in the sign bit. Then we
6120 : normalize with a shift, if needed.
6121 :
6122 : Two operations that can do the above actions are ABS and FFS, so try
6123 : them. If that doesn't work, and MODE is smaller than a full word,
6124 : we can use zero-extension to the wider mode (an unsigned conversion)
6125 : as the operation. */
6126 :
6127 : /* Note that ABS doesn't yield a positive number for INT_MIN, but
6128 : that is compensated by the subsequent overflow when subtracting
6129 : one / negating. */
6130 :
6131 142 : if (optab_handler (abs_optab, mode) != CODE_FOR_nothing)
6132 142 : tem = expand_unop (mode, abs_optab, op0, subtarget, 1);
6133 0 : else if (optab_handler (ffs_optab, mode) != CODE_FOR_nothing)
6134 0 : tem = expand_unop (mode, ffs_optab, op0, subtarget, 1);
6135 0 : else if (GET_MODE_SIZE (mode) < UNITS_PER_WORD)
6136 : {
6137 0 : tem = convert_modes (word_mode, mode, op0, 1);
6138 0 : mode = word_mode;
6139 : }
6140 :
6141 142 : if (tem != 0)
6142 : {
6143 142 : if (code == EQ)
6144 0 : tem = expand_binop (mode, sub_optab, tem, const1_rtx, subtarget,
6145 : 0, OPTAB_WIDEN);
6146 : else
6147 142 : tem = expand_unop (mode, neg_optab, tem, subtarget, 0);
6148 : }
6149 :
6150 : /* If we couldn't do it that way, for NE we can "or" the two's complement
6151 : of the value with itself. For EQ, we take the one's complement of
6152 : that "or", which is an extra insn, so we only handle EQ if branches
6153 : are expensive. */
6154 :
6155 142 : if (tem == 0
6156 142 : && (code == NE
6157 0 : || BRANCH_COST (optimize_insn_for_speed_p (),
6158 : false) > 1))
6159 : {
6160 0 : if (rtx_equal_p (subtarget, op0))
6161 0 : subtarget = 0;
6162 :
6163 0 : tem = expand_unop (mode, neg_optab, op0, subtarget, 0);
6164 0 : tem = expand_binop (mode, ior_optab, tem, op0, subtarget, 0,
6165 : OPTAB_WIDEN);
6166 :
6167 0 : if (tem && code == EQ)
6168 0 : tem = expand_unop (mode, one_cmpl_optab, tem, subtarget, 0);
6169 : }
6170 : }
6171 :
6172 233 : if (tem && normalizep)
6173 233 : tem = maybe_expand_shift (RSHIFT_EXPR, mode, tem,
6174 233 : GET_MODE_BITSIZE (mode) - 1,
6175 : subtarget, normalizep == 1);
6176 :
6177 233 : if (tem)
6178 : {
6179 233 : if (!target)
6180 : ;
6181 233 : else if (GET_MODE (tem) != target_mode)
6182 : {
6183 91 : convert_move (target, tem, 0);
6184 91 : tem = target;
6185 : }
6186 142 : else if (!subtarget)
6187 : {
6188 73 : emit_move_insn (target, tem);
6189 73 : tem = target;
6190 : }
6191 : }
6192 : else
6193 0 : delete_insns_since (last);
6194 :
6195 : return tem;
6196 : }
6197 :
6198 : /* Emit a store-flags instruction for comparison CODE on OP0 and OP1
6199 : and storing in TARGET. Normally return TARGET.
6200 : Return 0 if that cannot be done.
6201 :
6202 : MODE is the mode to use for OP0 and OP1 should they be CONST_INTs. If
6203 : it is VOIDmode, they cannot both be CONST_INT.
6204 :
6205 : UNSIGNEDP is for the case where we have to widen the operands
6206 : to perform the operation. It says to use zero-extension.
6207 :
6208 : NORMALIZEP is 1 if we should convert the result to be either zero
6209 : or one. Normalize is -1 if we should convert the result to be
6210 : either zero or -1. If NORMALIZEP is zero, the result will be left
6211 : "raw" out of the scc insn. */
6212 :
6213 : rtx
6214 649124 : emit_store_flag (rtx target, enum rtx_code code, rtx op0, rtx op1,
6215 : machine_mode mode, int unsignedp, int normalizep)
6216 : {
6217 649124 : machine_mode target_mode = target ? GET_MODE (target) : VOIDmode;
6218 649124 : enum rtx_code rcode;
6219 649124 : rtx subtarget;
6220 649124 : rtx tem, trueval;
6221 649124 : rtx_insn *last;
6222 :
6223 : /* If we compare constants, we shouldn't use a store-flag operation,
6224 : but a constant load. We can get there via the vanilla route that
6225 : usually generates a compare-branch sequence, but will in this case
6226 : fold the comparison to a constant, and thus elide the branch. */
6227 649124 : if (CONSTANT_P (op0) && CONSTANT_P (op1))
6228 : return NULL_RTX;
6229 :
6230 648809 : tem = emit_store_flag_1 (target, code, op0, op1, mode, unsignedp, normalizep,
6231 : target_mode);
6232 648809 : if (tem)
6233 : return tem;
6234 :
6235 : /* If we reached here, we can't do this with a scc insn, however there
6236 : are some comparisons that can be done in other ways. Don't do any
6237 : of these cases if branches are very cheap. */
6238 76645 : if (BRANCH_COST (optimize_insn_for_speed_p (), false) == 0)
6239 : return 0;
6240 :
6241 : /* See what we need to return. We can only return a 1, -1, or the
6242 : sign bit. */
6243 :
6244 76645 : if (normalizep == 0)
6245 : {
6246 0 : if (STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1)
6247 0 : normalizep = STORE_FLAG_VALUE;
6248 :
6249 : else if (val_signbit_p (mode, STORE_FLAG_VALUE))
6250 : ;
6251 : else
6252 : return 0;
6253 : }
6254 :
6255 76645 : last = get_last_insn ();
6256 :
6257 : /* If optimizing, use different pseudo registers for each insn, instead
6258 : of reusing the same pseudo. This leads to better CSE, but slows
6259 : down the compiler, since there are more pseudos. */
6260 76576 : subtarget = (!optimize
6261 76645 : && (target_mode == mode)) ? target : NULL_RTX;
6262 76645 : trueval = GEN_INT (normalizep ? normalizep : STORE_FLAG_VALUE);
6263 :
6264 : /* For floating-point comparisons, try the reverse comparison or try
6265 : changing the "orderedness" of the comparison. */
6266 76645 : if (GET_MODE_CLASS (mode) == MODE_FLOAT)
6267 : {
6268 68532 : enum rtx_code first_code;
6269 68532 : bool and_them;
6270 :
6271 68532 : rcode = reverse_condition_maybe_unordered (code);
6272 68532 : if (can_compare_p (rcode, mode, ccp_store_flag)
6273 68532 : && (code == ORDERED || code == UNORDERED
6274 0 : || (! HONOR_NANS (mode) && (code == LTGT || code == UNEQ))
6275 0 : || (! HONOR_SNANS (mode) && (code == EQ || code == NE))))
6276 : {
6277 0 : int want_add = ((STORE_FLAG_VALUE == 1 && normalizep == -1)
6278 : || (STORE_FLAG_VALUE == -1 && normalizep == 1));
6279 :
6280 : /* For the reverse comparison, use either an addition or a XOR. */
6281 0 : if (want_add
6282 0 : && rtx_cost (GEN_INT (normalizep), mode, PLUS, 1,
6283 : optimize_insn_for_speed_p ()) == 0)
6284 : {
6285 0 : tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6286 : STORE_FLAG_VALUE, target_mode);
6287 0 : if (tem)
6288 0 : return expand_binop (target_mode, add_optab, tem,
6289 0 : gen_int_mode (normalizep, target_mode),
6290 : target, 0, OPTAB_WIDEN);
6291 : }
6292 0 : else if (!want_add
6293 0 : && rtx_cost (trueval, mode, XOR, 1,
6294 : optimize_insn_for_speed_p ()) == 0)
6295 : {
6296 0 : tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6297 : normalizep, target_mode);
6298 0 : if (tem)
6299 0 : return expand_binop (target_mode, xor_optab, tem, trueval,
6300 0 : target, INTVAL (trueval) >= 0,
6301 0 : OPTAB_WIDEN);
6302 : }
6303 : }
6304 :
6305 68532 : delete_insns_since (last);
6306 :
6307 : /* Cannot split ORDERED and UNORDERED, only try the above trick. */
6308 68532 : if (code == ORDERED || code == UNORDERED)
6309 : return 0;
6310 :
6311 68390 : and_them = split_comparison (code, mode, &first_code, &code);
6312 :
6313 : /* If there are no NaNs, the first comparison should always fall through.
6314 : Effectively change the comparison to the other one. */
6315 68390 : if (!HONOR_NANS (mode))
6316 : {
6317 404 : gcc_assert (first_code == (and_them ? ORDERED : UNORDERED));
6318 243 : return emit_store_flag_1 (target, code, op0, op1, mode, 0, normalizep,
6319 243 : target_mode);
6320 : }
6321 :
6322 68147 : if (!HAVE_conditional_move)
6323 : return 0;
6324 :
6325 : /* Do not turn a trapping comparison into a non-trapping one. */
6326 68147 : if ((code != EQ && code != NE && code != UNEQ && code != LTGT)
6327 2573 : && flag_trapping_math)
6328 : return 0;
6329 :
6330 : /* Try using a setcc instruction for ORDERED/UNORDERED, followed by a
6331 : conditional move. */
6332 65574 : tem = emit_store_flag_1 (subtarget, first_code, op0, op1, mode, 0,
6333 : normalizep, target_mode);
6334 65574 : if (tem == 0)
6335 : return 0;
6336 :
6337 65047 : if (and_them)
6338 2737 : tem = emit_conditional_move (target, { code, op0, op1, mode },
6339 2737 : tem, const0_rtx, GET_MODE (tem), 0);
6340 : else
6341 62310 : tem = emit_conditional_move (target, { code, op0, op1, mode },
6342 62310 : trueval, tem, GET_MODE (tem), 0);
6343 :
6344 65047 : if (tem == 0)
6345 721 : delete_insns_since (last);
6346 : return tem;
6347 : }
6348 :
6349 : /* The remaining tricks only apply to integer comparisons. */
6350 :
6351 8113 : scalar_int_mode int_mode;
6352 8113 : if (is_int_mode (mode, &int_mode))
6353 4598 : return emit_store_flag_int (target, subtarget, code, op0, op1, int_mode,
6354 4598 : unsignedp, normalizep, trueval);
6355 :
6356 : return 0;
6357 : }
6358 :
6359 : /* Like emit_store_flag, but always succeeds. */
6360 :
6361 : rtx
6362 597056 : emit_store_flag_force (rtx target, enum rtx_code code, rtx op0, rtx op1,
6363 : machine_mode mode, int unsignedp, int normalizep)
6364 : {
6365 597056 : rtx tem;
6366 597056 : rtx_code_label *label;
6367 597056 : rtx trueval, falseval;
6368 :
6369 : /* First see if emit_store_flag can do the job. */
6370 597056 : tem = emit_store_flag (target, code, op0, op1, mode, unsignedp, normalizep);
6371 597056 : if (tem != 0)
6372 : return tem;
6373 :
6374 : /* If one operand is constant, make it the second one. Only do this
6375 : if the other operand is not constant as well. */
6376 12159 : if (swap_commutative_operands_p (op0, op1))
6377 : {
6378 8 : std::swap (op0, op1);
6379 8 : code = swap_condition (code);
6380 : }
6381 :
6382 12159 : if (mode == VOIDmode)
6383 0 : mode = GET_MODE (op0);
6384 :
6385 12159 : if (!target)
6386 0 : target = gen_reg_rtx (word_mode);
6387 :
6388 : /* If this failed, we have to do this with set/compare/jump/set code.
6389 : For foo != 0, if foo is in OP0, just replace it with 1 if nonzero. */
6390 12159 : trueval = normalizep ? GEN_INT (normalizep) : const1_rtx;
6391 12159 : if (code == NE
6392 1747 : && GET_MODE_CLASS (mode) == MODE_INT
6393 41 : && REG_P (target)
6394 41 : && op0 == target
6395 0 : && op1 == const0_rtx)
6396 : {
6397 0 : label = gen_label_rtx ();
6398 0 : do_compare_rtx_and_jump (target, const0_rtx, EQ, unsignedp, mode,
6399 : NULL_RTX, NULL, label,
6400 : profile_probability::uninitialized ());
6401 0 : emit_move_insn (target, trueval);
6402 0 : emit_label (label);
6403 0 : return target;
6404 : }
6405 :
6406 12159 : if (!REG_P (target)
6407 12159 : || reg_mentioned_p (target, op0) || reg_mentioned_p (target, op1))
6408 5 : target = gen_reg_rtx (GET_MODE (target));
6409 :
6410 : /* Jump in the right direction if the target cannot implement CODE
6411 : but can jump on its reverse condition. */
6412 12159 : falseval = const0_rtx;
6413 12159 : if (! can_compare_p (code, mode, ccp_jump)
6414 12159 : && (! FLOAT_MODE_P (mode)
6415 7448 : || code == ORDERED || code == UNORDERED
6416 7261 : || (! HONOR_NANS (mode) && (code == LTGT || code == UNEQ))
6417 7261 : || (! HONOR_SNANS (mode) && (code == EQ || code == NE))))
6418 : {
6419 2787 : enum rtx_code rcode;
6420 2787 : if (FLOAT_MODE_P (mode))
6421 2787 : rcode = reverse_condition_maybe_unordered (code);
6422 : else
6423 0 : rcode = reverse_condition (code);
6424 :
6425 : /* Canonicalize to UNORDERED for the libcall. */
6426 2787 : if (can_compare_p (rcode, mode, ccp_jump)
6427 2787 : || (code == ORDERED && ! can_compare_p (ORDERED, mode, ccp_jump)))
6428 : {
6429 155 : falseval = trueval;
6430 155 : trueval = const0_rtx;
6431 155 : code = rcode;
6432 : }
6433 : }
6434 :
6435 12159 : emit_move_insn (target, trueval);
6436 12159 : label = gen_label_rtx ();
6437 12159 : do_compare_rtx_and_jump (op0, op1, code, unsignedp, mode, NULL_RTX, NULL,
6438 : label, profile_probability::uninitialized ());
6439 :
6440 12159 : emit_move_insn (target, falseval);
6441 12159 : emit_label (label);
6442 :
6443 12159 : return target;
6444 : }
6445 :
6446 : /* Expand a vector (left) rotate of MODE of X by an immediate AMT as a vector
6447 : permute operation. Emit code to put the result in DST if successful and
6448 : return it. Otherwise return NULL. This is intended to implement vector
6449 : rotates by byte amounts using vector permutes when the target does not offer
6450 : native vector rotate operations. */
6451 : rtx
6452 0 : expand_rotate_as_vec_perm (machine_mode mode, rtx dst, rtx x, rtx amt)
6453 : {
6454 0 : rtx amt_unwrap = unwrap_const_vec_duplicate (amt);
6455 : /* For now handle only rotate by the same integer constant in all lanes.
6456 : In principle rotates by any constant vector are representable through
6457 : permutes as long as the individual rotate amounts are multiples of
6458 : BITS_PER_UNIT. */
6459 0 : if (!CONST_INT_P (amt_unwrap))
6460 : return NULL_RTX;
6461 :
6462 0 : int rotamnt = INTVAL (amt_unwrap);
6463 0 : if (rotamnt % BITS_PER_UNIT != 0)
6464 : return NULL_RTX;
6465 0 : machine_mode qimode;
6466 0 : if (!qimode_for_vec_perm (mode).exists (&qimode))
6467 0 : return NULL_RTX;
6468 :
6469 0 : vec_perm_builder builder;
6470 0 : unsigned nunits = GET_MODE_SIZE (GET_MODE_INNER (mode));
6471 0 : poly_uint64 total_units = GET_MODE_SIZE (mode);
6472 0 : builder.new_vector (total_units, nunits, 3);
6473 0 : unsigned rot_bytes = rotamnt / BITS_PER_UNIT;
6474 0 : unsigned rot_to_perm = BYTES_BIG_ENDIAN ? rot_bytes : nunits - rot_bytes;
6475 0 : for (unsigned j = 0; j < 3 * nunits; j += nunits)
6476 0 : for (unsigned i = 0; i < nunits; i++)
6477 0 : builder.quick_push ((rot_to_perm + i) % nunits + j);
6478 :
6479 0 : rtx perm_src = lowpart_subreg (qimode, x, mode);
6480 0 : rtx perm_dst = lowpart_subreg (qimode, dst, mode);
6481 0 : rtx res
6482 0 : = expand_vec_perm_const (qimode, perm_src, perm_src, builder,
6483 : qimode, perm_dst);
6484 0 : if (!res)
6485 : return NULL_RTX;
6486 0 : if (!rtx_equal_p (res, perm_dst))
6487 0 : emit_move_insn (dst, lowpart_subreg (mode, res, qimode));
6488 : return dst;
6489 0 : }
6490 :
6491 : /* Helper function for canonicalize_cmp_for_target. Swap between inclusive
6492 : and exclusive ranges in order to create an equivalent comparison. See
6493 : canonicalize_cmp_for_target for the possible cases. */
6494 :
6495 : static enum rtx_code
6496 47 : equivalent_cmp_code (enum rtx_code code)
6497 : {
6498 47 : switch (code)
6499 : {
6500 : case GT:
6501 : return GE;
6502 0 : case GE:
6503 0 : return GT;
6504 0 : case LT:
6505 0 : return LE;
6506 0 : case LE:
6507 0 : return LT;
6508 2 : case GTU:
6509 2 : return GEU;
6510 0 : case GEU:
6511 0 : return GTU;
6512 1 : case LTU:
6513 1 : return LEU;
6514 2 : case LEU:
6515 2 : return LTU;
6516 :
6517 0 : default:
6518 0 : return code;
6519 : }
6520 : }
6521 :
6522 : /* Choose the more appropriate immediate in scalar integer comparisons. The
6523 : purpose of this is to end up with an immediate which can be loaded into a
6524 : register in fewer moves, if possible.
6525 :
6526 : For each integer comparison there exists an equivalent choice:
6527 : i) a > b or a >= b + 1
6528 : ii) a <= b or a < b + 1
6529 : iii) a >= b or a > b - 1
6530 : iv) a < b or a <= b - 1
6531 :
6532 : MODE is the mode of the first operand.
6533 : CODE points to the comparison code.
6534 : IMM points to the rtx containing the immediate. *IMM must satisfy
6535 : CONST_SCALAR_INT_P on entry and continues to satisfy CONST_SCALAR_INT_P
6536 : on exit. */
6537 :
6538 : void
6539 4847230 : canonicalize_comparison (machine_mode mode, enum rtx_code *code, rtx *imm)
6540 : {
6541 4847230 : if (!SCALAR_INT_MODE_P (mode))
6542 3990395 : return;
6543 :
6544 4843411 : int to_add = 0;
6545 4843411 : enum signop sgn = unsigned_condition_p (*code) ? UNSIGNED : SIGNED;
6546 :
6547 : /* Extract the immediate value from the rtx. */
6548 4843411 : wide_int imm_val = rtx_mode_t (*imm, mode);
6549 :
6550 4843411 : if (*code == GT || *code == GTU || *code == LE || *code == LEU)
6551 : to_add = 1;
6552 : else if (*code == GE || *code == GEU || *code == LT || *code == LTU)
6553 : to_add = -1;
6554 : else
6555 : return;
6556 :
6557 : /* Check for overflow/underflow in the case of signed values and
6558 : wrapping around in the case of unsigned values. If any occur
6559 : cancel the optimization. */
6560 856979 : wi::overflow_type overflow = wi::OVF_NONE;
6561 856979 : wide_int imm_modif;
6562 :
6563 856979 : if (to_add == 1)
6564 618473 : imm_modif = wi::add (imm_val, 1, sgn, &overflow);
6565 : else
6566 238506 : imm_modif = wi::sub (imm_val, 1, sgn, &overflow);
6567 :
6568 856979 : if (overflow)
6569 144 : return;
6570 :
6571 856835 : rtx new_imm = immed_wide_int_const (imm_modif, mode);
6572 :
6573 856835 : int old_cost = rtx_cost (*imm, mode, COMPARE, 0, true);
6574 856835 : int new_cost = rtx_cost (new_imm, mode, COMPARE, 0, true);
6575 :
6576 856835 : if (dump_file && (dump_flags & TDF_DETAILS))
6577 : {
6578 7 : fprintf (dump_file, ";; cmp: %s, old cst: ",
6579 7 : GET_RTX_NAME (*code));
6580 7 : print_rtl (dump_file, *imm);
6581 7 : fprintf (dump_file, " new cst: ");
6582 7 : print_rtl (dump_file, new_imm);
6583 7 : fprintf (dump_file, "\n");
6584 7 : fprintf (dump_file, ";; old cst cost: %d, new cst cost: %d\n",
6585 : old_cost, new_cost);
6586 : }
6587 :
6588 : /* Update the immediate and the code. */
6589 856835 : if (old_cost > new_cost)
6590 : {
6591 47 : *code = equivalent_cmp_code (*code);
6592 47 : *imm = new_imm;
6593 : }
6594 4843555 : }
6595 :
6596 :
6597 :
6598 : /* Perform possibly multi-word comparison and conditional jump to LABEL
6599 : if ARG1 OP ARG2 true where ARG1 and ARG2 are of mode MODE. This is
6600 : now a thin wrapper around do_compare_rtx_and_jump. */
6601 :
6602 : static void
6603 2406 : do_cmp_and_jump (rtx arg1, rtx arg2, enum rtx_code op, machine_mode mode,
6604 : rtx_code_label *label)
6605 : {
6606 2406 : int unsignedp = (op == LTU || op == LEU || op == GTU || op == GEU);
6607 2406 : do_compare_rtx_and_jump (arg1, arg2, op, unsignedp, mode, NULL_RTX,
6608 : NULL, label, profile_probability::uninitialized ());
6609 2406 : }
|
